fluent-plugin-output-solr 0.4.5 → 0.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c3b1d94e9b198256826b6717b367f3d908f9cf2f
-  data.tar.gz: dff11f4040d0a6efb8edbcd6b4c9b7cdefd86ebd
+  metadata.gz: 47e3caa1f6aa5e15e129881af97b226e5ad19bf9
+  data.tar.gz: daf903c8e406d5122fae89a1472b782a1482785e
 SHA512:
-  metadata.gz: 774d85e8cca935a67f2c13e4c0948ee094b8dbd2161fb479c40c823c72e585658659fae1cc4d756c415e4a4628c445ee8f802ff1bb853d491f2e1e46a61513a8
-  data.tar.gz: b8a36d1441c7c956e9f5dde485f6a8f30663586990a9ca759e54073a43cf0d22280227fef99c33538b6fe4a03408405f8c63c2ef532d1520b0bfa614e620f4b1
+  metadata.gz: 1767955e6d29a293bf326f4712637b3c905cd2b28c00eef6ecaa387a47275b55ed2a0d41fa34ad0270be7022ad9f43550004f9312e2d9c9c1324477450dd1638
+  data.tar.gz: cfa4c10ae32925707d32e8e73e42c06732503798a813aa9e1aa38d826c36d8b2d00ed74227ab1c37e6537ce7f89abda8985d9bc93795dfe34e93454aeb91dfaf

data/README.md

@@ -80,6 +80,14 @@ A field name of unique key in the Solr schema.xml. If omitted, it will get uniqu
 unique_key_field id
 ```
 
+### string_field_value_max_length
+
+A string field value max length. If set -1, it means unlimited (default -1). However, there is a limit of Solr.
+
+```
+string_field_value_max_length -1
+```
+
 ### timestamp_field
 
 A field name of event timestamp in the Solr schema.xml (default event_timestamp).

data/fluent-plugin-output-solr.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-output-solr"
-  spec.version       = "0.4.5"
+  spec.version       = "0.4.6"
   spec.authors       = ["Minoru Osuka"]
   spec.email         = ["minoru.osuka@gmail.com"]
 

data/fluent.conf

@@ -13,46 +13,11 @@
   key_name message
 </filter>
 
-<filter messages>
-  @type geoip
-  log_level error
-  enable_auto_download true
-  md5_url http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.md5
-  download_url http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz
-  md5_path "#{ENV['HOME']}/fluentd/geoip/database/GeoLite2-City.md5"
-  database_path "#{ENV['HOME']}/fluentd/geoip/database/GeoLite2-City.mmdb"
-  lookup_field clientip
-  output_field geoip
-  field_delimiter _
-  flatten true
-  locale en
-  continent true
-  country true
-  city true
-  location true
-  postal true
-  registered_country true
-  represented_country true
-  subdivisions true
-  traits true
-  connection_type true
-</filter>
-
-<filter messages>
-  @type record_transformer
-  log_level error
-  enable_ruby
-  <record>
-    event_timestamp ${timestamp}
-    geoip_localtion_latitude_longitude ${geoip_location_latitude.nil? || geoip_location_longitude.nil? ? nil : geoip_location_latitude.to_s + ',' + geoip_location_longitude.to_s}
-  </record>
-</filter>
-
 <match messages>
   @type solr
-  log_level error
-  zk_host "#{ENV['SOLR_ZK_HOST']}"
-  collection "#{ENV['SOLR_COLLECTION']}"
+#  url http://localhost:8983/solr/collection1
+  zk_host localhost:2181/solr
+  collection collection1
   ignore_undefined_fields false
   flush_size 100
   buffer_type memory

data/lib/fluent/plugin/out_solr.rb

@@ -9,6 +9,7 @@ module Fluent
 
     DEFAULT_COLLECTION = 'collection1'
     DEFAULT_IGNORE_UNDEFINED_FIELDS = false
+    DEFAULT_STRING_FIELD_VALUE_MAX_LENGTH = -1
     DEFAULT_TAG_FIELD = 'tag'
     DEFAULT_TIMESTAMP_FIELD = 'event_timestamp'
     DEFAULT_FLUSH_SIZE = 100
@@ -34,7 +35,9 @@ module Fluent
     config_param :defined_fields, :array, :default => nil,
                  :desc => 'The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.'                 
     config_param :ignore_undefined_fields, :bool, :default => DEFAULT_IGNORE_UNDEFINED_FIELDS,
-                 :desc => 'Ignore undefined fields in the Solr schema.xml.'                 
+                 :desc => 'Ignore undefined fields in the Solr schema.xml.'
+    config_param :string_field_value_max_length, :integer, :default => DEFAULT_STRING_FIELD_VALUE_MAX_LENGTH,
+                 :desc => 'Field value max length.'
 
     config_param :unique_key_field, :string, :default => nil,
                  :desc => 'A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.'
@@ -125,6 +128,30 @@ module Fluent
           end
         end
 
+        if @string_field_value_max_length >= 0 then
+          record.each_key do |key|
+            if record[key].instance_of?(Array) then
+              values = []
+              record[key].each do |value|
+                if value.instance_of?(String) then
+                  if value.length > @string_field_value_max_length then
+                    log.warn "#{key} is too long (#{value.length}, max is #{@string_field_value_max_length})."
+                    values.push(value.slice(0, @string_field_value_max_length))
+                  else
+                    values.push(value)
+                  end
+                end
+              end
+              record[key] = values
+            elsif record[key].instance_of?(String) then
+              if record[key].length > @string_field_value_max_length then
+                log.warn "#{key} is too long (#{record[key].length}, max is #{@string_field_value_max_length})."
+                record[key] = record[key].slice(0, @string_field_value_max_length)
+              end
+            end
+          end
+        end
+
         documents << record
 
         if documents.count >= @flush_size
@@ -139,13 +166,13 @@ module Fluent
     def update(documents)
       if @mode == MODE_STANDALONE then
         @solr.add documents, :params => {:commit => @commit_with_flush}
-        log.debug "Added %d document(s) to Solr" % documents.count
+        log.debug "Added #{documents.count} document(s) to Solr"
       elsif @mode == MODE_SOLRCLOUD then
         @solr.add documents, collection: @collection, :params => {:commit => @commit_with_flush}
-        log.debug "Update: Added %d document(s) to Solr" % documents.count
+        log.debug "Added #{documents.count} document(s) to Solr"
       end
       rescue Exception => e
-        log.warn "Update: An error occurred while indexing: #{e.message}"
+        log.warn "An error occurred while indexing: #{e.message}"
     end
 
     def get_unique_key
@@ -163,7 +190,7 @@ module Fluent
       return unique_key
 
       rescue Exception => e
-        log.warn "Unique key: #{e.message}"
+        log.warn "An error occurred: #{e.message}"
     end
 
     def get_fields
@@ -184,7 +211,7 @@ module Fluent
       return fields
 
       rescue Exception => e
-        log.warn "Fields: #{e.message}"
+        log.warn "An error occurred: #{e.message}"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-output-solr
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.4.6
 platform: ruby
 authors:
 - Minoru Osuka
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-03-29 00:00:00.000000000 Z
+date: 2017-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd