fluent-plugin-output-solr 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 411fb997709bdee2ba7bdfacddd331dfacd82c0d
-  data.tar.gz: 481e8990f52260ad3d5ccabb03032ece66c7ad01
+  metadata.gz: 1919ebd408f73f5cf3ccd44214586fc15ac91bf1
+  data.tar.gz: a8f9e2f80d397a5f0578e407bb8a23027bcc25ae
 SHA512:
-  metadata.gz: e52a8fc04fd20bb336ddbcad6c7e12bf562180c52565736ba149334ba23afd61c8ceea989776d4ad2abb1d4fe8b37bcd0ac1bb4c989e5f251297e44b0328a776
-  data.tar.gz: 833671aa903c55254b03bf50ed4156f1e9b8a23e17027972a234991f14f2cc1176a6262777315aa4f7612ba7e296e9cff1b589cd01b625dac70a8a474bb24f69
+  metadata.gz: ab70f58bc5431a57e151e50ccfad286da2f4df3b911359c83f427a1b5806b39ec9759789c069151a560714eb643ee21fe17c34081bba838f06c594e32882338d
+  data.tar.gz: 49327b34e288c8f42f4d0aa93558b56c99b93b59419aa9fa1c31ee540544ed37d2e24423f3d90020941ef9d0ee56c358f3959073d8fc6bf6523581781ef9c6fb

data/README.md

@@ -1,6 +1,6 @@
 # Fluent::Plugin::SolrOutput
 
-This is a [Fluentd](http://fluentd.org/) output plugin for send data to [Apache Solr](http://lucene.apache.org/solr/). It support [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud) not only Standalone Solr.
+This is a [Fluentd](http://fluentd.org/) output plugin for sending data to [Apache Solr](http://lucene.apache.org/solr/). It supports standalone Solr and [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud).
 
 ## Requirements
 
@@ -72,6 +72,14 @@ Ignore undefined fields in the Solr schema.xml.
 ignore_undefined_fields false
 ```
 
+### string_field_value_max_length
+
+A string field value max length. If set -1, it means unlimited (default -1). However, there is a limit of Solr.
+
+```
+string_field_value_max_length -1
+```
+
 ### unique_key_field
 
 A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.
@@ -82,10 +90,10 @@ unique_key_field id
 
 ### timestamp_field
 
-A field name of event timestamp in the Solr schema.xml (default event_timestamp).
+A field name of event timestamp in the Solr schema.xml (default time).
 
 ```
-timestamp_field event_timestamp
+timestamp_field time
 ```
 
 ### flush_size

data/fluent-plugin-output-solr.gemspec

@@ -4,12 +4,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-output-solr"
-  spec.version       = "1.0.0"
+  spec.version       = "1.0.1"
   spec.authors       = ["Minoru Osuka"]
   spec.email         = ["minoru.osuka@gmail.com"]
 
   spec.summary       = "Fluent output plugin for sending data to Apache Solr."
-  spec.description   = "Fluent output plugin for sending data to Apache Solr. It support SolrCloud not only Standalone Solr."
+  spec.description   = "Fluent output plugin for sending data to Apache Solr. It supports standalone Solr and SolrCloud."
   spec.homepage      = "https://github.com/mosuka/fluent-plugin-output-solr"
 
   spec.license       = "Apache-2.0"
@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'rsolr', '~> 1.0.12'
   spec.add_runtime_dependency 'zk', '~> 1.9.5'
 
-  spec.add_development_dependency 'bundler', '~> 1.14.6'
+  spec.add_development_dependency 'bundler', '~> 1.15.1'
   spec.add_development_dependency 'rake', '~> 11.1.2'
   spec.add_development_dependency 'test-unit', '~> 3.1.5'
   spec.add_development_dependency 'minitest', '~> 5.8.3'

data/lib/fluent/plugin/out_solr.rb

@@ -12,8 +12,9 @@ module Fluent::Plugin
 
     DEFAULT_COLLECTION = 'collection1'
     DEFAULT_IGNORE_UNDEFINED_FIELDS = false
+    DEFAULT_STRING_FIELD_VALUE_MAX_LENGTH = -1
     DEFAULT_TAG_FIELD = 'tag'
-    DEFAULT_TIMESTAMP_FIELD = 'event_timestamp'
+    DEFAULT_TIMESTAMP_FIELD = 'time'
     DEFAULT_FLUSH_SIZE = 100
     DEFAULT_BUFFER_TYPE = "memory"
     DEFAULT_COMMIT_WITH_FLUSH = true
@@ -36,13 +37,15 @@ module Fluent::Plugin
                  :desc => 'The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.'
     config_param :ignore_undefined_fields, :bool, :default => DEFAULT_IGNORE_UNDEFINED_FIELDS,
                  :desc => 'Ignore undefined fields in the Solr schema.xml.'
+    config_param :string_field_value_max_length, :integer, :default => DEFAULT_STRING_FIELD_VALUE_MAX_LENGTH,
+                 :desc => 'Field value max length.'
 
     config_param :unique_key_field, :string, :default => nil,
                  :desc => 'A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.'
     config_param :tag_field, :string, :default => DEFAULT_TAG_FIELD,
-                 :desc => 'A field name of fluentd tag in the Solr schema.xml (default event_timestamp).'
+                 :desc => 'A field name of fluentd tag in the Solr schema.xml (default time).'
     config_param :timestamp_field, :string, :default => DEFAULT_TIMESTAMP_FIELD,
-                 :desc => 'A field name of event timestamp in the Solr schema.xml (default event_timestamp).'
+                 :desc => 'A field name of event timestamp in the Solr schema.xml (default time).'
 
     config_param :flush_size, :integer, :default => DEFAULT_FLUSH_SIZE,
                  :desc => 'A number of events to queue up before writing to Solr (default 100).'
@@ -62,6 +65,7 @@ module Fluent::Plugin
     def configure(conf)
       compat_parameters_convert(conf, :inject)
       super
+      raise Fluent::ConfigError, "'tag' in chunk_keys is required." if not @chunk_key_tag
     end
 
     def start
@@ -142,6 +146,30 @@ module Fluent::Plugin
           end
         end
 
+        if @string_field_value_max_length >= 0 then
+          record.each_key do |key|
+            if record[key].instance_of?(Array) then
+              values = []
+              record[key].each do |value|
+                if value.instance_of?(String) then
+                  if value.length > @string_field_value_max_length then
+                    log.warn "#{key} is too long (#{value.length}, max is #{@string_field_value_max_length})."
+                    values.push(value.slice(0, @string_field_value_max_length))
+                  else
+                    values.push(value)
+                  end
+                end
+              end
+              record[key] = values
+            elsif record[key].instance_of?(String) then
+              if record[key].length > @string_field_value_max_length then
+                log.warn "#{key} is too long (#{record[key].length}, max is #{@string_field_value_max_length})."
+                record[key] = record[key].slice(0, @string_field_value_max_length)
+              end
+            end
+          end
+        end
+
         documents << record
 
         if documents.count >= @flush_size
@@ -159,7 +187,7 @@ module Fluent::Plugin
         log.debug "Added %d document(s) to Solr" % documents.count
       elsif @mode == MODE_SOLRCLOUD then
         @solr.add documents, collection: @collection, :params => {:commit => @commit_with_flush}
-        log.debug "Update: Added %d document(s) to Solr" % documents.count
+        log.debug "Added #{documents.count} document(s) to Solr"
       end
       rescue Exception => e
         log.warn "Update: An error occurred while indexing: #{e.message}"
@@ -180,7 +208,7 @@ module Fluent::Plugin
       return unique_key
 
       rescue Exception => e
-        log.warn "Unique key: #{e.message}"
+        log.warn "An error occurred: #{e.message}"
     end
 
     def get_fields
@@ -201,7 +229,7 @@ module Fluent::Plugin
       return fields
 
       rescue Exception => e
-        log.warn "Fields: #{e.message}"
+        log.warn "An error occurred: #{e.message}"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-output-solr
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Minoru Osuka
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-03-29 00:00:00.000000000 Z
+date: 2017-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -78,14 +78,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.14.6
+        version: 1.15.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.14.6
+        version: 1.15.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -156,8 +156,8 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.8
-description: Fluent output plugin for sending data to Apache Solr. It support SolrCloud
-  not only Standalone Solr.
+description: Fluent output plugin for sending data to Apache Solr. It supports standalone
+  Solr and SolrCloud.
 email:
 - minoru.osuka@gmail.com
 executables: []