fluent-plugin-output-solr 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1919ebd408f73f5cf3ccd44214586fc15ac91bf1
-  data.tar.gz: a8f9e2f80d397a5f0578e407bb8a23027bcc25ae
+  metadata.gz: 83c3ee7ee757f55da3c9605b0c292d31cb41ca97
+  data.tar.gz: f01b0a4f320facb257e530772b5798ce38aa1c63
 SHA512:
-  metadata.gz: ab70f58bc5431a57e151e50ccfad286da2f4df3b911359c83f427a1b5806b39ec9759789c069151a560714eb643ee21fe17c34081bba838f06c594e32882338d
-  data.tar.gz: 49327b34e288c8f42f4d0aa93558b56c99b93b59419aa9fa1c31ee540544ed37d2e24423f3d90020941ef9d0ee56c358f3959073d8fc6bf6523581781ef9c6fb
+  metadata.gz: 5e957af523348177ef465b00f33762e9e566229afc980bf430f7aa59b6f82a1c72d027043077e204fd7b37b2984719eba055b2f7c2457795662570272a92707f
+  data.tar.gz: 820719027d5dfe43a39d8ed9771a3d069b15ed151d316db2172df5c1c8840798ee10de3be569b85620dc6e55922b06e7cfc7bf5e1cae4bfd2850fb9b6f2e1109

data/README.md

@@ -1,13 +1,13 @@
 # Fluent::Plugin::SolrOutput
 
-This is a [Fluentd](http://fluentd.org/) output plugin for sending data to [Apache Solr](http://lucene.apache.org/solr/). It supports standalone Solr and [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud).
+This is a [Fluentd](http://fluentd.org/) output plugin for send data to [Apache Solr](http://lucene.apache.org/solr/).
 
 ## Requirements
 
-| fluent-plugin-output-solr | fluentd         | td-agent | ruby    |
-| ------------------------- | --------------- | -------- | ------- |
-| 1.x.x                     | \>= 0.14.0, < 2 | 3        | \>= 2.1 |
-| 0.x.x                     | ~> 0.12.0       | 2        | \>= 1.9 |
+| fluent-plugin-output-solr | fluentd         | td-agent | ruby   |
+| ------------------------- | --------------- | -------- | ------ |
+| 1.x.x                     | >= 0.14.0, < 2  | 3        | >= 2.1 |
+| 0.x.x                     | ~> 0.12.0       | 2        | >= 1.9 |
 
 * The 1.x.x series is developed from this branch (master)
 * The 0.x.x series (compatible with fluentd v0.12, and td-agent 2) is developed on the [v0.x.x branch](https://github.com/mosuka/fluent-plugin-output-solr/tree/v0.x.x)
@@ -32,12 +32,12 @@ $ rake install
 
 ## Config parameters
 
-### url
+### base_url
 
-The Solr server url (for example http://localhost:8983/solr/collection1).
+The Solr base url (for example http://localhost:8983/solr).
 
 ```
-url http://localhost:8983/solr/collection1
+base_url http://localhost:8983/solr
 ```
 
 ### zk_host
@@ -50,50 +50,50 @@ zk_host localhost:2181/solr
 
 ### collection
 
-The SolrCloud collection name (default collection1).
+The Solr collection/core name (default collection1).
 
 ```
 collection collection1
 ```
 
-### defined_fields
+### ignore_undefined_fields
 
-The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.
+Ignore undefined fields in the Solr schema.xml.
 
 ```
-defined_fields ["id", "title"]
+ignore_undefined_fields false
 ```
 
-### ignore_undefined_fields
+### tag_field
 
-Ignore undefined fields in the Solr schema.xml.
+A field name of fluentd tag in the Solr schema.xml (default tag).
 
 ```
-ignore_undefined_fields false
+tag_field tag
 ```
 
-### string_field_value_max_length
+### time_field
 
-A string field value max length. If set -1, it means unlimited (default -1). However, there is a limit of Solr.
+A field name of event timestamp in the Solr schema.xml (default time).
 
 ```
-string_field_value_max_length -1
+time_field time
 ```
 
-### unique_key_field
+### time_format
 
-A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.
+The format of the time field (default %FT%TZ).
 
 ```
-unique_key_field id
+time_format %FT%TZ
 ```
 
-### timestamp_field
+### millisecond
 
-A field name of event timestamp in the Solr schema.xml (default time).
+Output millisecond to Solr (default false).
 
 ```
-timestamp_field time
+millisecond false
 ```
 
 ### flush_size
@@ -119,8 +119,11 @@ commit_with_flush true
 <match something.logs>
   @type solr
 
-  # The Solr server url (for example http://localhost:8983/solr/collection1).
-  url http://localhost:8983/solr/collection1
+  # The Solr base url (for example http://localhost:8983/solr).
+  base_url http://localhost:8983/solr
+
+  # The Solr collection/core name (default collection1).
+  collection collection1
 </match>
 ```
 
@@ -132,83 +135,11 @@ commit_with_flush true
   # The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).
   zk_host localhost:2181/solr
 
-  # The SolrCloud collection name (default collection1).
+  # The Solr collection/core name (default collection1).
   collection collection1
 </match>
 ```
 
-## Solr setup examples
-
-### How to setup Standalone Solr using data-driven schemaless mode.
-
-1.Download and install Solr
-
-```sh
-$ mkdir $HOME/solr
-$ cd $HOME/solr
-$ wget https://archive.apache.org/dist/lucene/solr/5.4.0/solr-5.4.0.tgz
-$ tar zxvf solr-5.4.0.tgz
-$ cd solr-5.4.0
-```
-
-2.Start standalone Solr
-
-```sh
-$ ./bin/solr start -p 8983 -s server/solr
-```
-
-3.Create core
-
-```sh
-$ ./bin/solr create -c collection1 -d server/solr/configsets/data_driven_schema_configs -n collection1_configs
-```
-
-### How to setup SolrCloud using data-driven schemaless mode (shards=1 and replicationfactor=2).
-
-1.Download and install ZooKeeper
-
-```sh
-$ mkdir $HOME/zookeeper
-$ cd $HOME/zookeeper
-$ wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz
-$ tar zxvf zookeeper-3.4.6.tar.gz
-$ cd zookeeper-3.4.6
-$ cp -p ./conf/zoo_sample.cfg ./conf/zoo.cfg
-```
-
-2.Start standalone ZooKeeper
-
-```sh
-$ ./bin/zkServer.sh start
-```
-
-3.Download an install Solr
-
-```sh
-$ mkdir $HOME/solr
-$ cd $HOME/solr
-$ wget https://archive.apache.org/dist/lucene/solr/5.4.0/solr-5.4.0.tgz
-$ tar zxvf solr-5.4.0.tgz
-$ cd solr-5.4.0
-$ ./server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd clear /solr
-$ ./server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd makepath /solr
-$ cp -pr server/solr server/solr1
-$ cp -pr server/solr server/solr2
-```
-
-4.Start SolrCloud
-
-```sh
-$ ./bin/solr start -h localhost -p 8983 -z localhost:2181/solr -s server/solr1
-$ ./bin/solr start -h localhost -p 8985 -z localhost:2181/solr -s server/solr2
-```
-
-5.Create collection
-
-```sh
-$ ./bin/solr create -c collection1 -d server/solr1/configsets/data_driven_schema_configs -n collection1_configs -shards 1 -replicationFactor 2
-```
-
 ## Development
 
 After checking out the repo, run `bundle install` to install dependencies. Then, run `rake test` to run the tests.

data/fluent-plugin-output-solr.gemspec

@@ -3,21 +3,21 @@ lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
-  spec.name          = "fluent-plugin-output-solr"
-  spec.version       = "1.0.1"
-  spec.authors       = ["Minoru Osuka"]
-  spec.email         = ["minoru.osuka@gmail.com"]
+  spec.name          = 'fluent-plugin-output-solr'
+  spec.version       = '1.0.2'
+  spec.authors       = ['Minoru Osuka']
+  spec.email         = ['minoru.osuka@gmail.com']
 
-  spec.summary       = "Fluent output plugin for sending data to Apache Solr."
-  spec.description   = "Fluent output plugin for sending data to Apache Solr. It supports standalone Solr and SolrCloud."
-  spec.homepage      = "https://github.com/mosuka/fluent-plugin-output-solr"
+  spec.summary       = 'Fluent output plugin for sending data to Apache Solr.'
+  spec.description   = 'Fluent output plugin for sending data to Apache Solr.'
+  spec.homepage      = 'https://github.com/mosuka/fluent-plugin-output-solr'
 
-  spec.license       = "Apache-2.0"
+  spec.license       = 'Apache-2.0'
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
   spec.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
   spec.add_runtime_dependency 'rsolr-cloud', '~> 1.1.0'

data/fluent.conf

@@ -15,11 +15,17 @@
 
 <match messages>
   @type solr
-#  url http://localhost:8983/solr/collection1
-  zk_host localhost:2181/solr
+  base_url http://localhost:8983/solr
+  # zk_host localhost:2181/solr
   collection collection1
   ignore_undefined_fields false
+  tag_field tag
+  time_field time
+  time_format %FT%TZ
+  millisecond true
   flush_size 100
+  commit_with_flush true
+
   buffer_type memory
   buffer_queue_limit 64m
   buffer_chunk_limit 8m

data/lib/fluent/plugin/out_solr.rb

@@ -11,12 +11,17 @@ module Fluent::Plugin
     helpers :inject, :compat_parameters
 
     DEFAULT_COLLECTION = 'collection1'
-    DEFAULT_IGNORE_UNDEFINED_FIELDS = false
-    DEFAULT_STRING_FIELD_VALUE_MAX_LENGTH = -1
+
     DEFAULT_TAG_FIELD = 'tag'
-    DEFAULT_TIMESTAMP_FIELD = 'time'
+
+    DEFAULT_TIME_FIELD = 'time'
+    DEFAULT_TIME_FORMAT = '%FT%TZ'
+    DEFAULT_MILLISECOND = false
+
+    DEFAULT_IGNORE_UNDEFINED_FIELDS = false
+
     DEFAULT_FLUSH_SIZE = 100
-    DEFAULT_BUFFER_TYPE = "memory"
+    DEFAULT_BUFFER_TYPE = 'memory'
     DEFAULT_COMMIT_WITH_FLUSH = true
 
     MODE_STANDALONE = 'Standalone'
@@ -25,27 +30,27 @@ module Fluent::Plugin
     config_set_default :include_tag_key, false
     config_set_default :include_time_key, false
 
-    config_param :url, :string, :default => nil,
-                 :desc => 'The Solr server url (for example http://localhost:8983/solr/collection1).'
+    config_param :base_url, :string, :default => nil,
+                 :desc => 'The Solr base url (for example http://localhost:8983/solr).'
 
     config_param :zk_host, :string, :default => nil,
                  :desc => 'The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).'
+
     config_param :collection, :string, :default => DEFAULT_COLLECTION,
-                 :desc => 'The SolrCloud collection name (default collection1).'
+                 :desc => 'The Solr collection/core name (default collection1).'
 
-    config_param :defined_fields, :array, :default => nil,
-                 :desc => 'The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.'
     config_param :ignore_undefined_fields, :bool, :default => DEFAULT_IGNORE_UNDEFINED_FIELDS,
                  :desc => 'Ignore undefined fields in the Solr schema.xml.'
-    config_param :string_field_value_max_length, :integer, :default => DEFAULT_STRING_FIELD_VALUE_MAX_LENGTH,
-                 :desc => 'Field value max length.'
 
-    config_param :unique_key_field, :string, :default => nil,
-                 :desc => 'A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.'
     config_param :tag_field, :string, :default => DEFAULT_TAG_FIELD,
                  :desc => 'A field name of fluentd tag in the Solr schema.xml (default time).'
-    config_param :timestamp_field, :string, :default => DEFAULT_TIMESTAMP_FIELD,
+
+    config_param :time_field, :string, :default => DEFAULT_TIME_FIELD,
                  :desc => 'A field name of event timestamp in the Solr schema.xml (default time).'
+    config_param :time_format, :string, :default => DEFAULT_TIME_FORMAT,
+                 :desc => 'The format of the time field (default %d/%b/%Y:%H:%M:%S %z).'
+    config_param :millisecond, :bool, :default => DEFAULT_MILLISECOND,
+                 :desc => 'Output millisecond to Solr (default false).'
 
     config_param :flush_size, :integer, :default => DEFAULT_FLUSH_SIZE,
                  :desc => 'A number of events to queue up before writing to Solr (default 100).'
@@ -72,7 +77,7 @@ module Fluent::Plugin
       super
 
       @mode = nil
-      if ! @url.nil? then
+      if ! @base_url.nil? then
         @mode = MODE_STANDALONE
       elsif ! @zk_host.nil?
         @mode = MODE_SOLRCLOUD
@@ -82,7 +87,7 @@ module Fluent::Plugin
       @zk = nil
 
       if @mode == MODE_STANDALONE then
-        @solr = RSolr.connect :url => @url
+        @solr = RSolr.connect :url => @base_url.end_with?('/') ? @base_url + @collection : @base_url + '/' + @collection
       elsif @mode == MODE_SOLRCLOUD then
         @zk = ZK.new(@zk_host)
         cloud_connection = RSolr::Cloud::Connection.new(@zk)
@@ -113,123 +118,119 @@ module Fluent::Plugin
     def write(chunk)
       documents = []
 
-      @fields = @defined_fields.nil? ? get_fields : @defined_fields
-      @unique_key = @unique_key_field.nil? ? get_unique_key : @unique_key_field
+      # Get fields from Solr
+      fields = get_fields
+
+      # Get unique key field from Solr
+      unique_key = get_unique_key
+
+      # Get fluentd tag
       tag = chunk.metadata.tag
+
       chunk.msgpack_each do |time, record|
         record = inject_values_to_record(tag, time, record)
 
-        unless record.has_key?(@unique_key) then
-          record.merge!({@unique_key => SecureRandom.uuid})
+        # Set unique key and value
+        unless record.has_key?(unique_key) then
+          record.merge!({unique_key => SecureRandom.uuid})
         end
 
+        # Set Fluentd tag to Solr tag field
         unless record.has_key?(@tag_field) then
           record.merge!({@tag_field => tag})
         end
 
-        if record.has_key?(@timestamp_field) then
+        # Set time
+        tmp_time = Time.at(time).utc
+        if record.has_key?(@time_field) then
+          # Parsing the time field in the record by the specified format.
           begin
-            event_timestamp_dt = DateTime.strptime(record[@timestamp_field], "%d/%b/%Y:%H:%M:%S %z").to_s
-            record.merge!({@timestamp_field => Time.parse(event_timestamp_dt.to_s).utc.strftime('%FT%TZ')})
-          rescue
-            record.merge!({@timestamp_field => Time.at(time).utc.strftime('%FT%TZ')})
+            tmp_time = Time.strptime(record[@time_field], @time_format).utc
+          rescue Exception => e
+            log.warn "An error occurred in parsing the time field: #{e.message}"
           end
+        end
+        if @millisecond then
+          record.merge!({@time_field => '%s.%03dZ' % [tmp_time.strftime('%FT%T'), tmp_time.usec / 1000.0]})
         else
-          record.merge!({@timestamp_field => Time.at(time).utc.strftime('%FT%TZ')})
+          record.merge!({@time_field => tmp_time.strftime('%FT%TZ')})
         end
 
+        # Ignore undefined fields
         if @ignore_undefined_fields then
           record.each_key do |key|
-            unless @fields.include?(key) then
+            unless fields.include?(key) then
               record.delete(key)
             end
           end
         end
 
-        if @string_field_value_max_length >= 0 then
-          record.each_key do |key|
-            if record[key].instance_of?(Array) then
-              values = []
-              record[key].each do |value|
-                if value.instance_of?(String) then
-                  if value.length > @string_field_value_max_length then
-                    log.warn "#{key} is too long (#{value.length}, max is #{@string_field_value_max_length})."
-                    values.push(value.slice(0, @string_field_value_max_length))
-                  else
-                    values.push(value)
-                  end
-                end
-              end
-              record[key] = values
-            elsif record[key].instance_of?(String) then
-              if record[key].length > @string_field_value_max_length then
-                log.warn "#{key} is too long (#{record[key].length}, max is #{@string_field_value_max_length})."
-                record[key] = record[key].slice(0, @string_field_value_max_length)
-              end
-            end
-          end
-        end
-
+        # Add record to documents
         documents << record
 
+        # Update when flash size is reached
         if documents.count >= @flush_size
           update documents
           documents.clear
         end
       end
 
+      # Update remaining documents
       update documents unless documents.empty?
     end
 
     def update(documents)
-      if @mode == MODE_STANDALONE then
-        @solr.add documents, :params => {:commit => @commit_with_flush}
-        log.debug "Added %d document(s) to Solr" % documents.count
-      elsif @mode == MODE_SOLRCLOUD then
-        @solr.add documents, collection: @collection, :params => {:commit => @commit_with_flush}
-        log.debug "Added #{documents.count} document(s) to Solr"
+      begin
+        if @mode == MODE_STANDALONE then
+          @solr.add documents, :params => {:commit => @commit_with_flush}
+        elsif @mode == MODE_SOLRCLOUD then
+          @solr.add documents, collection: @collection, :params => {:commit => @commit_with_flush}
+        end
+        log.debug "Sent #{documents.count} document(s) to Solr"
+      rescue Exception
+        log.warn "An error occurred while sending #{documents.count} document(s) to Solr"
       end
-      rescue Exception => e
-        log.warn "Update: An error occurred while indexing: #{e.message}"
     end
 
     def get_unique_key
-      response = nil
-
-      if @mode == MODE_STANDALONE then
-        response = @solr.get 'schema/uniquekey'
-      elsif @mode == MODE_SOLRCLOUD then
-        response = @solr.get 'schema/uniquekey', collection: @collection
+      unique_key = 'id'
+
+      begin
+        response = nil
+        if @mode == MODE_STANDALONE then
+          response = @solr.get 'schema/uniquekey'
+        elsif @mode == MODE_SOLRCLOUD then
+          response = @solr.get 'schema/uniquekey', collection: @collection
+        end
+        unique_key = response['uniqueKey']
+        log.debug "Unique key: #{unique_key}"
+      rescue Exception
+        log.warn 'An error occurred while getting unique key'
       end
 
-      unique_key = response['uniqueKey']
-      log.debug "Unique key: #{unique_key}"
-
       return unique_key
-
-      rescue Exception => e
-        log.warn "An error occurred: #{e.message}"
     end
 
     def get_fields
-      response = nil
+      fields = []
 
-      if @mode == MODE_STANDALONE then
-        response = @solr.get 'schema/fields'
-      elsif @mode == MODE_SOLRCLOUD then
-        response = @solr.get 'schema/fields', collection: @collection
-      end
+      begin
+        response = nil
 
-      fields = []
-      response['fields'].each do |field|
-        fields.push(field['name'])
+        if @mode == MODE_STANDALONE then
+          response = @solr.get 'schema/fields'
+        elsif @mode == MODE_SOLRCLOUD then
+          response = @solr.get 'schema/fields', collection: @collection
+        end
+        response['fields'].each do |field|
+          fields.push(field['name'])
+        end
+        log.debug "Fields: #{fields}"
+      rescue Exception
+        log.warn 'An error occurred while getting fields'
       end
-      log.debug "Fields: #{fields}"
 
       return fields
-
-      rescue Exception => e
-        log.warn "An error occurred: #{e.message}"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-output-solr
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Minoru Osuka
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-14 00:00:00.000000000 Z
+date: 2017-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -156,8 +156,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.8
-description: Fluent output plugin for sending data to Apache Solr. It supports standalone
-  Solr and SolrCloud.
+description: Fluent output plugin for sending data to Apache Solr.
 email:
 - minoru.osuka@gmail.com
 executables: []