fluent-plugin-output-solr 0.4.12 → 0.4.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 69c8450973f1699e1a6d085914b2aff6b33336c9
-  data.tar.gz: 1c45581028a1e2554c9c34859af1353b2078214c
+  metadata.gz: e7291cfc870c7f8be862adb23b42f1cf9072f697
+  data.tar.gz: 64a33feafa219c2798fd1a3897d9122f24bd3b08
 SHA512:
-  metadata.gz: 3b34e96a6bf48d9c402970c54feb4ebeab5dae6a3b1671e8508cb38343d418a46e2451446cb2fd6fbaa71942129def9bae6cc80c0b13ef3c072a315ed8990e46
-  data.tar.gz: 7f6687e3baba7bd6c7ae7cd97367bf82757a3383cb58c322720fa69c6bf5fd056805dbfb8d3459e2ed31b7d4a284069c495b7c7983e7c00a6000b91a7ba50a78
+  metadata.gz: f99cf5bfcb8dde3440bf65183594508923a20e09670e9ab78f122b64f53fa8efad0a82befa3cf95a26bf2e9438cbf8aa1bb3c681e4888e6b42fbbc4bc8844e54
+  data.tar.gz: b847b79fa7cfcd2311ad515c4732971dd47065ef745c237431fe0bb1b11324480cc08b4c0648c2528e06d41114c3c03efe7653d7f80d6111fa2fff3ceb7a61f7

data/README.md

@@ -1,6 +1,6 @@
 # Fluent::Plugin::SolrOutput
 
-This is a [Fluentd](http://fluentd.org/) output plugin for send data to [Apache Solr](http://lucene.apache.org/solr/). It support [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud) not only Standalone Solr.
+This is a [Fluentd](http://fluentd.org/) output plugin for send data to [Apache Solr](http://lucene.apache.org/solr/).
 
 ## Requirements
 
@@ -32,12 +32,12 @@ $ rake install
 
 ## Config parameters
 
-### url
+### base_url
 
-The Solr server url (for example http://localhost:8983/solr/collection1).
+The Solr base url (for example http://localhost:8983/solr).
 
 ```
-url http://localhost:8983/solr/collection1
+base_url http://localhost:8983/solr
 ```
 
 ### zk_host
@@ -50,20 +50,12 @@ zk_host localhost:2181/solr
 
 ### collection
 
-The SolrCloud collection name (default collection1).
+The Solr collection/core name (default collection1).
 
 ```
 collection collection1
 ```
 
-### defined_fields
-
-The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.
-
-```
-defined_fields ["id", "title"]
-```
-
 ### ignore_undefined_fields
 
 Ignore undefined fields in the Solr schema.xml.
@@ -72,20 +64,12 @@ Ignore undefined fields in the Solr schema.xml.
 ignore_undefined_fields false
 ```
 
-### unique_key_field
-
-A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.
-
-```
-unique_key_field id
-```
-
-### string_field_value_max_length
+### tag_field
 
-A string field value max length. If set -1, it means unlimited (default -1). However, there is a limit of Solr.
+A field name of fluentd tag in the Solr schema.xml (default tag).
 
 ```
-string_field_value_max_length -1
+tag_field tag
 ```
 
 ### time_field
@@ -135,8 +119,11 @@ commit_with_flush true
 <match something.logs>
   @type solr
 
-  # The Solr server url (for example http://localhost:8983/solr/collection1).
-  url http://localhost:8983/solr/collection1
+  # The Solr base url (for example http://localhost:8983/solr).
+  base_url http://localhost:8983/solr
+
+  # The Solr collection/core name (default collection1).
+  collection collection1
 </match>
 ```
 
@@ -148,83 +135,11 @@ commit_with_flush true
   # The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).
   zk_host localhost:2181/solr
 
-  # The SolrCloud collection name (default collection1).
+  # The Solr collection/core name (default collection1).
   collection collection1
 </match>
 ```
 
-## Solr setup examples
-
-### How to setup Standalone Solr using data-driven schemaless mode.
-
-1.Download and install Solr
-
-```sh
-$ mkdir $HOME/solr
-$ cd $HOME/solr
-$ wget https://archive.apache.org/dist/lucene/solr/5.4.0/solr-5.4.0.tgz
-$ tar zxvf solr-5.4.0.tgz
-$ cd solr-5.4.0
-```
-
-2.Start standalone Solr
-
-```sh
-$ ./bin/solr start -p 8983 -s server/solr
-```
-
-3.Create core
-
-```sh
-$ ./bin/solr create -c collection1 -d server/solr/configsets/data_driven_schema_configs -n collection1_configs
-```
-
-### How to setup SolrCloud using data-driven schemaless mode (shards=1 and replicationfactor=2).
-
-1.Download and install ZooKeeper
-
-```sh
-$ mkdir $HOME/zookeeper
-$ cd $HOME/zookeeper
-$ wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz
-$ tar zxvf zookeeper-3.4.6.tar.gz
-$ cd zookeeper-3.4.6
-$ cp -p ./conf/zoo_sample.cfg ./conf/zoo.cfg
-```
-
-2.Start standalone ZooKeeper
-
-```sh
-$ ./bin/zkServer.sh start
-```
-
-3.Download an install Solr
-
-```sh
-$ mkdir $HOME/solr
-$ cd $HOME/solr
-$ wget https://archive.apache.org/dist/lucene/solr/5.4.0/solr-5.4.0.tgz
-$ tar zxvf solr-5.4.0.tgz
-$ cd solr-5.4.0
-$ ./server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd clear /solr
-$ ./server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd makepath /solr
-$ cp -pr server/solr server/solr1
-$ cp -pr server/solr server/solr2
-```
-
-4.Start SolrCloud
-
-```sh
-$ ./bin/solr start -h localhost -p 8983 -z localhost:2181/solr -s server/solr1
-$ ./bin/solr start -h localhost -p 8985 -z localhost:2181/solr -s server/solr2
-```
-
-5.Create collection
-
-```sh
-$ ./bin/solr create -c collection1 -d server/solr1/configsets/data_driven_schema_configs -n collection1_configs -shards 1 -replicationFactor 2
-```
-
 ## Development
 
 After checking out the repo, run `bundle install` to install dependencies. Then, run `rake test` to run the tests.

data/fluent-plugin-output-solr.gemspec

@@ -3,21 +3,21 @@ lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
-  spec.name          = "fluent-plugin-output-solr"
-  spec.version       = "0.4.12"
-  spec.authors       = ["Minoru Osuka"]
-  spec.email         = ["minoru.osuka@gmail.com"]
+  spec.name          = 'fluent-plugin-output-solr'
+  spec.version       = '0.4.13'
+  spec.authors       = ['Minoru Osuka']
+  spec.email         = ['minoru.osuka@gmail.com']
 
-  spec.summary       = "Fluent output plugin for sending data to Apache Solr."
-  spec.description   = "Fluent output plugin for sending data to Apache Solr. It support SolrCloud not only Standalone Solr."
-  spec.homepage      = "https://github.com/mosuka/fluent-plugin-output-solr"
+  spec.summary       = 'Fluent output plugin for sending data to Apache Solr.'
+  spec.description   = 'Fluent output plugin for sending data to Apache Solr.'
+  spec.homepage      = 'https://github.com/mosuka/fluent-plugin-output-solr'
 
-  spec.license       = "Apache-2.0"
+  spec.license       = 'Apache-2.0'
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
   spec.add_runtime_dependency 'fluentd', '~> 0.12.0'
   spec.add_runtime_dependency 'rsolr-cloud', '~> 1.1.0'

data/fluent.conf

@@ -15,11 +15,17 @@
 
 <match messages>
   @type solr
-#  url http://localhost:8983/solr/collection1
-  zk_host localhost:2181/solr
+  base_url http://localhost:8983/solr
+  # zk_host localhost:2181/solr
   collection collection1
   ignore_undefined_fields false
+  tag_field tag
+  time_field time
+  time_format %FT%TZ
+  millisecond true
   flush_size 100
+  commit_with_flush true
+
   buffer_type memory
   buffer_queue_limit 64m
   buffer_chunk_limit 8m

data/lib/fluent/plugin/out_solr.rb

@@ -8,12 +8,15 @@ module Fluent
     Fluent::Plugin.register_output('solr', self)
 
     DEFAULT_COLLECTION = 'collection1'
-    DEFAULT_IGNORE_UNDEFINED_FIELDS = false
-    DEFAULT_STRING_FIELD_VALUE_MAX_LENGTH = -1
+
     DEFAULT_TAG_FIELD = 'tag'
+
     DEFAULT_TIME_FIELD = 'time'
     DEFAULT_TIME_FORMAT = '%FT%TZ'
     DEFAULT_MILLISECOND = false
+
+    DEFAULT_IGNORE_UNDEFINED_FIELDS = false
+
     DEFAULT_FLUSH_SIZE = 100
     DEFAULT_COMMIT_WITH_FLUSH = true
 
@@ -26,25 +29,21 @@ module Fluent
     include Fluent::SetTimeKeyMixin
     config_set_default :include_time_key, false
 
-    config_param :url, :string, :default => nil,
-                 :desc => 'The Solr server url (for example http://localhost:8983/solr/collection1).'
+    config_param :base_url, :string, :default => nil,
+                 :desc => 'The Solr base url (for example http://localhost:8983/solr).'
 
     config_param :zk_host, :string, :default => nil,
                  :desc => 'The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).'
+
     config_param :collection, :string, :default => DEFAULT_COLLECTION,
-                 :desc => 'The SolrCloud collection name (default collection1).'
+                 :desc => 'The Solr collection/core name (default collection1).'
 
-    config_param :defined_fields, :array, :default => nil,
-                 :desc => 'The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.'                 
     config_param :ignore_undefined_fields, :bool, :default => DEFAULT_IGNORE_UNDEFINED_FIELDS,
                  :desc => 'Ignore undefined fields in the Solr schema.xml.'
-    config_param :string_field_value_max_length, :integer, :default => DEFAULT_STRING_FIELD_VALUE_MAX_LENGTH,
-                 :desc => 'Field value max length.'
 
-    config_param :unique_key_field, :string, :default => nil,
-                 :desc => 'A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.'
     config_param :tag_field, :string, :default => DEFAULT_TAG_FIELD,
                  :desc => 'A field name of fluentd tag in the Solr schema.xml (default tag).'
+
     config_param :time_field, :string, :default => DEFAULT_TIME_FIELD,
                  :desc => 'A field name of event timestamp in the Solr schema.xml (default time).'
     config_param :time_format, :string, :default => DEFAULT_TIME_FORMAT,
@@ -70,7 +69,7 @@ module Fluent
       super
 
       @mode = nil
-      if ! @url.nil? then
+      if ! @base_url.nil? then
         @mode = MODE_STANDALONE
       elsif ! @zk_host.nil?
         @mode = MODE_SOLRCLOUD
@@ -80,7 +79,7 @@ module Fluent
       @zk = nil
 
       if @mode == MODE_STANDALONE then
-        @solr = RSolr.connect :url => @url
+        @solr = RSolr.connect :url => @base_url.end_with?('/') ? @base_url + @collection : @base_url + '/' + @collection
       elsif @mode == MODE_SOLRCLOUD then
         @zk = ZK.new(@zk_host)
         cloud_connection = RSolr::Cloud::Connection.new(@zk)
@@ -103,145 +102,114 @@ module Fluent
     def write(chunk)
       documents = []
 
-      @fields = @defined_fields.nil? ? get_fields : @defined_fields
-      @unique_key = @unique_key_field.nil? ? get_unique_key : @unique_key_field
+      # Get fields from Solr
+      fields = get_fields
+
+      # Get unique key field from Solr
+      unique_key = get_unique_key
 
       chunk.msgpack_each do |tag, time, record|
-        unless record.has_key?(@unique_key) then
-          record.merge!({@unique_key => SecureRandom.uuid})
+        # Set unique key and value
+        unless record.has_key?(unique_key) then
+          record.merge!({unique_key => SecureRandom.uuid})
         end
 
+        # Set Fluentd tag to Solr tag field
         unless record.has_key?(@tag_field) then
           record.merge!({@tag_field => tag})
         end
 
+        # Set time
+        tmp_time = Time.at(time).utc
         if record.has_key?(@time_field) then
+          # Parsing the time field in the record by the specified format.
           begin
             tmp_time = Time.strptime(record[@time_field], @time_format).utc
-            if @millisecond then
-              record.merge!({@time_field => '%s.%03dZ' % [tmp_time.strftime('%FT%T'), tmp_time.usec / 1000.0]})
-            else
-              record.merge!({@time_field => tmp_time.strftime('%FT%TZ')})
-            end
-          rescue
-            tmp_time = Time.at(time).utc
-            if @millisecond then
-              record.merge!({@time_field => '%s.%03dZ' % [tmp_time.strftime('%FT%T'), tmp_time.usec / 1000.0]})
-            else
-              record.merge!({@time_field => tmp_time.strftime('%FT%TZ')})
-            end
+          rescue Exception => e
+            log.warn "An error occurred in parsing the time field: #{e.message}"
           end
+        end
+        if @millisecond then
+          record.merge!({@time_field => '%s.%03dZ' % [tmp_time.strftime('%FT%T'), tmp_time.usec / 1000.0]})
         else
-          tmp_time = Time.at(time).utc
-          if @millisecond then
-            record.merge!({@time_field => '%s.%03dZ' % [tmp_time.strftime('%FT%T'), tmp_time.usec / 1000.0]})
-          else
-            record.merge!({@time_field => tmp_time.strftime('%FT%TZ')})
-          end
+          record.merge!({@time_field => tmp_time.strftime('%FT%TZ')})
         end
 
+        # Ignore undefined fields
         if @ignore_undefined_fields then
           record.each_key do |key|
-            unless @fields.include?(key) then
+            unless fields.include?(key) then
               record.delete(key)
             end
           end
         end
 
-        if @string_field_value_max_length >= 0 then
-          record.each_key do |key|
-            if record[key].instance_of?(Array) then
-              values = []
-              record[key].each do |value|
-                if value.instance_of?(String) then
-                  if value.length > @string_field_value_max_length then
-                    log.warn "#{key} is too long (#{value.length}, max is #{@string_field_value_max_length})."
-                    values.push(value.slice(0, @string_field_value_max_length))
-                  else
-                    values.push(value)
-                  end
-                end
-              end
-              record[key] = values
-            elsif record[key].instance_of?(String) then
-              if record[key].length > @string_field_value_max_length then
-                log.warn "#{key} is too long (#{record[key].length}, max is #{@string_field_value_max_length})."
-                record[key] = record[key].slice(0, @string_field_value_max_length)
-              end
-            end
-          end
-        end
-
-        #
-        # delete reserved fields
-        # https://cwiki.apache.org/confluence/display/solr/Defining+Fields
-        #
-        record.each_key do |key|
-          if key[0] == '_' and key[-1] == '_' then
-            record.delete(key)
-          end
-        end
-
+        # Add record to documents
         documents << record
 
+        # Update when flash size is reached
         if documents.count >= @flush_size
           update documents
           documents.clear
         end
       end
 
+      # Update remaining documents
       update documents unless documents.empty?
     end
 
     def update(documents)
-      if @mode == MODE_STANDALONE then
-        @solr.add documents, :params => {:commit => @commit_with_flush}
-        log.debug "Added #{documents.count} document(s) to Solr"
-      elsif @mode == MODE_SOLRCLOUD then
-        @solr.add documents, collection: @collection, :params => {:commit => @commit_with_flush}
-        log.debug "Added #{documents.count} document(s) to Solr"
+      begin
+        if @mode == MODE_STANDALONE then
+          @solr.add documents, :params => {:commit => @commit_with_flush}
+        elsif @mode == MODE_SOLRCLOUD then
+          @solr.add documents, collection: @collection, :params => {:commit => @commit_with_flush}
+        end
+        log.debug "Sent #{documents.count} document(s) to Solr"
+      rescue Exception
+        log.warn "An error occurred while sending #{documents.count} document(s) to Solr"
       end
-      rescue Exception => e
-        log.warn "An error occurred while indexing"
     end
 
     def get_unique_key
-      response = nil
-
-      if @mode == MODE_STANDALONE then
-        response = @solr.get 'schema/uniquekey'
-      elsif @mode == MODE_SOLRCLOUD then
-        response = @solr.get 'schema/uniquekey', collection: @collection
+      unique_key = 'id'
+
+      begin
+        response = nil
+        if @mode == MODE_STANDALONE then
+          response = @solr.get 'schema/uniquekey'
+        elsif @mode == MODE_SOLRCLOUD then
+          response = @solr.get 'schema/uniquekey', collection: @collection
+        end
+        unique_key = response['uniqueKey']
+        log.debug "Unique key: #{unique_key}"
+      rescue Exception
+        log.warn 'An error occurred while getting unique key'
       end
 
-      unique_key = response['uniqueKey']
-      log.debug "Unique key: #{unique_key}"
-
       return unique_key
-
-      rescue Exception => e
-        log.warn "An error occurred while getting unique key"
     end
 
     def get_fields
-      response = nil
+      fields = []
 
-      if @mode == MODE_STANDALONE then
-        response = @solr.get 'schema/fields'
-      elsif @mode == MODE_SOLRCLOUD then
-        response = @solr.get 'schema/fields', collection: @collection
-      end
+      begin
+        response = nil
 
-      fields = []
-      response['fields'].each do |field|
-        fields.push(field['name'])
+        if @mode == MODE_STANDALONE then
+          response = @solr.get 'schema/fields'
+        elsif @mode == MODE_SOLRCLOUD then
+          response = @solr.get 'schema/fields', collection: @collection
+        end
+        response['fields'].each do |field|
+          fields.push(field['name'])
+        end
+        log.debug "Fields: #{fields}"
+      rescue Exception
+        log.warn 'An error occurred while getting fields'
       end
-      log.debug "Fields: #{fields}"
 
       return fields
-
-      rescue Exception => e
-        log.warn "An error occurred while getting fields"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-output-solr
 version: !ruby/object:Gem::Version
-  version: 0.4.12
+  version: 0.4.13
 platform: ruby
 authors:
 - Minoru Osuka
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-21 00:00:00.000000000 Z
+date: 2017-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -150,8 +150,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.8
-description: Fluent output plugin for sending data to Apache Solr. It support SolrCloud
-  not only Standalone Solr.
+description: Fluent output plugin for sending data to Apache Solr.
 email:
 - minoru.osuka@gmail.com
 executables: []