fluent-plugin-output-solr 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ea700ae417e3291c67e294e9e64af1bd87315385
-  data.tar.gz: 42a95923a0023f2be86a63e247513607bd4fa259
+  metadata.gz: b2190d11bd62fbf4efbd390ae997e2ee4d962051
+  data.tar.gz: fd86da124f5ed9b94fdaffdb0c55e076e888806d
 SHA512:
-  metadata.gz: 148ec34675f9f1c1ba835848f88227786afba04f47ce709fee423c28f4dbed5a630f587cb2dd99c73f52e9496449bc72e2d4f2b7374270b697b3195c2a06f04b
-  data.tar.gz: 3ceb8cdfdec6e4105f0783414efaa7d217309d8f1e816839ea115779b48c72e0d9874651a6dde31ab227ac9d7aa4305ce9a908e2329791b3ae15ef165bb0bb27
+  metadata.gz: cc5031b355be897054a0577a9f2d0623e13274002ee8534ed0d20de9c4f7c8f851d8cf52017ce3c3d4d2fcf8d5f113a5090cb70ded850fa71f839ca293913c3e
+  data.tar.gz: 21da9bb4a7a7954ec3363e7acae5a11fe6fbf5ae0789e1bf48207b66be698766c5a6189c86288020f55ff20ef26be04da94be3f8253c44935935b5c9576b1594

data/README.md

@@ -1,6 +1,6 @@
 # Fluent::Plugin::OutSolr
 
-This is a [Fluentd](http://fluentd.org/) plugin for send data to [Apache Solr](http://lucene.apache.org/solr/).
+This is a [Fluentd](http://fluentd.org/) output plugin for send data to [Apache Solr](http://lucene.apache.org/solr/). It support [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud) not only Standalone Solr.
 
 ## Installation
 
@@ -18,17 +18,83 @@ Or install it yourself as:
 
     $ gem install fluent-plugin-output-solr
 
-## Examples
+## Config parameters
 
-### Sent to standalone Solr
+### url
+
+The Solr server url (for example http://localhost:8983/solr/collection1).
+
+```
+url http://localhost:8983/solr/collection1
+```
+
+### zk_host
+
+The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).
+
+```
+zk_host localhost:2181/solr
+```
+
+### collection
+
+The SolrCloud collection name (default collection1).
+
+```
+collection collection1
+```
+
+### defined_fields
+
+The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.
+
+```
+defined_fields ["id", "title"]
+```
+
+### ignore_undefined_fields
+
+Ignore undefined fields in the Solr schema.xml.
+
+```
+ignore_undefined_fields false
+```
+
+### unique_key_field
+
+A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.
+
+```
+unique_key_field id
+```
+
+### timestamp_field
+
+A field name of event timestamp in the Solr schema.xml (default event_timestamp).
+
+```
+timestamp_field event_timestamp
+```
+
+### flush_size
+
+A number of events to queue up before writing to Solr (default 100).
+
+```
+flush_size 100
+```
+
+
+## Plugin setup examples
+
+### Sent to standalone Solr using data-driven schemaless mode.
 ```
 <match something.logs>
   @type solr
 
+  # The Solr server url (for example http://localhost:8983/solr/collection1).
   url http://localhost:8983/solr/collection1
 
-  batch_size 100
-
   buffer_type memory
   buffer_queue_limit 64m
   buffer_chunk_limit 8m
@@ -36,15 +102,16 @@ Or install it yourself as:
 </match>
 ```
 
-### Sent to SolrCloud
+### Sent to SolrCloud using data-driven schemaless mode.
 ```
 <match something.logs>
   @type solr
 
+  # The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).
   zk_host localhost:2181/solr
-  collection collection1
 
-  batch_size 100
+  # The SolrCloud collection name (default collection1).
+  collection1 collection1
 
   buffer_type memory
   buffer_queue_limit 64m
@@ -53,6 +120,78 @@ Or install it yourself as:
 </match>
 ```
 
+## Solr setup examples
+
+### How to setup Standalone Solr using data-driven schemaless mode.
+
+1.Download and install Solr
+
+```sh
+$ mkdir $HOME/solr
+$ cd $HOME/solr
+$ wget https://archive.apache.org/dist/lucene/solr/5.4.0/solr-5.4.0.tgz
+$ tar zxvf solr-5.4.0.tgz
+$ cd solr-5.4.0
+```
+
+2.Start standalone Solr
+
+```sh
+$ ./bin/solr start -p 8983 -s server/solr
+```
+
+3.Create core
+
+```sh
+$ ./bin/solr create -c collection1 -d server/solr/configsets/data_driven_schema_configs -n collection1_configs
+```
+
+### How to setup SolrCloud using data-driven schemaless mode (shards=1 and replicationfactor=2).
+
+1.Download and install ZooKeeper
+
+```sh
+$ mkdir $HOME/zookeeper
+$ cd $HOME/zookeeper
+$ wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz
+$ tar zxvf zookeeper-3.4.6.tar.gz
+$ cd zookeeper-3.4.6
+$ cp -p ./conf/zoo_sample.cfg ./conf/zoo.cfg
+```
+
+2.Start standalone ZooKeeper
+
+```sh
+$ ./bin/zkServer.sh start
+```
+
+3.Download an install Solr
+
+```sh
+$ mkdir $HOME/solr
+$ cd $HOME/solr
+$ wget https://archive.apache.org/dist/lucene/solr/5.4.0/solr-5.4.0.tgz
+$ tar zxvf solr-5.4.0.tgz
+$ cd solr-5.4.0
+$ ./server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd clear /solr
+$ ./server/scripts/cloud-scripts/zkcli.sh -zkhost localhost:2181 -cmd makepath /solr
+$ cp -pr server/solr server/solr1
+$ cp -pr server/solr server/solr2
+```
+
+4.Start SolrCloud
+
+```sh
+$ ./bin/solr start -p 8983 -z localhost:2181/solr -s server/solr1
+$ ./bin/solr start -p 8985 -z localhost:2181/solr -s server/solr2
+```
+
+5.Create collection
+
+```sh
+$ ./bin/solr create -c collection1 -d server/solr1/configsets/data_driven_schema_configs -n collection1_configs -shards 1 -replicationFactor 2
+```
+
 ## Development
 
 After checking out the repo, run `bundle install` to install dependencies. Then, run `rake test` to run the tests.

data/fluent-plugin-output-solr.gemspec

@@ -4,12 +4,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-output-solr"
-  spec.version       = "0.1.1"
+  spec.version       = "0.2.0"
   spec.authors       = ["Minoru Osuka"]
   spec.email         = ["minoru.osuka@gmail.com"]
 
-  spec.summary       = %q{Solr output plugin for Fluent event collector}
-  spec.description   = spec.summary
+  spec.summary       = "Fluent output plugin for sending data to Apache Solr."
+  spec.description   = "Fluent output plugin for sending data to Apache Solr. It support SolrCloud not only Standalone Solr."
   spec.homepage      = "https://github.com/mosuka/fluent-plugin-output-solr"
 
   spec.license       = "Apache-2.0"

data/lib/fluent/plugin/out_solr.rb

@@ -7,19 +7,40 @@ module Fluent
   class SolrOutput < BufferedOutput
     Fluent::Plugin.register_output('solr', self)
 
+    DEFAULT_COLLECTION = 'collection1'
+    DEFAULT_IGNORE_UNDEFINED_FIELDS = false
+    DEFAULT_TIMESTAMP_FIELD = 'event_timestamp'
+    DEFAULT_FLUSH_SIZE = 100
+
+    MODE_STANDALONE = 'Standalone'
+    MODE_SOLRCLOUD = 'SolrCloud'
+
+    include Fluent::SetTagKeyMixin
+    config_set_default :include_tag_key, false
+
+    include Fluent::SetTimeKeyMixin
+    config_set_default :include_time_key, false
+
     config_param :url, :string, :default => nil,
-                  :desc => 'The Solr server url (for example http://localhost:8983/solr/collection1).'
+                 :desc => 'The Solr server url (for example http://localhost:8983/solr/collection1).'
 
     config_param :zk_host, :string, :default => nil,
-                  :desc => 'The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).'
-    config_param :collection, :string, :default => 'collection1',
-                  :desc => 'The SolrCloud collection name.'
+                 :desc => 'The ZooKeeper connection string that SolrCloud refers to (for example localhost:2181/solr).'
+    config_param :collection, :string, :default => DEFAULT_COLLECTION,
+                 :desc => 'The SolrCloud collection name (default collection1).'
 
-    config_param :batch_size, :integer, :default => 100,
-                  :desc => 'The batch size used in update.'
+    config_param :defined_fields, :array, :default => nil,
+                 :desc => 'The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.'                 
+    config_param :ignore_undefined_fields, :bool, :default => DEFAULT_IGNORE_UNDEFINED_FIELDS,
+                 :desc => 'Ignore undefined fields in the Solr schema.xml.'                 
 
-    MODE_STANDALONE = 'Standalone'
-    MODE_SOLRCLOUD = 'SolrCloud'
+    config_param :unique_key_field, :string, :default => nil,
+                 :desc => 'A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.'
+    config_param :timestamp_field, :string, :default => DEFAULT_TIMESTAMP_FIELD,
+                 :desc => 'A field name of event timestamp in the Solr schema.xml (default event_timestamp).'
+
+    config_param :flush_size, :integer, :default => DEFAULT_FLUSH_SIZE,
+                 :desc => 'A number of events to queue up before writing to Solr (default 100).'
 
     def initialize
       super
@@ -31,9 +52,15 @@ module Fluent
       @url = conf['url']
 
       @zk_host = conf['zk_host']
-      @collection = conf['collection']
+      @collection = conf.has_key?('collection') ? conf['collection'] : DEFAULT_COLLECTION
+
+      @defined_fields = conf['defined_fields']
+      @ignore_undefined_field = conf.has_key?('ignore_undefined_field') ? conf['ignore_undefined_field'] : DEFAULT_IGNORE_UNDEFINED_FIELDS
 
-      @batch_size = conf['batch_size'].to_i
+      @unique_key_field = conf['unique_key_field']
+      @timestamp_field = conf.has_key?('timestamp_field') ? conf['timestamp_field'] : DEFAULT_TIMESTAMP_FIELD
+
+      @flush_size = conf.has_key?('flush_size') ? conf['flush_size'].to_i : DEFAULT_FLUSH_SIZE
     end
 
     def start
@@ -73,19 +100,33 @@ module Fluent
     def write(chunk)
       documents = []
 
+      @fields = @defined_fields.nil? ? get_fields : @defined_fields
+      @unique_key = @unique_key_field.nil? ? get_unique_key : @unique_key_field
+
       chunk.msgpack_each do |tag, time, record|
-        unless record.has_key?('id') then
-          record.merge!({'id' => SecureRandom.uuid})
+
+        unless record.has_key?(@unique_key) then
+          record.merge!({@unique_key => SecureRandom.uuid})
+        end
+
+        record.merge!({@timestamp_field => Time.at(time).utc.strftime('%FT%TZ')})
+
+        if @ignore_undefined_fields then
+          record.each_key do |key|
+            unless @fields.include?(key) then
+              record.delete(key)
+            end
+          end
         end
 
         documents << record
-      
-        if documents.count >= @batch_size
+
+        if documents.count >= @flush_size
           update documents
           documents.clear
         end
       end
-      
+
       update documents unless documents.empty?
     end
 
@@ -97,6 +138,47 @@ module Fluent
         @solr.add documents, collection: @collection, :params => {:commit => true}
         log.info "Added %d document(s) to Solr" % documents.count
       end
+      rescue Exception => e
+        log.warn("An error occurred while indexing: #{e.message}")
+    end
+
+    def get_unique_key
+      response = nil
+
+      if @mode == MODE_STANDALONE then
+        response = @solr.get 'schema/uniquekey'
+      elsif @mode == MODE_SOLRCLOUD then
+        response = @solr.get 'schema/uniquekey', collection: @collection
+      end
+
+      unique_key = response['uniqueKey']
+      log.info ("Unique key: #{unique_key}")
+
+      return unique_key
+
+      rescue Exception => e
+        log.warn("An error occurred while indexing: #{e.message}")
+    end
+
+    def get_fields
+      response = nil
+
+      if @mode == MODE_STANDALONE then
+        response = @solr.get 'schema/fields'
+      elsif @mode == MODE_SOLRCLOUD then
+        response = @solr.get 'schema/fields', collection: @collection
+      end
+
+      fields = []
+      response['fields'].each do |field|
+        fields.push(field['name'])
+      end
+      log.info ("Fields: #{fields}")
+
+      return fields
+
+      rescue Exception => e
+        log.warn("An error occurred while indexing: #{e.message}")
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-output-solr
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Minoru Osuka
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-01-07 00:00:00.000000000 Z
+date: 2016-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -150,7 +150,8 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.1.8
-description: Solr output plugin for Fluent event collector
+description: Fluent output plugin for sending data to Apache Solr. It support SolrCloud
+  not only Standalone Solr.
 email:
 - minoru.osuka@gmail.com
 executables: []
@@ -188,5 +189,5 @@ rubyforge_project:
 rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
-summary: Solr output plugin for Fluent event collector
+summary: Fluent output plugin for sending data to Apache Solr.
 test_files: []