logstash-output-custom-solr 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e3205d97edee2f83daa2f520b29c26415162ab97a66f241c014a0d64004ba744
-  data.tar.gz: cf361397e3a85d1be0d8a97ab96576a7fbfe4a5e40585e3142f29635112a04bd
+  metadata.gz: 86a1d9be1b49383ea421b542e8f6b0e3c6c304b6508ec6e816d0b2909c8112a8
+  data.tar.gz: fbad8b4206fdb73820455507da630268669613dce94d8179a842a979188beb7c
 SHA512:
-  metadata.gz: c9da08f12bcb572406aa6777cccfa8b03b1d40338cac3a354326b2ebedae8c7daec70488d083cff9d60200b12afdf98b917d79f5e8e5f5638a51663da9e2271c
-  data.tar.gz: c54d0d8e30e69e8a6c73ec7c579255a00a7d6c85b05196f6173905e4d43e2301019200e8fd98aea1e4ac0e40a834772c05d5ec4e3f28bdc15d12b9f4abde9537
+  metadata.gz: '099c1b1bb09b013a0933bccf58e0f6535114b7ce062ef7ab6dfc130bea3fb2de6bbffb381f2b8e3594fa007994397291f416b8645645ec1184d2a188fad40e55'
+  data.tar.gz: 227b28125f95de3a9c11b0767f40f80bca346f2c7085ecdc638a81b0feaea8eec19d1dc4f3184cd2cf6a22115f37a5e1ef029bee0168b00e3bd1e4e34f2de11e

data/lib/logstash/outputs/solr.rb

@@ -22,15 +22,11 @@ class LogStash::Outputs::Solr < LogStash::Outputs::Base
   # The SolrCloud collection name.
   config :collection, :validate => :string, :default => 'collection1'
 
-  # The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.
-  config :defined_fields, :validate => :array, :default => nil
-  # Ignore undefined fields in the Solr schema.xml.
-  config :ignore_undefined_fields, :validate => :boolean, :default => false
+  # Commit every batch?
+  config :commit, :validate => :boolean, :default => false
 
-  # A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.
-  config :unique_key_field, :validate => :string, :default => nil
-  # A field name of event timestamp in the Solr schema.xml (default event_timestamp).
-  config :timestamp_field, :validate => :string, :default => 'event_timestamp'
+  # Solr commitWithin parameter
+  config :commitWithin, :validate => :number, :default => 10000
 
   # The batch size used in update.
   config :flush_size, :validate => :number, :default => 100
@@ -77,39 +73,24 @@ class LogStash::Outputs::Solr < LogStash::Outputs::Base
   def flush(events, close=false)
     documents = []
 
-    #@fields = @defined_fields.nil? || @defined_fields.empty? ? get_fields : @defined_fields
-
-    #@unique_key = @unique_key_field.nil? ? get_unique_key : @unique_key_field
-
     events.each do |event|
       document = event.to_hash()
 
-      #unless document.has_key?(@unique_key) then
-        #document.merge!({@unique_key => SecureRandom.uuid})
-      #end
-
-      #unless document.has_key?(@timestamp_field) then
-        #document.merge!({@timestamp_field => document['@timestamp']})
-      #end
-
-      #if @ignore_undefined_fields then
-        #document.each_key do |key|
-          #unless @fields.include?(key) then
-            #document.delete(key)
-          #end
-        #end
-      #end
-
       @logger.info 'Record: %s' % document.inspect
 
       documents.push(document)
     end
 
+    params = {}
+    if @commit
+      params[:commit] = true
+    end  
+    params[:commitWithin] = @commitWithin
     if @mode == MODE_STANDALONE then
-      @solr.add documents, :params => {:commit => true}
+      @solr.add documents, :params => params
       @logger.info 'Added %d document(s) to Solr' % documents.count
     elsif @mode == MODE_SOLRCLOUD then
-      @solr.add documents, collection: @collection, :params => {:commit => true}
+      @solr.add documents, collection: @collection, :params => params
       @logger.info 'Added %d document(s) to Solr' % documents.count
     end
 
@@ -124,44 +105,4 @@ class LogStash::Outputs::Solr < LogStash::Outputs::Base
     end
   end # def close
 
-  private
-  def get_unique_key
-    response = nil
-
-    if @mode == MODE_STANDALONE then
-      response = @solr.get 'schema/uniquekey'
-    elsif @mode == MODE_SOLRCLOUD then
-      response = @solr.get 'schema/uniquekey', collection: @collection
-    end
-
-    unique_key = response['uniqueKey']
-    @logger.info 'Unique key: #{unique_key}'
-
-    return unique_key
-
-    rescue Exception => e
-      @logger.warn("Unique key", :exception => e.inspect)
-  end # def get_unique_key
-
-  private
-  def get_fields
-    response = nil
-
-    if @mode == MODE_STANDALONE then
-      response = @solr.get 'schema/fields'
-    elsif @mode == MODE_SOLRCLOUD then
-      response = @solr.get 'schema/fields', collection: @collection
-    end
-
-    fields = []
-    response['fields'].each do |field|
-      fields.push(field['name'])
-    end
-    @logger.info 'Fields: #{fields}'
-
-    return fields
-
-    rescue Exception => e
-      @logger.warn("Could not get fields", :exception => e.inspect)
-  end # def get_fields
 end # class LogStash::Outputs::Solr

data/logstash-output-custom-solr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-custom-solr'
-  s.version = "0.1.5"
+  s.version = "0.1.6"
   s.licenses = ["Apache License (2.0)"]
   s.summary = "Logstash output plugin for sending data to Solr."
   s.description = "Logstash output plugin for sending data to Solr. It supports SolrCloud, not only Standalone Solr."
@@ -23,7 +23,7 @@ Gem::Specification.new do |s|
   
   s.add_runtime_dependency 'rsolr', '~> 1.1.2'
   s.add_runtime_dependency 'zk'
-  s.add_runtime_dependency 'rsolr-cloud'
+  s.add_runtime_dependency 'rsolr-cloud-custom'
   s.add_runtime_dependency 'stud'
 
   s.add_development_dependency "logstash-core", ">= 1.60", "<= 2.99"

data/spec/outputs/solr_spec.rb

@@ -19,11 +19,9 @@ describe LogStash::Outputs::Solr do
         'url' => 'http://localhost:8983/solr/collection1',
         'zk_host' => 'localhost:2181/solr',
         'collection' => 'collection1',
-        'defined_fields' => ['id', 'title'],
-        'ignore_undefined_fields' => true,
-        'unique_key_field' => 'id',
-        'timestamp_field' => 'event_timestamp',
-        'flush_size' => 100
+        'flush_size' => 100,
+        'commit' => false,
+        'commitWithin' => 10000
       }
     }
 
@@ -42,29 +40,19 @@ describe LogStash::Outputs::Solr do
       expect(output.config['collection']).to eq('collection1')
     end
 
-    it 'defined_fields' do
-      output = LogStash::Outputs::Solr.new(config)
-      expect(output.config['defined_fields']).to eq(['id', 'title'])
-    end
-
-    it 'ignore_undefined_fields' do
-      output = LogStash::Outputs::Solr.new(config)
-      expect(output.config['ignore_undefined_fields']).to eq(true)
-    end
-    
-    it 'unique_key_field' do
+    it 'flush_size' do
       output = LogStash::Outputs::Solr.new(config)
-      expect(output.config['unique_key_field']).to eq('id')
+      expect(output.config['flush_size']).to eq(100)
     end
 
-    it 'timestamp_field' do
+    it 'commit' do
       output = LogStash::Outputs::Solr.new(config)
-      expect(output.config['timestamp_field']).to eq('event_timestamp')
+      expect(output.config['commit']).to eq(false)
     end
 
-    it 'flush_size' do
+    it 'commitWithin' do
       output = LogStash::Outputs::Solr.new(config)
-      expect(output.config['flush_size']).to eq(100)
+      expect(output.config['commitWithin']).to eq(10000)
     end
   end
 
@@ -72,10 +60,6 @@ describe LogStash::Outputs::Solr do
     let(:config) {
       {
         'url' => 'http://localhost:8983/solr/collection1',
-        'defined_fields' => ['id', 'title'],
-        'ignore_undefined_fields' => true,
-        'unique_key_field' => 'id',
-        'timestamp_field' => 'event_timestamp',
         'flush_size' => 100
       }
     }
@@ -102,10 +86,6 @@ describe LogStash::Outputs::Solr do
       {
         'zk_host' => 'localhost:3292/solr',
         'collection' => 'collection1',
-        'defined_fields' => ['id', 'title'],
-        'ignore_undefined_fields' => true,
-        'unique_key_field' => 'id',
-        'timestamp_field' => 'event_timestamp',
         'flush_size' => 100
       }
     }
@@ -123,10 +103,6 @@ describe LogStash::Outputs::Solr do
     let(:config) {
       {
         'url' => 'http://localhost:8983/solr/collection1',
-        'defined_fields' => ['id', 'title'],
-        'ignore_undefined_fields' => true,
-        'unique_key_field' => 'id',
-        'timestamp_field' => 'event_timestamp',
         'flush_size' => 100
       }
     }
@@ -159,10 +135,6 @@ describe LogStash::Outputs::Solr do
       {
         'zk_host' => 'localhost:3292/solr',
         'collection' => 'collection1',
-        'defined_fields' => ['id', 'title'],
-        'ignore_undefined_fields' => true,
-        'unique_key_field' => 'id',
-        'timestamp_field' => 'event_timestamp',
         'flush_size' => 100
       }
     }
@@ -194,15 +166,9 @@ describe LogStash::Outputs::Solr do
     zk = ZK.new('localhost:3292')
     delete_nodes(zk, '/solr')
     create_nodes(zk, '/solr/live_nodes')
-    create_nodes(zk, '/solr/collections')
     ['localhost:8983_solr'].each do |node|
       zk.create("/solr/live_nodes/#{node}", '', mode: :ephemeral)
     end
-    ['collection1'].each do |collection|
-      zk.create("/solr/collections/#{collection}")
-      json = File.read("spec/files/collections/#{collection}/state.json")
-      zk.create("/solr/collections/#{collection}/state.json", json, mode: :ephemeral)
-    end
   end
 
   def stop_zookeeper

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-custom-solr
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Minoru Osuka
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-12-18 00:00:00.000000000 Z
+date: 2018-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -78,7 +78,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rsolr-cloud
+  name: rsolr-cloud-custom
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
@@ -177,7 +177,6 @@ files:
 - README.md
 - lib/logstash/outputs/solr.rb
 - logstash-output-custom-solr.gemspec
-- spec/files/collections/collection1/state.json
 - spec/outputs/solr_spec.rb
 homepage: https://github.com/mosuka/logstash-output-solr
 licenses:
@@ -206,5 +205,4 @@ signing_key:
 specification_version: 4
 summary: Logstash output plugin for sending data to Solr.
 test_files:
-- spec/files/collections/collection1/state.json
 - spec/outputs/solr_spec.rb

data/spec/files/collections/collection1/state.json

@@ -1,14 +0,0 @@
-{"collection1":{
-    "replicationFactor":"1",
-    "router":{"name":"compositeId"},
-    "maxShardsPerNode":"1",
-    "autoAddReplicas":"false",
-    "shards":{"shard1":{
-        "range":"80000000-7fffffff",
-        "state":"active",
-        "replicas":{"core_node1":{
-            "core":"collection1_shard1_replica1",
-            "base_url":"http://localhost:8983/solr",
-            "node_name":"localhost:8983_solr",
-            "state":"active",
-            "leader":"true"}}}}}}