logstash-input-elasticsearch 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f8380a3a71bd40498ef5838731f767fdcd095b75
-  data.tar.gz: e41dc06f80647729e588a1c36549269ca8aad48c
+  metadata.gz: 648519ea6b29fdcb5b95a9939080095fe7bf129f
+  data.tar.gz: 8e5c3a0d33d74007942e96806032fc08f7569ebf
 SHA512:
-  metadata.gz: 6e8802b10b181d74e845ca63ac6479c7d8d56bc012d0c163f7b84d10733b9acbd494b2de94c8960aa64d9eb6739b2d11babec180b0d64fa6c3a8d26528c08198
-  data.tar.gz: dd2988ee5fd12baf2b1753330cb4716797f62e97b9f34f9961615f60790fadaa19c7f09090b6aafa9b3d10f61446ff30f3f60e0f00176fec20d7a8bea8c0d574
+  metadata.gz: 40e81ca75e061380601bae58d415fa136ee59cce03de7c6dafc59402b117058659ac53d6de4e83d6f0d28acb030cc86cc3be7c27f115831bc59e8218f7393551
+  data.tar.gz: 56fa3228f843bdc67d2a4227e19989d8507f259c8ee8f9f589096923496860f22be0c6a15e8453ec1ee0aa27a08dbd45ef460d6cb56c25033fcd5af13316a7d7

data/CHANGELOG.md

@@ -1,2 +1,5 @@
+## 1.0.2 (September 3 - 2015)
+ - fix scan/scroll response handling
+
 ## 1.0.1
  - refactor request logic into own method (better memory gc perf)

data/lib/logstash/inputs/elasticsearch.rb

@@ -141,32 +141,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     @client = Elasticsearch::Client.new(:hosts => hosts, :transport_options => transport_options)
   end
 
-  private
-  def run_next(output_queue, scroll_id)
-    r = scroll_request(scroll_id)
-    r['hits']['hits'].each do |hit|
-      event = LogStash::Event.new(hit['_source'])
-      decorate(event)
-
-      if @docinfo
-        event[@docinfo_target] ||= {}
-
-        unless event[@docinfo_target].is_a?(Hash)
-          @logger.error("Elasticsearch Input: Incompatible Event, incompatible type for the `@metadata` field in the `_source` document, expected a hash got:", :metadata_type => event[@docinfo_target].class)
-
-          raise Exception.new("Elasticsearch input: incompatible event")
-        end
-
-        @docinfo_fields.each do |field|
-          event[@docinfo_target][field] = hit[field]
-        end
-      end
-      output_queue << event
-    end
-
-    {:has_hits => r['hits']['hits'].any?, :scroll_id => r['_scroll_id']}
-  end
-  
   public
   def run(output_queue)
 
@@ -175,14 +149,47 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
     # since 'scan' doesn't return data on the search call, do an extra scroll
     if @scan
-      resp = run_next(output_queue, r['_scroll_id'])
+      r = process_next_scroll(output_queue, r['_scroll_id'])
+      has_hits = r['has_hits']
+    else # not a scan, process the response
+      r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
+      has_hits = r['hits']['hits'].any?
     end
 
-    while resp[:has_hits] do
-      resp = run_next(output_queue, resp[:scroll_id])
+    while has_hits do
+      r = process_next_scroll(output_queue, r['_scroll_id'])
+      has_hits = r['has_hits']
     end
   end # def run
 
+  private
+  def process_next_scroll(output_queue, scroll_id)
+    r = scroll_request(scroll_id)
+    r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
+    {'has_hits' => r['hits']['hits'].any?, '_scroll_id' => r['_scroll_id']}
+  end
+
+  private
+  def push_hit(hit, output_queue)
+    event = LogStash::Event.new(hit['_source'])
+    decorate(event)
+
+    if @docinfo
+      event[@docinfo_target] ||= {}
+
+      unless event[@docinfo_target].is_a?(Hash)
+        @logger.error("Elasticsearch Input: Incompatible Event, incompatible type for the `@metadata` field in the `_source` document, expected a hash got:", :metadata_type => event[@docinfo_target].class)
+
+        raise Exception.new("Elasticsearch input: incompatible event")
+      end
+
+      @docinfo_fields.each do |field|
+        event[@docinfo_target][field] = hit[field]
+      end
+    end
+    output_queue << event
+  end
+
   private
   def scroll_request scroll_id
     @client.scroll(:body => scroll_id, :scroll => @scroll)

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '1.0.1'
+  s.version         = '1.0.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Read from an Elasticsearch cluster, based on search query results"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/elasticsearch_spec.rb

@@ -3,7 +3,7 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/elasticsearch"
 require "elasticsearch"
 
-describe "inputs/elasticsearch", :elasticsearch => true do
+describe LogStash::Inputs::Elasticsearch do
   it "should retrieve json event from elasticseach" do
     config = %q[
       input {
@@ -47,7 +47,9 @@ describe "inputs/elasticsearch", :elasticsearch => true do
     expect(client).to receive(:search).with(any_args).and_return(response)
     expect(client).to receive(:scroll).with({ :body => "cXVlcnlUaGVuRmV0Y2g", :scroll=> "1m" }).and_return(scroll_reponse)
 
-    event = fetch_event(config)
+    event = input(config) do |pipeline, queue|
+      queue.pop
+    end
 
     insist { event }.is_a?(LogStash::Event)
     insist { event["message"] } == [ "ohayo" ]
@@ -102,7 +104,9 @@ describe "inputs/elasticsearch", :elasticsearch => true do
     expect(client).to receive(:scroll).with({ :body => "DcrY3G1xff6SB", :scroll => "1m" }).and_return(scroll_responses.first)
     expect(client).to receive(:scroll).with({ :body=> "cXVlcnlUaGVuRmV0Y2g", :scroll => "1m" }).and_return(scroll_responses.last)
 
-    event = fetch_event(config)
+    event = input(config) do |pipeline, queue|
+      queue.pop
+    end
 
     insist { event }.is_a?(LogStash::Event)
     insist { event["message"] } == [ "ohayo" ]
@@ -181,7 +185,9 @@ describe "inputs/elasticsearch", :elasticsearch => true do
             }
         ]
 
-        event = fetch_event(config_metadata_with_hash)
+        event = input(config_metadata_with_hash) do |pipeline, queue|
+          queue.pop
+        end
 
         expect(event[metadata_field]["_index"]).to eq('logstash-2014.10.12')
         expect(event[metadata_field]["_type"]).to eq('logs')
@@ -214,7 +220,9 @@ describe "inputs/elasticsearch", :elasticsearch => true do
       end
 
       it "should move the document info to the @metadata field" do
-        event = fetch_event(config_metadata)
+        event = input(config_metadata) do |pipeline, queue|
+          queue.pop
+        end
 
         expect(event["[@metadata][_index]"]).to eq('logstash-2014.10.12')
         expect(event["[@metadata][_type]"]).to eq('logs')
@@ -233,7 +241,9 @@ describe "inputs/elasticsearch", :elasticsearch => true do
               }
             }
         ]
-        event = fetch_event(config)
+        event = input(config) do |pipeline, queue|
+          queue.pop
+        end
 
         expect(event["[meta][_index]"]).to eq('logstash-2014.10.12')
         expect(event["[meta][_type]"]).to eq('logs')
@@ -253,7 +263,9 @@ describe "inputs/elasticsearch", :elasticsearch => true do
               }
             }]
 
-        event = fetch_event(config)
+        event = input(config) do |pipeline, queue|
+          queue.pop
+        end
 
         expect(event["@metadata"].keys).to eq(fields)
         expect(event["[@metadata][_type]"]).to eq(nil)
@@ -273,7 +285,9 @@ describe "inputs/elasticsearch", :elasticsearch => true do
             }
           }
         ]
-        event = fetch_event(config)
+        event = input(config) do |pipeline, queue|
+          queue.pop
+        end
 
         expect(event["[@metadata][_index]"]).to eq(nil)
         expect(event["[@metadata][_type]"]).to eq(nil)
@@ -282,17 +296,3 @@ describe "inputs/elasticsearch", :elasticsearch => true do
     end
   end
 end
-
-def fetch_event(config)
-  pipeline = LogStash::Pipeline.new(config)
-  queue = Queue.new
-  pipeline.instance_eval do
-    @output_func = lambda { |event| queue << event }
-  end
-  pipeline_thread = Thread.new { pipeline.run }
-  event = queue.pop
-
-  pipeline_thread.join
-
-  return event
-end

metadata

@@ -1,18 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-08-28 00:00:00.000000000 Z
+date: 2015-09-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -20,7 +19,10 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -28,11 +30,8 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
-  name: elasticsearch
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -40,7 +39,10 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  requirement: !ruby/object:Gem::Requirement
+  name: elasticsearch
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -48,36 +50,34 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
-  name: logstash-codec-json
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-codec-json
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -117,7 +117,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key:
 specification_version: 4
 summary: Read from an Elasticsearch cluster, based on search query results