logstash-input-elasticsearch 4.2.1 → 4.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f67aae32541896feeb5a4e64d56c39585e9455ce68b5bc4a26b258a8468fdbf9
-  data.tar.gz: d9016dd6a8edf143be660c33ab207534e0932c19d8731e6d04ded2988098f4d1
+  metadata.gz: 2d8999f6e5261a2aedcf91e63941d0dbd0088af5969c7a132fdb9b2c64100985
+  data.tar.gz: e85ebd29d645319b5f498ce44e6bbea68f752ab480917e20237ff5482fd57492
 SHA512:
-  metadata.gz: 897449e042ff4061ccf7d3cbb2041af639e743e725351285ab374a23e0bc9d18083cfe4d901086404dd9bd405c20dda2c03c69aa6a35ae011f09dd562b71f6e4
-  data.tar.gz: e01e90e95e1c286c622590ad6a65f871fb4f1452e001faa9feffeba5958bd473521babed58084807e4f0510b6306543be9eaece93498be088bc06ef57b3fd5ae
+  metadata.gz: de04e26035cb7a0ab9448f630f6d833894c9c0c74de60f7a781efdf0eebafdb7c54692a31790543ca019932bd84751ff67366bcff78f9a739967c5a8df627452
+  data.tar.gz: 1984940fed8ca921ef26e8e4c30ce3c5c6bda4c0ed681676ca657c92a32e120518f2e4ac58d51a71e6951fde2fcd529a22227f05687517559b65991145217392

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.3.0
+  - Added managed slice scrolling with `slices` option
+
 ## 4.2.1
   - Docs: Set the default_codec doc attribute.
 

data/docs/index.asciidoc

@@ -97,6 +97,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-schedule>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-scroll>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-size>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-slices>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
 |=======================================================================
@@ -250,6 +251,30 @@ round trip (i.e. between the previous scroll request, to the next).
 
 This allows you to set the maximum number of hits returned per scroll.
 
+[id="plugins-{type}s-{plugin}-slices"]
+===== `slices`
+
+  * Value type is <<number,number>>
+  * There is no default value.
+  * Sensible values range from 2 to about 8.
+
+In some cases, it is possible to improve overall throughput by consuming multiple
+distinct slices of a query simultaneously using the
+https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html#sliced-scroll[Sliced Scroll API],
+especially if the pipeline is spending significant time waiting on Elasticsearch
+to provide results.
+
+If set, the `slices` parameter tells the plugin how many slices to divide the work
+into, and will produce events from the slices in parallel until all of them are done
+scrolling.
+
+NOTE: The Elasticsearch manual indicates that there can be _negative_ performance
+      implications to both the query and the Elasticsearch cluster when a scrolling
+      query uses more slices than shards in the index.
+
+If the `slices` parameter is left unset, the plugin will _not_ inject slice
+instructions into the query.
+
 [id="plugins-{type}s-{plugin}-ssl"]
 ===== `ssl` 
 

data/lib/logstash/inputs/elasticsearch.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "logstash/inputs/base"
 require "logstash/namespace"
+require "logstash/json"
 require "base64"
 
 # .Compatibility Note
@@ -83,6 +84,10 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # round trip (i.e. between the previous scroll request, to the next).
   config :scroll, :validate => :string, :default => "1m"
 
+  # This parameter controls the number of parallel slices to be consumed simultaneously
+  # by this pipeline input.
+  config :slices, :validate => :number
+
   # If set, include Elasticsearch document information such as index, type, and
   # the id in the event.
   #
@@ -147,10 +152,14 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
     @options = {
       :index => @index,
-      :body => @query,
       :scroll => @scroll,
       :size => @size
     }
+    @base_query = LogStash::Json.load(@query)
+    if @slices
+      @base_query.include?('slice') && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `query` option cannot specify specific `slice` when configured to manage parallel slices with `slices` option")
+      @slices < 1 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `slices` option must be greater than zero, got `#{@slices}`")
+    end
 
     transport_options = {}
 
@@ -196,16 +205,39 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   private
 
   def do_run(output_queue)
-    # get first wave of data
-    r = @client.search(@options)
+    # if configured to run a single slice, don't bother spinning up threads
+    return do_run_slice(output_queue) if @slices.nil? || @slices <= 1
+
+    logger.warn("managed slices for query is very large (#{@slices}); consider reducing") if @slices > 8
+
+    @slices.times.map do |slice_id|
+      Thread.new do
+        LogStash::Util::set_thread_name("#{@id}_slice_#{slice_id}")
+        do_run_slice(output_queue, slice_id)
+      end
+    end.map(&:join)
+  end
+
+  def do_run_slice(output_queue, slice_id=nil)
+    slice_query = @base_query
+    slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @slices}) unless slice_id.nil?
+
+    slice_options = @options.merge(:body => LogStash::Json.dump(slice_query) )
+
+    logger.info("Slice starting", slice_id: slice_id, slices: @slices) unless slice_id.nil?
+    r = search_request(slice_options)
 
     r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
+    logger.debug("Slice progress", slice_id: slice_id, slices: @slices) unless slice_id.nil?
+
     has_hits = r['hits']['hits'].any?
 
-    while has_hits && !stop?
+    while has_hits && r['_scroll_id'] && !stop?
       r = process_next_scroll(output_queue, r['_scroll_id'])
+      logger.debug("Slice progress", slice_id: slice_id, slices: @slices) unless slice_id.nil?
       has_hits = r['has_hits']
     end
+    logger.info("Slice complete", slice_id: slice_id, slices: @slices) unless slice_id.nil?
   end
 
   def process_next_scroll(output_queue, scroll_id)
@@ -243,4 +275,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   def scroll_request scroll_id
     @client.scroll(:body => { :scroll_id => scroll_id }, :scroll => @scroll)
   end
+
+  def search_request(options)
+    @client.search(options)
+  end
 end

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.2.1'
+  s.version         = '4.3.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/elasticsearch_spec.rb

@@ -84,6 +84,239 @@ describe LogStash::Inputs::Elasticsearch do
     insist { event.get("message") } == [ "ohayo" ]
   end
 
+
+  # This spec is an adapter-spec, ensuring that we send the right sequence of messages to our Elasticsearch Client
+  # to support sliced scrolling. The underlying implementation will spawn its own threads to consume, so we must be
+  # careful to use thread-safe constructs.
+  context "with managed sliced scrolling" do
+    let(:config) do
+      {
+          'query' => "#{LogStash::Json.dump(query)}",
+          'slices' => slices,
+          'docinfo' => true, # include ids
+      }
+    end
+    let(:query) do
+      {
+        "query" => {
+          "match" => { "city_name" => "Okinawa" }
+        },
+        "fields" => ["message"]
+      }
+    end
+    let(:slices) { 2 }
+
+    context 'with `slices => 0`' do
+      let(:slices) { 0 }
+      it 'fails to register' do
+        expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+
+    context 'with `slices => 1`' do
+      let(:slices) { 1 }
+      it 'runs just one slice' do
+        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
+        expect(Thread).to_not receive(:new)
+
+        plugin.register
+        plugin.run([])
+      end
+    end
+
+    context 'without slices directive' do
+      let(:config) { super().except('slices') }
+      it 'runs just one slice' do
+        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
+        expect(Thread).to_not receive(:new)
+
+        plugin.register
+        plugin.run([])
+      end
+    end
+
+    2.upto(8) do |slice_count|
+      context "with `slices => #{slice_count}`" do
+        let(:slices) { slice_count }
+        it "runs #{slice_count} independent slices" do
+          expect(Thread).to receive(:new).and_call_original.exactly(slice_count).times
+          slice_count.times do |slice_id|
+            expect(plugin).to receive(:do_run_slice).with(duck_type(:<<), slice_id)
+          end
+
+          plugin.register
+          plugin.run([])
+        end
+      end
+    end
+
+    # This section of specs heavily mocks the Elasticsearch::Client, and ensures that the Elasticsearch Input Plugin
+    # behaves as expected when handling a series of sliced, scrolled requests/responses.
+    context 'adapter/integration' do
+      let(:response_template) do
+        {
+            "took" => 12,
+            "timed_out" => false,
+            "shards" => {
+                "total" => 6,
+                "successful" => 6,
+                "failed" => 0
+            }
+        }
+      end
+
+      let(:hits_template) do
+        {
+            "total" => 4,
+            "max_score" => 1.0,
+            "hits" => []
+        }
+      end
+
+      let(:hit_template) do
+        {
+            "_index" => "logstash-2018.08.23",
+            "_type" => "logs",
+            "_score" => 1.0,
+            "_source" => { "message" => ["hello, world"] }
+        }
+      end
+
+      # BEGIN SLICE 0: a sequence of THREE scrolled responses containing 2, 1, and 0 items
+      # end-of-slice is reached when slice0_response2 is empty.
+      begin
+        let(:slice0_response0) do
+          response_template.merge({
+              "_scroll_id" => slice0_scroll1,
+              "hits" => hits_template.merge("hits" => [
+                  hit_template.merge('_id' => "slice0-response0-item0"),
+                  hit_template.merge('_id' => "slice0-response0-item1")
+                  ])
+          })
+        end
+        let(:slice0_scroll1) { 'slice:0,scroll:1' }
+        let(:slice0_response1) do
+          response_template.merge({
+              "_scroll_id" => slice0_scroll2,
+              "hits" => hits_template.merge("hits" => [
+                  hit_template.merge('_id' => "slice0-response1-item0")
+              ])
+          })
+        end
+        let(:slice0_scroll2) { 'slice:0,scroll:2' }
+        let(:slice0_response2) do
+          response_template.merge(
+              "_scroll_id" => slice0_scroll3,
+              "hits" => hits_template.merge({"hits" => []})
+          )
+        end
+        let(:slice0_scroll3) { 'slice:0,scroll:3' }
+      end
+      # END SLICE 0
+
+      # BEGIN SLICE 1: a sequence of TWO scrolled responses containing 2 and 2 items.
+      # end-of-slice is reached when slice1_response1 does not contain a next scroll id
+      begin
+        let(:slice1_response0) do
+          response_template.merge({
+              "_scroll_id" => slice1_scroll1,
+              "hits" => hits_template.merge("hits" => [
+                  hit_template.merge('_id' => "slice1-response0-item0"),
+                  hit_template.merge('_id' => "slice1-response0-item1")
+              ])
+          })
+        end
+        let(:slice1_scroll1) { 'slice:1,scroll:1' }
+        let(:slice1_response1) do
+          response_template.merge({
+              "hits" => hits_template.merge("hits" => [
+                  hit_template.merge('_id' => "slice1-response1-item0"),
+                  hit_template.merge('_id' => "slice1-response1-item1")
+              ])
+          })
+        end
+      end
+      # END SLICE 1
+
+      let(:client) { Elasticsearch::Client.new }
+
+      # RSpec mocks validations are not threadsafe.
+      # Allow caller to synchronize.
+      def synchronize_method!(object, method_name)
+        original_method = object.method(method_name)
+        mutex = Mutex.new
+        allow(object).to receive(method_name).with(any_args) do |*method_args, &method_block|
+          mutex.synchronize do
+            original_method.call(*method_args,&method_block)
+          end
+        end
+      end
+
+      before(:each) do
+        expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
+        plugin.register
+
+        # SLICE0 is a three-page scroll in which the last page is empty
+        slice0_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 0, 'max' => 2}))
+        expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
+        expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_return(slice0_response1)
+        expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll2 })).and_return(slice0_response2)
+
+        # SLICE1 is a two-page scroll in which the last page has no next scroll id
+        slice1_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 1, 'max' => 2}))
+        expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
+        expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_return(slice1_response1)
+
+        synchronize_method!(plugin, :scroll_request)
+        synchronize_method!(plugin, :search_request)
+      end
+
+      let(:emitted_events) do
+        queue = Queue.new # since we are running slices in threads, we need a thread-safe queue.
+        plugin.run(queue)
+        events = []
+        events << queue.pop until queue.empty?
+        events
+      end
+
+      let(:emitted_event_ids) do
+        emitted_events.map { |event| event.get('[@metadata][_id]') }
+      end
+
+      it 'emits the hits on the first page of the first slice' do
+        expect(emitted_event_ids).to include('slice0-response0-item0')
+        expect(emitted_event_ids).to include('slice0-response0-item1')
+      end
+      it 'emits the hits on the second page of the first slice' do
+        expect(emitted_event_ids).to include('slice0-response1-item0')
+      end
+
+      it 'emits the hits on the first page of the second slice' do
+        expect(emitted_event_ids).to include('slice1-response0-item0')
+        expect(emitted_event_ids).to include('slice1-response0-item1')
+      end
+
+      it 'emits the hitson the second page of the second slice' do
+        expect(emitted_event_ids).to include('slice1-response1-item0')
+        expect(emitted_event_ids).to include('slice1-response1-item1')
+      end
+
+      it 'does not double-emit' do
+        expect(emitted_event_ids.uniq).to eq(emitted_event_ids)
+      end
+
+      it 'emits events with appropriate fields' do
+        emitted_events.each do |event|
+          expect(event).to be_a(LogStash::Event)
+          expect(event.get('message')).to eq(['hello, world'])
+          expect(event.get('[@metadata][_id]')).to_not be_nil
+          expect(event.get('[@metadata][_id]')).to_not be_empty
+          expect(event.get('[@metadata][_index]')).to start_with('logstash-')
+        end
+      end
+    end
+  end
+
   context "with Elasticsearch document information" do
     let!(:response) do
       {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.2.1
+  version: 4.3.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-06 00:00:00.000000000 Z
+date: 2019-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -202,7 +202,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Reads query results from an Elasticsearch cluster