logstash-input-elasticsearch 4.14.0 → 4.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1bd8f2c20c31a39f42e8f32e32edbfef275a15de8040491b8d2549f55a3e9002
-  data.tar.gz: 68abc131e2c9ec958cc3360137b273b39303923d60b7dca4303aa43bf1425574
+  metadata.gz: 9379a31460bf649615c038b1b35d207f6a6db869bb1a27e963da70e278ae6bcd
+  data.tar.gz: f62823b8ddfb587ce9614a1d131be1bce2186dbd859287b1107757917a754c7e
 SHA512:
-  metadata.gz: 801739c8baf2655104a502176b42157c35c282dab719ec0bf0eb4274feef9ce6e79238870aa6ebb2604f4da726446089e94d4e5e8d48c50d731eaae55c447be9
-  data.tar.gz: 5af8ea0a963ce1d8d924b51428dbb70c9ab9e9d0df0146e2b5f0013ae5b13e56b934dc2540e7ce11080b4ed33c52afb66695bd08739a6a701c337b11d94a3c72
+  metadata.gz: c675ed6a5d1a4a104313611e4895171c6461149d0415c5ad4c276e42b5b99672584ae38ddc37f3e32521074727051c68f0d10a37b77064d0cea6ea03f041a3b9
+  data.tar.gz: beba1b98be77880e6e3712cebdd0474d68c33556a86236e828a3332d2e812b187e920e78f6fb006b27b82269faa0e0654568d050f7057ae7db48da7fd9a20e31

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.15.0
+  - Feat: add `retries` option. allow retry for failing query [#179](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/179)
+
 ## 4.14.0
   - Refactor: switch to using scheduler mixin [#177](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/177)
 

data/docs/index.asciidoc

@@ -124,6 +124,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-socket_timeout_seconds>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No
+| <<plugins-{type}s-{plugin}-retries>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
 |=======================================================================
 
@@ -339,6 +340,17 @@ The maximum amount of time, in seconds, for a single request to Elasticsearch.
 Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload
 documents and/or the <<plugins-{type}s-{plugin}-size>> has been specified as a large value.
 
+
+[id="plugins-{type}s-{plugin}-retries"]
+===== `retries`
+
+* Value type is <<number,number>>
+* Default value is `0`
+
+The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message.
+The default is 0 (no retry). This value should be equal to or greater than zero.
+
+
 [id="plugins-{type}s-{plugin}-schedule"]
 ===== `schedule` 
 
@@ -424,6 +436,7 @@ When the `target` is set to a field reference, the `_source` of the hit is place
 This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced.
 It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`).
 
+
 [id="plugins-{type}s-{plugin}-user"]
 ===== `user` 
 

data/lib/logstash/helpers/loggable_try.rb

@@ -0,0 +1,18 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+require 'stud/try'
+
+module LogStash module Helpers
+  class LoggableTry < Stud::Try
+    def initialize(logger, name)
+      @logger = logger
+      @name = name
+    end
+
+    def log_failure(exception, fail_count, message)
+      @logger.warn("Attempt to #{@name} but failed. #{message}", fail_count: fail_count, exception: exception.message)
+    end
+  end
+end end

data/lib/logstash/inputs/elasticsearch.rb

@@ -10,6 +10,7 @@ require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
 require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
 require "logstash/plugin_mixins/scheduler"
 require "base64"
+require 'logstash/helpers/loggable_try'
 
 require "elasticsearch"
 require "elasticsearch/transport/transport/http/manticore"
@@ -99,6 +100,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # This allows you to set the maximum number of hits returned per scroll.
   config :size, :validate => :number, :default => 1000
 
+  # The number of retries to run the query. If the query fails after all retries, it logs an error message.
+  config :retries, :validate => :number, :default => 0
+
   # This parameter controls the keepalive time in seconds of the scrolling
   # request and initiates the scrolling process. The timeout applies per
   # round trip (i.e. between the previous scroll request, to the next).
@@ -221,6 +225,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
       @slices < 1 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `slices` option must be greater than zero, got `#{@slices}`")
     end
 
+    @retries < 0 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `retries` option must be equal or greater than zero, got `#{@retries}`")
+
     validate_authentication
     fill_user_password_from_cloud_auth
     fill_hosts_from_cloud_id
@@ -262,22 +268,67 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   end
 
   private
-
+  JOB_NAME = "run query"
   def do_run(output_queue)
     # if configured to run a single slice, don't bother spinning up threads
-    return do_run_slice(output_queue) if @slices.nil? || @slices <= 1
+    if @slices.nil? || @slices <= 1
+      success, events = retryable_slice
+      success && events.each { |event| output_queue << event }
+      return
+    end
 
     logger.warn("managed slices for query is very large (#{@slices}); consider reducing") if @slices > 8
 
+    slice_results = parallel_slice # array of tuple(ok, events)
+
+    # insert events to queue if all slices success
+    if slice_results.all?(&:first)
+      slice_results.flat_map { |success, events| events }
+                  .each { |event| output_queue << event }
+    end
+
+    logger.trace("#{@slices} slices completed")
+  end
+
+  def retryable(job_name, &block)
+    begin
+      stud_try = ::LogStash::Helpers::LoggableTry.new(logger, job_name)
+      output = stud_try.try((@retries + 1).times) { yield }
+      [true, output]
+    rescue => e
+      error_details = {:message => e.message, :cause => e.cause}
+      error_details[:backtrace] = e.backtrace if logger.debug?
+      logger.error("Tried #{job_name} unsuccessfully", error_details)
+      [false, nil]
+    end
+  end
+
+
+  # @return [(ok, events)] : Array of tuple(Boolean, [Logstash::Event])
+  def parallel_slice
     pipeline_id = execution_context&.pipeline_id || 'main'
     @slices.times.map do |slice_id|
       Thread.new do
         LogStash::Util::set_thread_name("[#{pipeline_id}]|input|elasticsearch|slice_#{slice_id}")
-        do_run_slice(output_queue, slice_id)
+        retryable_slice(slice_id)
       end
-    end.map(&:join)
+    end.map do |t|
+      t.join
+      t.value
+    end
   end
 
+  # @param scroll_id [Integer]
+  # @return (ok, events) [Boolean, Array(Logstash::Event)]
+  def retryable_slice(slice_id=nil)
+    retryable(JOB_NAME) do
+      output = []
+      do_run_slice(output, slice_id)
+      output
+    end
+  end
+
+
   def do_run_slice(output_queue, slice_id=nil)
     slice_query = @base_query
     slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @slices}) unless slice_id.nil?
@@ -314,11 +365,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     r = scroll_request(scroll_id)
     r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
     [r['hits']['hits'].any?, r['_scroll_id']]
-  rescue => e
-    # this will typically be triggered by a scroll timeout
-    logger.error("Scroll request error, aborting scroll", message: e.message, exception: e.class)
-    # return no hits and original scroll_id so we can try to clear it
-    [false, scroll_id]
   end
 
   def push_hit(hit, output_queue)
@@ -353,7 +399,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     logger.warn("Ignoring clear_scroll exception", message: e.message, exception: e.class)
   end
 
-  def scroll_request scroll_id
+  def scroll_request(scroll_id)
     @client.scroll(:body => { :scroll_id => scroll_id }, :scroll => @scroll)
   end
 

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.14.0'
+  s.version         = '4.15.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/elasticsearch_spec.rb

@@ -51,6 +51,17 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
       end
     end
+
+    context "retry" do
+      let(:config) do
+        {
+          "retries" => -1
+        }
+      end
+      it "should raise an exception with negative number" do
+        expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
   end
 
   it_behaves_like "an interruptible input plugin" do
@@ -193,7 +204,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     context 'with `slices => 1`' do
       let(:slices) { 1 }
       it 'runs just one slice' do
-        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
+        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<), nil)
         expect(Thread).to_not receive(:new)
 
         plugin.register
@@ -204,7 +215,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     context 'without slices directive' do
       let(:config) { super().tap { |h| h.delete('slices') } }
       it 'runs just one slice' do
-        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
+        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<), nil)
         expect(Thread).to_not receive(:new)
 
         plugin.register
@@ -315,7 +326,6 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       end
       # END SLICE 1
 
-      let(:client) { Elasticsearch::Client.new }
 
       # RSpec mocks validations are not threadsafe.
       # Allow caller to synchronize.
@@ -329,69 +339,112 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         end
       end
 
-      before(:each) do
-        expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
-        plugin.register
+      describe "with normal response" do
+        before(:each) do
+          expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
+          plugin.register
 
-        expect(client).to receive(:clear_scroll).and_return(nil)
+          expect(client).to receive(:clear_scroll).and_return(nil)
 
-        # SLICE0 is a three-page scroll in which the last page is empty
-        slice0_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 0, 'max' => 2}))
-        expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
-        expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_return(slice0_response1)
-        expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll2 })).and_return(slice0_response2)
-        allow(client).to receive(:ping)
+          # SLICE0 is a three-page scroll in which the last page is empty
+          slice0_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 0, 'max' => 2}))
+          expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_return(slice0_response1)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll2 })).and_return(slice0_response2)
+          allow(client).to receive(:ping)
 
-        # SLICE1 is a two-page scroll in which the last page has no next scroll id
-        slice1_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 1, 'max' => 2}))
-        expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
-        expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_return(slice1_response1)
+          # SLICE1 is a two-page scroll in which the last page has no next scroll id
+          slice1_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 1, 'max' => 2}))
+          expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_return(slice1_response1)
 
-        synchronize_method!(plugin, :scroll_request)
-        synchronize_method!(plugin, :search_request)
-      end
+          synchronize_method!(plugin, :scroll_request)
+          synchronize_method!(plugin, :search_request)
+        end
 
-      let(:emitted_events) do
-        queue = Queue.new # since we are running slices in threads, we need a thread-safe queue.
-        plugin.run(queue)
-        events = []
-        events << queue.pop until queue.empty?
-        events
-      end
+        let(:client) { Elasticsearch::Client.new }
 
-      let(:emitted_event_ids) do
-        emitted_events.map { |event| event.get('[@metadata][_id]') }
-      end
+        let(:emitted_events) do
+          queue = Queue.new # since we are running slices in threads, we need a thread-safe queue.
+          plugin.run(queue)
+          events = []
+          events << queue.pop until queue.empty?
+          events
+        end
 
-      it 'emits the hits on the first page of the first slice' do
-        expect(emitted_event_ids).to include('slice0-response0-item0')
-        expect(emitted_event_ids).to include('slice0-response0-item1')
-      end
-      it 'emits the hits on the second page of the first slice' do
-        expect(emitted_event_ids).to include('slice0-response1-item0')
-      end
+        let(:emitted_event_ids) do
+          emitted_events.map { |event| event.get('[@metadata][_id]') }
+        end
 
-      it 'emits the hits on the first page of the second slice' do
-        expect(emitted_event_ids).to include('slice1-response0-item0')
-        expect(emitted_event_ids).to include('slice1-response0-item1')
-      end
+        it 'emits the hits on the first page of the first slice' do
+          expect(emitted_event_ids).to include('slice0-response0-item0')
+          expect(emitted_event_ids).to include('slice0-response0-item1')
+        end
+        it 'emits the hits on the second page of the first slice' do
+          expect(emitted_event_ids).to include('slice0-response1-item0')
+        end
 
-      it 'emits the hitson the second page of the second slice' do
-        expect(emitted_event_ids).to include('slice1-response1-item0')
-        expect(emitted_event_ids).to include('slice1-response1-item1')
-      end
+        it 'emits the hits on the first page of the second slice' do
+          expect(emitted_event_ids).to include('slice1-response0-item0')
+          expect(emitted_event_ids).to include('slice1-response0-item1')
+        end
+
+        it 'emits the hits on the second page of the second slice' do
+          expect(emitted_event_ids).to include('slice1-response1-item0')
+          expect(emitted_event_ids).to include('slice1-response1-item1')
+        end
+
+        it 'does not double-emit' do
+          expect(emitted_event_ids.uniq).to eq(emitted_event_ids)
+        end
 
-      it 'does not double-emit' do
-        expect(emitted_event_ids.uniq).to eq(emitted_event_ids)
+        it 'emits events with appropriate fields' do
+          emitted_events.each do |event|
+            expect(event).to be_a(LogStash::Event)
+            expect(event.get('message')).to eq(['hello, world'])
+            expect(event.get('[@metadata][_id]')).to_not be_nil
+            expect(event.get('[@metadata][_id]')).to_not be_empty
+            expect(event.get('[@metadata][_index]')).to start_with('logstash-')
+          end
+        end
       end
 
-      it 'emits events with appropriate fields' do
-        emitted_events.each do |event|
-          expect(event).to be_a(LogStash::Event)
-          expect(event.get('message')).to eq(['hello, world'])
-          expect(event.get('[@metadata][_id]')).to_not be_nil
-          expect(event.get('[@metadata][_id]')).to_not be_empty
-          expect(event.get('[@metadata][_index]')).to start_with('logstash-')
+      describe "with scroll request fail" do
+        before(:each) do
+          expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
+          plugin.register
+
+          expect(client).to receive(:clear_scroll).and_return(nil)
+
+          # SLICE0 is a three-page scroll in which the second page throw exception
+          slice0_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 0, 'max' => 2}))
+          expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_raise("boom")
+          allow(client).to receive(:ping)
+
+          # SLICE1 is a two-page scroll in which the last page has no next scroll id
+          slice1_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 1, 'max' => 2}))
+          expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_return(slice1_response1)
+
+          synchronize_method!(plugin, :scroll_request)
+          synchronize_method!(plugin, :search_request)
+        end
+
+        let(:client) { Elasticsearch::Client.new }
+
+        it 'does not insert event to queue' do
+          expect(plugin).to receive(:parallel_slice).and_wrap_original do |m, *args|
+            slice0, slice1 = m.call
+            expect(slice0[0]).to be_falsey
+            expect(slice1[0]).to be_truthy
+            expect(slice1[1].size).to eq(4) # four items from SLICE1
+            [slice0, slice1]
+          end
+
+          queue = Queue.new
+          plugin.run(queue)
+          expect(queue.size).to eq(0)
         end
       end
     end
@@ -890,16 +943,93 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
           queue << LogStash::Event.new({})
         }.at_least(:twice)
         runner = Thread.start { plugin.run(queue) }
-        sleep 3.0
+        expect(queue.pop).not_to be_nil
+        cron_jobs = plugin.instance_variable_get(:@_scheduler).instance_variable_get(:@impl).jobs
+        expect(cron_jobs[0].next_time - cron_jobs[0].last_time).to be <= 5.0
+        expect(queue.pop).not_to be_nil
       ensure
         plugin.do_stop
         runner.join if runner
       end
-      expect(queue.size).to be >= 2
     end
 
   end
 
+  context "retries" do
+    let(:mock_response) do
+      {
+        "_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
+        "took" => 27,
+        "timed_out" => false,
+        "_shards" => {
+          "total" => 169,
+          "successful" => 169,
+          "failed" => 0
+        },
+        "hits" => {
+          "total" => 1,
+          "max_score" => 1.0,
+          "hits" => [ {
+                        "_index" => "logstash-2014.10.12",
+                        "_type" => "logs",
+                        "_id" => "C5b2xLQwTZa76jBmHIbwHQ",
+                        "_score" => 1.0,
+                        "_source" => { "message" => ["ohayo"] }
+                      } ]
+        }
+      }
+    end
+
+    let(:mock_scroll_response) do
+      {
+        "_scroll_id" => "r453Wc1jh0caLJhSDg",
+        "hits" => { "hits" => [] }
+      }
+    end
+
+    before(:each) do
+      client = Elasticsearch::Client.new
+      allow(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
+      allow(client).to receive(:search).with(any_args).and_return(mock_response)
+      allow(client).to receive(:scroll).with({ :body => { :scroll_id => "cXVlcnlUaGVuRmV0Y2g" }, :scroll=> "1m" }).and_return(mock_scroll_response)
+      allow(client).to receive(:clear_scroll).and_return(nil)
+      allow(client).to receive(:ping)
+    end
+
+    let(:config) do
+      {
+        "hosts" => ["localhost"],
+        "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
+        "retries" => 1
+      }
+    end
+
+    it "retry and log error when all search request fail" do
+      expect(plugin.logger).to receive(:error).with(/Tried .* unsuccessfully/,
+                                                    hash_including(:message => 'Manticore::UnknownException'))
+      expect(plugin.logger).to receive(:warn).twice.with(/Attempt to .* but failed/,
+                                                         hash_including(:exception => "Manticore::UnknownException"))
+      expect(plugin).to receive(:search_request).with(instance_of(Hash)).and_raise(Manticore::UnknownException).at_least(:twice)
+
+      plugin.register
+
+      expect{ plugin.run(queue) }.not_to raise_error
+      expect(queue.size).to eq(0)
+    end
+
+    it "retry successfully when search request fail for one time" do
+      expect(plugin.logger).to receive(:warn).once.with(/Attempt to .* but failed/,
+                                                         hash_including(:exception => "Manticore::UnknownException"))
+      expect(plugin).to receive(:search_request).with(instance_of(Hash)).once.and_raise(Manticore::UnknownException)
+      expect(plugin).to receive(:search_request).with(instance_of(Hash)).once.and_call_original
+
+      plugin.register
+
+      expect{ plugin.run(queue) }.not_to raise_error
+      expect(queue.size).to eq(1)
+    end
+  end
+
   # @note can be removed once we depends on elasticsearch gem >= 6.x
   def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
     client.transport.respond_to?(:transport) ? client.transport.transport : client.transport

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.14.0
+  version: 4.15.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-08 00:00:00.000000000 Z
+date: 2022-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -255,6 +255,7 @@ files:
 - NOTICE.TXT
 - README.md
 - docs/index.asciidoc
+- lib/logstash/helpers/loggable_try.rb
 - lib/logstash/inputs/elasticsearch.rb
 - lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_connections_selector.rb
 - lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb