logstash-input-elasticsearch 4.13.0 → 4.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d87078bdf63844901c7684624bc15bd6af23c71c728adb20b86ba76738957926
-  data.tar.gz: 34d8ce81035665b93623aec9008c511000b80a463c0cf482fb459249c763aeaf
+  metadata.gz: 1c5466b50a56ec047ac20dc019d9e24862756b518613b63edf44583900a090e2
+  data.tar.gz: 6b3915bd640318ebd6fa4b87f676d7e43d4fba7fe54893efdb6eb39f70481da8
 SHA512:
-  metadata.gz: 4b2c5aa7a229bdbc89df35276c22a72f1cdbefc03a9069dfb74e64aad64d121aeb3ee67a9b632fd8ab445d1a256cce23f67feabb4ff379ed78a15507f28b1dce
-  data.tar.gz: 7c7a45cd3817d9e746a03eb166293e389eee16fc512bb2e0ecc546158d67f1f4e0f54bc191212e02ce1af206fe5f98374d70defa3f8ef218bed9e210ac8a8a65
+  metadata.gz: 930202045f30125060cce1f3d45b22c0336976a6cdcbf5931f4b98410785bd425cf665224edfe4994953ad2d53cdaa994b31be9db86fd05f538a722ff788ba97
+  data.tar.gz: 8cf1f45481575f653867831ab38fb7efd286c12432b2afc2c6940c32d4eb91f913dd7b89c284ed4e11e9205536cc75b318f399b92136777f84ac6dec75d2de2d

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 4.16.0
+  - Added `ssl_certificate_verification` option to control SSL certificate verification [#180](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/180)
+
+## 4.15.0
+  - Feat: add `retries` option. allow retry for failing query [#179](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/179)
+
+## 4.14.0
+  - Refactor: switch to using scheduler mixin [#177](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/177)
+
 ## 4.13.0
   - Added support for `ca_trusted_fingerprint` when run on Logstash 8.3+ [#178](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/178)
 

data/docs/index.asciidoc

@@ -122,8 +122,10 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-slices>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ssl_certificate_verification>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-socket_timeout_seconds>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No
+| <<plugins-{type}s-{plugin}-retries>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
 |=======================================================================
 
@@ -339,6 +341,17 @@ The maximum amount of time, in seconds, for a single request to Elasticsearch.
 Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload
 documents and/or the <<plugins-{type}s-{plugin}-size>> has been specified as a large value.
 
+
+[id="plugins-{type}s-{plugin}-retries"]
+===== `retries`
+
+* Value type is <<number,number>>
+* Default value is `0`
+
+The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message.
+The default is 0 (no retry). This value should be equal to or greater than zero.
+
+
 [id="plugins-{type}s-{plugin}-schedule"]
 ===== `schedule` 
 
@@ -402,6 +415,20 @@ instructions into the query.
 If enabled, SSL will be used when communicating with the Elasticsearch
 server (i.e. HTTPS will be used instead of plain HTTP).
 
+[id="plugins-{type}s-{plugin}-ssl_certificate_verification"]
+===== `ssl_certificate_verification`
+
+* Value type is <<boolean,boolean>>
+* Default value is `true`
+
+Option to validate the server's certificate. Disabling this severely compromises security.
+When certificate validation is disabled, this plugin implicitly trusts the machine
+resolved at the given address without validating its proof-of-identity.
+In this scenario, the plugin can transmit credentials to or process data from an untrustworthy
+man-in-the-middle or other compromised infrastructure.
+More information on the importance of certificate verification:
+**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**.
+
 [id="plugins-{type}s-{plugin}-socket_timeout_seconds"]
 ===== `socket_timeout_seconds`
 
@@ -424,6 +451,7 @@ When the `target` is set to a field reference, the `_source` of the hit is place
 This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced.
 It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`).
 
+
 [id="plugins-{type}s-{plugin}-user"]
 ===== `user` 
 

data/lib/logstash/helpers/loggable_try.rb

@@ -0,0 +1,18 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+require 'stud/try'
+
+module LogStash module Helpers
+  class LoggableTry < Stud::Try
+    def initialize(logger, name)
+      @logger = logger
+      @name = name
+    end
+
+    def log_failure(exception, fail_count, message)
+      @logger.warn("Attempt to #{@name} but failed. #{message}", fail_count: fail_count, exception: exception.message)
+    end
+  end
+end end

data/lib/logstash/inputs/elasticsearch.rb

@@ -8,7 +8,9 @@ require 'logstash/plugin_mixins/event_support/event_factory_adapter'
 require 'logstash/plugin_mixins/ecs_compatibility_support'
 require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
 require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
+require "logstash/plugin_mixins/scheduler"
 require "base64"
+require 'logstash/helpers/loggable_try'
 
 require "elasticsearch"
 require "elasticsearch/transport/transport/http/manticore"
@@ -78,6 +80,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
   extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
 
+  include LogStash::PluginMixins::Scheduler
+
   config_name "elasticsearch"
 
   # List of elasticsearch hosts to use for querying.
@@ -96,6 +100,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # This allows you to set the maximum number of hits returned per scroll.
   config :size, :validate => :number, :default => 1000
 
+  # The number of retries to run the query. If the query fails after all retries, it logs an error message.
+  config :retries, :validate => :number, :default => 0
+
   # This parameter controls the keepalive time in seconds of the scrolling
   # request and initiates the scrolling process. The timeout applies per
   # round trip (i.e. between the previous scroll request, to the next).
@@ -183,6 +190,11 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary 
   config :ca_file, :validate => :path
 
+  # Option to validate the server's certificate. Disabling this severely compromises security.
+  # For more information on the importance of certificate verification please read
+  # https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
+  config :ssl_certificate_verification, :validate => :boolean, :default => true
+
   # Schedule of when to periodically run statement, in Cron format
   # for example: "* * * * *" (execute query every minute, on the minute)
   #
@@ -218,6 +230,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
       @slices < 1 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `slices` option must be greater than zero, got `#{@slices}`")
     end
 
+    @retries < 0 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `retries` option must be equal or greater than zero, got `#{@retries}`")
+
     validate_authentication
     fill_user_password_from_cloud_auth
     fill_hosts_from_cloud_id
@@ -251,37 +265,75 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
   def run(output_queue)
     if @schedule
-      @scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
-      @scheduler.cron @schedule do
-        do_run(output_queue)
-      end
-
-      @scheduler.join
+      scheduler.cron(@schedule) { do_run(output_queue) }
+      scheduler.join
     else
       do_run(output_queue)
     end
   end
 
-  def stop
-    @scheduler.stop if @scheduler
-  end
-
   private
-
+  JOB_NAME = "run query"
   def do_run(output_queue)
     # if configured to run a single slice, don't bother spinning up threads
-    return do_run_slice(output_queue) if @slices.nil? || @slices <= 1
+    if @slices.nil? || @slices <= 1
+      success, events = retryable_slice
+      success && events.each { |event| output_queue << event }
+      return
+    end
 
     logger.warn("managed slices for query is very large (#{@slices}); consider reducing") if @slices > 8
 
+    slice_results = parallel_slice # array of tuple(ok, events)
+
+    # insert events to queue if all slices success
+    if slice_results.all?(&:first)
+      slice_results.flat_map { |success, events| events }
+                  .each { |event| output_queue << event }
+    end
+
+    logger.trace("#{@slices} slices completed")
+  end
+
+  def retryable(job_name, &block)
+    begin
+      stud_try = ::LogStash::Helpers::LoggableTry.new(logger, job_name)
+      output = stud_try.try((@retries + 1).times) { yield }
+      [true, output]
+    rescue => e
+      error_details = {:message => e.message, :cause => e.cause}
+      error_details[:backtrace] = e.backtrace if logger.debug?
+      logger.error("Tried #{job_name} unsuccessfully", error_details)
+      [false, nil]
+    end
+  end
+
+
+  # @return [(ok, events)] : Array of tuple(Boolean, [Logstash::Event])
+  def parallel_slice
+    pipeline_id = execution_context&.pipeline_id || 'main'
     @slices.times.map do |slice_id|
       Thread.new do
-        LogStash::Util::set_thread_name("#{@id}_slice_#{slice_id}")
-        do_run_slice(output_queue, slice_id)
+        LogStash::Util::set_thread_name("[#{pipeline_id}]|input|elasticsearch|slice_#{slice_id}")
+        retryable_slice(slice_id)
       end
-    end.map(&:join)
+    end.map do |t|
+      t.join
+      t.value
+    end
   end
 
+  # @param scroll_id [Integer]
+  # @return (ok, events) [Boolean, Array(Logstash::Event)]
+  def retryable_slice(slice_id=nil)
+    retryable(JOB_NAME) do
+      output = []
+      do_run_slice(output, slice_id)
+      output
+    end
+  end
+
+
   def do_run_slice(output_queue, slice_id=nil)
     slice_query = @base_query
     slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @slices}) unless slice_id.nil?
@@ -318,11 +370,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     r = scroll_request(scroll_id)
     r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
     [r['hits']['hits'].any?, r['_scroll_id']]
-  rescue => e
-    # this will typically be triggered by a scroll timeout
-    logger.error("Scroll request error, aborting scroll", message: e.message, exception: e.class)
-    # return no hits and original scroll_id so we can try to clear it
-    [false, scroll_id]
   end
 
   def push_hit(hit, output_queue)
@@ -357,7 +404,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     logger.warn("Ignoring clear_scroll exception", message: e.message, exception: e.class)
   end
 
-  def scroll_request scroll_id
+  def scroll_request(scroll_id)
     @client.scroll(:body => { :scroll_id => scroll_id }, :scroll => @scroll)
   end
 
@@ -390,6 +437,11 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     ssl_options[:ssl] = true if @ssl
     ssl_options[:ca_file] = @ca_file if @ssl && @ca_file
     ssl_options[:trust_strategy] = trust_strategy_for_ca_trusted_fingerprint
+    if @ssl && !@ssl_certificate_verification
+      logger.warn "You have enabled encryption but DISABLED certificate verification, " +
+                    "to make sure your data is secure remove `ssl_certificate_verification => false`"
+      ssl_options[:verify] = :disable
+    end
 
     ssl_options
   end

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.13.0'
+  s.version         = '4.16.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -24,13 +24,13 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
   s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0'
   s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
+  s.add_runtime_dependency "logstash-mixin-scheduler", '~> 1.0'
 
   s.add_runtime_dependency 'elasticsearch', '>= 7.17.1'
   s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
 
   s.add_runtime_dependency 'tzinfo'
   s.add_runtime_dependency 'tzinfo-data'
-  s.add_runtime_dependency 'rufus-scheduler'
   s.add_runtime_dependency 'manticore', ">= 0.7.1"
 
   s.add_development_dependency 'logstash-codec-plain'

data/spec/inputs/elasticsearch_spec.rb

@@ -51,6 +51,17 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
       end
     end
+
+    context "retry" do
+      let(:config) do
+        {
+          "retries" => -1
+        }
+      end
+      it "should raise an exception with negative number" do
+        expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
   end
 
   it_behaves_like "an interruptible input plugin" do
@@ -193,7 +204,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     context 'with `slices => 1`' do
       let(:slices) { 1 }
       it 'runs just one slice' do
-        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
+        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<), nil)
         expect(Thread).to_not receive(:new)
 
         plugin.register
@@ -204,7 +215,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     context 'without slices directive' do
       let(:config) { super().tap { |h| h.delete('slices') } }
       it 'runs just one slice' do
-        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
+        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<), nil)
         expect(Thread).to_not receive(:new)
 
         plugin.register
@@ -315,7 +326,6 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       end
       # END SLICE 1
 
-      let(:client) { Elasticsearch::Client.new }
 
       # RSpec mocks validations are not threadsafe.
       # Allow caller to synchronize.
@@ -329,69 +339,112 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         end
       end
 
-      before(:each) do
-        expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
-        plugin.register
+      describe "with normal response" do
+        before(:each) do
+          expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
+          plugin.register
 
-        expect(client).to receive(:clear_scroll).and_return(nil)
+          expect(client).to receive(:clear_scroll).and_return(nil)
 
-        # SLICE0 is a three-page scroll in which the last page is empty
-        slice0_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 0, 'max' => 2}))
-        expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
-        expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_return(slice0_response1)
-        expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll2 })).and_return(slice0_response2)
-        allow(client).to receive(:ping)
+          # SLICE0 is a three-page scroll in which the last page is empty
+          slice0_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 0, 'max' => 2}))
+          expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_return(slice0_response1)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll2 })).and_return(slice0_response2)
+          allow(client).to receive(:ping)
 
-        # SLICE1 is a two-page scroll in which the last page has no next scroll id
-        slice1_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 1, 'max' => 2}))
-        expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
-        expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_return(slice1_response1)
+          # SLICE1 is a two-page scroll in which the last page has no next scroll id
+          slice1_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 1, 'max' => 2}))
+          expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_return(slice1_response1)
 
-        synchronize_method!(plugin, :scroll_request)
-        synchronize_method!(plugin, :search_request)
-      end
+          synchronize_method!(plugin, :scroll_request)
+          synchronize_method!(plugin, :search_request)
+        end
 
-      let(:emitted_events) do
-        queue = Queue.new # since we are running slices in threads, we need a thread-safe queue.
-        plugin.run(queue)
-        events = []
-        events << queue.pop until queue.empty?
-        events
-      end
+        let(:client) { Elasticsearch::Client.new }
 
-      let(:emitted_event_ids) do
-        emitted_events.map { |event| event.get('[@metadata][_id]') }
-      end
+        let(:emitted_events) do
+          queue = Queue.new # since we are running slices in threads, we need a thread-safe queue.
+          plugin.run(queue)
+          events = []
+          events << queue.pop until queue.empty?
+          events
+        end
 
-      it 'emits the hits on the first page of the first slice' do
-        expect(emitted_event_ids).to include('slice0-response0-item0')
-        expect(emitted_event_ids).to include('slice0-response0-item1')
-      end
-      it 'emits the hits on the second page of the first slice' do
-        expect(emitted_event_ids).to include('slice0-response1-item0')
-      end
+        let(:emitted_event_ids) do
+          emitted_events.map { |event| event.get('[@metadata][_id]') }
+        end
 
-      it 'emits the hits on the first page of the second slice' do
-        expect(emitted_event_ids).to include('slice1-response0-item0')
-        expect(emitted_event_ids).to include('slice1-response0-item1')
-      end
+        it 'emits the hits on the first page of the first slice' do
+          expect(emitted_event_ids).to include('slice0-response0-item0')
+          expect(emitted_event_ids).to include('slice0-response0-item1')
+        end
+        it 'emits the hits on the second page of the first slice' do
+          expect(emitted_event_ids).to include('slice0-response1-item0')
+        end
 
-      it 'emits the hitson the second page of the second slice' do
-        expect(emitted_event_ids).to include('slice1-response1-item0')
-        expect(emitted_event_ids).to include('slice1-response1-item1')
-      end
+        it 'emits the hits on the first page of the second slice' do
+          expect(emitted_event_ids).to include('slice1-response0-item0')
+          expect(emitted_event_ids).to include('slice1-response0-item1')
+        end
+
+        it 'emits the hits on the second page of the second slice' do
+          expect(emitted_event_ids).to include('slice1-response1-item0')
+          expect(emitted_event_ids).to include('slice1-response1-item1')
+        end
 
-      it 'does not double-emit' do
-        expect(emitted_event_ids.uniq).to eq(emitted_event_ids)
+        it 'does not double-emit' do
+          expect(emitted_event_ids.uniq).to eq(emitted_event_ids)
+        end
+
+        it 'emits events with appropriate fields' do
+          emitted_events.each do |event|
+            expect(event).to be_a(LogStash::Event)
+            expect(event.get('message')).to eq(['hello, world'])
+            expect(event.get('[@metadata][_id]')).to_not be_nil
+            expect(event.get('[@metadata][_id]')).to_not be_empty
+            expect(event.get('[@metadata][_index]')).to start_with('logstash-')
+          end
+        end
       end
 
-      it 'emits events with appropriate fields' do
-        emitted_events.each do |event|
-          expect(event).to be_a(LogStash::Event)
-          expect(event.get('message')).to eq(['hello, world'])
-          expect(event.get('[@metadata][_id]')).to_not be_nil
-          expect(event.get('[@metadata][_id]')).to_not be_empty
-          expect(event.get('[@metadata][_index]')).to start_with('logstash-')
+      describe "with scroll request fail" do
+        before(:each) do
+          expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
+          plugin.register
+
+          expect(client).to receive(:clear_scroll).and_return(nil)
+
+          # SLICE0 is a three-page scroll in which the second page throw exception
+          slice0_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 0, 'max' => 2}))
+          expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_raise("boom")
+          allow(client).to receive(:ping)
+
+          # SLICE1 is a two-page scroll in which the last page has no next scroll id
+          slice1_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 1, 'max' => 2}))
+          expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_return(slice1_response1)
+
+          synchronize_method!(plugin, :scroll_request)
+          synchronize_method!(plugin, :search_request)
+        end
+
+        let(:client) { Elasticsearch::Client.new }
+
+        it 'does not insert event to queue' do
+          expect(plugin).to receive(:parallel_slice).and_wrap_original do |m, *args|
+            slice0, slice1 = m.call
+            expect(slice0[0]).to be_falsey
+            expect(slice1[0]).to be_truthy
+            expect(slice1[1].size).to eq(4) # four items from SLICE1
+            [slice0, slice1]
+          end
+
+          queue = Queue.new
+          plugin.run(queue)
+          expect(queue.size).to eq(0)
         end
       end
     end
@@ -645,6 +698,14 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
             expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
           end
         end
+        
+        context 'ssl verification disabled' do
+          let(:config) { super().merge({ 'ssl_certificate_verification' => false }) }
+          it 'should warn data security risk' do
+            expect(plugin.logger).to receive(:warn).once.with("You have enabled encryption but DISABLED certificate verification, to make sure your data is secure remove `ssl_certificate_verification => false`")
+            plugin.register
+          end
+        end
       end
     end if LOGSTASH_VERSION > '6.0'
 
@@ -890,16 +951,93 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
           queue << LogStash::Event.new({})
         }.at_least(:twice)
         runner = Thread.start { plugin.run(queue) }
-        sleep 3.0
+        expect(queue.pop).not_to be_nil
+        cron_jobs = plugin.instance_variable_get(:@_scheduler).instance_variable_get(:@impl).jobs
+        expect(cron_jobs[0].next_time - cron_jobs[0].last_time).to be <= 5.0
+        expect(queue.pop).not_to be_nil
       ensure
         plugin.do_stop
         runner.join if runner
       end
-      expect(queue.size).to be >= 2
     end
 
   end
 
+  context "retries" do
+    let(:mock_response) do
+      {
+        "_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
+        "took" => 27,
+        "timed_out" => false,
+        "_shards" => {
+          "total" => 169,
+          "successful" => 169,
+          "failed" => 0
+        },
+        "hits" => {
+          "total" => 1,
+          "max_score" => 1.0,
+          "hits" => [ {
+                        "_index" => "logstash-2014.10.12",
+                        "_type" => "logs",
+                        "_id" => "C5b2xLQwTZa76jBmHIbwHQ",
+                        "_score" => 1.0,
+                        "_source" => { "message" => ["ohayo"] }
+                      } ]
+        }
+      }
+    end
+
+    let(:mock_scroll_response) do
+      {
+        "_scroll_id" => "r453Wc1jh0caLJhSDg",
+        "hits" => { "hits" => [] }
+      }
+    end
+
+    before(:each) do
+      client = Elasticsearch::Client.new
+      allow(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
+      allow(client).to receive(:search).with(any_args).and_return(mock_response)
+      allow(client).to receive(:scroll).with({ :body => { :scroll_id => "cXVlcnlUaGVuRmV0Y2g" }, :scroll=> "1m" }).and_return(mock_scroll_response)
+      allow(client).to receive(:clear_scroll).and_return(nil)
+      allow(client).to receive(:ping)
+    end
+
+    let(:config) do
+      {
+        "hosts" => ["localhost"],
+        "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
+        "retries" => 1
+      }
+    end
+
+    it "retry and log error when all search request fail" do
+      expect(plugin.logger).to receive(:error).with(/Tried .* unsuccessfully/,
+                                                    hash_including(:message => 'Manticore::UnknownException'))
+      expect(plugin.logger).to receive(:warn).twice.with(/Attempt to .* but failed/,
+                                                         hash_including(:exception => "Manticore::UnknownException"))
+      expect(plugin).to receive(:search_request).with(instance_of(Hash)).and_raise(Manticore::UnknownException).at_least(:twice)
+
+      plugin.register
+
+      expect{ plugin.run(queue) }.not_to raise_error
+      expect(queue.size).to eq(0)
+    end
+
+    it "retry successfully when search request fail for one time" do
+      expect(plugin.logger).to receive(:warn).once.with(/Attempt to .* but failed/,
+                                                         hash_including(:exception => "Manticore::UnknownException"))
+      expect(plugin).to receive(:search_request).with(instance_of(Hash)).once.and_raise(Manticore::UnknownException)
+      expect(plugin).to receive(:search_request).with(instance_of(Hash)).once.and_call_original
+
+      plugin.register
+
+      expect{ plugin.run(queue) }.not_to raise_error
+      expect(queue.size).to eq(1)
+    end
+  end
+
   # @note can be removed once we depends on elasticsearch gem >= 6.x
   def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
     client.transport.respond_to?(:transport) ? client.transport.transport : client.transport

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.13.0
+  version: 4.16.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-05-24 00:00:00.000000000 Z
+date: 2022-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -72,6 +72,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-scheduler
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -128,20 +142,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: rufus-scheduler
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -255,6 +255,7 @@ files:
 - NOTICE.TXT
 - README.md
 - docs/index.asciidoc
+- lib/logstash/helpers/loggable_try.rb
 - lib/logstash/inputs/elasticsearch.rb
 - lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_connections_selector.rb
 - lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb