logstash-input-elasticsearch 4.12.3 → 4.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a4ed685c9f90446a47fc84cb4e52e585f164c5451c9dca3f7359b756077bbd4
-  data.tar.gz: 050b54e5c660a4ab436f57ab16ea20d6a64a6b8a8c063eaaca69cc2c77235bd4
+  metadata.gz: 9379a31460bf649615c038b1b35d207f6a6db869bb1a27e963da70e278ae6bcd
+  data.tar.gz: f62823b8ddfb587ce9614a1d131be1bce2186dbd859287b1107757917a754c7e
 SHA512:
-  metadata.gz: 902070db657622bd65bcba8c17470456366134aae23cd38fb70ec89c97686b74da591e6f47d34957f3e5753f0e029ad78b845dd6355d6d559b6c135e0c94d28e
-  data.tar.gz: 22391299a87f5ce0a3085b4be7886a7f7251c9e33bd7051996871ea7cc34cbd7fbc53a3d54d841722b319949c4422e0ab7e6ff7d04899db6255543363fc32f66
+  metadata.gz: c675ed6a5d1a4a104313611e4895171c6461149d0415c5ad4c276e42b5b99672584ae38ddc37f3e32521074727051c68f0d10a37b77064d0cea6ea03f041a3b9
+  data.tar.gz: beba1b98be77880e6e3712cebdd0474d68c33556a86236e828a3332d2e812b187e920e78f6fb006b27b82269faa0e0654568d050f7057ae7db48da7fd9a20e31

data/CHANGELOG.md

@@ -1,5 +1,14 @@
+## 4.15.0
+  - Feat: add `retries` option. allow retry for failing query [#179](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/179)
+
+## 4.14.0
+  - Refactor: switch to using scheduler mixin [#177](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/177)
+
+## 4.13.0
+  - Added support for `ca_trusted_fingerprint` when run on Logstash 8.3+ [#178](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/178)
+
 ## 4.12.3
-  - Fix: update Elasticsearch Ruby client to correctly customize 'user-agent' header[#171](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/171)
+  - Fix: update Elasticsearch Ruby client to correctly customize 'user-agent' header [#171](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/171)
 
 ## 4.12.2
   - Fix: hosts => "es_host:port" regression [#168](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/168)
@@ -24,7 +33,6 @@
     header isn't passed, this leads to the plugin not being able to leverage `user`/`password` credentials set by the user.
     [#153](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/153)
 
-
 ## 4.9.1
   - [DOC] Replaced hard-coded links with shared attributes [#143](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/143)
   - [DOC] Added missing quote to docinfo_fields example [#145](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/145)

data/docs/index.asciidoc

@@ -103,6 +103,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-api_key>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-connect_timeout_seconds>> | <<number,number>>|No
@@ -123,6 +124,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-socket_timeout_seconds>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No
+| <<plugins-{type}s-{plugin}-retries>> | <<number,number>>|No
 | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
 |=======================================================================
 
@@ -152,6 +154,15 @@ API key API].
 
 SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary.
 
+[id="plugins-{type}s-{plugin}-ca_trusted_fingerprint"]
+===== `ca_trusted_fingerprint`
+
+* Value type is <<string,string>>, and must contain exactly 64 hexadecimal characters.
+* There is no default value for this setting.
+* Use of this option _requires_ Logstash 8.3+
+
+The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster.
+
 [id="plugins-{type}s-{plugin}-cloud_auth"]
 ===== `cloud_auth`
 
@@ -329,6 +340,17 @@ The maximum amount of time, in seconds, for a single request to Elasticsearch.
 Request timeouts tend to occur when an individual page of data is very large, such as when it contains large-payload
 documents and/or the <<plugins-{type}s-{plugin}-size>> has been specified as a large value.
 
+
+[id="plugins-{type}s-{plugin}-retries"]
+===== `retries`
+
+* Value type is <<number,number>>
+* Default value is `0`
+
+The number of times to re-run the query after the first failure. If the query fails after all retries, it logs an error message.
+The default is 0 (no retry). This value should be equal to or greater than zero.
+
+
 [id="plugins-{type}s-{plugin}-schedule"]
 ===== `schedule` 
 
@@ -414,6 +436,7 @@ When the `target` is set to a field reference, the `_source` of the hit is place
 This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced.
 It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`).
 
+
 [id="plugins-{type}s-{plugin}-user"]
 ===== `user` 
 

data/lib/logstash/helpers/loggable_try.rb

@@ -0,0 +1,18 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+require 'stud/try'
+
+module LogStash module Helpers
+  class LoggableTry < Stud::Try
+    def initialize(logger, name)
+      @logger = logger
+      @name = name
+    end
+
+    def log_failure(exception, fail_count, message)
+      @logger.warn("Attempt to #{@name} but failed. #{message}", fail_count: fail_count, exception: exception.message)
+    end
+  end
+end end

data/lib/logstash/inputs/elasticsearch.rb

@@ -7,7 +7,10 @@ require 'logstash/plugin_mixins/validator_support/field_reference_validation_ada
 require 'logstash/plugin_mixins/event_support/event_factory_adapter'
 require 'logstash/plugin_mixins/ecs_compatibility_support'
 require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
+require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
+require "logstash/plugin_mixins/scheduler"
 require "base64"
+require 'logstash/helpers/loggable_try'
 
 require "elasticsearch"
 require "elasticsearch/transport/transport/http/manticore"
@@ -77,6 +80,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
   extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
 
+  include LogStash::PluginMixins::Scheduler
+
   config_name "elasticsearch"
 
   # List of elasticsearch hosts to use for querying.
@@ -95,6 +100,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # This allows you to set the maximum number of hits returned per scroll.
   config :size, :validate => :number, :default => 1000
 
+  # The number of retries to run the query. If the query fails after all retries, it logs an error message.
+  config :retries, :validate => :number, :default => 0
+
   # This parameter controls the keepalive time in seconds of the scrolling
   # request and initiates the scrolling process. The timeout applies per
   # round trip (i.e. between the previous scroll request, to the next).
@@ -192,6 +200,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # If set, the _source of each hit will be added nested under the target instead of at the top-level
   config :target, :validate => :field_reference
 
+  # config :ca_trusted_fingerprint, :validate => :sha_256_hex
+  include LogStash::PluginMixins::CATrustedFingerprintSupport
+
   def initialize(params={})
     super(params)
 
@@ -214,6 +225,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
       @slices < 1 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `slices` option must be greater than zero, got `#{@slices}`")
     end
 
+    @retries < 0 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `retries` option must be equal or greater than zero, got `#{@retries}`")
+
     validate_authentication
     fill_user_password_from_cloud_auth
     fill_hosts_from_cloud_id
@@ -247,37 +260,75 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
   def run(output_queue)
     if @schedule
-      @scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
-      @scheduler.cron @schedule do
-        do_run(output_queue)
-      end
-
-      @scheduler.join
+      scheduler.cron(@schedule) { do_run(output_queue) }
+      scheduler.join
     else
       do_run(output_queue)
     end
   end
 
-  def stop
-    @scheduler.stop if @scheduler
-  end
-
   private
-
+  JOB_NAME = "run query"
   def do_run(output_queue)
     # if configured to run a single slice, don't bother spinning up threads
-    return do_run_slice(output_queue) if @slices.nil? || @slices <= 1
+    if @slices.nil? || @slices <= 1
+      success, events = retryable_slice
+      success && events.each { |event| output_queue << event }
+      return
+    end
 
     logger.warn("managed slices for query is very large (#{@slices}); consider reducing") if @slices > 8
 
+    slice_results = parallel_slice # array of tuple(ok, events)
+
+    # insert events to queue if all slices success
+    if slice_results.all?(&:first)
+      slice_results.flat_map { |success, events| events }
+                  .each { |event| output_queue << event }
+    end
+
+    logger.trace("#{@slices} slices completed")
+  end
+
+  def retryable(job_name, &block)
+    begin
+      stud_try = ::LogStash::Helpers::LoggableTry.new(logger, job_name)
+      output = stud_try.try((@retries + 1).times) { yield }
+      [true, output]
+    rescue => e
+      error_details = {:message => e.message, :cause => e.cause}
+      error_details[:backtrace] = e.backtrace if logger.debug?
+      logger.error("Tried #{job_name} unsuccessfully", error_details)
+      [false, nil]
+    end
+  end
+
+
+  # @return [(ok, events)] : Array of tuple(Boolean, [Logstash::Event])
+  def parallel_slice
+    pipeline_id = execution_context&.pipeline_id || 'main'
     @slices.times.map do |slice_id|
       Thread.new do
-        LogStash::Util::set_thread_name("#{@id}_slice_#{slice_id}")
-        do_run_slice(output_queue, slice_id)
+        LogStash::Util::set_thread_name("[#{pipeline_id}]|input|elasticsearch|slice_#{slice_id}")
+        retryable_slice(slice_id)
       end
-    end.map(&:join)
+    end.map do |t|
+      t.join
+      t.value
+    end
   end
 
+  # @param scroll_id [Integer]
+  # @return (ok, events) [Boolean, Array(Logstash::Event)]
+  def retryable_slice(slice_id=nil)
+    retryable(JOB_NAME) do
+      output = []
+      do_run_slice(output, slice_id)
+      output
+    end
+  end
+
+
   def do_run_slice(output_queue, slice_id=nil)
     slice_query = @base_query
     slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @slices}) unless slice_id.nil?
@@ -314,11 +365,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     r = scroll_request(scroll_id)
     r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
     [r['hits']['hits'].any?, r['_scroll_id']]
-  rescue => e
-    # this will typically be triggered by a scroll timeout
-    logger.error("Scroll request error, aborting scroll", message: e.message, exception: e.class)
-    # return no hits and original scroll_id so we can try to clear it
-    [false, scroll_id]
   end
 
   def push_hit(hit, output_queue)
@@ -353,7 +399,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     logger.warn("Ignoring clear_scroll exception", message: e.message, exception: e.class)
   end
 
-  def scroll_request scroll_id
+  def scroll_request(scroll_id)
     @client.scroll(:body => { :scroll_id => scroll_id }, :scroll => @scroll)
   end
 
@@ -381,7 +427,13 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   end
 
   def setup_ssl
-    @ssl && @ca_file ? { :ssl  => true, :ca_file => @ca_file } : {}
+    ssl_options = {}
+
+    ssl_options[:ssl] = true if @ssl
+    ssl_options[:ca_file] = @ca_file if @ssl && @ca_file
+    ssl_options[:trust_strategy] = trust_strategy_for_ca_trusted_fingerprint
+
+    ssl_options
   end
 
   def setup_hosts

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.12.3'
+  s.version         = '4.15.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -24,12 +24,13 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
   s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0'
   s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
+  s.add_runtime_dependency "logstash-mixin-scheduler", '~> 1.0'
 
   s.add_runtime_dependency 'elasticsearch', '>= 7.17.1'
+  s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
 
   s.add_runtime_dependency 'tzinfo'
   s.add_runtime_dependency 'tzinfo-data'
-  s.add_runtime_dependency 'rufus-scheduler'
   s.add_runtime_dependency 'manticore', ">= 0.7.1"
 
   s.add_development_dependency 'logstash-codec-plain'
@@ -37,4 +38,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'timecop'
   s.add_development_dependency 'cabin', ['~> 0.6']
   s.add_development_dependency 'webrick'
+
+  # 3.8.0 has breaking changes WRT to joining, which break our specs
+  s.add_development_dependency 'rufus-scheduler', '~> 3.0.9'
 end

data/spec/fixtures/test_certs/ca.der.sha256

@@ -0,0 +1 @@
+195a7e7b1bc29f3d7913a918a44721704d27fa56facea0cd72a8093c7107c283

data/spec/inputs/elasticsearch_spec.rb

@@ -19,7 +19,14 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
   let(:queue) { Queue.new }
 
   before(:each) do
-     Elasticsearch::Client.send(:define_method, :ping) { } # define no-action ping method
+    Elasticsearch::Client.send(:define_method, :ping) { } # define no-action ping method
+  end
+
+  let(:base_config) do
+    {
+        'hosts' => ["localhost"],
+        'query' => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+    }
   end
 
   context "register" do
@@ -44,6 +51,17 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
       end
     end
+
+    context "retry" do
+      let(:config) do
+        {
+          "retries" => -1
+        }
+      end
+      it "should raise an exception with negative number" do
+        expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
   end
 
   it_behaves_like "an interruptible input plugin" do
@@ -78,14 +96,10 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     end
 
     let(:config) do
-      %q[
-        input {
-          elasticsearch {
-            hosts => ["localhost"]
-            query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-          }
-        }
-      ]
+      {
+          'hosts' => ["localhost"],
+          'query' => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+      }
     end
 
     let(:mock_response) do
@@ -128,10 +142,11 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       expect(client).to receive(:ping)
     end
 
+    before { plugin.register }
+
     it 'creates the events from the hits' do
-      event = input(config) do |pipeline, queue|
-        queue.pop
-      end
+      plugin.run queue
+      event = queue.pop
 
       expect(event).to be_a(LogStash::Event)
       expect(event.get("message")).to eql [ "ohayo" ]
@@ -139,21 +154,16 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
     context 'when a target is set' do
       let(:config) do
-        %q[
-          input {
-            elasticsearch {
-              hosts => ["localhost"]
-              query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-              target => "[@metadata][_source]"
-            }
-          }
-        ]
+        {
+            'hosts' => ["localhost"],
+            'query' => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
+            'target' => "[@metadata][_source]"
+        }
       end
 
       it 'creates the event using the target' do
-        event = input(config) do |pipeline, queue|
-          queue.pop
-        end
+        plugin.run queue
+        event = queue.pop
 
         expect(event).to be_a(LogStash::Event)
         expect(event.get("[@metadata][_source][message]")).to eql [ "ohayo" ]
@@ -194,7 +204,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     context 'with `slices => 1`' do
       let(:slices) { 1 }
       it 'runs just one slice' do
-        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
+        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<), nil)
         expect(Thread).to_not receive(:new)
 
         plugin.register
@@ -205,7 +215,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     context 'without slices directive' do
       let(:config) { super().tap { |h| h.delete('slices') } }
       it 'runs just one slice' do
-        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
+        expect(plugin).to receive(:do_run_slice).with(duck_type(:<<), nil)
         expect(Thread).to_not receive(:new)
 
         plugin.register
@@ -316,7 +326,6 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       end
       # END SLICE 1
 
-      let(:client) { Elasticsearch::Client.new }
 
       # RSpec mocks validations are not threadsafe.
       # Allow caller to synchronize.
@@ -330,69 +339,112 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         end
       end
 
-      before(:each) do
-        expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
-        plugin.register
+      describe "with normal response" do
+        before(:each) do
+          expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
+          plugin.register
 
-        expect(client).to receive(:clear_scroll).and_return(nil)
+          expect(client).to receive(:clear_scroll).and_return(nil)
 
-        # SLICE0 is a three-page scroll in which the last page is empty
-        slice0_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 0, 'max' => 2}))
-        expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
-        expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_return(slice0_response1)
-        expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll2 })).and_return(slice0_response2)
-        allow(client).to receive(:ping)
+          # SLICE0 is a three-page scroll in which the last page is empty
+          slice0_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 0, 'max' => 2}))
+          expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_return(slice0_response1)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll2 })).and_return(slice0_response2)
+          allow(client).to receive(:ping)
 
-        # SLICE1 is a two-page scroll in which the last page has no next scroll id
-        slice1_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 1, 'max' => 2}))
-        expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
-        expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_return(slice1_response1)
+          # SLICE1 is a two-page scroll in which the last page has no next scroll id
+          slice1_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 1, 'max' => 2}))
+          expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_return(slice1_response1)
 
-        synchronize_method!(plugin, :scroll_request)
-        synchronize_method!(plugin, :search_request)
-      end
+          synchronize_method!(plugin, :scroll_request)
+          synchronize_method!(plugin, :search_request)
+        end
 
-      let(:emitted_events) do
-        queue = Queue.new # since we are running slices in threads, we need a thread-safe queue.
-        plugin.run(queue)
-        events = []
-        events << queue.pop until queue.empty?
-        events
-      end
+        let(:client) { Elasticsearch::Client.new }
 
-      let(:emitted_event_ids) do
-        emitted_events.map { |event| event.get('[@metadata][_id]') }
-      end
+        let(:emitted_events) do
+          queue = Queue.new # since we are running slices in threads, we need a thread-safe queue.
+          plugin.run(queue)
+          events = []
+          events << queue.pop until queue.empty?
+          events
+        end
 
-      it 'emits the hits on the first page of the first slice' do
-        expect(emitted_event_ids).to include('slice0-response0-item0')
-        expect(emitted_event_ids).to include('slice0-response0-item1')
-      end
-      it 'emits the hits on the second page of the first slice' do
-        expect(emitted_event_ids).to include('slice0-response1-item0')
-      end
+        let(:emitted_event_ids) do
+          emitted_events.map { |event| event.get('[@metadata][_id]') }
+        end
 
-      it 'emits the hits on the first page of the second slice' do
-        expect(emitted_event_ids).to include('slice1-response0-item0')
-        expect(emitted_event_ids).to include('slice1-response0-item1')
-      end
+        it 'emits the hits on the first page of the first slice' do
+          expect(emitted_event_ids).to include('slice0-response0-item0')
+          expect(emitted_event_ids).to include('slice0-response0-item1')
+        end
+        it 'emits the hits on the second page of the first slice' do
+          expect(emitted_event_ids).to include('slice0-response1-item0')
+        end
 
-      it 'emits the hitson the second page of the second slice' do
-        expect(emitted_event_ids).to include('slice1-response1-item0')
-        expect(emitted_event_ids).to include('slice1-response1-item1')
-      end
+        it 'emits the hits on the first page of the second slice' do
+          expect(emitted_event_ids).to include('slice1-response0-item0')
+          expect(emitted_event_ids).to include('slice1-response0-item1')
+        end
+
+        it 'emits the hits on the second page of the second slice' do
+          expect(emitted_event_ids).to include('slice1-response1-item0')
+          expect(emitted_event_ids).to include('slice1-response1-item1')
+        end
 
-      it 'does not double-emit' do
-        expect(emitted_event_ids.uniq).to eq(emitted_event_ids)
+        it 'does not double-emit' do
+          expect(emitted_event_ids.uniq).to eq(emitted_event_ids)
+        end
+
+        it 'emits events with appropriate fields' do
+          emitted_events.each do |event|
+            expect(event).to be_a(LogStash::Event)
+            expect(event.get('message')).to eq(['hello, world'])
+            expect(event.get('[@metadata][_id]')).to_not be_nil
+            expect(event.get('[@metadata][_id]')).to_not be_empty
+            expect(event.get('[@metadata][_index]')).to start_with('logstash-')
+          end
+        end
       end
 
-      it 'emits events with appropriate fields' do
-        emitted_events.each do |event|
-          expect(event).to be_a(LogStash::Event)
-          expect(event.get('message')).to eq(['hello, world'])
-          expect(event.get('[@metadata][_id]')).to_not be_nil
-          expect(event.get('[@metadata][_id]')).to_not be_empty
-          expect(event.get('[@metadata][_index]')).to start_with('logstash-')
+      describe "with scroll request fail" do
+        before(:each) do
+          expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
+          plugin.register
+
+          expect(client).to receive(:clear_scroll).and_return(nil)
+
+          # SLICE0 is a three-page scroll in which the second page throw exception
+          slice0_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 0, 'max' => 2}))
+          expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_raise("boom")
+          allow(client).to receive(:ping)
+
+          # SLICE1 is a two-page scroll in which the last page has no next scroll id
+          slice1_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 1, 'max' => 2}))
+          expect(client).to receive(:search).with(hash_including(:body => slice1_query)).and_return(slice1_response0)
+          expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice1_scroll1 })).and_return(slice1_response1)
+
+          synchronize_method!(plugin, :scroll_request)
+          synchronize_method!(plugin, :search_request)
+        end
+
+        let(:client) { Elasticsearch::Client.new }
+
+        it 'does not insert event to queue' do
+          expect(plugin).to receive(:parallel_slice).and_wrap_original do |m, *args|
+            slice0, slice1 = m.call
+            expect(slice0[0]).to be_falsey
+            expect(slice1[0]).to be_truthy
+            expect(slice1[1].size).to eq(4) # four items from SLICE1
+            [slice0, slice1]
+          end
+
+          queue = Queue.new
+          plugin.run(queue)
+          expect(queue.size).to eq(0)
         end
       end
     end
@@ -450,24 +502,21 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
       end
 
-      context 'with docinfo enabled' do
-        let(:config_metadata) do
-          %q[
-              input {
-                elasticsearch {
-                  hosts => ["localhost"]
-                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                  docinfo => true
-                }
-              }
-          ]
+      before do
+        if do_register
+          plugin.register
+          plugin.run queue
         end
+      end
 
-        it "provides document info under metadata" do
-          event = input(config_metadata) do |pipeline, queue|
-            queue.pop
-          end
+      let(:do_register) { true }
+
+      let(:event) { queue.pop }
 
+      context 'with docinfo enabled' do
+        let(:config) { base_config.merge 'docinfo' => true }
+
+        it "provides document info under metadata" do
           if ecs_select.active_mode == :disabled
             expect(event.get("[@metadata][_index]")).to eq('logstash-2014.10.12')
             expect(event.get("[@metadata][_type]")).to eq('logs')
@@ -479,123 +528,72 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
           end
         end
 
-        it 'merges values if the `docinfo_target` already exist in the `_source` document' do
-          config_metadata_with_hash = %Q[
-              input {
-                elasticsearch {
-                  hosts => ["localhost"]
-                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                  docinfo => true
-                  docinfo_target => 'metadata_with_hash'
-                }
-              }
-          ]
+        context 'with docinfo_target' do
+          let(:config) { base_config.merge 'docinfo' => true, 'docinfo_target' => docinfo_target }
+          let(:docinfo_target) { 'metadata_with_hash' }
+
+          it 'merges values if the `docinfo_target` already exist in the `_source` document' do
+            expect(event.get("[metadata_with_hash][_index]")).to eq('logstash-2014.10.12')
+            expect(event.get("[metadata_with_hash][_type]")).to eq('logs')
+            expect(event.get("[metadata_with_hash][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+            expect(event.get("[metadata_with_hash][awesome]")).to eq("logstash")
+          end
+
+          context 'non-existent' do
+            let(:docinfo_target) { 'meta' }
+
+            it 'should move the document information to the specified field' do
+              expect(event.get("[meta][_index]")).to eq('logstash-2014.10.12')
+              expect(event.get("[meta][_type]")).to eq('logs')
+              expect(event.get("[meta][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+            end
 
-          event = input(config_metadata_with_hash) do |pipeline, queue|
-            queue.pop
           end
 
-          expect(event.get("[metadata_with_hash][_index]")).to eq('logstash-2014.10.12')
-          expect(event.get("[metadata_with_hash][_type]")).to eq('logs')
-          expect(event.get("[metadata_with_hash][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
-          expect(event.get("[metadata_with_hash][awesome]")).to eq("logstash")
         end
 
         context 'if the `docinfo_target` exist but is not of type hash' do
-          let (:config) { {
-              "hosts" => ["localhost"],
-              "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
-              "docinfo" => true,
-              "docinfo_target" => 'metadata_with_string'
-          } }
-          it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
+          let(:config) { base_config.merge 'docinfo' => true, "docinfo_target" => 'metadata_with_string' }
+          let(:do_register) { false }
+
+          it 'raises an exception if the `docinfo_target` exist but is not of type hash' do
             expect(client).not_to receive(:clear_scroll)
             plugin.register
             expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
           end
-        end
 
-        it 'should move the document information to the specified field' do
-          config = %q[
-              input {
-                elasticsearch {
-                  hosts => ["localhost"]
-                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                  docinfo => true
-                  docinfo_target => 'meta'
-                }
-              }
-          ]
-          event = input(config) do |pipeline, queue|
-            queue.pop
-          end
-
-          expect(event.get("[meta][_index]")).to eq('logstash-2014.10.12')
-          expect(event.get("[meta][_type]")).to eq('logs')
-          expect(event.get("[meta][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
         end
 
-        it "allows to specify which fields from the document info to save to metadata" do
-          fields = ["_index"]
-          config = %Q[
-              input {
-                elasticsearch {
-                  hosts => ["localhost"]
-                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                  docinfo => true
-                  docinfo_fields => #{fields}
-                }
-              }]
+        context 'with docinfo_fields' do
+          let(:config) { base_config.merge 'docinfo' => true, "docinfo_fields" => ["_index"] }
 
-          event = input(config) do |pipeline, queue|
-            queue.pop
+          it "allows to specify which fields from the document info to save to metadata" do
+            meta_base = event.get(ecs_select.active_mode == :disabled ? "@metadata" : "[@metadata][input][elasticsearch]")
+            expect(meta_base.keys).to eql ["_index"]
           end
 
-          meta_base = event.get(ecs_select.active_mode == :disabled ? "@metadata" : "[@metadata][input][elasticsearch]")
-          expect(meta_base.keys).to eq(fields)
         end
 
-        it 'should be able to reference metadata fields in `add_field` decorations' do
-          config = %q[
-            input {
-              elasticsearch {
-                hosts => ["localhost"]
-                query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                docinfo => true
-                add_field => {
-                  'identifier' => "foo:%{[@metadata][_type]}:%{[@metadata][_id]}"
-                }
-              }
-            }
-          ]
+        context 'add_field' do
+          let(:config) { base_config.merge 'docinfo' => true,
+                                           'add_field' => { 'identifier' => "foo:%{[@metadata][_type]}:%{[@metadata][_id]}" } }
 
-          event = input(config) do |pipeline, queue|
-            queue.pop
-          end
+          it 'should be able to reference metadata fields in `add_field` decorations' do
+            expect(event.get('identifier')).to eq('foo:logs:C5b2xLQwTZa76jBmHIbwHQ')
+          end if ecs_select.active_mode == :disabled
 
-          expect(event.get('identifier')).to eq('foo:logs:C5b2xLQwTZa76jBmHIbwHQ')
-        end if ecs_select.active_mode == :disabled
+        end
 
       end
 
-    end
+      context "when not defining the docinfo" do
+        let(:config) { base_config }
 
-    context "when not defining the docinfo" do
-      it 'should keep the document information in the root of the event' do
-        config = %q[
-          input {
-            elasticsearch {
-              hosts => ["localhost"]
-              query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-            }
-          }
-        ]
-        event = input(config) do |pipeline, queue|
-          queue.pop
+        it 'should keep the document information in the root of the event' do
+          expect(event.get("[@metadata]")).to be_empty
         end
-
-        expect(event.get("[@metadata]")).to be_empty
       end
+
     end
   end
 
@@ -740,9 +738,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
             begin
               @server = WEBrick::HTTPServer.new :Port => 0, :DocumentRoot => ".",
                        :Logger => Cabin::Channel.get, # silence WEBrick logging
-                       :StartCallback => Proc.new {
-                             queue.push("started")
-                           }
+                       :StartCallback => Proc.new { queue.push("started") }
               @port = @server.config[:Port]
               @server.mount_proc '/' do |req, res|
                 res.body = '''
@@ -811,11 +807,9 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
                 @first_req_waiter.countDown()
               end
 
-
-
               @server.start
             rescue => e
-              puts "Error in webserver thread #{e}"
+              warn "ERROR in webserver thread #{e.inspect}\n  #{e.backtrace.join("\n  ")}"
               # ignore
             end
           end
@@ -914,6 +908,8 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
           plugin.register
         end
+
+        after { plugin.do_stop }
       end
     end
 
@@ -942,18 +938,98 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     end
 
     it "should properly schedule" do
-      expect(plugin).to receive(:do_run) {
-        queue << LogStash::Event.new({})
-      }.at_least(:twice)
-      runner = Thread.start { plugin.run(queue) }
-      sleep 3.0
-      plugin.stop
-      runner.join
-      expect(queue.size).to be >= 2
+      begin
+        expect(plugin).to receive(:do_run) {
+          queue << LogStash::Event.new({})
+        }.at_least(:twice)
+        runner = Thread.start { plugin.run(queue) }
+        expect(queue.pop).not_to be_nil
+        cron_jobs = plugin.instance_variable_get(:@_scheduler).instance_variable_get(:@impl).jobs
+        expect(cron_jobs[0].next_time - cron_jobs[0].last_time).to be <= 5.0
+        expect(queue.pop).not_to be_nil
+      ensure
+        plugin.do_stop
+        runner.join if runner
+      end
     end
 
   end
 
+  context "retries" do
+    let(:mock_response) do
+      {
+        "_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
+        "took" => 27,
+        "timed_out" => false,
+        "_shards" => {
+          "total" => 169,
+          "successful" => 169,
+          "failed" => 0
+        },
+        "hits" => {
+          "total" => 1,
+          "max_score" => 1.0,
+          "hits" => [ {
+                        "_index" => "logstash-2014.10.12",
+                        "_type" => "logs",
+                        "_id" => "C5b2xLQwTZa76jBmHIbwHQ",
+                        "_score" => 1.0,
+                        "_source" => { "message" => ["ohayo"] }
+                      } ]
+        }
+      }
+    end
+
+    let(:mock_scroll_response) do
+      {
+        "_scroll_id" => "r453Wc1jh0caLJhSDg",
+        "hits" => { "hits" => [] }
+      }
+    end
+
+    before(:each) do
+      client = Elasticsearch::Client.new
+      allow(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
+      allow(client).to receive(:search).with(any_args).and_return(mock_response)
+      allow(client).to receive(:scroll).with({ :body => { :scroll_id => "cXVlcnlUaGVuRmV0Y2g" }, :scroll=> "1m" }).and_return(mock_scroll_response)
+      allow(client).to receive(:clear_scroll).and_return(nil)
+      allow(client).to receive(:ping)
+    end
+
+    let(:config) do
+      {
+        "hosts" => ["localhost"],
+        "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
+        "retries" => 1
+      }
+    end
+
+    it "retry and log error when all search request fail" do
+      expect(plugin.logger).to receive(:error).with(/Tried .* unsuccessfully/,
+                                                    hash_including(:message => 'Manticore::UnknownException'))
+      expect(plugin.logger).to receive(:warn).twice.with(/Attempt to .* but failed/,
+                                                         hash_including(:exception => "Manticore::UnknownException"))
+      expect(plugin).to receive(:search_request).with(instance_of(Hash)).and_raise(Manticore::UnknownException).at_least(:twice)
+
+      plugin.register
+
+      expect{ plugin.run(queue) }.not_to raise_error
+      expect(queue.size).to eq(0)
+    end
+
+    it "retry successfully when search request fail for one time" do
+      expect(plugin.logger).to receive(:warn).once.with(/Attempt to .* but failed/,
+                                                         hash_including(:exception => "Manticore::UnknownException"))
+      expect(plugin).to receive(:search_request).with(instance_of(Hash)).once.and_raise(Manticore::UnknownException)
+      expect(plugin).to receive(:search_request).with(instance_of(Hash)).once.and_call_original
+
+      plugin.register
+
+      expect{ plugin.run(queue) }.not_to raise_error
+      expect(queue.size).to eq(1)
+    end
+  end
+
   # @note can be removed once we depends on elasticsearch gem >= 6.x
   def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
     client.transport.respond_to?(:transport) ? client.transport.transport : client.transport

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -72,25 +72,45 @@ describe LogStash::Inputs::Elasticsearch do
 
   describe 'against a secured elasticsearch', secure_integration: true do
 
+    # client_options is for an out-of-band helper
     let(:client_options) { { :ca_file => ca_file, :user => user, :password => password } }
 
-    let(:config) { super().merge('user' => user, 'password' => password, 'ssl' => true, 'ca_file' => ca_file) }
+    let(:config) { super().merge('user' => user, 'password' => password) }
 
-    it_behaves_like 'an elasticsearch index plugin'
+    shared_examples 'secured_elasticsearch' do
+      it_behaves_like 'an elasticsearch index plugin'
 
-    context "incorrect auth credentials" do
+      context "incorrect auth credentials" do
 
-      let(:config) do
-        super().merge('user' => 'archer', 'password' => 'b0gus!')
-      end
+        let(:config) do
+          super().merge('user' => 'archer', 'password' => 'b0gus!')
+        end
 
-      let(:queue) { [] }
+        let(:queue) { [] }
 
-      it "fails to run the plugin" do
-        expect { plugin.register }.to raise_error Elasticsearch::Transport::Transport::Errors::Unauthorized
+        it "fails to run the plugin" do
+          expect { plugin.register }.to raise_error Elasticsearch::Transport::Transport::Errors::Unauthorized
+        end
       end
     end
 
+    context 'with ca_file' do
+      let(:config) { super().merge('ssl' => true, 'ca_file' => ca_file) }
+      it_behaves_like 'secured_elasticsearch'
+    end
+
+    context 'with `ca_trusted_fingerprint`' do
+      let(:ca_trusted_fingerprint) { File.read("spec/fixtures/test_certs/ca.der.sha256").chomp }
+      let(:config) { super().merge('ssl' => true, 'ca_trusted_fingerprint' => ca_trusted_fingerprint) }
+
+      if Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create("8.3.0")
+        it_behaves_like 'secured_elasticsearch'
+      else
+        it 'raises a configuration error' do
+          expect { plugin }.to raise_exception(LogStash::ConfigurationError, a_string_including("ca_trusted_fingerprint"))
+        end
+      end
+    end
   end
 
   context 'setting host:port', integration: true do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.12.3
+  version: 4.15.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-29 00:00:00.000000000 Z
+date: 2022-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -72,6 +72,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-scheduler
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -89,24 +103,24 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  name: tzinfo
+        version: '1.0'
+  name: logstash-mixin-ca_trusted_fingerprint_support
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: tzinfo-data
+  name: tzinfo
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
@@ -120,7 +134,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rufus-scheduler
+  name: tzinfo-data
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
@@ -212,6 +226,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.9
+  name: rufus-scheduler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.9
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program
@@ -227,12 +255,14 @@ files:
 - NOTICE.TXT
 - README.md
 - docs/index.asciidoc
+- lib/logstash/helpers/loggable_try.rb
 - lib/logstash/inputs/elasticsearch.rb
 - lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_connections_selector.rb
 - lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb
 - logstash-input-elasticsearch.gemspec
 - spec/es_helper.rb
 - spec/fixtures/test_certs/ca.crt
+- spec/fixtures/test_certs/ca.der.sha256
 - spec/fixtures/test_certs/ca.key
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
@@ -266,6 +296,7 @@ summary: Reads query results from an Elasticsearch cluster
 test_files:
 - spec/es_helper.rb
 - spec/fixtures/test_certs/ca.crt
+- spec/fixtures/test_certs/ca.der.sha256
 - spec/fixtures/test_certs/ca.key
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key