logstash-input-elasticsearch 4.12.2 → 4.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: caea2fedebb629d87471f67b2b03f43186e7e2b02e854e3f6df7154a64316f16
-  data.tar.gz: 1d75f76b97d6d8ea481637399b1ea82668f5fb47ea52a2f1a51034594b55375f
+  metadata.gz: 1bd8f2c20c31a39f42e8f32e32edbfef275a15de8040491b8d2549f55a3e9002
+  data.tar.gz: 68abc131e2c9ec958cc3360137b273b39303923d60b7dca4303aa43bf1425574
 SHA512:
-  metadata.gz: 07af682a46bc7aa7bd48b0d4c4f0a4baa9e8dd5a8848821863b67fe4b2ee5890655d76de1de782938be0b73bf3057228a94ccbf155b7fb9ecaa81077831362e5
-  data.tar.gz: 0031f21cc9d7e41dfdbf9de400b33e21a1b9abda7c9ca2159fb21623ff5cf21258b99670d22090eda39ad42141408166c10317874ea6682e4e3b756e031355a1
+  metadata.gz: 801739c8baf2655104a502176b42157c35c282dab719ec0bf0eb4274feef9ce6e79238870aa6ebb2604f4da726446089e94d4e5e8d48c50d731eaae55c447be9
+  data.tar.gz: 5af8ea0a963ce1d8d924b51428dbb70c9ab9e9d0df0146e2b5f0013ae5b13e56b934dc2540e7ce11080b4ed33c52afb66695bd08739a6a701c337b11d94a3c72

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 4.14.0
+  - Refactor: switch to using scheduler mixin [#177](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/177)
+
+## 4.13.0
+  - Added support for `ca_trusted_fingerprint` when run on Logstash 8.3+ [#178](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/178)
+
+## 4.12.3
+  - Fix: update Elasticsearch Ruby client to correctly customize 'user-agent' header [#171](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/171)
+
 ## 4.12.2
   - Fix: hosts => "es_host:port" regression [#168](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/168)
 
@@ -17,11 +26,10 @@
   - Fixed SSL handshake hang indefinitely with proxy setup [#156](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/156)
 
 ## 4.9.2
-  - Fix: a regression (in LS 7.14.0) where due the elasticsearch client update (from 5.0.5 to 7.5.0) the `Authorization` 
+  - Fix: a regression (in LS 7.14.0) where due the elasticsearch client update (from 5.0.5 to 7.5.0) the `Authorization`
     header isn't passed, this leads to the plugin not being able to leverage `user`/`password` credentials set by the user.
     [#153](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/153)
 
-
 ## 4.9.1
   - [DOC] Replaced hard-coded links with shared attributes [#143](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/143)
   - [DOC] Added missing quote to docinfo_fields example [#145](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/145)
@@ -61,7 +69,7 @@
   - Feat: Added support for cloud_id / cloud_auth configuration [#112](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/112)
 
 ## 4.4.0
-  - Changed Elasticsearch Client transport to use Manticore [#111](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/111) 
+  - Changed Elasticsearch Client transport to use Manticore [#111](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/111)
 
 ## 4.3.3
   - Loosen restrictions on Elasticsearch gem [#110](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/110)
@@ -100,7 +108,7 @@
 
 ## 4.0.2
   - Bump ES client to 5.0.2 to get content-type: json behavior
-  - Revert unneeded manticore change 
+  - Revert unneeded manticore change
 
 ## 4.0.1
   - Switch internal HTTP client to support TLSv1.2
@@ -113,7 +121,7 @@
     behavior
   - Improve documentation to show sort by \_doc, and how to add it to custom
     queries.
-    
+
 ## 3.0.2
   - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 

data/docs/index.asciidoc

@@ -103,6 +103,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-api_key>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-connect_timeout_seconds>> | <<number,number>>|No
@@ -152,6 +153,15 @@ API key API].
 
 SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary.
 
+[id="plugins-{type}s-{plugin}-ca_trusted_fingerprint"]
+===== `ca_trusted_fingerprint`
+
+* Value type is <<string,string>>, and must contain exactly 64 hexadecimal characters.
+* There is no default value for this setting.
+* Use of this option _requires_ Logstash 8.3+
+
+The SHA-256 fingerprint of an SSL Certificate Authority to trust, such as the autogenerated self-signed CA for an Elasticsearch cluster.
+
 [id="plugins-{type}s-{plugin}-cloud_auth"]
 ===== `cloud_auth`
 

data/lib/logstash/inputs/elasticsearch.rb

@@ -7,6 +7,8 @@ require 'logstash/plugin_mixins/validator_support/field_reference_validation_ada
 require 'logstash/plugin_mixins/event_support/event_factory_adapter'
 require 'logstash/plugin_mixins/ecs_compatibility_support'
 require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
+require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
+require "logstash/plugin_mixins/scheduler"
 require "base64"
 
 require "elasticsearch"
@@ -77,6 +79,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
   extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
 
+  include LogStash::PluginMixins::Scheduler
+
   config_name "elasticsearch"
 
   # List of elasticsearch hosts to use for querying.
@@ -192,6 +196,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # If set, the _source of each hit will be added nested under the target instead of at the top-level
   config :target, :validate => :field_reference
 
+  # config :ca_trusted_fingerprint, :validate => :sha_256_hex
+  include LogStash::PluginMixins::CATrustedFingerprintSupport
+
   def initialize(params={})
     super(params)
 
@@ -247,21 +254,13 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
   def run(output_queue)
     if @schedule
-      @scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
-      @scheduler.cron @schedule do
-        do_run(output_queue)
-      end
-
-      @scheduler.join
+      scheduler.cron(@schedule) { do_run(output_queue) }
+      scheduler.join
     else
       do_run(output_queue)
     end
   end
 
-  def stop
-    @scheduler.stop if @scheduler
-  end
-
   private
 
   def do_run(output_queue)
@@ -270,9 +269,10 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
     logger.warn("managed slices for query is very large (#{@slices}); consider reducing") if @slices > 8
 
+    pipeline_id = execution_context&.pipeline_id || 'main'
     @slices.times.map do |slice_id|
       Thread.new do
-        LogStash::Util::set_thread_name("#{@id}_slice_#{slice_id}")
+        LogStash::Util::set_thread_name("[#{pipeline_id}]|input|elasticsearch|slice_#{slice_id}")
         do_run_slice(output_queue, slice_id)
       end
     end.map(&:join)
@@ -381,7 +381,13 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   end
 
   def setup_ssl
-    @ssl && @ca_file ? { :ssl  => true, :ca_file => @ca_file } : {}
+    ssl_options = {}
+
+    ssl_options[:ssl] = true if @ssl
+    ssl_options[:ca_file] = @ca_file if @ssl && @ca_file
+    ssl_options[:trust_strategy] = trust_strategy_for_ca_trusted_fingerprint
+
+    ssl_options
   end
 
   def setup_hosts

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.12.2'
+  s.version         = '4.14.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -24,18 +24,21 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
   s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0'
   s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
+  s.add_runtime_dependency "logstash-mixin-scheduler", '~> 1.0'
 
-  s.add_runtime_dependency 'elasticsearch', '>= 7.14.0' # LS >= 6.7 and < 7.14 all used version 5.0.5
+  s.add_runtime_dependency 'elasticsearch', '>= 7.17.1'
+  s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
 
   s.add_runtime_dependency 'tzinfo'
   s.add_runtime_dependency 'tzinfo-data'
-  s.add_runtime_dependency 'rufus-scheduler'
   s.add_runtime_dependency 'manticore', ">= 0.7.1"
 
   s.add_development_dependency 'logstash-codec-plain'
-  s.add_development_dependency 'faraday', "~> 1"
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'timecop'
   s.add_development_dependency 'cabin', ['~> 0.6']
   s.add_development_dependency 'webrick'
+
+  # 3.8.0 has breaking changes WRT to joining, which break our specs
+  s.add_development_dependency 'rufus-scheduler', '~> 3.0.9'
 end

data/spec/es_helper.rb

@@ -7,25 +7,26 @@ module ESHelper
     end
   end
 
-  def self.get_client(options)
-    require 'elasticsearch/transport/transport/http/faraday' # supports user/password options
-    host, port = get_host_port.split(':')
-    host_opts = { host: host, port: port, scheme: 'http' }
-    ssl_opts = {}
-
-    if options[:ca_file]
-      ssl_opts = { ca_file: options[:ca_file], version: 'TLSv1.2', verify: false }
-      host_opts[:scheme] = 'https'
+  def self.curl_and_get_json_response(url, method: :get, args: nil); require 'open3'
+    cmd = "curl -s -v --show-error #{args} -X #{method.to_s.upcase} -k #{url}"
+    begin
+      out, err, status = Open3.capture3(cmd)
+    rescue Errno::ENOENT
+      fail "curl not available, make sure curl binary is installed and available on $PATH"
     end
 
-    if options[:user] && options[:password]
-      host_opts[:user] = options[:user]
-      host_opts[:password] = options[:password]
-    end
+    if status.success?
+      http_status = err.match(/< HTTP\/1.1 (.*?)/)[1] || '0' # < HTTP/1.1 200 OK\r\n
+      if http_status.strip[0].to_i > 2
+        warn out
+        fail "#{cmd.inspect} unexpected response: #{http_status}\n\n#{err}"
+      end
 
-    Elasticsearch::Client.new(hosts: [host_opts],
-                              transport_options: { ssl: ssl_opts },
-                              transport_class: Elasticsearch::Transport::Transport::HTTP::Faraday)
+      LogStash::Json.load(out)
+    else
+      warn out
+      fail "#{cmd.inspect} process failed: #{status}\n\n#{err}"
+    end
   end
 
   def self.doc_type

data/spec/fixtures/test_certs/ca.der.sha256

@@ -0,0 +1 @@
+195a7e7b1bc29f3d7913a918a44721704d27fa56facea0cd72a8093c7107c283

data/spec/inputs/elasticsearch_spec.rb

@@ -19,7 +19,14 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
   let(:queue) { Queue.new }
 
   before(:each) do
-     Elasticsearch::Client.send(:define_method, :ping) { } # define no-action ping method
+    Elasticsearch::Client.send(:define_method, :ping) { } # define no-action ping method
+  end
+
+  let(:base_config) do
+    {
+        'hosts' => ["localhost"],
+        'query' => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+    }
   end
 
   context "register" do
@@ -47,7 +54,6 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
   end
 
   it_behaves_like "an interruptible input plugin" do
-    let(:esclient) { double("elasticsearch-client") }
     let(:config) do
       {
         "schedule" => "* * * * * UTC"
@@ -55,7 +61,8 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     end
 
     before :each do
-      allow(Elasticsearch::Client).to receive(:new).and_return(esclient)
+      @esclient = double("elasticsearch-client")
+      allow(Elasticsearch::Client).to receive(:new).and_return(@esclient)
       hit = {
         "_index" => "logstash-2014.10.12",
         "_type" => "logs",
@@ -63,10 +70,10 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         "_score" => 1.0,
         "_source" => { "message" => ["ohayo"] }
       }
-      allow(esclient).to receive(:search) { { "hits" => { "hits" => [hit] } } }
-      allow(esclient).to receive(:scroll) { { "hits" => { "hits" => [hit] } } }
-      allow(esclient).to receive(:clear_scroll).and_return(nil)
-      allow(esclient).to receive(:ping)
+      allow(@esclient).to receive(:search) { { "hits" => { "hits" => [hit] } } }
+      allow(@esclient).to receive(:scroll) { { "hits" => { "hits" => [hit] } } }
+      allow(@esclient).to receive(:clear_scroll).and_return(nil)
+      allow(@esclient).to receive(:ping)
     end
   end
 
@@ -78,14 +85,10 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     end
 
     let(:config) do
-      %q[
-        input {
-          elasticsearch {
-            hosts => ["localhost"]
-            query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-          }
-        }
-      ]
+      {
+          'hosts' => ["localhost"],
+          'query' => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+      }
     end
 
     let(:mock_response) do
@@ -128,10 +131,11 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       expect(client).to receive(:ping)
     end
 
+    before { plugin.register }
+
     it 'creates the events from the hits' do
-      event = input(config) do |pipeline, queue|
-        queue.pop
-      end
+      plugin.run queue
+      event = queue.pop
 
       expect(event).to be_a(LogStash::Event)
       expect(event.get("message")).to eql [ "ohayo" ]
@@ -139,21 +143,16 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
     context 'when a target is set' do
       let(:config) do
-        %q[
-          input {
-            elasticsearch {
-              hosts => ["localhost"]
-              query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-              target => "[@metadata][_source]"
-            }
-          }
-        ]
+        {
+            'hosts' => ["localhost"],
+            'query' => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
+            'target' => "[@metadata][_source]"
+        }
       end
 
       it 'creates the event using the target' do
-        event = input(config) do |pipeline, queue|
-          queue.pop
-        end
+        plugin.run queue
+        event = queue.pop
 
         expect(event).to be_a(LogStash::Event)
         expect(event.get("[@metadata][_source][message]")).to eql [ "ohayo" ]
@@ -450,24 +449,21 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
       end
 
-      context 'with docinfo enabled' do
-        let(:config_metadata) do
-          %q[
-              input {
-                elasticsearch {
-                  hosts => ["localhost"]
-                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                  docinfo => true
-                }
-              }
-          ]
+      before do
+        if do_register
+          plugin.register
+          plugin.run queue
         end
+      end
 
-        it "provides document info under metadata" do
-          event = input(config_metadata) do |pipeline, queue|
-            queue.pop
-          end
+      let(:do_register) { true }
+
+      let(:event) { queue.pop }
 
+      context 'with docinfo enabled' do
+        let(:config) { base_config.merge 'docinfo' => true }
+
+        it "provides document info under metadata" do
           if ecs_select.active_mode == :disabled
             expect(event.get("[@metadata][_index]")).to eq('logstash-2014.10.12')
             expect(event.get("[@metadata][_type]")).to eq('logs')
@@ -479,123 +475,72 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
           end
         end
 
-        it 'merges values if the `docinfo_target` already exist in the `_source` document' do
-          config_metadata_with_hash = %Q[
-              input {
-                elasticsearch {
-                  hosts => ["localhost"]
-                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                  docinfo => true
-                  docinfo_target => 'metadata_with_hash'
-                }
-              }
-          ]
+        context 'with docinfo_target' do
+          let(:config) { base_config.merge 'docinfo' => true, 'docinfo_target' => docinfo_target }
+          let(:docinfo_target) { 'metadata_with_hash' }
+
+          it 'merges values if the `docinfo_target` already exist in the `_source` document' do
+            expect(event.get("[metadata_with_hash][_index]")).to eq('logstash-2014.10.12')
+            expect(event.get("[metadata_with_hash][_type]")).to eq('logs')
+            expect(event.get("[metadata_with_hash][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+            expect(event.get("[metadata_with_hash][awesome]")).to eq("logstash")
+          end
+
+          context 'non-existent' do
+            let(:docinfo_target) { 'meta' }
+
+            it 'should move the document information to the specified field' do
+              expect(event.get("[meta][_index]")).to eq('logstash-2014.10.12')
+              expect(event.get("[meta][_type]")).to eq('logs')
+              expect(event.get("[meta][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+            end
 
-          event = input(config_metadata_with_hash) do |pipeline, queue|
-            queue.pop
           end
 
-          expect(event.get("[metadata_with_hash][_index]")).to eq('logstash-2014.10.12')
-          expect(event.get("[metadata_with_hash][_type]")).to eq('logs')
-          expect(event.get("[metadata_with_hash][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
-          expect(event.get("[metadata_with_hash][awesome]")).to eq("logstash")
         end
 
         context 'if the `docinfo_target` exist but is not of type hash' do
-          let (:config) { {
-              "hosts" => ["localhost"],
-              "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
-              "docinfo" => true,
-              "docinfo_target" => 'metadata_with_string'
-          } }
-          it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
+          let(:config) { base_config.merge 'docinfo' => true, "docinfo_target" => 'metadata_with_string' }
+          let(:do_register) { false }
+
+          it 'raises an exception if the `docinfo_target` exist but is not of type hash' do
             expect(client).not_to receive(:clear_scroll)
             plugin.register
             expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
           end
-        end
 
-        it 'should move the document information to the specified field' do
-          config = %q[
-              input {
-                elasticsearch {
-                  hosts => ["localhost"]
-                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                  docinfo => true
-                  docinfo_target => 'meta'
-                }
-              }
-          ]
-          event = input(config) do |pipeline, queue|
-            queue.pop
-          end
-
-          expect(event.get("[meta][_index]")).to eq('logstash-2014.10.12')
-          expect(event.get("[meta][_type]")).to eq('logs')
-          expect(event.get("[meta][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
         end
 
-        it "allows to specify which fields from the document info to save to metadata" do
-          fields = ["_index"]
-          config = %Q[
-              input {
-                elasticsearch {
-                  hosts => ["localhost"]
-                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                  docinfo => true
-                  docinfo_fields => #{fields}
-                }
-              }]
+        context 'with docinfo_fields' do
+          let(:config) { base_config.merge 'docinfo' => true, "docinfo_fields" => ["_index"] }
 
-          event = input(config) do |pipeline, queue|
-            queue.pop
+          it "allows to specify which fields from the document info to save to metadata" do
+            meta_base = event.get(ecs_select.active_mode == :disabled ? "@metadata" : "[@metadata][input][elasticsearch]")
+            expect(meta_base.keys).to eql ["_index"]
           end
 
-          meta_base = event.get(ecs_select.active_mode == :disabled ? "@metadata" : "[@metadata][input][elasticsearch]")
-          expect(meta_base.keys).to eq(fields)
         end
 
-        it 'should be able to reference metadata fields in `add_field` decorations' do
-          config = %q[
-            input {
-              elasticsearch {
-                hosts => ["localhost"]
-                query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                docinfo => true
-                add_field => {
-                  'identifier' => "foo:%{[@metadata][_type]}:%{[@metadata][_id]}"
-                }
-              }
-            }
-          ]
+        context 'add_field' do
+          let(:config) { base_config.merge 'docinfo' => true,
+                                           'add_field' => { 'identifier' => "foo:%{[@metadata][_type]}:%{[@metadata][_id]}" } }
 
-          event = input(config) do |pipeline, queue|
-            queue.pop
-          end
+          it 'should be able to reference metadata fields in `add_field` decorations' do
+            expect(event.get('identifier')).to eq('foo:logs:C5b2xLQwTZa76jBmHIbwHQ')
+          end if ecs_select.active_mode == :disabled
 
-          expect(event.get('identifier')).to eq('foo:logs:C5b2xLQwTZa76jBmHIbwHQ')
-        end if ecs_select.active_mode == :disabled
+        end
 
       end
 
-    end
+      context "when not defining the docinfo" do
+        let(:config) { base_config }
 
-    context "when not defining the docinfo" do
-      it 'should keep the document information in the root of the event' do
-        config = %q[
-          input {
-            elasticsearch {
-              hosts => ["localhost"]
-              query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-            }
-          }
-        ]
-        event = input(config) do |pipeline, queue|
-          queue.pop
+        it 'should keep the document information in the root of the event' do
+          expect(event.get("[@metadata]")).to be_empty
         end
-
-        expect(event.get("[@metadata]")).to be_empty
       end
+
     end
   end
 
@@ -740,9 +685,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
             begin
               @server = WEBrick::HTTPServer.new :Port => 0, :DocumentRoot => ".",
                        :Logger => Cabin::Channel.get, # silence WEBrick logging
-                       :StartCallback => Proc.new {
-                             queue.push("started")
-                           }
+                       :StartCallback => Proc.new { queue.push("started") }
               @port = @server.config[:Port]
               @server.mount_proc '/' do |req, res|
                 res.body = '''
@@ -811,11 +754,9 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
                 @first_req_waiter.countDown()
               end
 
-
-
               @server.start
             rescue => e
-              puts "Error in webserver thread #{e}"
+              warn "ERROR in webserver thread #{e.inspect}\n  #{e.backtrace.join("\n  ")}"
               # ignore
             end
           end
@@ -914,6 +855,8 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
           plugin.register
         end
+
+        after { plugin.do_stop }
       end
     end
 
@@ -933,7 +876,7 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       {
         "hosts" => ["localhost"],
         "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
-        "schedule" => "* * * * * UTC"
+        "schedule" => "* * * * * * UTC" # every second
       }
     end
 
@@ -942,21 +885,17 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
     end
 
     it "should properly schedule" do
-      Timecop.travel(Time.new(2000))
-      Timecop.scale(60)
-      runner = Thread.new do
+      begin
         expect(plugin).to receive(:do_run) {
           queue << LogStash::Event.new({})
         }.at_least(:twice)
-
-        plugin.run(queue)
+        runner = Thread.start { plugin.run(queue) }
+        sleep 3.0
+      ensure
+        plugin.do_stop
+        runner.join if runner
       end
-      sleep 3
-      plugin.stop
-      runner.kill
-      runner.join
-      expect(queue.size).to eq(2)
-      Timecop.return
+      expect(queue.size).to be >= 2
     end
 
   end

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -11,6 +11,7 @@ describe LogStash::Inputs::Elasticsearch do
   let(:config)   { { 'hosts' => ["http#{SECURE_INTEGRATION ? 's' : nil}://#{ESHelper.get_host_port}"],
                      'index' => 'logs',
                      'query' => '{ "query": { "match": { "message": "Not found"} }}' } }
+
   let(:plugin) { described_class.new(config) }
   let(:event)  { LogStash::Event.new({}) }
   let(:client_options) { Hash.new }
@@ -19,22 +20,32 @@ describe LogStash::Inputs::Elasticsearch do
   let(:password) { ENV['ELASTIC_PASSWORD'] || 'abc123' }
   let(:ca_file) { "spec/fixtures/test_certs/ca.crt" }
 
+  let(:es_url) do
+    es_url = ESHelper.get_host_port
+    SECURE_INTEGRATION ? "https://#{es_url}" : "http://#{es_url}"
+  end
+
+  let(:curl_args) do
+    config['user'] ? "-u #{config['user']}:#{config['password']}" : ''
+  end
+
   before(:each) do
-    @es = ESHelper.get_client(client_options)
     # Delete all templates first.
     # Clean ES of data before we start.
-    @es.indices.delete_template(:name => "*")
+    ESHelper.curl_and_get_json_response "#{es_url}/_index_template/*", method: 'DELETE', args: curl_args
     # This can fail if there are no indexes, ignore failure.
-    @es.indices.delete(:index => "*") rescue nil
+    ESHelper.curl_and_get_json_response( "#{es_url}/_index/*", method: 'DELETE', args: curl_args) rescue nil
+    doc_args = "#{curl_args} -H 'Content-Type: application/json' -d '{\"response\": 404, \"message\":\"Not Found\"}'"
     10.times do
-      ESHelper.index_doc(@es, :index => 'logs', :body => { :response => 404, :message=> 'Not Found'})
+      ESHelper.curl_and_get_json_response "#{es_url}/logs/_doc", method: 'POST', args: doc_args
     end
-    @es.indices.refresh
+    ESHelper.curl_and_get_json_response "#{es_url}/_refresh", method: 'POST', args: curl_args
   end
 
   after(:each) do
-    @es.indices.delete_template(:name => "*")
-    @es.indices.delete(:index => "*") rescue nil
+    ESHelper.curl_and_get_json_response "#{es_url}/_index_template/*", method: 'DELETE', args: curl_args
+    # This can fail if there are no indexes, ignore failure.
+    ESHelper.curl_and_get_json_response( "#{es_url}/_index/*", method: 'DELETE', args: curl_args) rescue nil
   end
 
   shared_examples 'an elasticsearch index plugin' do
@@ -61,25 +72,45 @@ describe LogStash::Inputs::Elasticsearch do
 
   describe 'against a secured elasticsearch', secure_integration: true do
 
+    # client_options is for an out-of-band helper
     let(:client_options) { { :ca_file => ca_file, :user => user, :password => password } }
 
-    let(:config) { super().merge('user' => user, 'password' => password, 'ssl' => true, 'ca_file' => ca_file) }
+    let(:config) { super().merge('user' => user, 'password' => password) }
 
-    it_behaves_like 'an elasticsearch index plugin'
+    shared_examples 'secured_elasticsearch' do
+      it_behaves_like 'an elasticsearch index plugin'
 
-    context "incorrect auth credentials" do
+      context "incorrect auth credentials" do
 
-      let(:config) do
-        super().merge('user' => 'archer', 'password' => 'b0gus!')
-      end
+        let(:config) do
+          super().merge('user' => 'archer', 'password' => 'b0gus!')
+        end
 
-      let(:queue) { [] }
+        let(:queue) { [] }
 
-      it "fails to run the plugin" do
-        expect { plugin.register }.to raise_error Elasticsearch::Transport::Transport::Errors::Unauthorized
+        it "fails to run the plugin" do
+          expect { plugin.register }.to raise_error Elasticsearch::Transport::Transport::Errors::Unauthorized
+        end
       end
     end
 
+    context 'with ca_file' do
+      let(:config) { super().merge('ssl' => true, 'ca_file' => ca_file) }
+      it_behaves_like 'secured_elasticsearch'
+    end
+
+    context 'with `ca_trusted_fingerprint`' do
+      let(:ca_trusted_fingerprint) { File.read("spec/fixtures/test_certs/ca.der.sha256").chomp }
+      let(:config) { super().merge('ssl' => true, 'ca_trusted_fingerprint' => ca_trusted_fingerprint) }
+
+      if Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create("8.3.0")
+        it_behaves_like 'secured_elasticsearch'
+      else
+        it 'raises a configuration error' do
+          expect { plugin }.to raise_exception(LogStash::ConfigurationError, a_string_including("ca_trusted_fingerprint"))
+        end
+      end
+    end
   end
 
   context 'setting host:port', integration: true do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.12.2
+  version: 4.14.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-21 00:00:00.000000000 Z
+date: 2022-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -72,12 +72,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-scheduler
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.14.0
+        version: 7.17.1
   name: elasticsearch
   prerelease: false
   type: :runtime
@@ -85,28 +99,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 7.14.0
+        version: 7.17.1
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  name: tzinfo
+        version: '1.0'
+  name: logstash-mixin-ca_trusted_fingerprint_support
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: tzinfo-data
+  name: tzinfo
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
@@ -120,7 +134,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rufus-scheduler
+  name: tzinfo-data
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
@@ -156,20 +170,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1'
-  name: faraday
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -226,6 +226,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.9
+  name: rufus-scheduler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.9
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program
@@ -247,6 +261,7 @@ files:
 - logstash-input-elasticsearch.gemspec
 - spec/es_helper.rb
 - spec/fixtures/test_certs/ca.crt
+- spec/fixtures/test_certs/ca.der.sha256
 - spec/fixtures/test_certs/ca.key
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
@@ -280,6 +295,7 @@ summary: Reads query results from an Elasticsearch cluster
 test_files:
 - spec/es_helper.rb
 - spec/fixtures/test_certs/ca.crt
+- spec/fixtures/test_certs/ca.der.sha256
 - spec/fixtures/test_certs/ca.key
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key