logstash-input-elasticsearch 4.9.0 → 4.12.3

data/spec/inputs/elasticsearch_spec.rb

@@ -7,18 +7,46 @@ require "timecop"
 require "stud/temporary"
 require "time"
 require "date"
+require "cabin"
+require "webrick"
+require "uri"
 
-class LogStash::Inputs::TestableElasticsearch < LogStash::Inputs::Elasticsearch
-  attr_reader :client
-end
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 
-describe LogStash::Inputs::TestableElasticsearch do
+describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
-  let(:plugin) { LogStash::Inputs::TestableElasticsearch.new(config) }
+  let(:plugin) { described_class.new(config) }
   let(:queue) { Queue.new }
 
+  before(:each) do
+     Elasticsearch::Client.send(:define_method, :ping) { } # define no-action ping method
+  end
+
+  context "register" do
+    let(:config) do
+      {
+        "schedule" => "* * * * * UTC"
+      }
+    end
+
+    context "against authentic Elasticsearch" do
+      it "should not raise an exception" do
+       expect { plugin.register }.to_not raise_error
+     end
+    end
+
+    context "against not authentic Elasticsearch" do
+      before(:each) do
+         Elasticsearch::Client.send(:define_method, :ping) { raise Elasticsearch::UnsupportedProductError.new("Fake error") } # define error ping method
+      end
+
+      it "should raise ConfigurationError" do
+        expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+  end
+
   it_behaves_like "an interruptible input plugin" do
-    let(:esclient) { double("elasticsearch-client") }
     let(:config) do
       {
         "schedule" => "* * * * * UTC"
@@ -26,7 +54,8 @@ describe LogStash::Inputs::TestableElasticsearch do
     end
 
     before :each do
-      allow(Elasticsearch::Client).to receive(:new).and_return(esclient)
+      @esclient = double("elasticsearch-client")
+      allow(Elasticsearch::Client).to receive(:new).and_return(@esclient)
       hit = {
         "_index" => "logstash-2014.10.12",
         "_type" => "logs",
@@ -34,13 +63,20 @@ describe LogStash::Inputs::TestableElasticsearch do
         "_score" => 1.0,
         "_source" => { "message" => ["ohayo"] }
       }
-      allow(esclient).to receive(:search) { { "hits" => { "hits" => [hit] } } }
-      allow(esclient).to receive(:scroll) { { "hits" => { "hits" => [hit] } } }
-      allow(esclient).to receive(:clear_scroll).and_return(nil)
+      allow(@esclient).to receive(:search) { { "hits" => { "hits" => [hit] } } }
+      allow(@esclient).to receive(:scroll) { { "hits" => { "hits" => [hit] } } }
+      allow(@esclient).to receive(:clear_scroll).and_return(nil)
+      allow(@esclient).to receive(:ping)
     end
   end
 
-  context 'creating events from Elasticsearch' do
+
+  ecs_compatibility_matrix(:disabled, :v1, :v8) do |ecs_select|
+
+    before(:each) do
+      allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+    end
+
     let(:config) do
       %q[
         input {
@@ -89,6 +125,7 @@ describe LogStash::Inputs::TestableElasticsearch do
       expect(client).to receive(:search).with(any_args).and_return(mock_response)
       expect(client).to receive(:scroll).with({ :body => { :scroll_id => "cXVlcnlUaGVuRmV0Y2g" }, :scroll=> "1m" }).and_return(mock_scroll_response)
       expect(client).to receive(:clear_scroll).and_return(nil)
+      expect(client).to receive(:ping)
     end
 
     it 'creates the events from the hits' do
@@ -97,7 +134,6 @@ describe LogStash::Inputs::TestableElasticsearch do
       end
 
       expect(event).to be_a(LogStash::Event)
-      puts event.to_hash_with_metadata
       expect(event.get("message")).to eql [ "ohayo" ]
     end
 
@@ -120,10 +156,10 @@ describe LogStash::Inputs::TestableElasticsearch do
         end
 
         expect(event).to be_a(LogStash::Event)
-        puts event.to_hash_with_metadata
         expect(event.get("[@metadata][_source][message]")).to eql [ "ohayo" ]
       end
     end
+
   end
 
   # This spec is an adapter-spec, ensuring that we send the right sequence of messages to our Elasticsearch Client
@@ -135,6 +171,7 @@ describe LogStash::Inputs::TestableElasticsearch do
           'query' => "#{LogStash::Json.dump(query)}",
           'slices' => slices,
           'docinfo' => true, # include ids
+          'docinfo_target' => '[@metadata]'
       }
     end
     let(:query) do
@@ -166,7 +203,7 @@ describe LogStash::Inputs::TestableElasticsearch do
     end
 
     context 'without slices directive' do
-      let(:config) { super.tap { |h| h.delete('slices') } }
+      let(:config) { super().tap { |h| h.delete('slices') } }
       it 'runs just one slice' do
         expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
         expect(Thread).to_not receive(:new)
@@ -304,6 +341,7 @@ describe LogStash::Inputs::TestableElasticsearch do
         expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
         expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_return(slice0_response1)
         expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll2 })).and_return(slice0_response2)
+        allow(client).to receive(:ping)
 
         # SLICE1 is a two-page scroll in which the last page has no next scroll id
         slice1_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 1, 'max' => 2}))
@@ -403,129 +441,143 @@ describe LogStash::Inputs::TestableElasticsearch do
       expect(client).to receive(:search).with(any_args).and_return(response)
       allow(client).to receive(:scroll).with({ :body => {:scroll_id => "cXVlcnlUaGVuRmV0Y2g"}, :scroll => "1m" }).and_return(scroll_reponse)
       allow(client).to receive(:clear_scroll).and_return(nil)
+      allow(client).to receive(:ping).and_return(nil)
     end
 
-    context 'when defining docinfo' do
-      let(:config_metadata) do
-        %q[
-            input {
-              elasticsearch {
-                hosts => ["localhost"]
-                query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                docinfo => true
-              }
-            }
-        ]
-      end
+    ecs_compatibility_matrix(:disabled, :v1, :v8) do |ecs_select|
 
-      it 'merges the values if the `docinfo_target` already exist in the `_source` document' do
-        config_metadata_with_hash = %Q[
-            input {
-              elasticsearch {
-                hosts => ["localhost"]
-                query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                docinfo => true
-                docinfo_target => 'metadata_with_hash'
+      before(:each) do
+        allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+      end
+
+      context 'with docinfo enabled' do
+        let(:config_metadata) do
+          %q[
+              input {
+                elasticsearch {
+                  hosts => ["localhost"]
+                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+                  docinfo => true
+                }
               }
-            }
-        ]
-
-        event = input(config_metadata_with_hash) do |pipeline, queue|
-          queue.pop
+          ]
         end
 
-        expect(event.get("[metadata_with_hash][_index]")).to eq('logstash-2014.10.12')
-        expect(event.get("[metadata_with_hash][_type]")).to eq('logs')
-        expect(event.get("[metadata_with_hash][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
-        expect(event.get("[metadata_with_hash][awesome]")).to eq("logstash")
-      end
-
-      context 'if the `docinfo_target` exist but is not of type hash' do
-        let (:config) { {
-            "hosts" => ["localhost"],
-            "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
-            "docinfo" => true,
-            "docinfo_target" => 'metadata_with_string'
-        } }
-        it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
-          expect(client).not_to receive(:clear_scroll)
-          plugin.register
-          expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
+        it "provides document info under metadata" do
+          event = input(config_metadata) do |pipeline, queue|
+            queue.pop
+          end
+
+          if ecs_select.active_mode == :disabled
+            expect(event.get("[@metadata][_index]")).to eq('logstash-2014.10.12')
+            expect(event.get("[@metadata][_type]")).to eq('logs')
+            expect(event.get("[@metadata][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+          else
+            expect(event.get("[@metadata][input][elasticsearch][_index]")).to eq('logstash-2014.10.12')
+            expect(event.get("[@metadata][input][elasticsearch][_type]")).to eq('logs')
+            expect(event.get("[@metadata][input][elasticsearch][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+          end
         end
-      end
 
-      it "should move the document info to the @metadata field" do
-        event = input(config_metadata) do |pipeline, queue|
-          queue.pop
+        it 'merges values if the `docinfo_target` already exist in the `_source` document' do
+          config_metadata_with_hash = %Q[
+              input {
+                elasticsearch {
+                  hosts => ["localhost"]
+                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+                  docinfo => true
+                  docinfo_target => 'metadata_with_hash'
+                }
+              }
+          ]
+
+          event = input(config_metadata_with_hash) do |pipeline, queue|
+            queue.pop
+          end
+
+          expect(event.get("[metadata_with_hash][_index]")).to eq('logstash-2014.10.12')
+          expect(event.get("[metadata_with_hash][_type]")).to eq('logs')
+          expect(event.get("[metadata_with_hash][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+          expect(event.get("[metadata_with_hash][awesome]")).to eq("logstash")
         end
 
-        expect(event.get("[@metadata][_index]")).to eq('logstash-2014.10.12')
-        expect(event.get("[@metadata][_type]")).to eq('logs')
-        expect(event.get("[@metadata][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
-      end
+        context 'if the `docinfo_target` exist but is not of type hash' do
+          let (:config) { {
+              "hosts" => ["localhost"],
+              "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
+              "docinfo" => true,
+              "docinfo_target" => 'metadata_with_string'
+          } }
+          it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
+            expect(client).not_to receive(:clear_scroll)
+            plugin.register
+            expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
+          end
+        end
 
-      it 'should move the document information to the specified field' do
-        config = %q[
-            input {
-              elasticsearch {
-                hosts => ["localhost"]
-                query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                docinfo => true
-                docinfo_target => 'meta'
+        it 'should move the document information to the specified field' do
+          config = %q[
+              input {
+                elasticsearch {
+                  hosts => ["localhost"]
+                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+                  docinfo => true
+                  docinfo_target => 'meta'
+                }
               }
-            }
-        ]
-        event = input(config) do |pipeline, queue|
-          queue.pop
+          ]
+          event = input(config) do |pipeline, queue|
+            queue.pop
+          end
+
+          expect(event.get("[meta][_index]")).to eq('logstash-2014.10.12')
+          expect(event.get("[meta][_type]")).to eq('logs')
+          expect(event.get("[meta][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
         end
 
-        expect(event.get("[meta][_index]")).to eq('logstash-2014.10.12')
-        expect(event.get("[meta][_type]")).to eq('logs')
-        expect(event.get("[meta][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
-      end
+        it "allows to specify which fields from the document info to save to metadata" do
+          fields = ["_index"]
+          config = %Q[
+              input {
+                elasticsearch {
+                  hosts => ["localhost"]
+                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+                  docinfo => true
+                  docinfo_fields => #{fields}
+                }
+              }]
+
+          event = input(config) do |pipeline, queue|
+            queue.pop
+          end
 
-      it "should allow to specify which fields from the document info to save to the @metadata field" do
-        fields = ["_index"]
-        config = %Q[
+          meta_base = event.get(ecs_select.active_mode == :disabled ? "@metadata" : "[@metadata][input][elasticsearch]")
+          expect(meta_base.keys).to eq(fields)
+        end
+
+        it 'should be able to reference metadata fields in `add_field` decorations' do
+          config = %q[
             input {
               elasticsearch {
                 hosts => ["localhost"]
                 query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
                 docinfo => true
-                docinfo_fields => #{fields}
-              }
-            }]
-
-        event = input(config) do |pipeline, queue|
-          queue.pop
-        end
-
-        expect(event.get("@metadata").keys).to eq(fields)
-        expect(event.get("[@metadata][_type]")).to eq(nil)
-        expect(event.get("[@metadata][_index]")).to eq('logstash-2014.10.12')
-        expect(event.get("[@metadata][_id]")).to eq(nil)
-      end
-
-      it 'should be able to reference metadata fields in `add_field` decorations' do
-        config = %q[
-          input {
-            elasticsearch {
-              hosts => ["localhost"]
-              query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-              docinfo => true
-              add_field => {
-                'identifier' => "foo:%{[@metadata][_type]}:%{[@metadata][_id]}"
+                add_field => {
+                  'identifier' => "foo:%{[@metadata][_type]}:%{[@metadata][_id]}"
+                }
               }
             }
-          }
-        ]
+          ]
 
-        event = input(config) do |pipeline, queue|
-          queue.pop
-        end
+          event = input(config) do |pipeline, queue|
+            queue.pop
+          end
+
+          expect(event.get('identifier')).to eq('foo:logs:C5b2xLQwTZa76jBmHIbwHQ')
+        end if ecs_select.active_mode == :disabled
 
-        expect(event.get('identifier')).to eq('foo:logs:C5b2xLQwTZa76jBmHIbwHQ')
       end
+
     end
 
     context "when not defining the docinfo" do
@@ -542,9 +594,7 @@ describe LogStash::Inputs::TestableElasticsearch do
           queue.pop
         end
 
-        expect(event.get("[@metadata][_index]")).to eq(nil)
-        expect(event.get("[@metadata][_type]")).to eq(nil)
-        expect(event.get("[@metadata][_id]")).to eq(nil)
+        expect(event.get("[@metadata]")).to be_empty
       end
     end
   end
@@ -563,22 +613,23 @@ describe LogStash::Inputs::TestableElasticsearch do
         'sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=='
       end
 
-      let(:config) { super.merge({ 'cloud_id' => valid_cloud_id }) }
+      let(:config) { super().merge({ 'cloud_id' => valid_cloud_id }) }
 
       it "should set host(s)" do
         plugin.register
         client = plugin.send(:client)
-        expect( client.transport.hosts ).to eql [{
-                                                     :scheme => "https",
-                                                     :host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
-                                                     :port => 9243,
-                                                     :path => "",
-                                                     :protocol => "https"
-                                                 }]
+
+        expect( client.transport.instance_variable_get(:@seeds) ).to eql [{
+                                                                              :scheme => "https",
+                                                                              :host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
+                                                                              :port => 9243,
+                                                                              :path => "",
+                                                                              :protocol => "https"
+                                                                          }]
       end
 
       context 'invalid' do
-        let(:config) { super.merge({ 'cloud_id' => 'invalid:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlv' }) }
+        let(:config) { super().merge({ 'cloud_id' => 'invalid:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlv' }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id.*? is invalid/
@@ -586,7 +637,7 @@ describe LogStash::Inputs::TestableElasticsearch do
       end
 
       context 'hosts also set' do
-        let(:config) { super.merge({ 'cloud_id' => valid_cloud_id, 'hosts' => [ 'localhost:9200' ] }) }
+        let(:config) { super().merge({ 'cloud_id' => valid_cloud_id, 'hosts' => [ 'localhost:9200' ] }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id and hosts/
@@ -595,18 +646,18 @@ describe LogStash::Inputs::TestableElasticsearch do
     end if LOGSTASH_VERSION > '6.0'
 
     describe "cloud.auth" do
-      let(:config) { super.merge({ 'cloud_auth' => LogStash::Util::Password.new('elastic:my-passwd-00') }) }
+      let(:config) { super().merge({ 'cloud_auth' => LogStash::Util::Password.new('elastic:my-passwd-00') }) }
 
       it "should set authorization" do
         plugin.register
         client = plugin.send(:client)
-        auth_header = client.transport.options[:transport_options][:headers][:Authorization]
+        auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
 
         expect( auth_header ).to eql "Basic #{Base64.encode64('elastic:my-passwd-00').rstrip}"
       end
 
       context 'invalid' do
-        let(:config) { super.merge({ 'cloud_auth' => 'invalid-format' }) }
+        let(:config) { super().merge({ 'cloud_auth' => 'invalid-format' }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth.*? format/
@@ -614,7 +665,7 @@ describe LogStash::Inputs::TestableElasticsearch do
       end
 
       context 'user also set' do
-        let(:config) { super.merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
+        let(:config) { super().merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
@@ -624,7 +675,7 @@ describe LogStash::Inputs::TestableElasticsearch do
 
     describe "api_key" do
       context "without ssl" do
-        let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar') }) }
+        let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar') }) }
 
         it "should fail" do
           expect { plugin.register }.to raise_error LogStash::ConfigurationError, /api_key authentication requires SSL\/TLS/
@@ -632,18 +683,18 @@ describe LogStash::Inputs::TestableElasticsearch do
       end
 
       context "with ssl" do
-        let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) }
+        let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) }
 
         it "should set authorization" do
           plugin.register
           client = plugin.send(:client)
-          auth_header = client.transport.options[:transport_options][:headers][:Authorization]
+          auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
 
           expect( auth_header ).to eql "ApiKey #{Base64.strict_encode64('foo:bar')}"
         end
 
         context 'user also set' do
-          let(:config) { super.merge({ 'api_key' => 'foo:bar', 'user' => 'another' }) }
+          let(:config) { super().merge({ 'api_key' => 'foo:bar', 'user' => 'another' }) }
 
           it "should fail" do
             expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/
@@ -653,29 +704,179 @@ describe LogStash::Inputs::TestableElasticsearch do
     end if LOGSTASH_VERSION > '6.0'
 
     describe "proxy" do
-      let(:config) { super.merge({ 'proxy' => 'http://localhost:1234' }) }
+      let(:config) { super().merge({ 'proxy' => 'http://localhost:1234' }) }
 
       it "should set proxy" do
         plugin.register
         client = plugin.send(:client)
-        proxy = client.transport.options[:transport_options][:proxy]
+        proxy = extract_transport(client).options[:transport_options][:proxy]
 
         expect( proxy ).to eql "http://localhost:1234"
       end
 
       context 'invalid' do
-        let(:config) { super.merge({ 'proxy' => '${A_MISSING_ENV_VAR:}' }) }
+        let(:config) { super().merge({ 'proxy' => '${A_MISSING_ENV_VAR:}' }) }
 
         it "should not set proxy" do
           plugin.register
           client = plugin.send(:client)
 
-          expect( client.transport.options[:transport_options] ).to_not include(:proxy)
+          expect( extract_transport(client).options[:transport_options] ).to_not include(:proxy)
         end
       end
     end
 
-    shared_examples'configurable timeout' do |config_name, manticore_transport_option|
+    class StoppableServer
+
+      attr_reader :port
+
+      def initialize()
+        queue = Queue.new
+        @first_req_waiter = java.util.concurrent.CountDownLatch.new(1)
+        @first_request = nil
+
+        @t = java.lang.Thread.new(
+          proc do
+            begin
+              @server = WEBrick::HTTPServer.new :Port => 0, :DocumentRoot => ".",
+                       :Logger => Cabin::Channel.get, # silence WEBrick logging
+                       :StartCallback => Proc.new {
+                             queue.push("started")
+                           }
+              @port = @server.config[:Port]
+              @server.mount_proc '/' do |req, res|
+                res.body = '''
+                {
+                    "name": "ce7ccfb438e8",
+                    "cluster_name": "docker-cluster",
+                    "cluster_uuid": "DyR1hN03QvuCWXRy3jtb0g",
+                    "version": {
+                        "number": "7.13.1",
+                        "build_flavor": "default",
+                        "build_type": "docker",
+                        "build_hash": "9a7758028e4ea59bcab41c12004603c5a7dd84a9",
+                        "build_date": "2021-05-28T17:40:59.346932922Z",
+                        "build_snapshot": false,
+                        "lucene_version": "8.8.2",
+                        "minimum_wire_compatibility_version": "6.8.0",
+                        "minimum_index_compatibility_version": "6.0.0-beta1"
+                    },
+                    "tagline": "You Know, for Search"
+                }
+                '''
+                res.status = 200
+                res['Content-Type'] = 'application/json'
+                @first_request = req
+                @first_req_waiter.countDown()
+              end
+
+              @server.mount_proc '/logstash_unit_test/_search' do |req, res|
+                res.body = '''
+                {
+                  "took" : 1,
+                  "timed_out" : false,
+                  "_shards" : {
+                    "total" : 1,
+                    "successful" : 1,
+                    "skipped" : 0,
+                    "failed" : 0
+                  },
+                  "hits" : {
+                    "total" : {
+                      "value" : 10000,
+                      "relation" : "gte"
+                    },
+                    "max_score" : 1.0,
+                    "hits" : [
+                      {
+                        "_index" : "test_bulk_index_2",
+                        "_type" : "_doc",
+                        "_id" : "sHe6A3wBesqF7ydicQvG",
+                        "_score" : 1.0,
+                        "_source" : {
+                          "@timestamp" : "2021-09-20T15:02:02.557Z",
+                          "message" : "{\"name\": \"Andrea\"}",
+                          "@version" : "1",
+                          "host" : "kalispera",
+                          "sequence" : 5
+                        }
+                      }
+                    ]
+                  }
+                }
+                '''
+                res.status = 200
+                res['Content-Type'] = 'application/json'
+                @first_request = req
+                @first_req_waiter.countDown()
+              end
+
+
+
+              @server.start
+            rescue => e
+              puts "Error in webserver thread #{e}"
+              # ignore
+            end
+          end
+        )
+        @t.daemon = true
+        @t.start
+        queue.pop # blocks until the server is up
+      end
+
+      def stop
+        @server.shutdown
+      end
+
+      def wait_receive_request
+        @first_req_waiter.await(2, java.util.concurrent.TimeUnit::SECONDS)
+        @first_request
+      end
+    end
+
+    describe "'user-agent' header" do
+      let!(:webserver) { StoppableServer.new } # webserver must be started before the call, so no lazy "let"
+
+      after :each do
+        webserver.stop
+      end
+
+      it "server should be started" do
+        require 'net/http'
+        response = nil
+        Net::HTTP.start('localhost', webserver.port) {|http|
+          response = http.request_get('/')
+        }
+        expect(response.code.to_i).to eq(200)
+      end
+
+      context "used by plugin" do
+        let(:config) do
+          {
+            "hosts" => ["localhost:#{webserver.port}"],
+            "query" => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }',
+            "index" => "logstash_unit_test"
+          }
+        end
+        let(:plugin) { described_class.new(config) }
+        let(:event)  { LogStash::Event.new({}) }
+
+        it "client should sent the expect user-agent" do
+          plugin.register
+
+          queue = []
+          plugin.run(queue)
+
+          request = webserver.wait_receive_request
+
+          expect(request.header['user-agent'].size).to eq(1)
+          expect(request.header['user-agent'][0]).to match(/logstash\/\d*\.\d*\.\d* \(OS=.*; JVM=.*\) logstash-input-elasticsearch\/\d*\.\d*\.\d*/)
+        end
+      end
+    end
+
+    shared_examples 'configurable timeout' do |config_name, manticore_transport_option|
       let(:config_value) { fail NotImplementedError }
       let(:config) { super().merge(config_name => config_value) }
       {
@@ -706,6 +907,9 @@ describe LogStash::Inputs::TestableElasticsearch do
             transport_options = new_elasticsearch_client_params[:transport_options]
             expect(transport_options).to include(manticore_transport_option)
             expect(transport_options[manticore_transport_option]).to eq(config_value.to_i)
+            mock_client = double("fake_client")
+            allow(mock_client).to receive(:ping)
+            mock_client
           end
 
           plugin.register
@@ -729,7 +933,7 @@ describe LogStash::Inputs::TestableElasticsearch do
       {
         "hosts" => ["localhost"],
         "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
-        "schedule" => "* * * * * UTC"
+        "schedule" => "* * * * * * UTC" # every second
       }
     end
 
@@ -738,22 +942,21 @@ describe LogStash::Inputs::TestableElasticsearch do
     end
 
     it "should properly schedule" do
-      Timecop.travel(Time.new(2000))
-      Timecop.scale(60)
-      runner = Thread.new do
-        expect(plugin).to receive(:do_run) {
-          queue << LogStash::Event.new({})
-        }.at_least(:twice)
-
-        plugin.run(queue)
-      end
-      sleep 3
+      expect(plugin).to receive(:do_run) {
+        queue << LogStash::Event.new({})
+      }.at_least(:twice)
+      runner = Thread.start { plugin.run(queue) }
+      sleep 3.0
       plugin.stop
-      runner.kill
       runner.join
-      expect(queue.size).to eq(2)
-      Timecop.return
+      expect(queue.size).to be >= 2
     end
 
   end
+
+  # @note can be removed once we depends on elasticsearch gem >= 6.x
+  def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
+    client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
+  end
+
 end