logstash-input-elasticsearch 4.9.3 → 4.12.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc71917516d5297b151b09c3b26324272ee472fa8ae987fad6b30fa99492d382
-  data.tar.gz: cd72359f663375ad910dbf02a70d151160e449311bff81c0f77d062c7625c597
+  metadata.gz: e2427b28640265b075a0e21240cf410b2e2252d7516ac7bd0955d48087317f7f
+  data.tar.gz: dbc7d84f18348e7fa2292d2b68a5db59a5ffda724eb090ff17503e10e3ff130d
 SHA512:
-  metadata.gz: ea7bca9aa226958479061efab4172224039d10a93579ef482418ad6955c493c00a45b1372ce6bd61aaba07e6c89bc2e96d1236cfd4be2e326193b52805ca448f
-  data.tar.gz: 0f08585317a26f8a647be73e4e70b9a421025f1c81bcfa8518c1188cb60a0efea49f131a1e80645bd920c0806bb8c54542da07ab1cab44b540313a062b8b351d
+  metadata.gz: dd3f9693c355505fbe5a971a46899ba285e057406db5807d4226b4ace61449f41f20409926d49a5a69c19338020069de13e4a4da7028db9b8f7db6d3ce4e0e6c
+  data.tar.gz: 0c170d69801feac7d0df3a79ef4351a6237fa30d5ad3b69e0c8af660981f40a1d814e17d60cebd0e555cdee6613de93c7e407d2cba28f0092f85f5da7446b966

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 4.12.1
+  - Fixed too_long_frame_exception by passing scroll_id in the body [#159](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/159)
+
+## 4.12.0
+  - Feat: Update Elasticsearch client to 7.14.0 [#157](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/157)
+
+## 4.11.0
+  - Feat: add user-agent header passed to the Elasticsearch HTTP connection [#158](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/158)
+
+## 4.10.0
+  - Feat: added ecs_compatibility + event_factory support [#149](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/149)
+
 ## 4.9.3
   - Fixed SSL handshake hang indefinitely with proxy setup [#156](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/156)
 
@@ -6,6 +18,7 @@
     header isn't passed, this leads to the plugin not being able to leverage `user`/`password` credentials set by the user.
     [#153](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/153)
 
+
 ## 4.9.1
   - [DOC] Replaced hard-coded links with shared attributes [#143](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/143)
   - [DOC] Added missing quote to docinfo_fields example [#145](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/145)

data/docs/index.asciidoc

@@ -83,8 +83,18 @@ Authentication to a secure Elasticsearch cluster is possible using _one_ of the
 Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level.
 The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks.
 
+[id="plugins-{type}s-{plugin}-ecs"]
+==== Compatibility with the Elastic Common Schema (ECS)
+
+When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin
+uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information.
+
+The plugin logs a warning when ECS is enabled and `target` isn't set.
+
+TIP: Set the `target` option to avoid potential schema conflicts.
+
 [id="plugins-{type}s-{plugin}-options"]
-==== Elasticsearch Input Configuration Options
+==== Elasticsearch Input configuration options
 
 This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
 
@@ -99,6 +109,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-docinfo>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-docinfo_fields>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-docinfo_target>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-hosts>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-index>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
@@ -197,13 +208,14 @@ Example
         size => 500
         scroll => "5m"
         docinfo => true
+        docinfo_target => "[@metadata][doc]"
       }
     }
     output {
       elasticsearch {
-        index => "copy-of-production.%{[@metadata][_index]}"
-        document_type => "%{[@metadata][_type]}"
-        document_id => "%{[@metadata][_id]}"
+        index => "copy-of-production.%{[@metadata][doc][_index]}"
+        document_type => "%{[@metadata][doc][_type]}"
+        document_id => "%{[@metadata][doc][_id]}"
       }
     }
 
@@ -214,8 +226,9 @@ Example
     input {
       elasticsearch {
         docinfo => true
+        docinfo_target => "[@metadata][doc]"
         add_field => {
-          identifier => "%{[@metadata][_index]}:%{[@metadata][_type]}:%{[@metadata][_id]}"
+          identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}"
         }
       }
     }
@@ -236,11 +249,25 @@ more information.
 ===== `docinfo_target` 
 
   * Value type is <<string,string>>
-  * Default value is `"@metadata"`
+  * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
+    ** ECS Compatibility disabled: `"@metadata"`
+    ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"`
+
+If document metadata storage is requested by enabling the `docinfo` option,
+this option names the field under which to store the metadata fields as subfields.
+
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+  * Value type is <<string,string>>
+  * Supported values are:
+    ** `disabled`: CSV data added at root level
+    ** `v1`,`v8`: Elastic Common Schema compliant behavior
+  * Default value depends on which version of Logstash is running:
+    ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
+    ** Otherwise, the default value is `disabled`
 
-If document metadata storage is requested by enabling the `docinfo`
-option, this option names the field under which to store the metadata
-fields as subfields.
+Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
 
 [id="plugins-{type}s-{plugin}-hosts"]
 ===== `hosts` 
@@ -402,4 +429,4 @@ empty string authentication will be disabled.
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
 
-:default_codec!:
+:no_codec!:

data/lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb

@@ -20,7 +20,17 @@ if es_client_version >= Gem::Version.new('7.2') && es_client_version < Gem::Vers
             def apply_headers(request_options, options)
               headers = (options && options[:headers]) || {}
               headers[CONTENT_TYPE_STR] = find_value(headers, CONTENT_TYPE_REGEX) || DEFAULT_CONTENT_TYPE
-              headers[USER_AGENT_STR] = find_value(headers, USER_AGENT_REGEX) || user_agent_header
+
+              # this code is necessary to grab the correct user-agent header
+              # when this method is invoked with apply_headers(@request_options, options)
+              # from https://github.com/elastic/elasticsearch-ruby/blob/v7.14.0/elasticsearch-transport/lib/elasticsearch/transport/transport/http/manticore.rb#L113-L114
+              transport_user_agent = nil
+              if (options && options[:transport_options] && options[:transport_options][:headers])
+                transport_headers = options[:transport_options][:headers]
+                transport_user_agent = find_value(transport_headers, USER_AGENT_REGEX)
+              end
+
+              headers[USER_AGENT_STR] = transport_user_agent || find_value(headers, USER_AGENT_REGEX) || user_agent_header
               headers[ACCEPT_ENCODING] = GZIP if use_compression?
               (request_options[:headers] ||= {}).merge!(headers) # this line was changed
             end

data/lib/logstash/inputs/elasticsearch.rb

@@ -4,6 +4,9 @@ require "logstash/namespace"
 require "logstash/json"
 require "logstash/util/safe_uri"
 require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
+require 'logstash/plugin_mixins/event_support/event_factory_adapter'
+require 'logstash/plugin_mixins/ecs_compatibility_support'
+require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
 require "base64"
 
 require "elasticsearch"
@@ -66,12 +69,16 @@ require_relative "elasticsearch/patches/_elasticsearch_transport_connections_sel
 #
 #
 class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
+
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
+  include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
+
+  include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
+
   extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
 
   config_name "elasticsearch"
 
-  default :codec, "json"
-
   # List of elasticsearch hosts to use for querying.
   # Each host can be either IP, HOST, IP:port or HOST:port.
   # Port defaults to 9200
@@ -128,8 +135,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   #
   config :docinfo, :validate => :boolean, :default => false
 
-  # Where to move the Elasticsearch document information. By default we use the @metadata field.
-  config :docinfo_target, :validate=> :string, :default => LogStash::Event::METADATA
+  # Where to move the Elasticsearch document information.
+  # default: [@metadata][input][elasticsearch] in ECS mode, @metadata field otherwise
+  config :docinfo_target, :validate=> :field_reference
 
   # List of document metadata to move to the `docinfo_target` field.
   # To learn more about Elasticsearch metadata fields read
@@ -184,6 +192,14 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # If set, the _source of each hit will be added nested under the target instead of at the top-level
   config :target, :validate => :field_reference
 
+  def initialize(params={})
+    super(params)
+
+    if docinfo_target.nil?
+      @docinfo_target = ecs_select[disabled: '@metadata', v1: '[@metadata][input][elasticsearch]']
+    end
+  end
+
   def register
     require "rufus/scheduler"
 
@@ -206,6 +222,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     transport_options = {:headers => {}}
     transport_options[:headers].merge!(setup_basic_auth(user, password))
     transport_options[:headers].merge!(setup_api_key(api_key))
+    transport_options[:headers].merge!({'user-agent' => prepare_user_agent()})
     transport_options[:request_timeout] = @request_timeout_seconds unless @request_timeout_seconds.nil?
     transport_options[:connect_timeout] = @connect_timeout_seconds unless @connect_timeout_seconds.nil?
     transport_options[:socket_timeout]  = @socket_timeout_seconds  unless @socket_timeout_seconds.nil?
@@ -223,6 +240,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
       :transport_class => ::Elasticsearch::Transport::Transport::HTTP::Manticore,
       :ssl => ssl_options
     )
+    test_connection!
+    @client
   end
 
 
@@ -297,47 +316,41 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     [r['hits']['hits'].any?, r['_scroll_id']]
   rescue => e
     # this will typically be triggered by a scroll timeout
-    logger.error("Scroll request error, aborting scroll", error: e.inspect)
+    logger.error("Scroll request error, aborting scroll", message: e.message, exception: e.class)
     # return no hits and original scroll_id so we can try to clear it
     [false, scroll_id]
   end
 
   def push_hit(hit, output_queue)
-    if @target.nil?
-      event = LogStash::Event.new(hit['_source'])
-    else
-      event = LogStash::Event.new
-      event.set(@target, hit['_source'])
-    end
-
-    if @docinfo
-      # do not assume event[@docinfo_target] to be in-place updatable. first get it, update it, then at the end set it in the event.
-      docinfo_target = event.get(@docinfo_target) || {}
-
-      unless docinfo_target.is_a?(Hash)
-        @logger.error("Elasticsearch Input: Incompatible Event, incompatible type for the docinfo_target=#{@docinfo_target} field in the `_source` document, expected a hash got:", :docinfo_target_type => docinfo_target.class, :event => event)
+    event = targeted_event_factory.new_event hit['_source']
+    set_docinfo_fields(hit, event) if @docinfo
+    decorate(event)
+    output_queue << event
+  end
 
-        # TODO: (colin) I am not sure raising is a good strategy here?
-        raise Exception.new("Elasticsearch input: incompatible event")
-      end
+  def set_docinfo_fields(hit, event)
+    # do not assume event[@docinfo_target] to be in-place updatable. first get it, update it, then at the end set it in the event.
+    docinfo_target = event.get(@docinfo_target) || {}
 
-      @docinfo_fields.each do |field|
-        docinfo_target[field] = hit[field]
-      end
+    unless docinfo_target.is_a?(Hash)
+      @logger.error("Incompatible Event, incompatible type for the docinfo_target=#{@docinfo_target} field in the `_source` document, expected a hash got:", :docinfo_target_type => docinfo_target.class, :event => event.to_hash_with_metadata)
 
-      event.set(@docinfo_target, docinfo_target)
+      # TODO: (colin) I am not sure raising is a good strategy here?
+      raise Exception.new("Elasticsearch input: incompatible event")
     end
 
-    decorate(event)
+    @docinfo_fields.each do |field|
+      docinfo_target[field] = hit[field]
+    end
 
-    output_queue << event
+    event.set(@docinfo_target, docinfo_target)
   end
 
   def clear_scroll(scroll_id)
-    @client.clear_scroll(scroll_id: scroll_id) if scroll_id
+    @client.clear_scroll(:body => { :scroll_id => scroll_id }) if scroll_id
   rescue => e
     # ignore & log any clear_scroll errors
-    logger.warn("Ignoring clear_scroll exception", message: e.message)
+    logger.warn("Ignoring clear_scroll exception", message: e.message, exception: e.class)
   end
 
   def scroll_request scroll_id
@@ -397,6 +410,18 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     { 'Authorization' => "ApiKey #{token}" }
   end
 
+  def prepare_user_agent
+      os_name = java.lang.System.getProperty('os.name')
+      os_version = java.lang.System.getProperty('os.version')
+      os_arch = java.lang.System.getProperty('os.arch')
+      jvm_vendor = java.lang.System.getProperty('java.vendor')
+      jvm_version = java.lang.System.getProperty('java.version')
+
+      plugin_version = Gem.loaded_specs["logstash-input-elasticsearch"].version
+      # example: logstash/7.14.1 (OS=Linux-5.4.0-84-generic-amd64; JVM=AdoptOpenJDK-11.0.11) logstash-input-elasticsearch/4.10.0
+      "logstash/#{LOGSTASH_VERSION} (OS=#{os_name}-#{os_version}-#{os_arch}; JVM=#{jvm_vendor}-#{jvm_version}) logstash-#{@plugin_type}-#{config_name}/#{plugin_version}"
+  end
+
   def fill_user_password_from_cloud_auth
     return unless @cloud_auth
 
@@ -447,6 +472,15 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     [ cloud_auth.username, cloud_auth.password ]
   end
 
+  # @private used by unit specs
+  attr_reader :client
+
+  def test_connection!
+    @client.ping
+  rescue Elasticsearch::UnsupportedProductError
+    raise LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch"
+  end
+
   module URIOrEmptyValidator
     ##
     # @override to provide :uri_or_empty validator

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.9.3'
+  s.version         = '4.12.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,18 +20,22 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
+  s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0'
+  s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
 
-  s.add_runtime_dependency 'elasticsearch', '>= 5.0.5' # LS >= 6.7 and < 7.14 all used version 5.0.5
+  s.add_runtime_dependency 'elasticsearch', '>= 7.14.0' # LS >= 6.7 and < 7.14 all used version 5.0.5
 
-  s.add_runtime_dependency 'logstash-codec-json'
-  s.add_runtime_dependency 'sequel'
   s.add_runtime_dependency 'tzinfo'
   s.add_runtime_dependency 'tzinfo-data'
   s.add_runtime_dependency 'rufus-scheduler'
   s.add_runtime_dependency 'manticore', ">= 0.7.1"
-  s.add_development_dependency 'faraday', "~> 0.15.4"
+
+  s.add_development_dependency 'logstash-codec-plain'
+  s.add_development_dependency 'faraday', "~> 1"
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'timecop'
+  s.add_development_dependency 'cabin', ['~> 0.6']
+  s.add_development_dependency 'webrick'
 end

data/spec/inputs/elasticsearch_spec.rb

@@ -7,16 +7,45 @@ require "timecop"
 require "stud/temporary"
 require "time"
 require "date"
+require "cabin"
+require "webrick"
+require "uri"
 
-class LogStash::Inputs::TestableElasticsearch < LogStash::Inputs::Elasticsearch
-  attr_reader :client
-end
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 
-describe LogStash::Inputs::TestableElasticsearch do
+describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
 
-  let(:plugin) { LogStash::Inputs::TestableElasticsearch.new(config) }
+  let(:plugin) { described_class.new(config) }
   let(:queue) { Queue.new }
 
+  before(:each) do
+     Elasticsearch::Client.send(:define_method, :ping) { } # define no-action ping method
+  end
+
+  context "register" do
+    let(:config) do
+      {
+        "schedule" => "* * * * * UTC"
+      }
+    end
+
+    context "against authentic Elasticsearch" do
+      it "should not raise an exception" do
+       expect { plugin.register }.to_not raise_error
+     end
+    end
+
+    context "against not authentic Elasticsearch" do
+      before(:each) do
+         Elasticsearch::Client.send(:define_method, :ping) { raise Elasticsearch::UnsupportedProductError.new("Fake error") } # define error ping method
+      end
+
+      it "should raise ConfigurationError" do
+        expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+  end
+
   it_behaves_like "an interruptible input plugin" do
     let(:esclient) { double("elasticsearch-client") }
     let(:config) do
@@ -37,10 +66,17 @@ describe LogStash::Inputs::TestableElasticsearch do
       allow(esclient).to receive(:search) { { "hits" => { "hits" => [hit] } } }
       allow(esclient).to receive(:scroll) { { "hits" => { "hits" => [hit] } } }
       allow(esclient).to receive(:clear_scroll).and_return(nil)
+      allow(esclient).to receive(:ping)
     end
   end
 
-  context 'creating events from Elasticsearch' do
+
+  ecs_compatibility_matrix(:disabled, :v1, :v8) do |ecs_select|
+
+    before(:each) do
+      allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+    end
+
     let(:config) do
       %q[
         input {
@@ -89,6 +125,7 @@ describe LogStash::Inputs::TestableElasticsearch do
       expect(client).to receive(:search).with(any_args).and_return(mock_response)
       expect(client).to receive(:scroll).with({ :body => { :scroll_id => "cXVlcnlUaGVuRmV0Y2g" }, :scroll=> "1m" }).and_return(mock_scroll_response)
       expect(client).to receive(:clear_scroll).and_return(nil)
+      expect(client).to receive(:ping)
     end
 
     it 'creates the events from the hits' do
@@ -97,7 +134,6 @@ describe LogStash::Inputs::TestableElasticsearch do
       end
 
       expect(event).to be_a(LogStash::Event)
-      puts event.to_hash_with_metadata
       expect(event.get("message")).to eql [ "ohayo" ]
     end
 
@@ -120,10 +156,10 @@ describe LogStash::Inputs::TestableElasticsearch do
         end
 
         expect(event).to be_a(LogStash::Event)
-        puts event.to_hash_with_metadata
         expect(event.get("[@metadata][_source][message]")).to eql [ "ohayo" ]
       end
     end
+
   end
 
   # This spec is an adapter-spec, ensuring that we send the right sequence of messages to our Elasticsearch Client
@@ -135,6 +171,7 @@ describe LogStash::Inputs::TestableElasticsearch do
           'query' => "#{LogStash::Json.dump(query)}",
           'slices' => slices,
           'docinfo' => true, # include ids
+          'docinfo_target' => '[@metadata]'
       }
     end
     let(:query) do
@@ -304,6 +341,7 @@ describe LogStash::Inputs::TestableElasticsearch do
         expect(client).to receive(:search).with(hash_including(:body => slice0_query)).and_return(slice0_response0)
         expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll1 })).and_return(slice0_response1)
         expect(client).to receive(:scroll).with(hash_including(:body => { :scroll_id => slice0_scroll2 })).and_return(slice0_response2)
+        allow(client).to receive(:ping)
 
         # SLICE1 is a two-page scroll in which the last page has no next scroll id
         slice1_query = LogStash::Json.dump(query.merge('slice' => { 'id' => 1, 'max' => 2}))
@@ -403,129 +441,143 @@ describe LogStash::Inputs::TestableElasticsearch do
       expect(client).to receive(:search).with(any_args).and_return(response)
       allow(client).to receive(:scroll).with({ :body => {:scroll_id => "cXVlcnlUaGVuRmV0Y2g"}, :scroll => "1m" }).and_return(scroll_reponse)
       allow(client).to receive(:clear_scroll).and_return(nil)
+      allow(client).to receive(:ping).and_return(nil)
     end
 
-    context 'when defining docinfo' do
-      let(:config_metadata) do
-        %q[
-            input {
-              elasticsearch {
-                hosts => ["localhost"]
-                query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                docinfo => true
-              }
-            }
-        ]
-      end
+    ecs_compatibility_matrix(:disabled, :v1, :v8) do |ecs_select|
 
-      it 'merges the values if the `docinfo_target` already exist in the `_source` document' do
-        config_metadata_with_hash = %Q[
-            input {
-              elasticsearch {
-                hosts => ["localhost"]
-                query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                docinfo => true
-                docinfo_target => 'metadata_with_hash'
+      before(:each) do
+        allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+      end
+
+      context 'with docinfo enabled' do
+        let(:config_metadata) do
+          %q[
+              input {
+                elasticsearch {
+                  hosts => ["localhost"]
+                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+                  docinfo => true
+                }
               }
-            }
-        ]
-
-        event = input(config_metadata_with_hash) do |pipeline, queue|
-          queue.pop
+          ]
         end
 
-        expect(event.get("[metadata_with_hash][_index]")).to eq('logstash-2014.10.12')
-        expect(event.get("[metadata_with_hash][_type]")).to eq('logs')
-        expect(event.get("[metadata_with_hash][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
-        expect(event.get("[metadata_with_hash][awesome]")).to eq("logstash")
-      end
-
-      context 'if the `docinfo_target` exist but is not of type hash' do
-        let (:config) { {
-            "hosts" => ["localhost"],
-            "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
-            "docinfo" => true,
-            "docinfo_target" => 'metadata_with_string'
-        } }
-        it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
-          expect(client).not_to receive(:clear_scroll)
-          plugin.register
-          expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
+        it "provides document info under metadata" do
+          event = input(config_metadata) do |pipeline, queue|
+            queue.pop
+          end
+
+          if ecs_select.active_mode == :disabled
+            expect(event.get("[@metadata][_index]")).to eq('logstash-2014.10.12')
+            expect(event.get("[@metadata][_type]")).to eq('logs')
+            expect(event.get("[@metadata][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+          else
+            expect(event.get("[@metadata][input][elasticsearch][_index]")).to eq('logstash-2014.10.12')
+            expect(event.get("[@metadata][input][elasticsearch][_type]")).to eq('logs')
+            expect(event.get("[@metadata][input][elasticsearch][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+          end
         end
-      end
 
-      it "should move the document info to the @metadata field" do
-        event = input(config_metadata) do |pipeline, queue|
-          queue.pop
+        it 'merges values if the `docinfo_target` already exist in the `_source` document' do
+          config_metadata_with_hash = %Q[
+              input {
+                elasticsearch {
+                  hosts => ["localhost"]
+                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+                  docinfo => true
+                  docinfo_target => 'metadata_with_hash'
+                }
+              }
+          ]
+
+          event = input(config_metadata_with_hash) do |pipeline, queue|
+            queue.pop
+          end
+
+          expect(event.get("[metadata_with_hash][_index]")).to eq('logstash-2014.10.12')
+          expect(event.get("[metadata_with_hash][_type]")).to eq('logs')
+          expect(event.get("[metadata_with_hash][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+          expect(event.get("[metadata_with_hash][awesome]")).to eq("logstash")
         end
 
-        expect(event.get("[@metadata][_index]")).to eq('logstash-2014.10.12')
-        expect(event.get("[@metadata][_type]")).to eq('logs')
-        expect(event.get("[@metadata][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
-      end
+        context 'if the `docinfo_target` exist but is not of type hash' do
+          let (:config) { {
+              "hosts" => ["localhost"],
+              "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
+              "docinfo" => true,
+              "docinfo_target" => 'metadata_with_string'
+          } }
+          it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
+            expect(client).not_to receive(:clear_scroll)
+            plugin.register
+            expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
+          end
+        end
 
-      it 'should move the document information to the specified field' do
-        config = %q[
-            input {
-              elasticsearch {
-                hosts => ["localhost"]
-                query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                docinfo => true
-                docinfo_target => 'meta'
+        it 'should move the document information to the specified field' do
+          config = %q[
+              input {
+                elasticsearch {
+                  hosts => ["localhost"]
+                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+                  docinfo => true
+                  docinfo_target => 'meta'
+                }
               }
-            }
-        ]
-        event = input(config) do |pipeline, queue|
-          queue.pop
+          ]
+          event = input(config) do |pipeline, queue|
+            queue.pop
+          end
+
+          expect(event.get("[meta][_index]")).to eq('logstash-2014.10.12')
+          expect(event.get("[meta][_type]")).to eq('logs')
+          expect(event.get("[meta][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
         end
 
-        expect(event.get("[meta][_index]")).to eq('logstash-2014.10.12')
-        expect(event.get("[meta][_type]")).to eq('logs')
-        expect(event.get("[meta][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
-      end
+        it "allows to specify which fields from the document info to save to metadata" do
+          fields = ["_index"]
+          config = %Q[
+              input {
+                elasticsearch {
+                  hosts => ["localhost"]
+                  query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+                  docinfo => true
+                  docinfo_fields => #{fields}
+                }
+              }]
+
+          event = input(config) do |pipeline, queue|
+            queue.pop
+          end
+
+          meta_base = event.get(ecs_select.active_mode == :disabled ? "@metadata" : "[@metadata][input][elasticsearch]")
+          expect(meta_base.keys).to eq(fields)
+        end
 
-      it "should allow to specify which fields from the document info to save to the @metadata field" do
-        fields = ["_index"]
-        config = %Q[
+        it 'should be able to reference metadata fields in `add_field` decorations' do
+          config = %q[
             input {
               elasticsearch {
                 hosts => ["localhost"]
                 query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
                 docinfo => true
-                docinfo_fields => #{fields}
-              }
-            }]
-
-        event = input(config) do |pipeline, queue|
-          queue.pop
-        end
-
-        expect(event.get("@metadata").keys).to eq(fields)
-        expect(event.get("[@metadata][_type]")).to eq(nil)
-        expect(event.get("[@metadata][_index]")).to eq('logstash-2014.10.12')
-        expect(event.get("[@metadata][_id]")).to eq(nil)
-      end
-
-      it 'should be able to reference metadata fields in `add_field` decorations' do
-        config = %q[
-          input {
-            elasticsearch {
-              hosts => ["localhost"]
-              query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-              docinfo => true
-              add_field => {
-                'identifier' => "foo:%{[@metadata][_type]}:%{[@metadata][_id]}"
+                add_field => {
+                  'identifier' => "foo:%{[@metadata][_type]}:%{[@metadata][_id]}"
+                }
               }
             }
-          }
-        ]
+          ]
 
-        event = input(config) do |pipeline, queue|
-          queue.pop
-        end
+          event = input(config) do |pipeline, queue|
+            queue.pop
+          end
+
+          expect(event.get('identifier')).to eq('foo:logs:C5b2xLQwTZa76jBmHIbwHQ')
+        end if ecs_select.active_mode == :disabled
 
-        expect(event.get('identifier')).to eq('foo:logs:C5b2xLQwTZa76jBmHIbwHQ')
       end
+
     end
 
     context "when not defining the docinfo" do
@@ -542,9 +594,7 @@ describe LogStash::Inputs::TestableElasticsearch do
           queue.pop
         end
 
-        expect(event.get("[@metadata][_index]")).to eq(nil)
-        expect(event.get("[@metadata][_type]")).to eq(nil)
-        expect(event.get("[@metadata][_id]")).to eq(nil)
+        expect(event.get("[@metadata]")).to be_empty
       end
     end
   end
@@ -568,7 +618,8 @@ describe LogStash::Inputs::TestableElasticsearch do
       it "should set host(s)" do
         plugin.register
         client = plugin.send(:client)
-        expect( client.transport.instance_variable_get(:@hosts) ).to eql [{
+
+        expect( client.transport.instance_variable_get(:@seeds) ).to eql [{
                                                                               :scheme => "https",
                                                                               :host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
                                                                               :port => 9243,
@@ -600,7 +651,7 @@ describe LogStash::Inputs::TestableElasticsearch do
       it "should set authorization" do
         plugin.register
         client = plugin.send(:client)
-        auth_header = client.transport.instance_variable_get(:@options)[:transport_options][:headers]['Authorization']
+        auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
 
         expect( auth_header ).to eql "Basic #{Base64.encode64('elastic:my-passwd-00').rstrip}"
       end
@@ -637,7 +688,7 @@ describe LogStash::Inputs::TestableElasticsearch do
         it "should set authorization" do
           plugin.register
           client = plugin.send(:client)
-          auth_header = client.transport.instance_variable_get(:@options)[:transport_options][:headers]['Authorization']
+          auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
 
           expect( auth_header ).to eql "ApiKey #{Base64.strict_encode64('foo:bar')}"
         end
@@ -658,7 +709,7 @@ describe LogStash::Inputs::TestableElasticsearch do
       it "should set proxy" do
         plugin.register
         client = plugin.send(:client)
-        proxy = client.transport.instance_variable_get(:@options)[:transport_options][:proxy]
+        proxy = extract_transport(client).options[:transport_options][:proxy]
 
         expect( proxy ).to eql "http://localhost:1234"
       end
@@ -670,12 +721,162 @@ describe LogStash::Inputs::TestableElasticsearch do
           plugin.register
           client = plugin.send(:client)
 
-          expect( client.transport.instance_variable_get(:@options)[:transport_options] ).to_not include(:proxy)
+          expect( extract_transport(client).options[:transport_options] ).to_not include(:proxy)
         end
       end
     end
 
-    shared_examples'configurable timeout' do |config_name, manticore_transport_option|
+    class StoppableServer
+
+      attr_reader :port
+
+      def initialize()
+        queue = Queue.new
+        @first_req_waiter = java.util.concurrent.CountDownLatch.new(1)
+        @first_request = nil
+
+        @t = java.lang.Thread.new(
+          proc do
+            begin
+              @server = WEBrick::HTTPServer.new :Port => 0, :DocumentRoot => ".",
+                       :Logger => Cabin::Channel.get, # silence WEBrick logging
+                       :StartCallback => Proc.new {
+                             queue.push("started")
+                           }
+              @port = @server.config[:Port]
+              @server.mount_proc '/' do |req, res|
+                res.body = '''
+                {
+                    "name": "ce7ccfb438e8",
+                    "cluster_name": "docker-cluster",
+                    "cluster_uuid": "DyR1hN03QvuCWXRy3jtb0g",
+                    "version": {
+                        "number": "7.13.1",
+                        "build_flavor": "default",
+                        "build_type": "docker",
+                        "build_hash": "9a7758028e4ea59bcab41c12004603c5a7dd84a9",
+                        "build_date": "2021-05-28T17:40:59.346932922Z",
+                        "build_snapshot": false,
+                        "lucene_version": "8.8.2",
+                        "minimum_wire_compatibility_version": "6.8.0",
+                        "minimum_index_compatibility_version": "6.0.0-beta1"
+                    },
+                    "tagline": "You Know, for Search"
+                }
+                '''
+                res.status = 200
+                res['Content-Type'] = 'application/json'
+                @first_request = req
+                @first_req_waiter.countDown()
+              end
+
+              @server.mount_proc '/logstash_unit_test/_search' do |req, res|
+                res.body = '''
+                {
+                  "took" : 1,
+                  "timed_out" : false,
+                  "_shards" : {
+                    "total" : 1,
+                    "successful" : 1,
+                    "skipped" : 0,
+                    "failed" : 0
+                  },
+                  "hits" : {
+                    "total" : {
+                      "value" : 10000,
+                      "relation" : "gte"
+                    },
+                    "max_score" : 1.0,
+                    "hits" : [
+                      {
+                        "_index" : "test_bulk_index_2",
+                        "_type" : "_doc",
+                        "_id" : "sHe6A3wBesqF7ydicQvG",
+                        "_score" : 1.0,
+                        "_source" : {
+                          "@timestamp" : "2021-09-20T15:02:02.557Z",
+                          "message" : "{\"name\": \"Andrea\"}",
+                          "@version" : "1",
+                          "host" : "kalispera",
+                          "sequence" : 5
+                        }
+                      }
+                    ]
+                  }
+                }
+                '''
+                res.status = 200
+                res['Content-Type'] = 'application/json'
+                @first_request = req
+                @first_req_waiter.countDown()
+              end
+
+
+
+              @server.start
+            rescue => e
+              puts "Error in webserver thread #{e}"
+              # ignore
+            end
+          end
+        )
+        @t.daemon = true
+        @t.start
+        queue.pop # blocks until the server is up
+      end
+
+      def stop
+        @server.shutdown
+      end
+
+      def wait_receive_request
+        @first_req_waiter.await(2, java.util.concurrent.TimeUnit::SECONDS)
+        @first_request
+      end
+    end
+
+    describe "'user-agent' header" do
+      let!(:webserver) { StoppableServer.new } # webserver must be started before the call, so no lazy "let"
+
+      after :each do
+        webserver.stop
+      end
+
+      it "server should be started" do
+        require 'net/http'
+        response = nil
+        Net::HTTP.start('localhost', webserver.port) {|http|
+          response = http.request_get('/')
+        }
+        expect(response.code.to_i).to eq(200)
+      end
+
+      context "used by plugin" do
+        let(:config) do
+          {
+            "hosts" => ["localhost:#{webserver.port}"],
+            "query" => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }',
+            "index" => "logstash_unit_test"
+          }
+        end
+        let(:plugin) { described_class.new(config) }
+        let(:event)  { LogStash::Event.new({}) }
+
+        it "client should sent the expect user-agent" do
+          plugin.register
+
+          queue = []
+          plugin.run(queue)
+
+          request = webserver.wait_receive_request
+
+          expect(request.header['user-agent'].size).to eq(1)
+          expect(request.header['user-agent'][0]).to match(/logstash\/\d*\.\d*\.\d* \(OS=.*; JVM=.*\) logstash-input-elasticsearch\/\d*\.\d*\.\d*/)
+        end
+      end
+    end
+
+    shared_examples 'configurable timeout' do |config_name, manticore_transport_option|
       let(:config_value) { fail NotImplementedError }
       let(:config) { super().merge(config_name => config_value) }
       {
@@ -706,6 +907,9 @@ describe LogStash::Inputs::TestableElasticsearch do
             transport_options = new_elasticsearch_client_params[:transport_options]
             expect(transport_options).to include(manticore_transport_option)
             expect(transport_options[manticore_transport_option]).to eq(config_value.to_i)
+            mock_client = double("fake_client")
+            allow(mock_client).to receive(:ping)
+            mock_client
           end
 
           plugin.register
@@ -756,4 +960,10 @@ describe LogStash::Inputs::TestableElasticsearch do
     end
 
   end
+
+  # @note can be removed once we depends on elasticsearch gem >= 6.x
+  def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
+    client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
+  end
+
 end

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -24,7 +24,6 @@ describe LogStash::Inputs::Elasticsearch do
       ESHelper.index_doc(@es, :index => 'logs', :body => { :response => 404, :message=> 'Not Found'})
     end
     @es.indices.refresh
-    plugin.register
   end
 
   after(:each) do
@@ -33,6 +32,10 @@ describe LogStash::Inputs::Elasticsearch do
   end
 
   shared_examples 'an elasticsearch index plugin' do
+    before(:each) do
+      plugin.register
+    end
+
     it 'should retrieve json event from elasticsearch' do
       queue = []
       plugin.run(queue)
@@ -43,6 +46,10 @@ describe LogStash::Inputs::Elasticsearch do
   end
 
   describe 'against an unsecured elasticsearch', :integration => true do
+    before(:each) do
+      plugin.register
+    end
+
     it_behaves_like 'an elasticsearch index plugin'
   end
 
@@ -66,8 +73,7 @@ describe LogStash::Inputs::Elasticsearch do
       let(:queue) { [] }
 
       it "fails to run the plugin" do
-        plugin.register
-        expect { plugin.run queue }.to raise_error Elasticsearch::Transport::Transport::Errors::Unauthorized
+        expect { plugin.register }.to raise_error Elasticsearch::Transport::Transport::Errors::Unauthorized
       end
     end
 

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.9.3
+  version: 4.12.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-20 00:00:00.000000000 Z
+date: 2021-10-15 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  name: logstash-mixin-validator_support
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -47,45 +33,59 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.0.5
-  name: elasticsearch
+        version: '1.3'
+  name: logstash-mixin-ecs_compatibility_support
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.0.5
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  name: logstash-codec-json
+        version: '1.0'
+  name: logstash-mixin-event_support
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-validator_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  name: sequel
+        version: 7.14.0
+  name: elasticsearch
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 7.14.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -142,12 +142,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.7.1
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.15.4
+        version: '1'
   name: faraday
   prerelease: false
   type: :development
@@ -155,7 +169,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.15.4
+        version: '1'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -184,6 +198,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  name: cabin
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: webrick
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program