logstash-input-elasticsearch 4.9.0 → 4.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4091feeb0b3bf292cfb9afcde7496a72f95168eb570b21d29064ade174bb1352
-  data.tar.gz: cfd02af050bb495dceea16b4ffe5daa6828088d2bd7994639ee08374f87da69d
+  metadata.gz: 84fb0092d51b303586c275c6cfaef3222391d7aabe910450a989291b691930c0
+  data.tar.gz: 955e27896dff25ec6568685f91cc7e34252bc54fbb5182133db1e4dd273efffe
 SHA512:
-  metadata.gz: 873680bea22204e65d519d310f3952f26fd672ce336504e45e99b13988e2da2794b7c05f83b3e1b7a87f317eb51c079759f9df515dc99236577417ecfd379aa1
-  data.tar.gz: 85c66a665eb7a3b503ab7cec64ea2f24b0e16829c4b78a75560d69c6272b1861583dad41e04ea1157ce58435714a86eaa6f4d71b5a4a724edfb9604f77150b99
+  metadata.gz: 2be653a935384617b32905f441060326f2b45eadcaacbf727d1ebe5c82711e3e9f0f3a038b9ca39aad6377e67bb5caf78c8b83a625d82aa8fca267b738dc9cab
+  data.tar.gz: a19916b987c434dfeb396379fa9f552fd001901bc5b37ce60407e4b7df69f5bd3ff76d4122b2f71d27c5d37d983a1dd283dcdf3aa1693730e93c2875f2459a1a

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+## 4.10.0
+  - Feat: added ecs_compatibility + event_factory support [#149](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/149)
+
+## 4.9.3
+  - Fixed SSL handshake hang indefinitely with proxy setup [#156](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/156)
+
+## 4.9.2
+  - Fix: a regression (in LS 7.14.0) where due the elasticsearch client update (from 5.0.5 to 7.5.0) the `Authorization` 
+    header isn't passed, this leads to the plugin not being able to leverage `user`/`password` credentials set by the user.
+    [#153](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/153)
+
+
+## 4.9.1
+  - [DOC] Replaced hard-coded links with shared attributes [#143](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/143)
+  - [DOC] Added missing quote to docinfo_fields example [#145](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/145)
+
 ## 4.9.0
   - Added `target` option, allowing the hit's source to target a specific field instead of being expanded at the root of the event. This allows the input to play nicer with the Elastic Common Schema when the input does not follow the schema. [#117](https://github.com/logstash-plugins/logstash-input-elasticsearch/issues/117)
 

data/README.md

@@ -1,7 +1,7 @@
 # Logstash Plugin
 
 [![Gem Version](https://badge.fury.io/rb/logstash-input-elasticsearch.svg)](https://badge.fury.io/rb/logstash-input-elasticsearch)
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-elasticsearch.svg)](https://travis-ci.org/logstash-plugins/logstash-input-elasticsearch)
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-input-elasticsearch.svg)](https://travis-ci.com/logstash-plugins/logstash-input-elasticsearch)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/index.asciidoc

@@ -83,8 +83,18 @@ Authentication to a secure Elasticsearch cluster is possible using _one_ of the
 Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level.
 The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks.
 
+[id="plugins-{type}s-{plugin}-ecs"]
+==== Compatibility with the Elastic Common Schema (ECS)
+
+When ECS compatibility is disabled, `docinfo_target` uses the `"@metadata"` field as a default, with ECS enabled the plugin
+uses a naming convention `"[@metadata][input][elasticsearch]"` as a default target for placing document information.
+
+The plugin logs a warning when ECS is enabled and `target` isn't set.
+
+TIP: Set the `target` option to avoid potential schema conflicts.
+
 [id="plugins-{type}s-{plugin}-options"]
-==== Elasticsearch Input Configuration Options
+==== Elasticsearch Input configuration options
 
 This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
 
@@ -99,6 +109,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-docinfo>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-docinfo_fields>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-docinfo_target>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-hosts>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-index>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
@@ -111,7 +122,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-slices>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-socket_timeout_seconds>> | <<number,number>>|No
-| <<plugins-{type}s-{plugin}-target>> | https://www.elastic.co/guide/en/logstash/master/field-references-deepdive.html[field reference] | No
+| <<plugins-{type}s-{plugin}-target>> | {logstash-ref}/field-references-deepdive.html[field reference] | No
 | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
 |=======================================================================
 
@@ -130,7 +141,7 @@ Authenticate using Elasticsearch API key. Note that this option also requires en
 
 Format is `id:api_key` where `id` and `api_key` are as returned by the
 Elasticsearch
-https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create
+{ref}/security-api-create-api-key.html[Create
 API key API].
 
 [id="plugins-{type}s-{plugin}-ca_file"]
@@ -150,8 +161,7 @@ SSL Certificate Authority file in PEM encoded format, must also include any chai
 Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` pair.
 
 For more info, check out the
-https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html[Logstash-to-Cloud
-documentation]
+{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation].
 
 [id="plugins-{type}s-{plugin}-cloud_id"]
 ===== `cloud_id`
@@ -162,8 +172,7 @@ documentation]
 Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
 
 For more info, check out the
-https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html[Logstash-to-Cloud
-documentation]
+{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation].
 
 [id="plugins-{type}s-{plugin}-connect_timeout_seconds"]
 ===== `connect_timeout_seconds`
@@ -199,13 +208,14 @@ Example
         size => 500
         scroll => "5m"
         docinfo => true
+        docinfo_target => "[@metadata][doc]"
       }
     }
     output {
       elasticsearch {
-        index => "copy-of-production.%{[@metadata][_index]}"
-        document_type => "%{[@metadata][_type]}"
-        document_id => "%{[@metadata][_id]}"
+        index => "copy-of-production.%{[@metadata][doc][_index]}"
+        document_type => "%{[@metadata][doc][_type]}"
+        document_id => "%{[@metadata][doc][_id]}"
       }
     }
 
@@ -216,8 +226,9 @@ Example
     input {
       elasticsearch {
         docinfo => true
+        docinfo_target => "[@metadata][doc]"
         add_field => {
-          identifier => %{[@metadata][_index]}:%{[@metadata][_type]}:%{[@metadata][_id]}"
+          identifier => "%{[@metadata][doc][_index]}:%{[@metadata][doc][_type]}:%{[@metadata][doc][_id]}"
         }
       }
     }
@@ -238,11 +249,25 @@ more information.
 ===== `docinfo_target` 
 
   * Value type is <<string,string>>
-  * Default value is `"@metadata"`
+  * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
+    ** ECS Compatibility disabled: `"@metadata"`
+    ** ECS Compatibility enabled: `"[@metadata][input][elasticsearch]"`
+
+If document metadata storage is requested by enabling the `docinfo` option,
+this option names the field under which to store the metadata fields as subfields.
+
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+  * Value type is <<string,string>>
+  * Supported values are:
+    ** `disabled`: CSV data added at root level
+    ** `v1`,`v8`: Elastic Common Schema compliant behavior
+  * Default value depends on which version of Logstash is running:
+    ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
+    ** Otherwise, the default value is `disabled`
 
-If document metadata storage is requested by enabling the `docinfo`
-option, this option names the field under which to store the metadata
-fields as subfields.
+Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
 
 [id="plugins-{type}s-{plugin}-hosts"]
 ===== `hosts` 
@@ -260,10 +285,9 @@ can be either IP, HOST, IP:port, or HOST:port. The port defaults to
   * Value type is <<string,string>>
   * Default value is `"logstash-*"`
 
-The index or alias to search. See
-https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-index.html[Multi Indices documentation]
-in the Elasticsearch documentation for more information on how to reference
-multiple indices.
+The index or alias to search. See {ref}/multi-index.html[Multi Indices
+documentation] in the Elasticsearch documentation for more information on how to
+reference multiple indices.
 
 
 [id="plugins-{type}s-{plugin}-password"]
@@ -292,9 +316,8 @@ environment variables e.g. `proxy => '${LS_PROXY:}'`.
   * Value type is <<string,string>>
   * Default value is `'{ "sort": [ "_doc" ] }'`
 
-The query to be executed. Read the
-https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html[Elasticsearch query DSL documentation]
-for more information.
+The query to be executed. Read the {ref}/query-dsl.html[Elasticsearch query DSL
+documentation] for more information.
 
 [id="plugins-{type}s-{plugin}-request_timeout_seconds"]
 ===== `request_timeout_seconds`
@@ -345,7 +368,7 @@ This allows you to set the maximum number of hits returned per scroll.
 
 In some cases, it is possible to improve overall throughput by consuming multiple
 distinct slices of a query simultaneously using
-https://www.elastic.co/guide/en/elasticsearch/reference/current/paginate-search-results.html#slice-scroll[sliced scrolls],
+{ref}/paginate-search-results.html#slice-scroll[sliced scrolls],
 especially if the pipeline is spending significant time waiting on Elasticsearch
 to provide results.
 
@@ -382,7 +405,7 @@ Socket timeouts usually occur while waiting for the first byte of a response, su
 [id="plugins-{type}s-{plugin}-target"]
 ===== `target`
 
-* Value type is https://www.elastic.co/guide/en/logstash/master/field-references-deepdive.html[field reference]
+* Value type is {logstash-ref}/field-references-deepdive.html[field reference]
 * There is no default value for this setting.
 
 Without a `target`, events are created from each hit's `_source` at the root level.
@@ -406,4 +429,4 @@ empty string authentication will be disabled.
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
 
-:default_codec!:
+:no_codec!:

data/lib/logstash/inputs/elasticsearch.rb

@@ -4,9 +4,15 @@ require "logstash/namespace"
 require "logstash/json"
 require "logstash/util/safe_uri"
 require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
+require 'logstash/plugin_mixins/event_support/event_factory_adapter'
+require 'logstash/plugin_mixins/ecs_compatibility_support'
+require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
 require "base64"
-require_relative "patch"
 
+require "elasticsearch"
+require "elasticsearch/transport/transport/http/manticore"
+require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
+require_relative "elasticsearch/patches/_elasticsearch_transport_connections_selector"
 
 # .Compatibility Note
 # [NOTE]
@@ -63,12 +69,16 @@ require_relative "patch"
 #
 #
 class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
+
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
+  include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
+
+  include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
+
   extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
 
   config_name "elasticsearch"
 
-  default :codec, "json"
-
   # List of elasticsearch hosts to use for querying.
   # Each host can be either IP, HOST, IP:port or HOST:port.
   # Port defaults to 9200
@@ -125,8 +135,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   #
   config :docinfo, :validate => :boolean, :default => false
 
-  # Where to move the Elasticsearch document information. By default we use the @metadata field.
-  config :docinfo_target, :validate=> :string, :default => LogStash::Event::METADATA
+  # Where to move the Elasticsearch document information.
+  # default: [@metadata][input][elasticsearch] in ECS mode, @metadata field otherwise
+  config :docinfo_target, :validate=> :field_reference
 
   # List of document metadata to move to the `docinfo_target` field.
   # To learn more about Elasticsearch metadata fields read
@@ -181,10 +192,16 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # If set, the _source of each hit will be added nested under the target instead of at the top-level
   config :target, :validate => :field_reference
 
+  def initialize(params={})
+    super(params)
+
+    if docinfo_target.nil?
+      @docinfo_target = ecs_select[disabled: '@metadata', v1: '[@metadata][input][elasticsearch]']
+    end
+  end
+
   def register
-    require "elasticsearch"
     require "rufus/scheduler"
-    require "elasticsearch/transport/transport/http/manticore"
 
     @options = {
       :index => @index,
@@ -225,7 +242,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   end
 
 
-
   def run(output_queue)
     if @schedule
       @scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
@@ -267,7 +283,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
     logger.info("Slice starting", slice_id: slice_id, slices: @slices) unless slice_id.nil?
 
-    scroll_id = nil
     begin
       r = search_request(slice_options)
 
@@ -298,47 +313,41 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     [r['hits']['hits'].any?, r['_scroll_id']]
   rescue => e
     # this will typically be triggered by a scroll timeout
-    logger.error("Scroll request error, aborting scroll", error: e.inspect)
+    logger.error("Scroll request error, aborting scroll", message: e.message, exception: e.class)
     # return no hits and original scroll_id so we can try to clear it
     [false, scroll_id]
   end
 
   def push_hit(hit, output_queue)
-    if @target.nil?
-      event = LogStash::Event.new(hit['_source'])
-    else
-      event = LogStash::Event.new
-      event.set(@target, hit['_source'])
-    end
-
-    if @docinfo
-      # do not assume event[@docinfo_target] to be in-place updatable. first get it, update it, then at the end set it in the event.
-      docinfo_target = event.get(@docinfo_target) || {}
+    event = targeted_event_factory.new_event hit['_source']
+    set_docinfo_fields(hit, event) if @docinfo
+    decorate(event)
+    output_queue << event
+  end
 
-      unless docinfo_target.is_a?(Hash)
-        @logger.error("Elasticsearch Input: Incompatible Event, incompatible type for the docinfo_target=#{@docinfo_target} field in the `_source` document, expected a hash got:", :docinfo_target_type => docinfo_target.class, :event => event)
+  def set_docinfo_fields(hit, event)
+    # do not assume event[@docinfo_target] to be in-place updatable. first get it, update it, then at the end set it in the event.
+    docinfo_target = event.get(@docinfo_target) || {}
 
-        # TODO: (colin) I am not sure raising is a good strategy here?
-        raise Exception.new("Elasticsearch input: incompatible event")
-      end
+    unless docinfo_target.is_a?(Hash)
+      @logger.error("Incompatible Event, incompatible type for the docinfo_target=#{@docinfo_target} field in the `_source` document, expected a hash got:", :docinfo_target_type => docinfo_target.class, :event => event.to_hash_with_metadata)
 
-      @docinfo_fields.each do |field|
-        docinfo_target[field] = hit[field]
-      end
-
-      event.set(@docinfo_target, docinfo_target)
+      # TODO: (colin) I am not sure raising is a good strategy here?
+      raise Exception.new("Elasticsearch input: incompatible event")
     end
 
-    decorate(event)
+    @docinfo_fields.each do |field|
+      docinfo_target[field] = hit[field]
+    end
 
-    output_queue << event
+    event.set(@docinfo_target, docinfo_target)
   end
 
   def clear_scroll(scroll_id)
     @client.clear_scroll(scroll_id: scroll_id) if scroll_id
   rescue => e
     # ignore & log any clear_scroll errors
-    logger.warn("Ignoring clear_scroll exception", message: e.message)
+    logger.warn("Ignoring clear_scroll exception", message: e.message, exception: e.class)
   end
 
   def scroll_request scroll_id
@@ -388,14 +397,14 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     return {} unless user && password && password.value
 
     token = ::Base64.strict_encode64("#{user}:#{password.value}")
-    { Authorization: "Basic #{token}" }
+    { 'Authorization' => "Basic #{token}" }
   end
 
   def setup_api_key(api_key)
     return {} unless (api_key && api_key.value)
 
     token = ::Base64.strict_encode64(api_key.value)
-    { Authorization: "ApiKey #{token}" }
+    { 'Authorization' => "ApiKey #{token}" }
   end
 
   def fill_user_password_from_cloud_auth
@@ -448,6 +457,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     [ cloud_auth.username, cloud_auth.password ]
   end
 
+  # @private used by unit specs
+  attr_reader :client
+
   module URIOrEmptyValidator
     ##
     # @override to provide :uri_or_empty validator

data/lib/logstash/inputs/{patch.rb → elasticsearch/patches/_elasticsearch_transport_connections_selector.rb}

@@ -1,10 +1,13 @@
-if Gem.loaded_specs['elasticsearch-transport'].version >= Gem::Version.new("7.2.0")
+require 'elasticsearch'
+require 'elasticsearch/transport/transport/connections/selector'
+
+if Gem.loaded_specs['elasticsearch-transport'].version < Gem::Version.new("7.2.0")
   # elasticsearch-transport versions prior to 7.2.0 suffered of a race condition on accessing
-  # the connection pool. This issue was fixed with https://github.com/elastic/elasticsearch-ruby/commit/15f9d78591a6e8823948494d94b15b0ca38819d1
-   # This plugin, at the moment, is forced to use v5.x so we have to monkey patch the gem. When this requirement
-   # ceases, this patch could be removed.
-  puts "WARN remove the patch code into logstash-input-elasticsearch plugin"
-else
+  # the connection pool. This issue was fixed (in 7.2.0) with
+  # https://github.com/elastic/elasticsearch-ruby/commit/15f9d78591a6e8823948494d94b15b0ca38819d1
+  #
+  # This plugin, at the moment, is using elasticsearch >= 5.0.5
+  # When this requirement ceases, this patch could be removed.
   module Elasticsearch
     module Transport
       module Transport

data/lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+require "elasticsearch"
+require "elasticsearch/transport/transport/http/manticore"
+
+es_client_version = Gem.loaded_specs['elasticsearch-transport'].version
+if es_client_version >= Gem::Version.new('7.2') && es_client_version < Gem::Version.new('7.16')
+  # elasticsearch-transport 7.2.0 - 7.14.0 had a bug where setting http headers
+  #   ES::Client.new ..., transport_options: { headers: { 'Authorization' => ... } }
+  # would be lost https://github.com/elastic/elasticsearch-ruby/issues/1428
+  #
+  # NOTE: needs to be idempotent as filter ES plugin might apply the same patch!
+  #
+  # @private
+  module Elasticsearch
+    module Transport
+      module Transport
+        module HTTP
+          class Manticore
+
+            def apply_headers(request_options, options)
+              headers = (options && options[:headers]) || {}
+              headers[CONTENT_TYPE_STR] = find_value(headers, CONTENT_TYPE_REGEX) || DEFAULT_CONTENT_TYPE
+              headers[USER_AGENT_STR] = find_value(headers, USER_AGENT_REGEX) || user_agent_header
+              headers[ACCEPT_ENCODING] = GZIP if use_compression?
+              (request_options[:headers] ||= {}).merge!(headers) # this line was changed
+            end
+
+          end
+        end
+      end
+    end
+  end
+end

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.9.0'
+  s.version         = '4.10.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,20 +20,20 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
+  s.add_runtime_dependency 'logstash-mixin-event_support', '~> 1.0'
+  s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
 
-  s.add_runtime_dependency 'elasticsearch', '>= 5.0.3'
+  s.add_runtime_dependency 'elasticsearch', '>= 5.0.5' # LS >= 6.7 and < 7.14 all used version 5.0.5
 
-  s.add_runtime_dependency 'logstash-codec-json'
-  s.add_runtime_dependency 'logstash-codec-plain'
-  s.add_runtime_dependency 'sequel'
   s.add_runtime_dependency 'tzinfo'
   s.add_runtime_dependency 'tzinfo-data'
   s.add_runtime_dependency 'rufus-scheduler'
-  s.add_runtime_dependency 'manticore', "~> 0.6"
-  s.add_runtime_dependency 'faraday', "~> 0.15.4"
+  s.add_runtime_dependency 'manticore', ">= 0.7.1"
 
+  s.add_development_dependency 'logstash-codec-plain'
+  s.add_development_dependency 'faraday', "~> 0.15.4"
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'timecop'
 end