logstash-input-elasticsearch 4.20.5 → 5.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eaa000218cac721012284803638cb681c0eccf02107c34b03e597998a2e9eee3
-  data.tar.gz: eef84d9f24f1346b1d4793f6b10f200e790739f63d588888bae1bcdae33ec51f
+  metadata.gz: b34b6c6d814152e88f320525ea0bb80bbf1e63ff962e022aaac0a2385dd087b6
+  data.tar.gz: d142df9148ad69bf838d62badeec71382118741938db61e6aad0676bdb918a37
 SHA512:
-  metadata.gz: a9ed0836e937e4ceb9726d05be77e39a10cc11189bbf0cffb829e4445f5e6948e339187c2f088d1d9408657c26318375756d974299667fdf86d62b77ba6f00d5
-  data.tar.gz: 3e919b494b24d4d61a98222e27310a6d0e2f9d1d2b6209b2879c80199dd4c69e39b789a2d6e7503e07fe3097d50261f78d0c4ab7300bd48d6cf79a5d28c66c01
+  metadata.gz: 19b2b1325ded83b5b93966365f855f104ba1881f2c991ffdbe92216e08d12d18a7b3ddd4a14d755f6d55c85c98e00d12ca566188c63706d6db1f0aa5b085048b
+  data.tar.gz: ff5de17e75281d8ddd0be70167f2c4dee0a90eef328c7e486b704e79fe10db7b7108b733f77438386a7abb18d504efbef5aaf7b0f34a6c8edd62791640514b7b

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 5.0.0
+  - SSL settings that were marked deprecated in version `4.17.0` are now marked obsolete, and will prevent the plugin from starting.
+  - These settings are:
+    - `ssl`, which should bre replaced by `ssl_enabled`
+    - `ca_file`, which should bre replaced by `ssl_certificate_authorities`
+    - `ssl_certificate_verification`, which should bre replaced by `ssl_verification_mode`
+    - [#213](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/213)
+
 ## 4.20.5
   - Add `x-elastic-product-origin` header to Elasticsearch requests [#211](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/211)
 

data/docs/index.asciidoc

@@ -23,7 +23,7 @@ include::{include_path}/plugin_header.asciidoc[]
 
 Read from an Elasticsearch cluster, based on search query results.
 This is useful for replaying test logs, reindexing, etc.
-You can periodically schedule ingestion using a cron syntax 
+You can periodically schedule ingestion using a cron syntax
 (see `schedule` setting) or run the query one time to load
 data into Logstash.
 
@@ -96,7 +96,13 @@ TIP: Set the `target` option to avoid potential schema conflicts.
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Input configuration options
 
-This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> and the <<plugins-{type}s-{plugin}-deprecated-options>> described later.
+This plugin supports these configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+NOTE: As of version `5.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed.
+Please check out <<plugins-{type}s-{plugin}-obsolete-options>> for details.
+
+NOTE: As of version `5.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed.
+Please check out <<plugins-{type}s-{plugin}-obsolete-options>> for details.
 
 [cols="<,<,<",options="header",]
 |=======================================================================
@@ -106,6 +112,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-connect_timeout_seconds>> | <<number,number>>|No
+| <<plugins-{type}s-{plugin}-custom_headers>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-docinfo>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-docinfo_fields>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-docinfo_target>> |<<string,string>>|No
@@ -199,8 +206,18 @@ For more info, check out the
 The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch.
 Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool.
 
+[id="plugins-{type}s-{plugin}-custom_headers"]
+===== `custom_headers`
+
+  * Value type is <<hash,hash>>
+  * Default value is empty
+
+Pass a set of key value pairs as the headers sent in each request to an elasticsearch node.
+The headers will be used for any kind of request.
+These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers.
+
 [id="plugins-{type}s-{plugin}-docinfo"]
-===== `docinfo` 
+===== `docinfo`
 
   * Value type is <<boolean,boolean>>
   * Default value is `false`
@@ -251,7 +268,7 @@ Example
 
 
 [id="plugins-{type}s-{plugin}-docinfo_fields"]
-===== `docinfo_fields` 
+===== `docinfo_fields`
 
   * Value type is <<array,array>>
   * Default value is `["_index", "_type", "_id"]`
@@ -262,7 +279,7 @@ option lists the metadata fields to save in the current event. See
 more information.
 
 [id="plugins-{type}s-{plugin}-docinfo_target"]
-===== `docinfo_target` 
+===== `docinfo_target`
 
   * Value type is <<string,string>>
   * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
@@ -286,7 +303,7 @@ this option names the field under which to store the metadata fields as subfield
 Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
 
 [id="plugins-{type}s-{plugin}-hosts"]
-===== `hosts` 
+===== `hosts`
 
   * Value type is <<array,array>>
   * There is no default value for this setting.
@@ -296,18 +313,18 @@ can be either IP, HOST, IP:port, or HOST:port. The port defaults to
 9200.
 
 [id="plugins-{type}s-{plugin}-index"]
-===== `index` 
+===== `index`
 
   * Value type is <<string,string>>
   * Default value is `"logstash-*"`
 
-The index or alias to search. 
+The index or alias to search.
 Check out {ref}/api-conventions.html#api-multi-index[Multi Indices
 documentation] in the Elasticsearch documentation for info on
 referencing multiple indices.
 
 [id="plugins-{type}s-{plugin}-password"]
-===== `password` 
+===== `password`
 
   * Value type is <<password,password>>
   * There is no default value for this setting.
@@ -327,7 +344,7 @@ An empty string is treated as if proxy was not set, this is useful when using
 environment variables e.g. `proxy => '${LS_PROXY:}'`.
 
 [id="plugins-{type}s-{plugin}-query"]
-===== `query` 
+===== `query`
 
   * Value type is <<string,string>>
   * Default value is `'{ "sort": [ "_doc" ] }'`
@@ -375,7 +392,7 @@ The default is 0 (no retry). This value should be equal to or greater than zero.
 NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible.
 
 [id="plugins-{type}s-{plugin}-schedule"]
-===== `schedule` 
+===== `schedule`
 
   * Value type is <<string,string>>
   * There is no default value for this setting.
@@ -387,7 +404,7 @@ There is no schedule by default. If no schedule is given, then the statement is
 exactly once.
 
 [id="plugins-{type}s-{plugin}-scroll"]
-===== `scroll` 
+===== `scroll`
 
   * Value type is <<string,string>>
   * Default value is `"1m"`
@@ -410,7 +427,7 @@ The query requires at least one `sort` field, as described in the <<plugins-{typ
 `scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended.
 
 [id="plugins-{type}s-{plugin}-size"]
-===== `size` 
+===== `size`
 
   * Value type is <<number,number>>
   * Default value is `1000`
@@ -478,6 +495,8 @@ Enable SSL/TLS secured communication to Elasticsearch cluster.
 Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<plugins-{type}s-{plugin}-hosts>> or extracted from the <<plugins-{type}s-{plugin}-cloud_id>>.
 If no explicit protocol is specified plain HTTP will be used.
 
+When not explicitly set, SSL will be automatically enabled if any of the specified hosts use HTTPS.
+
 [id="plugins-{type}s-{plugin}-ssl_key"]
 ===== `ssl_key`
   * Value type is <<path,path>>
@@ -598,7 +617,7 @@ It is also possible to target an entry in the event's metadata, which will be av
 
 
 [id="plugins-{type}s-{plugin}-user"]
-===== `user` 
+===== `user`
 
   * Value type is <<string,string>>
   * There is no default value for this setting.
@@ -608,56 +627,21 @@ option when authenticating to the Elasticsearch server. If set to an
 empty string authentication will be disabled.
 
 
-[id="plugins-{type}s-{plugin}-deprecated-options"]
-==== Elasticsearch Input deprecated configuration options
+[id="plugins-{type}s-{plugin}-obsolete-options"]
+==== Elasticsearch Input Obsolete Configuration Options
 
-This plugin supports the following deprecated configurations.
+WARNING: As of version `5.0.0` of this plugin, some configuration options have been replaced.
+The plugin will fail to start if it contains any of these obsolete options.
 
-WARNING: Deprecated options are subject to removal in future releases.
 
-[cols="<,<,<",options="header",]
+[cols="<,<",options="header",]
 |=======================================================================
-|Setting|Input type|Replaced by
-| <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_certificate_authorities>>
-| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|<<plugins-{type}s-{plugin}-ssl_enabled>>
-| <<plugins-{type}s-{plugin}-ssl_certificate_verification>> |<<boolean,boolean>>|<<plugins-{type}s-{plugin}-ssl_verification_mode>>
+|Setting|Replaced by
+| ca_file | <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>
+| ssl | <<plugins-{type}s-{plugin}-ssl_enabled>>
+| ssl_certificate_verification | <<plugins-{type}s-{plugin}-ssl_verification_mode>>
 |=======================================================================
 
-[id="plugins-{type}s-{plugin}-ca_file"]
-===== `ca_file`
-deprecated[4.17.0, Replaced by <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>]
-
-* Value type is <<path,path>>
-* There is no default value for this setting.
-
-SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary.
-
-[id="plugins-{type}s-{plugin}-ssl"]
-===== `ssl`
-deprecated[4.17.0, Replaced by <<plugins-{type}s-{plugin}-ssl_enabled>>]
-
-* Value type is <<boolean,boolean>>
-* Default value is `false`
-
-If enabled, SSL will be used when communicating with the Elasticsearch
-server (i.e. HTTPS will be used instead of plain HTTP).
-
-
-[id="plugins-{type}s-{plugin}-ssl_certificate_verification"]
-===== `ssl_certificate_verification`
-deprecated[4.17.0, Replaced by <<plugins-{type}s-{plugin}-ssl_verification_mode>>]
-
-* Value type is <<boolean,boolean>>
-* Default value is `true`
-
-Option to validate the server's certificate. Disabling this severely compromises security.
-When certificate validation is disabled, this plugin implicitly trusts the machine
-resolved at the given address without validating its proof-of-identity.
-In this scenario, the plugin can transmit credentials to or process data from an untrustworthy
-man-in-the-middle or other compromised infrastructure.
-More information on the importance of certificate verification:
-**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**.
-
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
 

data/lib/logstash/inputs/elasticsearch.rb

@@ -24,9 +24,9 @@ require_relative "elasticsearch/patches/_elasticsearch_transport_connections_sel
 # called `http.content_type.required`. If this option is set to `true`, and you
 # are using Logstash 2.4 through 5.2, you need to update the Elasticsearch input
 # plugin to version 4.0.2 or higher.
-# 
+#
 # ================================================================================
-# 
+#
 # Read from an Elasticsearch cluster, based on search query results.
 # This is useful for replaying test logs, reindexing, etc.
 # It also supports periodically scheduling lookup enrichments
@@ -166,6 +166,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/_document_metadata.html
   config :docinfo_fields, :validate => :array, :default => ['_index', '_type', '_id']
 
+  # Custom headers for Elasticsearch requests
+  config :custom_headers, :validate => :hash, :default => {}
+
   # Basic Auth - username
   config :user, :validate => :string
 
@@ -198,23 +201,12 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # Set the address of a forward HTTP proxy.
   config :proxy, :validate => :uri_or_empty
 
-  # SSL
-  config :ssl, :validate => :boolean, :default => false, :deprecated => "Set 'ssl_enabled' instead."
-
-  # SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary
-  config :ca_file, :validate => :path, :deprecated => "Set 'ssl_certificate_authorities' instead."
-
   # OpenSSL-style X.509 certificate certificate to authenticate the client
   config :ssl_certificate, :validate => :path
 
   # SSL Certificate Authority files in PEM encoded format, must also include any chain certificates as necessary
   config :ssl_certificate_authorities, :validate => :path, :list => true
 
-  # Option to validate the server's certificate. Disabling this severely compromises security.
-  # For more information on the importance of certificate verification please read
-  # https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
-  config :ssl_certificate_verification, :validate => :boolean, :default => true, :deprecated => "Set 'ssl_verification_mode' instead."
-
   # The list of cipher suites to use, listed by priorities.
   # Supported cipher suites vary depending on which version of Java is used.
   config :ssl_cipher_suites, :validate => :string, :list => true
@@ -242,7 +234,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   config :ssl_truststore_password, :validate => :password
 
   # The JKS truststore to validate the server's certificate.
-  # Use either `:ssl_truststore_path` or `:ssl_certificate_authorities`
   config :ssl_truststore_path, :validate => :path
 
   # The format of the truststore file. It must be either jks or pkcs12
@@ -264,6 +255,11 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # If set, the _source of each hit will be added nested under the target instead of at the top-level
   config :target, :validate => :field_reference
 
+  # Obsolete Settings
+  config :ssl, :obsolete => "Set 'ssl_enabled' instead."
+  config :ca_file, :obsolete => "Set 'ssl_certificate_authorities' instead."
+  config :ssl_certificate_verification, :obsolete => "Set 'ssl_verification_mode' instead."
+
   # config :ca_trusted_fingerprint, :validate => :sha_256_hex
   include LogStash::PluginMixins::CATrustedFingerprintSupport
 
@@ -305,6 +301,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     transport_options[:headers].merge!(setup_basic_auth(user, password))
     transport_options[:headers].merge!(setup_api_key(api_key))
     transport_options[:headers].merge!({'user-agent' => prepare_user_agent()})
+    transport_options[:headers].merge!(@custom_headers) unless @custom_headers.empty?
     transport_options[:request_timeout] = @request_timeout_seconds unless @request_timeout_seconds.nil?
     transport_options[:connect_timeout] = @connect_timeout_seconds unless @connect_timeout_seconds.nil?
     transport_options[:socket_timeout]  = @socket_timeout_seconds  unless @socket_timeout_seconds.nil?
@@ -408,8 +405,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     ssl_options[:ssl] = true if @ssl_enabled
 
     unless @ssl_enabled
-      # Keep it backward compatible with the deprecated `ssl` option
-      ssl_options[:trust_strategy] = trust_strategy_for_ca_trusted_fingerprint if original_params.include?('ssl')
       return ssl_options
     end
 
@@ -473,38 +468,11 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   end
 
   def setup_ssl_params!
-    @ssl_enabled = normalize_config(:ssl_enabled) do |normalize|
-      normalize.with_deprecated_alias(:ssl)
-    end
-
-    # Infer the value if neither the deprecate `ssl` and `ssl_enabled` were set
-    infer_ssl_enabled_from_hosts
-
-    @ssl_certificate_authorities = normalize_config(:ssl_certificate_authorities) do |normalize|
-      normalize.with_deprecated_mapping(:ca_file) do |ca_file|
-        [ca_file]
-      end
-    end
-
-    @ssl_verification_mode = normalize_config(:ssl_verification_mode) do |normalize|
-      normalize.with_deprecated_mapping(:ssl_certificate_verification) do |ssl_certificate_verification|
-        if ssl_certificate_verification == true
-          "full"
-        else
-          "none"
-        end
-      end
+    # Only infer ssl_enabled if it wasn't explicitly set
+    unless original_params.include?('ssl_enabled')
+      @ssl_enabled = effectively_ssl?
+      params['ssl_enabled'] = @ssl_enabled
     end
-
-    params['ssl_enabled'] = @ssl_enabled
-    params['ssl_certificate_authorities'] = @ssl_certificate_authorities unless @ssl_certificate_authorities.nil?
-    params['ssl_verification_mode'] = @ssl_verification_mode unless @ssl_verification_mode.nil?
-  end
-
-  def infer_ssl_enabled_from_hosts
-    return if original_params.include?('ssl') || original_params.include?('ssl_enabled')
-
-    @ssl_enabled = params['ssl_enabled'] = effectively_ssl?
   end
 
   def setup_hosts

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.20.5'
+  s.version         = '5.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/elasticsearch_spec.rb

@@ -58,6 +58,19 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
       end
     end
 
+    describe 'handling obsolete settings' do
+      [{:name => 'ssl', :replacement => 'ssl_enabled', :sample_value => true},
+       {:name => 'ca_file', :replacement => 'ssl_certificate_authorities', :sample_value => 'spec/fixtures/test_certs/ca.crt'},
+       {:name => 'ssl_certificate_verification', :replacement => 'ssl_verification_mode', :sample_value => false }].each do | obsolete_setting|
+        context "with obsolete #{obsolete_setting[:name]}" do
+          let (:config) { {obsolete_setting[:name] => obsolete_setting[:sample_value]} }
+          it "should raise a config error with the appropriate message" do
+            expect { plugin.register }.to raise_error LogStash::ConfigurationError, /The setting `#{obsolete_setting[:name]}` in plugin `elasticsearch` is obsolete and is no longer available. Set '#{obsolete_setting[:replacement]}' instead/i
+          end
+        end
+      end
+    end
+
     context "against not authentic Elasticsearch" do
       before(:each) do
          Elasticsearch::Client.send(:define_method, :ping) { raise Elasticsearch::UnsupportedProductError.new("Fake error") } # define error ping method
@@ -103,6 +116,22 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
           expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_including("x-elastic-product-origin"=>"logstash-input-elasticsearch")
         end
       end
+
+      context "with custom headers" do
+        let(:config) do
+          {
+            "schedule" => "* * * * * UTC",
+            "custom_headers" => { "Custom-Header-1" => "Custom Value 1", "Custom-Header-2" => "Custom Value 2" }
+          }
+        end
+
+
+        it "sets custom headers" do
+          plugin.register
+          client = plugin.send(:client)
+          expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_including(config["custom_headers"])
+        end
+      end
     end
 
     context "retry" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.20.5
+  version: 5.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-05 00:00:00.000000000 Z
+date: 2024-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement