logstash-filter-elasticsearch 3.16.1 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ad7767b1d40fd59a496420b7e4256cdc8ce85a9dc5d438e5d495918e09f04b17
-  data.tar.gz: f1c7c730a52abc8344416b1116eda1701d2e118e37054eec3993e6cc7da505ca
+  metadata.gz: 015a98dbd36122dd3fc4c74da5744a31f6182f67023f802189cc73e837d5ba7a
+  data.tar.gz: 8f1c3a79c0af3fc4154501d16bf775e01b6aa7575627852cb18954a0dd952d91
 SHA512:
-  metadata.gz: 54b78fa510472eee856712d95e07e0d8ef599bc3376477aa37fbc2d6be0628e51d0fe64de43611ad398c24274fc91fb134c5fd7dd6f239bfdac32d23034657c7
-  data.tar.gz: 4e35f51081661f012577c72e35bc516f44a92608f49a60c01963e947b28324c4ce85612513848fdecd70260aa8f50000651fc560b524452814d0fd7785c0cd4c
+  metadata.gz: fc3971541568e34c0ef644c250243fefcb48c778b2471f81f5e87a1b59535a6f5ccc2a2c13d57f5123e0577efb80962b85f8096d3d6f70e0721df4f716d705e1
+  data.tar.gz: 5fe74aa6d8179e6dd3d9c69a6d0c81f284035bd82343543132f129be7bfb3355e01678eddbd051d9d4c360e9704f8a234737096e757f710b395b5d366f57936f

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+## 4.0.0
+  - SSL settings that were marked deprecated in version `3.15.0` are now marked obsolete, and will prevent the plugin from starting.
+  - These settings are:
+    - `ca_file`, which should be replaced by `ssl_certificate_authorities`
+    - `keystore`, which should be replaced by `ssl_keystore_path`
+    - `keystore_password`, which should be replaced by `ssl_keystore_password`
+    - `keystore_type`, which should be replaced by `ssl_keystore_password`
+    -  `ssl`, which should be replaced by `ssl_enabled`
+    - [#183](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/183)
+
+## 3.16.2
+  - Add `x-elastic-product-origin` header to Elasticsearch requests [#185](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/185)
+
 ## 3.16.1
   - Version bump to pick up doc fix in [#172](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/172)
 

data/docs/index.asciidoc

@@ -110,7 +110,7 @@ Authentication to a secure Elasticsearch cluster is possible using _one_ of the
 * <<plugins-{type}s-{plugin}-user>> AND <<plugins-{type}s-{plugin}-password>>
 * <<plugins-{type}s-{plugin}-cloud_auth>>
 * <<plugins-{type}s-{plugin}-api_key>>
-* <<plugins-{type}s-{plugin}-keystore>> and/or <<plugins-{type}s-{plugin}-keystore_password>>
+* <<plugins-{type}s-{plugin}-ssl_keystore_path>> and/or <<plugins-{type}s-{plugin}-ssl_keystore_password>>
 
 [id="plugins-{type}s-{plugin}-autz"]
 ==== Authorization
@@ -121,7 +121,10 @@ The `monitoring` permission at cluster level is necessary to perform periodic co
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Filter Configuration Options
 
-This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> and the <<plugins-{type}s-{plugin}-deprecated-options>> described later.
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+NOTE: As of version `4.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. Please see the
+<<plugins-{type}s-{plugin}-obsolete-options>> for more details.
 
 [cols="<,<,<",options="header",]
 |=======================================================================
@@ -144,7 +147,6 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-retry_on_failure>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-retry_on_status>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-sort>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|__Deprecated__
 | <<plugins-{type}s-{plugin}-ssl_certificate>> |<<path,path>>|No
 | <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of <<path,path>>|No
 | <<plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of <<string,string>>|No
@@ -519,57 +521,21 @@ Tags the event on failure to look up previous log event information. This can be
 Basic Auth - username
 
 
-[id="plugins-{type}s-{plugin}-deprecated-options"]
-==== Elasticsearch Filter Deprecated Configuration Options
-
-This plugin supports the following deprecated configurations.
+[id="plugins-{type}s-{plugin}-obsolete-options"]
+==== Elasticsearch Filter Obsolete Configuration Options
 
-WARNING: Deprecated options are subject to removal in future releases.
+WARNING: As of version `4.0.0` of this plugin, some configuration options have been replaced.
+The plugin will fail to start if it contains any of these obsolete options.
 
 [cols="<,<,<",options="header",]
 |=======================================================================
-|Setting|Input type|Replaced by
-| <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_certificate_authorities>>
-| <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_keystore_path>>
-| <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|<<plugins-{type}s-{plugin}-ssl_keystore_password>>
+|Setting|Replaced by
+| ca_file |<<plugins-{type}s-{plugin}-ssl_certificate_authorities>>
+| keystore |<<plugins-{type}s-{plugin}-ssl_keystore_path>>
+| keystore_password |<<plugins-{type}s-{plugin}-ssl_keystore_password>>
+| ssl |<<plugins-{type}s-{plugin}-ssl_enabled>>
 |=======================================================================
 
-[id="plugins-{type}s-{plugin}-ca_file"]
-===== `ca_file`
-deprecated[3.15.0, Replaced by <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>]
-
-* Value type is <<path,path>>
-* There is no default value for this setting.
-
-SSL Certificate Authority file
-
-[id="plugins-{type}s-{plugin}-ssl"]
-===== `ssl`
-deprecated[3.15.0, Replaced by <<plugins-{type}s-{plugin}-ssl_enabled>>]
-
-* Value type is <<boolean,boolean>>
-* Default value is `false`
-
-SSL
-
-[id="plugins-{type}s-{plugin}-keystore"]
-===== `keystore`
-deprecated[3.15.0, Replaced by <<plugins-{type}s-{plugin}-ssl_keystore_path>>]
-
-* Value type is <<path,path>>
-* There is no default value for this setting.
-
-The keystore used to present a certificate to the server. It can be either .jks or .p12
-
-[id="plugins-{type}s-{plugin}-keystore_password"]
-===== `keystore_password`
-deprecated[3.15.0, Replaced by <<plugins-{type}s-{plugin}-ssl_keystore_password>>]
-
-* Value type is <<password,password>>
-* There is no default value for this setting.
-
-Set the keystore password
-
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/elasticsearch/client.rb

@@ -12,6 +12,7 @@ module LogStash
 
       BUILD_FLAVOR_SERVERLESS = 'serverless'.freeze
       DEFAULT_EAV_HEADER = { "Elastic-Api-Version" => "2023-10-31" }.freeze
+      INTERNAL_ORIGIN_HEADER = { 'x-elastic-product-origin' => 'logstash-filter-elasticsearch'}.freeze
 
       def initialize(logger, hosts, options = {})
         user = options.fetch(:user, nil)
@@ -25,6 +26,7 @@ module LogStash
         transport_options[:headers].merge!(setup_basic_auth(user, password))
         transport_options[:headers].merge!(setup_api_key(api_key))
         transport_options[:headers].merge!({ 'user-agent' => "#{user_agent}" })
+        transport_options[:headers].merge!(INTERNAL_ORIGIN_HEADER)
 
         transport_options[:pool_max] = 1000
         transport_options[:pool_max_per_route] = 100

data/lib/logstash/filters/elasticsearch.rb

@@ -3,7 +3,6 @@ require "logstash/filters/base"
 require "logstash/namespace"
 require "logstash/json"
 require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
-require "logstash/plugin_mixins/normalize_config_support"
 require "monitor"
 
 require_relative "elasticsearch/client"
@@ -62,18 +61,6 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Set the address of a forward HTTP proxy.
   config :proxy, :validate => :uri_or_empty
 
-  # SSL
-  config :ssl, :validate => :boolean, :default => false, :deprecated => "Set 'ssl_enabled' instead."
-
-  # SSL Certificate Authority file
-  config :ca_file, :validate => :path, :deprecated => "Set 'ssl_certificate_authorities' instead."
-
-  # The keystore used to present a certificate to the server.
-  # It can be either .jks or .p12
-  config :keystore, :validate => :path, :deprecated => "Use 'ssl_keystore_path' instead."
-
-  # Set the keystore password
-  config :keystore_password, :validate => :password, :deprecated => "Use 'ssl_keystore_password' instead."
 
   # OpenSSL-style X.509 certificate certificate to authenticate the client
   config :ssl_certificate, :validate => :path
@@ -135,11 +122,15 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # What status codes to retry on?
   config :retry_on_status, :validate => :number, :list => true, :default => [500, 502, 503, 504]
 
+
+  config :ssl, :obsolete => "Set 'ssl_enabled' instead."
+  config :ca_file, :obsolete => "Set 'ssl_certificate_authorities' instead."
+  config :keystore, :obsolete => "Set 'ssl_keystore_path' instead."
+  config :keystore_password, :validate => :password, :obsolete => "Set 'ssl_keystore_password' instead."
+
   # config :ca_trusted_fingerprint, :validate => :sha_256_hex
   include LogStash::PluginMixins::CATrustedFingerprintSupport
 
-  include LogStash::PluginMixins::NormalizeConfigSupport
-
   include MonitorMixin
   attr_reader :shared_client
 
@@ -488,46 +479,9 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end
 
   def setup_ssl_params!
-    @ssl_enabled = normalize_config(:ssl_enabled) do |normalize|
-      normalize.with_deprecated_alias(:ssl)
-    end
-
-    # Infer the value if neither the deprecate `ssl` and `ssl_enabled` were set
-    infer_ssl_enabled_from_hosts
-
-    @ssl_keystore_path = normalize_config(:ssl_keystore_path) do |normalize|
-      normalize.with_deprecated_alias(:keystore)
-    end
-
-    @ssl_keystore_password = normalize_config(:ssl_keystore_password) do |normalize|
-      normalize.with_deprecated_alias(:keystore_password)
-    end
-
-    @ssl_certificate_authorities = normalize_config(:ssl_certificate_authorities) do |normalize|
-      normalize.with_deprecated_mapping(:ca_file) do |ca_file|
-        [ca_file]
-      end
-    end
-
-    params['ssl_enabled'] = @ssl_enabled
-    params['ssl_keystore_path'] = @ssl_keystore_path unless @ssl_keystore_path.nil?
-    params['ssl_keystore_password'] = @ssl_keystore_password unless @ssl_keystore_password.nil?
-    params['ssl_certificate_authorities'] = @ssl_certificate_authorities unless @ssl_certificate_authorities.nil?
-  end
-
-  def infer_ssl_enabled_from_hosts
-    return if original_params.include?('ssl') || original_params.include?('ssl_enabled')
-
-    @ssl_enabled = params['ssl_enabled'] = effectively_ssl?
-  end
-
-  def effectively_ssl?
-    return true if @ssl_enabled
-
-    hosts = Array(@hosts)
-    return false if hosts.nil? || hosts.empty?
-
-    hosts.all? { |host| host && host.to_s.start_with?("https") }
+    # Infer the value if neither `ssl_enabled` was not set
+    return if original_params.include?('ssl_enabled')
+    params['ssl_enabled'] = @ssl_enabled ||= Array(@hosts).all? { |host| host && host.to_s.start_with?("https") }
   end
 
 end #class LogStash::Filters::Elasticsearch

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.16.1'
+  s.version         = '4.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -24,7 +24,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'elasticsearch', ">= 7.14.9" # LS >= 6.7 and < 7.14 all used version 5.0.5
   s.add_runtime_dependency 'manticore', ">= 0.7.1"
   s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
-  s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'
   s.add_development_dependency 'cabin', ['~> 0.6']
   s.add_development_dependency 'webrick'
   s.add_development_dependency 'logstash-devutils'

data/spec/filters/elasticsearch_spec.rb

@@ -644,7 +644,7 @@ describe LogStash::Filters::Elasticsearch do
     end
   end
 
-  describe "Elastic Api Header" do
+  describe "Elastic Api and Product Origin Headers" do
     let(:config) { {"query" => "*"} }
     let(:plugin) { described_class.new(config) }
     let(:headers) {{'x-elastic-product' => 'Elasticsearch'}}
@@ -666,6 +666,8 @@ describe LogStash::Filters::Elasticsearch do
         plugin.register
         client = plugin.send(:get_client).client
         expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_including("Elastic-Api-Version" => "2023-10-31")
+        expect( extract_transport(client).options[:transport_options][:headers] )
+          .to match hash_including("x-elastic-product-origin" => "logstash-filter-elasticsearch")
       end
     end
 
@@ -676,10 +678,12 @@ describe LogStash::Filters::Elasticsearch do
         expect_any_instance_of(Elasticsearch::Client).to receive(:perform_request).with(any_args).and_return(mock_resp)
       end
 
-      it 'does not propagate header to es client' do
+      it 'does not propagate Elastic-Api-Version header to es client' do
         plugin.register
         client = plugin.send(:get_client).client
         expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_not_including("Elastic-Api-Version" => "2023-10-31")
+        expect( extract_transport(client).options[:transport_options][:headers] )
+          .to match hash_including("x-elastic-product-origin" => "logstash-filter-elasticsearch")
       end
     end
 

data/spec/filters/elasticsearch_ssl_spec.rb

@@ -24,6 +24,23 @@ describe "SSL options" do
     subject.close
   end
 
+  describe "obsolete settings" do
+    [{:name => 'ca_file', :canonical_name => 'ssl_certificate_authorities'},
+     {:name => "keystore", :canonical_name => 'ssl_keystore_path'},
+     {:name => "keystore_password", :canonical_name => "ssl_keystore_password"},
+     {:name => "ssl", :canonical_name => "ssl_enabled"}
+    ].each do |config_settings|
+      context "with option #{config_settings[:name]}" do
+        let(:obsolete_config) { settings.merge(config_settings[:name] => 'test_value') }
+        it "emits an error about the setting `#{config_settings[:name]}` now being obsolete and provides guidance to use `#{config_settings[:canonical_name]}`" do
+          error_text = /The setting `#{config_settings[:name]}` in plugin `elasticsearch` is obsolete and is no longer available. Set '#{config_settings[:canonical_name]}' instead/i
+          expect { LogStash::Filters::Elasticsearch.new(obsolete_config) }.to raise_error LogStash::ConfigurationError, error_text
+        end
+
+      end
+    end
+  end
+
   context "when ssl_enabled is" do
     context "true and there is no https hosts" do
       let(:hosts) { %w[http://es01 http://es01] }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.16.1
+  version: 4.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-09-29 00:00:00.000000000 Z
+date: 2025-01-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -20,8 +20,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -37,8 +37,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 7.14.9
   name: elasticsearch
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -51,8 +51,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.7.1
   name: manticore
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -65,22 +65,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-ca_trusted_fingerprint_support
-  prerelease: false
   type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  name: logstash-mixin-normalize_config_support
   prerelease: false
-  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -93,8 +79,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.6'
   name: cabin
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -107,8 +93,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: webrick
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -121,8 +107,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -195,7 +181,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.33
+rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
 summary: Copies fields from previous log events in Elasticsearch to current events