logstash-filter-elasticsearch 3.18.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f93bd5f45ce46abe35c9fb54f03ce2f2241a6a7e5c530671a2954324f7965b18
-  data.tar.gz: 3bd3ae0babbeb6e46a1e6001888aa85c8fc47d7fb5c794a0063ad7688f3d3bd0
+  metadata.gz: 015a98dbd36122dd3fc4c74da5744a31f6182f67023f802189cc73e837d5ba7a
+  data.tar.gz: 8f1c3a79c0af3fc4154501d16bf775e01b6aa7575627852cb18954a0dd952d91
 SHA512:
-  metadata.gz: a59b18ddbb4706e50c94c7af206886b430063370376adee33b5795635d813142e662d55830b8401cd17b2e4ecfc69bace939f2ffd16daf2ecedfc4e1f16e5717
-  data.tar.gz: eb3c6a279badcc7b9fdaea8831cfeff276cde141773d28f9ead912606c816f30320e706efd34799060349bb09df09c8a4e5f05bf5f6ed2ccb9b0d5b19207eaff
+  metadata.gz: fc3971541568e34c0ef644c250243fefcb48c778b2471f81f5e87a1b59535a6f5ccc2a2c13d57f5123e0577efb80962b85f8096d3d6f70e0721df4f716d705e1
+  data.tar.gz: 5fe74aa6d8179e6dd3d9c69a6d0c81f284035bd82343543132f129be7bfb3355e01678eddbd051d9d4c360e9704f8a234737096e757f710b395b5d366f57936f

data/CHANGELOG.md

@@ -1,11 +1,12 @@
-## 3.18.0
-  - Add `target` configuration option to store the result into it [#197](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/197)
-
-## 3.17.1
-  - Add elastic-transport client support used in elasticsearch-ruby 8.x [#193](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/193)
-
-## 3.17.0
-  - Added support for custom headers [#190](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/190)
+## 4.0.0
+  - SSL settings that were marked deprecated in version `3.15.0` are now marked obsolete, and will prevent the plugin from starting.
+  - These settings are:
+    - `ca_file`, which should be replaced by `ssl_certificate_authorities`
+    - `keystore`, which should be replaced by `ssl_keystore_path`
+    - `keystore_password`, which should be replaced by `ssl_keystore_password`
+    - `keystore_type`, which should be replaced by `ssl_keystore_password`
+    -  `ssl`, which should be replaced by `ssl_enabled`
+    - [#183](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/183)
 
 ## 3.16.2
   - Add `x-elastic-product-origin` header to Elasticsearch requests [#185](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/185)

data/docs/index.asciidoc

@@ -110,7 +110,7 @@ Authentication to a secure Elasticsearch cluster is possible using _one_ of the
 * <<plugins-{type}s-{plugin}-user>> AND <<plugins-{type}s-{plugin}-password>>
 * <<plugins-{type}s-{plugin}-cloud_auth>>
 * <<plugins-{type}s-{plugin}-api_key>>
-* <<plugins-{type}s-{plugin}-keystore>> and/or <<plugins-{type}s-{plugin}-keystore_password>>
+* <<plugins-{type}s-{plugin}-ssl_keystore_path>> and/or <<plugins-{type}s-{plugin}-ssl_keystore_password>>
 
 [id="plugins-{type}s-{plugin}-autz"]
 ==== Authorization
@@ -121,7 +121,10 @@ The `monitoring` permission at cluster level is necessary to perform periodic co
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Filter Configuration Options
 
-This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> and the <<plugins-{type}s-{plugin}-deprecated-options>> described later.
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+NOTE: As of version `4.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed. Please see the
+<<plugins-{type}s-{plugin}-obsolete-options>> for more details.
 
 [cols="<,<,<",options="header",]
 |=======================================================================
@@ -131,7 +134,6 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-ca_trusted_fingerprint>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-custom_headers>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-docinfo_fields>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-enable_sort>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-fields>> |<<array,array>>|No
@@ -145,7 +147,6 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-retry_on_failure>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-retry_on_status>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-sort>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|__Deprecated__
 | <<plugins-{type}s-{plugin}-ssl_certificate>> |<<path,path>>|No
 | <<plugins-{type}s-{plugin}-ssl_certificate_authorities>> |list of <<path,path>>|No
 | <<plugins-{type}s-{plugin}-ssl_cipher_suites>> |list of <<string,string>>|No
@@ -160,7 +161,6 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-ssl_truststore_type>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-ssl_verification_mode>> |<<string,string>>, one of `["full", "none"]`|No
 | <<plugins-{type}s-{plugin}-tag_on_failure>> |<<array,array>>|No
-| <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
 |=======================================================================
 
@@ -174,11 +174,8 @@ filter plugins.
 
   * Value type is <<hash,hash>>
   * Default value is `{}`
-  * Format: `"aggregation_name" => "[path][on][event]"`:
-  ** `aggregation_name`: aggregation name in result from {es}
-  ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation
 
-A mapping of aggregations to copy into the <<plugins-{type}s-{plugin}-target>> of the current event.
+Hash of aggregation names to copy from elasticsearch response into Logstash event fields
 
 Example:
 [source,ruby]
@@ -233,26 +230,13 @@ Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
 For more info, check out the
 {logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation].
 
-
-[id="plugins-{type}s-{plugin}-custom_headers"]
-===== `custom_headers`
-
-  * Value type is <<hash,hash>>
-  * Default value is empty
-
-Pass a set of key value pairs as the headers sent in each request to Elasticsearch.
-These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers.
-
 [id="plugins-{type}s-{plugin}-docinfo_fields"]
 ===== `docinfo_fields` 
 
   * Value type is <<hash,hash>>
   * Default value is `{}`
-  * Format: `"path.in.source" => "[path][on][event]"`:
-  ** `path.in.source`: field path in document source of result from {es}, using dot-notation
-  ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation
 
-A mapping of docinfo (`_source`) fields to copy into the <<plugins-{type}s-{plugin}-target>> of the current event.
+Hash of docinfo fields to copy from old event (found via elasticsearch) into new event
 
 Example:
 [source,ruby]
@@ -278,11 +262,9 @@ Whether results should be sorted or not
 
   * Value type is <<array,array>>
   * Default value is `{}`
-  * Format: `"path.in.result" => "[path][on][event]"`:
-  ** `path.in.result`: field path in indexed result from {es}, using dot-notation
-  ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation
 
-A mapping of indexed fields to copy into the <<plugins-{type}s-{plugin}-target>> of the current event.
+An array of fields to copy from the old event (found via elasticsearch) into the
+new event, currently being processed.
 
 In the following example, the values of `@timestamp` and `event_id` on the event
 found via elasticsearch are copied to the current event's
@@ -530,43 +512,6 @@ WARNING: Setting certificate verification to `none` disables many security benef
 
 Tags the event on failure to look up previous log event information. This can be used in later analysis.
 
-[id="plugins-{type}s-{plugin}-target"]
-===== `target`
-
-* Value type is <<string,string>>
-* There is no default value for this setting.
-
-Define the target field for placing the result data.
-If this setting is omitted, the target will be the root (top level) of the event.
-
-The destination fields specified in <<plugins-{type}s-{plugin}-fields>>, <<plugins-{type}s-{plugin}-aggregation_fields>>, and <<plugins-{type}s-{plugin}-docinfo_fields>> are relative to this target.
-
-For example, if you want the data to be put in the `operation` field:
-[source,ruby]
-if [type] == "end" {
-    filter {
-      query => "type:start AND transaction:%{[transactionId]}"
-      elasticsearch {
-        target => "transaction"
-        fields => {
-          "@timestamp" => "started"
-          "transaction_id" => "id"
-        }
-      }
-    }
-}
-
-`fields` fields will be expanded into a data structure in the `target` field, overall shape looks like this:
-[source,ruby]
-    {
-      "transaction" => {
-        "started" => "2025-04-29T12:01:46.263Z"
-        "id" => "1234567890"
-      }
-    }
-
-NOTE: when writing to a field that already exists on the event, the previous value will be overwritten.
-
 [id="plugins-{type}s-{plugin}-user"]
 ===== `user` 
 
@@ -576,57 +521,21 @@ NOTE: when writing to a field that already exists on the event, the previous val
 Basic Auth - username
 
 
-[id="plugins-{type}s-{plugin}-deprecated-options"]
-==== Elasticsearch Filter Deprecated Configuration Options
-
-This plugin supports the following deprecated configurations.
+[id="plugins-{type}s-{plugin}-obsolete-options"]
+==== Elasticsearch Filter Obsolete Configuration Options
 
-WARNING: Deprecated options are subject to removal in future releases.
+WARNING: As of version `4.0.0` of this plugin, some configuration options have been replaced.
+The plugin will fail to start if it contains any of these obsolete options.
 
 [cols="<,<,<",options="header",]
 |=======================================================================
-|Setting|Input type|Replaced by
-| <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_certificate_authorities>>
-| <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_keystore_path>>
-| <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|<<plugins-{type}s-{plugin}-ssl_keystore_password>>
+|Setting|Replaced by
+| ca_file |<<plugins-{type}s-{plugin}-ssl_certificate_authorities>>
+| keystore |<<plugins-{type}s-{plugin}-ssl_keystore_path>>
+| keystore_password |<<plugins-{type}s-{plugin}-ssl_keystore_password>>
+| ssl |<<plugins-{type}s-{plugin}-ssl_enabled>>
 |=======================================================================
 
-[id="plugins-{type}s-{plugin}-ca_file"]
-===== `ca_file`
-deprecated[3.15.0, Replaced by <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>]
-
-* Value type is <<path,path>>
-* There is no default value for this setting.
-
-SSL Certificate Authority file
-
-[id="plugins-{type}s-{plugin}-ssl"]
-===== `ssl`
-deprecated[3.15.0, Replaced by <<plugins-{type}s-{plugin}-ssl_enabled>>]
-
-* Value type is <<boolean,boolean>>
-* Default value is `false`
-
-SSL
-
-[id="plugins-{type}s-{plugin}-keystore"]
-===== `keystore`
-deprecated[3.15.0, Replaced by <<plugins-{type}s-{plugin}-ssl_keystore_path>>]
-
-* Value type is <<path,path>>
-* There is no default value for this setting.
-
-The keystore used to present a certificate to the server. It can be either .jks or .p12
-
-[id="plugins-{type}s-{plugin}-keystore_password"]
-===== `keystore_password`
-deprecated[3.15.0, Replaced by <<plugins-{type}s-{plugin}-ssl_keystore_password>>]
-
-* Value type is <<password,password>>
-* There is no default value for this setting.
-
-Set the keystore password
-
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/elasticsearch/client.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "elasticsearch"
 require "base64"
+require "elasticsearch/transport/transport/http/manticore"
 
 
 module LogStash
@@ -8,7 +9,6 @@ module LogStash
     class ElasticsearchClient
 
       attr_reader :client
-      attr_reader :es_transport_client_type
 
       BUILD_FLAVOR_SERVERLESS = 'serverless'.freeze
       DEFAULT_EAV_HEADER = { "Elastic-Api-Version" => "2023-10-31" }.freeze
@@ -20,8 +20,6 @@ module LogStash
         api_key = options.fetch(:api_key, nil)
         proxy = options.fetch(:proxy, nil)
         user_agent = options[:user_agent]
-        custom_headers = options[:custom_headers]
-
 
         transport_options = { }
         transport_options[:headers] = options.fetch(:serverless, false) ?  DEFAULT_EAV_HEADER.dup : {}
@@ -29,7 +27,6 @@ module LogStash
         transport_options[:headers].merge!(setup_api_key(api_key))
         transport_options[:headers].merge!({ 'user-agent' => "#{user_agent}" })
         transport_options[:headers].merge!(INTERNAL_ORIGIN_HEADER)
-        transport_options[:headers].merge!(custom_headers) unless custom_headers.empty?
 
         transport_options[:pool_max] = 1000
         transport_options[:pool_max_per_route] = 100
@@ -44,7 +41,7 @@ module LogStash
 
         client_options = {
                        hosts: hosts,
-             transport_class: get_transport_client_class,
+             transport_class: ::Elasticsearch::Transport::Transport::HTTP::Manticore,
            transport_options: transport_options,
                          ssl: ssl_options,
             retry_on_failure: options[:retry_on_failure],
@@ -98,20 +95,6 @@ module LogStash
         token = ::Base64.strict_encode64(api_key.value)
         { 'Authorization' => "ApiKey #{token}" }
       end
-
-      def get_transport_client_class
-        # LS-core includes `elasticsearch` gem. The gem is composed of two separate gems: `elasticsearch-api` and `elasticsearch-transport`
-        # And now `elasticsearch-transport` is old, instead we have `elastic-transport`.
-        # LS-core updated `elasticsearch` > 8: https://github.com/elastic/logstash/pull/17161
-        # Following source bits are for the compatibility to support both `elasticsearch-transport` and `elastic-transport` gems
-        require "elasticsearch/transport/transport/http/manticore"
-        es_transport_client_type = "elasticsearch_transport"
-        ::Elasticsearch::Transport::Transport::HTTP::Manticore
-      rescue ::LoadError
-        require "elastic/transport/transport/http/manticore"
-        es_transport_client_type = "elastic_transport"
-        ::Elastic::Transport::Transport::HTTP::Manticore
-      end
     end
   end
 end

data/lib/logstash/filters/elasticsearch.rb

@@ -2,20 +2,13 @@
 require "logstash/filters/base"
 require "logstash/namespace"
 require "logstash/json"
-require 'logstash/plugin_mixins/ecs_compatibility_support'
-require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
 require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
-require "logstash/plugin_mixins/normalize_config_support"
-require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
 require "monitor"
 
 require_relative "elasticsearch/client"
+require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
 
 class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
-
-  include LogStash::PluginMixins::ECSCompatibilitySupport
-  include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
-
   config_name "elasticsearch"
 
   # List of elasticsearch hosts to use for querying.
@@ -39,9 +32,6 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Array of fields to copy from old event (found via elasticsearch) into new event
   config :fields, :validate => :array, :default => {}
 
-  # Custom headers for Elasticsearch requests
-  config :custom_headers, :validate => :hash, :default => {}
-
   # Hash of docinfo fields to copy from old event (found via elasticsearch) into new event
   config :docinfo_fields, :validate => :hash, :default => {}
 
@@ -71,18 +61,6 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Set the address of a forward HTTP proxy.
   config :proxy, :validate => :uri_or_empty
 
-  # SSL
-  config :ssl, :validate => :boolean, :default => false, :deprecated => "Set 'ssl_enabled' instead."
-
-  # SSL Certificate Authority file
-  config :ca_file, :validate => :path, :deprecated => "Set 'ssl_certificate_authorities' instead."
-
-  # The keystore used to present a certificate to the server.
-  # It can be either .jks or .p12
-  config :keystore, :validate => :path, :deprecated => "Use 'ssl_keystore_path' instead."
-
-  # Set the keystore password
-  config :keystore_password, :validate => :password, :deprecated => "Use 'ssl_keystore_password' instead."
 
   # OpenSSL-style X.509 certificate certificate to authenticate the client
   config :ssl_certificate, :validate => :path
@@ -138,20 +116,21 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Tags the event on failure to look up geo information. This can be used in later analysis.
   config :tag_on_failure, :validate => :array, :default => ["_elasticsearch_lookup_failure"]
 
-  # If set, the the result set will be nested under the target field
-  config :target, :validate => :field_reference
-
   # How many times to retry on failure?
   config :retry_on_failure, :validate => :number, :default => 0
 
   # What status codes to retry on?
   config :retry_on_status, :validate => :number, :list => true, :default => [500, 502, 503, 504]
 
+
+  config :ssl, :obsolete => "Set 'ssl_enabled' instead."
+  config :ca_file, :obsolete => "Set 'ssl_certificate_authorities' instead."
+  config :keystore, :obsolete => "Set 'ssl_keystore_path' instead."
+  config :keystore_password, :validate => :password, :obsolete => "Set 'ssl_keystore_password' instead."
+
   # config :ca_trusted_fingerprint, :validate => :sha_256_hex
   include LogStash::PluginMixins::CATrustedFingerprintSupport
 
-  include LogStash::PluginMixins::NormalizeConfigSupport
-
   include MonitorMixin
   attr_reader :shared_client
 
@@ -192,9 +171,6 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
 
     test_connection!
     setup_serverless
-    if get_client.es_transport_client_type == "elasticsearch_transport"
-      require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
-    end
   end # def register
 
   def filter(event)
@@ -224,19 +200,17 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
         matched = true
         @fields.each do |old_key, new_key|
           old_key_path = extract_path(old_key)
-          extracted_hit_values = resultsHits.map do |doc|
+          set = resultsHits.map do |doc|
             extract_value(doc["_source"], old_key_path)
           end
-          value_to_set = extracted_hit_values.count > 1 ? extracted_hit_values : extracted_hit_values.first
-          set_to_event_target(event, new_key, value_to_set)
+          event.set(new_key, set.count > 1 ? set : set.first)
         end
         @docinfo_fields.each do |old_key, new_key|
           old_key_path = extract_path(old_key)
-          extracted_docs_info = resultsHits.map do |doc|
+          set = resultsHits.map do |doc|
             extract_value(doc, old_key_path)
           end
-          value_to_set = extracted_docs_info.count > 1 ? extracted_docs_info : extracted_docs_info.first
-          set_to_event_target(event, new_key, value_to_set)
+          event.set(new_key, set.count > 1 ? set : set.first)
         end
       end
 
@@ -244,7 +218,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       if !resultsAggs.nil? && !resultsAggs.empty?
         matched = true
         @aggregation_fields.each do |agg_name, ls_field|
-          set_to_event_target(event, ls_field, resultsAggs[agg_name])
+          event.set(ls_field, resultsAggs[agg_name])
         end
       end
 
@@ -277,18 +251,6 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
 
   private
 
-  # if @target is defined, creates a nested structure to inject result into target field
-  # if not defined, directly sets to the top-level event field
-  # @param event [LogStash::Event]
-  # @param new_key [String] name of the field to set
-  # @param value_to_set [Array] values to set
-  # @return [void]
-  def set_to_event_target(event, new_key, value_to_set)
-    key_to_set = target ? "[#{target}][#{new_key}]" : new_key
-
-    event.set(key_to_set, value_to_set)
-  end
-
   def client_options
     @client_options ||= {
       :user => @user,
@@ -298,8 +260,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       :ssl => client_ssl_options,
       :retry_on_failure => @retry_on_failure,
       :retry_on_status => @retry_on_status,
-      :user_agent => prepare_user_agent,
-      :custom_headers => @custom_headers
+      :user_agent => prepare_user_agent
     }
   end
 
@@ -518,46 +479,9 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end
 
   def setup_ssl_params!
-    @ssl_enabled = normalize_config(:ssl_enabled) do |normalize|
-      normalize.with_deprecated_alias(:ssl)
-    end
-
-    # Infer the value if neither the deprecate `ssl` and `ssl_enabled` were set
-    infer_ssl_enabled_from_hosts
-
-    @ssl_keystore_path = normalize_config(:ssl_keystore_path) do |normalize|
-      normalize.with_deprecated_alias(:keystore)
-    end
-
-    @ssl_keystore_password = normalize_config(:ssl_keystore_password) do |normalize|
-      normalize.with_deprecated_alias(:keystore_password)
-    end
-
-    @ssl_certificate_authorities = normalize_config(:ssl_certificate_authorities) do |normalize|
-      normalize.with_deprecated_mapping(:ca_file) do |ca_file|
-        [ca_file]
-      end
-    end
-
-    params['ssl_enabled'] = @ssl_enabled
-    params['ssl_keystore_path'] = @ssl_keystore_path unless @ssl_keystore_path.nil?
-    params['ssl_keystore_password'] = @ssl_keystore_password unless @ssl_keystore_password.nil?
-    params['ssl_certificate_authorities'] = @ssl_certificate_authorities unless @ssl_certificate_authorities.nil?
-  end
-
-  def infer_ssl_enabled_from_hosts
-    return if original_params.include?('ssl') || original_params.include?('ssl_enabled')
-
-    @ssl_enabled = params['ssl_enabled'] = effectively_ssl?
-  end
-
-  def effectively_ssl?
-    return true if @ssl_enabled
-
-    hosts = Array(@hosts)
-    return false if hosts.nil? || hosts.empty?
-
-    hosts.all? { |host| host && host.to_s.start_with?("https") }
+    # Infer the value if neither `ssl_enabled` was not set
+    return if original_params.include?('ssl_enabled')
+    params['ssl_enabled'] = @ssl_enabled ||= Array(@hosts).all? { |host| host && host.to_s.start_with?("https") }
   end
 
 end #class LogStash::Filters::Elasticsearch

data/logstash-filter-elasticsearch.gemspec

@@ -1,13 +1,13 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.18.0'
+  s.version         = '4.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
   s.email           = 'info@elastic.co'
-  s.homepage        = "https://elastic.co/logstash"
+  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
   s.require_paths = ["lib"]
 
   # Files
@@ -21,12 +21,9 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'elasticsearch', ">= 7.14.9", '< 9'
+  s.add_runtime_dependency 'elasticsearch', ">= 7.14.9" # LS >= 6.7 and < 7.14 all used version 5.0.5
   s.add_runtime_dependency 'manticore', ">= 0.7.1"
-  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
   s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
-  s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'
-  s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
   s.add_development_dependency 'cabin', ['~> 0.6']
   s.add_development_dependency 'webrick'
   s.add_development_dependency 'logstash-devutils'

data/spec/filters/elasticsearch_spec.rb

@@ -60,18 +60,9 @@ describe LogStash::Filters::Elasticsearch do
         allow(plugin).to receive(:get_client).and_return(filter_client)
         allow(filter_client).to receive(:serverless?).and_return(true)
         allow(filter_client).to receive(:client).and_return(es_client)
-
-        if elastic_ruby_v8_client_available?
-          allow(es_client).to receive(:info)
-                                .with(a_hash_including(
-                                        :headers => LogStash::Filters::ElasticsearchClient::DEFAULT_EAV_HEADER))
-                                .and_raise(Elastic::Transport::Transport::Errors::BadRequest.new)
-        else
-          allow(es_client).to receive(:info)
-                                .with(a_hash_including(
-                                        :headers => LogStash::Filters::ElasticsearchClient::DEFAULT_EAV_HEADER))
-                                .and_raise(Elasticsearch::Transport::Transport::Errors::BadRequest.new)
-        end
+        allow(es_client).to receive(:info).with(a_hash_including(:headers => LogStash::Filters::ElasticsearchClient::DEFAULT_EAV_HEADER)).and_raise(
+          Elasticsearch::Transport::Transport::Errors::BadRequest.new
+        )
       end
 
       it "raises an exception when Elastic Api Version is not supported" do
@@ -112,11 +103,6 @@ describe LogStash::Filters::Elasticsearch do
 
     before(:each) do
       allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
-      if elastic_ruby_v8_client_available?
-        allow(client).to receive(:es_transport_client_type).and_return('elastic_transport')
-      else
-        allow(client).to receive(:es_transport_client_type).and_return('elasticsearch_transport')
-      end
       allow(client).to receive(:search).and_return(response)
       allow(plugin).to receive(:test_connection!)
       allow(plugin).to receive(:setup_serverless)
@@ -347,35 +333,6 @@ describe LogStash::Filters::Elasticsearch do
       end
     end
 
-    context "with custom headers" do
-      let(:config) do
-        {
-          "query" => "*",
-          "custom_headers" => { "Custom-Header-1" => "Custom Value 1", "Custom-Header-2" => "Custom Value 2" }
-        }
-      end
-
-      let(:plugin) { LogStash::Filters::Elasticsearch.new(config) }
-      let(:client_double) { double("client") }
-      let(:transport_double) { double("transport", options: { transport_options: { headers: config["custom_headers"] } }) }
-
-      before do
-        allow(plugin).to receive(:get_client).and_return(client_double)
-        if elastic_ruby_v8_client_available?
-          allow(client_double).to receive(:es_transport_client_type).and_return('elastic_transport')
-        else
-          allow(client_double).to receive(:es_transport_client_type).and_return('elasticsearch_transport')
-        end
-        allow(client_double).to receive(:client).and_return(transport_double)
-      end
-
-      it "sets custom headers" do
-        plugin.register
-        client = plugin.send(:get_client).client
-        expect(client.options[:transport_options][:headers]).to match(hash_including(config["custom_headers"]))
-      end
-    end
-
     context "if query is on nested field" do
       let(:config) do
         {
@@ -525,12 +482,7 @@ describe LogStash::Filters::Elasticsearch do
       # this spec is a safeguard to trigger an assessment of thread-safety should
       # we choose a different transport adapter in the future.
       transport_class = extract_transport(client).options.fetch(:transport_class)
-      if elastic_ruby_v8_client_available?
-        allow(client).to receive(:es_transport_client_type).and_return("elastic_transport")
-        expect(transport_class).to equal ::Elastic::Transport::Transport::HTTP::Manticore
-      else
-        expect(transport_class).to equal ::Elasticsearch::Transport::Transport::HTTP::Manticore
-      end
+      expect(transport_class).to equal ::Elasticsearch::Transport::Transport::HTTP::Manticore
     end
 
     it 'uses a client with sufficient connection pool size' do
@@ -845,11 +797,6 @@ describe LogStash::Filters::Elasticsearch do
 
     before(:each) do
       allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
-      if elastic_ruby_v8_client_available?
-        allow(client).to receive(:es_transport_client_type).and_return('elastic_transport')
-      else
-        allow(client).to receive(:es_transport_client_type).and_return('elasticsearch_transport')
-      end
       allow(plugin).to receive(:test_connection!)
       allow(plugin).to receive(:setup_serverless)
       plugin.register
@@ -864,46 +811,11 @@ describe LogStash::Filters::Elasticsearch do
     end
   end
 
-  describe "#set_to_event_target" do
-
-    context "when `@target` is nil, default behavior" do
-      let(:config) {{ }}
-
-      it "sets the value directly to the top-level event field" do
-        plugin.send(:set_to_event_target, event, "new_field", %w[value1 value2])
-        expect(event.get("new_field")).to eq(%w[value1 value2])
-      end
-    end
-
-    context "when @target is defined" do
-      let(:config) {{ "target" => "nested" }}
-
-      it "creates a nested structure under the target field" do
-        plugin.send(:set_to_event_target, event, "new_field", %w[value1 value2])
-        expect(event.get("nested")).to eq({ "new_field" => %w[value1 value2] })
-      end
-
-      it "overwrites existing target field with new data" do
-        event.set("nested", { "existing_field" => "existing_value", "new_field" => "value0" })
-        plugin.send(:set_to_event_target, event, "new_field", ["value1"])
-        expect(event.get("nested")).to eq({ "existing_field" => "existing_value", "new_field" => ["value1"] })
-      end
-    end
-  end
-
-  def extract_transport(client)
-    # on 7x: client.transport.transport
-    # on >=8.x: client.transport
+  # @note can be removed once gem depends on elasticsearch >= 6.x
+  def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
     client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
   end
 
-  def elastic_ruby_v8_client_available?
-    Elasticsearch::Transport
-    false
-  rescue NameError # NameError: uninitialized constant Elasticsearch::Transport if Elastic Ruby client is not available
-    true
-  end
-
   class MockResponse
     attr_reader :code, :headers
 

data/spec/filters/elasticsearch_ssl_spec.rb

@@ -24,6 +24,23 @@ describe "SSL options" do
     subject.close
   end
 
+  describe "obsolete settings" do
+    [{:name => 'ca_file', :canonical_name => 'ssl_certificate_authorities'},
+     {:name => "keystore", :canonical_name => 'ssl_keystore_path'},
+     {:name => "keystore_password", :canonical_name => "ssl_keystore_password"},
+     {:name => "ssl", :canonical_name => "ssl_enabled"}
+    ].each do |config_settings|
+      context "with option #{config_settings[:name]}" do
+        let(:obsolete_config) { settings.merge(config_settings[:name] => 'test_value') }
+        it "emits an error about the setting `#{config_settings[:name]}` now being obsolete and provides guidance to use `#{config_settings[:canonical_name]}`" do
+          error_text = /The setting `#{config_settings[:name]}` in plugin `elasticsearch` is obsolete and is no longer available. Set '#{config_settings[:canonical_name]}' instead/i
+          expect { LogStash::Filters::Elasticsearch.new(obsolete_config) }.to raise_error LogStash::ConfigurationError, error_text
+        end
+
+      end
+    end
+  end
+
   context "when ssl_enabled is" do
     context "true and there is no https hosts" do
       let(:hosts) { %w[http://es01 http://es01] }

data/spec/filters/integration/elasticsearch_spec.rb

@@ -84,9 +84,7 @@ describe LogStash::Filters::Elasticsearch, :integration => true do
     end
 
     it "fails to register plugin" do
-      expect { plugin.register }.to raise_error elastic_ruby_v8_client_available? ?
-                                                  Elastic::Transport::Transport::Errors::Unauthorized :
-                                                  Elasticsearch::Transport::Transport::Errors::Unauthorized
+      expect { plugin.register }.to raise_error Elasticsearch::Transport::Transport::Errors::Unauthorized
     end
 
   end if ELASTIC_SECURITY_ENABLED
@@ -152,10 +150,5 @@ describe LogStash::Filters::Elasticsearch, :integration => true do
       end
     end
   end
-  def elastic_ruby_v8_client_available?
-    Elasticsearch::Transport
-    false
-  rescue NameError # NameError: uninitialized constant Elasticsearch::Transport if Elastic Ruby client is not available
-    true
-  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.18.0
+  version: 4.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-05-07 00:00:00.000000000 Z
+date: 2025-01-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -36,9 +36,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 7.14.9
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '9'
   name: elasticsearch
   type: :runtime
   prerelease: false
@@ -47,9 +44,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 7.14.9
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '9'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -64,20 +58,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.7.1
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.3'
-  name: logstash-mixin-ecs_compatibility_support
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -92,34 +72,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  name: logstash-mixin-normalize_config_support
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  name: logstash-mixin-validator_support
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -208,7 +160,7 @@ files:
 - spec/filters/fixtures/test_certs/ls.der.sha256
 - spec/filters/fixtures/test_certs/ls.key
 - spec/filters/integration/elasticsearch_spec.rb
-homepage: https://elastic.co/logstash
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
 metadata: