logstash-filter-elasticsearch 3.17.0 → 3.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aaf8d73f069f6550d53c38b1a27f5444f77b774bbe62d66b7b7dd276be4125fc
-  data.tar.gz: 19b2d5030a25fd03c78baf5edb6b18a649cc0751fa02f38fdfd8a9959669b9bc
+  metadata.gz: f93bd5f45ce46abe35c9fb54f03ce2f2241a6a7e5c530671a2954324f7965b18
+  data.tar.gz: 3bd3ae0babbeb6e46a1e6001888aa85c8fc47d7fb5c794a0063ad7688f3d3bd0
 SHA512:
-  metadata.gz: f3792d01a260e9dcada0ddc7df7aaebbfc1280025781f4c8f2640eaba3abd0d777982f31b794cd7e37b1ce9430bd1f2a60d73dc05ddb8fc0eedd778d9574f1c6
-  data.tar.gz: 0743bccc0a8da98dd91d697ab1b05e95fae2853fd24e8d6db6887ec7ad636de2ae970ff65e15d491388402c702d71adc2190b987a30e5ee2f30a39daf5014954
+  metadata.gz: a59b18ddbb4706e50c94c7af206886b430063370376adee33b5795635d813142e662d55830b8401cd17b2e4ecfc69bace939f2ffd16daf2ecedfc4e1f16e5717
+  data.tar.gz: eb3c6a279badcc7b9fdaea8831cfeff276cde141773d28f9ead912606c816f30320e706efd34799060349bb09df09c8a4e5f05bf5f6ed2ccb9b0d5b19207eaff

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.18.0
+  - Add `target` configuration option to store the result into it [#197](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/197)
+
+## 3.17.1
+  - Add elastic-transport client support used in elasticsearch-ruby 8.x [#193](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/193)
+
 ## 3.17.0
   - Added support for custom headers [#190](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/190)
 

data/docs/index.asciidoc

@@ -160,6 +160,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-ssl_truststore_type>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-ssl_verification_mode>> |<<string,string>>, one of `["full", "none"]`|No
 | <<plugins-{type}s-{plugin}-tag_on_failure>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
 |=======================================================================
 
@@ -173,8 +174,11 @@ filter plugins.
 
   * Value type is <<hash,hash>>
   * Default value is `{}`
+  * Format: `"aggregation_name" => "[path][on][event]"`:
+  ** `aggregation_name`: aggregation name in result from {es}
+  ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation
 
-Hash of aggregation names to copy from elasticsearch response into Logstash event fields
+A mapping of aggregations to copy into the <<plugins-{type}s-{plugin}-target>> of the current event.
 
 Example:
 [source,ruby]
@@ -244,8 +248,11 @@ These custom headers will override any headers previously set by the plugin such
 
   * Value type is <<hash,hash>>
   * Default value is `{}`
+  * Format: `"path.in.source" => "[path][on][event]"`:
+  ** `path.in.source`: field path in document source of result from {es}, using dot-notation
+  ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation
 
-Hash of docinfo fields to copy from old event (found via elasticsearch) into new event
+A mapping of docinfo (`_source`) fields to copy into the <<plugins-{type}s-{plugin}-target>> of the current event.
 
 Example:
 [source,ruby]
@@ -271,9 +278,11 @@ Whether results should be sorted or not
 
   * Value type is <<array,array>>
   * Default value is `{}`
+  * Format: `"path.in.result" => "[path][on][event]"`:
+  ** `path.in.result`: field path in indexed result from {es}, using dot-notation
+  ** `[path][on][event]`: path for where to place the value on the current event, using field-reference notation
 
-An array of fields to copy from the old event (found via elasticsearch) into the
-new event, currently being processed.
+A mapping of indexed fields to copy into the <<plugins-{type}s-{plugin}-target>> of the current event.
 
 In the following example, the values of `@timestamp` and `event_id` on the event
 found via elasticsearch are copied to the current event's
@@ -521,6 +530,43 @@ WARNING: Setting certificate verification to `none` disables many security benef
 
 Tags the event on failure to look up previous log event information. This can be used in later analysis.
 
+[id="plugins-{type}s-{plugin}-target"]
+===== `target`
+
+* Value type is <<string,string>>
+* There is no default value for this setting.
+
+Define the target field for placing the result data.
+If this setting is omitted, the target will be the root (top level) of the event.
+
+The destination fields specified in <<plugins-{type}s-{plugin}-fields>>, <<plugins-{type}s-{plugin}-aggregation_fields>>, and <<plugins-{type}s-{plugin}-docinfo_fields>> are relative to this target.
+
+For example, if you want the data to be put in the `operation` field:
+[source,ruby]
+if [type] == "end" {
+    filter {
+      query => "type:start AND transaction:%{[transactionId]}"
+      elasticsearch {
+        target => "transaction"
+        fields => {
+          "@timestamp" => "started"
+          "transaction_id" => "id"
+        }
+      }
+    }
+}
+
+`fields` fields will be expanded into a data structure in the `target` field, overall shape looks like this:
+[source,ruby]
+    {
+      "transaction" => {
+        "started" => "2025-04-29T12:01:46.263Z"
+        "id" => "1234567890"
+      }
+    }
+
+NOTE: when writing to a field that already exists on the event, the previous value will be overwritten.
+
 [id="plugins-{type}s-{plugin}-user"]
 ===== `user` 
 

data/lib/logstash/filters/elasticsearch/client.rb

@@ -1,7 +1,6 @@
 # encoding: utf-8
 require "elasticsearch"
 require "base64"
-require "elasticsearch/transport/transport/http/manticore"
 
 
 module LogStash
@@ -9,6 +8,7 @@ module LogStash
     class ElasticsearchClient
 
       attr_reader :client
+      attr_reader :es_transport_client_type
 
       BUILD_FLAVOR_SERVERLESS = 'serverless'.freeze
       DEFAULT_EAV_HEADER = { "Elastic-Api-Version" => "2023-10-31" }.freeze
@@ -44,7 +44,7 @@ module LogStash
 
         client_options = {
                        hosts: hosts,
-             transport_class: ::Elasticsearch::Transport::Transport::HTTP::Manticore,
+             transport_class: get_transport_client_class,
            transport_options: transport_options,
                          ssl: ssl_options,
             retry_on_failure: options[:retry_on_failure],
@@ -98,6 +98,20 @@ module LogStash
         token = ::Base64.strict_encode64(api_key.value)
         { 'Authorization' => "ApiKey #{token}" }
       end
+
+      def get_transport_client_class
+        # LS-core includes `elasticsearch` gem. The gem is composed of two separate gems: `elasticsearch-api` and `elasticsearch-transport`
+        # And now `elasticsearch-transport` is old, instead we have `elastic-transport`.
+        # LS-core updated `elasticsearch` > 8: https://github.com/elastic/logstash/pull/17161
+        # Following source bits are for the compatibility to support both `elasticsearch-transport` and `elastic-transport` gems
+        require "elasticsearch/transport/transport/http/manticore"
+        es_transport_client_type = "elasticsearch_transport"
+        ::Elasticsearch::Transport::Transport::HTTP::Manticore
+      rescue ::LoadError
+        require "elastic/transport/transport/http/manticore"
+        es_transport_client_type = "elastic_transport"
+        ::Elastic::Transport::Transport::HTTP::Manticore
+      end
     end
   end
 end

data/lib/logstash/filters/elasticsearch.rb

@@ -2,14 +2,20 @@
 require "logstash/filters/base"
 require "logstash/namespace"
 require "logstash/json"
+require 'logstash/plugin_mixins/ecs_compatibility_support'
+require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
 require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
 require "logstash/plugin_mixins/normalize_config_support"
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
 require "monitor"
 
 require_relative "elasticsearch/client"
-require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
 
 class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
+
+  include LogStash::PluginMixins::ECSCompatibilitySupport
+  include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
+
   config_name "elasticsearch"
 
   # List of elasticsearch hosts to use for querying.
@@ -132,6 +138,9 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Tags the event on failure to look up geo information. This can be used in later analysis.
   config :tag_on_failure, :validate => :array, :default => ["_elasticsearch_lookup_failure"]
 
+  # If set, the the result set will be nested under the target field
+  config :target, :validate => :field_reference
+
   # How many times to retry on failure?
   config :retry_on_failure, :validate => :number, :default => 0
 
@@ -183,6 +192,9 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
 
     test_connection!
     setup_serverless
+    if get_client.es_transport_client_type == "elasticsearch_transport"
+      require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
+    end
   end # def register
 
   def filter(event)
@@ -212,17 +224,19 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
         matched = true
         @fields.each do |old_key, new_key|
           old_key_path = extract_path(old_key)
-          set = resultsHits.map do |doc|
+          extracted_hit_values = resultsHits.map do |doc|
             extract_value(doc["_source"], old_key_path)
           end
-          event.set(new_key, set.count > 1 ? set : set.first)
+          value_to_set = extracted_hit_values.count > 1 ? extracted_hit_values : extracted_hit_values.first
+          set_to_event_target(event, new_key, value_to_set)
         end
         @docinfo_fields.each do |old_key, new_key|
           old_key_path = extract_path(old_key)
-          set = resultsHits.map do |doc|
+          extracted_docs_info = resultsHits.map do |doc|
             extract_value(doc, old_key_path)
           end
-          event.set(new_key, set.count > 1 ? set : set.first)
+          value_to_set = extracted_docs_info.count > 1 ? extracted_docs_info : extracted_docs_info.first
+          set_to_event_target(event, new_key, value_to_set)
         end
       end
 
@@ -230,7 +244,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       if !resultsAggs.nil? && !resultsAggs.empty?
         matched = true
         @aggregation_fields.each do |agg_name, ls_field|
-          event.set(ls_field, resultsAggs[agg_name])
+          set_to_event_target(event, ls_field, resultsAggs[agg_name])
         end
       end
 
@@ -263,6 +277,18 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
 
   private
 
+  # if @target is defined, creates a nested structure to inject result into target field
+  # if not defined, directly sets to the top-level event field
+  # @param event [LogStash::Event]
+  # @param new_key [String] name of the field to set
+  # @param value_to_set [Array] values to set
+  # @return [void]
+  def set_to_event_target(event, new_key, value_to_set)
+    key_to_set = target ? "[#{target}][#{new_key}]" : new_key
+
+    event.set(key_to_set, value_to_set)
+  end
+
   def client_options
     @client_options ||= {
       :user => @user,

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.17.0'
+  s.version         = '3.18.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,10 +21,12 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'elasticsearch', ">= 7.14.9" # LS >= 6.7 and < 7.14 all used version 5.0.5
+  s.add_runtime_dependency 'elasticsearch', ">= 7.14.9", '< 9'
   s.add_runtime_dependency 'manticore', ">= 0.7.1"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
   s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
   s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'
+  s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
   s.add_development_dependency 'cabin', ['~> 0.6']
   s.add_development_dependency 'webrick'
   s.add_development_dependency 'logstash-devutils'

data/spec/filters/elasticsearch_spec.rb

@@ -60,9 +60,18 @@ describe LogStash::Filters::Elasticsearch do
         allow(plugin).to receive(:get_client).and_return(filter_client)
         allow(filter_client).to receive(:serverless?).and_return(true)
         allow(filter_client).to receive(:client).and_return(es_client)
-        allow(es_client).to receive(:info).with(a_hash_including(:headers => LogStash::Filters::ElasticsearchClient::DEFAULT_EAV_HEADER)).and_raise(
-          Elasticsearch::Transport::Transport::Errors::BadRequest.new
-        )
+
+        if elastic_ruby_v8_client_available?
+          allow(es_client).to receive(:info)
+                                .with(a_hash_including(
+                                        :headers => LogStash::Filters::ElasticsearchClient::DEFAULT_EAV_HEADER))
+                                .and_raise(Elastic::Transport::Transport::Errors::BadRequest.new)
+        else
+          allow(es_client).to receive(:info)
+                                .with(a_hash_including(
+                                        :headers => LogStash::Filters::ElasticsearchClient::DEFAULT_EAV_HEADER))
+                                .and_raise(Elasticsearch::Transport::Transport::Errors::BadRequest.new)
+        end
       end
 
       it "raises an exception when Elastic Api Version is not supported" do
@@ -103,6 +112,11 @@ describe LogStash::Filters::Elasticsearch do
 
     before(:each) do
       allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
+      if elastic_ruby_v8_client_available?
+        allow(client).to receive(:es_transport_client_type).and_return('elastic_transport')
+      else
+        allow(client).to receive(:es_transport_client_type).and_return('elasticsearch_transport')
+      end
       allow(client).to receive(:search).and_return(response)
       allow(plugin).to receive(:test_connection!)
       allow(plugin).to receive(:setup_serverless)
@@ -347,6 +361,11 @@ describe LogStash::Filters::Elasticsearch do
 
       before do
         allow(plugin).to receive(:get_client).and_return(client_double)
+        if elastic_ruby_v8_client_available?
+          allow(client_double).to receive(:es_transport_client_type).and_return('elastic_transport')
+        else
+          allow(client_double).to receive(:es_transport_client_type).and_return('elasticsearch_transport')
+        end
         allow(client_double).to receive(:client).and_return(transport_double)
       end
 
@@ -506,7 +525,12 @@ describe LogStash::Filters::Elasticsearch do
       # this spec is a safeguard to trigger an assessment of thread-safety should
       # we choose a different transport adapter in the future.
       transport_class = extract_transport(client).options.fetch(:transport_class)
-      expect(transport_class).to equal ::Elasticsearch::Transport::Transport::HTTP::Manticore
+      if elastic_ruby_v8_client_available?
+        allow(client).to receive(:es_transport_client_type).and_return("elastic_transport")
+        expect(transport_class).to equal ::Elastic::Transport::Transport::HTTP::Manticore
+      else
+        expect(transport_class).to equal ::Elasticsearch::Transport::Transport::HTTP::Manticore
+      end
     end
 
     it 'uses a client with sufficient connection pool size' do
@@ -821,6 +845,11 @@ describe LogStash::Filters::Elasticsearch do
 
     before(:each) do
       allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
+      if elastic_ruby_v8_client_available?
+        allow(client).to receive(:es_transport_client_type).and_return('elastic_transport')
+      else
+        allow(client).to receive(:es_transport_client_type).and_return('elasticsearch_transport')
+      end
       allow(plugin).to receive(:test_connection!)
       allow(plugin).to receive(:setup_serverless)
       plugin.register
@@ -835,11 +864,46 @@ describe LogStash::Filters::Elasticsearch do
     end
   end
 
-  # @note can be removed once gem depends on elasticsearch >= 6.x
-  def extract_transport(client) # on 7.x client.transport is a ES::Transport::Client
+  describe "#set_to_event_target" do
+
+    context "when `@target` is nil, default behavior" do
+      let(:config) {{ }}
+
+      it "sets the value directly to the top-level event field" do
+        plugin.send(:set_to_event_target, event, "new_field", %w[value1 value2])
+        expect(event.get("new_field")).to eq(%w[value1 value2])
+      end
+    end
+
+    context "when @target is defined" do
+      let(:config) {{ "target" => "nested" }}
+
+      it "creates a nested structure under the target field" do
+        plugin.send(:set_to_event_target, event, "new_field", %w[value1 value2])
+        expect(event.get("nested")).to eq({ "new_field" => %w[value1 value2] })
+      end
+
+      it "overwrites existing target field with new data" do
+        event.set("nested", { "existing_field" => "existing_value", "new_field" => "value0" })
+        plugin.send(:set_to_event_target, event, "new_field", ["value1"])
+        expect(event.get("nested")).to eq({ "existing_field" => "existing_value", "new_field" => ["value1"] })
+      end
+    end
+  end
+
+  def extract_transport(client)
+    # on 7x: client.transport.transport
+    # on >=8.x: client.transport
     client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
   end
 
+  def elastic_ruby_v8_client_available?
+    Elasticsearch::Transport
+    false
+  rescue NameError # NameError: uninitialized constant Elasticsearch::Transport if Elastic Ruby client is not available
+    true
+  end
+
   class MockResponse
     attr_reader :code, :headers
 

data/spec/filters/integration/elasticsearch_spec.rb

@@ -84,7 +84,9 @@ describe LogStash::Filters::Elasticsearch, :integration => true do
     end
 
     it "fails to register plugin" do
-      expect { plugin.register }.to raise_error Elasticsearch::Transport::Transport::Errors::Unauthorized
+      expect { plugin.register }.to raise_error elastic_ruby_v8_client_available? ?
+                                                  Elastic::Transport::Transport::Errors::Unauthorized :
+                                                  Elasticsearch::Transport::Transport::Errors::Unauthorized
     end
 
   end if ELASTIC_SECURITY_ENABLED
@@ -150,5 +152,10 @@ describe LogStash::Filters::Elasticsearch, :integration => true do
       end
     end
   end
-
+  def elastic_ruby_v8_client_available?
+    Elasticsearch::Transport
+    false
+  rescue NameError # NameError: uninitialized constant Elasticsearch::Transport if Elastic Ruby client is not available
+    true
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.17.0
+  version: 3.18.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-01-29 00:00:00.000000000 Z
+date: 2025-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -36,6 +36,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 7.14.9
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
   name: elasticsearch
   type: :runtime
   prerelease: false
@@ -44,6 +47,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 7.14.9
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -58,6 +64,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.7.1
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  name: logstash-mixin-ecs_compatibility_support
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -86,6 +106,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-validator_support
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: