logstash-filter-elasticsearch 4.2.0 → 4.3.0

data/lib/logstash/filters/elasticsearch.rb

@@ -12,6 +12,9 @@ require_relative "elasticsearch/client"
 
 class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
 
+  require 'logstash/filters/elasticsearch/dsl_executor'
+  require 'logstash/filters/elasticsearch/esql_executor'
+
   include LogStash::PluginMixins::ECSCompatibilitySupport
   include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
 
@@ -24,8 +27,13 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Field substitution (e.g. `index-name-%{date_field}`) is available
   config :index, :validate => :string, :default => ""
 
-  # Elasticsearch query string. Read the Elasticsearch query string documentation.
-  # for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl-query-string-query.html#query-string-syntax
+  # A type of Elasticsearch query, provided by @query.
+  config :query_type, :validate => %w[esql dsl], :default => "dsl"
+
+  # Elasticsearch query string. This can be in DSL or ES|QL query shape defined by @query_type.
+  # Read the Elasticsearch query string documentation.
+  #   DSL: https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl-query-string-query.html#query-string-syntax
+  #   ES|QL: https://www.elastic.co/guide/en/elasticsearch/reference/current/esql.html
   config :query, :validate => :string
 
   # File path to elasticsearch query in DSL format. Read the Elasticsearch query documentation
@@ -125,7 +133,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Tags the event on failure to look up geo information. This can be used in later analysis.
   config :tag_on_failure, :validate => :array, :default => ["_elasticsearch_lookup_failure"]
 
-  # If set, the the result set will be nested under the target field
+  # If set, the result set will be nested under the target field
   config :target, :validate => :field_reference
 
   # How many times to retry on failure?
@@ -134,6 +142,15 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # What status codes to retry on?
   config :retry_on_status, :validate => :number, :list => true, :default => [500, 502, 503, 504]
 
+  # named placeholders in ES|QL query
+  # example,
+  #   if the query is "FROM my-index | WHERE some_type = ?type AND depth > ?min_depth"
+  #   named placeholders can be applied as the following in query_params:
+  #   query_params => [
+  #     {"type" => "%{[type]}"}
+  #     {"min_depth" => "%{[depth]}"}
+  #   ]
+  config :query_params, :validate => :array, :default => []
 
   config :ssl, :obsolete => "Set 'ssl_enabled' instead."
   config :ca_file, :obsolete => "Set 'ssl_certificate_authorities' instead."
@@ -146,6 +163,9 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   include MonitorMixin
   attr_reader :shared_client
 
+  LS_ESQL_SUPPORT_VERSION = "8.17.4" # the version started using elasticsearch-ruby v8
+  ES_ESQL_SUPPORT_VERSION = "8.11.0"
+
   ##
   # @override to handle proxy => '' as if none was set
   # @param value [Array<Object>]
@@ -163,17 +183,22 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     return super(value, :uri)
   end
 
+  attr_reader :query_dsl
+
   def register
-    #Load query if it exists
-    if @query_template
-      if File.zero?(@query_template)
-        raise "template is empty"
-      end
-      file = File.open(@query_template, 'r')
-      @query_dsl = file.read
+    case @query_type
+    when "esql"
+      invalid_params_with_esql = original_params.keys & %w(index query_template sort fields docinfo_fields aggregation_fields enable_sort result_size)
+      raise LogStash::ConfigurationError, "Configured #{invalid_params_with_esql} params cannot be used with ES|QL query" if invalid_params_with_esql.any?
+
+      validate_ls_version_for_esql_support!
+      validate_esql_query_and_params!
+      @esql_executor ||= LogStash::Filters::Elasticsearch::EsqlExecutor.new(self, @logger)
+    else # dsl
+      validate_dsl_query_settings!
+      @esql_executor ||= LogStash::Filters::Elasticsearch::DslExecutor.new(self, @logger)
     end
 
-    validate_query_settings
     fill_hosts_from_cloud_id
     setup_ssl_params!
     validate_authentication
@@ -182,6 +207,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
 
     test_connection!
+    validate_es_for_esql_support! if @query_type == "esql"
     setup_serverless
     if get_client.es_transport_client_type == "elasticsearch_transport"
       require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
@@ -189,71 +215,15 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end # def register
 
   def filter(event)
-    matched = false
-    begin
-      params = { :index => event.sprintf(@index) }
-
-      if @query_dsl
-        query = LogStash::Json.load(event.sprintf(@query_dsl))
-        params[:body] = query
-      else
-        query = event.sprintf(@query)
-        params[:q] = query
-        params[:size] = result_size
-        params[:sort] =  @sort if @enable_sort
-      end
-
-      @logger.debug("Querying elasticsearch for lookup", :params => params)
-
-      results = get_client.search(params)
-      raise "Elasticsearch query error: #{results["_shards"]["failures"]}" if results["_shards"].include? "failures"
-
-      event.set("[@metadata][total_hits]", extract_total_from_hits(results['hits']))
-
-      resultsHits = results["hits"]["hits"]
-      if !resultsHits.nil? && !resultsHits.empty?
-        matched = true
-        @fields.each do |old_key, new_key|
-          old_key_path = extract_path(old_key)
-          extracted_hit_values = resultsHits.map do |doc|
-            extract_value(doc["_source"], old_key_path)
-          end
-          value_to_set = extracted_hit_values.count > 1 ? extracted_hit_values : extracted_hit_values.first
-          set_to_event_target(event, new_key, value_to_set)
-        end
-        @docinfo_fields.each do |old_key, new_key|
-          old_key_path = extract_path(old_key)
-          extracted_docs_info = resultsHits.map do |doc|
-            extract_value(doc, old_key_path)
-          end
-          value_to_set = extracted_docs_info.count > 1 ? extracted_docs_info : extracted_docs_info.first
-          set_to_event_target(event, new_key, value_to_set)
-        end
-      end
-
-      resultsAggs = results["aggregations"]
-      if !resultsAggs.nil? && !resultsAggs.empty?
-        matched = true
-        @aggregation_fields.each do |agg_name, ls_field|
-          set_to_event_target(event, ls_field, resultsAggs[agg_name])
-        end
-      end
-
-    rescue => e
-      if @logger.trace?
-        @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query, :event => event.to_hash, :error => e.message, :backtrace => e.backtrace)
-      elsif @logger.debug?
-        @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :error => e.message, :backtrace => e.backtrace)
-      else
-        @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :error => e.message)
-      end
-      @tag_on_failure.each{|tag| event.tag(tag)}
-    else
-      filter_matched(event) if matched
-    end
+    @esql_executor.process(get_client, event)
   end # def filter
 
-  # public only to be reuse in testing
+  def decorate(event)
+    # this Elasticsearch class has access to `filter_matched`
+    filter_matched(event)
+  end
+
+  # public only to be reused in testing
   def prepare_user_agent
     os_name = java.lang.System.getProperty('os.name')
     os_version = java.lang.System.getProperty('os.version')
@@ -268,18 +238,6 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
 
   private
 
-  # if @target is defined, creates a nested structure to inject result into target field
-  # if not defined, directly sets to the top-level event field
-  # @param event [LogStash::Event]
-  # @param new_key [String] name of the field to set
-  # @param value_to_set [Array] values to set
-  # @return [void]
-  def set_to_event_target(event, new_key, value_to_set)
-    key_to_set = target ? "[#{target}][#{new_key}]" : new_key
-
-    event.set(key_to_set, value_to_set)
-  end
-
   def client_options
     @client_options ||= {
       :user => @user,
@@ -376,53 +334,10 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     end
   end
 
-  # get an array of path elements from a path reference
-  def extract_path(path_reference)
-    return [path_reference] unless path_reference.start_with?('[') && path_reference.end_with?(']')
-
-    path_reference[1...-1].split('][')
-  end
-
-  # given a Hash and an array of path fragments, returns the value at the path
-  # @param source [Hash{String=>Object}]
-  # @param path [Array{String}]
-  # @return [Object]
-  def extract_value(source, path)
-    path.reduce(source) do |memo, old_key_fragment|
-      break unless memo.include?(old_key_fragment)
-      memo[old_key_fragment]
-    end
-  end
-
-  # Given a "hits" object from an Elasticsearch response, return the total number of hits in
-  # the result set.
-  # @param hits [Hash{String=>Object}]
-  # @return [Integer]
-  def extract_total_from_hits(hits)
-    total = hits['total']
-
-    # Elasticsearch 7.x produces an object containing `value` and `relation` in order
-    # to enable unambiguous reporting when the total is only a lower bound; if we get
-    # an object back, return its `value`.
-    return total['value'] if total.kind_of?(Hash)
-
-    total
-  end
-
   def hosts_default?(hosts)
     hosts.is_a?(Array) && hosts.size == 1 && !original_params.key?('hosts')
   end
 
-  def validate_query_settings
-    unless @query || @query_template
-      raise LogStash::ConfigurationError, "Both `query` and `query_template` are empty. Require either `query` or `query_template`."
-    end
-
-    if @query && @query_template
-      raise LogStash::ConfigurationError, "Both `query` and `query_template` are set. Use either `query` or `query_template`."
-    end
-  end
-
   def validate_authentication
     authn_options = 0
     authn_options += 1 if @cloud_auth
@@ -514,4 +429,65 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     params['ssl_enabled'] = @ssl_enabled ||= Array(@hosts).all? { |host| host && host.to_s.start_with?("https") }
   end
 
+  def validate_dsl_query_settings!
+    #Load query if it exists
+    if @query_template
+      if File.zero?(@query_template)
+        raise "template is empty"
+      end
+      file = File.open(@query_template, 'r')
+      @query_dsl = file.read
+    end
+
+    validate_query_settings
+  end
+
+  def validate_query_settings
+    unless @query || @query_template
+      raise LogStash::ConfigurationError, "Both `query` and `query_template` are empty. Require either `query` or `query_template`."
+    end
+
+    if @query && @query_template
+      raise LogStash::ConfigurationError, "Both `query` and `query_template` are set. Use either `query` or `query_template`."
+    end
+
+    if original_params.keys.include?("query_params")
+      raise LogStash::ConfigurationError, "`query_params` is not allowed when `query_type => 'dsl'`."
+    end
+  end
+
+  def validate_ls_version_for_esql_support!
+    if Gem::Version.create(LOGSTASH_VERSION) < Gem::Version.create(LS_ESQL_SUPPORT_VERSION)
+      fail("Current version of Logstash does not include Elasticsearch client which supports ES|QL. Please upgrade Logstash to at least #{LS_ESQL_SUPPORT_VERSION}")
+    end
+  end
+
+  def validate_esql_query_and_params!
+    # If Array, validate that query_params needs to contain only single-entry hashes, convert it to a Hash
+    if @query_params.kind_of?(Array)
+      illegal_entries = @query_params.reject {|e| e.kind_of?(Hash) && e.size == 1 }
+      raise LogStash::ConfigurationError, "`query_params` must contain only single-entry hashes. Illegal placeholders: #{illegal_entries}" if illegal_entries.any?
+
+      @query_params = @query_params.reduce({}, :merge)
+    end
+
+    illegal_keys = @query_params.keys.reject {|k| k[/^[a-z_][a-z0-9_]*$/] }
+    if illegal_keys.any?
+      message = "Illegal #{illegal_keys} placeholder names in `query_params`. A valid parameter name starts with a letter and contains letters, digits and underscores only;"
+      raise LogStash::ConfigurationError, message
+    end
+
+    placeholders = @query.scan(/(?<=[?])[a-z_][a-z0-9_]*/i)
+    placeholders.each do |placeholder|
+      raise LogStash::ConfigurationError, "Placeholder #{placeholder} not found in query" unless @query_params.include?(placeholder)
+    end
+  end
+
+  def validate_es_for_esql_support!
+    # make sure connected ES supports ES|QL (8.11+)
+    @es_version ||= get_client.es_version
+    es_supports_esql = Gem::Version.create(@es_version) >= Gem::Version.create(ES_ESQL_SUPPORT_VERSION)
+    fail("Connected Elasticsearch #{@es_version} version does not supports ES|QL. ES|QL feature requires at least Elasticsearch #{ES_ESQL_SUPPORT_VERSION} version.") unless es_supports_esql
+  end
+
 end #class LogStash::Filters::Elasticsearch

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '4.2.0'
+  s.version         = '4.3.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/elasticsearch_dsl_spec.rb

@@ -0,0 +1,372 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/filters/elasticsearch"
+
+describe LogStash::Filters::Elasticsearch::DslExecutor do
+  let(:client) { instance_double(LogStash::Filters::ElasticsearchClient) }
+  let(:logger) { double("logger") }
+  let(:plugin) { LogStash::Filters::Elasticsearch.new(plugin_config) }
+  let(:plugin_config) do
+    {
+      "index" => "test_index",
+      "query" => "test_query",
+      "fields" => { "field1" => "field1_mapped" },
+      "result_size" => 10,
+      "docinfo_fields" => { "_id" => "doc_id" },
+      "tag_on_failure" => ["_failure"],
+      "enable_sort" => true,
+      "sort" => "@timestamp:desc",
+      "aggregation_fields" => { "agg1" => "agg1_mapped" }
+    }
+  end
+  let(:dsl_executor) { described_class.new(plugin, logger) }
+  let(:event) { LogStash::Event.new({}) }
+
+  describe "#initialize" do
+    it "initializes instance variables correctly" do
+      expect(dsl_executor.instance_variable_get(:@index)).to eq("test_index")
+      expect(dsl_executor.instance_variable_get(:@query)).to eq("test_query")
+      expect(dsl_executor.instance_variable_get(:@query_dsl)).to eq(nil)
+      expect(dsl_executor.instance_variable_get(:@fields)).to eq({ "field1" => "field1_mapped" })
+      expect(dsl_executor.instance_variable_get(:@result_size)).to eq(10)
+      expect(dsl_executor.instance_variable_get(:@docinfo_fields)).to eq({ "_id" => "doc_id" })
+      expect(dsl_executor.instance_variable_get(:@tag_on_failure)).to eq(["_failure"])
+      expect(dsl_executor.instance_variable_get(:@enable_sort)).to eq(true)
+      expect(dsl_executor.instance_variable_get(:@sort)).to eq("@timestamp:desc")
+      expect(dsl_executor.instance_variable_get(:@aggregation_fields)).to eq({ "agg1" => "agg1_mapped" })
+      expect(dsl_executor.instance_variable_get(:@logger)).to eq(logger)
+      expect(dsl_executor.instance_variable_get(:@event_decorator)).not_to be_nil
+    end
+  end
+
+  describe "data fetch" do
+    let(:plugin_config) do
+      {
+        "hosts" => ["localhost:9200"],
+        "query" => "response: 404",
+        "fields" => { "response" => "code" },
+        "docinfo_fields" => { "_index" => "es_index" },
+        "aggregation_fields" => { "bytes_avg" => "bytes_avg_ls_field" }
+      }
+    end
+
+    let(:response) do
+      LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_1.json")))
+    end
+
+    let(:client) { double(:client) }
+
+    before(:each) do
+      allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
+      if defined?(Elastic::Transport)
+        allow(client).to receive(:es_transport_client_type).and_return('elastic_transport')
+      else
+        allow(client).to receive(:es_transport_client_type).and_return('elasticsearch_transport')
+      end
+      allow(client).to receive(:search).and_return(response)
+      allow(plugin).to receive(:test_connection!)
+      allow(plugin).to receive(:setup_serverless)
+      plugin.register
+    end
+
+    after(:each) do
+      Thread.current[:filter_elasticsearch_client] = nil
+    end
+
+    it "should enhance the current event with new data" do
+      plugin.filter(event)
+      expect(event.get("code")).to eq(404)
+      expect(event.get("es_index")).to eq("logstash-2014.08.26")
+      expect(event.get("bytes_avg_ls_field")["value"]).to eq(294)
+    end
+
+    it "should receive all necessary params to perform the search" do
+      expect(client).to receive(:search).with({:q=>"response: 404", :size=>1, :index=>"", :sort=>"@timestamp:desc"})
+      plugin.filter(event)
+    end
+
+    context "when asking to hit specific index" do
+
+      let(:plugin_config) do
+        {
+          "index" => "foo*",
+          "hosts" => ["localhost:9200"],
+          "query" => "response: 404",
+          "fields" => { "response" => "code" }
+        }
+      end
+
+      it "should receive all necessary params to perform the search" do
+        expect(client).to receive(:search).with({:q=>"response: 404", :size=>1, :index=>"foo*", :sort=>"@timestamp:desc"})
+        plugin.filter(event)
+      end
+    end
+
+    context "when asking for more than one result" do
+
+      let(:plugin_config) do
+        {
+          "hosts" => ["localhost:9200"],
+          "query" => "response: 404",
+          "fields" => { "response" => "code" },
+          "result_size" => 10
+        }
+      end
+
+      let(:response) do
+        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_10.json")))
+      end
+
+      it "should enhance the current event with new data" do
+        plugin.filter(event)
+        expect(event.get("code")).to eq([404]*10)
+      end
+    end
+
+    context 'when Elasticsearch 7.x gives us a totals object instead of an integer' do
+      let(:plugin_config) do
+        {
+          "hosts" => ["localhost:9200"],
+          "query" => "response: 404",
+          "fields" => { "response" => "code" },
+          "result_size" => 10
+        }
+      end
+
+      let(:response) do
+        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "elasticsearch_7.x_hits_total_as_object.json")))
+      end
+
+      it "should enhance the current event with new data" do
+        plugin.filter(event)
+        expect(event.get("[@metadata][total_hits]")).to eq(13476)
+      end
+    end
+
+    context "if something wrong happen during connection" do
+
+      before(:each) do
+        allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
+        allow(client).to receive(:search).and_raise("connection exception")
+        plugin.register
+      end
+
+      it "tag the event as something happened, but still deliver it" do
+        expect(plugin.logger).to receive(:warn)
+        plugin.filter(event)
+        expect(event.to_hash["tags"]).to include("_elasticsearch_lookup_failure")
+      end
+    end
+
+    # Tagging test for positive results
+    context "Tagging should occur if query returns results" do
+      let(:plugin_config) do
+        {
+          "index" => "foo*",
+          "hosts" => ["localhost:9200"],
+          "query" => "response: 404",
+          "add_tag" => ["tagged"]
+        }
+      end
+
+      let(:response) do
+        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_10.json")))
+      end
+
+      it "should tag the current event if results returned" do
+        plugin.filter(event)
+        expect(event.to_hash["tags"]).to include("tagged")
+      end
+    end
+
+    context "an aggregation search with size 0 that matches" do
+      let(:plugin_config) do
+        {
+          "index" => "foo*",
+          "hosts" => ["localhost:9200"],
+          "query" => "response: 404",
+          "add_tag" => ["tagged"],
+          "result_size" => 0,
+          "aggregation_fields" => { "bytes_avg" => "bytes_avg_ls_field" }
+        }
+      end
+
+      let(:response) do
+        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_size0_agg.json")))
+      end
+
+      it "should tag the current event" do
+        plugin.filter(event)
+        expect(event.get("tags")).to include("tagged")
+        expect(event.get("bytes_avg_ls_field")["value"]).to eq(294)
+      end
+    end
+
+    # Tagging test for negative results
+    context "Tagging should not occur if query has no results" do
+      let(:plugin_config) do
+        {
+          "index" => "foo*",
+          "hosts" => ["localhost:9200"],
+          "query" => "response: 404",
+          "add_tag" => ["tagged"]
+        }
+      end
+
+      let(:response) do
+        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_error.json")))
+      end
+
+      it "should not tag the current event" do
+        plugin.filter(event)
+        expect(event.to_hash["tags"]).to_not include("tagged")
+      end
+    end
+    context "testing a simple query template" do
+      let(:plugin_config) do
+        {
+          "hosts" => ["localhost:9200"],
+          "query_template" => File.join(File.dirname(__FILE__), "fixtures", "query_template.json"),
+          "fields" => { "response" => "code" },
+          "result_size" => 1
+        }
+      end
+
+      let(:response) do
+        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_1.json")))
+      end
+
+      it "should enhance the current event with new data" do
+        plugin.filter(event)
+        expect(event.get("code")).to eq(404)
+      end
+
+    end
+
+    context "testing a simple index substitution" do
+      let(:event) {
+        LogStash::Event.new(
+          {
+            "subst_field" => "subst_value"
+          }
+        )
+      }
+      let(:plugin_config) do
+        {
+          "index" => "foo_%{subst_field}*",
+          "hosts" => ["localhost:9200"],
+          "query" => "response: 404",
+          "fields" => { "response" => "code" }
+        }
+      end
+
+      it "should receive substituted index name" do
+        expect(client).to receive(:search).with({:q => "response: 404", :size => 1, :index => "foo_subst_value*", :sort => "@timestamp:desc"})
+        plugin.filter(event)
+      end
+    end
+
+    context "if query result errored but no exception is thrown" do
+      let(:response) do
+        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_error.json")))
+      end
+
+      before(:each) do
+        allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
+        allow(client).to receive(:search).and_return(response)
+        plugin.register
+      end
+
+      it "tag the event as something happened, but still deliver it" do
+        expect(plugin.logger).to receive(:warn)
+        plugin.filter(event)
+        expect(event.to_hash["tags"]).to include("_elasticsearch_lookup_failure")
+      end
+    end
+
+    context 'with client-level retries' do
+      let(:plugin_config) do
+        super().merge(
+          "retry_on_failure" => 3,
+          "retry_on_status" => [500]
+        )
+      end
+    end
+
+    context "with custom headers" do
+      let(:plugin_config) do
+        {
+          "query" => "*",
+          "custom_headers" => { "Custom-Header-1" => "Custom Value 1", "Custom-Header-2" => "Custom Value 2" }
+        }
+      end
+
+      let(:plugin) { LogStash::Filters::Elasticsearch.new(plugin_config) }
+      let(:client_double) { double("client") }
+      let(:transport_double) { double("transport", options: { transport_options: { headers: plugin_config["custom_headers"] } }) }
+
+      before do
+        allow(plugin).to receive(:get_client).and_return(client_double)
+        if defined?(Elastic::Transport)
+          allow(client_double).to receive(:es_transport_client_type).and_return('elastic_transport')
+        else
+          allow(client_double).to receive(:es_transport_client_type).and_return('elasticsearch_transport')
+        end
+        allow(client_double).to receive(:client).and_return(transport_double)
+      end
+
+      it "sets custom headers" do
+        plugin.register
+        client = plugin.send(:get_client).client
+        expect(client.options[:transport_options][:headers]).to match(hash_including(plugin_config["custom_headers"]))
+      end
+    end
+
+    context "if query is on nested field" do
+      let(:plugin_config) do
+        {
+          "hosts" => ["localhost:9200"],
+          "query" => "response: 404",
+          "fields" => [ ["[geoip][ip]", "ip_address"] ]
+        }
+      end
+
+      it "should enhance the current event with new data" do
+        plugin.filter(event)
+        expect(event.get("ip_address")).to eq("66.249.73.185")
+      end
+
+    end
+  end
+
+  describe "#set_to_event_target" do
+    it 'is ready to set to `target`' do
+      expect(dsl_executor.apply_target("path")).to eq("path")
+    end
+
+    context "when `@target` is nil, default behavior" do
+      it "sets the value directly to the top-level event field" do
+        dsl_executor.send(:set_to_event_target, event, "new_field", %w[value1 value2])
+        expect(event.get("new_field")).to eq(%w[value1 value2])
+      end
+    end
+
+    context "when @target is defined" do
+      let(:plugin_config) {
+        super().merge({ "target" => "nested" })
+      }
+
+      it "creates a nested structure under the target field" do
+        dsl_executor.send(:set_to_event_target, event, "new_field", %w[value1 value2])
+        expect(event.get("nested")).to eq({ "new_field" => %w[value1 value2] })
+      end
+
+      it "overwrites existing target field with new data" do
+        event.set("nested", { "existing_field" => "existing_value", "new_field" => "value0" })
+        dsl_executor.send(:set_to_event_target, event, "new_field", ["value1"])
+        expect(event.get("nested")).to eq({ "existing_field" => "existing_value", "new_field" => ["value1"] })
+      end
+    end
+  end
+
+end