logstash-filter-elasticsearch 3.17.1 → 3.19.0

data/lib/logstash/filters/elasticsearch.rb

@@ -2,13 +2,23 @@
 require "logstash/filters/base"
 require "logstash/namespace"
 require "logstash/json"
+require 'logstash/plugin_mixins/ecs_compatibility_support'
+require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
 require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
 require "logstash/plugin_mixins/normalize_config_support"
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
 require "monitor"
 
 require_relative "elasticsearch/client"
 
 class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
+
+  require 'logstash/filters/elasticsearch/dsl_executor'
+  require 'logstash/filters/elasticsearch/esql_executor'
+
+  include LogStash::PluginMixins::ECSCompatibilitySupport
+  include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
+
   config_name "elasticsearch"
 
   # List of elasticsearch hosts to use for querying.
@@ -18,8 +28,13 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Field substitution (e.g. `index-name-%{date_field}`) is available
   config :index, :validate => :string, :default => ""
 
-  # Elasticsearch query string. Read the Elasticsearch query string documentation.
-  # for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl-query-string-query.html#query-string-syntax
+  # A type of Elasticsearch query, provided by @query.
+  config :query_type, :validate => %w[esql dsl], :default => "dsl"
+
+  # Elasticsearch query string. This can be in DSL or ES|QL query shape defined by @query_type.
+  # Read the Elasticsearch query string documentation.
+  #   DSL: https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl-query-string-query.html#query-string-syntax
+  #   ES|QL: https://www.elastic.co/guide/en/elasticsearch/reference/current/esql.html
   config :query, :validate => :string
 
   # File path to elasticsearch query in DSL format. Read the Elasticsearch query documentation
@@ -131,12 +146,25 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Tags the event on failure to look up geo information. This can be used in later analysis.
   config :tag_on_failure, :validate => :array, :default => ["_elasticsearch_lookup_failure"]
 
+  # If set, the result set will be nested under the target field
+  config :target, :validate => :field_reference
+
   # How many times to retry on failure?
   config :retry_on_failure, :validate => :number, :default => 0
 
   # What status codes to retry on?
   config :retry_on_status, :validate => :number, :list => true, :default => [500, 502, 503, 504]
 
+  # named placeholders in ES|QL query
+  # example,
+  #   if the query is "FROM my-index | WHERE some_type = ?type AND depth > ?min_depth"
+  #   named placeholders can be applied as the following in query_params:
+  #   query_params => [
+  #     {"type" => "%{[type]}"}
+  #     {"min_depth" => "%{[depth]}"}
+  #   ]
+  config :query_params, :validate => :array, :default => []
+
   # config :ca_trusted_fingerprint, :validate => :sha_256_hex
   include LogStash::PluginMixins::CATrustedFingerprintSupport
 
@@ -145,6 +173,9 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   include MonitorMixin
   attr_reader :shared_client
 
+  LS_ESQL_SUPPORT_VERSION = "8.17.4" # the version started using elasticsearch-ruby v8
+  ES_ESQL_SUPPORT_VERSION = "8.11.0"
+
   ##
   # @override to handle proxy => '' as if none was set
   # @param value [Array<Object>]
@@ -162,17 +193,22 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     return super(value, :uri)
   end
 
+  attr_reader :query_dsl
+
   def register
-    #Load query if it exists
-    if @query_template
-      if File.zero?(@query_template)
-        raise "template is empty"
-      end
-      file = File.open(@query_template, 'r')
-      @query_dsl = file.read
+    case @query_type
+    when "esql"
+      invalid_params_with_esql = original_params.keys & %w(index query_template sort fields docinfo_fields aggregation_fields enable_sort result_size)
+      raise LogStash::ConfigurationError, "Configured #{invalid_params_with_esql} params cannot be used with ES|QL query" if invalid_params_with_esql.any?
+
+      validate_ls_version_for_esql_support!
+      validate_esql_query_and_params!
+      @esql_executor ||= LogStash::Filters::Elasticsearch::EsqlExecutor.new(self, @logger)
+    else # dsl
+      validate_dsl_query_settings!
+      @esql_executor ||= LogStash::Filters::Elasticsearch::DslExecutor.new(self, @logger)
     end
 
-    validate_query_settings
     fill_hosts_from_cloud_id
     setup_ssl_params!
     validate_authentication
@@ -181,6 +217,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
 
     test_connection!
+    validate_es_for_esql_support! if @query_type == "esql"
     setup_serverless
     if get_client.es_transport_client_type == "elasticsearch_transport"
       require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
@@ -188,69 +225,15 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end # def register
 
   def filter(event)
-    matched = false
-    begin
-      params = { :index => event.sprintf(@index) }
-
-      if @query_dsl
-        query = LogStash::Json.load(event.sprintf(@query_dsl))
-        params[:body] = query
-      else
-        query = event.sprintf(@query)
-        params[:q] = query
-        params[:size] = result_size
-        params[:sort] =  @sort if @enable_sort
-      end
-
-      @logger.debug("Querying elasticsearch for lookup", :params => params)
-
-      results = get_client.search(params)
-      raise "Elasticsearch query error: #{results["_shards"]["failures"]}" if results["_shards"].include? "failures"
-
-      event.set("[@metadata][total_hits]", extract_total_from_hits(results['hits']))
-
-      resultsHits = results["hits"]["hits"]
-      if !resultsHits.nil? && !resultsHits.empty?
-        matched = true
-        @fields.each do |old_key, new_key|
-          old_key_path = extract_path(old_key)
-          set = resultsHits.map do |doc|
-            extract_value(doc["_source"], old_key_path)
-          end
-          event.set(new_key, set.count > 1 ? set : set.first)
-        end
-        @docinfo_fields.each do |old_key, new_key|
-          old_key_path = extract_path(old_key)
-          set = resultsHits.map do |doc|
-            extract_value(doc, old_key_path)
-          end
-          event.set(new_key, set.count > 1 ? set : set.first)
-        end
-      end
-
-      resultsAggs = results["aggregations"]
-      if !resultsAggs.nil? && !resultsAggs.empty?
-        matched = true
-        @aggregation_fields.each do |agg_name, ls_field|
-          event.set(ls_field, resultsAggs[agg_name])
-        end
-      end
-
-    rescue => e
-      if @logger.trace?
-        @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query, :event => event.to_hash, :error => e.message, :backtrace => e.backtrace)
-      elsif @logger.debug?
-        @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :error => e.message, :backtrace => e.backtrace)
-      else
-        @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :error => e.message)
-      end
-      @tag_on_failure.each{|tag| event.tag(tag)}
-    else
-      filter_matched(event) if matched
-    end
+    @esql_executor.process(get_client, event)
   end # def filter
 
-  # public only to be reuse in testing
+  def decorate(event)
+    # this Elasticsearch class has access to `filter_matched`
+    filter_matched(event)
+  end
+
+  # public only to be reused in testing
   def prepare_user_agent
     os_name = java.lang.System.getProperty('os.name')
     os_version = java.lang.System.getProperty('os.version')
@@ -361,53 +344,10 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     end
   end
 
-  # get an array of path elements from a path reference
-  def extract_path(path_reference)
-    return [path_reference] unless path_reference.start_with?('[') && path_reference.end_with?(']')
-
-    path_reference[1...-1].split('][')
-  end
-
-  # given a Hash and an array of path fragments, returns the value at the path
-  # @param source [Hash{String=>Object}]
-  # @param path [Array{String}]
-  # @return [Object]
-  def extract_value(source, path)
-    path.reduce(source) do |memo, old_key_fragment|
-      break unless memo.include?(old_key_fragment)
-      memo[old_key_fragment]
-    end
-  end
-
-  # Given a "hits" object from an Elasticsearch response, return the total number of hits in
-  # the result set.
-  # @param hits [Hash{String=>Object}]
-  # @return [Integer]
-  def extract_total_from_hits(hits)
-    total = hits['total']
-
-    # Elasticsearch 7.x produces an object containing `value` and `relation` in order
-    # to enable unambiguous reporting when the total is only a lower bound; if we get
-    # an object back, return its `value`.
-    return total['value'] if total.kind_of?(Hash)
-
-    total
-  end
-
   def hosts_default?(hosts)
     hosts.is_a?(Array) && hosts.size == 1 && !original_params.key?('hosts')
   end
 
-  def validate_query_settings
-    unless @query || @query_template
-      raise LogStash::ConfigurationError, "Both `query` and `query_template` are empty. Require either `query` or `query_template`."
-    end
-
-    if @query && @query_template
-      raise LogStash::ConfigurationError, "Both `query` and `query_template` are set. Use either `query` or `query_template`."
-    end
-  end
-
   def validate_authentication
     authn_options = 0
     authn_options += 1 if @cloud_auth
@@ -536,4 +476,65 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     hosts.all? { |host| host && host.to_s.start_with?("https") }
   end
 
+  def validate_dsl_query_settings!
+    #Load query if it exists
+    if @query_template
+      if File.zero?(@query_template)
+        raise "template is empty"
+      end
+      file = File.open(@query_template, 'r')
+      @query_dsl = file.read
+    end
+
+    validate_query_settings
+  end
+
+  def validate_query_settings
+    unless @query || @query_template
+      raise LogStash::ConfigurationError, "Both `query` and `query_template` are empty. Require either `query` or `query_template`."
+    end
+
+    if @query && @query_template
+      raise LogStash::ConfigurationError, "Both `query` and `query_template` are set. Use either `query` or `query_template`."
+    end
+
+    if original_params.keys.include?("query_params")
+      raise LogStash::ConfigurationError, "`query_params` is not allowed when `query_type => 'dsl'`."
+    end
+  end
+
+  def validate_ls_version_for_esql_support!
+    if Gem::Version.create(LOGSTASH_VERSION) < Gem::Version.create(LS_ESQL_SUPPORT_VERSION)
+      fail("Current version of Logstash does not include Elasticsearch client which supports ES|QL. Please upgrade Logstash to at least #{LS_ESQL_SUPPORT_VERSION}")
+    end
+  end
+
+  def validate_esql_query_and_params!
+    # If Array, validate that query_params needs to contain only single-entry hashes, convert it to a Hash
+    if @query_params.kind_of?(Array)
+      illegal_entries = @query_params.reject {|e| e.kind_of?(Hash) && e.size == 1 }
+      raise LogStash::ConfigurationError, "`query_params` must contain only single-entry hashes. Illegal placeholders: #{illegal_entries}" if illegal_entries.any?
+
+      @query_params = @query_params.reduce({}, :merge)
+    end
+
+    illegal_keys = @query_params.keys.reject {|k| k[/^[a-z_][a-z0-9_]*$/] }
+    if illegal_keys.any?
+      message = "Illegal #{illegal_keys} placeholder names in `query_params`. A valid parameter name starts with a letter and contains letters, digits and underscores only;"
+      raise LogStash::ConfigurationError, message
+    end
+
+    placeholders = @query.scan(/(?<=[?])[a-z_][a-z0-9_]*/i)
+    placeholders.each do |placeholder|
+      raise LogStash::ConfigurationError, "Placeholder #{placeholder} not found in query" unless @query_params.include?(placeholder)
+    end
+  end
+
+  def validate_es_for_esql_support!
+    # make sure connected ES supports ES|QL (8.11+)
+    @es_version ||= get_client.es_version
+    es_supports_esql = Gem::Version.create(@es_version) >= Gem::Version.create(ES_ESQL_SUPPORT_VERSION)
+    fail("Connected Elasticsearch #{@es_version} version does not supports ES|QL. ES|QL feature requires at least Elasticsearch #{ES_ESQL_SUPPORT_VERSION} version.") unless es_supports_esql
+  end
+
 end #class LogStash::Filters::Elasticsearch

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.17.1'
+  s.version         = '3.19.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -23,8 +23,10 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'elasticsearch', ">= 7.14.9", '< 9'
   s.add_runtime_dependency 'manticore', ">= 0.7.1"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
   s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
   s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'
+  s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
   s.add_development_dependency 'cabin', ['~> 0.6']
   s.add_development_dependency 'webrick'
   s.add_development_dependency 'logstash-devutils'