logstash-filter-elasticsearch 3.19.0 → 4.0.0

data/lib/logstash/filters/elasticsearch.rb

@@ -2,23 +2,13 @@
 require "logstash/filters/base"
 require "logstash/namespace"
 require "logstash/json"
-require 'logstash/plugin_mixins/ecs_compatibility_support'
-require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
 require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
-require "logstash/plugin_mixins/normalize_config_support"
-require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
 require "monitor"
 
 require_relative "elasticsearch/client"
+require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
 
 class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
-
-  require 'logstash/filters/elasticsearch/dsl_executor'
-  require 'logstash/filters/elasticsearch/esql_executor'
-
-  include LogStash::PluginMixins::ECSCompatibilitySupport
-  include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
-
   config_name "elasticsearch"
 
   # List of elasticsearch hosts to use for querying.
@@ -28,13 +18,8 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Field substitution (e.g. `index-name-%{date_field}`) is available
   config :index, :validate => :string, :default => ""
 
-  # A type of Elasticsearch query, provided by @query.
-  config :query_type, :validate => %w[esql dsl], :default => "dsl"
-
-  # Elasticsearch query string. This can be in DSL or ES|QL query shape defined by @query_type.
-  # Read the Elasticsearch query string documentation.
-  #   DSL: https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl-query-string-query.html#query-string-syntax
-  #   ES|QL: https://www.elastic.co/guide/en/elasticsearch/reference/current/esql.html
+  # Elasticsearch query string. Read the Elasticsearch query string documentation.
+  # for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl-query-string-query.html#query-string-syntax
   config :query, :validate => :string
 
   # File path to elasticsearch query in DSL format. Read the Elasticsearch query documentation
@@ -47,9 +32,6 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Array of fields to copy from old event (found via elasticsearch) into new event
   config :fields, :validate => :array, :default => {}
 
-  # Custom headers for Elasticsearch requests
-  config :custom_headers, :validate => :hash, :default => {}
-
   # Hash of docinfo fields to copy from old event (found via elasticsearch) into new event
   config :docinfo_fields, :validate => :hash, :default => {}
 
@@ -79,18 +61,6 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Set the address of a forward HTTP proxy.
   config :proxy, :validate => :uri_or_empty
 
-  # SSL
-  config :ssl, :validate => :boolean, :default => false, :deprecated => "Set 'ssl_enabled' instead."
-
-  # SSL Certificate Authority file
-  config :ca_file, :validate => :path, :deprecated => "Set 'ssl_certificate_authorities' instead."
-
-  # The keystore used to present a certificate to the server.
-  # It can be either .jks or .p12
-  config :keystore, :validate => :path, :deprecated => "Use 'ssl_keystore_path' instead."
-
-  # Set the keystore password
-  config :keystore_password, :validate => :password, :deprecated => "Use 'ssl_keystore_password' instead."
 
   # OpenSSL-style X.509 certificate certificate to authenticate the client
   config :ssl_certificate, :validate => :path
@@ -146,36 +116,24 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Tags the event on failure to look up geo information. This can be used in later analysis.
   config :tag_on_failure, :validate => :array, :default => ["_elasticsearch_lookup_failure"]
 
-  # If set, the result set will be nested under the target field
-  config :target, :validate => :field_reference
-
   # How many times to retry on failure?
   config :retry_on_failure, :validate => :number, :default => 0
 
   # What status codes to retry on?
   config :retry_on_status, :validate => :number, :list => true, :default => [500, 502, 503, 504]
 
-  # named placeholders in ES|QL query
-  # example,
-  #   if the query is "FROM my-index | WHERE some_type = ?type AND depth > ?min_depth"
-  #   named placeholders can be applied as the following in query_params:
-  #   query_params => [
-  #     {"type" => "%{[type]}"}
-  #     {"min_depth" => "%{[depth]}"}
-  #   ]
-  config :query_params, :validate => :array, :default => []
+
+  config :ssl, :obsolete => "Set 'ssl_enabled' instead."
+  config :ca_file, :obsolete => "Set 'ssl_certificate_authorities' instead."
+  config :keystore, :obsolete => "Set 'ssl_keystore_path' instead."
+  config :keystore_password, :validate => :password, :obsolete => "Set 'ssl_keystore_password' instead."
 
   # config :ca_trusted_fingerprint, :validate => :sha_256_hex
   include LogStash::PluginMixins::CATrustedFingerprintSupport
 
-  include LogStash::PluginMixins::NormalizeConfigSupport
-
   include MonitorMixin
   attr_reader :shared_client
 
-  LS_ESQL_SUPPORT_VERSION = "8.17.4" # the version started using elasticsearch-ruby v8
-  ES_ESQL_SUPPORT_VERSION = "8.11.0"
-
   ##
   # @override to handle proxy => '' as if none was set
   # @param value [Array<Object>]
@@ -193,22 +151,17 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     return super(value, :uri)
   end
 
-  attr_reader :query_dsl
-
   def register
-    case @query_type
-    when "esql"
-      invalid_params_with_esql = original_params.keys & %w(index query_template sort fields docinfo_fields aggregation_fields enable_sort result_size)
-      raise LogStash::ConfigurationError, "Configured #{invalid_params_with_esql} params cannot be used with ES|QL query" if invalid_params_with_esql.any?
-
-      validate_ls_version_for_esql_support!
-      validate_esql_query_and_params!
-      @esql_executor ||= LogStash::Filters::Elasticsearch::EsqlExecutor.new(self, @logger)
-    else # dsl
-      validate_dsl_query_settings!
-      @esql_executor ||= LogStash::Filters::Elasticsearch::DslExecutor.new(self, @logger)
+    #Load query if it exists
+    if @query_template
+      if File.zero?(@query_template)
+        raise "template is empty"
+      end
+      file = File.open(@query_template, 'r')
+      @query_dsl = file.read
     end
 
+    validate_query_settings
     fill_hosts_from_cloud_id
     setup_ssl_params!
     validate_authentication
@@ -217,23 +170,73 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
 
     test_connection!
-    validate_es_for_esql_support! if @query_type == "esql"
     setup_serverless
-    if get_client.es_transport_client_type == "elasticsearch_transport"
-      require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
-    end
   end # def register
 
   def filter(event)
-    @esql_executor.process(get_client, event)
-  end # def filter
+    matched = false
+    begin
+      params = { :index => event.sprintf(@index) }
+
+      if @query_dsl
+        query = LogStash::Json.load(event.sprintf(@query_dsl))
+        params[:body] = query
+      else
+        query = event.sprintf(@query)
+        params[:q] = query
+        params[:size] = result_size
+        params[:sort] =  @sort if @enable_sort
+      end
 
-  def decorate(event)
-    # this Elasticsearch class has access to `filter_matched`
-    filter_matched(event)
-  end
+      @logger.debug("Querying elasticsearch for lookup", :params => params)
+
+      results = get_client.search(params)
+      raise "Elasticsearch query error: #{results["_shards"]["failures"]}" if results["_shards"].include? "failures"
+
+      event.set("[@metadata][total_hits]", extract_total_from_hits(results['hits']))
+
+      resultsHits = results["hits"]["hits"]
+      if !resultsHits.nil? && !resultsHits.empty?
+        matched = true
+        @fields.each do |old_key, new_key|
+          old_key_path = extract_path(old_key)
+          set = resultsHits.map do |doc|
+            extract_value(doc["_source"], old_key_path)
+          end
+          event.set(new_key, set.count > 1 ? set : set.first)
+        end
+        @docinfo_fields.each do |old_key, new_key|
+          old_key_path = extract_path(old_key)
+          set = resultsHits.map do |doc|
+            extract_value(doc, old_key_path)
+          end
+          event.set(new_key, set.count > 1 ? set : set.first)
+        end
+      end
+
+      resultsAggs = results["aggregations"]
+      if !resultsAggs.nil? && !resultsAggs.empty?
+        matched = true
+        @aggregation_fields.each do |agg_name, ls_field|
+          event.set(ls_field, resultsAggs[agg_name])
+        end
+      end
+
+    rescue => e
+      if @logger.trace?
+        @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query, :event => event.to_hash, :error => e.message, :backtrace => e.backtrace)
+      elsif @logger.debug?
+        @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :error => e.message, :backtrace => e.backtrace)
+      else
+        @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :error => e.message)
+      end
+      @tag_on_failure.each{|tag| event.tag(tag)}
+    else
+      filter_matched(event) if matched
+    end
+  end # def filter
 
-  # public only to be reused in testing
+  # public only to be reuse in testing
   def prepare_user_agent
     os_name = java.lang.System.getProperty('os.name')
     os_version = java.lang.System.getProperty('os.version')
@@ -257,8 +260,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       :ssl => client_ssl_options,
       :retry_on_failure => @retry_on_failure,
       :retry_on_status => @retry_on_status,
-      :user_agent => prepare_user_agent,
-      :custom_headers => @custom_headers
+      :user_agent => prepare_user_agent
     }
   end
 
@@ -344,10 +346,53 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     end
   end
 
+  # get an array of path elements from a path reference
+  def extract_path(path_reference)
+    return [path_reference] unless path_reference.start_with?('[') && path_reference.end_with?(']')
+
+    path_reference[1...-1].split('][')
+  end
+
+  # given a Hash and an array of path fragments, returns the value at the path
+  # @param source [Hash{String=>Object}]
+  # @param path [Array{String}]
+  # @return [Object]
+  def extract_value(source, path)
+    path.reduce(source) do |memo, old_key_fragment|
+      break unless memo.include?(old_key_fragment)
+      memo[old_key_fragment]
+    end
+  end
+
+  # Given a "hits" object from an Elasticsearch response, return the total number of hits in
+  # the result set.
+  # @param hits [Hash{String=>Object}]
+  # @return [Integer]
+  def extract_total_from_hits(hits)
+    total = hits['total']
+
+    # Elasticsearch 7.x produces an object containing `value` and `relation` in order
+    # to enable unambiguous reporting when the total is only a lower bound; if we get
+    # an object back, return its `value`.
+    return total['value'] if total.kind_of?(Hash)
+
+    total
+  end
+
   def hosts_default?(hosts)
     hosts.is_a?(Array) && hosts.size == 1 && !original_params.key?('hosts')
   end
 
+  def validate_query_settings
+    unless @query || @query_template
+      raise LogStash::ConfigurationError, "Both `query` and `query_template` are empty. Require either `query` or `query_template`."
+    end
+
+    if @query && @query_template
+      raise LogStash::ConfigurationError, "Both `query` and `query_template` are set. Use either `query` or `query_template`."
+    end
+  end
+
   def validate_authentication
     authn_options = 0
     authn_options += 1 if @cloud_auth
@@ -434,107 +479,9 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   end
 
   def setup_ssl_params!
-    @ssl_enabled = normalize_config(:ssl_enabled) do |normalize|
-      normalize.with_deprecated_alias(:ssl)
-    end
-
-    # Infer the value if neither the deprecate `ssl` and `ssl_enabled` were set
-    infer_ssl_enabled_from_hosts
-
-    @ssl_keystore_path = normalize_config(:ssl_keystore_path) do |normalize|
-      normalize.with_deprecated_alias(:keystore)
-    end
-
-    @ssl_keystore_password = normalize_config(:ssl_keystore_password) do |normalize|
-      normalize.with_deprecated_alias(:keystore_password)
-    end
-
-    @ssl_certificate_authorities = normalize_config(:ssl_certificate_authorities) do |normalize|
-      normalize.with_deprecated_mapping(:ca_file) do |ca_file|
-        [ca_file]
-      end
-    end
-
-    params['ssl_enabled'] = @ssl_enabled
-    params['ssl_keystore_path'] = @ssl_keystore_path unless @ssl_keystore_path.nil?
-    params['ssl_keystore_password'] = @ssl_keystore_password unless @ssl_keystore_password.nil?
-    params['ssl_certificate_authorities'] = @ssl_certificate_authorities unless @ssl_certificate_authorities.nil?
-  end
-
-  def infer_ssl_enabled_from_hosts
-    return if original_params.include?('ssl') || original_params.include?('ssl_enabled')
-
-    @ssl_enabled = params['ssl_enabled'] = effectively_ssl?
-  end
-
-  def effectively_ssl?
-    return true if @ssl_enabled
-
-    hosts = Array(@hosts)
-    return false if hosts.nil? || hosts.empty?
-
-    hosts.all? { |host| host && host.to_s.start_with?("https") }
-  end
-
-  def validate_dsl_query_settings!
-    #Load query if it exists
-    if @query_template
-      if File.zero?(@query_template)
-        raise "template is empty"
-      end
-      file = File.open(@query_template, 'r')
-      @query_dsl = file.read
-    end
-
-    validate_query_settings
-  end
-
-  def validate_query_settings
-    unless @query || @query_template
-      raise LogStash::ConfigurationError, "Both `query` and `query_template` are empty. Require either `query` or `query_template`."
-    end
-
-    if @query && @query_template
-      raise LogStash::ConfigurationError, "Both `query` and `query_template` are set. Use either `query` or `query_template`."
-    end
-
-    if original_params.keys.include?("query_params")
-      raise LogStash::ConfigurationError, "`query_params` is not allowed when `query_type => 'dsl'`."
-    end
-  end
-
-  def validate_ls_version_for_esql_support!
-    if Gem::Version.create(LOGSTASH_VERSION) < Gem::Version.create(LS_ESQL_SUPPORT_VERSION)
-      fail("Current version of Logstash does not include Elasticsearch client which supports ES|QL. Please upgrade Logstash to at least #{LS_ESQL_SUPPORT_VERSION}")
-    end
-  end
-
-  def validate_esql_query_and_params!
-    # If Array, validate that query_params needs to contain only single-entry hashes, convert it to a Hash
-    if @query_params.kind_of?(Array)
-      illegal_entries = @query_params.reject {|e| e.kind_of?(Hash) && e.size == 1 }
-      raise LogStash::ConfigurationError, "`query_params` must contain only single-entry hashes. Illegal placeholders: #{illegal_entries}" if illegal_entries.any?
-
-      @query_params = @query_params.reduce({}, :merge)
-    end
-
-    illegal_keys = @query_params.keys.reject {|k| k[/^[a-z_][a-z0-9_]*$/] }
-    if illegal_keys.any?
-      message = "Illegal #{illegal_keys} placeholder names in `query_params`. A valid parameter name starts with a letter and contains letters, digits and underscores only;"
-      raise LogStash::ConfigurationError, message
-    end
-
-    placeholders = @query.scan(/(?<=[?])[a-z_][a-z0-9_]*/i)
-    placeholders.each do |placeholder|
-      raise LogStash::ConfigurationError, "Placeholder #{placeholder} not found in query" unless @query_params.include?(placeholder)
-    end
-  end
-
-  def validate_es_for_esql_support!
-    # make sure connected ES supports ES|QL (8.11+)
-    @es_version ||= get_client.es_version
-    es_supports_esql = Gem::Version.create(@es_version) >= Gem::Version.create(ES_ESQL_SUPPORT_VERSION)
-    fail("Connected Elasticsearch #{@es_version} version does not supports ES|QL. ES|QL feature requires at least Elasticsearch #{ES_ESQL_SUPPORT_VERSION} version.") unless es_supports_esql
+    # Infer the value if neither `ssl_enabled` was not set
+    return if original_params.include?('ssl_enabled')
+    params['ssl_enabled'] = @ssl_enabled ||= Array(@hosts).all? { |host| host && host.to_s.start_with?("https") }
   end
 
 end #class LogStash::Filters::Elasticsearch

data/logstash-filter-elasticsearch.gemspec

@@ -1,13 +1,13 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.19.0'
+  s.version         = '4.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
   s.email           = 'info@elastic.co'
-  s.homepage        = "https://elastic.co/logstash"
+  s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
   s.require_paths = ["lib"]
 
   # Files
@@ -21,12 +21,9 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'elasticsearch', ">= 7.14.9", '< 9'
+  s.add_runtime_dependency 'elasticsearch', ">= 7.14.9" # LS >= 6.7 and < 7.14 all used version 5.0.5
   s.add_runtime_dependency 'manticore', ">= 0.7.1"
-  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
   s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
-  s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'
-  s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
   s.add_development_dependency 'cabin', ['~> 0.6']
   s.add_development_dependency 'webrick'
   s.add_development_dependency 'logstash-devutils'