logstash-output-elasticsearch 10.8.1-java → 11.0.0-java

data/lib/logstash/outputs/elasticsearch.rb

@@ -3,8 +3,8 @@ require "logstash/namespace"
 require "logstash/environment"
 require "logstash/outputs/base"
 require "logstash/json"
-require "concurrent"
-require "stud/buffer"
+require "concurrent/atomic/atomic_boolean"
+require "stud/interval"
 require "socket" # for Socket.gethostname
 require "thread" # for safe queueing
 require "uri" # for escaping user input
@@ -92,6 +92,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   require "logstash/plugin_mixins/elasticsearch/api_configs"
   require "logstash/plugin_mixins/elasticsearch/common"
   require "logstash/outputs/elasticsearch/ilm"
+  require "logstash/outputs/elasticsearch/data_stream_support"
   require 'logstash/plugin_mixins/ecs_compatibility_support'
 
   # Protocol agnostic methods
@@ -106,6 +107,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # Generic/API config options that any document indexer output needs
   include(LogStash::PluginMixins::ElasticSearch::APIConfigs)
 
+  # DS support
+  include(LogStash::Outputs::ElasticSearch::DataStreamSupport)
+
   DEFAULT_POLICY = "logstash-policy"
 
   config_name "elasticsearch"
@@ -122,7 +126,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   #   would use the foo field for the action
   #
   # For more details on actions, check out the http://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html[Elasticsearch bulk API documentation]
-  config :action, :validate => :string, :default => "index"
+  config :action, :validate => :string # :default => "index" unless data_stream
 
   # The index to write events to. This can be dynamic using the `%{foo}` syntax.
   # The default value will partition your indices by day so you can more easily
@@ -247,6 +251,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # ILM policy to use, if undefined the default policy will be used.
   config :ilm_policy, :validate => :string, :default => DEFAULT_POLICY
 
+  attr_reader :client
   attr_reader :default_index
   attr_reader :default_ilm_rollover_alias
   attr_reader :default_template_name
@@ -257,26 +262,53 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   end
 
   def register
-    @template_installed = Concurrent::AtomicBoolean.new(false)
+    @after_successful_connection_done = Concurrent::AtomicBoolean.new(false)
     @stopping = Concurrent::AtomicBoolean.new(false)
-    # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
-    @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
 
     check_action_validity
 
+    @logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts.map(&:sanitized).map(&:to_s))
+
     # the license_checking behaviour in the Pool class is externalized in the LogStash::ElasticSearchOutputLicenseChecker
     # class defined in license_check.rb. This license checking is specific to the elasticsearch output here and passed
     # to build_client down to the Pool class.
-    build_client(LicenseChecker.new(@logger))
+    @client = build_client(LicenseChecker.new(@logger))
+
+    @after_successful_connection_thread = after_successful_connection do
+      begin
+        finish_register
+        true # thread.value
+      rescue => e
+        # we do not want to halt the thread with an exception as that has consequences for LS
+        e # thread.value
+      ensure
+        @after_successful_connection_done.make_true
+      end
+    end
 
-    @template_installer = setup_after_successful_connection do
-      discover_cluster_uuid
-      install_template
-      setup_ilm if ilm_in_use?
+    # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
+    @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
+
+    if data_stream_config?
+      @event_mapper = -> (e) { data_stream_event_action_tuple(e) }
+      @event_target = -> (e) { data_stream_name(e) }
+      @index = "#{data_stream_type}-#{data_stream_dataset}-#{data_stream_namespace}".freeze # default name
+    else
+      @event_mapper = -> (e) { event_action_tuple(e) }
+      @event_target = -> (e) { e.sprintf(@index) }
     end
+
     @bulk_request_metrics = metric.namespace(:bulk_requests)
     @document_level_metrics = metric.namespace(:documents)
-    @logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts.map(&:sanitized).map(&:to_s))
+  end
+
+  # @override post-register when ES connection established
+  def finish_register
+    assert_es_version_supports_data_streams if data_stream_config?
+    discover_cluster_uuid
+    install_template
+    setup_ilm if ilm_in_use?
+    super
   end
 
   # @override to handle proxy => '' as if none was set
@@ -297,46 +329,47 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
   # Receive an array of events and immediately attempt to index them (no buffering)
   def multi_receive(events)
-    until @template_installed.true?
-      sleep 1
+    wait_for_successful_connection if @after_successful_connection_done
+    retrying_submit map_events(events)
+  end
+
+  def map_events(events)
+    events.map(&@event_mapper)
+  end
+
+  def wait_for_successful_connection
+    after_successful_connection_done = @after_successful_connection_done
+    return unless after_successful_connection_done
+    stoppable_sleep 1 until after_successful_connection_done.true?
+
+    status = @after_successful_connection_thread && @after_successful_connection_thread.value
+    if status.is_a?(Exception) # check if thread 'halted' with an error
+      # keep logging that something isn't right (from every #multi_receive)
+      @logger.error "Elasticsearch setup did not complete normally, please review previously logged errors",
+                    message: status.message, exception: status.class
+    else
+      @after_successful_connection_done = nil # do not execute __method__ again if all went well
     end
-    retrying_submit(events.map {|e| event_action_tuple(e)})
   end
+  private :wait_for_successful_connection
 
   def close
     @stopping.make_true if @stopping
-    stop_template_installer
+    stop_after_successful_connection_thread
     @client.close if @client
   end
 
-  # not private because used by ILM specs
-  def stop_template_installer
-    @template_installer.join unless @template_installer.nil?
+  private
+
+  def stop_after_successful_connection_thread
+    @after_successful_connection_thread.join unless @after_successful_connection_thread.nil?
   end
 
-  # not private for elasticsearch_spec.rb
-  # Convert the event into a 3-tuple of action, params, and event
+  # Convert the event into a 3-tuple of action, params and event hash
   def event_action_tuple(event)
-    action = event.sprintf(@action)
-
-    params = {
-      :_id => @document_id ? event.sprintf(@document_id) : nil,
-      :_index => event.sprintf(@index),
-      routing_field_name => @routing ? event.sprintf(@routing) : nil
-    }
-
+    params = common_event_params(event)
     params[:_type] = get_event_type(event) if use_event_type?(nil)
 
-    if @pipeline
-      value = event.sprintf(@pipeline)
-      # convention: empty string equates to not using a pipeline
-      # this is useful when using a field reference in the pipeline setting, e.g.
-      #      elasticsearch {
-      #        pipeline => "%{[@metadata][pipeline]}"
-      #      }
-      params[:pipeline] = value unless value.empty?
-    end
-
     if @parent
       if @join_field
         join_value = event.get(@join_field)
@@ -348,26 +381,40 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
       end
     end
 
+    action = event.sprintf(@action || 'index')
+
     if action == 'update'
       params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != ""
       params[:_script] = event.sprintf(@script) if @script != ""
       params[retry_on_conflict_action_name] = @retry_on_conflict
     end
 
-    if @version
-      params[:version] = event.sprintf(@version)
-    end
-
-    if @version_type
-      params[:version_type] = event.sprintf(@version_type)
-    end
+    params[:version] = event.sprintf(@version) if @version
+    params[:version_type] = event.sprintf(@version_type) if @version_type
 
-    [action, params, event]
+    [action, params, event.to_hash]
   end
 
-  # not private for elasticsearch_spec.rb
-  def retry_on_conflict_action_name
-    maximum_seen_major_version >= 7 ? :retry_on_conflict : :_retry_on_conflict
+  # @return Hash (initial) parameters for given event
+  # @private shared event params factory between index and data_stream mode
+  def common_event_params(event)
+    params = {
+        :_id => @document_id ? event.sprintf(@document_id) : nil,
+        :_index => @event_target.call(event),
+        routing_field_name => @routing ? event.sprintf(@routing) : nil
+    }
+
+    if @pipeline
+      value = event.sprintf(@pipeline)
+      # convention: empty string equates to not using a pipeline
+      # this is useful when using a field reference in the pipeline setting, e.g.
+      #      elasticsearch {
+      #        pipeline => "%{[@metadata][pipeline]}"
+      #      }
+      params[:pipeline] = value unless value.empty?
+    end
+
+    params
   end
 
   @@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }
@@ -377,38 +424,47 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     require "logstash/outputs/elasticsearch/#{name}"
   end
 
-  private
+  def retry_on_conflict_action_name
+    maximum_seen_major_version >= 7 ? :retry_on_conflict : :_retry_on_conflict
+  end
 
   def routing_field_name
     maximum_seen_major_version >= 6 ? :routing : :_routing
   end
 
   # Determine the correct value for the 'type' field for the given event
-  DEFAULT_EVENT_TYPE_ES6="doc".freeze
-  DEFAULT_EVENT_TYPE_ES7="_doc".freeze
+  DEFAULT_EVENT_TYPE_ES6 = "doc".freeze
+  DEFAULT_EVENT_TYPE_ES7 = "_doc".freeze
+
   def get_event_type(event)
     # Set the 'type' value for the index.
     type = if @document_type
              event.sprintf(@document_type)
            else
-             if maximum_seen_major_version < 6
-               event.get("type") || DEFAULT_EVENT_TYPE_ES6
-             elsif maximum_seen_major_version == 6
+             major_version = maximum_seen_major_version
+             if major_version < 6
+               es5_event_type(event)
+             elsif major_version == 6
                DEFAULT_EVENT_TYPE_ES6
-             elsif maximum_seen_major_version == 7
+             elsif major_version == 7
                DEFAULT_EVENT_TYPE_ES7
              else
                nil
              end
            end
 
-    if !(type.is_a?(String) || type.is_a?(Numeric))
-      @logger.warn("Bad event type! Non-string/integer type value set!", :type_class => type.class, :type_value => type.to_s, :event => event)
-    end
-
     type.to_s
   end
 
+  def es5_event_type(event)
+    type = event.get('type')
+    return DEFAULT_EVENT_TYPE_ES6 unless type
+    if !type.is_a?(String) && !type.is_a?(Numeric)
+      @logger.warn("Bad event type (non-string/integer type value set)", :type_class => type.class, :type_value => type, :event => event.to_hash)
+    end
+    type
+  end
+
   ##
   # WARNING: This method is overridden in a subclass in Logstash Core 7.7-7.8's monitoring,
   #          where a `client` argument is both required and ignored. In later versions of
@@ -417,12 +473,15 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # @param noop_required_client [nil]: required `nil` for legacy reasons.
   # @return [Boolean]
   def use_event_type?(noop_required_client)
-    maximum_seen_major_version < 8
+    # always set type for ES <= 6
+    # for ES 7 only set it if the user defined it
+    (maximum_seen_major_version < 7) || (maximum_seen_major_version == 7 && @document_type)
   end
 
   def install_template
     TemplateManager.install_template(self)
-    @template_installed.make_true
+  rescue => e
+    @logger.error("Failed to install template", message: e.message, exception: e.class, backtrace: e.backtrace)
   end
 
   def setup_ecs_compatibility_related_defaults
@@ -445,13 +504,14 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   end
 
   # To be overidden by the -java version
-  VALID_HTTP_ACTIONS=["index", "delete", "create", "update"]
+  VALID_HTTP_ACTIONS = ["index", "delete", "create", "update"]
   def valid_actions
     VALID_HTTP_ACTIONS
   end
 
   def check_action_validity
-    raise LogStash::ConfigurationError, "No action specified!" unless @action
+    return if @action.nil? # not set
+    raise LogStash::ConfigurationError, "No action specified!" if @action.empty?
 
     # If we're using string interpolation, we're good!
     return if @action =~ /%{.+}/

data/lib/logstash/outputs/elasticsearch/data_stream_support.rb

@@ -0,0 +1,233 @@
+module LogStash module Outputs class ElasticSearch
+  # DS specific behavior/configuration.
+  module DataStreamSupport
+
+    def self.included(base)
+      # Defines whether data will be indexed into an Elasticsearch data stream,
+      # `data_stream_*` settings will only be used if this setting is enabled!
+      # This setting supports values `true`, `false`, and `auto`.
+      # Defaults to `false` in Logstash 7.x and `auto` starting in Logstash 8.0.
+      base.config :data_stream, :validate => ['true', 'false', 'auto']
+
+      base.config :data_stream_type, :validate => ['logs', 'metrics', 'synthetics'], :default => 'logs'
+      base.config :data_stream_dataset, :validate => :dataset_identifier, :default => 'generic'
+      base.config :data_stream_namespace, :validate => :namespace_identifier, :default => 'default'
+
+      base.config :data_stream_sync_fields, :validate => :boolean, :default => true
+      base.config :data_stream_auto_routing, :validate => :boolean, :default => true
+
+      base.extend(Validator)
+    end
+
+    # @note assumes to be running AFTER {after_successful_connection} completed, due ES version checks
+    def data_stream_config?
+      @data_stream_config.nil? ? @data_stream_config = check_data_stream_config! : @data_stream_config
+    end
+
+    private
+
+    def data_stream_name(event)
+      data_stream = event.get('data_stream')
+      return @index if !data_stream_auto_routing || !data_stream.is_a?(Hash)
+
+      type = data_stream['type'] || data_stream_type
+      dataset = data_stream['dataset'] || data_stream_dataset
+      namespace = data_stream['namespace'] || data_stream_namespace
+      "#{type}-#{dataset}-#{namespace}"
+    end
+
+    # @param params the user configuration for the ES output
+    # @note LS initialized configuration (with filled defaults) won't detect as data-stream
+    # compatible, only explicit (`original_params`) config should be tested.
+    # @return [TrueClass|FalseClass] whether given configuration is data-stream compatible
+    def check_data_stream_config!(params = original_params)
+      data_stream_params = params.select { |name, _| name.start_with?('data_stream_') } # exclude data_stream =>
+      invalid_data_stream_params = invalid_data_stream_params(params)
+
+      case data_stream_explicit_value
+      when false
+        if data_stream_params.any?
+          @logger.error "Ambiguous configuration; data stream settings must not be present when data streams is disabled (caused by: `data_stream => false`)", data_stream_params
+          raise LogStash::ConfigurationError, "Ambiguous configuration, please remove data stream specific settings: #{data_stream_params.keys}"
+        end
+        return false
+      when true
+        if invalid_data_stream_params.any?
+          @logger.error "Invalid data stream configuration, following parameters are not supported:", invalid_data_stream_params
+          raise LogStash::ConfigurationError, "Invalid data stream configuration: #{invalid_data_stream_params.keys}"
+        end
+        return true
+      else
+        use_data_stream = data_stream_default(data_stream_params, invalid_data_stream_params.empty?)
+        if !use_data_stream && data_stream_params.any?
+          # DS (auto) disabled but there's still some data-stream parameters (and no `data_stream => false`)
+          @logger.error "Ambiguous configuration; data stream settings are present, but data streams are not enabled", data_stream_params
+          raise LogStash::ConfigurationError, "Ambiguous configuration, please set data_stream => true " +
+              "or remove data stream specific settings: #{data_stream_params.keys}"
+        end
+        use_data_stream
+      end
+    end
+
+    def data_stream_explicit_value
+      case @data_stream
+      when 'true'
+        return true
+      when 'false'
+        return false
+      else
+        return nil # 'auto' or not set by user
+      end
+    end
+
+    def invalid_data_stream_params(params)
+      shared_params = LogStash::PluginMixins::ElasticSearch::APIConfigs::CONFIG_PARAMS.keys.map(&:to_s)
+      params.reject do |name, value|
+        # NOTE: intentionally do not support explicit DS configuration like:
+        # - `index => ...` identifier provided by data_stream_xxx settings
+        # - `manage_template => false` implied by not setting the parameter
+        case name
+        when 'action'
+          value == 'create'
+        when 'routing', 'pipeline'
+          true
+        when 'data_stream'
+          value.to_s == 'true'
+        else
+          name.start_with?('data_stream_') ||
+              shared_params.include?(name) ||
+                inherited_internal_config_param?(name) # 'id', 'enabled_metric' etc
+        end
+      end
+    end
+
+    def inherited_internal_config_param?(name)
+      self.class.superclass.get_config.key?(name.to_s) # superclass -> LogStash::Outputs::Base
+    end
+
+    DATA_STREAMS_ORIGIN_ES_VERSION = '7.9.0'
+
+    # @return [Gem::Version] if ES supports DS nil (or raise) otherwise
+    def assert_es_version_supports_data_streams
+      fail 'no last_es_version' unless last_es_version # assert - should not happen
+      es_version = Gem::Version.create(last_es_version)
+      if es_version < Gem::Version.create(DATA_STREAMS_ORIGIN_ES_VERSION)
+        @logger.error "Elasticsearch version does not support data streams, Logstash might end up writing to an index", es_version: es_version.version
+        # NOTE: when switching to synchronous check from register, this should be a ConfigurationError
+        raise LogStash::Error, "A data_stream configuration is only supported since Elasticsearch #{DATA_STREAMS_ORIGIN_ES_VERSION} " +
+                               "(detected version #{es_version.version}), please upgrade your cluster"
+      end
+      es_version # return truthy
+    end
+
+    DATA_STREAMS_ENABLED_BY_DEFAULT_LS_VERSION = '8.0.0'
+
+    # when data_stream => is either 'auto' or not set
+    def data_stream_default(data_stream_params, valid_data_stream_config)
+      ds_default = Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create(DATA_STREAMS_ENABLED_BY_DEFAULT_LS_VERSION)
+
+      if ds_default # LS 8.0
+        return false unless valid_data_stream_config
+
+        @logger.debug 'Configuration is data stream compliant'
+        return true
+      end
+
+      # LS 7.x
+      if valid_data_stream_config && !data_stream_params.any?
+        @logger.warn "Configuration is data stream compliant but due backwards compatibility Logstash 7.x will not assume " +
+                     "writing to a data-stream, default behavior will change on Logstash 8.0 " +
+                     "(set `data_stream => true/false` to disable this warning)"
+      end
+      false
+    end
+
+    # an {event_action_tuple} replacement when a data-stream configuration is detected
+    def data_stream_event_action_tuple(event)
+      event_data = event.to_hash
+      data_stream_event_sync(event_data) if data_stream_sync_fields
+      ['create', common_event_params(event), event_data] # action always 'create'
+    end
+
+    DATA_STREAM_SYNC_FIELDS = [ 'type', 'dataset', 'namespace' ].freeze
+
+    def data_stream_event_sync(event_data)
+      data_stream = event_data['data_stream']
+      if data_stream.is_a?(Hash)
+        unless data_stream_auto_routing
+          sync_fields = DATA_STREAM_SYNC_FIELDS.select { |name| data_stream.key?(name) && data_stream[name] != send(:"data_stream_#{name}") }
+          if sync_fields.any? # these fields will need to be overwritten
+            info = sync_fields.inject({}) { |info, name| info[name] = data_stream[name]; info }
+            info[:event] = event_data
+            @logger.warn "Some data_stream fields are out of sync, these will be updated to reflect data-stream name", info
+
+            # NOTE: we work directly with event.to_hash data thus fine to mutate the 'data_stream' hash
+            sync_fields.each { |name| data_stream[name] = nil } # fallback to ||= bellow
+          end
+        end
+      else
+        unless data_stream.nil?
+          @logger.warn "Invalid 'data_stream' field type, due fields sync will overwrite", value: data_stream, event: event_data
+        end
+        event_data['data_stream'] = data_stream = Hash.new
+      end
+
+      data_stream['type'] ||= data_stream_type
+      data_stream['dataset'] ||= data_stream_dataset
+      data_stream['namespace'] ||= data_stream_namespace
+
+      event_data
+    end
+
+    module Validator
+
+      # @override {LogStash::Config::Mixin::validate_value} to handle custom validators
+      # @param value [Array<Object>]
+      # @param validator [nil,Array,Symbol]
+      # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
+      # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
+      def validate_value(value, validator)
+        case validator
+        when :dataset_identifier   then validate_dataset_identifier(value)
+        when :namespace_identifier then validate_namespace_identifier(value)
+        else super
+        end
+      end
+
+      private
+
+      def validate_dataset_identifier(value)
+        valid, value = validate_value(value, :string)
+        return false, value unless valid
+
+        validate_identifier(value)
+      end
+
+      def validate_namespace_identifier(value)
+        valid, value = validate_value(value, :string)
+        return false, value unless valid
+
+        validate_identifier(value)
+      end
+
+      def validate_identifier(value, max_size = 100)
+        if value.empty?
+          return false, "Invalid identifier - empty string"
+        end
+        if value.bytesize > max_size
+          return false, "Invalid identifier - too long (#{value.bytesize} bytes)"
+        end
+        # cannot include \, /, *, ?, ", <, >, |, ' ' (space char), ',', #, :
+        if value.match? Regexp.union(INVALID_IDENTIFIER_CHARS)
+          return false, "Invalid characters detected #{INVALID_IDENTIFIER_CHARS.inspect} are not allowed"
+        end
+        return true, value
+      end
+
+      INVALID_IDENTIFIER_CHARS = [ '\\', '/', '*', '?', '"', '<', '>', '|', ' ', ',', '#', ':' ]
+      private_constant :INVALID_IDENTIFIER_CHARS
+
+    end
+
+  end
+end end end