logstash-output-elasticsearch 10.8.3-java → 11.0.2-java

data/lib/logstash/outputs/elasticsearch/data_stream_support.rb

@@ -0,0 +1,233 @@
+module LogStash module Outputs class ElasticSearch
+  # DS specific behavior/configuration.
+  module DataStreamSupport
+
+    def self.included(base)
+      # Defines whether data will be indexed into an Elasticsearch data stream,
+      # `data_stream_*` settings will only be used if this setting is enabled!
+      # This setting supports values `true`, `false`, and `auto`.
+      # Defaults to `false` in Logstash 7.x and `auto` starting in Logstash 8.0.
+      base.config :data_stream, :validate => ['true', 'false', 'auto']
+
+      base.config :data_stream_type, :validate => ['logs', 'metrics', 'synthetics'], :default => 'logs'
+      base.config :data_stream_dataset, :validate => :dataset_identifier, :default => 'generic'
+      base.config :data_stream_namespace, :validate => :namespace_identifier, :default => 'default'
+
+      base.config :data_stream_sync_fields, :validate => :boolean, :default => true
+      base.config :data_stream_auto_routing, :validate => :boolean, :default => true
+
+      base.extend(Validator)
+    end
+
+    # @note assumes to be running AFTER {after_successful_connection} completed, due ES version checks
+    def data_stream_config?
+      @data_stream_config.nil? ? @data_stream_config = check_data_stream_config! : @data_stream_config
+    end
+
+    private
+
+    def data_stream_name(event)
+      data_stream = event.get('data_stream')
+      return @index if !data_stream_auto_routing || !data_stream.is_a?(Hash)
+
+      type = data_stream['type'] || data_stream_type
+      dataset = data_stream['dataset'] || data_stream_dataset
+      namespace = data_stream['namespace'] || data_stream_namespace
+      "#{type}-#{dataset}-#{namespace}"
+    end
+
+    # @param params the user configuration for the ES output
+    # @note LS initialized configuration (with filled defaults) won't detect as data-stream
+    # compatible, only explicit (`original_params`) config should be tested.
+    # @return [TrueClass|FalseClass] whether given configuration is data-stream compatible
+    def check_data_stream_config!(params = original_params)
+      data_stream_params = params.select { |name, _| name.start_with?('data_stream_') } # exclude data_stream =>
+      invalid_data_stream_params = invalid_data_stream_params(params)
+
+      case data_stream_explicit_value
+      when false
+        if data_stream_params.any?
+          @logger.error "Ambiguous configuration; data stream settings must not be present when data streams is disabled (caused by: `data_stream => false`)", data_stream_params
+          raise LogStash::ConfigurationError, "Ambiguous configuration, please remove data stream specific settings: #{data_stream_params.keys}"
+        end
+        return false
+      when true
+        if invalid_data_stream_params.any?
+          @logger.error "Invalid data stream configuration, following parameters are not supported:", invalid_data_stream_params
+          raise LogStash::ConfigurationError, "Invalid data stream configuration: #{invalid_data_stream_params.keys}"
+        end
+        return true
+      else
+        use_data_stream = data_stream_default(data_stream_params, invalid_data_stream_params.empty?)
+        if !use_data_stream && data_stream_params.any?
+          # DS (auto) disabled but there's still some data-stream parameters (and no `data_stream => false`)
+          @logger.error "Ambiguous configuration; data stream settings are present, but data streams are not enabled", data_stream_params
+          raise LogStash::ConfigurationError, "Ambiguous configuration, please set data_stream => true " +
+              "or remove data stream specific settings: #{data_stream_params.keys}"
+        end
+        use_data_stream
+      end
+    end
+
+    def data_stream_explicit_value
+      case @data_stream
+      when 'true'
+        return true
+      when 'false'
+        return false
+      else
+        return nil # 'auto' or not set by user
+      end
+    end
+
+    def invalid_data_stream_params(params)
+      shared_params = LogStash::PluginMixins::ElasticSearch::APIConfigs::CONFIG_PARAMS.keys.map(&:to_s)
+      params.reject do |name, value|
+        # NOTE: intentionally do not support explicit DS configuration like:
+        # - `index => ...` identifier provided by data_stream_xxx settings
+        # - `manage_template => false` implied by not setting the parameter
+        case name
+        when 'action'
+          value == 'create'
+        when 'routing', 'pipeline'
+          true
+        when 'data_stream'
+          value.to_s == 'true'
+        else
+          name.start_with?('data_stream_') ||
+              shared_params.include?(name) ||
+                inherited_internal_config_param?(name) # 'id', 'enabled_metric' etc
+        end
+      end
+    end
+
+    def inherited_internal_config_param?(name)
+      self.class.superclass.get_config.key?(name.to_s) # superclass -> LogStash::Outputs::Base
+    end
+
+    DATA_STREAMS_ORIGIN_ES_VERSION = '7.9.0'
+
+    # @return [Gem::Version] if ES supports DS nil (or raise) otherwise
+    def assert_es_version_supports_data_streams
+      fail 'no last_es_version' unless last_es_version # assert - should not happen
+      es_version = Gem::Version.create(last_es_version)
+      if es_version < Gem::Version.create(DATA_STREAMS_ORIGIN_ES_VERSION)
+        @logger.error "Elasticsearch version does not support data streams, Logstash might end up writing to an index", es_version: es_version.version
+        # NOTE: when switching to synchronous check from register, this should be a ConfigurationError
+        raise LogStash::Error, "A data_stream configuration is only supported since Elasticsearch #{DATA_STREAMS_ORIGIN_ES_VERSION} " +
+                               "(detected version #{es_version.version}), please upgrade your cluster"
+      end
+      es_version # return truthy
+    end
+
+    DATA_STREAMS_ENABLED_BY_DEFAULT_LS_VERSION = '8.0.0'
+
+    # when data_stream => is either 'auto' or not set
+    def data_stream_default(data_stream_params, valid_data_stream_config)
+      ds_default = Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create(DATA_STREAMS_ENABLED_BY_DEFAULT_LS_VERSION)
+
+      if ds_default # LS 8.0
+        return false unless valid_data_stream_config
+
+        @logger.debug 'Configuration is data stream compliant'
+        return true
+      end
+
+      # LS 7.x
+      if valid_data_stream_config && !data_stream_params.any?
+        @logger.warn "Configuration is data stream compliant but due backwards compatibility Logstash 7.x will not assume " +
+                     "writing to a data-stream, default behavior will change on Logstash 8.0 " +
+                     "(set `data_stream => true/false` to disable this warning)"
+      end
+      false
+    end
+
+    # an {event_action_tuple} replacement when a data-stream configuration is detected
+    def data_stream_event_action_tuple(event)
+      event_data = event.to_hash
+      data_stream_event_sync(event_data) if data_stream_sync_fields
+      EventActionTuple.new('create', common_event_params(event), event, event_data)
+    end
+
+    DATA_STREAM_SYNC_FIELDS = [ 'type', 'dataset', 'namespace' ].freeze
+
+    def data_stream_event_sync(event_data)
+      data_stream = event_data['data_stream']
+      if data_stream.is_a?(Hash)
+        unless data_stream_auto_routing
+          sync_fields = DATA_STREAM_SYNC_FIELDS.select { |name| data_stream.key?(name) && data_stream[name] != send(:"data_stream_#{name}") }
+          if sync_fields.any? # these fields will need to be overwritten
+            info = sync_fields.inject({}) { |info, name| info[name] = data_stream[name]; info }
+            info[:event] = event_data
+            @logger.warn "Some data_stream fields are out of sync, these will be updated to reflect data-stream name", info
+
+            # NOTE: we work directly with event.to_hash data thus fine to mutate the 'data_stream' hash
+            sync_fields.each { |name| data_stream[name] = nil } # fallback to ||= bellow
+          end
+        end
+      else
+        unless data_stream.nil?
+          @logger.warn "Invalid 'data_stream' field type, due fields sync will overwrite", value: data_stream, event: event_data
+        end
+        event_data['data_stream'] = data_stream = Hash.new
+      end
+
+      data_stream['type'] ||= data_stream_type
+      data_stream['dataset'] ||= data_stream_dataset
+      data_stream['namespace'] ||= data_stream_namespace
+
+      event_data
+    end
+
+    module Validator
+
+      # @override {LogStash::Config::Mixin::validate_value} to handle custom validators
+      # @param value [Array<Object>]
+      # @param validator [nil,Array,Symbol]
+      # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
+      # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
+      def validate_value(value, validator)
+        case validator
+        when :dataset_identifier   then validate_dataset_identifier(value)
+        when :namespace_identifier then validate_namespace_identifier(value)
+        else super
+        end
+      end
+
+      private
+
+      def validate_dataset_identifier(value)
+        valid, value = validate_value(value, :string)
+        return false, value unless valid
+
+        validate_identifier(value)
+      end
+
+      def validate_namespace_identifier(value)
+        valid, value = validate_value(value, :string)
+        return false, value unless valid
+
+        validate_identifier(value)
+      end
+
+      def validate_identifier(value, max_size = 100)
+        if value.empty?
+          return false, "Invalid identifier - empty string"
+        end
+        if value.bytesize > max_size
+          return false, "Invalid identifier - too long (#{value.bytesize} bytes)"
+        end
+        # cannot include \, /, *, ?, ", <, >, |, ' ' (space char), ',', #, :
+        if value.match? Regexp.union(INVALID_IDENTIFIER_CHARS)
+          return false, "Invalid characters detected #{INVALID_IDENTIFIER_CHARS.inspect} are not allowed"
+        end
+        return true, value
+      end
+
+      INVALID_IDENTIFIER_CHARS = [ '\\', '/', '*', '?', '"', '<', '>', '|', ' ', ',', '#', ':' ]
+      private_constant :INVALID_IDENTIFIER_CHARS
+
+    end
+
+  end
+end end end

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -1,6 +1,4 @@
 require "logstash/outputs/elasticsearch"
-require "cabin"
-require "base64"
 require 'logstash/outputs/elasticsearch/http_client/pool'
 require 'logstash/outputs/elasticsearch/http_client/manticore_adapter'
 require 'cgi'
@@ -80,12 +78,16 @@ module LogStash; module Outputs; class ElasticSearch;
 
     def template_install(name, template, force=false)
       if template_exists?(name) && !force
-        @logger.debug("Found existing Elasticsearch template. Skipping template management", :name => name)
+        @logger.debug("Found existing Elasticsearch template, skipping template management", name: name)
         return
       end
       template_put(name, template)
     end
 
+    def last_es_version
+      @pool.last_es_version
+    end
+
     def maximum_seen_major_version
       @pool.maximum_seen_major_version
     end
@@ -109,27 +111,50 @@ module LogStash; module Outputs; class ElasticSearch;
       body_stream = StringIO.new
       if http_compression
         body_stream.set_encoding "BINARY"
-        stream_writer = Zlib::GzipWriter.new(body_stream, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY)
-      else 
+        stream_writer = gzip_writer(body_stream)
+      else
         stream_writer = body_stream
       end
       bulk_responses = []
-      bulk_actions.each do |action|
+      batch_actions = []
+      bulk_actions.each_with_index do |action, index|
         as_json = action.is_a?(Array) ?
                     action.map {|line| LogStash::Json.dump(line)}.join("\n") :
                     LogStash::Json.dump(action)
         as_json << "\n"
-        if (body_stream.size + as_json.bytesize) > TARGET_BULK_BYTES
-          bulk_responses << bulk_send(body_stream) unless body_stream.size == 0
+        if (stream_writer.pos + as_json.bytesize) > TARGET_BULK_BYTES && stream_writer.pos > 0
+          stream_writer.flush # ensure writer has sync'd buffers before reporting sizes
+          logger.debug("Sending partial bulk request for batch with one or more actions remaining.",
+                       :action_count => batch_actions.size,
+                       :payload_size => stream_writer.pos,
+                       :content_length => body_stream.size,
+                       :batch_offset => (index + 1 - batch_actions.size))
+          bulk_responses << bulk_send(body_stream, batch_actions)
+          body_stream.truncate(0) && body_stream.seek(0)
+          stream_writer = gzip_writer(body_stream) if http_compression
+          batch_actions.clear
         end
         stream_writer.write(as_json)
+        batch_actions << action
       end
       stream_writer.close if http_compression
-      bulk_responses << bulk_send(body_stream) if body_stream.size > 0
+      logger.debug("Sending final bulk request for batch.",
+                   :action_count => batch_actions.size,
+                   :payload_size => stream_writer.pos,
+                   :content_length => body_stream.size,
+                   :batch_offset => (actions.size - batch_actions.size))
+      bulk_responses << bulk_send(body_stream, batch_actions) if body_stream.size > 0
       body_stream.close if !http_compression
       join_bulk_responses(bulk_responses)
     end
 
+    def gzip_writer(io)
+      fail(ArgumentError, "Cannot create gzip writer on IO with unread bytes") unless io.eof?
+      fail(ArgumentError, "Cannot create gzip writer on non-empty IO") unless io.pos == 0
+
+      Zlib::GzipWriter.new(io, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY)
+    end
+
     def join_bulk_responses(bulk_responses)
       {
         "errors" => bulk_responses.any? {|r| r["errors"] == true},
@@ -137,25 +162,37 @@ module LogStash; module Outputs; class ElasticSearch;
       }
     end
 
-    def bulk_send(body_stream)
+    def bulk_send(body_stream, batch_actions)
       params = http_compression ? {:headers => {"Content-Encoding" => "gzip"}} : {}
-      # Discard the URL
       response = @pool.post(@bulk_path, params, body_stream.string)
-      if !body_stream.closed?
-        body_stream.truncate(0)
-        body_stream.seek(0)
-      end
 
       @bulk_response_metrics.increment(response.code.to_s)
 
-      if response.code != 200
+      case response.code
+      when 200 # OK
+        LogStash::Json.load(response.body)
+      when 413 # Payload Too Large
+        logger.warn("Bulk request rejected: `413 Payload Too Large`", :action_count => batch_actions.size, :content_length => body_stream.size)
+        emulate_batch_error_response(batch_actions, response.code, 'payload_too_large')
+      else
         url = ::LogStash::Util::SafeURI.new(response.final_url)
         raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
           response.code, url, body_stream.to_s, response.body
         )
       end
+    end
 
-      LogStash::Json.load(response.body)
+    def emulate_batch_error_response(actions, http_code, reason)
+      {
+          "errors" => true,
+          "items" => actions.map do |action|
+            action = action.first if action.is_a?(Array)
+            request_action, request_parameters = action.first
+            {
+                request_action => {"status" => http_code, "error" => { "type" => reason }}
+            }
+          end
+      }
     end
 
     def get(path)
@@ -349,7 +386,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
     def template_put(name, template)
       path = "#{template_endpoint}/#{name}"
-      logger.info("Installing elasticsearch template to #{path}")
+      logger.info("Installing Elasticsearch template", name: name)
       @pool.put(path, nil, LogStash::Json.dump(template))
     end
 
@@ -366,13 +403,13 @@ module LogStash; module Outputs; class ElasticSearch;
 
     # Create a new rollover alias
     def rollover_alias_put(alias_name, alias_definition)
-      logger.info("Creating rollover alias #{alias_name}")
       begin
         @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition))
+        logger.info("Created rollover alias", name: alias_name)
         # If the rollover alias already exists, ignore the error that comes back from Elasticsearch
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
         if e.response_code == 400
-            logger.info("Rollover Alias #{alias_name} already exists. Skipping")
+            logger.info("Rollover alias already exists, skipping", name: alias_name)
             return
         end
         raise e
@@ -393,13 +430,14 @@ module LogStash; module Outputs; class ElasticSearch;
 
     def ilm_policy_put(name, policy)
       path = "_ilm/policy/#{name}"
-      logger.info("Installing ILM policy #{policy} to #{path}")
+      logger.info("Installing ILM policy #{policy}", name: name)
       @pool.put(path, nil, LogStash::Json.dump(policy))
     end
 
 
     # Build a bulk item for an elasticsearch update action
     def update_action_builder(args, source)
+      args = args.clone()
       if args[:_script]
         # Use the event as a hash from your script with variable name defined
         # by script_var_name (default: "event")

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -1,3 +1,4 @@
+require "concurrent/atomic/atomic_reference"
 require "logstash/plugin_mixins/elasticsearch/noop_license_checker"
 
 module LogStash; module Outputs; class ElasticSearch; class HttpClient;
@@ -71,6 +72,8 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @stopping = false
 
       @license_checker = options[:license_checker] || LogStash::PluginMixins::ElasticSearch::NoopLicenseChecker::INSTANCE
+
+      @last_es_version = Concurrent::AtomicReference.new
     end
 
     def start
@@ -118,12 +121,6 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @state_mutex.synchronize { @url_info }
     end
 
-    def maximum_seen_major_version
-      @state_mutex.synchronize do
-        @maximum_seen_major_version
-      end
-    end
-
     def urls
       url_info.keys
     end
@@ -252,11 +249,12 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       response = perform_request_to_url(url, :get, LICENSE_PATH)
       LogStash::Json.load(response.body)
     rescue => e
-      logger.error("Unable to get license information", url: url.sanitized.to_s, error_type: e.class, error: e.message)
+      logger.error("Unable to get license information", url: url.sanitized.to_s, exception: e.class, message: e.message)
       {}
     end
 
     def health_check_request(url)
+      logger.debug("Running health check to see if an ES connection is working", url: url.sanitized.to_s, path: @healthcheck_path)
       perform_request_to_url(url, :head, @healthcheck_path)
     end
 
@@ -264,29 +262,20 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       # Try to keep locking granularity low such that we don't affect IO...
       @state_mutex.synchronize { @url_info.select {|url,meta| meta[:state] != :alive } }.each do |url,meta|
         begin
-          logger.debug("Running health check to see if an Elasticsearch connection is working",
-                        :healthcheck_url => url, :path => @healthcheck_path)
           health_check_request(url)
           # If no exception was raised it must have succeeded!
-          logger.warn("Restored connection to ES instance", :url => url.sanitized.to_s)
+          logger.warn("Restored connection to ES instance", url: url.sanitized.to_s)
           # We reconnected to this node, check its ES version
           es_version = get_es_version(url)
           @state_mutex.synchronize do
             meta[:version] = es_version
-            major = major_version(es_version)
-            if !@maximum_seen_major_version
-              @logger.info("ES Output version determined", :es_version => major)
-              set_new_major_version(major)
-            elsif major > @maximum_seen_major_version
-              @logger.warn("Detected a node with a higher major version than previously observed. This could be the result of an elasticsearch cluster upgrade.", :previous_major => @maximum_seen_major_version, :new_major => major, :node_url => url.sanitized.to_s)
-              set_new_major_version(major)
-            end
+            set_last_es_version(es_version, url)
 
             alive = @license_checker.appropriate_license?(self, url)
             meta[:state] = alive ? :alive : :dead
           end
         rescue HostUnreachableError, BadResponseCodeError => e
-          logger.warn("Attempted to resurrect connection to dead ES instance, but got an error.", url: url.sanitized.to_s, error_type: e.class, error: e.message)
+          logger.warn("Attempted to resurrect connection to dead ES instance, but got an error", url: url.sanitized.to_s, exception: e.class, message: e.message)
         end
       end
     end
@@ -355,9 +344,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       end
 
       if state_changes[:removed].size > 0 || state_changes[:added].size > 0
-        if logger.info?
-          logger.info("Elasticsearch pool URLs updated", :changes => state_changes)
-        end
+        logger.info? && logger.info("Elasticsearch pool URLs updated", :changes => state_changes)
       end
       
       # Run an inline healthcheck anytime URLs are updated
@@ -371,10 +358,6 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @state_mutex.synchronize { @url_info.size }
     end
 
-    def es_versions
-      @state_mutex.synchronize { @url_info.size }
-    end
-
     def add_url(url)
       @url_info[url] ||= empty_url_meta
     end
@@ -459,22 +442,52 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
 
     def return_connection(url)
       @state_mutex.synchronize do
-        if @url_info[url] # Guard against the condition where the connection has already been deleted
-          @url_info[url][:in_use] -= 1
-        end
+        info = @url_info[url]
+        info[:in_use] -= 1 if info # Guard against the condition where the connection has already been deleted
       end
     end
 
     def get_es_version(url)
       request = perform_request_to_url(url, :get, ROOT_URI_PATH)
-      LogStash::Json.load(request.body)["version"]["number"]
+      LogStash::Json.load(request.body)["version"]["number"] # e.g. "7.10.0"
+    end
+
+    def last_es_version
+      @last_es_version.get
+    end
+
+    def maximum_seen_major_version
+      @state_mutex.synchronize { @maximum_seen_major_version }
+    end
+
+    private
+
+    # @private executing within @state_mutex
+    def set_last_es_version(version, url)
+      @last_es_version.set(version)
+
+      major = major_version(version)
+      if @maximum_seen_major_version.nil?
+        @logger.info("Elasticsearch version determined (#{version})", es_version: major)
+        set_maximum_seen_major_version(major)
+      elsif major > @maximum_seen_major_version
+        warn_on_higher_major_version(major, url)
+        @maximum_seen_major_version = major
+      end
     end
 
-    def set_new_major_version(version)
-      @maximum_seen_major_version = version
-      if @maximum_seen_major_version >= 6
-        @logger.warn("Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type", :es_version => @maximum_seen_major_version)
+    def set_maximum_seen_major_version(major)
+      if major >= 6
+        @logger.warn("Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type", es_version: major)
       end
+      @maximum_seen_major_version = major
     end
+
+    def warn_on_higher_major_version(major, url)
+      @logger.warn("Detected a node with a higher major version than previously observed, " +
+                   "this could be the result of an Elasticsearch cluster upgrade",
+                   previous_major: @maximum_seen_major_version, new_major: major, node_url: url.sanitized.to_s)
+    end
+
   end
 end; end; end; end;