logstash-output-elasticsearch-test 11.16.0-x86_64-linux

data/lib/logstash/outputs/elasticsearch/data_stream_support.rb

@@ -0,0 +1,282 @@
+module LogStash module Outputs class ElasticSearch
+  # DS specific behavior/configuration.
+  module DataStreamSupport
+
+    # @api private
+    ENABLING_ECS_GUIDANCE = <<~END.tr("\n", " ")
+      Elasticsearch data streams require that events adhere to the Elastic Common Schema.
+      While `ecs_compatibility` can be set for this individual Elasticsearch output plugin, doing so will not fix schema conflicts caused by upstream plugins in your pipeline.
+      To avoid mapping conflicts, you will need to use ECS-compatible field names and datatypes throughout your pipeline.
+      Many plugins support an `ecs_compatibility` mode, and the `pipeline.ecs_compatibility` setting can be used to opt-in for all plugins in a pipeline.
+    END
+    private_constant :ENABLING_ECS_GUIDANCE
+
+    def self.included(base)
+      # Defines whether data will be indexed into an Elasticsearch data stream,
+      # `data_stream_*` settings will only be used if this setting is enabled!
+      # This setting supports values `true`, `false`, and `auto`.
+      # Defaults to `false` in Logstash 7.x and `auto` starting in Logstash 8.0.
+      base.config :data_stream, :validate => ['true', 'false', 'auto']
+
+      base.config :data_stream_type, :validate => ['logs', 'metrics', 'synthetics', 'traces'], :default => 'logs'
+      base.config :data_stream_dataset, :validate => :dataset_identifier, :default => 'generic'
+      base.config :data_stream_namespace, :validate => :namespace_identifier, :default => 'default'
+
+      base.config :data_stream_sync_fields, :validate => :boolean, :default => true
+      base.config :data_stream_auto_routing, :validate => :boolean, :default => true
+
+      base.extend(Validator)
+    end
+
+    def data_stream_config?
+      @data_stream_config.nil? ? @data_stream_config = check_data_stream_config! : @data_stream_config
+    end
+
+    private
+
+    def data_stream_name(event)
+      data_stream = event.get('data_stream')
+      return @index if !data_stream_auto_routing || !data_stream.is_a?(Hash)
+
+      type = data_stream['type'] || data_stream_type
+      dataset = data_stream['dataset'] || data_stream_dataset
+      namespace = data_stream['namespace'] || data_stream_namespace
+      "#{type}-#{dataset}-#{namespace}"
+    end
+
+    DATA_STREAMS_AND_ECS_ENABLED_BY_DEFAULT_LS_VERSION = '8.0.0'
+
+    # @param params the user configuration for the ES output
+    # @note LS initialized configuration (with filled defaults) won't detect as data-stream
+    # compatible, only explicit (`original_params`) config should be tested.
+    # @return [Boolean] whether given configuration is data-stream compatible
+    def check_data_stream_config!(params = original_params)
+      case data_stream_explicit_value
+      when false
+        check_disabled_data_stream_config!(params)
+        return false
+      when true
+        check_enabled_data_stream_config!(params)
+        return true
+      else # data_stream => auto or not set
+        use_data_stream = data_stream_default(params)
+
+        check_disabled_data_stream_config!(params) unless use_data_stream
+
+        @logger.info("Data streams auto configuration (`data_stream => auto` or unset) resolved to `#{use_data_stream}`")
+        return use_data_stream
+      end
+    end
+
+    def check_enabled_data_stream_config!(params)
+      invalid_data_stream_params = invalid_data_stream_params(params)
+
+      if invalid_data_stream_params.any?
+        @logger.error "Invalid data stream configuration, the following parameters are not supported:", invalid_data_stream_params
+        raise LogStash::ConfigurationError, "Invalid data stream configuration: #{invalid_data_stream_params.keys}"
+      end
+
+      if ecs_compatibility == :disabled
+        if ecs_compatibility_required?
+          @logger.error "Invalid data stream configuration; `ecs_compatibility` must not be `disabled`. " + ENABLING_ECS_GUIDANCE
+          raise LogStash::ConfigurationError, "Invalid data stream configuration: `ecs_compatibility => disabled`"
+        end
+
+        @deprecation_logger.deprecated "In a future release of Logstash, the Elasticsearch output plugin's `data_stream => true` will require the plugin to be run in ECS compatibility mode. " + ENABLING_ECS_GUIDANCE
+      end
+    end
+
+    def check_disabled_data_stream_config!(params)
+      data_stream_params = data_stream_params(params)
+
+      if data_stream_params.any?
+        @logger.error "Ambiguous configuration; data stream settings must not be present when data streams are disabled (caused by `data_stream => false`, `data_stream => auto` or unset resolved to false). " \
+                      "You can either manually set `data_stream => true` or remove the following specific data stream settings: ", data_stream_params
+
+        raise LogStash::ConfigurationError,
+              "Ambiguous configuration; data stream settings must not be present when data streams are disabled: #{data_stream_params.keys}"
+      end
+    end
+
+    def data_stream_params(params)
+      params.select { |name, _| name.start_with?('data_stream_') }
+    end
+
+    def data_stream_explicit_value
+      case @data_stream
+      when 'true'
+        return true
+      when 'false'
+        return false
+      else
+        return nil # 'auto' or not set by user
+      end
+    end
+
+    def invalid_data_stream_params(params)
+      shared_params = LogStash::PluginMixins::ElasticSearch::APIConfigs::CONFIG_PARAMS.keys.map(&:to_s)
+      params.reject do |name, value|
+        # NOTE: intentionally do not support explicit DS configuration like:
+        # - `index => ...` identifier provided by data_stream_xxx settings
+        case name
+        when 'action'
+          value == 'create'
+        when 'routing', 'pipeline'
+          true
+        when 'data_stream'
+          value.to_s == 'true'
+        when 'manage_template'
+          value.to_s == 'false'
+        when 'ecs_compatibility' then true # required for LS <= 6.x
+        else
+          name.start_with?('data_stream_') ||
+              shared_params.include?(name) ||
+                inherited_internal_config_param?(name) # 'id', 'enabled_metric' etc
+        end
+      end
+    end
+
+    def inherited_internal_config_param?(name)
+      self.class.superclass.get_config.key?(name.to_s) # superclass -> LogStash::Outputs::Base
+    end
+
+    DATA_STREAMS_ORIGIN_ES_VERSION = '7.9.0'
+
+    # @note assumes to be running AFTER {after_successful_connection} completed, due ES version checks
+    # @return [Gem::Version] if ES supports DS nil (or raise) otherwise
+    def assert_es_version_supports_data_streams
+      fail 'no last_es_version' unless last_es_version # assert - should not happen
+      es_version = ::Gem::Version.create(last_es_version)
+      if es_version < ::Gem::Version.create(DATA_STREAMS_ORIGIN_ES_VERSION)
+        @logger.error "Elasticsearch version does not support data streams, Logstash might end up writing to an index", es_version: es_version.version
+        # NOTE: when switching to synchronous check from register, this should be a ConfigurationError
+        raise LogStash::Error, "A data_stream configuration is only supported since Elasticsearch #{DATA_STREAMS_ORIGIN_ES_VERSION} " +
+                               "(detected version #{es_version.version}), please upgrade your cluster"
+      end
+      es_version # return truthy
+    end
+
+    # when data_stream => is either 'auto' or not set
+    def data_stream_default(params)
+      if ecs_compatibility == :disabled
+        @logger.info("Not eligible for data streams because ecs_compatibility is not enabled. " + ENABLING_ECS_GUIDANCE)
+        return false
+      end
+
+      invalid_data_stream_params = invalid_data_stream_params(params)
+
+      if data_stream_and_ecs_enabled_by_default?
+        if invalid_data_stream_params.any?
+          @logger.info("Not eligible for data streams because config contains one or more settings that are not compatible with data streams: #{invalid_data_stream_params.inspect}")
+          return false
+        end
+
+        return true
+      end
+
+      # LS 7.x
+      if !invalid_data_stream_params.any? && !data_stream_params(params).any?
+        @logger.warn "Configuration is data stream compliant but due backwards compatibility Logstash 7.x will not assume " +
+                     "writing to a data-stream, default behavior will change on Logstash 8.0 " +
+                     "(set `data_stream => true/false` to disable this warning)"
+      end
+      false
+    end
+
+    def ecs_compatibility_required?
+      data_stream_and_ecs_enabled_by_default?
+    end
+
+    def data_stream_and_ecs_enabled_by_default?
+      ::Gem::Version.create(LOGSTASH_VERSION) >= ::Gem::Version.create(DATA_STREAMS_AND_ECS_ENABLED_BY_DEFAULT_LS_VERSION)
+    end
+
+    # an {event_action_tuple} replacement when a data-stream configuration is detected
+    def data_stream_event_action_tuple(event)
+      event_data = event.to_hash
+      data_stream_event_sync(event_data) if data_stream_sync_fields
+      EventActionTuple.new('create', common_event_params(event), event, event_data)
+    end
+
+    DATA_STREAM_SYNC_FIELDS = [ 'type', 'dataset', 'namespace' ].freeze
+
+    def data_stream_event_sync(event_data)
+      data_stream = event_data['data_stream']
+      if data_stream.is_a?(Hash)
+        unless data_stream_auto_routing
+          sync_fields = DATA_STREAM_SYNC_FIELDS.select { |name| data_stream.key?(name) && data_stream[name] != send(:"data_stream_#{name}") }
+          if sync_fields.any? # these fields will need to be overwritten
+            info = sync_fields.inject({}) { |info, name| info[name] = data_stream[name]; info }
+            info[:event] = event_data
+            @logger.warn "Some data_stream fields are out of sync, these will be updated to reflect data-stream name", info
+
+            # NOTE: we work directly with event.to_hash data thus fine to mutate the 'data_stream' hash
+            sync_fields.each { |name| data_stream[name] = nil } # fallback to ||= bellow
+          end
+        end
+      else
+        unless data_stream.nil?
+          @logger.warn "Invalid 'data_stream' field type, due fields sync will overwrite", value: data_stream, event: event_data
+        end
+        event_data['data_stream'] = data_stream = Hash.new
+      end
+
+      data_stream['type'] ||= data_stream_type
+      data_stream['dataset'] ||= data_stream_dataset
+      data_stream['namespace'] ||= data_stream_namespace
+
+      event_data
+    end
+
+    module Validator
+
+      # @override {LogStash::Config::Mixin::validate_value} to handle custom validators
+      # @param value [Array<Object>]
+      # @param validator [nil,Array,Symbol]
+      # @return [Array(true,Object)]: if validation is a success, a tuple containing `true` and the coerced value
+      # @return [Array(false,String)]: if validation is a failure, a tuple containing `false` and the failure reason.
+      def validate_value(value, validator)
+        case validator
+        when :dataset_identifier   then validate_dataset_identifier(value)
+        when :namespace_identifier then validate_namespace_identifier(value)
+        else super
+        end
+      end
+
+      private
+
+      def validate_dataset_identifier(value)
+        valid, value = validate_value(value, :string)
+        return false, value unless valid
+
+        validate_identifier(value)
+      end
+
+      def validate_namespace_identifier(value)
+        valid, value = validate_value(value, :string)
+        return false, value unless valid
+
+        validate_identifier(value)
+      end
+
+      def validate_identifier(value, max_size = 100)
+        if value.empty?
+          return false, "Invalid identifier - empty string"
+        end
+        if value.bytesize > max_size
+          return false, "Invalid identifier - too long (#{value.bytesize} bytes)"
+        end
+        # cannot include \, /, *, ?, ", <, >, |, ' ' (space char), ',', #, :
+        if value.match? Regexp.union(INVALID_IDENTIFIER_CHARS)
+          return false, "Invalid characters detected #{INVALID_IDENTIFIER_CHARS.inspect} are not allowed"
+        end
+        return true, value
+      end
+
+      INVALID_IDENTIFIER_CHARS = [ '\\', '/', '*', '?', '"', '<', '>', '|', ' ', ',', '#', ':' ]
+      private_constant :INVALID_IDENTIFIER_CHARS
+
+    end
+
+  end
+end end end

data/lib/logstash/outputs/elasticsearch/default-ilm-policy.json

@@ -0,0 +1,14 @@
+{
+  "policy" : {
+    "phases": {
+      "hot" : {
+        "actions" : {
+          "rollover" : {
+            "max_size" : "50gb",
+            "max_age":"30d"
+          }
+        }
+      }
+    }
+  }
+}

data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -0,0 +1,155 @@
+require 'manticore'
+require 'cgi'
+
+module LogStash; module Outputs; class ElasticSearch; class HttpClient;
+  DEFAULT_HEADERS = { "Content-Type" => "application/json" }
+  
+  class ManticoreAdapter
+    attr_reader :manticore, :logger
+
+    def initialize(logger, options)
+      @logger = logger
+      options = options.dup
+      options[:ssl] = options[:ssl] || {}
+
+      # We manage our own retries directly, so let's disable them here
+      options[:automatic_retries] = 0
+      # We definitely don't need cookies
+      options[:cookies] = false
+
+      @client_params = {:headers => DEFAULT_HEADERS.merge(options[:headers] || {})}
+      
+      if options[:proxy]
+        options[:proxy] = manticore_proxy_hash(options[:proxy])
+      end
+      
+      @manticore = ::Manticore::Client.new(options)
+    end
+    
+    # Transform the proxy option to a hash. Manticore's support for non-hash
+    # proxy options is broken. This was fixed in https://github.com/cheald/manticore/commit/34a00cee57a56148629ed0a47c329181e7319af5
+    # but this is not yet released
+    def manticore_proxy_hash(proxy_uri)
+      [:scheme, :port, :user, :password, :path].reduce(:host => proxy_uri.host) do |acc,opt|
+        value = proxy_uri.send(opt)
+        acc[opt] = value unless value.nil? || (value.is_a?(String) && value.empty?)
+        acc
+      end
+    end
+
+    def client
+      @manticore
+    end
+
+    # Performs the request by invoking {Transport::Base#perform_request} with a block.
+    #
+    # @return [Response]
+    # @see    Transport::Base#perform_request
+    #
+    def perform_request(url, method, path, params={}, body=nil)
+      # Perform 2-level deep merge on the params, so if the passed params and client params will both have hashes stored on a key they
+      # will be merged as well, instead of choosing just one of the values
+      params = (params || {}).merge(@client_params) { |key, oldval, newval|
+        (oldval.is_a?(Hash) && newval.is_a?(Hash)) ? oldval.merge(newval) : newval
+      }
+      params[:body] = body if body
+
+      if url.user
+        params[:auth] = { 
+          :user => CGI.unescape(url.user),
+          # We have to unescape the password here since manticore won't do it
+          # for us unless its part of the URL
+          :password => CGI.unescape(url.password), 
+          :eager => true 
+        }
+      end
+
+      request_uri = format_url(url, path)
+      request_uri_as_string = remove_double_escaping(request_uri.to_s)
+      begin
+        resp = @manticore.send(method.downcase, request_uri_as_string, params)
+        # Manticore returns lazy responses by default
+        # We want to block for our usage, this will wait for the response to finish
+        resp.call
+      rescue ::Manticore::ManticoreException => e
+        log_request_error(e)
+        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new(e, request_uri_as_string)
+      end
+
+      # 404s are excluded because they are valid codes in the case of
+      # template installation. We might need a better story around this later
+      # but for our current purposes this is correct
+      code = resp.code
+      if code < 200 || code > 299 && code != 404
+        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(code, request_uri, body, resp.body)
+      end
+
+      resp
+    end
+
+    def log_request_error(e)
+      details = { message: e.message, exception: e.class }
+      details[:cause] = e.cause if e.respond_to?(:cause)
+      details[:backtrace] = e.backtrace if @logger.debug?
+
+      level = case e
+      when ::Manticore::Timeout
+        :debug
+      when ::Manticore::UnknownException
+        :warn
+      else
+        :info
+      end
+
+      @logger.send level, "Failed to perform request", details
+      log_java_exception(details[:cause], :debug) if details[:cause] && @logger.debug?
+    end
+
+    def log_java_exception(e, level = :debug)
+      return unless e.is_a?(java.lang.Exception)
+      # @logger.name using the same convention as LS does
+      logger = self.class.name.gsub('::', '.').downcase
+      logger = org.apache.logging.log4j.LogManager.getLogger(logger)
+      logger.send(level, '', e) # logger.error('', e) - prints nested causes
+    end
+
+    # Returned urls from this method should be checked for double escaping.
+    def format_url(url, path_and_query=nil)
+      request_uri = url.clone
+      
+      # We excise auth info from the URL in case manticore itself tries to stick
+      # sensitive data in a thrown exception or log data
+      request_uri.user = nil
+      request_uri.password = nil
+
+      return request_uri.to_s if path_and_query.nil?
+
+      parsed_path_and_query = java.net.URI.new(path_and_query)
+
+      new_query_parts = [request_uri.query, parsed_path_and_query.query].select do |part|
+        part && !part.empty? # Skip empty nil and ""
+      end
+      
+      request_uri.query = new_query_parts.join("&") unless new_query_parts.empty?
+
+      # use `raw_path`` as `path` will unescape any escaped '/' in the path
+      request_uri.path = "#{request_uri.path}/#{parsed_path_and_query.raw_path}".gsub(/\/{2,}/, "/")
+      request_uri
+    end
+
+    # Later versions of SafeURI will also escape the '%' sign in an already escaped URI.
+    # (If the path variable is used, it constructs a new java.net.URI object using the multi-arg constructor,
+    # which will escape any '%' characters in the path, as opposed to the single-arg constructor which requires illegal
+    # characters to be already escaped, and will throw otherwise)
+    # The URI needs to have been previously escaped, as it does not play nice with an escaped '/' in the
+    # middle of a URI, as required by date math, treating it as a path separator
+    def remove_double_escaping(url)
+      url.gsub(/%25([0-9A-F]{2})/i, '%\1')
+    end
+
+    def close
+      @manticore.close
+    end
+
+  end
+end; end; end; end