logstash-output-elasticsearch 11.3.0-java → 11.4.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a9b6a7eacf857e3f0e0953d2aa1628f3ea50f9928d73dc71fd0d09db2f2cd147
-  data.tar.gz: 16c0222ff642f203c5a9c0553faac90b6fbf02875910191cd9b921e07b52f1df
+  metadata.gz: 3932944b80f3ae81063657ab9255cb9504624d1c6c0634c547b69b38d4a9b4a9
+  data.tar.gz: b850f5abbe8ae00b62e379d6f7c90802172602f23a3ab0e3f67b1a4d8319bbea
 SHA512:
-  metadata.gz: 15b8d3c36d59e2bdb087058e24a85ddd67ad09f2708ccad357b032b7c3e6bf04beb224138738ca3be09a28cd5e24fb9d5b2883b510ee406500393132825d79f1
-  data.tar.gz: e1cc0b009714cead91f295baf08dee61fb76938abd2c1456a48440d78ab62d4beffffd761d4a831057757d090052e8bb83d670ad72805171435b056f1184500f
+  metadata.gz: 5bc64de78f7ca0808e377595774f63b874d753801bbee8722bcd58af6a556e9c8baae683359179b65ee5a282c321c40dbeda838a9caacb5fc7ff14e7fe63e294
+  data.tar.gz: 0ff0a9731418cb7858e51109c8392d4a7692629d85141eedfa7019d99bbc12d7b9f016bd21d976f390ba5d745e039be6998742107b0eded2525a8c4dc7d0bf5f

data/CHANGELOG.md

@@ -1,13 +1,30 @@
+## 11.4.0
+ - Updates ECS templates [#1062](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1062)
+   - Updates v1 templates to 1.12.1 for use with Elasticsearch 7.x and 8.x
+   - Updates BETA preview of ECS v8 templates for Elasticsearch 7.x and 8.x
+
+## 11.3.3
+ - Feat: add support for 'traces' data stream type [#1057](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1057)
+
+## 11.3.2
+ - Refactor: review manticore error handling/logging, logging originating cause in case of connection related error when debug level is enabled [#1029](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1029)
+   - Java causes on connection related exceptions will now be extra logged when plugin is logging at debug level
+
+## 11.3.1
+ - ECS-related fixes [#1046](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1046)
+   - Data Streams requirement on ECS is properly enforced when running on Logstash 8, and warned about when running on Logstash 7.
+   - ECS Compatibility v8 can now be selected
+
 ## 11.3.0
  - Adds ECS templates [#1048](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1048)
    - Adds templates for ECS v1 for Elasticsearch 8.x
    - Adds templates for BETA preview of ECS v8 for both Elasticsearch 7.x and 8.x
 
 ## 11.2.3
- - Downgrade ECS templates, pinning to v1.10.0 of upstream; fixes an issue where ECS templates cannot be installed in Elasticsearch 6.x or 7.1-7.2, since the generated templates include fields of `type: flattened` that was introduced in Elasticsearch 7.3
+ - Downgrade ECS templates, pinning to v1.10.0 of upstream; fixes an issue where ECS templates cannot be installed in Elasticsearch 6.x or 7.1-7.2, since the generated templates include fields of `type: flattened` that was introduced in Elasticsearch 7.3. [#1049](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1049)
 
 ## 11.2.2
- - Update ECS templates from upstream; `ecs_compatiblity => v1` now resolves to templates for ECS v1.12.1 [#1027](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/1027)
+ - Update ECS templates from upstream; `ecs_compatiblity => v1` now resolves to templates for ECS v1.12.1 [#1047](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1047). Fixes [#1027](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/1027)
 
 ## 11.2.1
  - Fix referencing Gem classes from global lexical scope [#1044](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1044)

data/docs/index.asciidoc

@@ -117,12 +117,8 @@ output {
 
 ==== Writing to different indices: best practices
 
-[NOTE]
-================================================================================
-You cannot use dynamic variable substitution when `ilm_enabled` is `true` and
-when using `ilm_rollover_alias`.
-
-================================================================================
+NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` 
+and when using `ilm_rollover_alias`.
 
 If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can:
 
@@ -509,7 +505,7 @@ overwritten with a warning.
 * Default value is `logs`.
 
 The data stream type used to construct the data stream at index time.
-Currently, only `logs`, `metrics` and `synthetics` are supported.
+Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported.
 
 [id="plugins-{type}s-{plugin}-doc_as_upsert"]
 ===== `doc_as_upsert`

data/lib/logstash/outputs/elasticsearch/data_stream_support.rb

@@ -2,6 +2,15 @@ module LogStash module Outputs class ElasticSearch
   # DS specific behavior/configuration.
   module DataStreamSupport
 
+    # @api private
+    ENABLING_ECS_GUIDANCE = <<~END.tr("\n", " ")
+      Elasticsearch data streams require that events adhere to the Elastic Common Schema.
+      While `ecs_compatibility` can be set for this individual Elasticsearch output plugin, doing so will not fix schema conflicts caused by upstream plugins in your pipeline.
+      To avoid mapping conflicts, you will need to use ECS-compatible field names and datatypes throughout your pipeline.
+      Many plugins support an `ecs_compatibility` mode, and the `pipeline.ecs_compatibility` setting can be used to opt-in for all plugins in a pipeline.
+    END
+    private_constant :ENABLING_ECS_GUIDANCE
+
     def self.included(base)
       # Defines whether data will be indexed into an Elasticsearch data stream,
       # `data_stream_*` settings will only be used if this setting is enabled!
@@ -9,7 +18,7 @@ module LogStash module Outputs class ElasticSearch
       # Defaults to `false` in Logstash 7.x and `auto` starting in Logstash 8.0.
       base.config :data_stream, :validate => ['true', 'false', 'auto']
 
-      base.config :data_stream_type, :validate => ['logs', 'metrics', 'synthetics'], :default => 'logs'
+      base.config :data_stream_type, :validate => ['logs', 'metrics', 'synthetics', 'traces'], :default => 'logs'
       base.config :data_stream_dataset, :validate => :dataset_identifier, :default => 'generic'
       base.config :data_stream_namespace, :validate => :namespace_identifier, :default => 'default'
 
@@ -36,6 +45,8 @@ module LogStash module Outputs class ElasticSearch
       "#{type}-#{dataset}-#{namespace}"
     end
 
+    DATA_STREAMS_REQUIRES_ECS_LS_VERSION = '8.0.0'
+
     # @param params the user configuration for the ES output
     # @note LS initialized configuration (with filled defaults) won't detect as data-stream
     # compatible, only explicit (`original_params`) config should be tested.
@@ -56,14 +67,26 @@ module LogStash module Outputs class ElasticSearch
           @logger.error "Invalid data stream configuration, following parameters are not supported:", invalid_data_stream_params
           raise LogStash::ConfigurationError, "Invalid data stream configuration: #{invalid_data_stream_params.keys}"
         end
+        if ecs_compatibility == :disabled
+          if ::Gem::Version.create(LOGSTASH_VERSION) < ::Gem::Version.create(DATA_STREAMS_REQUIRES_ECS_LS_VERSION)
+            @deprecation_logger.deprecated "In a future release of Logstash, the Elasticsearch output plugin's `data_stream => true` will require the plugin to be run in ECS compatibility mode. " + ENABLING_ECS_GUIDANCE
+          else
+            @logger.error "Invalid data stream configuration; `ecs_compatibility` must not be `disabled`. " + ENABLING_ECS_GUIDANCE
+            raise LogStash::ConfigurationError, "Invalid data stream configuration: `ecs_compatibility => disabled`"
+          end
+        end
         return true
       else
-        use_data_stream = data_stream_default(data_stream_params, invalid_data_stream_params.empty?)
-        if !use_data_stream && data_stream_params.any?
+        use_data_stream = data_stream_default(data_stream_params, invalid_data_stream_params)
+        if use_data_stream
+          @logger.info("Config is compliant with data streams. `data_stream => auto` resolved to `true`")
+        elsif data_stream_params.any?
           # DS (auto) disabled but there's still some data-stream parameters (and no `data_stream => false`)
           @logger.error "Ambiguous configuration; data stream settings are present, but data streams are not enabled", data_stream_params
           raise LogStash::ConfigurationError, "Ambiguous configuration, please set data_stream => true " +
               "or remove data stream specific settings: #{data_stream_params.keys}"
+        else
+          @logger.info("Config is not compliant with data streams. `data_stream => auto` resolved to `false`")
         end
         use_data_stream
       end
@@ -93,6 +116,7 @@ module LogStash module Outputs class ElasticSearch
           true
         when 'data_stream'
           value.to_s == 'true'
+        when 'ecs_compatibility' then true # required for LS <= 6.x
         else
           name.start_with?('data_stream_') ||
               shared_params.include?(name) ||
@@ -110,8 +134,8 @@ module LogStash module Outputs class ElasticSearch
     # @return [Gem::Version] if ES supports DS nil (or raise) otherwise
     def assert_es_version_supports_data_streams
       fail 'no last_es_version' unless last_es_version # assert - should not happen
-      es_version = Gem::Version.create(last_es_version)
-      if es_version < Gem::Version.create(DATA_STREAMS_ORIGIN_ES_VERSION)
+      es_version = ::Gem::Version.create(last_es_version)
+      if es_version < ::Gem::Version.create(DATA_STREAMS_ORIGIN_ES_VERSION)
         @logger.error "Elasticsearch version does not support data streams, Logstash might end up writing to an index", es_version: es_version.version
         # NOTE: when switching to synchronous check from register, this should be a ConfigurationError
         raise LogStash::Error, "A data_stream configuration is only supported since Elasticsearch #{DATA_STREAMS_ORIGIN_ES_VERSION} " +
@@ -123,18 +147,28 @@ module LogStash module Outputs class ElasticSearch
     DATA_STREAMS_ENABLED_BY_DEFAULT_LS_VERSION = '8.0.0'
 
     # when data_stream => is either 'auto' or not set
-    def data_stream_default(data_stream_params, valid_data_stream_config)
-      ds_default = Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create(DATA_STREAMS_ENABLED_BY_DEFAULT_LS_VERSION)
+    # @param data_stream_params [#any?]
+    # @param invalid_data_stream_config [#any?#inspect]
+    def data_stream_default(data_stream_params, invalid_data_stream_config)
+      if ecs_compatibility == :disabled
+        @logger.debug("Not eligible for data streams because ecs_compatibility is not enabled. " + ENABLING_ECS_GUIDANCE)
+        return false
+      end
+
+      ds_default = ::Gem::Version.create(LOGSTASH_VERSION) >= ::Gem::Version.create(DATA_STREAMS_ENABLED_BY_DEFAULT_LS_VERSION)
 
       if ds_default # LS 8.0
-        return false unless valid_data_stream_config
+        if invalid_data_stream_config.any?
+          @logger.debug("Not eligible for data streams because config contains one or more settings that are not compatible with data streams: #{invalid_data_stream_config.inspect}")
+          return false
+        end
 
         @logger.debug 'Configuration is data stream compliant'
         return true
       end
 
       # LS 7.x
-      if valid_data_stream_config && !data_stream_params.any?
+      if !invalid_data_stream_config.any? && !data_stream_params.any?
         @logger.warn "Configuration is data stream compliant but due backwards compatibility Logstash 7.x will not assume " +
                      "writing to a data-stream, default behavior will change on Logstash 8.0 " +
                      "(set `data_stream => true/false` to disable this warning)"

data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -7,9 +7,9 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
   class ManticoreAdapter
     attr_reader :manticore, :logger
 
-    def initialize(logger, options={})
+    def initialize(logger, options)
       @logger = logger
-      options = options.clone || {}
+      options = options.dup
       options[:ssl] = options[:ssl] || {}
 
       # We manage our own retries directly, so let's disable them here
@@ -66,23 +66,53 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
 
       request_uri = format_url(url, path)
       request_uri_as_string = remove_double_escaping(request_uri.to_s)
-      resp = @manticore.send(method.downcase, request_uri_as_string, params)
-
-      # Manticore returns lazy responses by default
-      # We want to block for our usage, this will wait for the repsonse
-      # to finish
-      resp.call
+      begin
+        resp = @manticore.send(method.downcase, request_uri_as_string, params)
+        # Manticore returns lazy responses by default
+        # We want to block for our usage, this will wait for the response to finish
+        resp.call
+      rescue ::Manticore::ManticoreException => e
+        log_request_error(e)
+        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new(e, request_uri_as_string)
+      end
 
       # 404s are excluded because they are valid codes in the case of
       # template installation. We might need a better story around this later
       # but for our current purposes this is correct
-      if resp.code < 200 || resp.code > 299 && resp.code != 404
-        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(resp.code, request_uri, body, resp.body)
+      code = resp.code
+      if code < 200 || code > 299 && code != 404
+        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(code, request_uri, body, resp.body)
       end
 
       resp
     end
 
+    def log_request_error(e)
+      details = { message: e.message, exception: e.class }
+      details[:cause] = e.cause if e.respond_to?(:cause)
+      details[:backtrace] = e.backtrace if @logger.debug?
+
+      level = case e
+      when ::Manticore::Timeout
+        :debug
+      when ::Manticore::UnknownException
+        :warn
+      else
+        :info
+      end
+
+      @logger.send level, "Failed to perform request", details
+      log_java_exception(details[:cause], :debug) if details[:cause] && @logger.debug?
+    end
+
+    def log_java_exception(e, level = :debug)
+      return unless e.is_a?(java.lang.Exception)
+      # @logger.name using the same convention as LS does
+      logger = self.class.name.gsub('::', '.').downcase
+      logger = org.apache.logging.log4j.LogManager.getLogger(logger)
+      logger.send(level, '', e) # logger.error('', e) - prints nested causes
+    end
+
     # Returned urls from this method should be checked for double escaping.
     def format_url(url, path_and_query=nil)
       request_uri = url.clone
@@ -96,9 +126,6 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
 
       parsed_path_and_query = java.net.URI.new(path_and_query)
 
-      query = request_uri.query
-      parsed_query = parsed_path_and_query.query
-
       new_query_parts = [request_uri.query, parsed_path_and_query.query].select do |part|
         part && !part.empty? # Skip empty nil and ""
       end
@@ -124,8 +151,5 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @manticore.close
     end
 
-    def host_unreachable_exceptions
-      [::Manticore::Timeout,::Manticore::SocketException, ::Manticore::ClientProtocolException, ::Manticore::ResolutionFailure, Manticore::SocketTimeout]
-    end
   end
 end; end; end; end

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -8,27 +8,25 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       attr_reader :url, :response_code, :request_body, :response_body
 
       def initialize(response_code, url, request_body, response_body)
+        super("Got response code '#{response_code}' contacting Elasticsearch at URL '#{url}'")
+
         @response_code = response_code
         @url = url
         @request_body = request_body
         @response_body = response_body
       end
 
-      def message
-        "Got response code '#{response_code}' contacting Elasticsearch at URL '#{@url}'"
-      end
     end
     class HostUnreachableError < Error;
       attr_reader :original_error, :url
 
       def initialize(original_error, url)
+        super("Elasticsearch Unreachable: [#{url}][#{original_error.class}] #{original_error.message}")
+
         @original_error = original_error
         @url = url
       end
 
-      def message
-        "Elasticsearch Unreachable: [#{@url}][#{original_error.class}] #{original_error.message}"
-      end
     end
 
     attr_reader :logger, :adapter, :sniffing, :sniffer_delay, :resurrect_delay, :healthcheck_path, :sniffing_path, :bulk_path
@@ -323,9 +321,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def perform_request_to_url(url, method, path, params={}, body=nil)
-      res = @adapter.perform_request(url, method, path, params, body)
-    rescue *@adapter.host_unreachable_exceptions => e
-      raise HostUnreachableError.new(e, url), "Could not reach host #{e.class}: #{e.message}"
+      @adapter.perform_request(url, method, path, params, body)
     end
 
     def normalize_url(uri)

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -322,8 +322,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
       adapter_options[:headers] = client_settings[:headers] if client_settings[:headers]
 
-      adapter_class = ::LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter
-      adapter = adapter_class.new(@logger, adapter_options)
+      ::LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(@logger, adapter_options)
     end
 
     def prepare_user_agent