logstash-output-elasticsearch 11.2.1-java → 11.3.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 55d5bf15c3c83c50fc630b6de9b6e150b84bc82ef981b18b62043294323be07a
-  data.tar.gz: ed957435bcf63f85835d7eabf5f5a5c249f8dfe28183d204574bc8553e960907
+  metadata.gz: dccee6612b454d47b0596a9feb04db71f51f368a34906f559b8f7a8df7cf166e
+  data.tar.gz: 27d8ab736dd7c3ffa46d1229ad70afd5e8b23813ee09aa3e9d4ec05b4f263a76
 SHA512:
-  metadata.gz: 664ba0d3d5e26089b8a9b29a87cec5fb17da3ae7ead430b10572ca540236da0918f24a4df874d499074060683b8f57937fb5d7239c7c32778a51728d1378b431
-  data.tar.gz: d513fa0801d91b80b3434c7f7576e7c9874c7f36a699d35408981013a64a0762309371f2aeaecac5720e82225a44301c435f1328d63b1d056b6beccf0b5163db
+  metadata.gz: d7ab3e32387faf0cc2680fc1446e9a330314d2ead466487fd6f63a27ab2757cac8c2d90cb41bb0eba422986b22821d0e0c813617ba153225058b1e1d09765598
+  data.tar.gz: df0a5b6cbcbdc592e1c18906f6a54a97d309afbbd1c36529f812501df31f78c7e4c2ba2fb8a3eab72e9fd53dfbca8388b710dc772a20ba9084c59ce1f5aa43fc

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+## 11.3.1
+ - ECS-related fixes [#1046](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1046)
+   - Data Streams requirement on ECS is properly enforced when running on Logstash 8, and warned about when running on Logstash 7.
+   - ECS Compatibility v8 can now be selected
+
+## 11.3.0
+ - Adds ECS templates [#1048](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1048)
+   - Adds templates for ECS v1 for Elasticsearch 8.x
+   - Adds templates for BETA preview of ECS v8 for both Elasticsearch 7.x and 8.x
+
+## 11.2.3
+ - Downgrade ECS templates, pinning to v1.10.0 of upstream; fixes an issue where ECS templates cannot be installed in Elasticsearch 6.x or 7.1-7.2, since the generated templates include fields of `type: flattened` that was introduced in Elasticsearch 7.3
+
+## 11.2.2
+ - Update ECS templates from upstream; `ecs_compatiblity => v1` now resolves to templates for ECS v1.12.1 [#1027](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/1027)
+
 ## 11.2.1
  - Fix referencing Gem classes from global lexical scope [#1044](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1044)
 

data/lib/logstash/outputs/elasticsearch/data_stream_support.rb

@@ -2,6 +2,15 @@ module LogStash module Outputs class ElasticSearch
   # DS specific behavior/configuration.
   module DataStreamSupport
 
+    # @api private
+    ENABLING_ECS_GUIDANCE = <<~END.tr("\n", " ")
+      Elasticsearch data streams require that events adhere to the Elastic Common Schema.
+      While `ecs_compatibility` can be set for this individual Elasticsearch output plugin, doing so will not fix schema conflicts caused by upstream plugins in your pipeline.
+      To avoid mapping conflicts, you will need to use ECS-compatible field names and datatypes throughout your pipeline.
+      Many plugins support an `ecs_compatibility` mode, and the `pipeline.ecs_compatibility` setting can be used to opt-in for all plugins in a pipeline.
+    END
+    private_constant :ENABLING_ECS_GUIDANCE
+
     def self.included(base)
       # Defines whether data will be indexed into an Elasticsearch data stream,
       # `data_stream_*` settings will only be used if this setting is enabled!
@@ -36,6 +45,8 @@ module LogStash module Outputs class ElasticSearch
       "#{type}-#{dataset}-#{namespace}"
     end
 
+    DATA_STREAMS_REQUIRES_ECS_LS_VERSION = '8.0.0'
+
     # @param params the user configuration for the ES output
     # @note LS initialized configuration (with filled defaults) won't detect as data-stream
     # compatible, only explicit (`original_params`) config should be tested.
@@ -56,14 +67,26 @@ module LogStash module Outputs class ElasticSearch
           @logger.error "Invalid data stream configuration, following parameters are not supported:", invalid_data_stream_params
           raise LogStash::ConfigurationError, "Invalid data stream configuration: #{invalid_data_stream_params.keys}"
         end
+        if ecs_compatibility == :disabled
+          if ::Gem::Version.create(LOGSTASH_VERSION) < ::Gem::Version.create(DATA_STREAMS_REQUIRES_ECS_LS_VERSION)
+            @deprecation_logger.deprecated "In a future release of Logstash, the Elasticsearch output plugin's `data_stream => true` will require the plugin to be run in ECS compatibility mode. " + ENABLING_ECS_GUIDANCE
+          else
+            @logger.error "Invalid data stream configuration; `ecs_compatibility` must not be `disabled`. " + ENABLING_ECS_GUIDANCE
+            raise LogStash::ConfigurationError, "Invalid data stream configuration: `ecs_compatibility => disabled`"
+          end
+        end
         return true
       else
-        use_data_stream = data_stream_default(data_stream_params, invalid_data_stream_params.empty?)
-        if !use_data_stream && data_stream_params.any?
+        use_data_stream = data_stream_default(data_stream_params, invalid_data_stream_params)
+        if use_data_stream
+          @logger.info("Config is compliant with data streams. `data_stream => auto` resolved to `true`")
+        elsif data_stream_params.any?
           # DS (auto) disabled but there's still some data-stream parameters (and no `data_stream => false`)
           @logger.error "Ambiguous configuration; data stream settings are present, but data streams are not enabled", data_stream_params
           raise LogStash::ConfigurationError, "Ambiguous configuration, please set data_stream => true " +
               "or remove data stream specific settings: #{data_stream_params.keys}"
+        else
+          @logger.info("Config is not compliant with data streams. `data_stream => auto` resolved to `false`")
         end
         use_data_stream
       end
@@ -93,6 +116,7 @@ module LogStash module Outputs class ElasticSearch
           true
         when 'data_stream'
           value.to_s == 'true'
+        when 'ecs_compatibility' then true # required for LS <= 6.x
         else
           name.start_with?('data_stream_') ||
               shared_params.include?(name) ||
@@ -110,8 +134,8 @@ module LogStash module Outputs class ElasticSearch
     # @return [Gem::Version] if ES supports DS nil (or raise) otherwise
     def assert_es_version_supports_data_streams
       fail 'no last_es_version' unless last_es_version # assert - should not happen
-      es_version = Gem::Version.create(last_es_version)
-      if es_version < Gem::Version.create(DATA_STREAMS_ORIGIN_ES_VERSION)
+      es_version = ::Gem::Version.create(last_es_version)
+      if es_version < ::Gem::Version.create(DATA_STREAMS_ORIGIN_ES_VERSION)
         @logger.error "Elasticsearch version does not support data streams, Logstash might end up writing to an index", es_version: es_version.version
         # NOTE: when switching to synchronous check from register, this should be a ConfigurationError
         raise LogStash::Error, "A data_stream configuration is only supported since Elasticsearch #{DATA_STREAMS_ORIGIN_ES_VERSION} " +
@@ -123,18 +147,28 @@ module LogStash module Outputs class ElasticSearch
     DATA_STREAMS_ENABLED_BY_DEFAULT_LS_VERSION = '8.0.0'
 
     # when data_stream => is either 'auto' or not set
-    def data_stream_default(data_stream_params, valid_data_stream_config)
-      ds_default = Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create(DATA_STREAMS_ENABLED_BY_DEFAULT_LS_VERSION)
+    # @param data_stream_params [#any?]
+    # @param invalid_data_stream_config [#any?#inspect]
+    def data_stream_default(data_stream_params, invalid_data_stream_config)
+      if ecs_compatibility == :disabled
+        @logger.debug("Not eligible for data streams because ecs_compatibility is not enabled. " + ENABLING_ECS_GUIDANCE)
+        return false
+      end
+
+      ds_default = ::Gem::Version.create(LOGSTASH_VERSION) >= ::Gem::Version.create(DATA_STREAMS_ENABLED_BY_DEFAULT_LS_VERSION)
 
       if ds_default # LS 8.0
-        return false unless valid_data_stream_config
+        if invalid_data_stream_config.any?
+          @logger.debug("Not eligible for data streams because config contains one or more settings that are not compatible with data streams: #{invalid_data_stream_config.inspect}")
+          return false
+        end
 
         @logger.debug 'Configuration is data stream compliant'
         return true
       end
 
       # LS 7.x
-      if valid_data_stream_config && !data_stream_params.any?
+      if !invalid_data_stream_config.any? && !data_stream_params.any?
         @logger.warn "Configuration is data stream compliant but due backwards compatibility Logstash 7.x will not assume " +
                      "writing to a data-stream, default behavior will change on Logstash 8.0 " +
                      "(set `data_stream => true/false` to disable this warning)"