logstash-output-elasticsearch 11.12.2-java → 11.12.4-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f7955021889fbf4d2de6e6e167dba2b1e1231349a324d65ceb9d11b507c2405
-  data.tar.gz: 1cba0bd70fa6fe4d22631d473fee7d39f92cb3bca709aa61659c00de5992f30a
+  metadata.gz: a912267d383b51082d4c1b68e21d83dcb506433abc4f3f12bce0465eef84f439
+  data.tar.gz: dc69623c94ad47986ed07f25818eeabfbfdd49532dfd9970f8c638dfe950f615
 SHA512:
-  metadata.gz: de2eda2f2efb0564ca1f5a2165fa8e3b430f8c404c5b1640e8dce90ffa05885776c0ebaa7abd45a397129f4c08f146a1c1e2bdea68b07f5cbe624c28b26e8bfd
-  data.tar.gz: 219300f8c64b45079fcf9daa81f2afd219baa588a252f6905bb2ab234615ea0a99986c4b0e9aaf7bfc8b4d3e7afe5c5e51db7e287b4351a17a12abda52bece9a
+  metadata.gz: 99568149dc8d313460046ca2e247c43e2e505e6649441a22a365e0d5484ba493d5d91c4eedfb5700fc447f64122555db3b92d6b59513a6b42e07dcc43dda3ada
+  data.tar.gz: 56d2ce9c942f2dffbfc850f8feec0368dc2456ee08f7d7f46b83de94f9439fdd9047a1bae9231cc41caf45105a1abda0d1e60429a37059709f337e5ea8f8b6e4

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 11.12.4
+ - Changed the `manage_template` default value to `false` when data streams is enabled [#1111](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1111)
+   - Added the `manage_template => false` as a valid data stream option
+
+## 11.12.3
+ - Changed the log messages for data stream checks [#1109](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1109)
+   - Added more details about incompatible data streams supplied configurations
+   - Changed the data stream auto-configuration log levels from `debug` to `info`
+
 ## 11.12.2
  - [Doc] Fixes the broken apache http client link [#1101](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1101)
 

data/docs/index.asciidoc

@@ -790,7 +790,7 @@ Set the keystore password
 ===== `manage_template`
 
   * Value type is <<boolean,boolean>>
-  * Default value is `true`
+  * Default value is `true` for non-time series data, and `false` for data streams.
 
 From Logstash 1.3 onwards, a template is applied to Elasticsearch during
 Logstash's startup if one with the name <<plugins-{type}s-{plugin}-template_name>>

data/lib/logstash/outputs/elasticsearch/data_stream_support.rb

@@ -28,7 +28,6 @@ module LogStash module Outputs class ElasticSearch
       base.extend(Validator)
     end
 
-    # @note assumes to be running AFTER {after_successful_connection} completed, due ES version checks
     def data_stream_config?
       @data_stream_config.nil? ? @data_stream_config = check_data_stream_config! : @data_stream_config
     end
@@ -45,53 +44,64 @@ module LogStash module Outputs class ElasticSearch
       "#{type}-#{dataset}-#{namespace}"
     end
 
-    DATA_STREAMS_REQUIRES_ECS_LS_VERSION = '8.0.0'
+    DATA_STREAMS_AND_ECS_ENABLED_BY_DEFAULT_LS_VERSION = '8.0.0'
 
     # @param params the user configuration for the ES output
     # @note LS initialized configuration (with filled defaults) won't detect as data-stream
     # compatible, only explicit (`original_params`) config should be tested.
-    # @return [TrueClass|FalseClass] whether given configuration is data-stream compatible
+    # @return [Boolean] whether given configuration is data-stream compatible
     def check_data_stream_config!(params = original_params)
-      data_stream_params = params.select { |name, _| name.start_with?('data_stream_') } # exclude data_stream =>
-      invalid_data_stream_params = invalid_data_stream_params(params)
-
       case data_stream_explicit_value
       when false
-        if data_stream_params.any?
-          @logger.error "Ambiguous configuration; data stream settings must not be present when data streams is disabled (caused by: `data_stream => false`)", data_stream_params
-          raise LogStash::ConfigurationError, "Ambiguous configuration, please remove data stream specific settings: #{data_stream_params.keys}"
-        end
+        check_disabled_data_stream_config!(params)
         return false
       when true
-        if invalid_data_stream_params.any?
-          @logger.error "Invalid data stream configuration, following parameters are not supported:", invalid_data_stream_params
-          raise LogStash::ConfigurationError, "Invalid data stream configuration: #{invalid_data_stream_params.keys}"
-        end
-        if ecs_compatibility == :disabled
-          if ::Gem::Version.create(LOGSTASH_VERSION) < ::Gem::Version.create(DATA_STREAMS_REQUIRES_ECS_LS_VERSION)
-            @deprecation_logger.deprecated "In a future release of Logstash, the Elasticsearch output plugin's `data_stream => true` will require the plugin to be run in ECS compatibility mode. " + ENABLING_ECS_GUIDANCE
-          else
-            @logger.error "Invalid data stream configuration; `ecs_compatibility` must not be `disabled`. " + ENABLING_ECS_GUIDANCE
-            raise LogStash::ConfigurationError, "Invalid data stream configuration: `ecs_compatibility => disabled`"
-          end
-        end
+        check_enabled_data_stream_config!(params)
         return true
-      else
-        use_data_stream = data_stream_default(data_stream_params, invalid_data_stream_params)
-        if use_data_stream
-          @logger.info("Config is compliant with data streams. `data_stream => auto` resolved to `true`")
-        elsif data_stream_params.any?
-          # DS (auto) disabled but there's still some data-stream parameters (and no `data_stream => false`)
-          @logger.error "Ambiguous configuration; data stream settings are present, but data streams are not enabled", data_stream_params
-          raise LogStash::ConfigurationError, "Ambiguous configuration, please set data_stream => true " +
-              "or remove data stream specific settings: #{data_stream_params.keys}"
-        else
-          @logger.info("Config is not compliant with data streams. `data_stream => auto` resolved to `false`")
+      else # data_stream => auto or not set
+        use_data_stream = data_stream_default(params)
+
+        check_disabled_data_stream_config!(params) unless use_data_stream
+
+        @logger.info("Data streams auto configuration (`data_stream => auto` or unset) resolved to `#{use_data_stream}`")
+        return use_data_stream
+      end
+    end
+
+    def check_enabled_data_stream_config!(params)
+      invalid_data_stream_params = invalid_data_stream_params(params)
+
+      if invalid_data_stream_params.any?
+        @logger.error "Invalid data stream configuration, the following parameters are not supported:", invalid_data_stream_params
+        raise LogStash::ConfigurationError, "Invalid data stream configuration: #{invalid_data_stream_params.keys}"
+      end
+
+      if ecs_compatibility == :disabled
+        if ecs_compatibility_required?
+          @logger.error "Invalid data stream configuration; `ecs_compatibility` must not be `disabled`. " + ENABLING_ECS_GUIDANCE
+          raise LogStash::ConfigurationError, "Invalid data stream configuration: `ecs_compatibility => disabled`"
         end
-        use_data_stream
+
+        @deprecation_logger.deprecated "In a future release of Logstash, the Elasticsearch output plugin's `data_stream => true` will require the plugin to be run in ECS compatibility mode. " + ENABLING_ECS_GUIDANCE
       end
     end
 
+    def check_disabled_data_stream_config!(params)
+      data_stream_params = data_stream_params(params)
+
+      if data_stream_params.any?
+        @logger.error "Ambiguous configuration; data stream settings must not be present when data streams are disabled (caused by `data_stream => false`, `data_stream => auto` or unset resolved to false). " \
+                      "You can either manually set `data_stream => true` or remove the following specific data stream settings: ", data_stream_params
+
+        raise LogStash::ConfigurationError,
+              "Ambiguous configuration; data stream settings must not be present when data streams are disabled: #{data_stream_params.keys}"
+      end
+    end
+
+    def data_stream_params(params)
+      params.select { |name, _| name.start_with?('data_stream_') }
+    end
+
     def data_stream_explicit_value
       case @data_stream
       when 'true'
@@ -108,7 +118,6 @@ module LogStash module Outputs class ElasticSearch
       params.reject do |name, value|
         # NOTE: intentionally do not support explicit DS configuration like:
         # - `index => ...` identifier provided by data_stream_xxx settings
-        # - `manage_template => false` implied by not setting the parameter
         case name
         when 'action'
           value == 'create'
@@ -116,6 +125,8 @@ module LogStash module Outputs class ElasticSearch
           true
         when 'data_stream'
           value.to_s == 'true'
+        when 'manage_template'
+          value.to_s == 'false'
         when 'ecs_compatibility' then true # required for LS <= 6.x
         else
           name.start_with?('data_stream_') ||
@@ -131,6 +142,7 @@ module LogStash module Outputs class ElasticSearch
 
     DATA_STREAMS_ORIGIN_ES_VERSION = '7.9.0'
 
+    # @note assumes to be running AFTER {after_successful_connection} completed, due ES version checks
     # @return [Gem::Version] if ES supports DS nil (or raise) otherwise
     def assert_es_version_supports_data_streams
       fail 'no last_es_version' unless last_es_version # assert - should not happen
@@ -144,31 +156,26 @@ module LogStash module Outputs class ElasticSearch
       es_version # return truthy
     end
 
-    DATA_STREAMS_ENABLED_BY_DEFAULT_LS_VERSION = '8.0.0'
-
     # when data_stream => is either 'auto' or not set
-    # @param data_stream_params [#any?]
-    # @param invalid_data_stream_config [#any?#inspect]
-    def data_stream_default(data_stream_params, invalid_data_stream_config)
+    def data_stream_default(params)
       if ecs_compatibility == :disabled
-        @logger.debug("Not eligible for data streams because ecs_compatibility is not enabled. " + ENABLING_ECS_GUIDANCE)
+        @logger.info("Not eligible for data streams because ecs_compatibility is not enabled. " + ENABLING_ECS_GUIDANCE)
         return false
       end
 
-      ds_default = ::Gem::Version.create(LOGSTASH_VERSION) >= ::Gem::Version.create(DATA_STREAMS_ENABLED_BY_DEFAULT_LS_VERSION)
+      invalid_data_stream_params = invalid_data_stream_params(params)
 
-      if ds_default # LS 8.0
-        if invalid_data_stream_config.any?
-          @logger.debug("Not eligible for data streams because config contains one or more settings that are not compatible with data streams: #{invalid_data_stream_config.inspect}")
+      if data_stream_and_ecs_enabled_by_default?
+        if invalid_data_stream_params.any?
+          @logger.info("Not eligible for data streams because config contains one or more settings that are not compatible with data streams: #{invalid_data_stream_params.inspect}")
           return false
         end
 
-        @logger.debug 'Configuration is data stream compliant'
         return true
       end
 
       # LS 7.x
-      if !invalid_data_stream_config.any? && !data_stream_params.any?
+      if !invalid_data_stream_params.any? && !data_stream_params(params).any?
         @logger.warn "Configuration is data stream compliant but due backwards compatibility Logstash 7.x will not assume " +
                      "writing to a data-stream, default behavior will change on Logstash 8.0 " +
                      "(set `data_stream => true/false` to disable this warning)"
@@ -176,6 +183,14 @@ module LogStash module Outputs class ElasticSearch
       false
     end
 
+    def ecs_compatibility_required?
+      data_stream_and_ecs_enabled_by_default?
+    end
+
+    def data_stream_and_ecs_enabled_by_default?
+      ::Gem::Version.create(LOGSTASH_VERSION) >= ::Gem::Version.create(DATA_STREAMS_AND_ECS_ENABLED_BY_DEFAULT_LS_VERSION)
+    end
+
     # an {event_action_tuple} replacement when a data-stream configuration is detected
     def data_stream_event_action_tuple(event)
       event_data = event.to_hash

data/lib/logstash/outputs/elasticsearch.rb

@@ -158,6 +158,8 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # control over template creation, (e.g. creating indices dynamically based on
   # field names) you should set `manage_template` to false and use the REST
   # API to apply your templates manually.
+  #
+  # Default value is `true` unless data streams is enabled
   config :manage_template, :validate => :boolean, :default => true
 
   # This configuration option defines how the template is named inside Elasticsearch.
@@ -300,6 +302,12 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     # to build_client down to the Pool class.
     @client = build_client(LicenseChecker.new(@logger))
 
+    # Avoids race conditions in the @data_stream_config initialization (invoking check_data_stream_config! twice).
+    # It's being concurrently invoked by this register method and by the finish_register on the @after_successful_connection_thread
+    data_stream_enabled = data_stream_config?
+
+    setup_template_manager_defaults(data_stream_enabled)
+
     @after_successful_connection_thread = after_successful_connection do
       begin
         finish_register
@@ -324,7 +332,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
       raise LogStash::ConfigurationError, "DLQ feature (dlq_custom_codes) is configured while DLQ is not enabled" unless dlq_custom_codes.empty?
     end
 
-    if data_stream_config?
+    if data_stream_enabled
       @event_mapper = -> (e) { data_stream_event_action_tuple(e) }
       @event_target = -> (e) { data_stream_name(e) }
       @index = "#{data_stream_type}-#{data_stream_dataset}-#{data_stream_namespace}".freeze # default name
@@ -604,6 +612,13 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     @template_name ||= default_template_name
   end
 
+  def setup_template_manager_defaults(data_stream_enabled)
+    if original_params["manage_template"].nil? && data_stream_enabled
+      logger.debug("Disabling template management since data streams are enabled")
+      @manage_template = false
+    end
+  end
+
   # To be overidden by the -java version
   VALID_HTTP_ACTIONS = ["index", "delete", "create", "update"]
   def valid_actions

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.12.2'
+  s.version         = '11.12.4'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb

@@ -84,7 +84,8 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
         context "on LS #{ls_version_desc}" do
           around(:each) { |example| change_constant(:LOGSTASH_VERSION, ls_version, &example) }
           it "does not use data-streams" do
-            expect( subject.logger ).to receive(:debug).with(a_string_including "ecs_compatibility is not enabled")
+            expect( subject.logger ).to receive(:info).with(a_string_including "ecs_compatibility is not enabled")
+            expect( subject.logger ).to receive(:info).with(a_string_including "Data streams auto configuration (`data_stream => auto` or unset) resolved to `false`")
             expect( subject.data_stream_config? ).to be false
           end
         end
@@ -151,6 +152,39 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
 
   end
 
+  context 'ds value-dependent configuration' do
+    context 'with valid values' do
+      let(:options) { super().merge(
+        'action' => 'create',
+        'routing' => 'any',
+        'pipeline' => 'any',
+        'manage_template' => 'false',
+        'data_stream' => 'true',
+        'data_stream_type' => 'logs',
+        'data_stream_dataset' => 'any',
+        'data_stream_namespace' => 'any',
+        'data_stream_sync_fields' => true,
+        'data_stream_auto_routing' => true)
+      }
+
+      it 'should enable data-streams by default' do
+        expect( subject.data_stream_config? ).to be_truthy
+      end
+    end
+
+    context 'with invalid values' do
+      let(:options) { super().merge(
+        'data_stream' => 'true',
+        'action' => 'index',
+        'manage_template' => 'true')
+      }
+
+      it 'should raise a configuration error' do
+        expect { subject.data_stream_config? }.to raise_error(LogStash::ConfigurationError, 'Invalid data stream configuration: ["action", "manage_template"]')
+      end
+    end
+  end
+
   context "default (non data-stream) configuration (on 7.x)" do
 
     let(:options) do
@@ -161,7 +195,7 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
 
     it "does not default to data-streams" do
       expect( subject.logger ).to receive(:error) do |msg|
-        expect(msg).to include "Ambiguous configuration; data stream settings are present, but data streams are not enabled"
+        expect(msg).to include "Ambiguous configuration; data stream settings must not be present when data streams are disabled"
       end
       change_constant :LOGSTASH_VERSION, '7.10.2' do
         expect { subject.data_stream_config? }.to raise_error(LogStash::ConfigurationError, /Ambiguous configuration/i)
@@ -173,8 +207,8 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
       let(:options) { super().merge('data_stream' => 'false') }
 
       it "raises a configuration error (due ds specific settings)" do
-        expect( subject.logger ).to receive(:error).with(/Ambiguous configuration; data stream settings must not be present when data streams is disabled/,
-                                                         {"data_stream_auto_routing"=>"false", "data_stream_dataset"=>"test"})
+        expect( subject.logger ).to receive(:error).with(/Ambiguous configuration; data stream settings must not be present when data streams are disabled/,
+                                                          {"data_stream_auto_routing"=>"false", "data_stream_dataset"=>"test"})
         change_constant :LOGSTASH_VERSION, '7.10.2' do
           expect { subject.data_stream_config? }.to raise_error(LogStash::ConfigurationError, /Ambiguous configuration/i)
         end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -606,6 +606,26 @@ describe LogStash::Outputs::ElasticSearch do
     end
   end
 
+  describe "the manage_template option" do
+    context "with data stream enabled" do
+      let(:options) { {"data_stream" => "true", "data_stream_type" => "logs" } }
+      let(:do_register) { true }
+
+      it "should default to false" do
+        expect(subject).to have_attributes(manage_template: false)
+      end
+    end
+
+    context "with data stream disabled" do
+      let(:options) { {"data_stream" => "false", "index" => "logs" } }
+      let(:do_register) { true }
+
+      it "should default to true" do
+        expect(subject).to have_attributes(manage_template: true)
+      end
+    end
+  end
+
   describe "SSL end to end" do
     let(:do_register) { false } # skip the register in the global before block, as is called here.
 
@@ -870,7 +890,7 @@ describe LogStash::Outputs::ElasticSearch do
   end if LOGSTASH_VERSION > '6.0'
 
   context 'handling elasticsearch document-level status meant for the DLQ' do
-    let(:options) { { "manage_template" => false } }
+    let(:options) { { "manage_template" => false, "data_stream" => 'false' } }
     let(:action) { LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, LogStash::Event.new("foo" => "bar")) }
 
     context 'when @dlq_writer is nil' do
@@ -1140,7 +1160,7 @@ describe LogStash::Outputs::ElasticSearch do
   describe "post-register ES setup" do
     let(:do_register) { false }
     let(:es_version) { '7.10.0' } # DS default on LS 8.x
-    let(:options) { { 'hosts' => '127.0.0.1:9999' } }
+    let(:options) { { 'hosts' => '127.0.0.1:9999', 'data_stream' => 'false' } }
     let(:logger) { subject.logger }
 
     before do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.12.2
+  version: 11.12.4
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-16 00:00:00.000000000 Z
+date: 2023-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement