logstash-output-elasticsearch 11.3.0-java → 11.3.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a9b6a7eacf857e3f0e0953d2aa1628f3ea50f9928d73dc71fd0d09db2f2cd147
-  data.tar.gz: 16c0222ff642f203c5a9c0553faac90b6fbf02875910191cd9b921e07b52f1df
+  metadata.gz: dccee6612b454d47b0596a9feb04db71f51f368a34906f559b8f7a8df7cf166e
+  data.tar.gz: 27d8ab736dd7c3ffa46d1229ad70afd5e8b23813ee09aa3e9d4ec05b4f263a76
 SHA512:
-  metadata.gz: 15b8d3c36d59e2bdb087058e24a85ddd67ad09f2708ccad357b032b7c3e6bf04beb224138738ca3be09a28cd5e24fb9d5b2883b510ee406500393132825d79f1
-  data.tar.gz: e1cc0b009714cead91f295baf08dee61fb76938abd2c1456a48440d78ab62d4beffffd761d4a831057757d090052e8bb83d670ad72805171435b056f1184500f
+  metadata.gz: d7ab3e32387faf0cc2680fc1446e9a330314d2ead466487fd6f63a27ab2757cac8c2d90cb41bb0eba422986b22821d0e0c813617ba153225058b1e1d09765598
+  data.tar.gz: df0a5b6cbcbdc592e1c18906f6a54a97d309afbbd1c36529f812501df31f78c7e4c2ba2fb8a3eab72e9fd53dfbca8388b710dc772a20ba9084c59ce1f5aa43fc

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 11.3.1
+ - ECS-related fixes [#1046](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1046)
+   - Data Streams requirement on ECS is properly enforced when running on Logstash 8, and warned about when running on Logstash 7.
+   - ECS Compatibility v8 can now be selected
+
 ## 11.3.0
  - Adds ECS templates [#1048](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1048)
    - Adds templates for ECS v1 for Elasticsearch 8.x

data/lib/logstash/outputs/elasticsearch/data_stream_support.rb

@@ -2,6 +2,15 @@ module LogStash module Outputs class ElasticSearch
   # DS specific behavior/configuration.
   module DataStreamSupport
 
+    # @api private
+    ENABLING_ECS_GUIDANCE = <<~END.tr("\n", " ")
+      Elasticsearch data streams require that events adhere to the Elastic Common Schema.
+      While `ecs_compatibility` can be set for this individual Elasticsearch output plugin, doing so will not fix schema conflicts caused by upstream plugins in your pipeline.
+      To avoid mapping conflicts, you will need to use ECS-compatible field names and datatypes throughout your pipeline.
+      Many plugins support an `ecs_compatibility` mode, and the `pipeline.ecs_compatibility` setting can be used to opt-in for all plugins in a pipeline.
+    END
+    private_constant :ENABLING_ECS_GUIDANCE
+
     def self.included(base)
       # Defines whether data will be indexed into an Elasticsearch data stream,
       # `data_stream_*` settings will only be used if this setting is enabled!
@@ -36,6 +45,8 @@ module LogStash module Outputs class ElasticSearch
       "#{type}-#{dataset}-#{namespace}"
     end
 
+    DATA_STREAMS_REQUIRES_ECS_LS_VERSION = '8.0.0'
+
     # @param params the user configuration for the ES output
     # @note LS initialized configuration (with filled defaults) won't detect as data-stream
     # compatible, only explicit (`original_params`) config should be tested.
@@ -56,14 +67,26 @@ module LogStash module Outputs class ElasticSearch
           @logger.error "Invalid data stream configuration, following parameters are not supported:", invalid_data_stream_params
           raise LogStash::ConfigurationError, "Invalid data stream configuration: #{invalid_data_stream_params.keys}"
         end
+        if ecs_compatibility == :disabled
+          if ::Gem::Version.create(LOGSTASH_VERSION) < ::Gem::Version.create(DATA_STREAMS_REQUIRES_ECS_LS_VERSION)
+            @deprecation_logger.deprecated "In a future release of Logstash, the Elasticsearch output plugin's `data_stream => true` will require the plugin to be run in ECS compatibility mode. " + ENABLING_ECS_GUIDANCE
+          else
+            @logger.error "Invalid data stream configuration; `ecs_compatibility` must not be `disabled`. " + ENABLING_ECS_GUIDANCE
+            raise LogStash::ConfigurationError, "Invalid data stream configuration: `ecs_compatibility => disabled`"
+          end
+        end
         return true
       else
-        use_data_stream = data_stream_default(data_stream_params, invalid_data_stream_params.empty?)
-        if !use_data_stream && data_stream_params.any?
+        use_data_stream = data_stream_default(data_stream_params, invalid_data_stream_params)
+        if use_data_stream
+          @logger.info("Config is compliant with data streams. `data_stream => auto` resolved to `true`")
+        elsif data_stream_params.any?
           # DS (auto) disabled but there's still some data-stream parameters (and no `data_stream => false`)
           @logger.error "Ambiguous configuration; data stream settings are present, but data streams are not enabled", data_stream_params
           raise LogStash::ConfigurationError, "Ambiguous configuration, please set data_stream => true " +
               "or remove data stream specific settings: #{data_stream_params.keys}"
+        else
+          @logger.info("Config is not compliant with data streams. `data_stream => auto` resolved to `false`")
         end
         use_data_stream
       end
@@ -93,6 +116,7 @@ module LogStash module Outputs class ElasticSearch
           true
         when 'data_stream'
           value.to_s == 'true'
+        when 'ecs_compatibility' then true # required for LS <= 6.x
         else
           name.start_with?('data_stream_') ||
               shared_params.include?(name) ||
@@ -110,8 +134,8 @@ module LogStash module Outputs class ElasticSearch
     # @return [Gem::Version] if ES supports DS nil (or raise) otherwise
     def assert_es_version_supports_data_streams
       fail 'no last_es_version' unless last_es_version # assert - should not happen
-      es_version = Gem::Version.create(last_es_version)
-      if es_version < Gem::Version.create(DATA_STREAMS_ORIGIN_ES_VERSION)
+      es_version = ::Gem::Version.create(last_es_version)
+      if es_version < ::Gem::Version.create(DATA_STREAMS_ORIGIN_ES_VERSION)
         @logger.error "Elasticsearch version does not support data streams, Logstash might end up writing to an index", es_version: es_version.version
         # NOTE: when switching to synchronous check from register, this should be a ConfigurationError
         raise LogStash::Error, "A data_stream configuration is only supported since Elasticsearch #{DATA_STREAMS_ORIGIN_ES_VERSION} " +
@@ -123,18 +147,28 @@ module LogStash module Outputs class ElasticSearch
     DATA_STREAMS_ENABLED_BY_DEFAULT_LS_VERSION = '8.0.0'
 
     # when data_stream => is either 'auto' or not set
-    def data_stream_default(data_stream_params, valid_data_stream_config)
-      ds_default = Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create(DATA_STREAMS_ENABLED_BY_DEFAULT_LS_VERSION)
+    # @param data_stream_params [#any?]
+    # @param invalid_data_stream_config [#any?#inspect]
+    def data_stream_default(data_stream_params, invalid_data_stream_config)
+      if ecs_compatibility == :disabled
+        @logger.debug("Not eligible for data streams because ecs_compatibility is not enabled. " + ENABLING_ECS_GUIDANCE)
+        return false
+      end
+
+      ds_default = ::Gem::Version.create(LOGSTASH_VERSION) >= ::Gem::Version.create(DATA_STREAMS_ENABLED_BY_DEFAULT_LS_VERSION)
 
       if ds_default # LS 8.0
-        return false unless valid_data_stream_config
+        if invalid_data_stream_config.any?
+          @logger.debug("Not eligible for data streams because config contains one or more settings that are not compatible with data streams: #{invalid_data_stream_config.inspect}")
+          return false
+        end
 
         @logger.debug 'Configuration is data stream compliant'
         return true
       end
 
       # LS 7.x
-      if valid_data_stream_config && !data_stream_params.any?
+      if !invalid_data_stream_config.any? && !data_stream_params.any?
         @logger.warn "Configuration is data stream compliant but due backwards compatibility Logstash 7.x will not assume " +
                      "writing to a data-stream, default behavior will change on Logstash 8.0 " +
                      "(set `data_stream => true/false` to disable this warning)"

data/lib/logstash/outputs/elasticsearch.rb

@@ -94,6 +94,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   require "logstash/outputs/elasticsearch/ilm"
   require "logstash/outputs/elasticsearch/data_stream_support"
   require 'logstash/plugin_mixins/ecs_compatibility_support'
+  require 'logstash/plugin_mixins/deprecation_logger_support'
 
   # Protocol agnostic methods
   include(LogStash::PluginMixins::ElasticSearch::Common)
@@ -104,6 +105,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # ecs_compatibility option, provided by Logstash core or the support adapter.
   include(LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8))
 
+  # deprecation logger adapter for older Logstashes
+  include(LogStash::PluginMixins::DeprecationLoggerSupport)
+
   # Generic/API config options that any document indexer output needs
   include(LogStash::PluginMixins::ElasticSearch::APIConfigs)
 
@@ -498,7 +502,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
       @default_index = "logstash-%{+yyyy.MM.dd}"
       @default_ilm_rollover_alias = "logstash"
       @default_template_name = 'logstash'
-    when :v1
+    when :v1, :v8
       @default_index = "ecs-logstash-%{+yyyy.MM.dd}"
       @default_ilm_rollover_alias = "ecs-logstash"
       @default_template_name = 'ecs-logstash'

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.3.0'
+  s.version         = '11.3.1'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
+  s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
 
   s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'logstash-devutils'

data/spec/integration/outputs/data_stream_spec.rb

@@ -12,6 +12,13 @@ describe "data streams", :integration => true do
 
   subject { LogStash::Outputs::ElasticSearch.new(options) }
 
+  # All data-streams features require that the plugin be run in a non-disabled ECS compatibility mode.
+  # We run the plugin in ECS by default, and add test scenarios specifically for it being disabled.
+  let(:ecs_compatibility) { :v1 }
+  before(:each) do
+    allow( subject ).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+  end
+
   before :each do
     @es = get_client
     @es.delete_by_query(index: ".ds-#{ds_name}-*", expand_wildcards: :all, body: { query: { match_all: {} } }) rescue nil

data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb

@@ -4,9 +4,17 @@ require "logstash/outputs/elasticsearch/data_stream_support"
 describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
 
   subject { LogStash::Outputs::ElasticSearch.new(options) }
+
   let(:options) { { 'hosts' => [ 'localhost:12345' ] } }
   let(:es_version) { '7.10.1' }
 
+  # All data-streams features require that the plugin be run in a non-disabled ECS compatibility mode.
+  # We run the plugin in ECS by default, and add test scenarios specifically for it being disabled.
+  let(:ecs_compatibility) { :v1 }
+  before(:each) do
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+  end
+
   let(:do_register) { false }
   let(:stub_plugin_register!) do
     allow(subject).to receive(:last_es_version).and_return(es_version)
@@ -52,9 +60,7 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
     end
 
     it "warns when configuration is data-stream compliant (LS 7.x)" do
-      expect( subject.logger ).to receive(:warn) do |msg|
-        expect(msg).to include "Configuration is data stream compliant but due backwards compatibility Logstash 7.x"
-      end
+      expect( subject.logger ).to receive(:warn).with(a_string_including "Configuration is data stream compliant but due backwards compatibility Logstash 7.x")
       change_constant :LOGSTASH_VERSION, '7.11.0' do
         expect( subject.data_stream_config? ).to be false
       end
@@ -66,6 +72,25 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
       end
     end
 
+    context 'ecs_compatibility disabled' do
+      let(:ecs_compatibility) { :disabled }
+
+      {
+        '7.x (pre-DS)' => '7.9.0',
+        '7.x (with DS)' => '7.11.0',
+        '8.0' => '8.0.0',
+        '8.x' => '8.1.2',
+      }.each do |ls_version_desc, ls_version|
+        context "on LS #{ls_version_desc}" do
+          around(:each) { |example| change_constant(:LOGSTASH_VERSION, ls_version, &example) }
+          it "does not use data-streams" do
+            expect( subject.logger ).to receive(:debug).with(a_string_including "ecs_compatibility is not enabled")
+            expect( subject.data_stream_config? ).to be false
+          end
+        end
+      end
+    end
+
     context 'non-compatible ES' do
 
       let(:es_version) { '7.8.0' }
@@ -95,6 +120,7 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
     before { allow(subject).to receive(:last_es_version).and_return(es_version) }
 
     it "does not use data-streams on LS 7.x" do
+      expect( subject.logger ).to receive(:warn).with(a_string_including "Logstash 7.x will not assume writing to a data-stream")
       change_constant :LOGSTASH_VERSION, '7.10.0' do
         expect( subject.data_stream_config? ).to be false
       end
@@ -197,12 +223,34 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
       end
     end
 
-    it "does use data-streams on LS 8.0" do
+    it "does use data-streams on LS 8.x" do
       change_constant :LOGSTASH_VERSION, '8.1.0' do
         expect( subject.data_stream_config? ).to be true
       end
     end
 
+    context 'with ecs_compatibility disabled' do
+      let(:ecs_compatibility) { :disabled }
+
+      context 'when running on LS 7.x' do
+        around(:each) { |example| change_constant(:LOGSTASH_VERSION, '7.15.1', &example) }
+
+        it "emits a deprecation warning and uses data streams anway" do
+          expect( subject.deprecation_logger ).to receive(:deprecated).with(a_string_including "`data_stream => true` will require the plugin to be run in ECS compatibility mode")
+          expect( subject.data_stream_config? ).to be true
+        end
+      end
+
+      context 'when running on LS 8.x' do
+        around(:each) { |example| change_constant(:LOGSTASH_VERSION, '8.0.0', &example) }
+
+        it "errors helpfully" do
+          expect{ subject.data_stream_config? }.to raise_error(LogStash::ConfigurationError, a_string_including("Invalid data stream configuration: `ecs_compatibility => disabled`"))
+        end
+      end
+
+    end
+
     context 'non-compatible ES' do
 
       let(:es_version) { '6.8.11' }

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -4,6 +4,8 @@ require "flores/random"
 require 'concurrent/atomic/count_down_latch'
 require "logstash/outputs/elasticsearch"
 
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
+
 describe LogStash::Outputs::ElasticSearch do
   subject(:elasticsearch_output_instance) { described_class.new(options) }
   let(:options) { {} }
@@ -922,6 +924,25 @@ describe LogStash::Outputs::ElasticSearch do
     end
   end
 
+  describe 'ECS Compatibility Support', :ecs_compatibility_support do
+    [
+      :disabled,
+      :v1,
+      :v8,
+    ].each do |ecs_compatibility|
+      context "When initialized with `ecs_compatibility => #{ecs_compatibility}`" do
+        let(:options) { Hash.new }
+        subject(:output) { described_class.new(options.merge("ecs_compatibility" => "#{ecs_compatibility}")) }
+        context 'when registered' do
+          before(:each) { output.register }
+          it 'has the correct effective ECS compatibility setting' do
+            expect(output.ecs_compatibility).to eq(ecs_compatibility)
+          end
+        end
+      end
+    end
+  end
+
   describe "post-register ES setup" do
     let(:do_register) { false }
     let(:es_version) { '7.10.0' } # DS default on LS 8.x
@@ -959,7 +980,7 @@ describe LogStash::Outputs::ElasticSearch do
     context 'error raised' do
 
       let(:es_version) { '7.8.0' }
-      let(:options) { super().merge('data_stream' => 'true') }
+      let(:options) { super().merge('data_stream' => 'true', 'ecs_compatibility' => 'v1') }
       let(:latch) { Concurrent::CountDownLatch.new }
 
       before do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.3.0
+  version: 11.3.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-03 00:00:00.000000000 Z
+date: 2021-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -84,6 +84,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-deprecation_logger_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: