logstash-output-elasticsearch 11.3.0-java → 11.4.0-java

data/lib/logstash/outputs/elasticsearch.rb

@@ -94,6 +94,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   require "logstash/outputs/elasticsearch/ilm"
   require "logstash/outputs/elasticsearch/data_stream_support"
   require 'logstash/plugin_mixins/ecs_compatibility_support'
+  require 'logstash/plugin_mixins/deprecation_logger_support'
 
   # Protocol agnostic methods
   include(LogStash::PluginMixins::ElasticSearch::Common)
@@ -104,6 +105,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # ecs_compatibility option, provided by Logstash core or the support adapter.
   include(LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8))
 
+  # deprecation logger adapter for older Logstashes
+  include(LogStash::PluginMixins::DeprecationLoggerSupport)
+
   # Generic/API config options that any document indexer output needs
   include(LogStash::PluginMixins::ElasticSearch::APIConfigs)
 
@@ -498,7 +502,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
       @default_index = "logstash-%{+yyyy.MM.dd}"
       @default_ilm_rollover_alias = "logstash"
       @default_template_name = 'logstash'
-    when :v1
+    when :v1, :v8
       @default_index = "ecs-logstash-%{+yyyy.MM.dd}"
       @default_ilm_rollover_alias = "ecs-logstash"
       @default_template_name = 'ecs-logstash'

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.3.0'
+  s.version         = '11.4.0'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
+  s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
 
   s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'logstash-devutils'

data/spec/es_spec_helper.rb

@@ -59,7 +59,11 @@ module ESHelper
   end
 
   def self.es_version
-    RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
+    [
+      nilify(RSpec.configuration.filter[:es_version]),
+      nilify(ENV['ES_VERSION']),
+      nilify(ENV['ELASTIC_STACK_VERSION']),
+    ].compact.first
   end
 
   RSpec::Matchers.define :have_hits do |expected|

data/spec/integration/outputs/data_stream_spec.rb

@@ -12,6 +12,13 @@ describe "data streams", :integration => true do
 
   subject { LogStash::Outputs::ElasticSearch.new(options) }
 
+  # All data-streams features require that the plugin be run in a non-disabled ECS compatibility mode.
+  # We run the plugin in ECS by default, and add test scenarios specifically for it being disabled.
+  let(:ecs_compatibility) { :v1 }
+  before(:each) do
+    allow( subject ).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+  end
+
   before :each do
     @es = get_client
     @es.delete_by_query(index: ".ds-#{ds_name}-*", expand_wildcards: :all, body: { query: { match_all: {} } }) rescue nil

data/spec/integration/outputs/ilm_spec.rb

@@ -236,7 +236,8 @@ describe 'Elasticsearch has index lifecycle management enabled', :integration =>
   let (:settings) {
     {
         "ilm_enabled" => ilm_enabled,
-        "hosts" => "#{get_host_port()}"
+        "hosts" => "#{get_host_port()}",
+        "ecs_compatibility" => "disabled", # specs are tightly tied to non-ECS defaults
     }
   }
   let (:small_max_doc_policy) { max_docs_policy(3) }

data/spec/integration/outputs/ingest_pipeline_spec.rb

@@ -6,7 +6,8 @@ describe "Ingest pipeline execution behavior", :integration => true do
     settings = {
       "hosts" => "#{get_host_port()}",
       "pipeline" => "apache-logs",
-      "data_stream" => 'false'
+      "data_stream" => 'false',
+      "ecs_compatibility" => "disabled", # specs are tightly tied to non-ECS defaults
     }
     next LogStash::Outputs::ElasticSearch.new(settings)
   end

data/spec/integration/outputs/retry_spec.rb

@@ -45,7 +45,8 @@ describe "failures in bulk class expected behavior", :integration => true do
       "template_overwrite" => true,
       "hosts" => get_host_port(),
       "retry_max_interval" => 64,
-      "retry_initial_interval" => 2
+      "retry_initial_interval" => 2,
+      "ecs_compatibility" => "disabled", # specs are tightly tied to non-ECS defaults
     }
     next LogStash::Outputs::ElasticSearch.new(settings)
   end

data/spec/integration/outputs/sniffer_spec.rb

@@ -5,7 +5,7 @@ require "socket"
 
 describe "pool sniffer", :integration => true do
   let(:logger) { Cabin::Channel.get }
-  let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger) }
+  let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger, {}) }
   let(:es_host) { get_host_port.split(":").first }
   let(:es_port) { get_host_port.split(":").last }
   let(:es_ip) { IPSocket.getaddress(es_host) }

data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb

@@ -4,9 +4,17 @@ require "logstash/outputs/elasticsearch/data_stream_support"
 describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
 
   subject { LogStash::Outputs::ElasticSearch.new(options) }
+
   let(:options) { { 'hosts' => [ 'localhost:12345' ] } }
   let(:es_version) { '7.10.1' }
 
+  # All data-streams features require that the plugin be run in a non-disabled ECS compatibility mode.
+  # We run the plugin in ECS by default, and add test scenarios specifically for it being disabled.
+  let(:ecs_compatibility) { :v1 }
+  before(:each) do
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+  end
+
   let(:do_register) { false }
   let(:stub_plugin_register!) do
     allow(subject).to receive(:last_es_version).and_return(es_version)
@@ -52,9 +60,7 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
     end
 
     it "warns when configuration is data-stream compliant (LS 7.x)" do
-      expect( subject.logger ).to receive(:warn) do |msg|
-        expect(msg).to include "Configuration is data stream compliant but due backwards compatibility Logstash 7.x"
-      end
+      expect( subject.logger ).to receive(:warn).with(a_string_including "Configuration is data stream compliant but due backwards compatibility Logstash 7.x")
       change_constant :LOGSTASH_VERSION, '7.11.0' do
         expect( subject.data_stream_config? ).to be false
       end
@@ -66,6 +72,25 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
       end
     end
 
+    context 'ecs_compatibility disabled' do
+      let(:ecs_compatibility) { :disabled }
+
+      {
+        '7.x (pre-DS)' => '7.9.0',
+        '7.x (with DS)' => '7.11.0',
+        '8.0' => '8.0.0',
+        '8.x' => '8.1.2',
+      }.each do |ls_version_desc, ls_version|
+        context "on LS #{ls_version_desc}" do
+          around(:each) { |example| change_constant(:LOGSTASH_VERSION, ls_version, &example) }
+          it "does not use data-streams" do
+            expect( subject.logger ).to receive(:debug).with(a_string_including "ecs_compatibility is not enabled")
+            expect( subject.data_stream_config? ).to be false
+          end
+        end
+      end
+    end
+
     context 'non-compatible ES' do
 
       let(:es_version) { '7.8.0' }
@@ -95,6 +120,7 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
     before { allow(subject).to receive(:last_es_version).and_return(es_version) }
 
     it "does not use data-streams on LS 7.x" do
+      expect( subject.logger ).to receive(:warn).with(a_string_including "Logstash 7.x will not assume writing to a data-stream")
       change_constant :LOGSTASH_VERSION, '7.10.0' do
         expect( subject.data_stream_config? ).to be false
       end
@@ -197,12 +223,34 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
       end
     end
 
-    it "does use data-streams on LS 8.0" do
+    it "does use data-streams on LS 8.x" do
       change_constant :LOGSTASH_VERSION, '8.1.0' do
         expect( subject.data_stream_config? ).to be true
       end
     end
 
+    context 'with ecs_compatibility disabled' do
+      let(:ecs_compatibility) { :disabled }
+
+      context 'when running on LS 7.x' do
+        around(:each) { |example| change_constant(:LOGSTASH_VERSION, '7.15.1', &example) }
+
+        it "emits a deprecation warning and uses data streams anway" do
+          expect( subject.deprecation_logger ).to receive(:deprecated).with(a_string_including "`data_stream => true` will require the plugin to be run in ECS compatibility mode")
+          expect( subject.data_stream_config? ).to be true
+        end
+      end
+
+      context 'when running on LS 8.x' do
+        around(:each) { |example| change_constant(:LOGSTASH_VERSION, '8.0.0', &example) }
+
+        it "errors helpfully" do
+          expect{ subject.data_stream_config? }.to raise_error(LogStash::ConfigurationError, a_string_including("Invalid data stream configuration: `ecs_compatibility => disabled`"))
+        end
+      end
+
+    end
+
     context 'non-compatible ES' do
 
       let(:es_version) { '6.8.11' }

data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb

@@ -10,11 +10,12 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
 
   it "should raise an exception if requests are issued after close" do
     subject.close
-    expect { subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, '/') }.to raise_error(::Manticore::ClientStoppedException)
-  end
-
-  it "should implement host unreachable exceptions" do
-    expect(subject.host_unreachable_exceptions).to be_a(Array)
+    begin
+      subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, '/')
+      fail 'expected to raise a HostUnreachableError'
+    rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError => e
+      expect( e.original_error ).to be_a ::Manticore::ClientStoppedException
+    end
   end
   
   describe "auth" do

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -4,7 +4,7 @@ require 'cabin'
 
 describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   let(:logger) { Cabin::Channel.get }
-  let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger) }
+  let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger, {}) }
   let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
   let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
   let(:es_node_versions) { [ "0.0.0" ] }

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -4,6 +4,8 @@ require "flores/random"
 require 'concurrent/atomic/count_down_latch'
 require "logstash/outputs/elasticsearch"
 
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
+
 describe LogStash::Outputs::ElasticSearch do
   subject(:elasticsearch_output_instance) { described_class.new(options) }
   let(:options) { {} }
@@ -922,6 +924,25 @@ describe LogStash::Outputs::ElasticSearch do
     end
   end
 
+  describe 'ECS Compatibility Support', :ecs_compatibility_support do
+    [
+      :disabled,
+      :v1,
+      :v8,
+    ].each do |ecs_compatibility|
+      context "When initialized with `ecs_compatibility => #{ecs_compatibility}`" do
+        let(:options) { Hash.new }
+        subject(:output) { described_class.new(options.merge("ecs_compatibility" => "#{ecs_compatibility}")) }
+        context 'when registered' do
+          before(:each) { output.register }
+          it 'has the correct effective ECS compatibility setting' do
+            expect(output.ecs_compatibility).to eq(ecs_compatibility)
+          end
+        end
+      end
+    end
+  end
+
   describe "post-register ES setup" do
     let(:do_register) { false }
     let(:es_version) { '7.10.0' } # DS default on LS 8.x
@@ -959,7 +980,7 @@ describe LogStash::Outputs::ElasticSearch do
     context 'error raised' do
 
       let(:es_version) { '7.8.0' }
-      let(:options) { super().merge('data_stream' => 'true') }
+      let(:options) { super().merge('data_stream' => 'true', 'ecs_compatibility' => 'v1') }
       let(:latch) { Concurrent::CountDownLatch.new }
 
       before do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.3.0
+  version: 11.4.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-03 00:00:00.000000000 Z
+date: 2022-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -84,6 +84,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-deprecation_logger_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: