logstash-output-elasticsearch 11.2.1-java → 11.3.1-java

data/lib/logstash/outputs/elasticsearch.rb

@@ -94,6 +94,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   require "logstash/outputs/elasticsearch/ilm"
   require "logstash/outputs/elasticsearch/data_stream_support"
   require 'logstash/plugin_mixins/ecs_compatibility_support'
+  require 'logstash/plugin_mixins/deprecation_logger_support'
 
   # Protocol agnostic methods
   include(LogStash::PluginMixins::ElasticSearch::Common)
@@ -102,7 +103,10 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   include(LogStash::Outputs::ElasticSearch::Ilm)
 
   # ecs_compatibility option, provided by Logstash core or the support adapter.
-  include(LogStash::PluginMixins::ECSCompatibilitySupport)
+  include(LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8))
+
+  # deprecation logger adapter for older Logstashes
+  include(LogStash::PluginMixins::DeprecationLoggerSupport)
 
   # Generic/API config options that any document indexer output needs
   include(LogStash::PluginMixins::ElasticSearch::APIConfigs)
@@ -300,6 +304,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
     @bulk_request_metrics = metric.namespace(:bulk_requests)
     @document_level_metrics = metric.namespace(:documents)
+
+    if ecs_compatibility == :v8
+      @logger.warn("Elasticsearch Output configured with `ecs_compatibility => v8`, which resolved to an UNRELEASED preview of version 8.0.0 of the Elastic Common Schema. " +
+                   "Once ECS v8 and an updated release of this plugin are publicly available, you will need to update this plugin to resolve this warning.")
+    end
   end
 
   # @override post-register when ES connection established
@@ -493,7 +502,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
       @default_index = "logstash-%{+yyyy.MM.dd}"
       @default_ilm_rollover_alias = "logstash"
       @default_template_name = 'logstash'
-    when :v1
+    when :v1, :v8
       @default_index = "ecs-logstash-%{+yyyy.MM.dd}"
       @default_ilm_rollover_alias = "ecs-logstash"
       @default_template_name = 'ecs-logstash'

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.2.1'
+  s.version         = '11.3.1'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
+  s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~>1.0'
 
   s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'logstash-devutils'

data/spec/integration/outputs/data_stream_spec.rb

@@ -12,6 +12,13 @@ describe "data streams", :integration => true do
 
   subject { LogStash::Outputs::ElasticSearch.new(options) }
 
+  # All data-streams features require that the plugin be run in a non-disabled ECS compatibility mode.
+  # We run the plugin in ECS by default, and add test scenarios specifically for it being disabled.
+  let(:ecs_compatibility) { :v1 }
+  before(:each) do
+    allow( subject ).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+  end
+
   before :each do
     @es = get_client
     @es.delete_by_query(index: ".ds-#{ds_name}-*", expand_wildcards: :all, body: { query: { match_all: {} } }) rescue nil

data/spec/integration/outputs/templates_spec.rb

@@ -1,8 +1,12 @@
 require_relative "../../../spec/es_spec_helper"
 
 describe "index template expected behavior", :integration => true do
+  let(:ecs_compatibility) { fail('spec group does not define `ecs_compatibility`!') }
+
   subject! do
     require "logstash/outputs/elasticsearch"
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+
     settings = {
       "manage_template" => true,
       "template_overwrite" => true,
@@ -11,86 +15,117 @@ describe "index template expected behavior", :integration => true do
     next LogStash::Outputs::ElasticSearch.new(settings)
   end
 
-  before :each do
-    # Delete all templates first.
-    require "elasticsearch"
+  let(:elasticsearch_client) { get_client }
 
-    # Clean ES of data before we start.
-    @es = get_client
-    @es.indices.delete_template(:name => "*")
+  before(:each) do
+    # delete indices and templates
+    require "elasticsearch"
 
+    elasticsearch_client.indices.delete_template(:name => '*')
     # This can fail if there are no indexes, ignore failure.
-    @es.indices.delete(:index => "*") rescue nil
-
-    subject.register
-
-    subject.multi_receive([
-      LogStash::Event.new("message" => "sample message here"),
-      LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
-      LogStash::Event.new("somevalue" => 100),
-      LogStash::Event.new("somevalue" => 10),
-      LogStash::Event.new("somevalue" => 1),
-      LogStash::Event.new("country" => "us"),
-      LogStash::Event.new("country" => "at"),
-      LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
-    ])
-
-    @es.indices.refresh
-
-    # Wait or fail until everything's indexed.
-    Stud::try(20.times) do
-      r = @es.search(index: 'logstash-*')
-      expect(r).to have_hits(8)
-    end
+    elasticsearch_client.indices.delete(:index => '*') rescue nil
   end
 
-  it "permits phrase searching on string fields" do
-    results = @es.search(:q => "message:\"sample message\"")
-    expect(results).to have_hits(1)
-    expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
-  end
+  context 'with ecs_compatibility => disabled' do
+    let(:ecs_compatibility) { :disabled }
+    before :each do
+      @es = elasticsearch_client # cache as ivar for tests...
+
+      subject.register
+
+      subject.multi_receive([
+        LogStash::Event.new("message" => "sample message here"),
+        LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
+        LogStash::Event.new("somevalue" => 100),
+        LogStash::Event.new("somevalue" => 10),
+        LogStash::Event.new("somevalue" => 1),
+        LogStash::Event.new("country" => "us"),
+        LogStash::Event.new("country" => "at"),
+        LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
+      ])
+
+      @es.indices.refresh
+
+      # Wait or fail until everything's indexed.
+      Stud::try(20.times) do
+        r = @es.search(index: 'logstash-*')
+        expect(r).to have_hits(8)
+      end
+    end
 
-  it "numbers dynamically map to a numeric type and permit range queries" do
-    results = @es.search(:q => "somevalue:[5 TO 105]")
-    expect(results).to have_hits(2)
+    it "permits phrase searching on string fields" do
+      results = @es.search(:q => "message:\"sample message\"")
+      expect(results).to have_hits(1)
+      expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
+    end
 
-    values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
-    expect(values).to include(10)
-    expect(values).to include(100)
-    expect(values).to_not include(1)
-  end
+    it "numbers dynamically map to a numeric type and permit range queries" do
+      results = @es.search(:q => "somevalue:[5 TO 105]")
+      expect(results).to have_hits(2)
 
-  it "does not create .keyword field for top-level message field" do
-    results = @es.search(:q => "message.keyword:\"sample message here\"")
-    expect(results).to have_hits(0)
-  end
+      values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
+      expect(values).to include(10)
+      expect(values).to include(100)
+      expect(values).to_not include(1)
+    end
 
-  it "creates .keyword field for nested message fields" do
-    results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
-    expect(results).to have_hits(1)
-  end
+    it "does not create .keyword field for top-level message field" do
+      results = @es.search(:q => "message.keyword:\"sample message here\"")
+      expect(results).to have_hits(0)
+    end
 
-  it "creates .keyword field from any string field which is not_analyzed" do
-    results = @es.search(:q => "country.keyword:\"us\"")
-    expect(results).to have_hits(1)
-    expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
+    it "creates .keyword field for nested message fields" do
+      results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
+      expect(results).to have_hits(1)
+    end
 
-    # partial or terms should not work.
-    results = @es.search(:q => "country.keyword:\"u\"")
-    expect(results).to have_hits(0)
-  end
+    it "creates .keyword field from any string field which is not_analyzed" do
+      results = @es.search(:q => "country.keyword:\"us\"")
+      expect(results).to have_hits(1)
+      expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
+
+      # partial or terms should not work.
+      results = @es.search(:q => "country.keyword:\"u\"")
+      expect(results).to have_hits(0)
+    end
+
+    it "make [geoip][location] a geo_point" do
+      expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
+    end
+
+    it "aggregate .keyword results correctly " do
+      results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
+      terms = results["buckets"].collect { |b| b["key"] }
 
-  it "make [geoip][location] a geo_point" do
-    expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
+      expect(terms).to include("us")
+
+      # 'at' is a stopword, make sure stopwords are not ignored.
+      expect(terms).to include("at")
+    end
   end
 
-  it "aggregate .keyword results correctly " do
-    results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
-    terms = results["buckets"].collect { |b| b["key"] }
+  context 'with ECS enabled' do
+    let(:ecs_compatibility) { :v1 }
+
+    before(:each) do
+      subject.register # should load template?
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+    end
 
-    expect(terms).to include("us")
+    let(:elasticsearch_cluster_major_version) do
+      elasticsearch_client.info&.dig("version", "number" )&.split('.')&.map(&:to_i)&.first
+    end
 
-    # 'at' is a stopword, make sure stopwords are not ignored.
-    expect(terms).to include("at")
+    it 'loads the templates' do
+      aggregate_failures do
+        if elasticsearch_cluster_major_version >= 8
+          # In ES 8+ we use the _index_template API
+          expect(elasticsearch_client.indices.exists_index_template(name: 'ecs-logstash')).to be_truthy
+        else
+          # Otherwise, we used the legacy _template API
+          expect(elasticsearch_client.indices.exists_template(name: 'ecs-logstash')).to be_truthy
+        end
+      end
+    end
   end
 end

data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb

@@ -4,9 +4,17 @@ require "logstash/outputs/elasticsearch/data_stream_support"
 describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
 
   subject { LogStash::Outputs::ElasticSearch.new(options) }
+
   let(:options) { { 'hosts' => [ 'localhost:12345' ] } }
   let(:es_version) { '7.10.1' }
 
+  # All data-streams features require that the plugin be run in a non-disabled ECS compatibility mode.
+  # We run the plugin in ECS by default, and add test scenarios specifically for it being disabled.
+  let(:ecs_compatibility) { :v1 }
+  before(:each) do
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+  end
+
   let(:do_register) { false }
   let(:stub_plugin_register!) do
     allow(subject).to receive(:last_es_version).and_return(es_version)
@@ -52,9 +60,7 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
     end
 
     it "warns when configuration is data-stream compliant (LS 7.x)" do
-      expect( subject.logger ).to receive(:warn) do |msg|
-        expect(msg).to include "Configuration is data stream compliant but due backwards compatibility Logstash 7.x"
-      end
+      expect( subject.logger ).to receive(:warn).with(a_string_including "Configuration is data stream compliant but due backwards compatibility Logstash 7.x")
       change_constant :LOGSTASH_VERSION, '7.11.0' do
         expect( subject.data_stream_config? ).to be false
       end
@@ -66,6 +72,25 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
       end
     end
 
+    context 'ecs_compatibility disabled' do
+      let(:ecs_compatibility) { :disabled }
+
+      {
+        '7.x (pre-DS)' => '7.9.0',
+        '7.x (with DS)' => '7.11.0',
+        '8.0' => '8.0.0',
+        '8.x' => '8.1.2',
+      }.each do |ls_version_desc, ls_version|
+        context "on LS #{ls_version_desc}" do
+          around(:each) { |example| change_constant(:LOGSTASH_VERSION, ls_version, &example) }
+          it "does not use data-streams" do
+            expect( subject.logger ).to receive(:debug).with(a_string_including "ecs_compatibility is not enabled")
+            expect( subject.data_stream_config? ).to be false
+          end
+        end
+      end
+    end
+
     context 'non-compatible ES' do
 
       let(:es_version) { '7.8.0' }
@@ -95,6 +120,7 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
     before { allow(subject).to receive(:last_es_version).and_return(es_version) }
 
     it "does not use data-streams on LS 7.x" do
+      expect( subject.logger ).to receive(:warn).with(a_string_including "Logstash 7.x will not assume writing to a data-stream")
       change_constant :LOGSTASH_VERSION, '7.10.0' do
         expect( subject.data_stream_config? ).to be false
       end
@@ -197,12 +223,34 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
       end
     end
 
-    it "does use data-streams on LS 8.0" do
+    it "does use data-streams on LS 8.x" do
       change_constant :LOGSTASH_VERSION, '8.1.0' do
         expect( subject.data_stream_config? ).to be true
       end
     end
 
+    context 'with ecs_compatibility disabled' do
+      let(:ecs_compatibility) { :disabled }
+
+      context 'when running on LS 7.x' do
+        around(:each) { |example| change_constant(:LOGSTASH_VERSION, '7.15.1', &example) }
+
+        it "emits a deprecation warning and uses data streams anway" do
+          expect( subject.deprecation_logger ).to receive(:deprecated).with(a_string_including "`data_stream => true` will require the plugin to be run in ECS compatibility mode")
+          expect( subject.data_stream_config? ).to be true
+        end
+      end
+
+      context 'when running on LS 8.x' do
+        around(:each) { |example| change_constant(:LOGSTASH_VERSION, '8.0.0', &example) }
+
+        it "errors helpfully" do
+          expect{ subject.data_stream_config? }.to raise_error(LogStash::ConfigurationError, a_string_including("Invalid data stream configuration: `ecs_compatibility => disabled`"))
+        end
+      end
+
+    end
+
     context 'non-compatible ES' do
 
       let(:es_version) { '6.8.11' }

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -27,6 +27,12 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
     end
   end
 
+  context 'when ECS v8 is requested' do
+    it 'resolves' do
+      expect(described_class.default_template_path(7, :v8)).to end_with("/templates/ecs-v8/elasticsearch-7x.json")
+    end
+  end
+
   describe "index template with ilm settings" do
     let(:plugin_settings) { {"manage_template" => true, "template_overwrite" => true} }
     let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -4,6 +4,8 @@ require "flores/random"
 require 'concurrent/atomic/count_down_latch'
 require "logstash/outputs/elasticsearch"
 
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
+
 describe LogStash::Outputs::ElasticSearch do
   subject(:elasticsearch_output_instance) { described_class.new(options) }
   let(:options) { {} }
@@ -922,6 +924,25 @@ describe LogStash::Outputs::ElasticSearch do
     end
   end
 
+  describe 'ECS Compatibility Support', :ecs_compatibility_support do
+    [
+      :disabled,
+      :v1,
+      :v8,
+    ].each do |ecs_compatibility|
+      context "When initialized with `ecs_compatibility => #{ecs_compatibility}`" do
+        let(:options) { Hash.new }
+        subject(:output) { described_class.new(options.merge("ecs_compatibility" => "#{ecs_compatibility}")) }
+        context 'when registered' do
+          before(:each) { output.register }
+          it 'has the correct effective ECS compatibility setting' do
+            expect(output.ecs_compatibility).to eq(ecs_compatibility)
+          end
+        end
+      end
+    end
+  end
+
   describe "post-register ES setup" do
     let(:do_register) { false }
     let(:es_version) { '7.10.0' } # DS default on LS 8.x
@@ -959,7 +980,7 @@ describe LogStash::Outputs::ElasticSearch do
     context 'error raised' do
 
       let(:es_version) { '7.8.0' }
-      let(:options) { super().merge('data_stream' => 'true') }
+      let(:options) { super().merge('data_stream' => 'true', 'ecs_compatibility' => 'v1') }
       let(:latch) { Concurrent::CountDownLatch.new }
 
       before do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.2.1
+  version: 11.3.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-14 00:00:00.000000000 Z
+date: 2021-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -84,6 +84,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-deprecation_logger_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -212,6 +226,9 @@ files:
 - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-6x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-7x.json
+- lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json
+- lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-7x.json
+- lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-8x.json
 - lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
 - lib/logstash/plugin_mixins/elasticsearch/common.rb
 - lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb