logstash-output-elasticsearch 11.2.0-java → 11.3.0-java

data/lib/logstash/outputs/elasticsearch.rb

@@ -102,7 +102,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   include(LogStash::Outputs::ElasticSearch::Ilm)
 
   # ecs_compatibility option, provided by Logstash core or the support adapter.
-  include(LogStash::PluginMixins::ECSCompatibilitySupport)
+  include(LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8))
 
   # Generic/API config options that any document indexer output needs
   include(LogStash::PluginMixins::ElasticSearch::APIConfigs)
@@ -300,6 +300,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
     @bulk_request_metrics = metric.namespace(:bulk_requests)
     @document_level_metrics = metric.namespace(:documents)
+
+    if ecs_compatibility == :v8
+      @logger.warn("Elasticsearch Output configured with `ecs_compatibility => v8`, which resolved to an UNRELEASED preview of version 8.0.0 of the Elastic Common Schema. " +
+                   "Once ECS v8 and an updated release of this plugin are publicly available, you will need to update this plugin to resolve this warning.")
+    end
   end
 
   # @override post-register when ES connection established

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.2.0'
+  s.version         = '11.3.0'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"

data/spec/integration/outputs/templates_spec.rb

@@ -1,8 +1,12 @@
 require_relative "../../../spec/es_spec_helper"
 
 describe "index template expected behavior", :integration => true do
+  let(:ecs_compatibility) { fail('spec group does not define `ecs_compatibility`!') }
+
   subject! do
     require "logstash/outputs/elasticsearch"
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+
     settings = {
       "manage_template" => true,
       "template_overwrite" => true,
@@ -11,86 +15,117 @@ describe "index template expected behavior", :integration => true do
     next LogStash::Outputs::ElasticSearch.new(settings)
   end
 
-  before :each do
-    # Delete all templates first.
-    require "elasticsearch"
+  let(:elasticsearch_client) { get_client }
 
-    # Clean ES of data before we start.
-    @es = get_client
-    @es.indices.delete_template(:name => "*")
+  before(:each) do
+    # delete indices and templates
+    require "elasticsearch"
 
+    elasticsearch_client.indices.delete_template(:name => '*')
     # This can fail if there are no indexes, ignore failure.
-    @es.indices.delete(:index => "*") rescue nil
-
-    subject.register
-
-    subject.multi_receive([
-      LogStash::Event.new("message" => "sample message here"),
-      LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
-      LogStash::Event.new("somevalue" => 100),
-      LogStash::Event.new("somevalue" => 10),
-      LogStash::Event.new("somevalue" => 1),
-      LogStash::Event.new("country" => "us"),
-      LogStash::Event.new("country" => "at"),
-      LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
-    ])
-
-    @es.indices.refresh
-
-    # Wait or fail until everything's indexed.
-    Stud::try(20.times) do
-      r = @es.search(index: 'logstash-*')
-      expect(r).to have_hits(8)
-    end
+    elasticsearch_client.indices.delete(:index => '*') rescue nil
   end
 
-  it "permits phrase searching on string fields" do
-    results = @es.search(:q => "message:\"sample message\"")
-    expect(results).to have_hits(1)
-    expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
-  end
+  context 'with ecs_compatibility => disabled' do
+    let(:ecs_compatibility) { :disabled }
+    before :each do
+      @es = elasticsearch_client # cache as ivar for tests...
+
+      subject.register
+
+      subject.multi_receive([
+        LogStash::Event.new("message" => "sample message here"),
+        LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
+        LogStash::Event.new("somevalue" => 100),
+        LogStash::Event.new("somevalue" => 10),
+        LogStash::Event.new("somevalue" => 1),
+        LogStash::Event.new("country" => "us"),
+        LogStash::Event.new("country" => "at"),
+        LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
+      ])
+
+      @es.indices.refresh
+
+      # Wait or fail until everything's indexed.
+      Stud::try(20.times) do
+        r = @es.search(index: 'logstash-*')
+        expect(r).to have_hits(8)
+      end
+    end
 
-  it "numbers dynamically map to a numeric type and permit range queries" do
-    results = @es.search(:q => "somevalue:[5 TO 105]")
-    expect(results).to have_hits(2)
+    it "permits phrase searching on string fields" do
+      results = @es.search(:q => "message:\"sample message\"")
+      expect(results).to have_hits(1)
+      expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
+    end
 
-    values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
-    expect(values).to include(10)
-    expect(values).to include(100)
-    expect(values).to_not include(1)
-  end
+    it "numbers dynamically map to a numeric type and permit range queries" do
+      results = @es.search(:q => "somevalue:[5 TO 105]")
+      expect(results).to have_hits(2)
 
-  it "does not create .keyword field for top-level message field" do
-    results = @es.search(:q => "message.keyword:\"sample message here\"")
-    expect(results).to have_hits(0)
-  end
+      values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
+      expect(values).to include(10)
+      expect(values).to include(100)
+      expect(values).to_not include(1)
+    end
 
-  it "creates .keyword field for nested message fields" do
-    results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
-    expect(results).to have_hits(1)
-  end
+    it "does not create .keyword field for top-level message field" do
+      results = @es.search(:q => "message.keyword:\"sample message here\"")
+      expect(results).to have_hits(0)
+    end
 
-  it "creates .keyword field from any string field which is not_analyzed" do
-    results = @es.search(:q => "country.keyword:\"us\"")
-    expect(results).to have_hits(1)
-    expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
+    it "creates .keyword field for nested message fields" do
+      results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
+      expect(results).to have_hits(1)
+    end
 
-    # partial or terms should not work.
-    results = @es.search(:q => "country.keyword:\"u\"")
-    expect(results).to have_hits(0)
-  end
+    it "creates .keyword field from any string field which is not_analyzed" do
+      results = @es.search(:q => "country.keyword:\"us\"")
+      expect(results).to have_hits(1)
+      expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
+
+      # partial or terms should not work.
+      results = @es.search(:q => "country.keyword:\"u\"")
+      expect(results).to have_hits(0)
+    end
+
+    it "make [geoip][location] a geo_point" do
+      expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
+    end
+
+    it "aggregate .keyword results correctly " do
+      results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
+      terms = results["buckets"].collect { |b| b["key"] }
 
-  it "make [geoip][location] a geo_point" do
-    expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
+      expect(terms).to include("us")
+
+      # 'at' is a stopword, make sure stopwords are not ignored.
+      expect(terms).to include("at")
+    end
   end
 
-  it "aggregate .keyword results correctly " do
-    results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
-    terms = results["buckets"].collect { |b| b["key"] }
+  context 'with ECS enabled' do
+    let(:ecs_compatibility) { :v1 }
+
+    before(:each) do
+      subject.register # should load template?
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+    end
 
-    expect(terms).to include("us")
+    let(:elasticsearch_cluster_major_version) do
+      elasticsearch_client.info&.dig("version", "number" )&.split('.')&.map(&:to_i)&.first
+    end
 
-    # 'at' is a stopword, make sure stopwords are not ignored.
-    expect(terms).to include("at")
+    it 'loads the templates' do
+      aggregate_failures do
+        if elasticsearch_cluster_major_version >= 8
+          # In ES 8+ we use the _index_template API
+          expect(elasticsearch_client.indices.exists_index_template(name: 'ecs-logstash')).to be_truthy
+        else
+          # Otherwise, we used the legacy _template API
+          expect(elasticsearch_client.indices.exists_template(name: 'ecs-logstash')).to be_truthy
+        end
+      end
+    end
   end
 end

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -27,6 +27,12 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
     end
   end
 
+  context 'when ECS v8 is requested' do
+    it 'resolves' do
+      expect(described_class.default_template_path(7, :v8)).to end_with("/templates/ecs-v8/elasticsearch-7x.json")
+    end
+  end
+
   describe "index template with ilm settings" do
     let(:plugin_settings) { {"manage_template" => true, "template_overwrite" => true} }
     let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.2.0
+  version: 11.3.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-12 00:00:00.000000000 Z
+date: 2021-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -212,6 +212,9 @@ files:
 - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-6x.json
 - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-7x.json
+- lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json
+- lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-7x.json
+- lib/logstash/outputs/elasticsearch/templates/ecs-v8/elasticsearch-8x.json
 - lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
 - lib/logstash/plugin_mixins/elasticsearch/common.rb
 - lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb