logstash-output-elasticsearch 10.5.1-java → 10.7.3-java

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '10.5.1'
+  s.version         = '10.7.3'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency 'cabin', ['~> 0.6']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
 
   s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'logstash-devutils'

data/spec/es_spec_helper.rb

@@ -49,19 +49,10 @@ module ESHelper
     Time.now.strftime("%Y.%m.%d")
   end
 
-
-  def default_mapping_from_mappings(mappings)
-    if ESHelper.es_version_satisfies?(">=7")
-      mappings
-    else
-      mappings["_default_"]
-    end
-  end
-
   def field_properties_from_template(template_name, field)
-    mappings = @es.indices.get_template(name: template_name)[template_name]["mappings"]
-    mapping = default_mapping_from_mappings(mappings)
-    mapping["properties"][field]["properties"]
+    template = get_template(@es, template_name)
+    mappings = get_template_mappings(template)
+    mappings["properties"][field]["properties"]
   end
 
   def routing_field_name
@@ -105,6 +96,7 @@ module ESHelper
 
   def clean(client)
     client.indices.delete_template(:name => "*")
+    client.indices.delete_index_template(:name => "logstash*") rescue nil
     # This can fail if there are no indexes, ignore failure.
     client.indices.delete(:index => "*") rescue nil
     clean_ilm(client) if supports_ilm?(client)
@@ -182,6 +174,34 @@ module ESHelper
     }
   }
   end
+
+  def get_template(client, name)
+    if ESHelper.es_version_satisfies?(">=8")
+      t = client.indices.get_index_template(name: name)
+      t['index_templates'][0]['index_template']
+    else
+      t = client.indices.get_template(name: name)
+      t[name]
+    end
+  end
+
+  def get_template_settings(template)
+    if ESHelper.es_version_satisfies?(">=8")
+      template['template']['settings']
+    else
+      template['settings']
+    end
+  end
+
+  def get_template_mappings(template)
+    if ESHelper.es_version_satisfies?(">=8")
+      template['template']['mappings']
+    elsif ESHelper.es_version_satisfies?(">=7")
+      template['mappings']
+    else
+      template['mappings']["_default_"]
+    end
+  end
 end
 
 RSpec.configure do |config|

data/spec/fixtures/template-with-policy-es8x.json

@@ -0,0 +1,50 @@
+{
+  "index_patterns" : "overwrite-*",
+  "version" : 80001,
+  "template" : {
+    "settings" : {
+      "index.refresh_interval" : "1s",
+      "number_of_shards": 1
+    },
+    "mappings" : {
+      "dynamic_templates" : [ {
+        "message_field" : {
+          "path_match" : "message",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text",
+            "norms" : false
+          }
+        }
+      }, {
+        "string_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text", "norms" : false,
+            "fields" : {
+              "keyword" : { "type": "keyword", "ignore_above": 256 }
+            }
+          }
+        }
+      } ],
+      "properties" : {
+        "@timestamp": { "type": "date" },
+        "@version": { "type": "keyword" },
+        "geoip"  : {
+          "dynamic": true,
+          "properties" : {
+            "ip": { "type": "ip" },
+            "location" : { "type" : "geo_point" },
+            "latitude" : { "type" : "half_float" },
+            "longitude" : { "type" : "half_float" }
+          }
+        }
+      }
+    }
+  },
+  "priority": 200,
+  "_meta" : {
+    "description": "index template for logstash-output-elasticsearch"
+  }
+}

data/spec/integration/outputs/ilm_spec.rb

@@ -8,7 +8,7 @@ shared_examples_for 'an ILM enabled Logstash' do
     let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
 
     it 'should rollover when the policy max docs is reached' do
-      put_policy(@es,ilm_policy_name, policy)
+      put_policy(@es, ilm_policy_name, policy)
       subject.register
 
       subject.multi_receive([
@@ -108,9 +108,11 @@ shared_examples_for 'an ILM disabled Logstash' do
   it 'should not write the ILM settings into the template' do
     subject.register
     sleep(1)
-    expect(@es.indices.get_template(name: "logstash")["logstash"]).to have_index_pattern("logstash-*")
+
+    template = get_template(@es, "logstash")
+    expect(template).to have_index_pattern("logstash-*")
     if ESHelper.es_version_satisfies?(">= 2")
-      expect(@es.indices.get_template(name: "logstash")["logstash"]["settings"]['index']['lifecycle']).to be_nil
+      expect(get_template_settings(template)['index']['lifecycle']).to be_nil
     end
   end
 
@@ -152,16 +154,17 @@ shared_examples_for 'an ILM disabled Logstash' do
   end
 
   context 'with a custom template name' do
-    let (:template_name) { "custom_template_name" }
+    let (:template_name) { "logstash_custom_template_name" }
     let (:settings) { super.merge('template_name' => template_name)}
 
     it 'should not write the ILM settings into the template' do
       subject.register
       sleep(1)
 
-      expect(@es.indices.get_template(name: template_name)[template_name]).to have_index_pattern("logstash-*")
+      template = get_template(@es, template_name)
+      expect(template).to have_index_pattern("logstash-*")
       if ESHelper.es_version_satisfies?(">= 2")
-        expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']).to be_nil
+        expect(get_template_settings(template)['index']['lifecycle']).to be_nil
       end
     end
   end
@@ -249,9 +252,9 @@ if ESHelper.es_version_satisfies?(">= 6.6")
     }
     let (:small_max_doc_policy) { max_docs_policy(3) }
     let (:large_max_doc_policy) { max_docs_policy(1000000) }
-    let (:expected_index) { LogStash::Outputs::ElasticSearch::DEFAULT_ROLLOVER_ALIAS }
+    let (:expected_index) { elasticsearch_output_plugin.default_ilm_rollover_alias }
 
-    subject { LogStash::Outputs::ElasticSearch.new(settings) }
+    subject(:elasticsearch_output_plugin) { LogStash::Outputs::ElasticSearch.new(settings) }
 
     before :each do
       # Delete all templates first.
@@ -387,16 +390,20 @@ if ESHelper.es_version_satisfies?(">= 6.6")
         it 'should write the ILM settings into the template' do
           subject.register
           sleep(1)
-          expect(@es.indices.get_template(name: "logstash")["logstash"]).to have_index_pattern("logstash-*")
-          expect(@es.indices.get_template(name: "logstash")["logstash"]["settings"]['index']['lifecycle']['name']).to eq("logstash-policy")
-          expect(@es.indices.get_template(name: "logstash")["logstash"]["settings"]['index']['lifecycle']['rollover_alias']).to eq("logstash")
+
+          template = get_template(@es, "logstash")
+          expect(template).to have_index_pattern("logstash-*")
+          expect(get_template_settings(template)['index']['lifecycle']['name']).to eq("logstash-policy")
+          expect(get_template_settings(template)['index']['lifecycle']['rollover_alias']).to eq("logstash")
         end
 
         it_behaves_like 'an ILM enabled Logstash'
       end
 
       context 'with a set index and a custom index pattern' do
-        if ESHelper.es_version_satisfies?(">= 7.0")
+        if ESHelper.es_version_satisfies?(">= 8.0")
+          let (:template) { "spec/fixtures/template-with-policy-es8x.json" }
+        elsif ESHelper.es_version_satisfies?(">= 7.0")
           let (:template) { "spec/fixtures/template-with-policy-es7x.json" }
         else
           let (:template) { "spec/fixtures/template-with-policy-es6x.json" }
@@ -408,13 +415,15 @@ if ESHelper.es_version_satisfies?(">= 6.6")
         it 'should not overwrite the index patterns' do
           subject.register
           sleep(1)
-          expect(@es.indices.get_template(name: "logstash")["logstash"]).to have_index_pattern("overwrite-*")
+
+          template = get_template(@es, "logstash")
+          expect(template).to have_index_pattern("overwrite-*")
         end
       end
 
 
       context 'with a custom template' do
-        let (:ilm_rollover_alias) { "the_cat_in_the_hat" }
+        let (:ilm_rollover_alias) { "logstash_the_cat_in_the_hat" }
         let (:index) { ilm_rollover_alias }
         let(:expected_index) { index }
         let (:settings) { super.merge("ilm_policy" => ilm_policy_name,
@@ -422,7 +431,9 @@ if ESHelper.es_version_satisfies?(">= 6.6")
                                       "ilm_rollover_alias" => ilm_rollover_alias)}
 
 
-        if ESHelper.es_version_satisfies?(">= 7.0")
+        if ESHelper.es_version_satisfies?(">= 8.0")
+          let (:template) { "spec/fixtures/template-with-policy-es8x.json" }
+        elsif ESHelper.es_version_satisfies?(">= 7.0")
           let (:template) { "spec/fixtures/template-with-policy-es7x.json" }
         else
           let (:template) { "spec/fixtures/template-with-policy-es6x.json" }
@@ -460,13 +471,15 @@ if ESHelper.es_version_satisfies?(">= 6.6")
         it 'should write the ILM settings into the template' do
           subject.register
           sleep(1)
-          expect(@es.indices.get_template(name: ilm_rollover_alias)[ilm_rollover_alias]).to have_index_pattern("#{ilm_rollover_alias}-*")
-          expect(@es.indices.get_template(name: ilm_rollover_alias)[ilm_rollover_alias]["settings"]['index']['lifecycle']['name']).to eq(ilm_policy_name)
-          expect(@es.indices.get_template(name: ilm_rollover_alias)[ilm_rollover_alias]["settings"]['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias)
+
+          template = get_template(@es, ilm_rollover_alias)
+          expect(template).to have_index_pattern("#{ilm_rollover_alias}-*")
+          expect(get_template_settings(template)['index']['lifecycle']['name']).to eq(ilm_policy_name)
+          expect(get_template_settings(template)['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias)
         end
 
         context 'with a different template_name' do
-          let (:template_name) { "custom_template_name" }
+          let (:template_name) { "logstash_custom_template_name" }
           let (:settings) { super.merge('template_name' => template_name)}
 
           it_behaves_like 'an ILM enabled Logstash'
@@ -474,9 +487,10 @@ if ESHelper.es_version_satisfies?(">= 6.6")
           it 'should write the ILM settings into the template' do
             subject.register
             sleep(1)
-            expect(@es.indices.get_template(name: template_name)[template_name]).to have_index_pattern("#{ilm_rollover_alias}-*")
-            expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['name']).to eq(ilm_policy_name)
-            expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias)
+            template = get_template(@es, template_name)
+            expect(template).to have_index_pattern("#{ilm_rollover_alias}-*")
+            expect(get_template_settings(template)['index']['lifecycle']['name']).to eq(ilm_policy_name)
+            expect(get_template_settings(template)['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias)
           end
         end
 

data/spec/integration/outputs/metrics_spec.rb

@@ -19,11 +19,7 @@ describe "metrics", :integration => true do
 
     # Clean ES of data before we start.
     @es = get_client
-    @es.indices.delete_template(:name => "*")
-
-    # This can fail if there are no indexes, ignore failure.
-    @es.indices.delete(:index => "*") rescue nil
-    #@es.indices.refresh
+    clean(@es)
     subject.register
   end
 

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -1,3 +1,4 @@
+require_relative "../../../../spec/es_spec_helper"
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/elasticsearch/http_client"
 require "java"
@@ -138,6 +139,27 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
     end
   end
 
+  describe "index template" do
+    subject { described_class.new(base_options) }
+    let(:template_name) { "logstash" }
+    let(:template) { {} }
+    let(:get_response) {
+      double("response", :body => {})
+    }
+
+    it "should call composable index template in version 8+" do
+      expect(subject).to receive(:maximum_seen_major_version).and_return(8)
+      expect(subject.pool).to receive(:put).with("_index_template/#{template_name}", nil, anything).and_return(get_response)
+      subject.template_put(template_name, template)
+    end
+
+    it "should call index template in version < 8" do
+      expect(subject).to receive(:maximum_seen_major_version).and_return(7)
+      expect(subject.pool).to receive(:put).with("_template/#{template_name}", nil, anything).and_return(get_response)
+      subject.template_put(template_name, template)
+    end
+  end
+
   describe "join_bulk_responses" do
     subject { described_class.new(base_options) }
 

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -8,17 +8,54 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
   describe ".default_template_path" do
     context "elasticsearch 1.x" do
       it "chooses the 2x template" do
-        expect(described_class.default_template_path(1)).to match(/elasticsearch-template-es2x.json/)
+        expect(described_class.default_template_path(1)).to end_with("/templates/ecs-disabled/elasticsearch-2x.json")
       end
     end
     context "elasticsearch 2.x" do
       it "chooses the 2x template" do
-        expect(described_class.default_template_path(2)).to match(/elasticsearch-template-es2x.json/)
+        expect(described_class.default_template_path(2)).to end_with("/templates/ecs-disabled/elasticsearch-2x.json")
       end
     end
     context "elasticsearch 5.x" do
       it "chooses the 5x template" do
-        expect(described_class.default_template_path(5)).to match(/elasticsearch-template-es5x.json/)
+        expect(described_class.default_template_path(5)).to end_with("/templates/ecs-disabled/elasticsearch-5x.json")
+      end
+    end
+  end
+
+  context 'when ECS v1 is requested' do
+    it 'resolves' do
+      expect(described_class.default_template_path(7, :v1)).to end_with("/templates/ecs-v1/elasticsearch-7x.json")
+    end
+  end
+
+  describe "index template with ilm settings" do
+    let(:plugin_settings) { {"manage_template" => true, "template_overwrite" => true} }
+    let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
+
+    describe "in version 8+" do
+      let(:file_path) { described_class.default_template_path(8) }
+      let(:template) { described_class.read_template_file(file_path)}
+
+      it "should update settings" do
+        expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
+        described_class.add_ilm_settings_to_template(plugin, template)
+        expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil)
+        expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
+        expect(template.include?('settings')).to be_falsey
+      end
+    end
+
+    describe "in version < 8" do
+      let(:file_path) { described_class.default_template_path(7) }
+      let(:template) { described_class.read_template_file(file_path)}
+
+      it "should update settings" do
+        expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
+        described_class.add_ilm_settings_to_template(plugin, template)
+        expect(template['settings']['index.lifecycle.name']).not_to eq(nil)
+        expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
+        expect(template.include?('template')).to be_falsey
       end
     end
   end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -6,7 +6,7 @@ require "logstash/outputs/elasticsearch"
 describe LogStash::Outputs::ElasticSearch do
   subject { described_class.new(options) }
   let(:options) { {} }
-  let(:maximum_seen_major_version) { rand(100) }
+  let(:maximum_seen_major_version) { [1,2,5,6,7,8].sample }
 
   let(:do_register) { true }
 
@@ -355,6 +355,28 @@ describe LogStash::Outputs::ElasticSearch do
     end
   end
 
+  describe "the pipeline option" do
+    context "with a sprintf and set pipeline" do
+      let(:options) { {"pipeline" => "%{pipeline}" } }
+
+      let(:event) { LogStash::Event.new("pipeline" => "my-ingest-pipeline") }
+
+      it "should interpolate the pipeline value and set it" do
+        expect(subject.event_action_tuple(event)[1]).to include(:pipeline => "my-ingest-pipeline")
+      end
+    end
+
+    context "with a sprintf and empty pipeline" do
+      let(:options) { {"pipeline" => "%{pipeline}" } }
+
+      let(:event) { LogStash::Event.new("pipeline" => "") }
+
+      it "should interpolate the pipeline value but not set it because it is empty" do
+        expect(subject.event_action_tuple(event)[1]).not_to include(:pipeline)
+      end
+    end
+  end
+
   describe "SSL end to end" do
     let(:do_register) { false } # skip the register in the global before block, as is called here.
 

data/spec/unit/outputs/error_whitelist_spec.rb

@@ -11,10 +11,10 @@ describe "whitelisting error types in expected behavior" do
 
   before :each do
     allow(subject.logger).to receive(:warn)
+    allow(subject).to receive(:maximum_seen_major_version).and_return(0)
 
     subject.register
 
-    allow(subject.client).to receive(:maximum_seen_major_version).and_return(0)
     allow(subject.client).to receive(:get_xpack_info)
     allow(subject.client).to receive(:bulk).and_return(
       {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 10.5.1
+  version: 10.7.3
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-04-30 00:00:00.000000000 Z
+date: 2020-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -84,6 +84,20 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-ecs_compatibility_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -159,17 +173,19 @@ files:
 - lib/logstash/outputs/elasticsearch/common.rb
 - lib/logstash/outputs/elasticsearch/common_configs.rb
 - lib/logstash/outputs/elasticsearch/default-ilm-policy.json
-- lib/logstash/outputs/elasticsearch/elasticsearch-template-es2x.json
-- lib/logstash/outputs/elasticsearch/elasticsearch-template-es5x.json
-- lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json
-- lib/logstash/outputs/elasticsearch/elasticsearch-template-es7x.json
-- lib/logstash/outputs/elasticsearch/elasticsearch-template-es8x.json
 - lib/logstash/outputs/elasticsearch/http_client.rb
 - lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb
 - lib/logstash/outputs/elasticsearch/http_client/pool.rb
 - lib/logstash/outputs/elasticsearch/http_client_builder.rb
 - lib/logstash/outputs/elasticsearch/ilm.rb
 - lib/logstash/outputs/elasticsearch/template_manager.rb
+- lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-2x.json
+- lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-5x.json
+- lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-6x.json
+- lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-7x.json
+- lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json
+- lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-6x.json
+- lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-7x.json
 - logstash-output-elasticsearch.gemspec
 - spec/es_spec_helper.rb
 - spec/fixtures/_nodes/2x_1x.json
@@ -185,6 +201,7 @@ files:
 - spec/fixtures/scripts/painless/scripted_upsert.painless
 - spec/fixtures/template-with-policy-es6x.json
 - spec/fixtures/template-with-policy-es7x.json
+- spec/fixtures/template-with-policy-es8x.json
 - spec/fixtures/test_certs/ca/ca.crt
 - spec/fixtures/test_certs/ca/ca.key
 - spec/fixtures/test_certs/test.crt
@@ -262,6 +279,7 @@ test_files:
 - spec/fixtures/scripts/painless/scripted_upsert.painless
 - spec/fixtures/template-with-policy-es6x.json
 - spec/fixtures/template-with-policy-es7x.json
+- spec/fixtures/template-with-policy-es8x.json
 - spec/fixtures/test_certs/ca/ca.crt
 - spec/fixtures/test_certs/ca/ca.key
 - spec/fixtures/test_certs/test.crt