logstash-output-elasticsearch 11.12.4-java → 11.13.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a912267d383b51082d4c1b68e21d83dcb506433abc4f3f12bce0465eef84f439
-  data.tar.gz: dc69623c94ad47986ed07f25818eeabfbfdd49532dfd9970f8c638dfe950f615
+  metadata.gz: d83bbddeedf7f5674416d431b0a54d7d939a7fd4a21853847f96c2ecf44659c8
+  data.tar.gz: 5d92c4dfd6e5843c7298021b98dcc86dcdfa3f5e8f474255c0358723a673d9cb
 SHA512:
-  metadata.gz: 99568149dc8d313460046ca2e247c43e2e505e6649441a22a365e0d5484ba493d5d91c4eedfb5700fc447f64122555db3b92d6b59513a6b42e07dcc43dda3ada
-  data.tar.gz: 56d2ce9c942f2dffbfc850f8feec0368dc2456ee08f7d7f46b83de94f9439fdd9047a1bae9231cc41caf45105a1abda0d1e60429a37059709f337e5ea8f8b6e4
+  metadata.gz: 47194a7711b93f6a1dcca191dc37a7c0bb14ddcf5591940520ad3e24e046df67850ec0fbf1c9ed769ff6706c7bd9b087bd44cd2d7b97be444bd7d8c5c48ad295
+  data.tar.gz: b7344d10ba9a8a09a5acb65348a4e7152407a3d98126843a8266112b67bc8e6a4859e7ab60e4a7d1bd448cde8d00d65c2e5dda9572eec175fc5cc2baab17441d

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.13.1
+ - Avoid crash by ensuring ILM settings are injected in the correct location depending on the default (or custom) template format, template_api setting and ES version [#1102](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1102)
+
+## 11.13.0
+ - add technology preview support for allowing events to individually encode a default pipeline with `[@metadata][target_ingest_pipeline]` (as part of a technology preview, this feature may change without notice) [#1113](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1113)
+
 ## 11.12.4
  - Changed the `manage_template` default value to `false` when data streams is enabled [#1111](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1111)
    - Added the `manage_template => false` as a valid data stream option

data/docs/index.asciidoc

@@ -849,12 +849,11 @@ not also set this field. That will raise an error at startup
 ===== `pipeline`
 
   * Value type is <<string,string>>
-  * Default value is `nil`
+  * There is no default value.
 
 Set which ingest pipeline you wish to execute for an event. You can also use
-event dependent configuration here like `pipeline =>
-"%{[@metadata][pipeline]}"`. The pipeline parameter won't be set if the value
-resolves to empty string ("").
+event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`.
+The pipeline parameter won't be set if the value resolves to empty string ("").
 
 [id="plugins-{type}s-{plugin}-pool_max"]
 ===== `pool_max`

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -46,15 +46,38 @@ module LogStash; module Outputs; class ElasticSearch
       # definition - remove any existing definition of 'template'
       template.delete('template') if template.include?('template') if plugin.maximum_seen_major_version < 8
       template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*"
-      settings = template_settings(plugin, template)
+      settings = resolve_template_settings(plugin, template)
       if settings && (settings['index.lifecycle.name'] || settings['index.lifecycle.rollover_alias'])
         plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled")
       end
       settings.update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias})
     end
 
-    def self.template_settings(plugin, template)
-      plugin.maximum_seen_major_version < 8 ? template['settings']: template['template']['settings']
+    def self.resolve_template_settings(plugin, template)
+      if template.key?('template')
+        plugin.logger.trace("Resolving ILM template settings: under 'template' key", :template => template, :template_api => plugin.template_api, :es_version => plugin.maximum_seen_major_version)
+        composable_index_template_settings(template)
+      elsif template.key?('settings')
+        plugin.logger.trace("Resolving ILM template settings: under 'settings' key", :template => template, :template_api => plugin.template_api, :es_version => plugin.maximum_seen_major_version)
+        legacy_index_template_settings(template)
+      else
+        template_endpoint = template_endpoint(plugin)
+        plugin.logger.trace("Resolving ILM template settings: template doesn't have 'settings' or 'template' fields, falling back to auto detection", :template => template, :template_api => plugin.template_api, :es_version => plugin.maximum_seen_major_version, :template_endpoint => template_endpoint)
+        template_endpoint == INDEX_TEMPLATE_ENDPOINT ?
+          composable_index_template_settings(template) :
+          legacy_index_template_settings(template)
+      end
+    end
+
+    # Sets ['settings'] field to be compatible with _template API structure
+    def self.legacy_index_template_settings(template)
+      template['settings'] ||= {}
+    end
+
+    # Sets the ['template']['settings'] fields if not exist to be compatible with _index_template API structure
+    def self.composable_index_template_settings(template)
+      template['template'] ||= {}
+      template['template']['settings'] ||= {}
     end
 
     # Template name - if template_name set, use it

data/lib/logstash/outputs/elasticsearch.rb

@@ -524,19 +524,22 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
         routing_field_name => @routing ? event.sprintf(@routing) : nil
     }
 
-    if @pipeline
-      value = event.sprintf(@pipeline)
-      # convention: empty string equates to not using a pipeline
-      # this is useful when using a field reference in the pipeline setting, e.g.
-      #      elasticsearch {
-      #        pipeline => "%{[@metadata][pipeline]}"
-      #      }
-      params[:pipeline] = value unless value.empty?
-    end
+    target_pipeline = resolve_pipeline(event)
+    # convention: empty string equates to not using a pipeline
+    # this is useful when using a field reference in the pipeline setting, e.g.
+    #      elasticsearch {
+    #        pipeline => "%{[@metadata][pipeline]}"
+    #      }
+    params[:pipeline] = target_pipeline unless (target_pipeline.nil? || target_pipeline.empty?)
 
     params
   end
 
+  def resolve_pipeline(event)
+    pipeline_template = @pipeline || event.get("[@metadata][target_ingest_pipeline]")&.to_s
+    pipeline_template && event.sprintf(pipeline_template)
+  end
+
   @@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }
 
   @@plugins.each do |plugin|

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.12.4'
+  s.version         = '11.13.1'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -1,4 +1,4 @@
-require "logstash/devutils/rspec/spec_helper"
+require_relative "../../../../spec/spec_helper"
 require "logstash/outputs/elasticsearch/template_manager"
 
 describe LogStash::Outputs::ElasticSearch::TemplateManager do
@@ -33,33 +33,85 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
     end
   end
 
-  describe "index template with ilm settings" do
+  context "index template with ilm settings" do
     let(:plugin_settings) { {"manage_template" => true, "template_overwrite" => true} }
     let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
 
-    describe "in version 8+" do
-      let(:file_path) { described_class.default_template_path(8) }
-      let(:template) { described_class.read_template_file(file_path)}
+    describe "with custom template" do
 
-      it "should update settings" do
-        expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
-        described_class.add_ilm_settings_to_template(plugin, template)
-        expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil)
-        expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
-        expect(template.include?('settings')).to be_falsey
+      describe "in version 8+" do
+        let(:file_path) { described_class.default_template_path(8) }
+        let(:template) { described_class.read_template_file(file_path)}
+
+        it "should update settings" do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
+          described_class.add_ilm_settings_to_template(plugin, template)
+          expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil)
+          expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
+          expect(template.include?('settings')).to be_falsey
+        end
+      end
+
+      describe "in version < 8" do
+        let(:file_path) { described_class.default_template_path(7) }
+        let(:template) { described_class.read_template_file(file_path)}
+
+        it "should update settings" do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
+          described_class.add_ilm_settings_to_template(plugin, template)
+          expect(template['settings']['index.lifecycle.name']).not_to eq(nil)
+          expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
+          expect(template.include?('template')).to be_falsey
+        end
       end
     end
 
-    describe "in version < 8" do
-      let(:file_path) { described_class.default_template_path(7) }
-      let(:template) { described_class.read_template_file(file_path)}
+    context "resolve template setting" do
+      let(:plugin_settings) { super().merge({"template_api" => template_api}) }
+
+      describe "with composable template API" do
+        let(:template_api) { "composable" }
 
-      it "should update settings" do
-        expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
-        described_class.add_ilm_settings_to_template(plugin, template)
-        expect(template['settings']['index.lifecycle.name']).not_to eq(nil)
-        expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
-        expect(template.include?('template')).to be_falsey
+        it 'resolves composable index template API compatible setting' do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8) # required to log
+          template = {}
+          described_class.resolve_template_settings(plugin, template)
+          expect(template["template"]["settings"]).not_to eq(nil)
+        end
+      end
+
+      describe "with legacy template API" do
+        let(:template_api) { "legacy" }
+
+        it 'resolves legacy index template API compatible setting' do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7) # required to log
+          template = {}
+          described_class.resolve_template_settings(plugin, template)
+          expect(template["settings"]).not_to eq(nil)
+        end
+      end
+
+      describe "with `template_api => 'auto'`" do
+        let(:template_api) { "auto" }
+
+        describe "with ES < 8 versions" do
+
+          it 'resolves legacy index template API compatible setting' do
+            expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
+            template = {}
+            described_class.resolve_template_settings(plugin, template)
+            expect(template["settings"]).not_to eq(nil)
+          end
+        end
+
+        describe "with ES >= 8 versions" do
+          it 'resolves composable index template API compatible setting' do
+            expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
+            template = {}
+            described_class.resolve_template_settings(plugin, template)
+            expect(template["template"]["settings"]).not_to eq(nil)
+          end
+        end
       end
     end
   end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -590,7 +590,7 @@ describe LogStash::Outputs::ElasticSearch do
 
       let(:event) { LogStash::Event.new("pipeline" => "my-ingest-pipeline") }
 
-      it "should interpolate the pipeline value and set it" do
+      it "interpolate the pipeline value and set it" do
         expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "my-ingest-pipeline")
       end
     end
@@ -600,7 +600,66 @@ describe LogStash::Outputs::ElasticSearch do
 
       let(:event) { LogStash::Event.new("pipeline" => "") }
 
-      it "should interpolate the pipeline value but not set it because it is empty" do
+      it "interpolates the pipeline value but not set it because it is empty" do
+        expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
+      end
+    end
+
+    context "with both pipeline and target_ingest_pipeline" do
+      let(:options) { {"pipeline" => "%{pipeline}" } }
+      let(:event) { LogStash::Event.new({"pipeline" => "my-ingest-pipeline", "[@metadata][target_ingest_pipeline]" => "meta-ingest-pipeline"}) }
+
+      it "interpolates the plugin's pipeline value" do
+        expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "my-ingest-pipeline")
+      end
+
+      context "when the plugin's `pipeline` is constant" do
+        let(:options) { super().merge("pipeline" => "my-constant-pipeline") }
+         it "uses plugin's pipeline value" do
+          expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "my-constant-pipeline")
+        end
+      end
+
+      context "when the plugin's `pipeline` includes an unresolvable sprintf placeholder" do
+        let(:options) { super().merge("pipeline" => "reference-%{unset}-field") }
+        it "does not use the target_ingest_pipeline" do
+          # when sprintf doesn't resolve a placeholder, the behaviour of our `pipeline` is UNSPECIFIED.
+          # here we only validate that the presence of the magic field does not
+          # override an explicitly-configured pipeline.
+          expect(subject.send(:event_action_tuple, event)[1]).to_not include(:pipeline => "my-ingest-pipeline")
+        end
+      end
+    end
+
+    context "with empty pipeline and target_ingest_pipeline" do
+      let(:options) { {"pipeline" => "%{pipeline}" } }
+      let(:event) { LogStash::Event.new({"pipeline" => "", "[@metadata][target_ingest_pipeline]" => "meta-ingest-pipeline"}) }
+
+      it "interpolates the pipeline value but not set it because pipeline is empty" do
+        expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
+      end
+    end
+
+    context "with target_ingest_pipeline" do
+      let(:event) { LogStash::Event.new({"pipeline" => "", "@metadata" => {"target_ingest_pipeline" => "meta-ingest-pipeline"}}) }
+
+      it "interpolates the target_ingest_pipeline value and set it" do
+        expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "meta-ingest-pipeline")
+      end
+    end
+
+    context "with empty target_ingest_pipeline" do
+      let(:event) { LogStash::Event.new({"pipeline" => "", "@metadata" => {"host" => "elastic"}}) }
+
+      it "does not set pipeline" do
+        expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
+      end
+    end
+
+    context "with empty pipeline and empty target_ingest_pipeline" do
+      let(:event) { LogStash::Event.new }
+
+      it "does not set pipeline" do
         expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.12.4
+  version: 11.13.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-01 00:00:00.000000000 Z
+date: 2023-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement