logstash-output-elasticsearch 11.12.3-java → 11.13.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a71925d993ccbf151c0cb88f41641e76926df0a2a398914dbe5d4ce54224c8f5
-  data.tar.gz: 97b17aad17959c0319ce94916570ce12f9ef67b7a22d583221ac26688e7cbe76
+  metadata.gz: 2e8764e50f36193ca77db2e6abe008e64732e52fd102554d6325d53180041ca6
+  data.tar.gz: 8e2525721fdf4a3e82c5983ea815125c407b91ab7e57e18d11256346c54d02c6
 SHA512:
-  metadata.gz: '0817002e4b468d47ea5540e8a9ed63efbee2506350da2f08a3171d0e26037a4e99d5bff613f6f6fca91ea441c88ae08602b211a656e10f13f5b9a91895838bab'
-  data.tar.gz: 60312f326a381d28b895c8f1f7f8fae611d7fdb1ccf4f1782da15949eb84c2fb93411e998f727ba91a922052cfe785bb9a3227aa798c7e1434eeca8369024471
+  metadata.gz: ef4c7059810d99d9aaa5d36a06a87eaf771ab1472056f8ed4bc7b05dfaf9280b6cc1a02a623bd1f5d25b7bb2f6458310450ebef91d8301757f8759512d1da625
+  data.tar.gz: e0970b2d8fc1413bcc7ab25bb02393f8be1ffe12fa0bf6c2349ad6ba3790ee844cfcf42136b0c10ea1b46e2c3bd2238da7a285869255076da28ba47ab6d84f9d

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 11.13.0
+ - add technology preview support for allowing events to individually encode a default pipeline with `[@metadata][target_ingest_pipeline]` (as part of a technology preview, this feature may change without notice) [#1113](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1113)
+
+## 11.12.4
+ - Changed the `manage_template` default value to `false` when data streams is enabled [#1111](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1111)
+   - Added the `manage_template => false` as a valid data stream option
+
 ## 11.12.3
  - Changed the log messages for data stream checks [#1109](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1109)
    - Added more details about incompatible data streams supplied configurations

data/docs/index.asciidoc

@@ -790,7 +790,7 @@ Set the keystore password
 ===== `manage_template`
 
   * Value type is <<boolean,boolean>>
-  * Default value is `true`
+  * Default value is `true` for non-time series data, and `false` for data streams.
 
 From Logstash 1.3 onwards, a template is applied to Elasticsearch during
 Logstash's startup if one with the name <<plugins-{type}s-{plugin}-template_name>>
@@ -849,12 +849,11 @@ not also set this field. That will raise an error at startup
 ===== `pipeline`
 
   * Value type is <<string,string>>
-  * Default value is `nil`
+  * There is no default value.
 
 Set which ingest pipeline you wish to execute for an event. You can also use
-event dependent configuration here like `pipeline =>
-"%{[@metadata][pipeline]}"`. The pipeline parameter won't be set if the value
-resolves to empty string ("").
+event dependent configuration here like `pipeline => "%{[@metadata][pipeline]}"`.
+The pipeline parameter won't be set if the value resolves to empty string ("").
 
 [id="plugins-{type}s-{plugin}-pool_max"]
 ===== `pool_max`

data/lib/logstash/outputs/elasticsearch/data_stream_support.rb

@@ -118,7 +118,6 @@ module LogStash module Outputs class ElasticSearch
       params.reject do |name, value|
         # NOTE: intentionally do not support explicit DS configuration like:
         # - `index => ...` identifier provided by data_stream_xxx settings
-        # - `manage_template => false` implied by not setting the parameter
         case name
         when 'action'
           value == 'create'
@@ -126,6 +125,8 @@ module LogStash module Outputs class ElasticSearch
           true
         when 'data_stream'
           value.to_s == 'true'
+        when 'manage_template'
+          value.to_s == 'false'
         when 'ecs_compatibility' then true # required for LS <= 6.x
         else
           name.start_with?('data_stream_') ||

data/lib/logstash/outputs/elasticsearch.rb

@@ -158,6 +158,8 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # control over template creation, (e.g. creating indices dynamically based on
   # field names) you should set `manage_template` to false and use the REST
   # API to apply your templates manually.
+  #
+  # Default value is `true` unless data streams is enabled
   config :manage_template, :validate => :boolean, :default => true
 
   # This configuration option defines how the template is named inside Elasticsearch.
@@ -304,6 +306,8 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     # It's being concurrently invoked by this register method and by the finish_register on the @after_successful_connection_thread
     data_stream_enabled = data_stream_config?
 
+    setup_template_manager_defaults(data_stream_enabled)
+
     @after_successful_connection_thread = after_successful_connection do
       begin
         finish_register
@@ -520,19 +524,22 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
         routing_field_name => @routing ? event.sprintf(@routing) : nil
     }
 
-    if @pipeline
-      value = event.sprintf(@pipeline)
-      # convention: empty string equates to not using a pipeline
-      # this is useful when using a field reference in the pipeline setting, e.g.
-      #      elasticsearch {
-      #        pipeline => "%{[@metadata][pipeline]}"
-      #      }
-      params[:pipeline] = value unless value.empty?
-    end
+    target_pipeline = resolve_pipeline(event)
+    # convention: empty string equates to not using a pipeline
+    # this is useful when using a field reference in the pipeline setting, e.g.
+    #      elasticsearch {
+    #        pipeline => "%{[@metadata][pipeline]}"
+    #      }
+    params[:pipeline] = target_pipeline unless (target_pipeline.nil? || target_pipeline.empty?)
 
     params
   end
 
+  def resolve_pipeline(event)
+    pipeline_template = @pipeline || event.get("[@metadata][target_ingest_pipeline]")&.to_s
+    pipeline_template && event.sprintf(pipeline_template)
+  end
+
   @@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }
 
   @@plugins.each do |plugin|
@@ -608,6 +615,13 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     @template_name ||= default_template_name
   end
 
+  def setup_template_manager_defaults(data_stream_enabled)
+    if original_params["manage_template"].nil? && data_stream_enabled
+      logger.debug("Disabling template management since data streams are enabled")
+      @manage_template = false
+    end
+  end
+
   # To be overidden by the -java version
   VALID_HTTP_ACTIONS = ["index", "delete", "create", "update"]
   def valid_actions

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.12.3'
+  s.version         = '11.13.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/unit/outputs/elasticsearch/data_stream_support_spec.rb

@@ -152,6 +152,39 @@ describe LogStash::Outputs::ElasticSearch::DataStreamSupport do
 
   end
 
+  context 'ds value-dependent configuration' do
+    context 'with valid values' do
+      let(:options) { super().merge(
+        'action' => 'create',
+        'routing' => 'any',
+        'pipeline' => 'any',
+        'manage_template' => 'false',
+        'data_stream' => 'true',
+        'data_stream_type' => 'logs',
+        'data_stream_dataset' => 'any',
+        'data_stream_namespace' => 'any',
+        'data_stream_sync_fields' => true,
+        'data_stream_auto_routing' => true)
+      }
+
+      it 'should enable data-streams by default' do
+        expect( subject.data_stream_config? ).to be_truthy
+      end
+    end
+
+    context 'with invalid values' do
+      let(:options) { super().merge(
+        'data_stream' => 'true',
+        'action' => 'index',
+        'manage_template' => 'true')
+      }
+
+      it 'should raise a configuration error' do
+        expect { subject.data_stream_config? }.to raise_error(LogStash::ConfigurationError, 'Invalid data stream configuration: ["action", "manage_template"]')
+      end
+    end
+  end
+
   context "default (non data-stream) configuration (on 7.x)" do
 
     let(:options) do

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -590,7 +590,7 @@ describe LogStash::Outputs::ElasticSearch do
 
       let(:event) { LogStash::Event.new("pipeline" => "my-ingest-pipeline") }
 
-      it "should interpolate the pipeline value and set it" do
+      it "interpolate the pipeline value and set it" do
         expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "my-ingest-pipeline")
       end
     end
@@ -600,10 +600,89 @@ describe LogStash::Outputs::ElasticSearch do
 
       let(:event) { LogStash::Event.new("pipeline" => "") }
 
-      it "should interpolate the pipeline value but not set it because it is empty" do
+      it "interpolates the pipeline value but not set it because it is empty" do
         expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
       end
     end
+
+    context "with both pipeline and target_ingest_pipeline" do
+      let(:options) { {"pipeline" => "%{pipeline}" } }
+      let(:event) { LogStash::Event.new({"pipeline" => "my-ingest-pipeline", "[@metadata][target_ingest_pipeline]" => "meta-ingest-pipeline"}) }
+
+      it "interpolates the plugin's pipeline value" do
+        expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "my-ingest-pipeline")
+      end
+
+      context "when the plugin's `pipeline` is constant" do
+        let(:options) { super().merge("pipeline" => "my-constant-pipeline") }
+         it "uses plugin's pipeline value" do
+          expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "my-constant-pipeline")
+        end
+      end
+
+      context "when the plugin's `pipeline` includes an unresolvable sprintf placeholder" do
+        let(:options) { super().merge("pipeline" => "reference-%{unset}-field") }
+        it "does not use the target_ingest_pipeline" do
+          # when sprintf doesn't resolve a placeholder, the behaviour of our `pipeline` is UNSPECIFIED.
+          # here we only validate that the presence of the magic field does not
+          # override an explicitly-configured pipeline.
+          expect(subject.send(:event_action_tuple, event)[1]).to_not include(:pipeline => "my-ingest-pipeline")
+        end
+      end
+    end
+
+    context "with empty pipeline and target_ingest_pipeline" do
+      let(:options) { {"pipeline" => "%{pipeline}" } }
+      let(:event) { LogStash::Event.new({"pipeline" => "", "[@metadata][target_ingest_pipeline]" => "meta-ingest-pipeline"}) }
+
+      it "interpolates the pipeline value but not set it because pipeline is empty" do
+        expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
+      end
+    end
+
+    context "with target_ingest_pipeline" do
+      let(:event) { LogStash::Event.new({"pipeline" => "", "@metadata" => {"target_ingest_pipeline" => "meta-ingest-pipeline"}}) }
+
+      it "interpolates the target_ingest_pipeline value and set it" do
+        expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "meta-ingest-pipeline")
+      end
+    end
+
+    context "with empty target_ingest_pipeline" do
+      let(:event) { LogStash::Event.new({"pipeline" => "", "@metadata" => {"host" => "elastic"}}) }
+
+      it "does not set pipeline" do
+        expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
+      end
+    end
+
+    context "with empty pipeline and empty target_ingest_pipeline" do
+      let(:event) { LogStash::Event.new }
+
+      it "does not set pipeline" do
+        expect(subject.send(:event_action_tuple, event)[1]).not_to include(:pipeline)
+      end
+    end
+  end
+
+  describe "the manage_template option" do
+    context "with data stream enabled" do
+      let(:options) { {"data_stream" => "true", "data_stream_type" => "logs" } }
+      let(:do_register) { true }
+
+      it "should default to false" do
+        expect(subject).to have_attributes(manage_template: false)
+      end
+    end
+
+    context "with data stream disabled" do
+      let(:options) { {"data_stream" => "false", "index" => "logs" } }
+      let(:do_register) { true }
+
+      it "should default to true" do
+        expect(subject).to have_attributes(manage_template: true)
+      end
+    end
   end
 
   describe "SSL end to end" do
@@ -870,7 +949,7 @@ describe LogStash::Outputs::ElasticSearch do
   end if LOGSTASH_VERSION > '6.0'
 
   context 'handling elasticsearch document-level status meant for the DLQ' do
-    let(:options) { { "manage_template" => false } }
+    let(:options) { { "manage_template" => false, "data_stream" => 'false' } }
     let(:action) { LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, LogStash::Event.new("foo" => "bar")) }
 
     context 'when @dlq_writer is nil' do
@@ -1140,7 +1219,7 @@ describe LogStash::Outputs::ElasticSearch do
   describe "post-register ES setup" do
     let(:do_register) { false }
     let(:es_version) { '7.10.0' } # DS default on LS 8.x
-    let(:options) { { 'hosts' => '127.0.0.1:9999' } }
+    let(:options) { { 'hosts' => '127.0.0.1:9999', 'data_stream' => 'false' } }
     let(:logger) { subject.logger }
 
     before do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.12.3
+  version: 11.13.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-24 00:00:00.000000000 Z
+date: 2023-02-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement