logstash-output-elasticsearch 11.20.1-java → 11.22.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f3f97949dede5ce24e127df3a0cf4c809f1436d1ff34831cea6d0b873784968
-  data.tar.gz: cadc8d310da90f1cc853b24734520859101cd787efe29b5371394248303999f9
+  metadata.gz: 717e2454d150e9f3ae5c7ad9a7cd6d7e4e4451f538e276c1fa1e42a0b6c37f71
+  data.tar.gz: 714e7b7df621a80edd1a0e3eb0978b7a5b8a3789b2652e76c3336fcf8b7a6dfe
 SHA512:
-  metadata.gz: 9ef570b20433e6b3d69c24c9e402c39aa59a9f5556cb8c03f5643be65a7800b1a66b744be3095d11fe640a262902506fa78a7a10c3ea5b6ce1dfa81da98ea4e9
-  data.tar.gz: c323b23e45dfa795df06fdd206134ba340c5745529c690543bf94cfdb81526306c415723bb7a9cba759df658f38cbe6b6c62a4e6d1bbc6d777b224da830e8613
+  metadata.gz: 4d84c7f34a39b0566770881161f50c41f1270ab94131d6d70a35f4264a8e32631ac2da4e607dc61acc565b8e2ab87ca539949046233bce135ffc52c2caf6870e
+  data.tar.gz: b544f8e2e0f577970464e1bf72adda1e10254c361e3c749c0bfd84ca0184e59736d53dc5e6891f8e8369d2cc101a87af661aec90cc3038aaf2261893cf25eba2

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.22.0
+  - Added support for propagating event processing metadata when this output is downstream of an Elastic Integration Filter and configured _without_ explicit `version`, `version_type`, or `routing` directives [#1158](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1158)
+
+## 11.21.0
+  - Added support for propagating event processing metadata when this output is downstream of an Elastic Integration Filter and configured _without_ explicit `index`, `document_id`, or `pipeline` directives [#1155](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1155)
+
 ## 11.20.1
   - Doc: Replace `document_already_exist_exception` with `version_conflict_engine_exception` in the `silence_errors_in_log` setting example [#1159](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1159)
 

data/lib/logstash/outputs/elasticsearch.rb

@@ -499,9 +499,6 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
       params[retry_on_conflict_action_name] = @retry_on_conflict
     end
 
-    params[:version] = event.sprintf(@version) if @version
-    params[:version_type] = event.sprintf(@version_type) if @version_type
-
     EventActionTuple.new(action, params, event)
   end
 
@@ -540,15 +537,16 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # @return Hash (initial) parameters for given event
   # @private shared event params factory between index and data_stream mode
   def common_event_params(event)
-    sprintf_index = @event_target.call(event)
-    raise IndexInterpolationError, sprintf_index if sprintf_index.match(/%{.*?}/) && dlq_on_failed_indexname_interpolation
+    event_control = event.get("[@metadata][_ingest_document]")
+    event_id, event_pipeline, event_index, event_routing, event_version, event_version_type = event_control&.values_at("id","pipeline","index", "routing", "version", "version_type") rescue nil
+
     params = {
-        :_id => @document_id ? event.sprintf(@document_id) : nil,
-        :_index => sprintf_index,
-        routing_field_name => @routing ? event.sprintf(@routing) : nil
+        :_id => resolve_document_id(event, event_id),
+        :_index => resolve_index!(event, event_index),
+        routing_field_name => resolve_routing(event, event_routing)
     }
 
-    target_pipeline = resolve_pipeline(event)
+    target_pipeline = resolve_pipeline(event, event_pipeline)
     # convention: empty string equates to not using a pipeline
     # this is useful when using a field reference in the pipeline setting, e.g.
     #      elasticsearch {
@@ -556,10 +554,53 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     #      }
     params[:pipeline] = target_pipeline unless (target_pipeline.nil? || target_pipeline.empty?)
 
+    resolved_version = resolve_version(event, event_version)
+    resolved_version_type = resolve_version_type(event, event_version_type)
+    # avoid to add nil valued key-value pairs
+    params[:version] = resolved_version unless resolved_version.nil?
+    params[:version_type] = resolved_version_type unless resolved_version_type.nil?
+
     params
   end
 
-  def resolve_pipeline(event)
+  def resolve_version(event, event_version)
+    return event_version if event_version && !@version
+    event.sprintf(@version) if @version
+  end
+  private :resolve_version
+
+  def resolve_version_type(event, event_version_type)
+    return event_version_type if event_version_type && !@version_type
+    event.sprintf(@version_type) if @version_type
+  end
+  private :resolve_version_type
+
+  def resolve_routing(event, event_routing)
+    return event_routing if event_routing && !@routing
+    @routing ? event.sprintf(@routing) : nil
+  end
+  private :resolve_routing
+
+  def resolve_document_id(event, event_id)
+    return event.sprintf(@document_id) if @document_id
+    return event_id || nil
+  end
+  private :resolve_document_id
+
+  def resolve_index!(event, event_index)
+    sprintf_index = @event_target.call(event)
+    raise IndexInterpolationError, sprintf_index if sprintf_index.match(/%{.*?}/) && dlq_on_failed_indexname_interpolation
+    # if it's not a data stream, sprintf_index is the @index with resolved placeholders.
+    # if is a data stream, sprintf_index could be either the name of a data stream or the value contained in
+    # @index without placeholders substitution. If event's metadata index is provided, it takes precedence
+    # on datastream name or whatever is returned by the event_target provider.
+    return event_index if @index == @default_index && event_index
+    return sprintf_index
+  end
+  private :resolve_index!
+
+  def resolve_pipeline(event, event_pipeline)
+    return event_pipeline if event_pipeline && !@pipeline
     pipeline_template = @pipeline || event.get("[@metadata][target_ingest_pipeline]")&.to_s
     pipeline_template && event.sprintf(pipeline_template)
   end

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.20.1'
+  s.version         = '11.22.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -271,6 +271,260 @@ describe LogStash::Outputs::ElasticSearch do
       end
     end
 
+    describe "with event integration metadata" do
+      let(:event_fields) {{}}
+      let(:event) { LogStash::Event.new(event_fields)}
+
+      context "when plugin's version is specified" do
+        let(:options) { super().merge("version" => "123")}
+
+        context "when the event contains an integration metadata version" do
+          let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"version" => "456"}}}) }
+
+          it "plugin's version is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:version => "123")
+          end
+        end
+
+        context "when the event DOESN'T contains an integration metadata version" do
+          it "plugin's version is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:version => "123")
+          end
+        end
+      end
+
+      context "when plugin's version is NOT specified" do
+        context "when the event contains an integration metadata version" do
+          let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"version" => "456"}}}) }
+
+          it "event's metadata version is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:version => "456")
+          end
+        end
+
+        context "when the event DOESN'T contain an integration metadata version" do
+          it "plugin's default id mechanism is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to_not include(:version)
+          end
+        end
+      end
+
+      context "when plugin's version_type is specified" do
+        let(:options) { super().merge("version_type" => "internal")}
+
+        context "when the event contains an integration metadata version_type" do
+          let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"version_type" => "external"}}}) }
+
+          it "plugin's version_type is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:version_type => "internal")
+          end
+        end
+
+        context "when the event DOESN'T contains an integration metadata version_type" do
+          it "plugin's version_type is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:version_type => "internal")
+          end
+        end
+      end
+
+      context "when plugin's version_type is NOT specified" do
+        context "when the event contains an integration metadata version_type" do
+          let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"version_type" => "external"}}}) }
+
+          it "event's metadata version_type is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:version_type => "external")
+          end
+        end
+
+        context "when the event DOESN'T contain an integration metadata version_type" do
+          it "plugin's default id mechanism is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to_not include(:version_type)
+          end
+        end
+      end
+
+      context "when plugin's routing is specified" do
+        let(:options) { super().merge("routing" => "settings_routing")}
+
+        context "when the event contains an integration metadata routing" do
+          let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"routing" => "meta-document-routing"}}}) }
+
+          it "plugin's routing is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:routing => "settings_routing")
+          end
+        end
+
+        context "when the event DOESN'T contains an integration metadata routing" do
+          it "plugin's routing is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:routing => "settings_routing")
+          end
+        end
+      end
+
+      context "when plugin's routing is NOT specified" do
+        context "when the event contains an integration metadata routing" do
+          let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"routing" => "meta-document-routing"}}}) }
+
+          it "event's metadata routing is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:routing => "meta-document-routing")
+          end
+        end
+
+        context "when the event DOESN'T contain an integration metadata routing" do
+          it "plugin's default id mechanism is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:routing => nil)
+          end
+        end
+      end
+
+      context "when plugin's index is specified" do
+        let(:options) { super().merge("index" => "index_from_settings")}
+
+        context "when the event contains an integration metadata index" do
+          let(:event_fields) { super().merge({"@metadata" => {"_ingest_document" => {"index" => "meta-document-index"}}}) }
+
+          it "plugin's index is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:_index => "index_from_settings")
+          end
+        end
+
+        context "when the event doesn't contains an integration metadata index" do
+          it "plugin's index is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:_index => "index_from_settings")
+          end
+        end
+      end
+
+      context "when plugin's index is NOT specified" do
+        let(:options) { super().merge("index" => nil)}
+        
+        context "when the event contains an integration metadata index" do
+          let(:event_fields) { super().merge({"@metadata" => {"_ingest_document" => {"index" => "meta-document-index"}}}) }
+        
+          it "event's metadata index is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:_index => "meta-document-index")
+          end
+
+          context "when datastream settings are NOT configured" do
+            it "event's metadata index is used" do
+              expect(subject.send(:event_action_tuple, event)[1]).to include(:_index => "meta-document-index")
+            end
+          end
+
+          context "when datastream settings are configured" do
+            let(:event_fields) { super().merge({"data_stream" => {"type" => "logs", "dataset" => "generic", "namespace" => "default"}}) }
+
+            it "event's metadata index is used" do
+              expect(subject.send(:event_action_tuple, event)[1]).to include(:_index => "meta-document-index")
+            end
+          end
+        end
+
+        context "when the event DOESN'T contain integration metadata index" do
+          let(:default_index_resolved) { event.sprintf(subject.default_index) }
+
+          it "default index is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:_index => default_index_resolved)
+          end
+          
+          context "when datastream settings are NOT configured" do
+            it "default index is used" do
+              expect(subject.send(:event_action_tuple, event)[1]).to include(:_index => default_index_resolved)
+            end
+          end
+          
+          context "when datastream settings are configured" do
+            let(:event_fields) { super().merge({"data_stream" => {"type" => "logs", "dataset" => "generic", "namespace" => "default"}}) }
+          
+            it "default index is used" do
+              expect(subject.send(:event_action_tuple, event)[1]).to include(:_index => default_index_resolved)
+            end
+          end
+        end
+      end
+
+      context "when plugin's document_id is specified" do
+        let(:options) { super().merge("document_id" => "id_from_settings")}
+
+        context "when the event contains an integration metadata document_id" do
+          let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"id" => "meta-document-id"}}}) }
+
+          it "plugin's document_id is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:_id => "id_from_settings")
+          end
+        end
+
+        context "when the event DOESN'T contains an integration metadata document_id" do
+          it "plugin's document_id is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:_id => "id_from_settings")
+          end
+        end
+      end
+
+      context "when plugin's document_id is NOT specified" do
+        let(:options) { super().merge("document_id" => nil)}
+
+        context "when the event contains an integration metadata document_id" do
+          let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"id" => "meta-document-id"}}}) }
+
+          it "event's metadata document_id is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:_id => "meta-document-id")
+          end
+        end
+
+        context "when the event DOESN'T contains an integration metadata document_id" do
+          it "plugin's default id mechanism is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:_id => nil)
+          end
+        end
+      end
+
+      context "when plugin's pipeline is specified" do
+        let(:options) { {"pipeline" => "pipeline_from_settings" } }
+
+        context "when the event contains an integration metadata pipeline" do
+          let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"pipeline" => "integration-pipeline"}}}) }
+
+          it "plugin's pipeline is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "pipeline_from_settings")
+          end
+        end
+
+        context "when the event DOESN'T contains an integration metadata pipeline" do
+          it "plugin's pipeline is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "pipeline_from_settings")
+          end
+        end
+      end
+
+      context "when plugin's pipeline is NOT specified" do
+        let(:options) { super().merge("pipeline" => nil)}
+
+        context "when the event contains an integration metadata pipeline" do
+          let(:metadata) { {"_ingest_document" => {"pipeline" => "integration-pipeline"}} }
+          let(:event) { LogStash::Event.new({"@metadata" => metadata}) }
+
+          it "event's metadata pipeline is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "integration-pipeline")
+          end
+
+          context "when also target_ingest_pipeline id defined" do
+            let(:metadata) { super().merge({"target_ingest_pipeline" => "meta-ingest-pipeline"}) }
+
+            it "then event's pipeline from _ingest_document is used" do
+              expect(subject.send(:event_action_tuple, event)[1]).to include(:pipeline => "integration-pipeline")
+            end
+          end
+        end
+
+        context "when the event DOESN'T contains an integration metadata pipeline" do
+          it "plugin's default pipeline mechanism is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to_not have_key(:pipeline)
+          end
+        end
+      end
+    end
+
     describe "with auth" do
       let(:user) { "myuser" }
       let(:password) { ::LogStash::Util::Password.new("mypassword") }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.20.1
+  version: 11.22.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-09 00:00:00.000000000 Z
+date: 2023-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement