logstash-output-elasticsearch 11.21.0-java → 11.22.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 670f82b7bbfea1744a5daaf77a069c8872f87e1b15226131a786f826a931cd9c
-  data.tar.gz: 9aa78b93f19d995562192466ad7b816870706383da69b5050613079a76cd1e92
+  metadata.gz: 343fd4ddc3b4bef5b44a75dab05e92e88e745d4ed45c8bac1a1264c49ebea26c
+  data.tar.gz: 4db3cccc3c815d076ea5d060735934faedb903dd546fd812db9dfcb51b420815
 SHA512:
-  metadata.gz: 71d5c1b1558c4ba2c0bb29ab761980dfaad7005410a5bda436ac7f1d6a637b523aa058176e14eed491e8ffb99503f963c2eb5a9c3423e3d7ebd550f74902913e
-  data.tar.gz: aa32f8838b2526ae3fbcd3288ef6f848c90d926b0206416c5f2c348855e98494e36721ad9992cfbc238290a7c3e5baa90a518208c4b5cdc6187a80918f7b8716
+  metadata.gz: 106914085af2ec4e9eaab10d765eab7aca2f4967eb33bc76c11c87b6cbe77795a10966c3d314b612913c34fd91d9a7320f1c1f6990e2cc39f610ecd3dbefce0e
+  data.tar.gz: 507d0193320fdf5f5e48f3afc983f287bf69a8f122c3e1c05ee9854a203a90fc45ace2a337fd187b8744ac8693a4963e19da4fa11daafc1d0100d4346a29191f

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.22.1
+  - Fix, avoid to populate `version` and `version_type` attributes when processing integration metadata and datastream is enabled. [#1161](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1161)
+
+## 11.22.0
+  - Added support for propagating event processing metadata when this output is downstream of an Elastic Integration Filter and configured _without_ explicit `version`, `version_type`, or `routing` directives [#1158](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1158)
+
 ## 11.21.0
   - Added support for propagating event processing metadata when this output is downstream of an Elastic Integration Filter and configured _without_ explicit `index`, `document_id`, or `pipeline` directives [#1155](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1155)
 

data/lib/logstash/outputs/elasticsearch.rb

@@ -499,8 +499,15 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
       params[retry_on_conflict_action_name] = @retry_on_conflict
     end
 
-    params[:version] = event.sprintf(@version) if @version
-    params[:version_type] = event.sprintf(@version_type) if @version_type
+    event_control = event.get("[@metadata][_ingest_document]")
+    event_version, event_version_type = event_control&.values_at("version", "version_type") rescue nil
+
+    resolved_version = resolve_version(event, event_version)
+    resolved_version_type = resolve_version_type(event, event_version_type)
+
+    # avoid to add nil valued key-value pairs
+    params[:version] = resolved_version unless resolved_version.nil?
+    params[:version_type] = resolved_version_type unless resolved_version_type.nil?
 
     EventActionTuple.new(action, params, event)
   end
@@ -541,12 +548,12 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # @private shared event params factory between index and data_stream mode
   def common_event_params(event)
     event_control = event.get("[@metadata][_ingest_document]")
-    event_id, event_pipeline, event_index = event_control&.values_at("id","pipeline","index") rescue nil
+    event_id, event_pipeline, event_index, event_routing = event_control&.values_at("id","pipeline","index", "routing") rescue nil
 
     params = {
         :_id => resolve_document_id(event, event_id),
         :_index => resolve_index!(event, event_index),
-        routing_field_name => @routing ? event.sprintf(@routing) : nil
+        routing_field_name => resolve_routing(event, event_routing)
     }
 
     target_pipeline = resolve_pipeline(event, event_pipeline)
@@ -560,6 +567,24 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     params
   end
 
+  def resolve_version(event, event_version)
+    return event_version if event_version && !@version
+    event.sprintf(@version) if @version
+  end
+  private :resolve_version
+
+  def resolve_version_type(event, event_version_type)
+    return event_version_type if event_version_type && !@version_type
+    event.sprintf(@version_type) if @version_type
+  end
+  private :resolve_version_type
+
+  def resolve_routing(event, event_routing)
+    return event_routing if event_routing && !@routing
+    @routing ? event.sprintf(@routing) : nil
+  end
+  private :resolve_routing
+
   def resolve_document_id(event, event_id)
     return event.sprintf(@document_id) if @document_id
     return event_id || nil

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.21.0'
+  s.version         = '11.22.1'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -275,6 +275,130 @@ describe LogStash::Outputs::ElasticSearch do
       let(:event_fields) {{}}
       let(:event) { LogStash::Event.new(event_fields)}
 
+      context "when plugin's version is specified" do
+        let(:options) { super().merge("version" => "123")}
+
+        context "when the event contains an integration metadata version" do
+          let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"version" => "456"}}}) }
+
+          it "plugin's version is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:version => "123")
+          end
+        end
+
+        context "when the event DOESN'T contains an integration metadata version" do
+          it "plugin's version is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:version => "123")
+          end
+        end
+      end
+
+      context "when plugin's version is NOT specified" do
+        context "when the event contains an integration metadata version" do
+          let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"version" => "456"}}}) }
+
+          context "when datastream settings are NOT configured" do
+            it "event's metadata version is used" do
+              expect(subject.send(:event_action_tuple, event)[1]).to include(:version => "456")
+            end
+          end
+
+          context "when datastream settings are configured" do
+            # NOTE: we validate with datastream-specific `data_stream_event_action_tuple`
+            let(:event_fields) { super().merge({"data_stream" => {"type" => "logs", "dataset" => "generic", "namespace" => "default"}}) }
+
+            it "no version is used" do
+              expect(subject.send(:data_stream_event_action_tuple, event)[1]).to_not include(:version)
+            end
+          end
+        end
+
+        context "when the event DOESN'T contain an integration metadata version" do
+          it "plugin's default id mechanism is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to_not include(:version)
+          end
+        end
+      end
+
+      context "when plugin's version_type is specified" do
+        let(:options) { super().merge("version_type" => "internal")}
+
+        context "when the event contains an integration metadata version_type" do
+          let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"version_type" => "external"}}}) }
+
+          context "when datastream settings are NOT configured" do
+            it "plugin's version_type is used" do
+              expect(subject.send(:event_action_tuple, event)[1]).to include(:version_type => "internal")
+            end
+          end
+
+          context "when datastream settings are configured" do
+            # NOTE: we validate with datastream-specific `data_stream_event_action_tuple`
+            let(:event_fields) { super().merge({"data_stream" => {"type" => "logs", "dataset" => "generic", "namespace" => "default"}}) }
+
+            it "no version_type is used" do
+              expect(subject.send(:data_stream_event_action_tuple, event)[1]).to_not include(:version_type)
+            end
+          end
+        end
+
+        context "when the event DOESN'T contains an integration metadata version_type" do
+          it "plugin's version_type is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:version_type => "internal")
+          end
+        end
+      end
+
+      context "when plugin's version_type is NOT specified" do
+        context "when the event contains an integration metadata version_type" do
+          let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"version_type" => "external"}}}) }
+
+          it "event's metadata version_type is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:version_type => "external")
+          end
+        end
+
+        context "when the event DOESN'T contain an integration metadata version_type" do
+          it "plugin's default id mechanism is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to_not include(:version_type)
+          end
+        end
+      end
+
+      context "when plugin's routing is specified" do
+        let(:options) { super().merge("routing" => "settings_routing")}
+
+        context "when the event contains an integration metadata routing" do
+          let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"routing" => "meta-document-routing"}}}) }
+
+          it "plugin's routing is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:routing => "settings_routing")
+          end
+        end
+
+        context "when the event DOESN'T contains an integration metadata routing" do
+          it "plugin's routing is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:routing => "settings_routing")
+          end
+        end
+      end
+
+      context "when plugin's routing is NOT specified" do
+        context "when the event contains an integration metadata routing" do
+          let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"routing" => "meta-document-routing"}}}) }
+
+          it "event's metadata routing is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:routing => "meta-document-routing")
+          end
+        end
+
+        context "when the event DOESN'T contain an integration metadata routing" do
+          it "plugin's default id mechanism is used" do
+            expect(subject.send(:event_action_tuple, event)[1]).to include(:routing => nil)
+          end
+        end
+      end
+
       context "when plugin's index is specified" do
         let(:options) { super().merge("index" => "index_from_settings")}
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.21.0
+  version: 11.22.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-10 00:00:00.000000000 Z
+date: 2023-11-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement