logstash-output-elasticsearch 11.22.0-java → 11.22.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 717e2454d150e9f3ae5c7ad9a7cd6d7e4e4451f538e276c1fa1e42a0b6c37f71
-  data.tar.gz: 714e7b7df621a80edd1a0e3eb0978b7a5b8a3789b2652e76c3336fcf8b7a6dfe
+  metadata.gz: 011acede8b368a5fcf578665eacc4393b1d0bc531fc2f5c47814345826534d6f
+  data.tar.gz: 00b17bcff6d2100a03e801da6e4dff1c97d58adee514764c0c6c7a11588d2bd8
 SHA512:
-  metadata.gz: 4d84c7f34a39b0566770881161f50c41f1270ab94131d6d70a35f4264a8e32631ac2da4e607dc61acc565b8e2ab87ca539949046233bce135ffc52c2caf6870e
-  data.tar.gz: b544f8e2e0f577970464e1bf72adda1e10254c361e3c749c0bfd84ca0184e59736d53dc5e6891f8e8369d2cc101a87af661aec90cc3038aaf2261893cf25eba2
+  metadata.gz: c01b55a7daa6609256624e44f2f05f5528adc114a928c150dae3a06f6596d8cc0473252b207b60bd0dbaaeb801e4f44bc1b8338900030d8acfa981ee4cc1807a
+  data.tar.gz: b422aca67422e08a8627f7b276f32d88ca0320823248bf89b6124eb2f205866718cbdc7f208b36ed595f6810737813c9ce291343885621622bd7cc7dbf427842

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.22.2
+  - [DOC] Add content for sending data to Elasticsearch on serverless [#1164](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1164)
+
+## 11.22.1
+  - Fix, avoid to populate `version` and `version_type` attributes when processing integration metadata and datastream is enabled. [#1161](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1161)
+
 ## 11.22.0
   - Added support for propagating event processing metadata when this output is downstream of an Elastic Integration Filter and configured _without_ explicit `version`, `version_type`, or `routing` directives [#1158](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1158)
 

data/docs/index.asciidoc

@@ -30,7 +30,7 @@ the website landing page or in the {ref}[Elasticsearch documentation].
 
 .Compatibility Note
 [NOTE]
-================================================================================
+=====
 When connected to Elasticsearch 7.x, modern versions of this plugin
 don't use the document-type when inserting documents, unless the user
 explicitly sets <<plugins-{type}s-{plugin}-document_type>>.
@@ -42,9 +42,30 @@ picks up changes to the Elasticsearch index template.
 If you are using a custom <<plugins-{type}s-{plugin}-template>>,
 ensure your template uses the `_doc` document-type before
 connecting to Elasticsearch 7.x.
-================================================================================
+=====
 
-===== Hosted {es} Service on Elastic Cloud
+[id="plugins-{type}s-{plugin}-serverless"]
+==== {ls} to {es-serverless}
+
+You can use this plugin to send your {ls} data to {es-serverless}.
+Some differences to note between {es-serverless} and self-managed {es}:
+
+* Use *API keys* to access {serverless-full} from {ls}. 
+Any user-based security settings in your in your <<plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors.
+* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). 
+Any {ilm-init} settings in your <<plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors.
+* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}.
+
+.Known issue for {ls} to {es-serverless}
+****
+The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. 
+Set the value to port :443 instead.
+****
+
+For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs].
+
+[id="plugins-{type}s-{plugin}-ess"]
+==== Hosted {es} Service on Elastic Cloud
 
 {ess-leadin}
 
@@ -68,8 +89,8 @@ and will correctly reject events with fields that conflict and cannot be coerced
 The {es} output plugin can store both time series datasets (such
 as logs, events, and metrics) and non-time series data in Elasticsearch.
 
-The data stream options are recommended for indexing time series datasets (such
-as logs, metrics, and events) into {es}:
+Use the data stream options for indexing time series datasets (such
+as logs, metrics, and events) into {es} and {es-serverless}:
 
 * <<plugins-{type}s-{plugin}-data_stream>>
 * <<plugins-{type}s-{plugin}-data_stream_auto_routing>> 
@@ -200,27 +221,28 @@ The list of error codes accepted for DLQ could be customized with <<plugins-{typ
 but should be used only in motivated cases.
 
 [id="plugins-{type}s-{plugin}-ilm"]
-==== Index Lifecycle Management
+==== {ilm-cap} ({ilm-init})
 
 [NOTE]
-The Index Lifecycle Management feature requires plugin version `9.3.1` or higher.
+--
+* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. 
+Any {ilm-init} settings in your plugin configuration are ignored and may cause errors.
+* The {ilm-init} feature requires plugin version `9.3.1` or higher.
+* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license
+--
 
-[NOTE]
-This feature requires an Elasticsearch instance of 6.6.0 or higher with at least a Basic license
+{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time.
 
-Logstash can use {ref}/index-lifecycle-management.html[Index Lifecycle
-Management] to automate the management of indices over time.
-
-The use of Index Lifecycle Management is controlled by the `ilm_enabled`
+The use of {ilm} is controlled by the `ilm_enabled`
 setting. By default, this setting detects whether the Elasticsearch instance
-supports ILM, and uses it if it is available. `ilm_enabled` can also be set to
-`true` or `false` to override the automatic detection, or disable ILM.
+supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to
+`true` or `false` to override the automatic detection, or disable {ilm-init}.
 
-This will overwrite the index settings and adjust the Logstash template to write
-the necessary settings for the template to support index lifecycle management,
+This will overwrite the index settings and adjust the {ls} template to write
+the necessary settings for the template to support {ilm},
 including the index policy and rollover alias to be used.
 
-Logstash will create a rollover alias for the indices to be written to,
+{ls} creates a rollover alias for the indices to be written to,
 including a pattern for how the actual indices will be named, and unless an ILM
 policy that already exists has been specified, a default policy will also be
 created. The default policy is configured to rollover an index when it reaches
@@ -246,14 +268,14 @@ See config below for an example:
       }
     }
 
-NOTE: Custom ILM policies must already exist on the Elasticsearch cluster before they can be used.
-
-NOTE: If the rollover alias or pattern is modified, the index template will need to be
+[NOTE]
+--
+* Custom ILM policies must already exist on the {es} cluster before they can be used.
+* If the rollover alias or pattern is modified, the index template will need to be
 overwritten as the settings `index.lifecycle.name` and
 `index.lifecycle.rollover_alias` are automatically written to the template
-
-NOTE: If the index property is supplied in the output definition, it will be overwritten by the rollover alias.
-
+* If the index property is supplied in the output definition, it will be overwritten by the rollover alias.
+--
 
 ==== Batch Sizes
 

data/lib/logstash/outputs/elasticsearch.rb

@@ -499,6 +499,16 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
       params[retry_on_conflict_action_name] = @retry_on_conflict
     end
 
+    event_control = event.get("[@metadata][_ingest_document]")
+    event_version, event_version_type = event_control&.values_at("version", "version_type") rescue nil
+
+    resolved_version = resolve_version(event, event_version)
+    resolved_version_type = resolve_version_type(event, event_version_type)
+
+    # avoid to add nil valued key-value pairs
+    params[:version] = resolved_version unless resolved_version.nil?
+    params[:version_type] = resolved_version_type unless resolved_version_type.nil?
+
     EventActionTuple.new(action, params, event)
   end
 
@@ -538,7 +548,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # @private shared event params factory between index and data_stream mode
   def common_event_params(event)
     event_control = event.get("[@metadata][_ingest_document]")
-    event_id, event_pipeline, event_index, event_routing, event_version, event_version_type = event_control&.values_at("id","pipeline","index", "routing", "version", "version_type") rescue nil
+    event_id, event_pipeline, event_index, event_routing = event_control&.values_at("id","pipeline","index", "routing") rescue nil
 
     params = {
         :_id => resolve_document_id(event, event_id),
@@ -554,12 +564,6 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     #      }
     params[:pipeline] = target_pipeline unless (target_pipeline.nil? || target_pipeline.empty?)
 
-    resolved_version = resolve_version(event, event_version)
-    resolved_version_type = resolve_version_type(event, event_version_type)
-    # avoid to add nil valued key-value pairs
-    params[:version] = resolved_version unless resolved_version.nil?
-    params[:version_type] = resolved_version_type unless resolved_version_type.nil?
-
     params
   end
 

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.22.0'
+  s.version         = '11.22.2'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -297,8 +297,19 @@ describe LogStash::Outputs::ElasticSearch do
         context "when the event contains an integration metadata version" do
           let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"version" => "456"}}}) }
 
-          it "event's metadata version is used" do
-            expect(subject.send(:event_action_tuple, event)[1]).to include(:version => "456")
+          context "when datastream settings are NOT configured" do
+            it "event's metadata version is used" do
+              expect(subject.send(:event_action_tuple, event)[1]).to include(:version => "456")
+            end
+          end
+
+          context "when datastream settings are configured" do
+            # NOTE: we validate with datastream-specific `data_stream_event_action_tuple`
+            let(:event_fields) { super().merge({"data_stream" => {"type" => "logs", "dataset" => "generic", "namespace" => "default"}}) }
+
+            it "no version is used" do
+              expect(subject.send(:data_stream_event_action_tuple, event)[1]).to_not include(:version)
+            end
           end
         end
 
@@ -315,8 +326,19 @@ describe LogStash::Outputs::ElasticSearch do
         context "when the event contains an integration metadata version_type" do
           let(:event) { LogStash::Event.new({"@metadata" => {"_ingest_document" => {"version_type" => "external"}}}) }
 
-          it "plugin's version_type is used" do
-            expect(subject.send(:event_action_tuple, event)[1]).to include(:version_type => "internal")
+          context "when datastream settings are NOT configured" do
+            it "plugin's version_type is used" do
+              expect(subject.send(:event_action_tuple, event)[1]).to include(:version_type => "internal")
+            end
+          end
+
+          context "when datastream settings are configured" do
+            # NOTE: we validate with datastream-specific `data_stream_event_action_tuple`
+            let(:event_fields) { super().merge({"data_stream" => {"type" => "logs", "dataset" => "generic", "namespace" => "default"}}) }
+
+            it "no version_type is used" do
+              expect(subject.send(:data_stream_event_action_tuple, event)[1]).to_not include(:version_type)
+            end
           end
         end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.22.0
+  version: 11.22.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-16 00:00:00.000000000 Z
+date: 2023-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement