logstash-output-elasticsearch 11.22.1-java → 11.22.3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 343fd4ddc3b4bef5b44a75dab05e92e88e745d4ed45c8bac1a1264c49ebea26c
-  data.tar.gz: 4db3cccc3c815d076ea5d060735934faedb903dd546fd812db9dfcb51b420815
+  metadata.gz: f11cbc421fcbb44c5d993d70f97a8a32c79a278b5770805d5d82c3f63bdde681
+  data.tar.gz: 38ea003c71ed10ac5bffe531a16404c27670c67a6f06fbd637ed5d1544970e59
 SHA512:
-  metadata.gz: 106914085af2ec4e9eaab10d765eab7aca2f4967eb33bc76c11c87b6cbe77795a10966c3d314b612913c34fd91d9a7320f1c1f6990e2cc39f610ecd3dbefce0e
-  data.tar.gz: 507d0193320fdf5f5e48f3afc983f287bf69a8f122c3e1c05ee9854a203a90fc45ace2a337fd187b8744ac8693a4963e19da4fa11daafc1d0100d4346a29191f
+  metadata.gz: c8efac5e9e84066e1dbe7272b9bb04276d10b34983d8f8dc49824663c67c55be1e5a1bad3c5f7f13bdb2512e995ca1f0dab13f890a596f1ca73ff49a99414fc3
+  data.tar.gz: 442f1690d2d96f87536a95d1827bba4e8029891209cf27fe5f380b126100288e9c3ca08dcfab6e0ce256ed24ddc74f91f318d81f0885b32fb8f9a45c1a388735

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.22.3
+  - Fixes an issue where events containing non-unicode strings could fail to serialize correctly when compression is enabled [#1169](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1169)
+
+## 11.22.2
+  - [DOC] Add content for sending data to Elasticsearch on serverless [#1164](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1164)
+
 ## 11.22.1
   - Fix, avoid to populate `version` and `version_type` attributes when processing integration metadata and datastream is enabled. [#1161](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1161)
 

data/docs/index.asciidoc

@@ -30,7 +30,7 @@ the website landing page or in the {ref}[Elasticsearch documentation].
 
 .Compatibility Note
 [NOTE]
-================================================================================
+=====
 When connected to Elasticsearch 7.x, modern versions of this plugin
 don't use the document-type when inserting documents, unless the user
 explicitly sets <<plugins-{type}s-{plugin}-document_type>>.
@@ -42,9 +42,30 @@ picks up changes to the Elasticsearch index template.
 If you are using a custom <<plugins-{type}s-{plugin}-template>>,
 ensure your template uses the `_doc` document-type before
 connecting to Elasticsearch 7.x.
-================================================================================
+=====
 
-===== Hosted {es} Service on Elastic Cloud
+[id="plugins-{type}s-{plugin}-serverless"]
+==== {ls} to {es-serverless}
+
+You can use this plugin to send your {ls} data to {es-serverless}.
+Some differences to note between {es-serverless} and self-managed {es}:
+
+* Use *API keys* to access {serverless-full} from {ls}. 
+Any user-based security settings in your <<plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors.
+* {es-serverless} uses *data streams* and {ref}/data-stream-lifecycle.html[{dlm} ({dlm-init})] instead of {ilm} ({ilm-init}). 
+Any {ilm-init} settings in your <<plugins-outputs-elasticsearch,{es} output plugin>> configuration are ignored and may cause errors.
+* *{ls} monitoring* is available through the https://github.com/elastic/integrations/blob/main/packages/logstash/_dev/build/docs/README.md[{ls} Integration] in {serverless-docs}/observability/what-is-observability-serverless[Elastic Observability] on {serverless-full}.
+
+.Known issue for {ls} to {es-serverless}
+****
+The logstash-output-elasticsearch `hosts` setting on {serverless-short} defaults the port to 9200 when omitted. 
+Set the value to port :443 instead.
+****
+
+For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs].
+
+[id="plugins-{type}s-{plugin}-ess"]
+==== Hosted {es} Service on Elastic Cloud
 
 {ess-leadin}
 
@@ -68,8 +89,8 @@ and will correctly reject events with fields that conflict and cannot be coerced
 The {es} output plugin can store both time series datasets (such
 as logs, events, and metrics) and non-time series data in Elasticsearch.
 
-The data stream options are recommended for indexing time series datasets (such
-as logs, metrics, and events) into {es}:
+Use the data stream options for indexing time series datasets (such
+as logs, metrics, and events) into {es} and {es-serverless}:
 
 * <<plugins-{type}s-{plugin}-data_stream>>
 * <<plugins-{type}s-{plugin}-data_stream_auto_routing>> 
@@ -200,27 +221,28 @@ The list of error codes accepted for DLQ could be customized with <<plugins-{typ
 but should be used only in motivated cases.
 
 [id="plugins-{type}s-{plugin}-ilm"]
-==== Index Lifecycle Management
-
-[NOTE]
-The Index Lifecycle Management feature requires plugin version `9.3.1` or higher.
+==== {ilm-cap} ({ilm-init})
 
 [NOTE]
-This feature requires an Elasticsearch instance of 6.6.0 or higher with at least a Basic license
+--
+* The {ilm-cap} ({ilm-init}) feature does not apply for {es-serverless}. 
+Any {ilm-init} settings in your plugin configuration are ignored and may cause errors.
+* The {ilm-init} feature requires plugin version `9.3.1` or higher.
+* This feature requires an {es} instance of 6.6.0 or higher with at least a Basic license
+--
 
-Logstash can use {ref}/index-lifecycle-management.html[Index Lifecycle
-Management] to automate the management of indices over time.
+{ls} can use {ref}/index-lifecycle-management.html[{ilm}] to automate the management of indices over time.
 
-The use of Index Lifecycle Management is controlled by the `ilm_enabled`
+The use of {ilm} is controlled by the `ilm_enabled`
 setting. By default, this setting detects whether the Elasticsearch instance
-supports ILM, and uses it if it is available. `ilm_enabled` can also be set to
-`true` or `false` to override the automatic detection, or disable ILM.
+supports {ilm-init}, and uses it if it is available. `ilm_enabled` can also be set to
+`true` or `false` to override the automatic detection, or disable {ilm-init}.
 
-This will overwrite the index settings and adjust the Logstash template to write
-the necessary settings for the template to support index lifecycle management,
+This will overwrite the index settings and adjust the {ls} template to write
+the necessary settings for the template to support {ilm},
 including the index policy and rollover alias to be used.
 
-Logstash will create a rollover alias for the indices to be written to,
+{ls} creates a rollover alias for the indices to be written to,
 including a pattern for how the actual indices will be named, and unless an ILM
 policy that already exists has been specified, a default policy will also be
 created. The default policy is configured to rollover an index when it reaches
@@ -246,14 +268,14 @@ See config below for an example:
       }
     }
 
-NOTE: Custom ILM policies must already exist on the Elasticsearch cluster before they can be used.
-
-NOTE: If the rollover alias or pattern is modified, the index template will need to be
+[NOTE]
+--
+* Custom ILM policies must already exist on the {es} cluster before they can be used.
+* If the rollover alias or pattern is modified, the index template will need to be
 overwritten as the settings `index.lifecycle.name` and
 `index.lifecycle.rollover_alias` are automatically written to the template
-
-NOTE: If the index property is supplied in the output definition, it will be overwritten by the rollover alias.
-
+* If the index property is supplied in the output definition, it will be overwritten by the rollover alias.
+--
 
 ==== Batch Sizes
 
@@ -294,6 +316,11 @@ index level and `monitoring` permissions at cluster level. The `monitoring`
 permission at cluster level is necessary to perform periodic connectivity
 checks.
 
+[id="plugins-{type}s-{plugin}-handling-non-utf-8"]
+==== Handling non UTF-8 data
+
+This plugin transmits events to Elasticsearch using a JSON API, and therefore requires that all string values in events to be valid UTF-8.
+When a string value on an event contains one or more byte sequences that are not valid in UTF-8, each offending byte sequence is replaced with the UTF-8 replacement character (`\uFFFD`).
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Output Configuration Options

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -22,6 +22,7 @@ module LogStash; module Outputs; class ElasticSearch;
   # made sense. We picked one on the lowish side to not use too much heap.
   TARGET_BULK_BYTES = 20 * 1024 * 1024 # 20MiB
 
+
   class HttpClient
     attr_reader :client, :options, :logger, :pool, :action_count, :recv_count
     # This is here in case we use DEFAULT_OPTIONS in the future
@@ -37,7 +38,7 @@ module LogStash; module Outputs; class ElasticSearch;
     # * `:user` - String. The user to use for authentication.
     # * `:password` - String. The password to use for authentication.
     # * `:timeout` - Float. A duration value, in seconds, after which a socket
-    #    operation or request will be aborted if not yet successfull
+    #    operation or request will be aborted if not yet successful
     # * `:client_settings` - a hash; see below for keys.
     #
     # The `client_settings` key is a has that can contain other settings:
@@ -132,6 +133,9 @@ module LogStash; module Outputs; class ElasticSearch;
                     action.map {|line| LogStash::Json.dump(line)}.join("\n") :
                     LogStash::Json.dump(action)
         as_json << "\n"
+
+        as_json.scrub! # ensure generated JSON is valid UTF-8
+
         if (stream_writer.pos + as_json.bytesize) > TARGET_BULK_BYTES && stream_writer.pos > 0
           stream_writer.flush # ensure writer has sync'd buffers before reporting sizes
           logger.debug("Sending partial bulk request for batch with one or more actions remaining.",
@@ -496,5 +500,6 @@ module LogStash; module Outputs; class ElasticSearch;
       end
       [args, source]
     end
+
   end
 end end end

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.22.1'
+  s.version         = '11.22.3'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -11,10 +11,13 @@ end
 [ {"http_compression" => true}, {"compression_level" => 1} ].each do |compression_config|
   describe "indexing with http_compression turned on", :integration => true do
     let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
+    let(:event_with_invalid_utf_8_bytes) { LogStash::Event.new("message" => "Message from spacecraft which contains \xAC invalid \xD7 byte sequences.", "type" => type) }
+
     let(:index) { 10.times.collect { rand(10).to_s }.join("") }
     let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
     let(:event_count) { 10000 + rand(500) }
-    let(:events) { event_count.times.map { event }.to_a }
+    # mix the events with valid and invalid UTF-8 payloads
+    let(:events) { event_count.times.map { |i| i%3 == 0 ? event : event_with_invalid_utf_8_bytes }.to_a }
     let(:config) {
       {
         "hosts" => get_host_port,

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -242,12 +242,14 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
           end
         end
 
-        context "with two messages" do
-          let(:message1) { "hey" }
-          let(:message2) { "you" }
+        context "with multiple messages" do
+          let(:message_head) { "Spacecraft message" }
+          let(:message_tail) { "byte sequence" }
+          let(:invalid_utf_8_message) { "contains invalid \xAC" }
           let(:actions) { [
-            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message1}],
-            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message2}],
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message_head}],
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> invalid_utf_8_message}],
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message_tail}],
           ]}
           it "executes one bulk_send operation" do
             allow(subject).to receive(:join_bulk_responses)
@@ -257,7 +259,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
 
           context "if one exceeds TARGET_BULK_BYTES" do
             let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
-            let(:message1) { "a" * (target_bulk_bytes + 1) }
+            let(:message_head) { "a" * (target_bulk_bytes + 1) }
             it "executes two bulk_send operations" do
               allow(subject).to receive(:join_bulk_responses)
               expect(subject).to receive(:bulk_send).twice

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.22.1
+  version: 11.22.3
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-21 00:00:00.000000000 Z
+date: 2024-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement