logstash-output-elasticsearch 12.1.2-java → 12.1.5-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59124c8f411f8bcb4742b43cc81cc075676732feef3275eac6898e3bf2341021
-  data.tar.gz: 26a524985bbf66a83b19807b6ebcb9fab824b3172a1718c1cedd85cc8f142be6
+  metadata.gz: 84a9fb2d6f4345422640ad3b59e9c0cb59b330569cbd33dc79cc1356b828bd34
+  data.tar.gz: 73375c880a26de3986cc1b984f3f898af055f040e1b3ea8e9772c1be78c03a3f
 SHA512:
-  metadata.gz: c9afff55af20a6b184a5ec912b8f96387bf3310b4de884972a3e98b7146bf2873b4a60e18f6781692cfa4c92c99fd25ef6da9ccc8a0ea4391c70eb41e81a9b5c
-  data.tar.gz: 1599a3d08006c0ca14c7b671eea25772aeb0f6b419682ac5d6ee030fd2dfda2aef34a68dea12cd30e6f0643260b81e360063f28b46278ccb2b4be2232dfe1b1b
+  metadata.gz: b5c8e15da6a79b51df996e6295687281d1dd19d668b6443455aeee3218e2c88f1aea271560cdad6bbde6a5c1ce44085e9bac4acb8dd65f4b84a596f589be1369
+  data.tar.gz: 5ae3d4135fd0f18460b7872b58f8385ed254c9154cd150116f43cae5b54557e92a0ac7403fac7e51161b1ffc24171cf49571662c25684a46adb02a866e73e720

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 12.1.5
+  - Support Elastic Cloud API keys in the `api_key` option, which now accepts an `id:api_key` pair, its base64-encoded form, or an `essu_` Cloud API key, and rejects an unrecognized format at startup [#1274](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1274)
+
+## 12.1.4
+  - [Doc] Add note for index option [#1269](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1269)
+
+## 12.1.3
+ - Improves the logging experience when DLQ used [#1253](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1253)
+
 ## 12.1.2
  - Fix: replace deprecated `File.exists?` with `File.exist?` for Ruby 3.4 (JRuby 10) compatibility [#1238](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1238)
 

data/README.md

@@ -1,6 +1,8 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-output-elasticsearch.svg)](https://travis-ci.com/logstash-plugins/logstash-output-elasticsearch)
+[![Unit Tests](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/unit-tests.yml/badge.svg)](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/unit-tests.yml)
+[![Secure Integration Tests](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/secure-integration-tests.yml/badge.svg)](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/secure-integration-tests.yml)
+[![Integration Tests](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/integration-tests.yml/badge.svg)](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/integration-tests.yml)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/index.asciidoc

@@ -467,8 +467,11 @@ For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bu
 Authenticate using Elasticsearch API key.
 Note that this option also requires SSL/TLS, which can be enabled by supplying a <<plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<plugins-{type}s-{plugin}-hosts>>, or by setting <<plugins-{type}s-{plugin}-ssl_enabled,`ssl_enabled => true`>>.
 
-Format is `id:api_key` where `id` and `api_key` are as returned by the
-Elasticsearch {ref}/security-api-create-api-key.html[Create API key API].
+The format is `id:api_key`, where `id` and `api_key` are as returned by the
+Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. The
+base64-encoded form of that pair is also accepted, as is an
+https://www.elastic.co/docs/deploy-manage/api-keys/elastic-cloud-api-keys[Elastic Cloud API key]
+(prefixed with `essu_`), which is used as-is.
 
 [id="plugins-{type}s-{plugin}-bulk_path"]
 ===== `bulk_path`
@@ -842,10 +845,15 @@ NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as
 The indexing target to write events to. 
 Can point to an {ref}/index-mgmt.html[index], {ref}/aliases.html[alias], or {ref}/data-streams.html[data stream]. 
 This can be dynamic using the `%{foo}` syntax.
+
+NOTE: When using dynamic syntax, verify that the resolved index names align with your cluster index templates and Index Lifecycle Management (ILM) configuration.
+
+WARNING: Date math (e.g. `%{+yyyy.MM.dd}`) is not recommended for use with aliases or data streams.
+
 The default value will partition your indices by day so you can more easily
 delete old data or only search specific date ranges.
 Indexes may not contain uppercase characters.
-For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
+For weekly indexes ISO 8601 format is recommended, eg. `logstash-%{+xxxx.ww}`.
 Logstash uses
 http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda
 formats] and the `@timestamp` field of each event is being used as source for the date.

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -190,8 +190,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
       return {} unless (api_key&.value)
 
-      value = is_base64?(api_key.value) ?  api_key.value : Base64.strict_encode64(api_key.value)
-
+      value = resolve_api_key(api_key.value)
       { "Authorization" => "ApiKey #{value}" }
     end
 
@@ -212,6 +211,28 @@ module LogStash; module Outputs; class ElasticSearch;
         url.match?(/\?[^\s#]+/) ? '&' : '?'
       end
 
+      # Resolves the `api_key` value into the credential used in the
+      # `Authorization: ApiKey` header. An already base64-encoded key and an
+      # Elastic Cloud API key are used as-is; a raw `id:api_key` pair is
+      # base64-encoded. An unrecognized value is rejected so a malformed key
+      # surfaces at startup rather than as a later authentication failure.
+      def resolve_api_key(key_value)
+        if is_base64?(key_value) || cloud_api_key?(key_value)
+          key_value
+        elsif key_value.match?(/\A[^:]+:[^:]+\z/)
+          Base64.strict_encode64(key_value)
+        else
+          raise LogStash::ConfigurationError, "Invalid api_key format. Expected a base64-encoded key, an 'id:api_key' pair, or a Cloud API key (essu_ prefix)."
+        end
+      end
+
+      # Elastic Cloud API keys (such as the unified Serverless keys) are opaque
+      # tokens prefixed with `essu_` that Elasticsearch accepts verbatim in the
+      # `Authorization: ApiKey` header, with no base64 encoding.
+      def cloud_api_key?(string)
+        string.match?(/\Aessu_.+/)
+      end
+
       def is_base64?(string)
         begin
           string == Base64.strict_encode64(Base64.strict_decode64(string))

data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb

@@ -16,7 +16,8 @@ module LogStash; module PluginMixins; module ElasticSearch
         :password => { :validate => :password },
 
         # Authenticate using Elasticsearch API key.
-        # format is id:api_key (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key])
+        # Format is either the `id:api_key` pair (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key]),
+        # its base64-encoded form, or an https://www.elastic.co/docs/deploy-manage/api-keys/elastic-cloud-api-keys[Elastic Cloud API key] (prefixed with `essu_`) can be used.
         :api_key => { :validate => :password },
 
         # Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -241,7 +241,6 @@ module LogStash; module PluginMixins; module ElasticSearch
         @dlq_writer.write(event, "#{detailed_message}")
       else
         log_level = dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception' ? :error : :warn
-
         @logger.public_send(log_level, message, status: status, action: action_params, response: response)
       end
     end
@@ -273,6 +272,7 @@ module LogStash; module PluginMixins; module ElasticSearch
       end
 
       actions_to_retry = []
+      dlq_routed_stats = {}
       responses.each_with_index do |response,idx|
         action_type, action_props = response.first
 
@@ -296,6 +296,11 @@ module LogStash; module PluginMixins; module ElasticSearch
         elsif @dlq_codes.include?(status)
           handle_dlq_response("Could not index event to Elasticsearch.", action, status, response)
           @document_level_metrics.increment(:dlq_routed)
+          if dlq_routed_stats.key?(status)
+            dlq_routed_stats[status][:count] += 1
+          else
+            dlq_routed_stats[status] = { :count => 1, :sample_event => { :action => action, :response => response } }
+          end
           next
         else
           # only log what the user whitelisted
@@ -305,6 +310,11 @@ module LogStash; module PluginMixins; module ElasticSearch
         end
       end
 
+      if @dlq_writer && !dlq_routed_stats.empty?
+        total = dlq_routed_stats.values.sum { |entry| entry[:count] }
+        @logger.warn("Events could not be indexed and routing to DLQ", :count => total, :dlq_routed_stats => dlq_routed_stats)
+      end
+
       actions_to_retry
     end
 

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '12.1.2'
+  s.version         = ::File.read('version').split("\n").first
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
   s.platform = RUBY_PLATFORM
 
   # Files
-  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "version", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

data/spec/unit/http_client_builder_spec.rb

@@ -62,6 +62,40 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
         expect(api_key_header["Authorization"]).to eql(expected)
       end
     end
+
+    context "when api-key is an Elastic Cloud API key (essu_ prefix)" do
+      # The suffix is intentionally not canonical base64: a Cloud key is opaque
+      # and must be forwarded verbatim regardless of its payload encoding.
+      let(:api_key) { "essu_VFZGblZreFhTekJ4ZDB4M2NHUnZRMEU2YzNWd1pYSnpaV055WlhRPQ==AAAAAAAA" }
+      let(:api_key_secured) do
+        secured = double("api_key")
+        allow(secured).to receive(:value).and_return(api_key)
+        secured
+      end
+      let(:options) { { "api_key" => api_key_secured } }
+      let(:logger) { double("logger") }
+      let(:api_key_header) { klass.setup_api_key(logger, options) }
+
+      it "returns the cloud api-key header verbatim without re-encoding" do
+        expected = "ApiKey #{api_key}"
+        expect(api_key_header["Authorization"]).to eql(expected)
+      end
+    end
+
+    context "when api-key has an unrecognized format" do
+      let(:api_key) { "not-a-valid-key" }
+      let(:api_key_secured) do
+        secured = double("api_key")
+        allow(secured).to receive(:value).and_return(api_key)
+        secured
+      end
+      let(:options) { { "api_key" => api_key_secured } }
+      let(:logger) { double("logger") }
+
+      it "raises a configuration error" do
+        expect { klass.setup_api_key(logger, options) }.to raise_error(LogStash::ConfigurationError, /Invalid api_key format/)
+      end
+    end
   end
 
   describe "customizing action paths" do

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -901,6 +901,9 @@ describe LogStash::Outputs::ElasticSearch do
 
     before(:each) do
       allow(elasticsearch_output_instance.client).to receive(:logger).and_return(logger_stub)
+      # Stub the output plugin's own @logger to prevent submit()/retrying_submit()
+      # from logging the full 15MB action payload on each retry, which floods stdout.
+      elasticsearch_output_instance.instance_variable_set(:@logger, logger_stub)
 
       allow(elasticsearch_output_instance.client).to receive(:bulk).and_call_original
 
@@ -1475,7 +1478,7 @@ describe LogStash::Outputs::ElasticSearch do
             expect( event ).to be_a LogStash::Event
             expect( event ).to be events.first
             expect( reason ).to start_with "Could not index event to Elasticsearch. status: #{error_code}, action: [\"index\""
-            expect( reason ).to match /_id=>"bar".*"foo"=>"bar".*response:.*"reason"=>"TEST"/
+            expect( reason ).to match /_id(?::| ?=>) ?"bar".*"foo" ?=> ?"bar".*response:.*"reason" ?=> ?"TEST"/
 
             method.call(*args) # won't hurt to call LogStash::Util::DummyDeadLetterQueueWriter
           end.once

data/version

@@ -0,0 +1 @@
+12.1.5

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 12.1.2
+  version: 12.1.5
 platform: java
 authors:
 - Elastic
 bindir: bin
 cert_chain: []
-date: 2026-02-11 00:00:00.000000000 Z
+date: 2026-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: manticore
@@ -334,6 +334,7 @@ files:
 - spec/unit/outputs/elasticsearch_ssl_spec.rb
 - spec/unit/outputs/error_whitelist_spec.rb
 - spec/unit/outputs/license_check_spec.rb
+- version
 homepage: https://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - apache-2.0