logstash-output-elasticsearch 11.22.15-java → 11.22.17-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a0b7ec8d5dbede6389a4e59f97d8e98c812091e9f9a1f49ed11a443c4c7b4a6b
-  data.tar.gz: e0a1b09ca555fed7cd9340ce843e837b3abb0ceb0d21a577d43a3b89462042cc
+  metadata.gz: d4bb66bdcf7a4f0d3557d0999e3c3737a1a75d8cb7fa26830339cf6e0e28b82d
+  data.tar.gz: 12bfde3746577a042a3ec3962f39b76e530a65a7cf34e856146e8b3fc2933495
 SHA512:
-  metadata.gz: 14dd08f7aa74dee3c9cadb7ff96a86a0e65e4fcb425b1292749005faac2e7153a580ab6212dabb7664f1d83997404993ec1a2f0f9efdabd834aecd5447b728f8
-  data.tar.gz: 1c21dd49841e4f05ddaca95e7a3f91bb9d7e9c3ee9887249ef2b365c3066db666645b3b508a173c2c9fa55256c906fa202068b2942646cac8e2cc1e3ab444b65
+  metadata.gz: 95d48b93014a46d8412add41d4e753ee1c0ef091f86ad2138825802bbaf416c5865abd7d8afa7df825361f2db010d0f5b189df4274991d02f4f67325ddab56fc
+  data.tar.gz: 607b5b63bc6a41067bd0d4a9c2f627dbb9ea007b43f6d69f8ec53e2d1e8177b9b37be0b76249e0cc310d0da541c981ac753db9cb1cece249404a7c33b8c92618

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.22.17
+  - Support base64-encoded and Elastic Cloud (`essu_`-prefixed) API keys in the `api_key` option; reject unrecognized formats at startup. [#1278](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1278)
+
+## 11.22.16
+ - Fix serverless compatibility: guard against nil params in pool requests and strip unsupported template settings (number_of_shards, number_of_replicas) on serverless
+
 ## 11.22.15
  - Improves the logging experience when DLQ used [#1258](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1258)
 

data/README.md

@@ -1,6 +1,8 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-output-elasticsearch.svg)](https://travis-ci.com/logstash-plugins/logstash-output-elasticsearch)
+[![Unit Tests](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/unit-tests.yml/badge.svg)](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/unit-tests.yml)
+[![Integration Tests](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/integration-tests.yml/badge.svg)](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/integration-tests.yml)
+[![Secure Integration Tests](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/secure-integration-tests.yml/badge.svg)](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/secure-integration-tests.yml)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/index.asciidoc

@@ -458,8 +458,12 @@ For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bu
 Authenticate using Elasticsearch API key.
 Note that this option also requires SSL/TLS, which can be enabled by supplying a <<plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<plugins-{type}s-{plugin}-hosts>>, or by setting <<plugins-{type}s-{plugin}-ssl,`ssl_enabled => true`>>.
 
-Format is `id:api_key` where `id` and `api_key` are as returned by the
-Elasticsearch {ref}/security-api-create-api-key.html[Create API key API].
+The format is `id:api_key`, where `id` and `api_key` are as returned by the
+Elasticsearch
+{ref}/security-api-create-api-key.html[Create
+API key API]. The base64-encoded form of that pair is also accepted, as is an
+https://www.elastic.co/docs/deploy-manage/api-keys/elastic-cloud-api-keys[Elastic Cloud API key]
+(prefixed with `essu_`), which is used as-is.
 
 [id="plugins-{type}s-{plugin}-bulk_path"]
 ===== `bulk_path`

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -347,6 +347,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def perform_request_to_url(url, method, path, params={}, body=nil)
+      params ||= {}
       params[:headers] = DEFAULT_EAV_HEADER.merge(params[:headers] || {}) if serverless?
       @adapter.perform_request(url, method, path, params, body)
     end

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -433,7 +433,7 @@ module LogStash; module Outputs; class ElasticSearch;
     def template_put(template_endpoint, name, template)
       path = "#{template_endpoint}/#{name}"
       logger.info("Installing Elasticsearch template", name: name)
-      @pool.put(path, nil, LogStash::Json.dump(template))
+      @pool.put(path, {}, LogStash::Json.dump(template))
     rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
       raise e unless e.response_code == 404
     end
@@ -447,7 +447,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
     # Create a new rollover alias
     def rollover_alias_put(alias_name, alias_definition)
-      @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition))
+      @pool.put(CGI::escape(alias_name), {}, LogStash::Json.dump(alias_definition))
       logger.info("Created rollover alias", name: alias_name)
       # If the rollover alias already exists, ignore the error that comes back from Elasticsearch
     rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
@@ -473,7 +473,7 @@ module LogStash; module Outputs; class ElasticSearch;
     def ilm_policy_put(name, policy)
       path = "_ilm/policy/#{name}"
       logger.info("Installing ILM policy #{policy}", name: name)
-      @pool.put(path, nil, LogStash::Json.dump(policy))
+      @pool.put(path, {}, LogStash::Json.dump(policy))
     end
 
 

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -188,9 +188,38 @@ module LogStash; module Outputs; class ElasticSearch;
     def self.setup_api_key(logger, params)
       api_key = params["api_key"]
 
-      return {} unless (api_key && api_key.value)
+      return {} unless (api_key&.value)
 
-      { "Authorization" => "ApiKey " + Base64.strict_encode64(api_key.value) }
+      token = resolve_api_key(api_key.value)
+      { "Authorization" => "ApiKey #{token}" }
+    end
+
+    # Resolves the `api_key` value into the credential used in the
+    # `Authorization: ApiKey` header. An already base64-encoded key and an Elastic
+    # Cloud API key are used as-is; a raw `id:api_key` pair is base64-encoded. An
+    # unrecognized value is rejected so a malformed key surfaces at startup rather
+    # than as a later authentication failure.
+    def self.resolve_api_key(key_value)
+      if base64?(key_value) || cloud_api_key?(key_value)
+        key_value
+      elsif key_value.match?(/\A[^:]+:[^:]+\z/)
+        Base64.strict_encode64(key_value)
+      else
+        raise LogStash::ConfigurationError, "Invalid api_key format. Expected a base64-encoded key, an 'id:api_key' pair, or a Cloud API key (essu_ prefix)."
+      end
+    end
+
+    # Elastic Cloud API keys (such as the unified Serverless keys) are opaque
+    # tokens prefixed with `essu_` that Elasticsearch accepts verbatim in the
+    # `Authorization: ApiKey` header, with no base64 encoding.
+    def self.cloud_api_key?(string)
+      string.match?(/\Aessu_.+/)
+    end
+
+    def self.base64?(string)
+      string == Base64.strict_encode64(Base64.strict_decode64(string))
+    rescue ArgumentError
+      false
     end
 
     private

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -28,6 +28,7 @@ module LogStash; module Outputs; class ElasticSearch
       end
 
       add_ilm_settings_to_template(plugin, template) if plugin.ilm_in_use?
+      strip_serverless_incompatible_settings(template) if plugin.serverless?
       plugin.logger.debug("Attempting to install template", template: template)
       install(plugin.client, template_endpoint(plugin), template_name(plugin), template, plugin.template_overwrite)
     end
@@ -129,5 +130,19 @@ module LogStash; module Outputs; class ElasticSearch
       end
     end
 
+    SERVERLESS_INCOMPATIBLE_SETTINGS = %w[
+      number_of_shards
+      index.number_of_shards
+      number_of_replicas
+      index.number_of_replicas
+    ].freeze
+
+    def self.strip_serverless_incompatible_settings(template)
+      settings = template.dig('template', 'settings') || template['settings']
+      return unless settings
+
+      SERVERLESS_INCOMPATIBLE_SETTINGS.each { |s| settings.delete(s) }
+    end
+
   end
 end end end

data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb

@@ -16,7 +16,8 @@ module LogStash; module PluginMixins; module ElasticSearch
         :password => { :validate => :password },
 
         # Authenticate using Elasticsearch API key.
-        # format is id:api_key (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key])
+        # Format is either the `id:api_key` pair (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key]),
+        # its base64-encoded form, or an https://www.elastic.co/docs/deploy-manage/api-keys/elastic-cloud-api-keys[Elastic Cloud API key] (prefixed with `essu_`) can be used.
         :api_key => { :validate => :password },
 
         # Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.22.15'
+  s.version         = ::File.read('version').split("\n").first
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
   s.platform = RUBY_PLATFORM
 
   # Files
-  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "version", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -186,4 +186,75 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
 
     end
   end
+
+  describe ".strip_serverless_incompatible_settings" do
+    context "with composable template (template.settings)" do
+      let(:template) do
+        {
+          "template" => {
+            "settings" => {
+              "index.refresh_interval" => "5s",
+              "number_of_shards" => 1,
+              "number_of_replicas" => 0
+            }
+          }
+        }
+      end
+
+      it "removes number_of_shards and number_of_replicas" do
+        described_class.strip_serverless_incompatible_settings(template)
+        expect(template["template"]["settings"]).not_to have_key("number_of_shards")
+        expect(template["template"]["settings"]).not_to have_key("number_of_replicas")
+      end
+
+      it "preserves compatible settings" do
+        described_class.strip_serverless_incompatible_settings(template)
+        expect(template["template"]["settings"]["index.refresh_interval"]).to eq("5s")
+      end
+    end
+
+    context "with index-prefixed setting keys" do
+      let(:template) do
+        {
+          "template" => {
+            "settings" => {
+              "index.number_of_shards" => 1,
+              "index.number_of_replicas" => 0
+            }
+          }
+        }
+      end
+
+      it "removes index-prefixed variants" do
+        described_class.strip_serverless_incompatible_settings(template)
+        expect(template["template"]["settings"]).not_to have_key("index.number_of_shards")
+        expect(template["template"]["settings"]).not_to have_key("index.number_of_replicas")
+      end
+    end
+
+    context "with legacy template (top-level settings)" do
+      let(:template) do
+        {
+          "settings" => {
+            "index.refresh_interval" => "5s",
+            "number_of_shards" => 1
+          }
+        }
+      end
+
+      it "removes incompatible settings from top-level settings" do
+        described_class.strip_serverless_incompatible_settings(template)
+        expect(template["settings"]).not_to have_key("number_of_shards")
+        expect(template["settings"]["index.refresh_interval"]).to eq("5s")
+      end
+    end
+
+    context "with no settings" do
+      let(:template) { { "mappings" => {} } }
+
+      it "does not raise" do
+        expect { described_class.strip_serverless_incompatible_settings(template) }.not_to raise_error
+      end
+    end
+  end
 end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -908,6 +908,9 @@ describe LogStash::Outputs::ElasticSearch do
 
     before(:each) do
       allow(elasticsearch_output_instance.client).to receive(:logger).and_return(logger_stub)
+      # Stub the output plugin's own @logger to prevent submit()/retrying_submit()
+      # from logging the full 15MB action payload on each retry, which floods stdout.
+      elasticsearch_output_instance.instance_variable_set(:@logger, logger_stub)
 
       allow(elasticsearch_output_instance.client).to receive(:bulk).and_call_original
 
@@ -1615,6 +1618,32 @@ describe LogStash::Outputs::ElasticSearch do
       end; end
     end
 
+    context "with an already base64-encoded API key" do
+      let(:api_key) { Base64.strict_encode64("some_id:some_api_key") }
+      let(:options) { { "ssl_enabled" => true, "api_key" => ::LogStash::Util::Password.new(api_key) } }
+      let(:base64_api_key) { "ApiKey #{api_key}" }
+
+      it_behaves_like 'secure api-key authenticated client'
+    end
+
+    context "with an Elastic Cloud API key (essu_ prefix)" do
+      let(:api_key) { "essu_VFZGblZreFhTekJ4ZDB4M2NHUnZRMEU2YzNWd1pYSnpaV055WlhRPQ==AAAAAAAA" }
+      let(:options) { { "ssl_enabled" => true, "api_key" => ::LogStash::Util::Password.new(api_key) } }
+
+      it "sets the Authorization header verbatim without re-encoding" do
+        expect(manticore_options[:headers]).to eq({ "Authorization" => "ApiKey #{api_key}" })
+      end
+    end
+
+    context "with an unrecognized api_key format" do
+      let(:do_register) { false }
+      let(:options) { { "ssl_enabled" => true, "api_key" => ::LogStash::Util::Password.new("not-a-valid-key") } }
+
+      it "fails registration with a configuration error" do
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError, /Invalid api_key format/)
+      end
+    end
+
     context "when set without ssl_enabled => true" do
       let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
       let(:options) { { "api_key" => api_key } }

data/version

@@ -0,0 +1 @@
+11.22.17

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.22.15
+  version: 11.22.17
 platform: java
 authors:
 - Elastic
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-04-17 00:00:00.000000000 Z
+date: 2026-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: manticore
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -19,7 +19,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
-  name: manticore
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -31,6 +30,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.0.0
 - !ruby/object:Gem::Dependency
+  name: stud
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,7 +39,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.0'
-  name: stud
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -51,6 +50,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -59,7 +59,6 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  name: logstash-core-plugin-api
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -71,12 +70,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-ecs_compatibility_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-ecs_compatibility_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -85,12 +84,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-deprecation_logger_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-deprecation_logger_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -99,12 +98,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-ca_trusted_fingerprint_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-ca_trusted_fingerprint_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -113,12 +112,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-normalize_config_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-normalize_config_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -127,12 +126,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -141,12 +140,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -155,12 +154,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: flores
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: flores
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -169,12 +168,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: cabin
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
-  name: cabin
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -183,12 +182,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.6'
 - !ruby/object:Gem::Dependency
+  name: webrick
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: webrick
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -197,12 +196,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: webmock
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -211,12 +210,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: rspec-collection_matchers
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rspec-collection_matchers
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -225,12 +224,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: elasticsearch
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -339,13 +338,13 @@ files:
 - spec/unit/outputs/elasticsearch_ssl_spec.rb
 - spec/unit/outputs/error_whitelist_spec.rb
 - spec/unit/outputs/license_check_spec.rb
+- version
 homepage: https://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - apache-2.0
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -360,8 +359,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
-signing_key:
+rubygems_version: 3.6.3
 specification_version: 4
 summary: Stores logs in Elasticsearch
 test_files: