logstash-filter-elasticsearch 3.19.0 → 3.19.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 28c0544d8e61fe99078cabecc2ce4cf898aefc35775e80a71a5bb34221d534d5
-  data.tar.gz: 6f42fd12be09e9aa4b619c95ffff8cc9dba427f5f71f915d2e7c30a583f83fec
+  metadata.gz: 9a0e21ee3a328a2c9f6ad6a7cba6035f09fdc3a760787441efc9746b0db53a00
+  data.tar.gz: 66facd522d9a822e62529bd6fc99207db46070232f13558fef87565002c24a9c
 SHA512:
-  metadata.gz: 3aaa9a9b8b31e1782e927fe4bccb26fd28063206dfeaf00c574657e0d91e3c7c7544565c657a65d37a5e164c0e2477b8fdab3f4dfb2c7feb79971962e94cbb87
-  data.tar.gz: 339a881e589fe44d5657e4a4417f9e72f38921a08ff99b2cbc1534785fcddb00ae484960311a70d653f3ca058f38ceed0a290d789bb0d728d7db1ff8de8a6d46
+  metadata.gz: baebe8460aaaa98b3c84da38a52e344a840beed5cfe6ae7359a07b4a86e2403d01ed693755b5e7750c6f8c56586be0ad51aac306e7f5a112c03398ffe4138015
+  data.tar.gz: 97afa7a8436f4a27472c60670230ed6746f84c12e7daeff3ff08723cddcf57402e1918ef4dca1d8fe0b51aa9fb2318d0f529f89fd148b29bf4d9aec146502ec6

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.19.1
+  - Support base64-encoded and Elastic Cloud (`essu_`-prefixed) API keys in the `api_key` option; reject unrecognized formats at startup. [#229](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/229)
+
 ## 3.19.0
   - ES|QL support [#199](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/199)
 

data/docs/index.asciidoc

@@ -305,8 +305,12 @@ Example:
 Authenticate using Elasticsearch API key. Note that this option also requires
 enabling the <<plugins-{type}s-{plugin}-ssl_enabled>> option.
 
-Format is `id:api_key` where `id` and `api_key` are as returned by the
-Elasticsearch {ref}/security-api-create-api-key.html[Create API key API].
+The format is `id:api_key`, where `id` and `api_key` are as returned by the
+Elasticsearch
+{ref}/security-api-create-api-key.html[Create
+API key API]. The base64-encoded form of that pair is also accepted, as is an
+https://www.elastic.co/docs/deploy-manage/api-keys/elastic-cloud-api-keys[Elastic Cloud API key]
+(prefixed with `essu_`), which is used as-is.
 
 [id="plugins-{type}s-{plugin}-ca_trusted_fingerprint"]
 ===== `ca_trusted_fingerprint`

data/lib/logstash/filters/elasticsearch/client.rb

@@ -101,12 +101,40 @@ module LogStash
       end
 
       def setup_api_key(api_key)
-        return {} unless (api_key && api_key.value)
+        return {} unless (api_key&.value)
 
-        token = ::Base64.strict_encode64(api_key.value)
+        token = resolve_api_key(api_key.value)
         { 'Authorization' => "ApiKey #{token}" }
       end
 
+      # Resolves the `api_key` value into the credential used in the
+      # `Authorization: ApiKey` header. An already base64-encoded key and an Elastic
+      # Cloud API key are used as-is; a raw `id:api_key` pair is base64-encoded. An
+      # unrecognized value is rejected so a malformed key surfaces at startup rather
+      # than as a later authentication failure.
+      def resolve_api_key(key_value)
+        if base64?(key_value) || cloud_api_key?(key_value)
+          key_value
+        elsif key_value.match?(/\A[^:]+:[^:]+\z/)
+          Base64.strict_encode64(key_value)
+        else
+          raise LogStash::ConfigurationError, "Invalid api_key format. Expected a base64-encoded key, an 'id:api_key' pair, or a Cloud API key (essu_ prefix)."
+        end
+      end
+
+      # Elastic Cloud API keys (such as the unified Serverless keys) are opaque
+      # tokens prefixed with `essu_` that Elasticsearch accepts verbatim in the
+      # `Authorization: ApiKey` header, with no base64 encoding.
+      def cloud_api_key?(string)
+        string.match?(/\Aessu_.+/)
+      end
+
+      def base64?(string)
+        string == Base64.strict_encode64(Base64.strict_decode64(string))
+      rescue ArgumentError
+        false
+      end
+
       def get_transport_client_class
         # LS-core includes `elasticsearch` gem. The gem is composed of two separate gems: `elasticsearch-api` and `elasticsearch-transport`
         # And now `elasticsearch-transport` is old, instead we have `elastic-transport`.

data/lib/logstash/filters/elasticsearch.rb

@@ -73,7 +73,8 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   config :cloud_auth, :validate => :password
 
   # Authenticate using Elasticsearch API key.
-  # format is id:api_key (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key])
+  # Format is either the `id:api_key` pair (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key]),
+  # its base64-encoded form, or an https://www.elastic.co/docs/deploy-manage/api-keys/elastic-cloud-api-keys[Elastic Cloud API key] (prefixed with `essu_`) can be used.
   config :api_key, :validate => :password
 
   # Set the address of a forward HTTP proxy.

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.19.0'
+  s.version         = ::File.read('version').split("\n").first
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "version", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

data/spec/filters/elasticsearch_spec.rb

@@ -4,6 +4,7 @@ require "logstash/plugin"
 require "logstash/filters/elasticsearch"
 require "logstash/json"
 require "cabin"
+require "manticore"
 require "webrick"
 require "uri"
 
@@ -324,14 +325,50 @@ describe LogStash::Filters::Elasticsearch do
       end
 
       context "with ssl" do
-        let(:config) { super().merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl_enabled" => true }) }
+        let(:api_key_value) { nil }
+        let(:config) { super().merge("ssl_enabled" => true, 'api_key' => LogStash::Util::Password.new(api_key_value)) }
+        let(:encoded_api_key) { Base64.strict_encode64('foo:bar') }
 
-        it "should set authorization" do
-          plugin.register
-          client = plugin.send(:get_client).client
-          auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
+        shared_examples "a plugin that sets the ApiKey authorization header" do
+          it "correctly sets the Authorization header" do
+            plugin.register
+            client = plugin.send(:get_client).client
+            auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
+
+            expect(auth_header).to eql("ApiKey #{encoded_api_key}")
+          end
+        end
+
+        context "with a non-encoded API key" do
+          let(:api_key_value) { "foo:bar" }
+          it_behaves_like "a plugin that sets the ApiKey authorization header"
+        end
+
+        context "with an encoded API key" do
+          let(:api_key_value) { encoded_api_key }
+          it_behaves_like "a plugin that sets the ApiKey authorization header"
+        end
+
+        context "with an Elastic Cloud API key (essu_ prefix)" do
+          # The suffix is intentionally not canonical base64: a Cloud key is opaque
+          # and must be forwarded verbatim regardless of its payload encoding.
+          let(:api_key_value) { "essu_VFZGblZreFhTekJ4ZDB4M2NHUnZRMEU2YzNWd1pYSnpaV055WlhRPQ==AAAAAAAA" }
+
+          it "sets the Authorization header verbatim without re-encoding" do
+            plugin.register
+            client = plugin.send(:get_client).client
+            auth_header = extract_transport(client).options[:transport_options][:headers]['Authorization']
+
+            expect(auth_header).to eql("ApiKey #{api_key_value}")
+          end
+        end
 
-          expect( auth_header ).to eql "ApiKey #{Base64.strict_encode64('foo:bar')}"
+        context "with an unrecognized api_key format" do
+          let(:api_key_value) { "not-a-valid-key" }
+
+          it "fails registration with a configuration error" do
+            expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Invalid api_key format/)
+          end
         end
 
         context 'user also set' do

data/version

@@ -0,0 +1 @@
+3.19.1

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.19.0
+  version: 3.19.1
 platform: ruby
 authors:
 - Elastic
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-07-21 00:00:00.000000000 Z
+date: 2026-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -19,7 +19,6 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  name: logstash-core-plugin-api
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -31,6 +30,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
+  name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,7 +39,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '9'
-  name: elasticsearch
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -51,12 +50,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '9'
 - !ruby/object:Gem::Dependency
+  name: manticore
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.7.1
-  name: manticore
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -65,12 +64,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.7.1
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-ecs_compatibility_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
-  name: logstash-mixin-ecs_compatibility_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -79,12 +78,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-ca_trusted_fingerprint_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-ca_trusted_fingerprint_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -93,12 +92,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-normalize_config_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-normalize_config_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -107,12 +106,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-validator_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-validator_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -121,12 +120,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: cabin
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
-  name: cabin
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -135,12 +134,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.6'
 - !ruby/object:Gem::Dependency
+  name: webrick
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: webrick
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -149,12 +148,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -213,13 +212,13 @@ files:
 - spec/filters/fixtures/test_certs/ls.key
 - spec/filters/integration/elasticsearch_esql_spec.rb
 - spec/filters/integration/elasticsearch_spec.rb
+- version
 homepage: https://elastic.co/logstash
 licenses:
 - Apache License (2.0)
 metadata:
   logstash_plugin: 'true'
   logstash_group: filter
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -234,8 +233,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
-signing_key:
+rubygems_version: 3.6.3
 specification_version: 4
 summary: Copies fields from previous log events in Elasticsearch to current events
 test_files: