logstash-input-elasticsearch 5.3.0 → 5.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90dc46d1bcadf70173792a9c2319925ea5e78bd3e6322e7c323de178afb72fa4
-  data.tar.gz: 21548a5935b0aa5b153b6e1bc00c7f9f3f6ade7f6546a0beb0ce7cc6cef5a984
+  metadata.gz: f16f7eb70ca4239c5d779c5fa8c77d0778ab48277c7f6ca4003d20894d0d7108
+  data.tar.gz: 59baf110729b3f8e353087dda028b6e6dc7c6973449307756729d451ddc4192e
 SHA512:
-  metadata.gz: 59e8a03c25d8ee9a1a02014b75d1096b81250b85c7f6bd7e78114764534d9ad244d5d758e974c84bc20d9aa0c8fad669a3ef636b17037426c30f2705d74083d2
-  data.tar.gz: 8b986365b25b8884e3aae581b37ce35f9190e50bf3ce4d17263887c31721efe16f16b97dcf2571dcb446991f7ea4406c6011a1d1dbd3afd7f3e07ec0c4b9b743
+  metadata.gz: 5450236e4c130d4a7737520de8571d9364327703c8b715ed3aa6de638f604e83c0062ebc2d1f36fecbb7d11d6e29931f8e713008ae89529f843d048c8f82227d
+  data.tar.gz: 1e5fdda8c6feafa77d40eda3940155995fba96b1c6d925b74e3392e246eb893a46dc38912335011cc9bd9235a4bf9ad68a0378b8799e48947ea98f68c1cb2207

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 5.3.2
+  - Support Elastic Cloud API keys in the `api_key` option, which now accepts an `id:api_key` pair, its base64-encoded form, or an `essu_` Cloud API key, and rejects an unrecognized format at startup [#274](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/274)
+
+## 5.3.1
+  - Fix serverless request failure caused by conflicting `compatible-with` and `Elastic-Api-Version` headers when using elasticsearch-ruby v9 [#269](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/269)
+
 ## 5.3.0
   - Drop a support for Logstash 7.x by requiring elasticsearch gem >= 8. Logstash 8+ continues to work as before [#252](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/252)
 

data/docs/index.asciidoc

@@ -404,10 +404,12 @@ input plugins.
 
 Authenticate using Elasticsearch API key. Note that this option also requires enabling the <<plugins-{type}s-{plugin}-ssl_enabled>> option.
 
-Format is `id:api_key` where `id` and `api_key` are as returned by the
+The format is `id:api_key`, where `id` and `api_key` are as returned by the
 Elasticsearch
 {ref}/security-api-create-api-key.html[Create
-API key API].
+API key API]. The base64-encoded form of that pair is also accepted, as is an
+https://www.elastic.co/docs/deploy-manage/api-keys/elastic-cloud-api-keys[Elastic Cloud API key]
+(prefixed with `essu_`), which is used as-is.
 
 [id="plugins-{type}s-{plugin}-ca_trusted_fingerprint"]
 ===== `ca_trusted_fingerprint`

data/lib/logstash/inputs/elasticsearch.rb

@@ -216,7 +216,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   config :cloud_auth, :validate => :password
 
   # Authenticate using Elasticsearch API key.
-  # format is id:api_key (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key])
+  # Format is either the `id:api_key` pair (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key]),
+  # its base64-encoded form, or an https://www.elastic.co/docs/deploy-manage/api-keys/elastic-cloud-api-keys[Elastic Cloud API key] (prefixed with `essu_`) can be used.
   config :api_key, :validate => :password
 
   # Set the address of a forward HTTP proxy.
@@ -331,31 +332,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     validate_authentication
     fill_user_password_from_cloud_auth
 
-    transport_options = {:headers => {}}
-    transport_options[:headers].merge!(INTERNAL_ORIGIN_HEADER)
-    transport_options[:headers].merge!(setup_basic_auth(user, password))
-    transport_options[:headers].merge!(setup_api_key(api_key))
-    transport_options[:headers].merge!({'user-agent' => prepare_user_agent()})
-    transport_options[:headers].merge!(@custom_headers) unless @custom_headers.empty?
-    transport_options[:request_timeout] = @request_timeout_seconds unless @request_timeout_seconds.nil?
-    transport_options[:connect_timeout] = @connect_timeout_seconds unless @connect_timeout_seconds.nil?
-    transport_options[:socket_timeout]  = @socket_timeout_seconds  unless @socket_timeout_seconds.nil?
-
-    hosts = setup_hosts
-    ssl_options = setup_client_ssl
-
-    @logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
-
-    transport_options[:proxy] = @proxy.to_s if @proxy && !@proxy.eql?('')
-
-    @client_options = {
-      :hosts => hosts,
-      :transport_options => transport_options,
-      :transport_class => ::Elastic::Transport::Transport::HTTP::Manticore,
-      :ssl => ssl_options
-    }
-
-    @client = Elasticsearch::Client.new(@client_options)
+    @client = new_client
 
     test_connection!
 
@@ -566,10 +543,32 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   def setup_api_key(api_key)
     return {} unless (api_key&.value)
 
-    token = base64?(api_key.value) ? api_key.value : Base64.strict_encode64(api_key.value)
+    token = resolve_api_key(api_key.value)
     { 'Authorization' => "ApiKey #{token}" }
   end
 
+  # Resolves the `api_key` value into the credential used in the
+  # `Authorization: ApiKey` header. An already base64-encoded key and an Elastic
+  # Cloud API key are used as-is; a raw `id:api_key` pair is base64-encoded. An
+  # unrecognized value is rejected so a malformed key surfaces at startup rather
+  # than as a later authentication failure.
+  def resolve_api_key(key_value)
+    if base64?(key_value) || cloud_api_key?(key_value)
+      key_value
+    elsif key_value.match?(/\A[^:]+:[^:]+\z/)
+      Base64.strict_encode64(key_value)
+    else
+      raise LogStash::ConfigurationError, "Invalid api_key format. Expected a base64-encoded key, an 'id:api_key' pair, or a Cloud API key (essu_ prefix)."
+    end
+  end
+
+  # Elastic Cloud API keys (such as the unified Serverless keys) are opaque
+  # tokens prefixed with `essu_` that Elasticsearch accepts verbatim in the
+  # `Authorization: ApiKey` header, with no base64 encoding.
+  def cloud_api_key?(string)
+    string.match?(/\Aessu_.+/)
+  end
+
   def base64?(string)
     string == Base64.strict_encode64(Base64.strict_decode64(string))
   rescue ArgumentError
@@ -641,6 +640,31 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # @private used by unit specs
   attr_reader :client
 
+  def new_client(serverless: false)
+    headers = {}
+    headers.merge!(INTERNAL_ORIGIN_HEADER)
+    headers.merge!(setup_basic_auth(user, password))
+    headers.merge!(setup_api_key(api_key))
+    headers.merge!({'user-agent' => prepare_user_agent()})
+    headers.merge!(@custom_headers) unless @custom_headers.empty?
+    headers.merge!(DEFAULT_EAV_HEADER) if serverless
+
+    transport_options = { headers: headers }
+    transport_options[:request_timeout] = @request_timeout_seconds unless @request_timeout_seconds.nil?
+    transport_options[:connect_timeout] = @connect_timeout_seconds unless @connect_timeout_seconds.nil?
+    transport_options[:socket_timeout]  = @socket_timeout_seconds  unless @socket_timeout_seconds.nil?
+
+    @logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
+    transport_options[:proxy] = @proxy.to_s if @proxy && !@proxy.eql?('')
+
+    Elasticsearch::Client.new(
+      hosts: setup_hosts,
+      transport_options: transport_options,
+      transport_class: ::Elastic::Transport::Transport::HTTP::Manticore,
+      ssl: setup_client_ssl
+    )
+  end
+
   def test_connection!
     @client.ping
   rescue Elasticsearch::UnsupportedProductError
@@ -663,8 +687,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # verify the header by sending GET /
   def setup_serverless
     if serverless?
-      @client_options[:transport_options][:headers].merge!(DEFAULT_EAV_HEADER)
-      @client = Elasticsearch::Client.new(@client_options)
+      @client = new_client(serverless: true)
       @client.info
     end
   rescue => e

data/logstash-input-elasticsearch.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "version", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

data/spec/inputs/elasticsearch_spec.rb

@@ -835,6 +835,28 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
           it_behaves_like "a plugin that sets the ApiKey authorization header"
         end
 
+        context "with an Elastic Cloud API key (essu_ prefix)" do
+          # The suffix is intentionally not canonical base64: a Cloud key is opaque
+          # and must be forwarded verbatim regardless of its payload encoding.
+          let(:api_key_value) { "essu_VFZGblZreFhTekJ4ZDB4M2NHUnZRMEU2YzNWd1pYSnpaV055WlhRPQ==AAAAAAAA" }
+
+          it "sets the Authorization header verbatim without re-encoding" do
+            plugin.register
+            client = plugin.send(:client)
+            auth_header = client.transport.options[:transport_options][:headers]['Authorization']
+
+            expect(auth_header).to eql("ApiKey #{api_key_value}")
+          end
+        end
+
+        context "with an unrecognized api_key format" do
+          let(:api_key_value) { "not-a-valid-key" }
+
+          it "fails registration with a configuration error" do
+            expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Invalid api_key format/)
+          end
+        end
+
         context 'user also set' do
           let(:config) { super().merge({ 'api_key' => 'foo:bar', 'user' => 'another' }) }
 

data/version

@@ -0,0 +1 @@
+5.3.2

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.3.0
+  version: 5.3.2
 platform: ruby
 authors:
 - Elastic
 bindir: bin
 cert_chain: []
-date: 1980-01-02 00:00:00.000000000 Z
+date: 2026-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -283,6 +283,7 @@ files:
 - spec/inputs/integration/elasticsearch_esql_spec.rb
 - spec/inputs/integration/elasticsearch_spec.rb
 - spec/inputs/paginated_search_spec.rb
+- version
 homepage: https://elastic.co/logstash
 licenses:
 - Apache License (2.0)
@@ -303,7 +304,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.7.2
+rubygems_version: 3.6.3
 specification_version: 4
 summary: Reads query results from an Elasticsearch cluster
 test_files: