logstash-output-elasticsearch 12.1.3-java → 12.1.6-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 295ebf9ba0c0c79766056646cba92ea6f037d2adf61379a53c9f7d080de34c1d
-  data.tar.gz: 26e45497d1b2e77894ebd0ccdd9b3f55492f18a2932ef938dabd95065007f37c
+  metadata.gz: 1cdbd34639c856955089e567feefe731b38ca8c6990207f5eb23bf4b82e7b655
+  data.tar.gz: 6f6faff70fc33714ae9ae49b44d046abcbf45d51cf3d986d03580d63f104d859
 SHA512:
-  metadata.gz: 8c1afa9504cb9ec4a7e15aab9068456185cf54d2b1bd7b323c43462a721747a2aeb329ed68e0d3e519a7322554c155e1c32b4501ff854558c25bea3eed612c82
-  data.tar.gz: 2c617369a30a0d485d510ad460e9b6a4e55b8d7265795e24129a6472c7d688de0ef9562238f7bda0621cad54277d65199e6826ce8f76f70b94e057e7671f237a
+  metadata.gz: c78e7bb0162c33b4ec4ae39ceb9a9fe16f16a8f3c3ef6cda426ceaf9adf2a5d415bea2398a1d2e20ec414dab258c3d95135666f3dbfffa532a455c3ffeb91f32
+  data.tar.gz: 620d9d85ab8425dffbdc89f8dfa45d5b65b741ab29aca914d6039ea9619e1706893a2b6dc4b7f301a3a631f4842ce1681c3cbb024352c6e31f0429b4118cee16

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 12.1.6
+  - Fix serverless compatibility: nil params in pool requests and unsupported template settings [#1276](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1276)
+
+## 12.1.5
+  - Support Elastic Cloud API keys in the `api_key` option, which now accepts an `id:api_key` pair, its base64-encoded form, or an `essu_` Cloud API key, and rejects an unrecognized format at startup [#1274](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1274)
+
+## 12.1.4
+  - [Doc] Add note for index option [#1269](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1269)
+
 ## 12.1.3
  - Improves the logging experience when DLQ used [#1253](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1253)
 

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 
-[![Unit Tests](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/tests.yml/badge.svg)](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/tests.yml)
+[![Unit Tests](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/unit-tests.yml/badge.svg)](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/unit-tests.yml)
 [![Secure Integration Tests](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/secure-integration-tests.yml/badge.svg)](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/secure-integration-tests.yml)
 [![Integration Tests](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/integration-tests.yml/badge.svg)](https://github.com/logstash-plugins/logstash-output-elasticsearch/actions/workflows/integration-tests.yml)
 

data/docs/index.asciidoc

@@ -467,8 +467,11 @@ For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bu
 Authenticate using Elasticsearch API key.
 Note that this option also requires SSL/TLS, which can be enabled by supplying a <<plugins-{type}s-{plugin}-cloud_id>>, a list of HTTPS <<plugins-{type}s-{plugin}-hosts>>, or by setting <<plugins-{type}s-{plugin}-ssl_enabled,`ssl_enabled => true`>>.
 
-Format is `id:api_key` where `id` and `api_key` are as returned by the
-Elasticsearch {ref}/security-api-create-api-key.html[Create API key API].
+The format is `id:api_key`, where `id` and `api_key` are as returned by the
+Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. The
+base64-encoded form of that pair is also accepted, as is an
+https://www.elastic.co/docs/deploy-manage/api-keys/elastic-cloud-api-keys[Elastic Cloud API key]
+(prefixed with `essu_`), which is used as-is.
 
 [id="plugins-{type}s-{plugin}-bulk_path"]
 ===== `bulk_path`
@@ -842,10 +845,15 @@ NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as
 The indexing target to write events to. 
 Can point to an {ref}/index-mgmt.html[index], {ref}/aliases.html[alias], or {ref}/data-streams.html[data stream]. 
 This can be dynamic using the `%{foo}` syntax.
+
+NOTE: When using dynamic syntax, verify that the resolved index names align with your cluster index templates and Index Lifecycle Management (ILM) configuration.
+
+WARNING: Date math (e.g. `%{+yyyy.MM.dd}`) is not recommended for use with aliases or data streams.
+
 The default value will partition your indices by day so you can more easily
 delete old data or only search specific date ranges.
 Indexes may not contain uppercase characters.
-For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
+For weekly indexes ISO 8601 format is recommended, eg. `logstash-%{+xxxx.ww}`.
 Logstash uses
 http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[Joda
 formats] and the `@timestamp` field of each event is being used as source for the date.

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -350,6 +350,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def perform_request_to_url(url, method, path, params={}, body=nil)
+      params ||= {}
       params[:headers] = DEFAULT_EAV_HEADER.merge(params[:headers] || {}) if serverless?
       @adapter.perform_request(url, method, path, params, body)
     end

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -433,7 +433,7 @@ module LogStash; module Outputs; class ElasticSearch;
     def template_put(template_endpoint, name, template)
       path = "#{template_endpoint}/#{name}"
       logger.info("Installing Elasticsearch template", name: name)
-      @pool.put(path, nil, LogStash::Json.dump(template))
+      @pool.put(path, {}, LogStash::Json.dump(template))
     rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
       raise e unless e.response_code == 404
     end
@@ -447,7 +447,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
     # Create a new rollover alias
     def rollover_alias_put(alias_name, alias_definition)
-      @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition))
+      @pool.put(CGI::escape(alias_name), {}, LogStash::Json.dump(alias_definition))
       logger.info("Created rollover alias", name: alias_name)
       # If the rollover alias already exists, ignore the error that comes back from Elasticsearch
     rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
@@ -473,7 +473,7 @@ module LogStash; module Outputs; class ElasticSearch;
     def ilm_policy_put(name, policy)
       path = "_ilm/policy/#{name}"
       logger.info("Installing ILM policy #{policy}", name: name)
-      @pool.put(path, nil, LogStash::Json.dump(policy))
+      @pool.put(path, {}, LogStash::Json.dump(policy))
     end
 
 

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -190,8 +190,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
       return {} unless (api_key&.value)
 
-      value = is_base64?(api_key.value) ?  api_key.value : Base64.strict_encode64(api_key.value)
-
+      value = resolve_api_key(api_key.value)
       { "Authorization" => "ApiKey #{value}" }
     end
 
@@ -212,6 +211,28 @@ module LogStash; module Outputs; class ElasticSearch;
         url.match?(/\?[^\s#]+/) ? '&' : '?'
       end
 
+      # Resolves the `api_key` value into the credential used in the
+      # `Authorization: ApiKey` header. An already base64-encoded key and an
+      # Elastic Cloud API key are used as-is; a raw `id:api_key` pair is
+      # base64-encoded. An unrecognized value is rejected so a malformed key
+      # surfaces at startup rather than as a later authentication failure.
+      def resolve_api_key(key_value)
+        if is_base64?(key_value) || cloud_api_key?(key_value)
+          key_value
+        elsif key_value.match?(/\A[^:]+:[^:]+\z/)
+          Base64.strict_encode64(key_value)
+        else
+          raise LogStash::ConfigurationError, "Invalid api_key format. Expected a base64-encoded key, an 'id:api_key' pair, or a Cloud API key (essu_ prefix)."
+        end
+      end
+
+      # Elastic Cloud API keys (such as the unified Serverless keys) are opaque
+      # tokens prefixed with `essu_` that Elasticsearch accepts verbatim in the
+      # `Authorization: ApiKey` header, with no base64 encoding.
+      def cloud_api_key?(string)
+        string.match?(/\Aessu_.+/)
+      end
+
       def is_base64?(string)
         begin
           string == Base64.strict_encode64(Base64.strict_decode64(string))

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -28,6 +28,7 @@ module LogStash; module Outputs; class ElasticSearch
       end
 
       add_ilm_settings_to_template(plugin, template) if plugin.ilm_in_use?
+      strip_serverless_incompatible_settings(template) if plugin.serverless?
       plugin.logger.debug("Attempting to install template", template: template)
       install(plugin.client, template_endpoint(plugin), template_name(plugin), template, plugin.template_overwrite)
     end
@@ -129,5 +130,19 @@ module LogStash; module Outputs; class ElasticSearch
       end
     end
 
+    SERVERLESS_INCOMPATIBLE_SETTINGS = %w[
+      number_of_shards
+      index.number_of_shards
+      number_of_replicas
+      index.number_of_replicas
+    ].freeze
+
+    def self.strip_serverless_incompatible_settings(template)
+      settings = template.dig('template', 'settings') || template['settings']
+      return unless settings
+
+      SERVERLESS_INCOMPATIBLE_SETTINGS.each { |s| settings.delete(s) }
+    end
+
   end
 end end end

data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb

@@ -16,7 +16,8 @@ module LogStash; module PluginMixins; module ElasticSearch
         :password => { :validate => :password },
 
         # Authenticate using Elasticsearch API key.
-        # format is id:api_key (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key])
+        # Format is either the `id:api_key` pair (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key]),
+        # its base64-encoded form, or an https://www.elastic.co/docs/deploy-manage/api-keys/elastic-cloud-api-keys[Elastic Cloud API key] (prefixed with `essu_`) can be used.
         :api_key => { :validate => :password },
 
         # Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '12.1.3'
+  s.version         = ::File.read('version').split("\n").first
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
   s.platform = RUBY_PLATFORM
 
   # Files
-  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "version", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

data/spec/unit/http_client_builder_spec.rb

@@ -62,6 +62,40 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
         expect(api_key_header["Authorization"]).to eql(expected)
       end
     end
+
+    context "when api-key is an Elastic Cloud API key (essu_ prefix)" do
+      # The suffix is intentionally not canonical base64: a Cloud key is opaque
+      # and must be forwarded verbatim regardless of its payload encoding.
+      let(:api_key) { "essu_VFZGblZreFhTekJ4ZDB4M2NHUnZRMEU2YzNWd1pYSnpaV055WlhRPQ==AAAAAAAA" }
+      let(:api_key_secured) do
+        secured = double("api_key")
+        allow(secured).to receive(:value).and_return(api_key)
+        secured
+      end
+      let(:options) { { "api_key" => api_key_secured } }
+      let(:logger) { double("logger") }
+      let(:api_key_header) { klass.setup_api_key(logger, options) }
+
+      it "returns the cloud api-key header verbatim without re-encoding" do
+        expected = "ApiKey #{api_key}"
+        expect(api_key_header["Authorization"]).to eql(expected)
+      end
+    end
+
+    context "when api-key has an unrecognized format" do
+      let(:api_key) { "not-a-valid-key" }
+      let(:api_key_secured) do
+        secured = double("api_key")
+        allow(secured).to receive(:value).and_return(api_key)
+        secured
+      end
+      let(:options) { { "api_key" => api_key_secured } }
+      let(:logger) { double("logger") }
+
+      it "raises a configuration error" do
+        expect { klass.setup_api_key(logger, options) }.to raise_error(LogStash::ConfigurationError, /Invalid api_key format/)
+      end
+    end
   end
 
   describe "customizing action paths" do

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -181,4 +181,75 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
 
     end
   end
+
+  describe ".strip_serverless_incompatible_settings" do
+    context "with composable template (template.settings)" do
+      let(:template) do
+        {
+          "template" => {
+            "settings" => {
+              "index.refresh_interval" => "5s",
+              "number_of_shards" => 1,
+              "number_of_replicas" => 0
+            }
+          }
+        }
+      end
+
+      it "removes number_of_shards and number_of_replicas" do
+        described_class.strip_serverless_incompatible_settings(template)
+        expect(template["template"]["settings"]).not_to have_key("number_of_shards")
+        expect(template["template"]["settings"]).not_to have_key("number_of_replicas")
+      end
+
+      it "preserves compatible settings" do
+        described_class.strip_serverless_incompatible_settings(template)
+        expect(template["template"]["settings"]["index.refresh_interval"]).to eq("5s")
+      end
+    end
+
+    context "with index-prefixed setting keys" do
+      let(:template) do
+        {
+          "template" => {
+            "settings" => {
+              "index.number_of_shards" => 1,
+              "index.number_of_replicas" => 0
+            }
+          }
+        }
+      end
+
+      it "removes index-prefixed variants" do
+        described_class.strip_serverless_incompatible_settings(template)
+        expect(template["template"]["settings"]).not_to have_key("index.number_of_shards")
+        expect(template["template"]["settings"]).not_to have_key("index.number_of_replicas")
+      end
+    end
+
+    context "with legacy template (top-level settings)" do
+      let(:template) do
+        {
+          "settings" => {
+            "index.refresh_interval" => "5s",
+            "number_of_shards" => 1
+          }
+        }
+      end
+
+      it "removes incompatible settings from top-level settings" do
+        described_class.strip_serverless_incompatible_settings(template)
+        expect(template["settings"]).not_to have_key("number_of_shards")
+        expect(template["settings"]["index.refresh_interval"]).to eq("5s")
+      end
+    end
+
+    context "with no settings" do
+      let(:template) { { "mappings" => {} } }
+
+      it "does not raise" do
+        expect { described_class.strip_serverless_incompatible_settings(template) }.not_to raise_error
+      end
+    end
+  end
 end

data/version

@@ -0,0 +1 @@
+12.1.6

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 12.1.3
+  version: 12.1.6
 platform: java
 authors:
 - Elastic
 bindir: bin
 cert_chain: []
-date: 2026-04-17 00:00:00.000000000 Z
+date: 2026-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: manticore
@@ -334,6 +334,7 @@ files:
 - spec/unit/outputs/elasticsearch_ssl_spec.rb
 - spec/unit/outputs/error_whitelist_spec.rb
 - spec/unit/outputs/license_check_spec.rb
+- version
 homepage: https://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - apache-2.0