logstash-output-elasticsearch 11.22.10-java → 11.22.12-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1ce26f1744954843bcf5738f6ea07c57ecde6a2b5e63e827d9117bffeb8d2857
-  data.tar.gz: cb9306e7fa13d3aeb1edf089035047f1db95debd5490675ccaabf28771dabbb6
+  metadata.gz: b9a7b78791991372bdd4cbeb0f1d6c5f6d9f7a4db88b599ae090f09ecf548377
+  data.tar.gz: 169225b31802647c4e00070e465d24dc0222ad3d1095fdb47c37db3cab64048e
 SHA512:
-  metadata.gz: 7a1b4dfe4d8788794073b9cda092cad34d8a0a3f5a28e9b7ed77de1c4a88dc668c7db3c88279bab9c31d8a33f1ebe60eb70136d3a5a8c5d378cc9f0dd53d7c6b
-  data.tar.gz: b59c8d820a59cc7ef5c33d434714077568cb6fe20651d027893054f6c6bd0d89f6eeef6989313613a65f23cdab99abbf9594ea2ecd4cb4dc4f5edbed25733d98
+  metadata.gz: 79ebc09c5483def4e6104b90f01bca4780f499cacc4c598e19e773900f9f7dde048242f5302f26798aefbf9e85e84555f23b9d3cbce1d862292c25723d1dc10f
+  data.tar.gz: 369dc07aa0a5474cd34feda727013fc49a7013c39ef0d28ccc586e6eaf268a37097dc650636d4720d89bd0b017ea37a9d15d9036290a80848432c0526c443748

data/CHANGELOG.md

@@ -1,5 +1,10 @@
+## 11.22.12
+ - Properly handle http code 413 (Payload Too Large) [#1199](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1199)
+## 11.22.11
+ - Remove irrelevant log warning about elastic stack version [#1202](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1202)
+
 ## 11.22.10
- - Add `x-elastic-product-origin` header to Elasticsearch requests [#1194](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1194)
+ - Add `x-elastic-product-origin` header to Elasticsearch requests [#1195](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1195)
 
 ## 11.22.9
  - Vendor ECS template for Elasticsearch 9.x in built gem [#1188](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1188)

data/docs/index.asciidoc

@@ -196,7 +196,22 @@ This plugin uses the Elasticsearch bulk API to optimize its imports into Elastic
 either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP
 request are handled differently than error codes for individual documents.
 
-HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely.
+
+HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely,
+including 413 (Payload Too Large) responses.
+
+If you want to handle large payloads differently, you can configure 413 responses to go to the Dead Letter Queue instead:
+
+[source,ruby]
+-----
+output {
+  elasticsearch {
+    hosts => ["localhost:9200"]
+    dlq_custom_codes => [413]  # Send 413 errors to DLQ instead of retrying
+  }
+-----
+
+This will capture oversized payloads in the DLQ for analysis rather than retrying them.
 
 The following document errors are handled as follows:
 

data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -76,11 +76,8 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new(e, request_uri_as_string)
       end
 
-      # 404s are excluded because they are valid codes in the case of
-      # template installation. We might need a better story around this later
-      # but for our current purposes this is correct
       code = resp.code
-      if code < 200 || code > 299 && code != 404
+      if code < 200 || code > 299 # assume anything not 2xx is an error that the layer above needs to interpret
         raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(code, request_uri, body, resp.body)
       end
 

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -253,13 +253,11 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     def health_check_request(url)
       logger.debug("Running health check to see if an Elasticsearch connection is working",
                    :healthcheck_url => url.sanitized.to_s, :path => @healthcheck_path)
-      begin
-        response = perform_request_to_url(url, :head, @healthcheck_path)
-        return response, nil
-      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-        logger.warn("Health check failed", code: e.response_code, url: e.url, message: e.message)
-        return nil, e
-      end
+      response = perform_request_to_url(url, :head, @healthcheck_path)
+      return response, nil
+    rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+      logger.warn("Health check failed", code: e.response_code, url: e.url, message: e.message)
+      return nil, e
     end
 
     def healthcheck!(register_phase = true)
@@ -312,13 +310,11 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def get_root_path(url, params={})
-      begin
-        resp = perform_request_to_url(url, :get, ROOT_URI_PATH, params)
-        return resp, nil
-      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-        logger.warn("Elasticsearch main endpoint returns #{e.response_code}", message: e.message, body: e.response_body)
-        return nil, e
-      end
+      resp = perform_request_to_url(url, :get, ROOT_URI_PATH, params)
+      return resp, nil
+    rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+      logger.warn("Elasticsearch main endpoint returns #{e.response_code}", message: e.message, body: e.response_body)
+      return nil, e
     end
 
     def test_serverless_connection(url, root_response)
@@ -515,20 +511,13 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       major = major_version(version)
       if @maximum_seen_major_version.nil?
         @logger.info("Elasticsearch version determined (#{version})", es_version: major)
-        set_maximum_seen_major_version(major)
+        @maximum_seen_major_version = major
       elsif major > @maximum_seen_major_version
         warn_on_higher_major_version(major, url)
         @maximum_seen_major_version = major
       end
     end
 
-    def set_maximum_seen_major_version(major)
-      if major >= 6
-        @logger.warn("Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type", es_version: major)
-      end
-      @maximum_seen_major_version = major
-    end
-
     def warn_on_higher_major_version(major, url)
       @logger.warn("Detected a node with a higher major version than previously observed, " +
                    "this could be the result of an Elasticsearch cluster upgrade",

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -182,22 +182,20 @@ module LogStash; module Outputs; class ElasticSearch;
     def bulk_send(body_stream, batch_actions)
       params = compression_level? ? {:headers => {"Content-Encoding" => "gzip"}} : {}
 
-      response = @pool.post(@bulk_path, params, body_stream.string)
-
-      @bulk_response_metrics.increment(response.code.to_s)
-
-      case response.code
-      when 200 # OK
-        LogStash::Json.load(response.body)
-      when 413 # Payload Too Large
+      begin
+        response = @pool.post(@bulk_path, params, body_stream.string)
+        @bulk_response_metrics.increment(response.code.to_s)
+      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+        @bulk_response_metrics.increment(e.response_code.to_s)
+        raise e unless e.response_code == 413
+        # special handling for 413, treat it as a document level issue
         logger.warn("Bulk request rejected: `413 Payload Too Large`", :action_count => batch_actions.size, :content_length => body_stream.size)
-        emulate_batch_error_response(batch_actions, response.code, 'payload_too_large')
-      else
-        url = ::LogStash::Util::SafeURI.new(response.final_url)
-        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
-          response.code, url, body_stream.to_s, response.body
-        )
+        return emulate_batch_error_response(batch_actions, 413, 'payload_too_large')
+      rescue => e # it may be a network issue instead, re-raise
+        raise e
       end
+
+      LogStash::Json.load(response.body)
     end
 
     def emulate_batch_error_response(actions, http_code, reason)
@@ -411,6 +409,9 @@ module LogStash; module Outputs; class ElasticSearch;
     def exists?(path, use_get=false)
       response = use_get ? @pool.get(path) : @pool.head(path)
       response.code >= 200 && response.code <= 299
+    rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+      return false if e.response_code == 404
+      raise e
     end
 
     def template_exists?(template_endpoint, name)
@@ -421,6 +422,8 @@ module LogStash; module Outputs; class ElasticSearch;
       path = "#{template_endpoint}/#{name}"
       logger.info("Installing Elasticsearch template", name: name)
       @pool.put(path, nil, LogStash::Json.dump(template))
+    rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+      raise e unless e.response_code == 404
     end
 
     # ILM methods
@@ -432,17 +435,15 @@ module LogStash; module Outputs; class ElasticSearch;
 
     # Create a new rollover alias
     def rollover_alias_put(alias_name, alias_definition)
-      begin
-        @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition))
-        logger.info("Created rollover alias", name: alias_name)
-        # If the rollover alias already exists, ignore the error that comes back from Elasticsearch
-      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-        if e.response_code == 400
-            logger.info("Rollover alias already exists, skipping", name: alias_name)
-            return
-        end
-        raise e
+      @pool.put(CGI::escape(alias_name), nil, LogStash::Json.dump(alias_definition))
+      logger.info("Created rollover alias", name: alias_name)
+      # If the rollover alias already exists, ignore the error that comes back from Elasticsearch
+    rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+      if e.response_code == 400
+        logger.info("Rollover alias already exists, skipping", name: alias_name)
+        return
       end
+      raise e
     end
 
     def get_xpack_info

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.22.10'
+  s.version         = '11.22.12'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -915,7 +915,12 @@ describe LogStash::Outputs::ElasticSearch do
       allow(elasticsearch_output_instance.client.pool).to receive(:post) do |path, params, body|
         if body.length > max_bytes
           max_bytes *= 2 # ensure a successful retry
-          double("Response", :code => 413, :body => "")
+          raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
+            413,
+            "test-url",
+            body,
+            ""
+          )
         else
           double("Response", :code => 200, :body => '{"errors":false,"items":[{"index":{"status":200,"result":"created"}}]}')
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.22.10
+  version: 11.22.12
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-09 00:00:00.000000000 Z
+date: 2025-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement