logstash-output-elasticsearch 11.3.1-java → 11.3.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dccee6612b454d47b0596a9feb04db71f51f368a34906f559b8f7a8df7cf166e
-  data.tar.gz: 27d8ab736dd7c3ffa46d1229ad70afd5e8b23813ee09aa3e9d4ec05b4f263a76
+  metadata.gz: b716e89a4ab884412f633b3c0e745ab787006466a44b09ca0c20c45410603eea
+  data.tar.gz: '09387e7905fcb56e033cf3b9d8c2db7df3f7d11d8e4dc8e638d3e9ff570f4231'
 SHA512:
-  metadata.gz: d7ab3e32387faf0cc2680fc1446e9a330314d2ead466487fd6f63a27ab2757cac8c2d90cb41bb0eba422986b22821d0e0c813617ba153225058b1e1d09765598
-  data.tar.gz: df0a5b6cbcbdc592e1c18906f6a54a97d309afbbd1c36529f812501df31f78c7e4c2ba2fb8a3eab72e9fd53dfbca8388b710dc772a20ba9084c59ce1f5aa43fc
+  metadata.gz: df0e1ef992ead1c347ef7d528309c732092037186310623aa2b538e086c6eea5cba24bb9aa354d7deeb1718904ff1965a40b3bed2d4c5597c14a8d3f64276a58
+  data.tar.gz: bec8153c3242e4de19410eac06f8e1ee716df545a0d0a2d49b69023901a8b928315409d863581a16f3fb1feb6648ec623ef5f35faf6f707fea64cceddf136e6b

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 11.3.2
+ - Refactor: review manticore error handling/logging, logging originating cause in case of connection related error when debug level is enabled [#1029](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1029)
+   - Java causes on connection related exceptions will now be extra logged when plugin is logging at debug level
+
 ## 11.3.1
  - ECS-related fixes [#1046](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1046)
    - Data Streams requirement on ECS is properly enforced when running on Logstash 8, and warned about when running on Logstash 7.
@@ -9,10 +13,10 @@
    - Adds templates for BETA preview of ECS v8 for both Elasticsearch 7.x and 8.x
 
 ## 11.2.3
- - Downgrade ECS templates, pinning to v1.10.0 of upstream; fixes an issue where ECS templates cannot be installed in Elasticsearch 6.x or 7.1-7.2, since the generated templates include fields of `type: flattened` that was introduced in Elasticsearch 7.3
+ - Downgrade ECS templates, pinning to v1.10.0 of upstream; fixes an issue where ECS templates cannot be installed in Elasticsearch 6.x or 7.1-7.2, since the generated templates include fields of `type: flattened` that was introduced in Elasticsearch 7.3. [#1049](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1049)
 
 ## 11.2.2
- - Update ECS templates from upstream; `ecs_compatiblity => v1` now resolves to templates for ECS v1.12.1 [#1027](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/1027)
+ - Update ECS templates from upstream; `ecs_compatiblity => v1` now resolves to templates for ECS v1.12.1 [#1047](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1047). Fixes [#1027](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/1027)
 
 ## 11.2.1
  - Fix referencing Gem classes from global lexical scope [#1044](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1044)

data/docs/index.asciidoc

@@ -117,12 +117,8 @@ output {
 
 ==== Writing to different indices: best practices
 
-[NOTE]
-================================================================================
-You cannot use dynamic variable substitution when `ilm_enabled` is `true` and
-when using `ilm_rollover_alias`.
-
-================================================================================
+NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` 
+and when using `ilm_rollover_alias`.
 
 If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can:
 

data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -7,9 +7,9 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
   class ManticoreAdapter
     attr_reader :manticore, :logger
 
-    def initialize(logger, options={})
+    def initialize(logger, options)
       @logger = logger
-      options = options.clone || {}
+      options = options.dup
       options[:ssl] = options[:ssl] || {}
 
       # We manage our own retries directly, so let's disable them here
@@ -66,23 +66,53 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
 
       request_uri = format_url(url, path)
       request_uri_as_string = remove_double_escaping(request_uri.to_s)
-      resp = @manticore.send(method.downcase, request_uri_as_string, params)
-
-      # Manticore returns lazy responses by default
-      # We want to block for our usage, this will wait for the repsonse
-      # to finish
-      resp.call
+      begin
+        resp = @manticore.send(method.downcase, request_uri_as_string, params)
+        # Manticore returns lazy responses by default
+        # We want to block for our usage, this will wait for the response to finish
+        resp.call
+      rescue ::Manticore::ManticoreException => e
+        log_request_error(e)
+        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new(e, request_uri_as_string)
+      end
 
       # 404s are excluded because they are valid codes in the case of
       # template installation. We might need a better story around this later
       # but for our current purposes this is correct
-      if resp.code < 200 || resp.code > 299 && resp.code != 404
-        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(resp.code, request_uri, body, resp.body)
+      code = resp.code
+      if code < 200 || code > 299 && code != 404
+        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(code, request_uri, body, resp.body)
       end
 
       resp
     end
 
+    def log_request_error(e)
+      details = { message: e.message, exception: e.class }
+      details[:cause] = e.cause if e.respond_to?(:cause)
+      details[:backtrace] = e.backtrace if @logger.debug?
+
+      level = case e
+      when ::Manticore::Timeout
+        :debug
+      when ::Manticore::UnknownException
+        :warn
+      else
+        :info
+      end
+
+      @logger.send level, "Failed to perform request", details
+      log_java_exception(details[:cause], :debug) if details[:cause] && @logger.debug?
+    end
+
+    def log_java_exception(e, level = :debug)
+      return unless e.is_a?(java.lang.Exception)
+      # @logger.name using the same convention as LS does
+      logger = self.class.name.gsub('::', '.').downcase
+      logger = org.apache.logging.log4j.LogManager.getLogger(logger)
+      logger.send(level, '', e) # logger.error('', e) - prints nested causes
+    end
+
     # Returned urls from this method should be checked for double escaping.
     def format_url(url, path_and_query=nil)
       request_uri = url.clone
@@ -96,9 +126,6 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
 
       parsed_path_and_query = java.net.URI.new(path_and_query)
 
-      query = request_uri.query
-      parsed_query = parsed_path_and_query.query
-
       new_query_parts = [request_uri.query, parsed_path_and_query.query].select do |part|
         part && !part.empty? # Skip empty nil and ""
       end
@@ -124,8 +151,5 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @manticore.close
     end
 
-    def host_unreachable_exceptions
-      [::Manticore::Timeout,::Manticore::SocketException, ::Manticore::ClientProtocolException, ::Manticore::ResolutionFailure, Manticore::SocketTimeout]
-    end
   end
 end; end; end; end

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -8,27 +8,25 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       attr_reader :url, :response_code, :request_body, :response_body
 
       def initialize(response_code, url, request_body, response_body)
+        super("Got response code '#{response_code}' contacting Elasticsearch at URL '#{url}'")
+
         @response_code = response_code
         @url = url
         @request_body = request_body
         @response_body = response_body
       end
 
-      def message
-        "Got response code '#{response_code}' contacting Elasticsearch at URL '#{@url}'"
-      end
     end
     class HostUnreachableError < Error;
       attr_reader :original_error, :url
 
       def initialize(original_error, url)
+        super("Elasticsearch Unreachable: [#{url}][#{original_error.class}] #{original_error.message}")
+
         @original_error = original_error
         @url = url
       end
 
-      def message
-        "Elasticsearch Unreachable: [#{@url}][#{original_error.class}] #{original_error.message}"
-      end
     end
 
     attr_reader :logger, :adapter, :sniffing, :sniffer_delay, :resurrect_delay, :healthcheck_path, :sniffing_path, :bulk_path
@@ -323,9 +321,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def perform_request_to_url(url, method, path, params={}, body=nil)
-      res = @adapter.perform_request(url, method, path, params, body)
-    rescue *@adapter.host_unreachable_exceptions => e
-      raise HostUnreachableError.new(e, url), "Could not reach host #{e.class}: #{e.message}"
+      @adapter.perform_request(url, method, path, params, body)
     end
 
     def normalize_url(uri)

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -322,8 +322,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
       adapter_options[:headers] = client_settings[:headers] if client_settings[:headers]
 
-      adapter_class = ::LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter
-      adapter = adapter_class.new(@logger, adapter_options)
+      ::LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(@logger, adapter_options)
     end
 
     def prepare_user_agent

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.3.1'
+  s.version         = '11.3.2'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"

data/spec/integration/outputs/sniffer_spec.rb

@@ -5,7 +5,7 @@ require "socket"
 
 describe "pool sniffer", :integration => true do
   let(:logger) { Cabin::Channel.get }
-  let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger) }
+  let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger, {}) }
   let(:es_host) { get_host_port.split(":").first }
   let(:es_port) { get_host_port.split(":").last }
   let(:es_ip) { IPSocket.getaddress(es_host) }

data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb

@@ -10,11 +10,12 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
 
   it "should raise an exception if requests are issued after close" do
     subject.close
-    expect { subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, '/') }.to raise_error(::Manticore::ClientStoppedException)
-  end
-
-  it "should implement host unreachable exceptions" do
-    expect(subject.host_unreachable_exceptions).to be_a(Array)
+    begin
+      subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, '/')
+      fail 'expected to raise a HostUnreachableError'
+    rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError => e
+      expect( e.original_error ).to be_a ::Manticore::ClientStoppedException
+    end
   end
   
   describe "auth" do

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -4,7 +4,7 @@ require 'cabin'
 
 describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   let(:logger) { Cabin::Channel.get }
-  let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger) }
+  let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger, {}) }
   let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
   let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
   let(:es_node_versions) { [ "0.0.0" ] }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.3.1
+  version: 11.3.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-05 00:00:00.000000000 Z
+date: 2021-12-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement