logstash-output-elasticsearch 11.3.1-java → 11.4.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dccee6612b454d47b0596a9feb04db71f51f368a34906f559b8f7a8df7cf166e
-  data.tar.gz: 27d8ab736dd7c3ffa46d1229ad70afd5e8b23813ee09aa3e9d4ec05b4f263a76
+  metadata.gz: c1ce0504aa5082dc3fc958b72019f243c6cad0174bb883612c25eb5d7f8aedfe
+  data.tar.gz: 4d28c34d89b5a70a02c10ebd3b8d425111d3902bc58fb9e428acadee4a471646
 SHA512:
-  metadata.gz: d7ab3e32387faf0cc2680fc1446e9a330314d2ead466487fd6f63a27ab2757cac8c2d90cb41bb0eba422986b22821d0e0c813617ba153225058b1e1d09765598
-  data.tar.gz: df0a5b6cbcbdc592e1c18906f6a54a97d309afbbd1c36529f812501df31f78c7e4c2ba2fb8a3eab72e9fd53dfbca8388b710dc772a20ba9084c59ce1f5aa43fc
+  metadata.gz: d1d0680cf639e01b6c2a2a3b8eb87a2ad525764d006fb3cbfadc06c845925d9f1055075265d7327794e326b1a9df67308d85ebefafb38297bd2121499ecc2279
+  data.tar.gz: 285bafc2ffa2ae726bdafdb3176c193c7cd20979625e59cd7526445ae5243137e6dad37cc64208bb3494fc45afe44d913f7b7b4188fcf6b25faf1e9fb6a04d06

data/CHANGELOG.md

@@ -1,3 +1,20 @@
+## 11.4.1
+ - Feat: upgrade manticore (http-client) library [#1063](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1063)
+   - the underlying changes include latest HttpClient (4.5.13)
+   - resolves an old issue with `ssl_certificate_verification => false` still doing some verification logic
+
+## 11.4.0
+ - Updates ECS templates [#1062](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1062)
+   - Updates v1 templates to 1.12.1 for use with Elasticsearch 7.x and 8.x
+   - Updates BETA preview of ECS v8 templates for Elasticsearch 7.x and 8.x
+
+## 11.3.3
+ - Feat: add support for 'traces' data stream type [#1057](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1057)
+
+## 11.3.2
+ - Refactor: review manticore error handling/logging, logging originating cause in case of connection related error when debug level is enabled [#1029](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1029)
+   - Java causes on connection related exceptions will now be extra logged when plugin is logging at debug level
+
 ## 11.3.1
  - ECS-related fixes [#1046](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1046)
    - Data Streams requirement on ECS is properly enforced when running on Logstash 8, and warned about when running on Logstash 7.
@@ -9,10 +26,10 @@
    - Adds templates for BETA preview of ECS v8 for both Elasticsearch 7.x and 8.x
 
 ## 11.2.3
- - Downgrade ECS templates, pinning to v1.10.0 of upstream; fixes an issue where ECS templates cannot be installed in Elasticsearch 6.x or 7.1-7.2, since the generated templates include fields of `type: flattened` that was introduced in Elasticsearch 7.3
+ - Downgrade ECS templates, pinning to v1.10.0 of upstream; fixes an issue where ECS templates cannot be installed in Elasticsearch 6.x or 7.1-7.2, since the generated templates include fields of `type: flattened` that was introduced in Elasticsearch 7.3. [#1049](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1049)
 
 ## 11.2.2
- - Update ECS templates from upstream; `ecs_compatiblity => v1` now resolves to templates for ECS v1.12.1 [#1027](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/1027)
+ - Update ECS templates from upstream; `ecs_compatiblity => v1` now resolves to templates for ECS v1.12.1 [#1047](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1047). Fixes [#1027](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/1027)
 
 ## 11.2.1
  - Fix referencing Gem classes from global lexical scope [#1044](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1044)

data/Gemfile

@@ -12,4 +12,5 @@ end
 
 if RUBY_VERSION == "1.9.3"
   gem 'rake', '12.2.1'
-end
+end
+

data/docs/index.asciidoc

@@ -117,12 +117,8 @@ output {
 
 ==== Writing to different indices: best practices
 
-[NOTE]
-================================================================================
-You cannot use dynamic variable substitution when `ilm_enabled` is `true` and
-when using `ilm_rollover_alias`.
-
-================================================================================
+NOTE: You cannot use dynamic variable substitution when `ilm_enabled` is `true` 
+and when using `ilm_rollover_alias`.
 
 If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can:
 
@@ -509,7 +505,7 @@ overwritten with a warning.
 * Default value is `logs`.
 
 The data stream type used to construct the data stream at index time.
-Currently, only `logs`, `metrics` and `synthetics` are supported.
+Currently, only `logs`, `metrics`, `synthetics` and `traces` are supported.
 
 [id="plugins-{type}s-{plugin}-doc_as_upsert"]
 ===== `doc_as_upsert`

data/lib/logstash/outputs/elasticsearch/data_stream_support.rb

@@ -18,7 +18,7 @@ module LogStash module Outputs class ElasticSearch
       # Defaults to `false` in Logstash 7.x and `auto` starting in Logstash 8.0.
       base.config :data_stream, :validate => ['true', 'false', 'auto']
 
-      base.config :data_stream_type, :validate => ['logs', 'metrics', 'synthetics'], :default => 'logs'
+      base.config :data_stream_type, :validate => ['logs', 'metrics', 'synthetics', 'traces'], :default => 'logs'
       base.config :data_stream_dataset, :validate => :dataset_identifier, :default => 'generic'
       base.config :data_stream_namespace, :validate => :namespace_identifier, :default => 'default'
 

data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -7,9 +7,9 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
   class ManticoreAdapter
     attr_reader :manticore, :logger
 
-    def initialize(logger, options={})
+    def initialize(logger, options)
       @logger = logger
-      options = options.clone || {}
+      options = options.dup
       options[:ssl] = options[:ssl] || {}
 
       # We manage our own retries directly, so let's disable them here
@@ -66,23 +66,53 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
 
       request_uri = format_url(url, path)
       request_uri_as_string = remove_double_escaping(request_uri.to_s)
-      resp = @manticore.send(method.downcase, request_uri_as_string, params)
-
-      # Manticore returns lazy responses by default
-      # We want to block for our usage, this will wait for the repsonse
-      # to finish
-      resp.call
+      begin
+        resp = @manticore.send(method.downcase, request_uri_as_string, params)
+        # Manticore returns lazy responses by default
+        # We want to block for our usage, this will wait for the response to finish
+        resp.call
+      rescue ::Manticore::ManticoreException => e
+        log_request_error(e)
+        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new(e, request_uri_as_string)
+      end
 
       # 404s are excluded because they are valid codes in the case of
       # template installation. We might need a better story around this later
       # but for our current purposes this is correct
-      if resp.code < 200 || resp.code > 299 && resp.code != 404
-        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(resp.code, request_uri, body, resp.body)
+      code = resp.code
+      if code < 200 || code > 299 && code != 404
+        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(code, request_uri, body, resp.body)
       end
 
       resp
     end
 
+    def log_request_error(e)
+      details = { message: e.message, exception: e.class }
+      details[:cause] = e.cause if e.respond_to?(:cause)
+      details[:backtrace] = e.backtrace if @logger.debug?
+
+      level = case e
+      when ::Manticore::Timeout
+        :debug
+      when ::Manticore::UnknownException
+        :warn
+      else
+        :info
+      end
+
+      @logger.send level, "Failed to perform request", details
+      log_java_exception(details[:cause], :debug) if details[:cause] && @logger.debug?
+    end
+
+    def log_java_exception(e, level = :debug)
+      return unless e.is_a?(java.lang.Exception)
+      # @logger.name using the same convention as LS does
+      logger = self.class.name.gsub('::', '.').downcase
+      logger = org.apache.logging.log4j.LogManager.getLogger(logger)
+      logger.send(level, '', e) # logger.error('', e) - prints nested causes
+    end
+
     # Returned urls from this method should be checked for double escaping.
     def format_url(url, path_and_query=nil)
       request_uri = url.clone
@@ -96,9 +126,6 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
 
       parsed_path_and_query = java.net.URI.new(path_and_query)
 
-      query = request_uri.query
-      parsed_query = parsed_path_and_query.query
-
       new_query_parts = [request_uri.query, parsed_path_and_query.query].select do |part|
         part && !part.empty? # Skip empty nil and ""
       end
@@ -124,8 +151,5 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @manticore.close
     end
 
-    def host_unreachable_exceptions
-      [::Manticore::Timeout,::Manticore::SocketException, ::Manticore::ClientProtocolException, ::Manticore::ResolutionFailure, Manticore::SocketTimeout]
-    end
   end
 end; end; end; end

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -8,27 +8,25 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       attr_reader :url, :response_code, :request_body, :response_body
 
       def initialize(response_code, url, request_body, response_body)
+        super("Got response code '#{response_code}' contacting Elasticsearch at URL '#{url}'")
+
         @response_code = response_code
         @url = url
         @request_body = request_body
         @response_body = response_body
       end
 
-      def message
-        "Got response code '#{response_code}' contacting Elasticsearch at URL '#{@url}'"
-      end
     end
     class HostUnreachableError < Error;
       attr_reader :original_error, :url
 
       def initialize(original_error, url)
+        super("Elasticsearch Unreachable: [#{url}][#{original_error.class}] #{original_error.message}")
+
         @original_error = original_error
         @url = url
       end
 
-      def message
-        "Elasticsearch Unreachable: [#{@url}][#{original_error.class}] #{original_error.message}"
-      end
     end
 
     attr_reader :logger, :adapter, :sniffing, :sniffer_delay, :resurrect_delay, :healthcheck_path, :sniffing_path, :bulk_path
@@ -323,9 +321,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def perform_request_to_url(url, method, path, params={}, body=nil)
-      res = @adapter.perform_request(url, method, path, params, body)
-    rescue *@adapter.host_unreachable_exceptions => e
-      raise HostUnreachableError.new(e, url), "Could not reach host #{e.class}: #{e.message}"
+      @adapter.perform_request(url, method, path, params, body)
     end
 
     def normalize_url(uri)

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -322,8 +322,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
       adapter_options[:headers] = client_settings[:headers] if client_settings[:headers]
 
-      adapter_class = ::LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter
-      adapter = adapter_class.new(@logger, adapter_options)
+      ::LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(@logger, adapter_options)
     end
 
     def prepare_user_agent

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -133,12 +133,9 @@ module LogStash; module Outputs; class ElasticSearch;
         ssl_options[:keystore_password] = keystore_password.value if keystore_password
       end
       if !params["ssl_certificate_verification"]
-        logger.warn [
-                       "** WARNING ** Detected UNSAFE options in elasticsearch output configuration!",
-                       "** WARNING ** You have enabled encryption but DISABLED certificate verification.",
-                       "** WARNING ** To make sure your data is secure change :ssl_certificate_verification to true"
-                     ].join("\n")
-        ssl_options[:verify] = false
+        logger.warn "You have enabled encryption but DISABLED certificate verification, " +
+                    "to make sure your data is secure remove `ssl_certificate_verification => false`"
+        ssl_options[:verify] = :disable # false accepts self-signed but still validates hostname
       end
       { ssl: ssl_options }
     end