logstash-output-elasticsearch 10.8.1-java → 11.0.0-java

data/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb

@@ -5,159 +5,163 @@ module LogStash; module PluginMixins; module ElasticSearch
 
     DEFAULT_HOST = ::LogStash::Util::SafeURI.new("//127.0.0.1")
 
-    def self.included(mod)
-      # Username to authenticate to a secure Elasticsearch cluster
-      mod.config :user, :validate => :string
-      # Password to authenticate to a secure Elasticsearch cluster
-      mod.config :password, :validate => :password
-
-      # Authenticate using Elasticsearch API key.
-      # format is id:api_key (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key])
-      mod.config :api_key, :validate => :password
-
-      # Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.
-      #
-      # For more details, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[cloud documentation]
-      mod.config :cloud_auth, :validate => :password
-
-      # The document ID for the index. Useful for overwriting existing entries in
-      # Elasticsearch with the same ID.
-      mod.config :document_id, :validate => :string
-
-      # HTTP Path at which the Elasticsearch server lives. Use this if you must run Elasticsearch behind a proxy that remaps
-      # the root path for the Elasticsearch HTTP API lives.
-      # Note that if you use paths as components of URLs in the 'hosts' field you may
-      # not also set this field. That will raise an error at startup
-      mod.config :path, :validate => :string
-
-      # HTTP Path to perform the _bulk requests to
-      # this defaults to a concatenation of the path parameter and "_bulk"
-      mod.config :bulk_path, :validate => :string
-
-      # Pass a set of key value pairs as the URL query string. This query string is added
-      # to every host listed in the 'hosts' configuration. If the 'hosts' list contains
-      # urls that already have query strings, the one specified here will be appended.
-      mod.config :parameters, :validate => :hash
-
-      # Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
-      # is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
-      # If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
-      mod.config :ssl, :validate => :boolean
-
-      # Option to validate the server's certificate. Disabling this severely compromises security.
-      # For more information on disabling certificate verification please read
-      # https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
-      mod.config :ssl_certificate_verification, :validate => :boolean, :default => true
-
-      # The .cer or .pem file to validate the server's certificate
-      mod.config :cacert, :validate => :path
-
-      # The JKS truststore to validate the server's certificate.
-      # Use either `:truststore` or `:cacert`
-      mod.config :truststore, :validate => :path
-
-      # Set the truststore password
-      mod.config :truststore_password, :validate => :password
-
-      # The keystore used to present a certificate to the server.
-      # It can be either .jks or .p12
-      mod.config :keystore, :validate => :path
-
-      # Set the keystore password
-      mod.config :keystore_password, :validate => :password
-
-      # This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
-      # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
-      # this with master nodes, you probably want to disable HTTP on them by setting
-      # `http.enabled` to false in their elasticsearch.yml. You can either use the `sniffing` option or
-      # manually enter multiple Elasticsearch hosts using the `hosts` parameter.
-      mod.config :sniffing, :validate => :boolean, :default => false
-
-      # How long to wait, in seconds, between sniffing attempts
-      mod.config :sniffing_delay, :validate => :number, :default => 5
-
-      # HTTP Path to be used for the sniffing requests
-      # the default value is computed by concatenating the path value and "_nodes/http"
-      # if sniffing_path is set it will be used as an absolute path
-      # do not use full URL here, only paths, e.g. "/sniff/_nodes/http"
-      mod.config :sniffing_path, :validate => :string
-
-      # Set the address of a forward HTTP proxy.
-      # This used to accept hashes as arguments but now only accepts
-      # arguments of the URI type to prevent leaking credentials.
-      mod.config :proxy, :validate => :uri # but empty string is allowed
-
-      # Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If
-      # a timeout occurs, the request will be retried.
-      mod.config :timeout, :validate => :number, :default => 60
-
-      # Set the Elasticsearch errors in the whitelist that you don't want to log.
-      # A useful example is when you want to skip all 409 errors
-      # which are `document_already_exists_exception`.
-      mod.config :failure_type_logging_whitelist, :validate => :array, :default => []
-
-      # While the output tries to reuse connections efficiently we have a maximum.
-      # This sets the maximum number of open connections the output will create.
-      # Setting this too low may mean frequently closing / opening connections
-      # which is bad.
-      mod.config :pool_max, :validate => :number, :default => 1000
-
-      # While the output tries to reuse connections efficiently we have a maximum per endpoint.
-      # This sets the maximum number of open connections per endpoint the output will create.
-      # Setting this too low may mean frequently closing / opening connections
-      # which is bad.
-      mod.config :pool_max_per_route, :validate => :number, :default => 100
-
-      # HTTP Path where a HEAD request is sent when a backend is marked down
-      # the request is sent in the background to see if it has come back again
-      # before it is once again eligible to service requests.
-      # If you have custom firewall rules you may need to change this
-      mod.config :healthcheck_path, :validate => :string
-
-      # How frequently, in seconds, to wait between resurrection attempts.
-      # Resurrection is the process by which backend endpoints marked 'down' are checked
-      # to see if they have come back to life
-      mod.config :resurrect_delay, :validate => :number, :default => 5
-
-      # How long to wait before checking if the connection is stale before executing a request on a connection using keepalive.
-      # You may want to set this lower, if you get connection errors regularly
-      # Quoting the Apache commons docs (this client is based Apache Commmons):
-      # 'Defines period of inactivity in milliseconds after which persistent connections must
-      # be re-validated prior to being leased to the consumer. Non-positive value passed to
-      # this method disables connection validation. This check helps detect connections that
-      # have become stale (half-closed) while kept inactive in the pool.'
-      # See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info]
-      mod.config :validate_after_inactivity, :validate => :number, :default => 10000
-
-      # Enable gzip compression on requests. Note that response compression is on by default for Elasticsearch v5.0 and beyond
-      mod.config :http_compression, :validate => :boolean, :default => false
-
-      # Custom Headers to send on each request to elasticsearch nodes
-      mod.config :custom_headers, :validate => :hash, :default => {}
-
-      # Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `hosts` parameter.
-      # Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
-      #     `"127.0.0.1"`
-      #     `["127.0.0.1:9200","127.0.0.2:9200"]`
-      #     `["http://127.0.0.1"]`
-      #     `["https://127.0.0.1:9200"]`
-      #     `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath)
-      # It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `hosts` list
-      # to prevent LS from sending bulk requests to the master nodes.  So this parameter should only reference either data or client nodes in Elasticsearch.
-      #
-      # Any special characters present in the URLs here MUST be URL escaped! This means `#` should be put in as `%23` for instance.
-      mod.config :hosts, :validate => :uri, :default => [ DEFAULT_HOST ], :list => true
-
-      # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
-      #
-      # For more details, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[cloud documentation]
-      mod.config :cloud_id, :validate => :string
-
-      # Set initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval`
-      mod.config :retry_initial_interval, :validate => :number, :default => 2
-
-      # Set max interval in seconds between bulk retries.
-      mod.config :retry_max_interval, :validate => :number, :default => 64
+    CONFIG_PARAMS = {
+        # Username to authenticate to a secure Elasticsearch cluster
+        :user => { :validate => :string },
+        # Password to authenticate to a secure Elasticsearch cluster
+        :password => { :validate => :password },
+
+        # Authenticate using Elasticsearch API key.
+        # format is id:api_key (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key])
+        :api_key => { :validate => :password },
+
+        # Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.
+        #
+        # For more details, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[cloud documentation]
+        :cloud_auth => { :validate => :password },
+
+        # The document ID for the index. Useful for overwriting existing entries in
+        # Elasticsearch with the same ID.
+        :document_id => { :validate => :string },
+
+        # HTTP Path at which the Elasticsearch server lives. Use this if you must run Elasticsearch behind a proxy that remaps
+        # the root path for the Elasticsearch HTTP API lives.
+        # Note that if you use paths as components of URLs in the 'hosts' field you may
+        # not also set this field. That will raise an error at startup
+        :path => { :validate => :string },
+
+        # HTTP Path to perform the _bulk requests to
+        # this defaults to a concatenation of the path parameter and "_bulk"
+        :bulk_path => { :validate => :string },
+
+        # Pass a set of key value pairs as the URL query string. This query string is added
+        # to every host listed in the 'hosts' configuration. If the 'hosts' list contains
+        # urls that already have query strings, the one specified here will be appended.
+        :parameters => { :validate => :hash },
+
+        # Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
+        # is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
+        # If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
+        :ssl => { :validate => :boolean },
+
+        # Option to validate the server's certificate. Disabling this severely compromises security.
+        # For more information on disabling certificate verification please read
+        # https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
+        :ssl_certificate_verification => { :validate => :boolean, :default => true },
+
+        # The .cer or .pem file to validate the server's certificate
+        :cacert => { :validate => :path },
+
+        # The JKS truststore to validate the server's certificate.
+        # Use either `:truststore` or `:cacert`
+        :truststore => { :validate => :path },
+
+        # Set the truststore password
+        :truststore_password => { :validate => :password },
+
+        # The keystore used to present a certificate to the server.
+        # It can be either .jks or .p12
+        :keystore => { :validate => :path },
+
+        # Set the keystore password
+        :keystore_password => { :validate => :password },
+
+        # This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
+        # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
+        # this with master nodes, you probably want to disable HTTP on them by setting
+        # `http.enabled` to false in their elasticsearch.yml. You can either use the `sniffing` option or
+        # manually enter multiple Elasticsearch hosts using the `hosts` parameter.
+        :sniffing => { :validate => :boolean, :default => false },
+
+        # How long to wait, in seconds, between sniffing attempts
+        :sniffing_delay => { :validate => :number, :default => 5 },
+
+        # HTTP Path to be used for the sniffing requests
+        # the default value is computed by concatenating the path value and "_nodes/http"
+        # if sniffing_path is set it will be used as an absolute path
+        # do not use full URL here, only paths, e.g. "/sniff/_nodes/http"
+        :sniffing_path => { :validate => :string },
+
+        # Set the address of a forward HTTP proxy.
+        # This used to accept hashes as arguments but now only accepts
+        # arguments of the URI type to prevent leaking credentials.
+        :proxy => { :validate => :uri }, # but empty string is allowed
+
+        # Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If
+        # a timeout occurs, the request will be retried.
+        :timeout => { :validate => :number, :default => 60 },
+
+        # Set the Elasticsearch errors in the whitelist that you don't want to log.
+        # A useful example is when you want to skip all 409 errors
+        # which are `document_already_exists_exception`.
+        :failure_type_logging_whitelist => { :validate => :array, :default => [] },
+
+        # While the output tries to reuse connections efficiently we have a maximum.
+        # This sets the maximum number of open connections the output will create.
+        # Setting this too low may mean frequently closing / opening connections
+        # which is bad.
+        :pool_max => { :validate => :number, :default => 1000 },
+
+        # While the output tries to reuse connections efficiently we have a maximum per endpoint.
+        # This sets the maximum number of open connections per endpoint the output will create.
+        # Setting this too low may mean frequently closing / opening connections
+        # which is bad.
+        :pool_max_per_route => { :validate => :number, :default => 100 },
+
+        # HTTP Path where a HEAD request is sent when a backend is marked down
+        # the request is sent in the background to see if it has come back again
+        # before it is once again eligible to service requests.
+        # If you have custom firewall rules you may need to change this
+        :healthcheck_path => { :validate => :string },
+
+        # How frequently, in seconds, to wait between resurrection attempts.
+        # Resurrection is the process by which backend endpoints marked 'down' are checked
+        # to see if they have come back to life
+        :resurrect_delay => { :validate => :number, :default => 5 },
+
+        # How long to wait before checking if the connection is stale before executing a request on a connection using keepalive.
+        # You may want to set this lower, if you get connection errors regularly
+        # Quoting the Apache commons docs (this client is based Apache Commmons):
+        # 'Defines period of inactivity in milliseconds after which persistent connections must
+        # be re-validated prior to being leased to the consumer. Non-positive value passed to
+        # this method disables connection validation. This check helps detect connections that
+        # have become stale (half-closed) while kept inactive in the pool.'
+        # See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info]
+        :validate_after_inactivity => { :validate => :number, :default => 10000 },
+
+        # Enable gzip compression on requests. Note that response compression is on by default for Elasticsearch v5.0 and beyond
+        :http_compression => { :validate => :boolean, :default => false },
+
+        # Custom Headers to send on each request to elasticsearch nodes
+        :custom_headers => { :validate => :hash, :default => {} },
+
+        # Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `hosts` parameter.
+        # Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
+        #     `"127.0.0.1"`
+        #     `["127.0.0.1:9200","127.0.0.2:9200"]`
+        #     `["http://127.0.0.1"]`
+        #     `["https://127.0.0.1:9200"]`
+        #     `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath)
+        # It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `hosts` list
+        # to prevent LS from sending bulk requests to the master nodes.  So this parameter should only reference either data or client nodes in Elasticsearch.
+        #
+        # Any special characters present in the URLs here MUST be URL escaped! This means `#` should be put in as `%23` for instance.
+        :hosts => { :validate => :uri, :default => [ DEFAULT_HOST ], :list => true },
+
+        # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
+        #
+        # For more details, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[cloud documentation]
+        :cloud_id => { :validate => :string },
+
+        # Set initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval`
+        :retry_initial_interval => { :validate => :number, :default => 2 },
+
+        # Set max interval in seconds between bulk retries.
+        :retry_max_interval => { :validate => :number, :default => 64 }
+    }.freeze
+
+    def self.included(base)
+      CONFIG_PARAMS.each { |name, opts| base.config(name, opts) }
     end
   end
 end; end; end

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -5,7 +5,7 @@ module LogStash; module PluginMixins; module ElasticSearch
 
     # This module defines common methods that can be reused by alternate elasticsearch output plugins such as the elasticsearch_data_streams output.
 
-    attr_reader :client, :hosts
+    attr_reader :hosts
 
     # These codes apply to documents, not at the request level
     DOC_DLQ_CODES = [400, 404]
@@ -31,7 +31,7 @@ module LogStash; module PluginMixins; module ElasticSearch
       if @proxy.eql?('')
         @logger.warn "Supplied proxy setting (proxy => '') has no effect"
       end
-      @client ||= ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
+      ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
     end
 
     def validate_authentication
@@ -115,6 +115,15 @@ module LogStash; module PluginMixins; module ElasticSearch
     end
     private :parse_user_password_from_cloud_auth
 
+    # Plugin initialization extension point (after a successful ES connection).
+    def finish_register
+    end
+    protected :finish_register
+
+    def last_es_version
+      client.last_es_version
+    end
+
     def maximum_seen_major_version
       client.maximum_seen_major_version
     end
@@ -126,25 +135,24 @@ module LogStash; module PluginMixins; module ElasticSearch
     # launch a thread that waits for an initial successful connection to the ES cluster to call the given block
     # @param block [Proc] the block to execute upon initial successful connection
     # @return [Thread] the successful connection wait thread
-    def setup_after_successful_connection(&block)
+    def after_successful_connection(&block)
       Thread.new do
         sleep_interval = @retry_initial_interval
         until successful_connection? || @stopping.true?
-          @logger.debug("Waiting for connectivity to Elasticsearch cluster. Retrying in #{sleep_interval}s")
-          Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
-          sleep_interval = next_sleep_interval(sleep_interval)
+          @logger.debug("Waiting for connectivity to Elasticsearch cluster, retrying in #{sleep_interval}s")
+          sleep_interval = sleep_for_interval(sleep_interval)
         end
         block.call if successful_connection?
       end
     end
+    private :after_successful_connection
 
     def discover_cluster_uuid
       return unless defined?(plugin_metadata)
       cluster_info = client.get('/')
       plugin_metadata.set(:cluster_uuid, cluster_info['cluster_uuid'])
     rescue => e
-      # TODO introducing this logging message breaks many tests that need refactoring
-      # @logger.error("Unable to retrieve elasticsearch cluster uuid", error => e.message)
+      @logger.error("Unable to retrieve Elasticsearch cluster uuid", message: e.message, exception: e.class, backtrace: e.backtrace)
     end
 
     def retrying_submit(actions)
@@ -159,13 +167,11 @@ module LogStash; module PluginMixins; module ElasticSearch
         begin
           submit_actions = submit(submit_actions)
           if submit_actions && submit_actions.size > 0
-            @logger.info("Retrying individual bulk actions that failed or were rejected by the previous bulk request.", :count => submit_actions.size)
+            @logger.info("Retrying individual bulk actions that failed or were rejected by the previous bulk request", count: submit_actions.size)
           end
         rescue => e
-          @logger.error("Encountered an unexpected error submitting a bulk request! Will retry.",
-                       :error_message => e.message,
-                       :class => e.class.name,
-                       :backtrace => e.backtrace)
+          @logger.error("Encountered an unexpected error submitting a bulk request, will retry",
+                        message: e.message, exception: e.class, backtrace: e.backtrace)
         end
 
         # Everything was a success!
@@ -173,21 +179,41 @@ module LogStash; module PluginMixins; module ElasticSearch
 
         # If we're retrying the action sleep for the recommended interval
         # Double the interval for the next time through to achieve exponential backoff
-        Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
-        sleep_interval = next_sleep_interval(sleep_interval)
+        sleep_interval = sleep_for_interval(sleep_interval)
       end
     end
 
     def sleep_for_interval(sleep_interval)
-      Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
+      stoppable_sleep(sleep_interval)
       next_sleep_interval(sleep_interval)
     end
 
+    def stoppable_sleep(interval)
+      Stud.stoppable_sleep(interval) { @stopping.true? }
+    end
+
     def next_sleep_interval(current_interval)
       doubled = current_interval * 2
       doubled > @retry_max_interval ? @retry_max_interval : doubled
     end
 
+    def handle_dlq_status(message, action, status, response)
+      # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
+      if @dlq_writer
+        # TODO: Change this to send a map with { :status => status, :action => action } in the future
+        @dlq_writer.write(action[2], "#{message} status: #{status}, action: #{action}, response: #{response}")
+      else
+        if dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception'
+          level = :error
+        else
+          level = :warn
+        end
+        @logger.send level, message, status: status, action: action, response: response
+      end
+    end
+
+    private
+
     def submit(actions)
       bulk_response = safe_bulk(actions)
 
@@ -204,12 +230,20 @@ module LogStash; module PluginMixins; module ElasticSearch
         return
       end
 
+      responses = bulk_response["items"]
+      if responses.size != actions.size # can not map action -> response reliably
+        # an ES bug (on 7.10.2, 7.11.1) where a _bulk request to index X documents would return Y (> X) items
+        msg = "Sent #{actions.size} documents but Elasticsearch returned #{responses.size} responses"
+        @logger.warn(msg, actions: actions, responses: responses)
+        fail("#{msg} (likely a bug with _bulk endpoint)")
+      end
+
       actions_to_retry = []
-      bulk_response["items"].each_with_index do |response,idx|
+      responses.each_with_index do |response,idx|
         action_type, action_props = response.first
 
         status = action_props["status"]
-        failure  = action_props["error"]
+        error  = action_props["error"]
         action = actions[idx]
         action_params = action[1]
 
@@ -222,7 +256,7 @@ module LogStash; module PluginMixins; module ElasticSearch
           next
         elsif DOC_CONFLICT_CODE == status
           @document_level_metrics.increment(:non_retryable_failures)
-          @logger.warn "Failed action.", status: status, action: action, response: response if !failure_type_logging_whitelist.include?(failure["type"])
+          @logger.warn "Failed action", status: status, action: action, response: response if log_failure_type?(error)
           next
         elsif DOC_DLQ_CODES.include?(status)
           handle_dlq_status("Could not index event to Elasticsearch.", action, status, response)
@@ -231,7 +265,7 @@ module LogStash; module PluginMixins; module ElasticSearch
         else
           # only log what the user whitelisted
           @document_level_metrics.increment(:retryable_failures)
-          @logger.info "retrying failed action with response code: #{status} (#{failure})" if !failure_type_logging_whitelist.include?(failure["type"])
+          @logger.info "Retrying failed action", status: status, action: action, error: error if log_failure_type?(error)
           actions_to_retry << action
         end
       end
@@ -239,40 +273,25 @@ module LogStash; module PluginMixins; module ElasticSearch
       actions_to_retry
     end
 
-    def handle_dlq_status(message, action, status, response)
-      # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
-      if @dlq_writer
-        # TODO: Change this to send a map with { :status => status, :action => action } in the future
-        @dlq_writer.write(action[2], "#{message} status: #{status}, action: #{action}, response: #{response}")
-      else
-        error_type = response.fetch('index', {}).fetch('error', {})['type']
-        if 'invalid_index_name_exception' == error_type
-          level = :error
-        else
-          level = :warn
-        end
-        @logger.send level, message, status: status, action: action, response: response
-      end
+    def log_failure_type?(failure)
+      !failure_type_logging_whitelist.include?(failure["type"])
     end
 
     # Rescue retryable errors during bulk submission
+    # @param actions a [action, params, event.to_hash] tuple
+    # @return response [Hash] which contains 'errors' and processed 'items' entries
     def safe_bulk(actions)
       sleep_interval = @retry_initial_interval
       begin
-        es_actions = actions.map {|action_type, params, event| [action_type, params, event.to_hash]}
-        response = @client.bulk(es_actions)
-        response
+        @client.bulk(actions) # returns { 'errors': ..., 'items': ... }
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError => e
         # If we can't even connect to the server let's just print out the URL (:hosts is actually a URL)
         # and let the user sort it out from there
         @logger.error(
-          "Attempted to send a bulk request to elasticsearch'"+
-            " but Elasticsearch appears to be unreachable or down!",
-          :error_message => e.message,
-          :class => e.class.name,
-          :will_retry_in_seconds => sleep_interval
+          "Attempted to send a bulk request but Elasticsearch appears to be unreachable or down",
+          message: e.message, exception: e.class, will_retry_in_seconds: sleep_interval
         )
-        @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
+        @logger.debug? && @logger.debug("Failed actions for last bad bulk request", :actions => actions)
 
         # We retry until there are no errors! Errors should all go to the retry queue
         sleep_interval = sleep_for_interval(sleep_interval)
@@ -280,20 +299,19 @@ module LogStash; module PluginMixins; module ElasticSearch
         retry unless @stopping.true?
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError => e
         @logger.error(
-          "Attempted to send a bulk request to elasticsearch, but no there are no living connections in the connection pool. Perhaps Elasticsearch is unreachable or down?",
-          :error_message => e.message,
-          :class => e.class.name,
-          :will_retry_in_seconds => sleep_interval
+          "Attempted to send a bulk request but there are no living connections in the pool " +
+          "(perhaps Elasticsearch is unreachable or down?)",
+          message: e.message, exception: e.class, will_retry_in_seconds: sleep_interval
         )
-        Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
-        sleep_interval = next_sleep_interval(sleep_interval)
+
+        sleep_interval = sleep_for_interval(sleep_interval)
         @bulk_request_metrics.increment(:failures)
         retry unless @stopping.true?
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
         @bulk_request_metrics.increment(:failures)
-        log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s}
+        log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s, :content_length => e.request_body.bytesize}
         log_hash[:body] = e.response_body if @logger.debug? # Generally this is too verbose
-        message = "Encountered a retryable error. Will Retry with exponential backoff "
+        message = "Encountered a retryable error (will retry with exponential backoff)"
 
         # We treat 429s as a special case because these really aren't errors, but
         # rather just ES telling us to back off a bit, which we do.
@@ -307,17 +325,12 @@ module LogStash; module PluginMixins; module ElasticSearch
 
         sleep_interval = sleep_for_interval(sleep_interval)
         retry
-      rescue => e
-        # Stuff that should never happen
-        # For all other errors print out full connection issues
+      rescue => e # Stuff that should never happen - print out full connection issues
         @logger.error(
-          "An unknown error occurred sending a bulk request to Elasticsearch. We will retry indefinitely",
-          :error_message => e.message,
-          :error_class => e.class.name,
-          :backtrace => e.backtrace
+          "An unknown error occurred sending a bulk request to Elasticsearch (will retry indefinitely)",
+          message: e.message, exception: e.class, backtrace: e.backtrace
         )
-
-        @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
+        @logger.debug? && @logger.debug("Failed actions for last bad bulk request", :actions => actions)
 
         sleep_interval = sleep_for_interval(sleep_interval)
         @bulk_request_metrics.increment(:failures)
@@ -331,5 +344,12 @@ module LogStash; module PluginMixins; module ElasticSearch
       respond_to?(:execution_context) && execution_context.respond_to?(:dlq_writer) &&
         !execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter)
     end
+
+    def dig_value(val, first_key, *rest_keys)
+      fail(TypeError, "cannot dig value from #{val.class}") unless val.kind_of?(Hash)
+      val = val[first_key]
+      return val if rest_keys.empty? || val == nil
+      dig_value(val, *rest_keys)
+    end
   end
 end; end; end