logstash-output-elasticsearch 10.8.2-java → 11.0.1-java

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -5,7 +5,7 @@ module LogStash; module PluginMixins; module ElasticSearch
 
     # This module defines common methods that can be reused by alternate elasticsearch output plugins such as the elasticsearch_data_streams output.
 
-    attr_reader :client, :hosts
+    attr_reader :hosts
 
     # These codes apply to documents, not at the request level
     DOC_DLQ_CODES = [400, 404]
@@ -31,7 +31,7 @@ module LogStash; module PluginMixins; module ElasticSearch
       if @proxy.eql?('')
         @logger.warn "Supplied proxy setting (proxy => '') has no effect"
       end
-      @client ||= ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
+      ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
     end
 
     def validate_authentication
@@ -115,6 +115,15 @@ module LogStash; module PluginMixins; module ElasticSearch
     end
     private :parse_user_password_from_cloud_auth
 
+    # Plugin initialization extension point (after a successful ES connection).
+    def finish_register
+    end
+    protected :finish_register
+
+    def last_es_version
+      client.last_es_version
+    end
+
     def maximum_seen_major_version
       client.maximum_seen_major_version
     end
@@ -126,25 +135,24 @@ module LogStash; module PluginMixins; module ElasticSearch
     # launch a thread that waits for an initial successful connection to the ES cluster to call the given block
     # @param block [Proc] the block to execute upon initial successful connection
     # @return [Thread] the successful connection wait thread
-    def setup_after_successful_connection(&block)
+    def after_successful_connection(&block)
       Thread.new do
         sleep_interval = @retry_initial_interval
         until successful_connection? || @stopping.true?
-          @logger.debug("Waiting for connectivity to Elasticsearch cluster. Retrying in #{sleep_interval}s")
-          Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
-          sleep_interval = next_sleep_interval(sleep_interval)
+          @logger.debug("Waiting for connectivity to Elasticsearch cluster, retrying in #{sleep_interval}s")
+          sleep_interval = sleep_for_interval(sleep_interval)
         end
         block.call if successful_connection?
       end
     end
+    private :after_successful_connection
 
     def discover_cluster_uuid
       return unless defined?(plugin_metadata)
       cluster_info = client.get('/')
       plugin_metadata.set(:cluster_uuid, cluster_info['cluster_uuid'])
     rescue => e
-      # TODO introducing this logging message breaks many tests that need refactoring
-      # @logger.error("Unable to retrieve elasticsearch cluster uuid", error => e.message)
+      @logger.error("Unable to retrieve Elasticsearch cluster uuid", message: e.message, exception: e.class, backtrace: e.backtrace)
     end
 
     def retrying_submit(actions)
@@ -159,13 +167,11 @@ module LogStash; module PluginMixins; module ElasticSearch
         begin
           submit_actions = submit(submit_actions)
           if submit_actions && submit_actions.size > 0
-            @logger.info("Retrying individual bulk actions that failed or were rejected by the previous bulk request.", :count => submit_actions.size)
+            @logger.info("Retrying individual bulk actions that failed or were rejected by the previous bulk request", count: submit_actions.size)
           end
         rescue => e
-          @logger.error("Encountered an unexpected error submitting a bulk request! Will retry.",
-                       :error_message => e.message,
-                       :class => e.class.name,
-                       :backtrace => e.backtrace)
+          @logger.error("Encountered an unexpected error submitting a bulk request, will retry",
+                        message: e.message, exception: e.class, backtrace: e.backtrace)
         end
 
         # Everything was a success!
@@ -173,21 +179,42 @@ module LogStash; module PluginMixins; module ElasticSearch
 
         # If we're retrying the action sleep for the recommended interval
         # Double the interval for the next time through to achieve exponential backoff
-        Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
-        sleep_interval = next_sleep_interval(sleep_interval)
+        sleep_interval = sleep_for_interval(sleep_interval)
       end
     end
 
     def sleep_for_interval(sleep_interval)
-      Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
+      stoppable_sleep(sleep_interval)
       next_sleep_interval(sleep_interval)
     end
 
+    def stoppable_sleep(interval)
+      Stud.stoppable_sleep(interval) { @stopping.true? }
+    end
+
     def next_sleep_interval(current_interval)
       doubled = current_interval * 2
       doubled > @retry_max_interval ? @retry_max_interval : doubled
     end
 
+    def handle_dlq_status(message, action, status, response)
+      # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
+      if @dlq_writer
+        event, action = action.event, [action[0], action[1], action[2]]
+        # TODO: Change this to send a map with { :status => status, :action => action } in the future
+        @dlq_writer.write(event, "#{message} status: #{status}, action: #{action}, response: #{response}")
+      else
+        if dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception'
+          level = :error
+        else
+          level = :warn
+        end
+        @logger.send level, message, status: status, action: action, response: response
+      end
+    end
+
+    private
+
     def submit(actions)
       bulk_response = safe_bulk(actions)
 
@@ -204,12 +231,20 @@ module LogStash; module PluginMixins; module ElasticSearch
         return
       end
 
+      responses = bulk_response["items"]
+      if responses.size != actions.size # can not map action -> response reliably
+        # an ES bug (on 7.10.2, 7.11.1) where a _bulk request to index X documents would return Y (> X) items
+        msg = "Sent #{actions.size} documents but Elasticsearch returned #{responses.size} responses"
+        @logger.warn(msg, actions: actions, responses: responses)
+        fail("#{msg} (likely a bug with _bulk endpoint)")
+      end
+
       actions_to_retry = []
-      bulk_response["items"].each_with_index do |response,idx|
+      responses.each_with_index do |response,idx|
         action_type, action_props = response.first
 
         status = action_props["status"]
-        failure  = action_props["error"]
+        error  = action_props["error"]
         action = actions[idx]
         action_params = action[1]
 
@@ -222,7 +257,7 @@ module LogStash; module PluginMixins; module ElasticSearch
           next
         elsif DOC_CONFLICT_CODE == status
           @document_level_metrics.increment(:non_retryable_failures)
-          @logger.warn "Failed action.", status: status, action: action, response: response if !failure_type_logging_whitelist.include?(failure["type"])
+          @logger.warn "Failed action", status: status, action: action, response: response if log_failure_type?(error)
           next
         elsif DOC_DLQ_CODES.include?(status)
           handle_dlq_status("Could not index event to Elasticsearch.", action, status, response)
@@ -231,7 +266,7 @@ module LogStash; module PluginMixins; module ElasticSearch
         else
           # only log what the user whitelisted
           @document_level_metrics.increment(:retryable_failures)
-          @logger.info "retrying failed action with response code: #{status} (#{failure})" if !failure_type_logging_whitelist.include?(failure["type"])
+          @logger.info "Retrying failed action", status: status, action: action, error: error if log_failure_type?(error)
           actions_to_retry << action
         end
       end
@@ -239,40 +274,25 @@ module LogStash; module PluginMixins; module ElasticSearch
       actions_to_retry
     end
 
-    def handle_dlq_status(message, action, status, response)
-      # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
-      if @dlq_writer
-        # TODO: Change this to send a map with { :status => status, :action => action } in the future
-        @dlq_writer.write(action[2], "#{message} status: #{status}, action: #{action}, response: #{response}")
-      else
-        error_type = response.fetch('index', {}).fetch('error', {})['type']
-        if 'invalid_index_name_exception' == error_type
-          level = :error
-        else
-          level = :warn
-        end
-        @logger.send level, message, status: status, action: action, response: response
-      end
+    def log_failure_type?(failure)
+      !failure_type_logging_whitelist.include?(failure["type"])
     end
 
     # Rescue retryable errors during bulk submission
+    # @param actions a [action, params, event.to_hash] tuple
+    # @return response [Hash] which contains 'errors' and processed 'items' entries
     def safe_bulk(actions)
       sleep_interval = @retry_initial_interval
       begin
-        es_actions = actions.map {|action_type, params, event| [action_type, params, event.to_hash]}
-        response = @client.bulk(es_actions)
-        response
+        @client.bulk(actions) # returns { 'errors': ..., 'items': ... }
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError => e
         # If we can't even connect to the server let's just print out the URL (:hosts is actually a URL)
         # and let the user sort it out from there
         @logger.error(
-          "Attempted to send a bulk request to elasticsearch'"+
-            " but Elasticsearch appears to be unreachable or down!",
-          :error_message => e.message,
-          :class => e.class.name,
-          :will_retry_in_seconds => sleep_interval
+          "Attempted to send a bulk request but Elasticsearch appears to be unreachable or down",
+          message: e.message, exception: e.class, will_retry_in_seconds: sleep_interval
         )
-        @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
+        @logger.debug? && @logger.debug("Failed actions for last bad bulk request", :actions => actions)
 
         # We retry until there are no errors! Errors should all go to the retry queue
         sleep_interval = sleep_for_interval(sleep_interval)
@@ -280,20 +300,19 @@ module LogStash; module PluginMixins; module ElasticSearch
         retry unless @stopping.true?
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError => e
         @logger.error(
-          "Attempted to send a bulk request to elasticsearch, but no there are no living connections in the connection pool. Perhaps Elasticsearch is unreachable or down?",
-          :error_message => e.message,
-          :class => e.class.name,
-          :will_retry_in_seconds => sleep_interval
+          "Attempted to send a bulk request but there are no living connections in the pool " +
+          "(perhaps Elasticsearch is unreachable or down?)",
+          message: e.message, exception: e.class, will_retry_in_seconds: sleep_interval
         )
-        Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
-        sleep_interval = next_sleep_interval(sleep_interval)
+
+        sleep_interval = sleep_for_interval(sleep_interval)
         @bulk_request_metrics.increment(:failures)
         retry unless @stopping.true?
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
         @bulk_request_metrics.increment(:failures)
-        log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s}
+        log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s, :content_length => e.request_body.bytesize}
         log_hash[:body] = e.response_body if @logger.debug? # Generally this is too verbose
-        message = "Encountered a retryable error. Will Retry with exponential backoff "
+        message = "Encountered a retryable error (will retry with exponential backoff)"
 
         # We treat 429s as a special case because these really aren't errors, but
         # rather just ES telling us to back off a bit, which we do.
@@ -307,17 +326,12 @@ module LogStash; module PluginMixins; module ElasticSearch
 
         sleep_interval = sleep_for_interval(sleep_interval)
         retry
-      rescue => e
-        # Stuff that should never happen
-        # For all other errors print out full connection issues
+      rescue => e # Stuff that should never happen - print out full connection issues
         @logger.error(
-          "An unknown error occurred sending a bulk request to Elasticsearch. We will retry indefinitely",
-          :error_message => e.message,
-          :error_class => e.class.name,
-          :backtrace => e.backtrace
+          "An unknown error occurred sending a bulk request to Elasticsearch (will retry indefinitely)",
+          message: e.message, exception: e.class, backtrace: e.backtrace
         )
-
-        @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
+        @logger.debug? && @logger.debug("Failed actions for last bad bulk request", :actions => actions)
 
         sleep_interval = sleep_for_interval(sleep_interval)
         @bulk_request_metrics.increment(:failures)
@@ -331,5 +345,12 @@ module LogStash; module PluginMixins; module ElasticSearch
       respond_to?(:execution_context) && execution_context.respond_to?(:dlq_writer) &&
         !execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter)
     end
+
+    def dig_value(val, first_key, *rest_keys)
+      fail(TypeError, "cannot dig value from #{val.class}") unless val.kind_of?(Hash)
+      val = val[first_key]
+      return val if rest_keys.empty? || val == nil
+      dig_value(val, *rest_keys)
+    end
   end
 end; end; end

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '10.8.2'
+  s.version         = '11.0.1'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
@@ -23,13 +23,13 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
-  s.add_runtime_dependency 'cabin', ['~> 0.6']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
 
   s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'flores'
+  s.add_development_dependency 'cabin', ['~> 0.6']
   # Still used in some specs, we should remove this ASAP
   s.add_development_dependency 'elasticsearch'
 end

data/spec/es_spec_helper.rb

@@ -1,5 +1,5 @@
-require "logstash/devutils/rspec/spec_helper"
-require 'manticore'
+require_relative './spec_helper'
+
 require 'elasticsearch'
 require_relative "support/elasticsearch/api/actions/delete_ilm_policy"
 require_relative "support/elasticsearch/api/actions/get_alias"
@@ -8,10 +8,7 @@ require_relative "support/elasticsearch/api/actions/get_ilm_policy"
 require_relative "support/elasticsearch/api/actions/put_ilm_policy"
 
 require 'json'
-
-unless defined?(LogStash::OSS)
-  LogStash::OSS = ENV['DISTRIBUTION'] != "default"
-end
+require 'cabin'
 
 module ESHelper
   def get_host_port

data/spec/integration/outputs/data_stream_spec.rb

@@ -0,0 +1,61 @@
+require_relative "../../../spec/es_spec_helper"
+require "logstash/outputs/elasticsearch"
+
+describe "data streams", :integration => true do
+
+  let(:ds_name) { "logs-#{ds_dataset}-default" }
+  let(:ds_dataset) { 'integration_test' }
+
+  let(:options) do
+    { "data_stream" => 'true', "data_stream_dataset" => ds_dataset, "hosts" => get_host_port() }
+  end
+
+  subject { LogStash::Outputs::ElasticSearch.new(options) }
+
+  before :each do
+    @es = get_client
+    @es.delete_by_query(index: ".ds-#{ds_name}-*", expand_wildcards: :all, body: { query: { match_all: {} } }) rescue nil
+
+    es_version = @es.info['version']['number']
+    if Gem::Version.create(es_version) < Gem::Version.create('7.9.0')
+      skip "ES version #{es_version} does not support data-streams"
+    end
+  end
+
+  it "creates a new document" do
+    subject.register
+    subject.multi_receive([LogStash::Event.new("message" => "MSG 111")])
+
+    @es.indices.refresh
+
+    Stud::try(3.times) do
+      r = @es.search(index: ds_name)
+
+      expect( r['hits']['total']['value'] ).to eq 1
+      doc = r['hits']['hits'].first
+      expect( doc['_source'] ).to include "message"=>"MSG 111"
+      expect( doc['_source'] ).to include "data_stream"=>{"dataset"=>ds_dataset, "type"=>"logs", "namespace"=>"default"}
+    end
+  end
+
+  context "with document_id" do
+
+    let(:document_id) { '1234567890' }
+    let(:options) { super().merge("document_id" => document_id) }
+
+    it "creates a new document" do
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "foo")])
+
+      @es.indices.refresh
+
+      Stud::try(3.times) do
+        r = @es.search(index: ds_name, body: { query: { match: { _id: document_id } } })
+        expect( r['hits']['total']['value'] ).to eq 1
+        doc = r['hits']['hits'].first
+        expect( doc['_source'] ).to include "message"=>"foo"
+      end
+    end
+
+  end
+end

data/spec/integration/outputs/ilm_spec.rb

@@ -5,7 +5,7 @@ shared_examples_for 'an ILM enabled Logstash' do
   context 'with a policy with a maximum number of documents' do
     let (:policy) { small_max_doc_policy }
     let (:ilm_policy_name) { "logstash-policy-custom"}
-    let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+    let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
 
     it 'should rollover when the policy max docs is reached' do
       put_policy(@es, ilm_policy_name, policy)
@@ -54,7 +54,7 @@ shared_examples_for 'an ILM enabled Logstash' do
   context 'with a policy where the maximum number of documents is not reached' do
     let (:policy) { large_max_doc_policy }
     let (:ilm_policy_name) { "logstash-policy-custom-policy"}
-    let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+    let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
 
     it 'should ingest into a single index when max docs is not reached' do
       put_policy(@es,ilm_policy_name, policy)
@@ -119,7 +119,7 @@ shared_examples_for 'an ILM disabled Logstash' do
   context 'with an existing policy that will roll over' do
     let (:policy) { small_max_doc_policy }
     let (:ilm_policy_name) { "logstash-policy-3_docs"}
-    let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+    let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
 
     it 'should not roll over indices' do
       subject.register
@@ -155,7 +155,7 @@ shared_examples_for 'an ILM disabled Logstash' do
 
   context 'with a custom template name' do
     let (:template_name) { "logstash_custom_template_name" }
-    let (:settings) { super.merge('template_name' => template_name)}
+    let (:settings) { super().merge('template_name' => template_name)}
 
     it 'should not write the ILM settings into the template' do
       subject.register
@@ -195,28 +195,32 @@ shared_examples_for 'an Elasticsearch instance that does not support index lifec
   subject { LogStash::Outputs::ElasticSearch.new(settings) }
 
   context 'when ilm is enabled in Logstash' do
-    let (:settings) { super.merge!({ 'ilm_enabled' => true }) }
+    let (:settings) { super().merge!({ 'ilm_enabled' => true }) }
 
     it 'should raise a configuration error' do
+      # TODO should be refactored not to rely on plugin internals
+      finish_register = subject.method(:finish_register)
+      expect(subject).to receive(:finish_register)
       expect do
         begin
           subject.register
-          sleep(1)
+          finish_register.call
+          sleep(1.5) # wait_for_successful_connection (for the thread to raise)
         ensure
-          subject.stop_template_installer
+          subject.send :stop_after_successful_connection_thread
         end
       end.to raise_error(LogStash::ConfigurationError)
     end
   end
 
   context 'when ilm is disabled in Logstash' do
-    let (:settings) { super.merge!({ 'ilm_enabled' => false }) }
+    let (:settings) { super().merge!({ 'ilm_enabled' => false }) }
 
     it_behaves_like 'an ILM disabled Logstash'
   end
 
   context 'when ilm is set to auto in Logstash' do
-    let (:settings) { super.merge!({ 'ilm_enabled' => 'auto' }) }
+    let (:settings) { super().merge!({ 'ilm_enabled' => 'auto' }) }
 
     it_behaves_like 'an ILM disabled Logstash'
   end
@@ -286,7 +290,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
 
       context 'when using the default policy' do
         context 'with a custom pattern' do
-          let (:settings) { super.merge("ilm_pattern" => "000001")}
+          let (:settings) { super().merge("ilm_pattern" => "000001")}
           it 'should create a rollover alias' do
             expect(@es.indices.exists_alias(name: "logstash")).to be_falsey
             subject.register
@@ -346,7 +350,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
 
       context 'when not using the default policy' do
         let (:ilm_policy_name) {"logstash-policy-small"}
-        let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+        let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
         let (:policy) { small_max_doc_policy }
 
         before do
@@ -363,7 +367,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
 
       context 'when using a time based policy' do
         let (:ilm_policy_name) {"logstash-policy-time"}
-        let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+        let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
         let (:policy) { max_age_policy("1d") }
 
         before do
@@ -409,7 +413,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
           let (:template) { "spec/fixtures/template-with-policy-es6x.json" }
         end
 
-        let (:settings) { super.merge("template" => template,
+        let (:settings) { super().merge("template" => template,
                                       "index" => "overwrite-4")}
 
         it 'should not overwrite the index patterns' do
@@ -426,7 +430,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
         let (:ilm_rollover_alias) { "logstash_the_cat_in_the_hat" }
         let (:index) { ilm_rollover_alias }
         let(:expected_index) { index }
-        let (:settings) { super.merge("ilm_policy" => ilm_policy_name,
+        let (:settings) { super().merge("ilm_policy" => ilm_policy_name,
                                       "template" => template,
                                       "ilm_rollover_alias" => ilm_rollover_alias)}
 
@@ -480,7 +484,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
 
         context 'with a different template_name' do
           let (:template_name) { "logstash_custom_template_name" }
-          let (:settings) { super.merge('template_name' => template_name)}
+          let (:settings) { super().merge('template_name' => template_name)}
 
           it_behaves_like 'an ILM enabled Logstash'
 
@@ -514,7 +518,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
     end
 
     context 'when ilm_enabled is the default' do
-      let (:settings) { super.tap{|x|x.delete('ilm_enabled')}}
+      let (:settings) { super().tap{|x|x.delete('ilm_enabled')}}
 
       if ESHelper.es_version_satisfies?(">=7.0")
         context 'when Elasticsearch is version 7 or above' do
@@ -530,13 +534,13 @@ if ESHelper.es_version_satisfies?(">= 6.6")
     end
 
     context 'with ilm disabled' do
-      let (:settings) { super.merge('ilm_enabled' => false )}
+      let (:settings) { super().merge('ilm_enabled' => false )}
 
       it_behaves_like 'an ILM disabled Logstash'
     end
 
     context 'with ilm disabled using a string' do
-      let (:settings) { super.merge('ilm_enabled' => 'false' )}
+      let (:settings) { super().merge('ilm_enabled' => 'false' )}
 
       it_behaves_like 'an ILM disabled Logstash'
     end