logstash-output-elasticsearch 10.8.6-java → 11.0.3-java

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -5,7 +5,7 @@ module LogStash; module PluginMixins; module ElasticSearch
 
     # This module defines common methods that can be reused by alternate elasticsearch output plugins such as the elasticsearch_data_streams output.
 
-    attr_reader :client, :hosts
+    attr_reader :hosts
 
     # These codes apply to documents, not at the request level
     DOC_DLQ_CODES = [400, 404]
@@ -31,7 +31,7 @@ module LogStash; module PluginMixins; module ElasticSearch
       if @proxy.eql?('')
         @logger.warn "Supplied proxy setting (proxy => '') has no effect"
       end
-      @client ||= ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
+      ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
     end
 
     def validate_authentication
@@ -115,6 +115,15 @@ module LogStash; module PluginMixins; module ElasticSearch
     end
     private :parse_user_password_from_cloud_auth
 
+    # Plugin initialization extension point (after a successful ES connection).
+    def finish_register
+    end
+    protected :finish_register
+
+    def last_es_version
+      client.last_es_version
+    end
+
     def maximum_seen_major_version
       client.maximum_seen_major_version
     end
@@ -126,25 +135,24 @@ module LogStash; module PluginMixins; module ElasticSearch
     # launch a thread that waits for an initial successful connection to the ES cluster to call the given block
     # @param block [Proc] the block to execute upon initial successful connection
     # @return [Thread] the successful connection wait thread
-    def setup_after_successful_connection(&block)
+    def after_successful_connection(&block)
       Thread.new do
         sleep_interval = @retry_initial_interval
         until successful_connection? || @stopping.true?
-          @logger.debug("Waiting for connectivity to Elasticsearch cluster. Retrying in #{sleep_interval}s")
-          Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
-          sleep_interval = next_sleep_interval(sleep_interval)
+          @logger.debug("Waiting for connectivity to Elasticsearch cluster, retrying in #{sleep_interval}s")
+          sleep_interval = sleep_for_interval(sleep_interval)
         end
         block.call if successful_connection?
       end
     end
+    private :after_successful_connection
 
     def discover_cluster_uuid
       return unless defined?(plugin_metadata)
       cluster_info = client.get('/')
       plugin_metadata.set(:cluster_uuid, cluster_info['cluster_uuid'])
     rescue => e
-      # TODO introducing this logging message breaks many tests that need refactoring
-      # @logger.error("Unable to retrieve elasticsearch cluster uuid", error => e.message)
+      @logger.error("Unable to retrieve Elasticsearch cluster uuid", message: e.message, exception: e.class, backtrace: e.backtrace)
     end
 
     def retrying_submit(actions)
@@ -159,13 +167,11 @@ module LogStash; module PluginMixins; module ElasticSearch
         begin
           submit_actions = submit(submit_actions)
           if submit_actions && submit_actions.size > 0
-            @logger.info("Retrying individual bulk actions that failed or were rejected by the previous bulk request.", :count => submit_actions.size)
+            @logger.info("Retrying individual bulk actions that failed or were rejected by the previous bulk request", count: submit_actions.size)
           end
         rescue => e
-          @logger.error("Encountered an unexpected error submitting a bulk request! Will retry.",
-                       :error_message => e.message,
-                       :class => e.class.name,
-                       :backtrace => e.backtrace)
+          @logger.error("Encountered an unexpected error submitting a bulk request, will retry",
+                        message: e.message, exception: e.class, backtrace: e.backtrace)
         end
 
         # Everything was a success!
@@ -173,21 +179,42 @@ module LogStash; module PluginMixins; module ElasticSearch
 
         # If we're retrying the action sleep for the recommended interval
         # Double the interval for the next time through to achieve exponential backoff
-        Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
-        sleep_interval = next_sleep_interval(sleep_interval)
+        sleep_interval = sleep_for_interval(sleep_interval)
       end
     end
 
     def sleep_for_interval(sleep_interval)
-      Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
+      stoppable_sleep(sleep_interval)
       next_sleep_interval(sleep_interval)
     end
 
+    def stoppable_sleep(interval)
+      Stud.stoppable_sleep(interval) { @stopping.true? }
+    end
+
     def next_sleep_interval(current_interval)
       doubled = current_interval * 2
       doubled > @retry_max_interval ? @retry_max_interval : doubled
     end
 
+    def handle_dlq_status(message, action, status, response)
+      # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
+      if @dlq_writer
+        event, action = action.event, [action[0], action[1], action[2]]
+        # TODO: Change this to send a map with { :status => status, :action => action } in the future
+        @dlq_writer.write(event, "#{message} status: #{status}, action: #{action}, response: #{response}")
+      else
+        if dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception'
+          level = :error
+        else
+          level = :warn
+        end
+        @logger.send level, message, status: status, action: action, response: response
+      end
+    end
+
+    private
+
     def submit(actions)
       bulk_response = safe_bulk(actions)
 
@@ -217,7 +244,7 @@ module LogStash; module PluginMixins; module ElasticSearch
         action_type, action_props = response.first
 
         status = action_props["status"]
-        failure  = action_props["error"]
+        error  = action_props["error"]
         action = actions[idx]
         action_params = action[1]
 
@@ -230,7 +257,7 @@ module LogStash; module PluginMixins; module ElasticSearch
           next
         elsif DOC_CONFLICT_CODE == status
           @document_level_metrics.increment(:non_retryable_failures)
-          @logger.warn "Failed action.", status: status, action: action, response: response if !failure_type_logging_whitelist.include?(failure["type"])
+          @logger.warn "Failed action", status: status, action: action, response: response if log_failure_type?(error)
           next
         elsif DOC_DLQ_CODES.include?(status)
           handle_dlq_status("Could not index event to Elasticsearch.", action, status, response)
@@ -239,7 +266,7 @@ module LogStash; module PluginMixins; module ElasticSearch
         else
           # only log what the user whitelisted
           @document_level_metrics.increment(:retryable_failures)
-          @logger.info "retrying failed action with response code: #{status} (#{failure})" if !failure_type_logging_whitelist.include?(failure["type"])
+          @logger.info "Retrying failed action", status: status, action: action, error: error if log_failure_type?(error)
           actions_to_retry << action
         end
       end
@@ -247,40 +274,25 @@ module LogStash; module PluginMixins; module ElasticSearch
       actions_to_retry
     end
 
-    def handle_dlq_status(message, action, status, response)
-      # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
-      if @dlq_writer
-        # TODO: Change this to send a map with { :status => status, :action => action } in the future
-        @dlq_writer.write(action[2], "#{message} status: #{status}, action: #{action}, response: #{response}")
-      else
-        error_type = response.fetch('index', {}).fetch('error', {})['type']
-        if 'invalid_index_name_exception' == error_type
-          level = :error
-        else
-          level = :warn
-        end
-        @logger.send level, message, status: status, action: action, response: response
-      end
+    def log_failure_type?(failure)
+      !failure_type_logging_whitelist.include?(failure["type"])
     end
 
     # Rescue retryable errors during bulk submission
+    # @param actions a [action, params, event.to_hash] tuple
+    # @return response [Hash] which contains 'errors' and processed 'items' entries
     def safe_bulk(actions)
       sleep_interval = @retry_initial_interval
       begin
-        es_actions = actions.map {|action_type, params, event| [action_type, params, event.to_hash]}
-        response = @client.bulk(es_actions)
-        response
+        @client.bulk(actions) # returns { 'errors': ..., 'items': ... }
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError => e
         # If we can't even connect to the server let's just print out the URL (:hosts is actually a URL)
         # and let the user sort it out from there
         @logger.error(
-          "Attempted to send a bulk request to elasticsearch'"+
-            " but Elasticsearch appears to be unreachable or down!",
-          :error_message => e.message,
-          :class => e.class.name,
-          :will_retry_in_seconds => sleep_interval
+          "Attempted to send a bulk request but Elasticsearch appears to be unreachable or down",
+          message: e.message, exception: e.class, will_retry_in_seconds: sleep_interval
         )
-        @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
+        @logger.debug? && @logger.debug("Failed actions for last bad bulk request", :actions => actions)
 
         # We retry until there are no errors! Errors should all go to the retry queue
         sleep_interval = sleep_for_interval(sleep_interval)
@@ -288,20 +300,19 @@ module LogStash; module PluginMixins; module ElasticSearch
         retry unless @stopping.true?
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError => e
         @logger.error(
-          "Attempted to send a bulk request to elasticsearch, but no there are no living connections in the connection pool. Perhaps Elasticsearch is unreachable or down?",
-          :error_message => e.message,
-          :class => e.class.name,
-          :will_retry_in_seconds => sleep_interval
+          "Attempted to send a bulk request but there are no living connections in the pool " +
+          "(perhaps Elasticsearch is unreachable or down?)",
+          message: e.message, exception: e.class, will_retry_in_seconds: sleep_interval
         )
-        Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
-        sleep_interval = next_sleep_interval(sleep_interval)
+
+        sleep_interval = sleep_for_interval(sleep_interval)
         @bulk_request_metrics.increment(:failures)
         retry unless @stopping.true?
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
         @bulk_request_metrics.increment(:failures)
         log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s, :content_length => e.request_body.bytesize}
         log_hash[:body] = e.response_body if @logger.debug? # Generally this is too verbose
-        message = "Encountered a retryable error. Will Retry with exponential backoff "
+        message = "Encountered a retryable error (will retry with exponential backoff)"
 
         # We treat 429s as a special case because these really aren't errors, but
         # rather just ES telling us to back off a bit, which we do.
@@ -315,17 +326,12 @@ module LogStash; module PluginMixins; module ElasticSearch
 
         sleep_interval = sleep_for_interval(sleep_interval)
         retry
-      rescue => e
-        # Stuff that should never happen
-        # For all other errors print out full connection issues
+      rescue => e # Stuff that should never happen - print out full connection issues
         @logger.error(
-          "An unknown error occurred sending a bulk request to Elasticsearch. We will retry indefinitely",
-          :error_message => e.message,
-          :error_class => e.class.name,
-          :backtrace => e.backtrace
+          "An unknown error occurred sending a bulk request to Elasticsearch (will retry indefinitely)",
+          message: e.message, exception: e.class, backtrace: e.backtrace
         )
-
-        @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
+        @logger.debug? && @logger.debug("Failed actions for last bad bulk request", :actions => actions)
 
         sleep_interval = sleep_for_interval(sleep_interval)
         @bulk_request_metrics.increment(:failures)
@@ -339,5 +345,12 @@ module LogStash; module PluginMixins; module ElasticSearch
       respond_to?(:execution_context) && execution_context.respond_to?(:dlq_writer) &&
         !execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter)
     end
+
+    def dig_value(val, first_key, *rest_keys)
+      fail(TypeError, "cannot dig value from #{val.class}") unless val.kind_of?(Hash)
+      val = val[first_key]
+      return val if rest_keys.empty? || val == nil
+      dig_value(val, *rest_keys)
+    end
   end
 end; end; end

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '10.8.6'
+  s.version         = '11.0.3'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
@@ -21,15 +21,15 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
-  s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
+  s.add_runtime_dependency "manticore", '>= 0.7.1', '< 1.0.0'
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
-  s.add_runtime_dependency 'cabin', ['~> 0.6']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
 
   s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'flores'
+  s.add_development_dependency 'cabin', ['~> 0.6']
   # Still used in some specs, we should remove this ASAP
   s.add_development_dependency 'elasticsearch'
 end

data/spec/es_spec_helper.rb

@@ -1,5 +1,5 @@
-require "logstash/devutils/rspec/spec_helper"
-require 'manticore'
+require_relative './spec_helper'
+
 require 'elasticsearch'
 require_relative "support/elasticsearch/api/actions/delete_ilm_policy"
 require_relative "support/elasticsearch/api/actions/get_alias"
@@ -8,10 +8,7 @@ require_relative "support/elasticsearch/api/actions/get_ilm_policy"
 require_relative "support/elasticsearch/api/actions/put_ilm_policy"
 
 require 'json'
-
-unless defined?(LogStash::OSS)
-  LogStash::OSS = ENV['DISTRIBUTION'] != "default"
-end
+require 'cabin'
 
 module ESHelper
   def get_host_port
@@ -23,7 +20,9 @@ module ESHelper
   end
 
   def get_client
-    Elasticsearch::Client.new(:hosts => [get_host_port])
+    Elasticsearch::Client.new(:hosts => [get_host_port]).tap do |client|
+      allow(client).to receive(:verify_elasticsearch).and_return(true) # bypass client side version checking
+    end
   end
 
   def doc_type
@@ -56,11 +55,7 @@ module ESHelper
   end
 
   def routing_field_name
-    if ESHelper.es_version_satisfies?(">=6")
-      :routing
-    else
-      :_routing
-    end
+    :routing
   end
 
   def self.es_version

data/spec/fixtures/_nodes/{5x_6x.json → 6x.json}

@@ -11,7 +11,7 @@
       "transport_address" : "http://localhost:9200",
       "host" : "localhost",
       "ip" : "127.0.0.1",
-      "version" : "5.5.1",
+      "version" : "6.8.10",
       "build_hash" : "19c13d0",
       "roles" : [
         "master"
@@ -29,7 +29,7 @@
       "transport_address" : "http://localhost:9201",
       "host" : "localhost",
       "ip" : "127.0.0.1",
-      "version" : "5.5.1",
+      "version" : "6.8.10",
       "build_hash" : "19c13d0",
       "roles" : [
         "data"
@@ -47,7 +47,7 @@
       "transport_address" : "http://localhost:9202",
       "host" : "localhost",
       "ip" : "127.0.0.1",
-      "version" : "5.5.1",
+      "version" : "6.8.10",
       "build_hash" : "19c13d0",
       "roles" : [
         "data",
@@ -66,7 +66,7 @@
       "transport_address" : "http://localhost:9203",
       "host" : "localhost",
       "ip" : "127.0.0.1",
-      "version" : "5.5.1",
+      "version" : "6.8.10",
       "build_hash" : "19c13d0",
       "roles" : [ ],
       "http" : {
@@ -78,4 +78,4 @@
       }
     }
   }
-}
+}

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -8,62 +8,63 @@ RSpec::Matchers.define :a_valid_gzip_encoded_string do
   }
 end
 
-if ESHelper.es_version_satisfies?(">= 5")
-  describe "indexing with http_compression turned on", :integration => true do
-    let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
-    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
-    let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
-    let(:event_count) { 10000 + rand(500) }
-    let(:events) { event_count.times.map { event }.to_a }
-    let(:config) {
-      {
-        "hosts" => get_host_port,
-        "index" => index,
-        "http_compression" => true
-      }
+describe "indexing with http_compression turned on", :integration => true do
+  let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
+  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
+  let(:event_count) { 10000 + rand(500) }
+  let(:events) { event_count.times.map { event }.to_a }
+  let(:config) {
+    {
+      "hosts" => get_host_port,
+      "index" => index,
+      "http_compression" => true
     }
-    subject { LogStash::Outputs::ElasticSearch.new(config) }
+  }
+  subject { LogStash::Outputs::ElasticSearch.new(config) }
 
-    let(:es_url) { "http://#{get_host_port}" }
-    let(:index_url) {"#{es_url}/#{index}"}
-    let(:http_client_options) { {} }
-    let(:http_client) do
-      Manticore::Client.new(http_client_options)
-    end
+  let(:es_url) { "http://#{get_host_port}" }
+  let(:index_url) {"#{es_url}/#{index}"}
+  let(:http_client_options) { {} }
+  let(:http_client) do
+    Manticore::Client.new(http_client_options)
+  end
 
-    before do
-      subject.register
-      subject.multi_receive([])
-    end
+  before do
+    subject.register
+    subject.multi_receive([])
+  end
 
-    shared_examples "an indexer" do
-      it "ships events" do
-        subject.multi_receive(events)
+  shared_examples "an indexer" do
+    it "ships events" do
+      subject.multi_receive(events)
 
-        http_client.post("#{es_url}/_refresh").call
+      http_client.post("#{es_url}/_refresh").call
 
-        response = http_client.get("#{index_url}/_count?q=*")
-        result = LogStash::Json.load(response.body)
-        cur_count = result["count"]
-        expect(cur_count).to eq(event_count)
+      response = http_client.get("#{index_url}/_count?q=*")
+      result = LogStash::Json.load(response.body)
+      cur_count = result["count"]
+      expect(cur_count).to eq(event_count)
 
-        response = http_client.get("#{index_url}/_search?q=*&size=1000")
-        result = LogStash::Json.load(response.body)
-        result["hits"]["hits"].each do |doc|
-          expect(doc["_type"]).to eq(type) if ESHelper.es_version_satisfies?(">= 6", "< 8")
-          expect(doc).not_to include("_type") if ESHelper.es_version_satisfies?(">= 8")
-          expect(doc["_index"]).to eq(index)
+      response = http_client.get("#{index_url}/_search?q=*&size=1000")
+      result = LogStash::Json.load(response.body)
+      result["hits"]["hits"].each do |doc|
+        if ESHelper.es_version_satisfies?("< 8")
+          expect(doc["_type"]).to eq(type)
+        else
+          expect(doc).not_to include("_type")
         end
+        expect(doc["_index"]).to eq(index)
       end
     end
+  end
 
-    it "sets the correct content-encoding header and body is compressed" do
-      expect(subject.client.pool.adapter.client).to receive(:send).
-        with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "Content-Type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
-        and_call_original
-      subject.multi_receive(events)
-    end
-
-    it_behaves_like("an indexer")
+  it "sets the correct content-encoding header and body is compressed" do
+    expect(subject.client.pool.adapter.client).to receive(:send).
+      with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "Content-Type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
+      and_call_original
+    subject.multi_receive(events)
   end
+
+  it_behaves_like("an indexer")
 end