logstash-output-elasticsearch 11.15.1-java → 11.15.4-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ad6b80e4c6db7a54c8b7a830f6a922a7057b2e7585820db8a6c093b57db3800
-  data.tar.gz: 82cd893394fc7ac08ae7a79b2cd953f80fbf258ec2faad225300a2ce516ab208
+  metadata.gz: 35c442c72cb3629cc3d17b28bffce6351c6e61a6332c254bef41ad89c9cc2810
+  data.tar.gz: 396df2b1625d0b62f8f71982203f01f1fa8901a6ccaf2098e10b51c4f77c9d80
 SHA512:
-  metadata.gz: b861d195377c3eeadf4489d21780be6627597aa0b93992bd7378b7cf680f6f99ac120ae342a4b48739ee6ef2652aa9238ee9fc08838cbed0f6dae0c887b06daa
-  data.tar.gz: 16091be406132dbb590ad76a0ba2e83cb0729717d8229b410bccbcabc660a7a26146fd2c7aaae6976f954bd17f1f763c6353552699ffd67173efb9917b913658
+  metadata.gz: 3acda17f89403df63a709bedb84481c259178c97963120cce9452565d1d93981ac4a8b3766c9de2ee356d540f38109afbaf03cba3de17402551ee55cc09e74ce
+  data.tar.gz: fb405743b1746631fe84e4b346810df84b21955bb7ef0d5adf122d03d9e7fc739fde8623947fc9ae94d1139cb34ecef869eb18141344231183d844a334b3b6dd

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 11.15.4
+  - Improved connection handling under several partial-failure scenarios [#1130](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1130)
+   - Ensures an HTTP connection can be established before adding the connection to the pool
+   - Ensures that the version of the connected Elasticsearch is retrieved _successfully_ before the connection is added to the pool.
+   - Fixes a crash that could occur when the plugin is configured to connect to a live HTTP resource that is _not_ Elasticsearch
+
+## 11.15.3
+  -  Removes the ECS v8 unreleased preview warning [#1131](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1131)
+
+## 11.15.2
+ - Restores DLQ logging behavior from 11.8.x to include the action-tuple as structured [#1105](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1105)
+
 ## 11.15.1
  - Move async finish_register to bottom of register to avoid race condition [#1125](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1125)
 

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -229,14 +229,16 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def health_check_request(url)
-      logger.debug("Running health check to see if an ES connection is working", url: url.sanitized.to_s, path: @healthcheck_path)
-      perform_request_to_url(url, :head, @healthcheck_path)
+      response = perform_request_to_url(url, :head, @healthcheck_path)
+      raise BadResponseCodeError.new(response.code, url, nil, response.body) unless (200..299).cover?(response.code)
     end
 
     def healthcheck!(register_phase = true)
       # Try to keep locking granularity low such that we don't affect IO...
       @state_mutex.synchronize { @url_info.select {|url,meta| meta[:state] != :alive } }.each do |url,meta|
         begin
+          logger.debug("Running health check to see if an Elasticsearch connection is working",
+                        :healthcheck_url => url.sanitized.to_s, :path => @healthcheck_path)
           health_check_request(url)
 
           # when called from resurrectionist skip the product check done during register phase
@@ -249,6 +251,10 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
           logger.warn("Restored connection to ES instance", url: url.sanitized.to_s)
           # We reconnected to this node, check its ES version
           es_version = get_es_version(url)
+          if es_version.nil?
+            logger.warn("Failed to retrieve Elasticsearch version data from connected endpoint, connection aborted", :url => url.sanitized.to_s)
+            next
+          end
           @state_mutex.synchronize do
             meta[:version] = es_version
             set_last_es_version(es_version, url)
@@ -464,8 +470,12 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def get_es_version(url)
-      request = perform_request_to_url(url, :get, ROOT_URI_PATH)
-      LogStash::Json.load(request.body)["version"]["number"] # e.g. "7.10.0"
+      response = perform_request_to_url(url, :get, ROOT_URI_PATH)
+      return nil unless (200..299).cover?(response.code)
+
+      response = LogStash::Json.load(response.body)
+
+      response.fetch('version', {}).fetch('number', nil)
     end
 
     def last_es_version

data/lib/logstash/outputs/elasticsearch.rb

@@ -330,11 +330,6 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     @bulk_request_metrics = metric.namespace(:bulk_requests)
     @document_level_metrics = metric.namespace(:documents)
 
-    if ecs_compatibility == :v8
-      @logger.warn("Elasticsearch Output configured with `ecs_compatibility => v8`, which resolved to an UNRELEASED preview of version 8.0.0 of the Elastic Common Schema. " +
-                   "Once ECS v8 and an updated release of this plugin are publicly available, you will need to update this plugin to resolve this warning.")
-    end
-
     @after_successful_connection_thread = after_successful_connection do
       begin
         finish_register
@@ -405,7 +400,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
     event_mapping_errors.each do |event_mapping_error|
       detailed_message = "#{event_mapping_error.message}; event: `#{event_mapping_error.event.to_hash_with_metadata}`"
-      handle_dlq_status(event_mapping_error.event, :warn, detailed_message)
+      @dlq_writer ? @dlq_writer.write(event_mapping_error.event, detailed_message) : @logger.warn(detailed_message)
     end
     @document_level_metrics.increment(:non_retryable_failures, event_mapping_errors.size)
   end
@@ -422,7 +417,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
         successful_events << @event_mapper.call(event)
       rescue EventMappingError => ie
         event_mapping_errors << FailedEventMapping.new(event, ie.message)
-       end
+      end
     end
     MapEventsResult.new(successful_events, event_mapping_errors)
   end

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -227,22 +227,16 @@ module LogStash; module PluginMixins; module ElasticSearch
     end
 
     def handle_dlq_response(message, action, status, response)
-      _, action_params = action.event, [action[0], action[1], action[2]]
+      event, action_params = action.event, [action[0], action[1], action[2]]
 
-      # TODO: Change this to send a map with { :status => status, :action => action } in the future
-      detailed_message = "#{message} status: #{status}, action: #{action_params}, response: #{response}"
-
-      log_level = dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception' ? :error : :warn
-
-      handle_dlq_status(action.event, log_level, detailed_message)
-    end
-
-    def handle_dlq_status(event, log_level, message)
-      # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
       if @dlq_writer
-        @dlq_writer.write(event, "#{message}")
+        # TODO: Change this to send a map with { :status => status, :action => action } in the future
+        detailed_message = "#{message} status: #{status}, action: #{action_params}, response: #{response}"
+        @dlq_writer.write(event, "#{detailed_message}")
       else
-        @logger.send log_level, message
+        log_level = dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception' ? :error : :warn
+
+        @logger.public_send(log_level, message, status: status, action: action_params, response: response)
       end
     end
 

data/logstash-output-elasticsearch.gemspec

@@ -1,12 +1,12 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.15.1'
+  s.version         = '11.15.4'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
   s.email           = 'info@elastic.co'
-  s.homepage        = "http://logstash.net/"
+  s.homepage        = "https://www.elastic.co/guide/en/logstash/current/index.html"
   s.require_paths = ["lib"]
 
   s.platform = RUBY_PLATFORM

data/spec/es_spec_helper.rb

@@ -67,19 +67,24 @@ module ESHelper
   end
 
   RSpec::Matchers.define :have_hits do |expected|
+    hits_count_path = ESHelper.es_version_satisfies?(">=7") ? %w(hits total value) : %w(hits total)
+
     match do |actual|
-      if ESHelper.es_version_satisfies?(">=7")
-        expected == actual['hits']['total']['value']
-      else
-        expected == actual['hits']['total']
-      end
+      @actual_hits_count = actual&.dig(*hits_count_path)
+      values_match? expected, @actual_hits_count
+    end
+    failure_message do |actual|
+      "expected that #{actual} with #{@actual_hits_count || "UNKNOWN" } hits would have #{expected} hits"
     end
   end
 
   RSpec::Matchers.define :have_index_pattern do |expected|
     match do |actual|
-      test_against = Array(actual['index_patterns'].nil? ? actual['template'] : actual['index_patterns'])
-      test_against.include?(expected)
+      @actual_index_pattterns = Array(actual['index_patterns'].nil? ? actual['template'] : actual['index_patterns'])
+      @actual_index_pattterns.any? { |v| values_match? expected, v }
+    end
+    failure_message do |actual|
+      "expected that #{actual} with index patterns #{@actual_index_pattterns} would have included `#{expected}`"
     end
   end
 

data/spec/integration/outputs/templates_spec.rb

@@ -23,7 +23,9 @@ describe "index template expected behavior", :integration => true do
 
     elasticsearch_client.indices.delete_template(:name => '*')
     # This can fail if there are no indexes, ignore failure.
-    elasticsearch_client.indices.delete(:index => '*') rescue nil
+    elasticsearch_client.indices.delete(:index => '*') rescue puts("DELETE INDICES ERROR: #{$!}")
+    # Since we are pinned to ES client 7.x, we need to delete data streams the hard way...
+    elasticsearch_client.perform_request("DELETE", "/_data_stream/*") rescue puts("DELETE DATA STREAMS ERROR: #{$!}")
   end
 
   context 'with ecs_compatibility => disabled' do
@@ -48,19 +50,19 @@ describe "index template expected behavior", :integration => true do
 
       # Wait or fail until everything's indexed.
       Stud::try(20.times) do
-        r = @es.search(index: 'logstash-*')
+        r = @es.search(index: 'logstash*')
         expect(r).to have_hits(8)
       end
     end
 
     it "permits phrase searching on string fields" do
-      results = @es.search(:q => "message:\"sample message\"")
+      results = @es.search(index: 'logstash*', q: "message:\"sample message\"")
       expect(results).to have_hits(1)
       expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
     end
 
     it "numbers dynamically map to a numeric type and permit range queries" do
-      results = @es.search(:q => "somevalue:[5 TO 105]")
+      results = @es.search(index: 'logstash*', q: "somevalue:[5 TO 105]")
       expect(results).to have_hits(2)
 
       values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
@@ -70,22 +72,22 @@ describe "index template expected behavior", :integration => true do
     end
 
     it "does not create .keyword field for top-level message field" do
-      results = @es.search(:q => "message.keyword:\"sample message here\"")
+      results = @es.search(index: 'logstash*', q: "message.keyword:\"sample message here\"")
       expect(results).to have_hits(0)
     end
 
     it "creates .keyword field for nested message fields" do
-      results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
+      results = @es.search(index: 'logstash*', q: "somemessage.message.keyword:\"sample nested message here\"")
       expect(results).to have_hits(1)
     end
 
     it "creates .keyword field from any string field which is not_analyzed" do
-      results = @es.search(:q => "country.keyword:\"us\"")
+      results = @es.search(index: 'logstash*', q: "country.keyword:\"us\"")
       expect(results).to have_hits(1)
       expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
 
       # partial or terms should not work.
-      results = @es.search(:q => "country.keyword:\"u\"")
+      results = @es.search(index: 'logstash*', q: "country.keyword:\"u\"")
       expect(results).to have_hits(0)
     end
 
@@ -94,7 +96,7 @@ describe "index template expected behavior", :integration => true do
     end
 
     it "aggregate .keyword results correctly " do
-      results = @es.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
+      results = @es.search(index: 'logstash*', body: { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
       terms = results["buckets"].collect { |b| b["key"] }
 
       expect(terms).to include("us")

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -50,6 +50,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
 
     describe "healthcheck url handling" do
       let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
+      let(:success_response) { double("Response", :code => 200) }
+
       before(:example) do
         expect(adapter).to receive(:perform_request).with(anything, :get, "/", anything, anything) do |url, _, _, _, _|
           expect(url.path).to be_empty
@@ -60,6 +62,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
         it "performs the healthcheck to the root" do
           expect(adapter).to receive(:perform_request).with(anything, :head, "/", anything, anything) do |url, _, _, _, _|
             expect(url.path).to be_empty
+
+            success_response
           end
           expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
         end
@@ -71,6 +75,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
         it "performs the healthcheck to the healthcheck_path" do
           expect(adapter).to receive(:perform_request).with(anything, :head, eq(healthcheck_path), anything, anything) do |url, _, _, _, _|
             expect(url.path).to be_empty
+
+            success_response
           end
           expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
         end
@@ -197,9 +203,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
                                  "build_flavor" => 'default'}
                                })
       end
+      let(:success_response) { double("head_req", :code => 200)}
 
       before :each do
-        allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil)
+        allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil).and_return(success_response)
         allow(adapter).to receive(:perform_request).with(anything, :get, subject.healthcheck_path, {}, nil).and_return(version_ok)
       end
       let(:initial_urls) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200"), ::LogStash::Util::SafeURI.new("http://localhost:9201"), ::LogStash::Util::SafeURI.new("http://localhost:9202") ] }
@@ -225,7 +232,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
         u,m = subject.get_connection
 
         # The resurrectionist will call this to check on the backend
-        response = double("response")
+        response = double("response", :code => 200)
         expect(adapter).to receive(:perform_request).with(u, :head, subject.healthcheck_path, {}, nil).and_return(response)
 
         subject.return_connection(u)

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -1121,41 +1121,63 @@ describe LogStash::Outputs::ElasticSearch do
   end if LOGSTASH_VERSION > '6.0'
 
   context 'handling elasticsearch document-level status meant for the DLQ' do
+    let(:es_api_action) { "CUSTOM_ACTION" }
+    let(:es_api_params) { Hash['_index' => 'MY_INDEX'] }
+
     let(:options) { { "manage_template" => false, "data_stream" => 'false' } }
-    let(:action) { LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, LogStash::Event.new("foo" => "bar")) }
+    let(:action) { LogStash::Outputs::ElasticSearch::EventActionTuple.new(es_api_action, es_api_params, LogStash::Event.new("foo" => "bar")) }
+
+    let(:logger) { double('logger').as_null_object }
+    before(:each) { subject.instance_variable_set(:@logger, logger) }
 
     context 'when @dlq_writer is nil' do
       before { subject.instance_variable_set '@dlq_writer', nil }
-      let(:action) { LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, LogStash::Event.new("foo" => "bar")) }
 
       context 'resorting to previous behaviour of logging the error' do
         context 'getting an invalid_index_name_exception' do
           it 'should log at ERROR level' do
-            subject.instance_variable_set(:@logger, double("logger").as_null_object)
+            # logger = double("logger").as_null_object
+            # subject.instance_variable_set(:@logger, logger)
+
             mock_response = { 'index' => { 'error' => { 'type' => 'invalid_index_name_exception' } } }
             subject.handle_dlq_response("Could not index event to Elasticsearch.", action, :some_status, mock_response)
+
+            expect(logger).to have_received(:error).with(a_string_including("Could not index event to Elasticsearch"),
+                                                         a_hash_including(:status => :some_status,
+                                                                          :action => [es_api_action, es_api_params, action.event.to_hash],
+                                                                          :response => mock_response))
           end
         end
 
         context 'when getting any other exception' do
           it 'should log at WARN level' do
-            logger = double("logger").as_null_object
-            subject.instance_variable_set(:@logger, logger)
-            expect(logger).to receive(:warn).with(a_string_including "Could not index event to Elasticsearch. status: some_status, action: [:action, :params, {")
+            # logger = double("logger").as_null_object
+            # subject.instance_variable_set(:@logger, logger)
+
             mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
             subject.handle_dlq_response("Could not index event to Elasticsearch.", action, :some_status, mock_response)
+
+            expect(logger).to have_received(:warn).with(a_string_including("Could not index event to Elasticsearch"),
+                                                        a_hash_including(:status => :some_status,
+                                                                         :action => [es_api_action, es_api_params, action.event.to_hash],
+                                                                         :response => mock_response))
           end
         end
 
         context 'when the response does not include [error]' do
           it 'should not fail, but just log a warning' do
-            logger = double("logger").as_null_object
-            subject.instance_variable_set(:@logger, logger)
-            expect(logger).to receive(:warn).with(a_string_including "Could not index event to Elasticsearch. status: some_status, action: [:action, :params, {")
+            # logger = double("logger").as_null_object
+            # subject.instance_variable_set(:@logger, logger)
+
             mock_response = { 'index' => {} }
             expect do
               subject.handle_dlq_response("Could not index event to Elasticsearch.", action, :some_status, mock_response)
             end.to_not raise_error
+
+            expect(logger).to have_received(:warn).with(a_string_including("Could not index event to Elasticsearch"),
+                                                        a_hash_including(:status => :some_status,
+                                                                         :action => [es_api_action, es_api_params, action.event.to_hash],
+                                                                         :response => mock_response))
           end
         end
       end
@@ -1175,6 +1197,8 @@ describe LogStash::Outputs::ElasticSearch do
         mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
         action = LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, event)
         subject.handle_dlq_response("Could not index event to Elasticsearch.", action, 404, mock_response)
+
+        expect(logger).to_not have_received(:warn).with(a_string_including("Could not index event to Elasticsearch"))
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.15.1
+  version: 11.15.4
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-20 00:00:00.000000000 Z
+date: 2023-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -335,7 +335,7 @@ files:
 - spec/unit/outputs/elasticsearch_ssl_spec.rb
 - spec/unit/outputs/error_whitelist_spec.rb
 - spec/unit/outputs/license_check_spec.rb
-homepage: http://logstash.net/
+homepage: https://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - apache-2.0
 metadata:
@@ -356,7 +356,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.29
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Stores logs in Elasticsearch