logstash-output-elasticsearch 11.15.0-java → 11.15.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e99f37b77580192819e3215149302e093a99906de26cdf613e4b6c6b24dacda5
-  data.tar.gz: 6a18546d9e7f1fab675c4a88c1728646613cff63dc87dab5a2bac1756642b91f
+  metadata.gz: d000a73f7c5397c4e4643c0d1dbb4efb0c04b290054b0d455f020ec3443765d5
+  data.tar.gz: 9330ee2b0e8a903cc966c502fdc321e39fb2ee15b7b2782f2539567743346d71
 SHA512:
-  metadata.gz: 3fb0cf262bf06830ae5089095f5421920f4ffad6936060df74286bf6aadbd18f729bca32021df94f7cb6251b48a6d25d7e00b849d83a5534718f2b0248c73e9c
-  data.tar.gz: 50ddf8e65adb4d9b4e67519edaf19bf9b1f127fecc3d8b7736505d20bde0eac822c7d554f208aeed812efe3166940fba7655e484b40ce217785e92136594ad6a
+  metadata.gz: ddcee04fc7e64d4dd77b6c868c34e65e9fff345c7d1d2c37da5752ae612f09ae8147d0686cfdcdee8494c9c6e6c6045a204ddc157db959733c677f719ca958f9
+  data.tar.gz: '032939b34d7fbf6b4107295ea12c80ea651da05b29290154da37e6ff2b04cb123dcc07dee3bdce2ba184f06d2a2d0b4036dac9b3d8099e403bfda836efb4f29e'

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.15.2
+ - Restores DLQ logging behavior from 11.8.x to include the action-tuple as structured [#1105](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1105)
+
+## 11.15.1
+ - Move async finish_register to bottom of register to avoid race condition [#1125](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1125)
+
 ## 11.15.0
  - Added the ability to negatively acknowledge the batch under processing if the plugin is blocked in a retry-error-loop and a shutdown is requested. [#1119](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1119)
 

data/lib/logstash/outputs/elasticsearch.rb

@@ -313,6 +313,27 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     data_stream_enabled = data_stream_config?
 
     setup_template_manager_defaults(data_stream_enabled)
+    # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
+    @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
+
+    @dlq_codes = DOC_DLQ_CODES.to_set
+
+    if dlq_enabled?
+      check_dlq_custom_codes
+      @dlq_codes.merge(dlq_custom_codes)
+    else
+      raise LogStash::ConfigurationError, "DLQ feature (dlq_custom_codes) is configured while DLQ is not enabled" unless dlq_custom_codes.empty?
+    end
+
+    setup_mapper_and_target(data_stream_enabled)
+
+    @bulk_request_metrics = metric.namespace(:bulk_requests)
+    @document_level_metrics = metric.namespace(:documents)
+
+    if ecs_compatibility == :v8
+      @logger.warn("Elasticsearch Output configured with `ecs_compatibility => v8`, which resolved to an UNRELEASED preview of version 8.0.0 of the Elastic Common Schema. " +
+                   "Once ECS v8 and an updated release of this plugin are publicly available, you will need to update this plugin to resolve this warning.")
+    end
 
     @after_successful_connection_thread = after_successful_connection do
       begin
@@ -326,18 +347,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
       end
     end
 
-    # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
-    @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
-
-    @dlq_codes = DOC_DLQ_CODES.to_set
-
-    if dlq_enabled?
-      check_dlq_custom_codes
-      @dlq_codes.merge(dlq_custom_codes)
-    else
-      raise LogStash::ConfigurationError, "DLQ feature (dlq_custom_codes) is configured while DLQ is not enabled" unless dlq_custom_codes.empty?
-    end
+  end
 
+  def setup_mapper_and_target(data_stream_enabled)
     if data_stream_enabled
       @event_mapper = -> (e) { data_stream_event_action_tuple(e) }
       @event_target = -> (e) { data_stream_name(e) }
@@ -346,14 +358,6 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
       @event_mapper = -> (e) { event_action_tuple(e) }
       @event_target = -> (e) { e.sprintf(@index) }
     end
-
-    @bulk_request_metrics = metric.namespace(:bulk_requests)
-    @document_level_metrics = metric.namespace(:documents)
-
-    if ecs_compatibility == :v8
-      @logger.warn("Elasticsearch Output configured with `ecs_compatibility => v8`, which resolved to an UNRELEASED preview of version 8.0.0 of the Elastic Common Schema. " +
-                   "Once ECS v8 and an updated release of this plugin are publicly available, you will need to update this plugin to resolve this warning.")
-    end
   end
 
   # @override post-register when ES connection established
@@ -401,7 +405,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
     event_mapping_errors.each do |event_mapping_error|
       detailed_message = "#{event_mapping_error.message}; event: `#{event_mapping_error.event.to_hash_with_metadata}`"
-      handle_dlq_status(event_mapping_error.event, :warn, detailed_message)
+      @dlq_writer ? @dlq_writer.write(event_mapping_error.event, detailed_message) : @logger.warn(detailed_message)
     end
     @document_level_metrics.increment(:non_retryable_failures, event_mapping_errors.size)
   end
@@ -418,7 +422,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
         successful_events << @event_mapper.call(event)
       rescue EventMappingError => ie
         event_mapping_errors << FailedEventMapping.new(event, ie.message)
-       end
+      end
     end
     MapEventsResult.new(successful_events, event_mapping_errors)
   end

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -227,22 +227,16 @@ module LogStash; module PluginMixins; module ElasticSearch
     end
 
     def handle_dlq_response(message, action, status, response)
-      _, action_params = action.event, [action[0], action[1], action[2]]
+      event, action_params = action.event, [action[0], action[1], action[2]]
 
-      # TODO: Change this to send a map with { :status => status, :action => action } in the future
-      detailed_message = "#{message} status: #{status}, action: #{action_params}, response: #{response}"
-
-      log_level = dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception' ? :error : :warn
-
-      handle_dlq_status(action.event, log_level, detailed_message)
-    end
-
-    def handle_dlq_status(event, log_level, message)
-      # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
       if @dlq_writer
-        @dlq_writer.write(event, "#{message}")
+        # TODO: Change this to send a map with { :status => status, :action => action } in the future
+        detailed_message = "#{message} status: #{status}, action: #{action_params}, response: #{response}"
+        @dlq_writer.write(event, "#{detailed_message}")
       else
-        @logger.send log_level, message
+        log_level = dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception' ? :error : :warn
+
+        @logger.public_send(log_level, message, status: status, action: action_params, response: response)
       end
     end
 

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.15.0'
+  s.version         = '11.15.2'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/es_spec_helper.rb

@@ -67,19 +67,24 @@ module ESHelper
   end
 
   RSpec::Matchers.define :have_hits do |expected|
+    hits_count_path = ESHelper.es_version_satisfies?(">=7") ? %w(hits total value) : %w(hits total)
+
     match do |actual|
-      if ESHelper.es_version_satisfies?(">=7")
-        expected == actual['hits']['total']['value']
-      else
-        expected == actual['hits']['total']
-      end
+      @actual_hits_count = actual&.dig(*hits_count_path)
+      values_match? expected, @actual_hits_count
+    end
+    failure_message do |actual|
+      "expected that #{actual} with #{@actual_hits_count || "UNKNOWN" } hits would have #{expected} hits"
     end
   end
 
   RSpec::Matchers.define :have_index_pattern do |expected|
     match do |actual|
-      test_against = Array(actual['index_patterns'].nil? ? actual['template'] : actual['index_patterns'])
-      test_against.include?(expected)
+      @actual_index_pattterns = Array(actual['index_patterns'].nil? ? actual['template'] : actual['index_patterns'])
+      @actual_index_pattterns.any? { |v| values_match? expected, v }
+    end
+    failure_message do |actual|
+      "expected that #{actual} with index patterns #{@actual_index_pattterns} would have included `#{expected}`"
     end
   end
 

data/spec/integration/outputs/templates_spec.rb

@@ -23,7 +23,9 @@ describe "index template expected behavior", :integration => true do
 
     elasticsearch_client.indices.delete_template(:name => '*')
     # This can fail if there are no indexes, ignore failure.
-    elasticsearch_client.indices.delete(:index => '*') rescue nil
+    elasticsearch_client.indices.delete(:index => '*') rescue puts("DELETE INDICES ERROR: #{$!}")
+    # Since we are pinned to ES client 7.x, we need to delete data streams the hard way...
+    elasticsearch_client.perform_request("DELETE", "/_data_stream/*") rescue puts("DELETE DATA STREAMS ERROR: #{$!}")
   end
 
   context 'with ecs_compatibility => disabled' do

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -1121,41 +1121,63 @@ describe LogStash::Outputs::ElasticSearch do
   end if LOGSTASH_VERSION > '6.0'
 
   context 'handling elasticsearch document-level status meant for the DLQ' do
+    let(:es_api_action) { "CUSTOM_ACTION" }
+    let(:es_api_params) { Hash['_index' => 'MY_INDEX'] }
+
     let(:options) { { "manage_template" => false, "data_stream" => 'false' } }
-    let(:action) { LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, LogStash::Event.new("foo" => "bar")) }
+    let(:action) { LogStash::Outputs::ElasticSearch::EventActionTuple.new(es_api_action, es_api_params, LogStash::Event.new("foo" => "bar")) }
+
+    let(:logger) { double('logger').as_null_object }
+    before(:each) { subject.instance_variable_set(:@logger, logger) }
 
     context 'when @dlq_writer is nil' do
       before { subject.instance_variable_set '@dlq_writer', nil }
-      let(:action) { LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, LogStash::Event.new("foo" => "bar")) }
 
       context 'resorting to previous behaviour of logging the error' do
         context 'getting an invalid_index_name_exception' do
           it 'should log at ERROR level' do
-            subject.instance_variable_set(:@logger, double("logger").as_null_object)
+            # logger = double("logger").as_null_object
+            # subject.instance_variable_set(:@logger, logger)
+
             mock_response = { 'index' => { 'error' => { 'type' => 'invalid_index_name_exception' } } }
             subject.handle_dlq_response("Could not index event to Elasticsearch.", action, :some_status, mock_response)
+
+            expect(logger).to have_received(:error).with(a_string_including("Could not index event to Elasticsearch"),
+                                                         a_hash_including(:status => :some_status,
+                                                                          :action => [es_api_action, es_api_params, action.event.to_hash],
+                                                                          :response => mock_response))
           end
         end
 
         context 'when getting any other exception' do
           it 'should log at WARN level' do
-            logger = double("logger").as_null_object
-            subject.instance_variable_set(:@logger, logger)
-            expect(logger).to receive(:warn).with(a_string_including "Could not index event to Elasticsearch. status: some_status, action: [:action, :params, {")
+            # logger = double("logger").as_null_object
+            # subject.instance_variable_set(:@logger, logger)
+
             mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
             subject.handle_dlq_response("Could not index event to Elasticsearch.", action, :some_status, mock_response)
+
+            expect(logger).to have_received(:warn).with(a_string_including("Could not index event to Elasticsearch"),
+                                                        a_hash_including(:status => :some_status,
+                                                                         :action => [es_api_action, es_api_params, action.event.to_hash],
+                                                                         :response => mock_response))
           end
         end
 
         context 'when the response does not include [error]' do
           it 'should not fail, but just log a warning' do
-            logger = double("logger").as_null_object
-            subject.instance_variable_set(:@logger, logger)
-            expect(logger).to receive(:warn).with(a_string_including "Could not index event to Elasticsearch. status: some_status, action: [:action, :params, {")
+            # logger = double("logger").as_null_object
+            # subject.instance_variable_set(:@logger, logger)
+
             mock_response = { 'index' => {} }
             expect do
               subject.handle_dlq_response("Could not index event to Elasticsearch.", action, :some_status, mock_response)
             end.to_not raise_error
+
+            expect(logger).to have_received(:warn).with(a_string_including("Could not index event to Elasticsearch"),
+                                                        a_hash_including(:status => :some_status,
+                                                                         :action => [es_api_action, es_api_params, action.event.to_hash],
+                                                                         :response => mock_response))
           end
         end
       end
@@ -1175,6 +1197,8 @@ describe LogStash::Outputs::ElasticSearch do
         mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
         action = LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, event)
         subject.handle_dlq_response("Could not index event to Elasticsearch.", action, 404, mock_response)
+
+        expect(logger).to_not have_received(:warn).with(a_string_including("Could not index event to Elasticsearch"))
       end
     end
 
@@ -1456,6 +1480,42 @@ describe LogStash::Outputs::ElasticSearch do
       end
 
     end
+
+    context 'during register/finish_register' do
+
+      let(:options) { { 'hosts' => '127.0.0.1:9999', 'data_stream' => 'true' } }
+      let(:es_version) { '8.7.0' } # DS default on LS 8.x
+
+      before do
+        allow(subject).to receive(:discover_cluster_uuid)
+        allow(subject).to receive(:maybe_create_rollover_alias)
+        allow(subject).to receive(:maybe_create_ilm_policy)
+        allow(subject).to receive(:build_client)
+      end
+
+      # this test could not have been done using latches as the fix to the race
+      # condition alters the order of the instructions in elasticsearch.rb
+      #
+      # the race condition happened when the @index was set to the datastream
+      # after `ilm_in_use?` is called but before `setup_ilm`
+      it 'doesn\'t have a race condition leading to resetting back to ILM' do
+        ilm_in_use = subject.method(:ilm_in_use?)
+        expect(subject).to receive(:ilm_in_use?) do |params|
+          ret = ilm_in_use.call
+          sleep 3
+          ret
+        end
+        setup_mapper_and_target = subject.method(:setup_mapper_and_target)
+        expect(subject).to receive(:setup_mapper_and_target).once do |params|
+          sleep 1
+          setup_mapper_and_target.call(params)
+        end
+        subject.register
+        sleep 6
+        expect(subject.index).to eq("logs-generic-default")
+      end
+
+    end
   end
 
   @private

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.15.0
+  version: 11.15.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-04 00:00:00.000000000 Z
+date: 2023-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement