logstash-output-elasticsearch 11.0.0-java → 11.0.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd0e368ed484aa214da94fcdc6978f919b3139cf90f8db462aba17f9c1e86670
-  data.tar.gz: 67b475fdd703d50d7bbb806adebd46c3b6657ead3019e76c7517e3c2428335be
+  metadata.gz: beee28e62f0872a041adc3c0f2c8546dbca09ced5c8a8e261cf1c9d07a0e64fe
+  data.tar.gz: f3f88dbbdf310557151a2f819ea566413d9f798be38d4717261c9dc544db97f1
 SHA512:
-  metadata.gz: f1e85fea62c9173d0ffdbf739487d8bcbbcfb41f304893e37333d22a8f42b0f527506d926c9f983480ef1c76eee43869ded32b84df98e19fdd999b9d5a26baa9
-  data.tar.gz: 0d1df95d541fa6e1d1161763051a3b4dc8fed10f62d878e7b4aa556d00c17bf6daf44a2fdc0c356c14c9c5431457112da601942dcf34dbf5386b05235933146e
+  metadata.gz: b43c800a66632c08a8c9c6d35367de3a127ebbf0abdfb0757cea7fb22501f4dc97e395e4b1ae2d57065c2c913a58e7f62dfa9a5cc4761c2f5c449d1f54448a4d
+  data.tar.gz: ff012fb8f3133de9f6002a35d733ea178828970f530c5c8f36239fdda605756f6d76b8a550690e2ac52aa64c551b73889453443ddc17f945a33b1f02be2abd0b

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 11.0.1
+ - Fix: DLQ regression shipped in 11.0.0 [#1012](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1012)
+ - [DOC] Fixed broken link in list item [#1011](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1011)
+
 ## 11.0.0
  - Feat: Data stream support [#988](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/988)
  - Refactor: reviewed logging format + restored ES (initial) setup error logging

data/docs/index.asciidoc

@@ -71,7 +71,7 @@ as logs, events, and metrics) and non-time series data in Elasticsearch.
 The data stream options are recommended for indexing time series datasets (such
 as logs, metrics, and events) into {es}:
 
-* <<plugins-{type}s-{plugin}-data_stream>> |<<string,string>>
+* <<plugins-{type}s-{plugin}-data_stream>>
 * <<plugins-{type}s-{plugin}-data_stream_auto_routing>> 
 * <<plugins-{type}s-{plugin}-data_stream_dataset>> 
 * <<plugins-{type}s-{plugin}-data_stream_namespace>>

data/lib/logstash/outputs/elasticsearch.rb

@@ -392,7 +392,21 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     params[:version] = event.sprintf(@version) if @version
     params[:version_type] = event.sprintf(@version_type) if @version_type
 
-    [action, params, event.to_hash]
+    EventActionTuple.new(action, params, event)
+  end
+
+  class EventActionTuple < Array # TODO: acting as an array for compatibility
+
+    def initialize(action, params, event, event_data = nil)
+      super(3)
+      self[0] = action
+      self[1] = params
+      self[2] = event_data || event.to_hash
+      @event = event
+    end
+
+    attr_reader :event
+
   end
 
   # @return Hash (initial) parameters for given event

data/lib/logstash/outputs/elasticsearch/data_stream_support.rb

@@ -146,7 +146,7 @@ module LogStash module Outputs class ElasticSearch
     def data_stream_event_action_tuple(event)
       event_data = event.to_hash
       data_stream_event_sync(event_data) if data_stream_sync_fields
-      ['create', common_event_params(event), event_data] # action always 'create'
+      EventActionTuple.new('create', common_event_params(event), event, event_data)
     end
 
     DATA_STREAM_SYNC_FIELDS = [ 'type', 'dataset', 'namespace' ].freeze

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -200,8 +200,9 @@ module LogStash; module PluginMixins; module ElasticSearch
     def handle_dlq_status(message, action, status, response)
       # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
       if @dlq_writer
+        event, action = action.event, [action[0], action[1], action[2]]
         # TODO: Change this to send a map with { :status => status, :action => action } in the future
-        @dlq_writer.write(action[2], "#{message} status: #{status}, action: #{action}, response: #{response}")
+        @dlq_writer.write(event, "#{message} status: #{status}, action: #{action}, response: #{response}")
       else
         if dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception'
           level = :error

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.0.0'
+  s.version         = '11.0.1'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -343,7 +343,7 @@ describe LogStash::Outputs::ElasticSearch do
                               }
                     }]
         }
-    end
+      end
 
       before(:each) do
         allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array)) do |stream, actions|
@@ -771,9 +771,9 @@ describe LogStash::Outputs::ElasticSearch do
 
         context 'when getting any other exception' do
           it 'should log at WARN level' do
-            dlog = double_logger = double("logger").as_null_object
-            subject.instance_variable_set(:@logger, dlog)
-            expect(dlog).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
+            logger = double("logger").as_null_object
+            subject.instance_variable_set(:@logger, logger)
+            expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
             mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
             subject.handle_dlq_status("Could not index event to Elasticsearch.",
               [:action, :params, :event], :some_status, mock_response)
@@ -782,9 +782,9 @@ describe LogStash::Outputs::ElasticSearch do
 
         context 'when the response does not include [error]' do
           it 'should not fail, but just log a warning' do
-            dlog = double_logger = double("logger").as_null_object
-            subject.instance_variable_set(:@logger, dlog)
-            expect(dlog).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
+            logger = double("logger").as_null_object
+            subject.instance_variable_set(:@logger, logger)
+            expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
             mock_response = { 'index' => {} }
             expect do
               subject.handle_dlq_status("Could not index event to Elasticsearch.",
@@ -804,12 +804,55 @@ describe LogStash::Outputs::ElasticSearch do
       # We should still log when sending to the DLQ.
       # This shall be solved by another issue, however: logstash-output-elasticsearch#772
       it 'should send the event to the DLQ instead, and not log' do
-        expect(dlq_writer).to receive(:write).once.with(:event, /Could not index/)
+        event = LogStash::Event.new("foo" => "bar")
+        expect(dlq_writer).to receive(:write).once.with(event, /Could not index/)
         mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
-        subject.handle_dlq_status("Could not index event to Elasticsearch.",
-          [:action, :params, :event], :some_status, mock_response)
+        action = LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, event)
+        subject.handle_dlq_status("Could not index event to Elasticsearch.", action, 404, mock_response)
       end
     end
+
+    context 'with response status 400' do
+
+      let(:options) { super().merge 'document_id' => '%{foo}' }
+
+      let(:events) { [ LogStash::Event.new("foo" => "bar") ] }
+
+      let(:dlq_writer) { subject.instance_variable_get(:@dlq_writer) }
+
+      let(:bulk_response) do
+        {
+            "took"=>1, "ingest_took"=>11, "errors"=>true, "items"=>
+            [{
+                 "index"=>{"_index"=>"bar", "_type"=>"_doc", "_id"=>'bar', "status"=>400,
+                           "error"=>{"type" => "illegal_argument_exception", "reason" => "TEST" }
+                  }
+            }]
+        }
+      end
+
+      before(:each) do
+        allow(subject.client).to receive(:bulk_send).and_return(bulk_response)
+      end
+
+      it "should write event to DLQ" do
+        expect(dlq_writer).to receive(:write).and_wrap_original do |method, *args|
+          expect( args.size ).to eql 2
+
+          event, reason = *args
+          expect( event ).to be_a LogStash::Event
+          expect( event ).to be events.first
+          expect( reason ).to start_with 'Could not index event to Elasticsearch. status: 400, action: ["index"'
+          expect( reason ).to match /_id=>"bar".*"foo"=>"bar".*response:.*"reason"=>"TEST"/
+
+          method.call(*args) # won't hurt to call LogStash::Util::DummyDeadLetterQueueWriter
+        end.once
+
+        event_action_tuples = subject.map_events(events)
+        subject.send(:submit, event_action_tuples)
+      end
+
+    end if LOGSTASH_VERSION > '7.0'
   end
 
   describe "custom headers" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.0.0
+  version: 11.0.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-19 00:00:00.000000000 Z
+date: 2021-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement