logstash-output-elasticsearch 11.0.0-java → 11.0.1-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: dd0e368ed484aa214da94fcdc6978f919b3139cf90f8db462aba17f9c1e86670
4
- data.tar.gz: 67b475fdd703d50d7bbb806adebd46c3b6657ead3019e76c7517e3c2428335be
3
+ metadata.gz: beee28e62f0872a041adc3c0f2c8546dbca09ced5c8a8e261cf1c9d07a0e64fe
4
+ data.tar.gz: f3f88dbbdf310557151a2f819ea566413d9f798be38d4717261c9dc544db97f1
5
5
  SHA512:
6
- metadata.gz: f1e85fea62c9173d0ffdbf739487d8bcbbcfb41f304893e37333d22a8f42b0f527506d926c9f983480ef1c76eee43869ded32b84df98e19fdd999b9d5a26baa9
7
- data.tar.gz: 0d1df95d541fa6e1d1161763051a3b4dc8fed10f62d878e7b4aa556d00c17bf6daf44a2fdc0c356c14c9c5431457112da601942dcf34dbf5386b05235933146e
6
+ metadata.gz: b43c800a66632c08a8c9c6d35367de3a127ebbf0abdfb0757cea7fb22501f4dc97e395e4b1ae2d57065c2c913a58e7f62dfa9a5cc4761c2f5c449d1f54448a4d
7
+ data.tar.gz: ff012fb8f3133de9f6002a35d733ea178828970f530c5c8f36239fdda605756f6d76b8a550690e2ac52aa64c551b73889453443ddc17f945a33b1f02be2abd0b
data/CHANGELOG.md CHANGED
@@ -1,3 +1,7 @@
1
+ ## 11.0.1
2
+ - Fix: DLQ regression shipped in 11.0.0 [#1012](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1012)
3
+ - [DOC] Fixed broken link in list item [#1011](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1011)
4
+
1
5
  ## 11.0.0
2
6
  - Feat: Data stream support [#988](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/988)
3
7
  - Refactor: reviewed logging format + restored ES (initial) setup error logging
data/docs/index.asciidoc CHANGED
@@ -71,7 +71,7 @@ as logs, events, and metrics) and non-time series data in Elasticsearch.
71
71
  The data stream options are recommended for indexing time series datasets (such
72
72
  as logs, metrics, and events) into {es}:
73
73
 
74
- * <<plugins-{type}s-{plugin}-data_stream>> |<<string,string>>
74
+ * <<plugins-{type}s-{plugin}-data_stream>>
75
75
  * <<plugins-{type}s-{plugin}-data_stream_auto_routing>>
76
76
  * <<plugins-{type}s-{plugin}-data_stream_dataset>>
77
77
  * <<plugins-{type}s-{plugin}-data_stream_namespace>>
@@ -392,7 +392,21 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
392
392
  params[:version] = event.sprintf(@version) if @version
393
393
  params[:version_type] = event.sprintf(@version_type) if @version_type
394
394
 
395
- [action, params, event.to_hash]
395
+ EventActionTuple.new(action, params, event)
396
+ end
397
+
398
+ class EventActionTuple < Array # TODO: acting as an array for compatibility
399
+
400
+ def initialize(action, params, event, event_data = nil)
401
+ super(3)
402
+ self[0] = action
403
+ self[1] = params
404
+ self[2] = event_data || event.to_hash
405
+ @event = event
406
+ end
407
+
408
+ attr_reader :event
409
+
396
410
  end
397
411
 
398
412
  # @return Hash (initial) parameters for given event
@@ -146,7 +146,7 @@ module LogStash module Outputs class ElasticSearch
146
146
  def data_stream_event_action_tuple(event)
147
147
  event_data = event.to_hash
148
148
  data_stream_event_sync(event_data) if data_stream_sync_fields
149
- ['create', common_event_params(event), event_data] # action always 'create'
149
+ EventActionTuple.new('create', common_event_params(event), event, event_data)
150
150
  end
151
151
 
152
152
  DATA_STREAM_SYNC_FIELDS = [ 'type', 'dataset', 'namespace' ].freeze
@@ -200,8 +200,9 @@ module LogStash; module PluginMixins; module ElasticSearch
200
200
  def handle_dlq_status(message, action, status, response)
201
201
  # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
202
202
  if @dlq_writer
203
+ event, action = action.event, [action[0], action[1], action[2]]
203
204
  # TODO: Change this to send a map with { :status => status, :action => action } in the future
204
- @dlq_writer.write(action[2], "#{message} status: #{status}, action: #{action}, response: #{response}")
205
+ @dlq_writer.write(event, "#{message} status: #{status}, action: #{action}, response: #{response}")
205
206
  else
206
207
  if dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception'
207
208
  level = :error
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-elasticsearch'
3
- s.version = '11.0.0'
3
+ s.version = '11.0.1'
4
4
 
5
5
  s.licenses = ['apache-2.0']
6
6
  s.summary = "Stores logs in Elasticsearch"
@@ -343,7 +343,7 @@ describe LogStash::Outputs::ElasticSearch do
343
343
  }
344
344
  }]
345
345
  }
346
- end
346
+ end
347
347
 
348
348
  before(:each) do
349
349
  allow(subject.client).to receive(:bulk_send).with(instance_of(StringIO), instance_of(Array)) do |stream, actions|
@@ -771,9 +771,9 @@ describe LogStash::Outputs::ElasticSearch do
771
771
 
772
772
  context 'when getting any other exception' do
773
773
  it 'should log at WARN level' do
774
- dlog = double_logger = double("logger").as_null_object
775
- subject.instance_variable_set(:@logger, dlog)
776
- expect(dlog).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
774
+ logger = double("logger").as_null_object
775
+ subject.instance_variable_set(:@logger, logger)
776
+ expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
777
777
  mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
778
778
  subject.handle_dlq_status("Could not index event to Elasticsearch.",
779
779
  [:action, :params, :event], :some_status, mock_response)
@@ -782,9 +782,9 @@ describe LogStash::Outputs::ElasticSearch do
782
782
 
783
783
  context 'when the response does not include [error]' do
784
784
  it 'should not fail, but just log a warning' do
785
- dlog = double_logger = double("logger").as_null_object
786
- subject.instance_variable_set(:@logger, dlog)
787
- expect(dlog).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
785
+ logger = double("logger").as_null_object
786
+ subject.instance_variable_set(:@logger, logger)
787
+ expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
788
788
  mock_response = { 'index' => {} }
789
789
  expect do
790
790
  subject.handle_dlq_status("Could not index event to Elasticsearch.",
@@ -804,12 +804,55 @@ describe LogStash::Outputs::ElasticSearch do
804
804
  # We should still log when sending to the DLQ.
805
805
  # This shall be solved by another issue, however: logstash-output-elasticsearch#772
806
806
  it 'should send the event to the DLQ instead, and not log' do
807
- expect(dlq_writer).to receive(:write).once.with(:event, /Could not index/)
807
+ event = LogStash::Event.new("foo" => "bar")
808
+ expect(dlq_writer).to receive(:write).once.with(event, /Could not index/)
808
809
  mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
809
- subject.handle_dlq_status("Could not index event to Elasticsearch.",
810
- [:action, :params, :event], :some_status, mock_response)
810
+ action = LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, event)
811
+ subject.handle_dlq_status("Could not index event to Elasticsearch.", action, 404, mock_response)
811
812
  end
812
813
  end
814
+
815
+ context 'with response status 400' do
816
+
817
+ let(:options) { super().merge 'document_id' => '%{foo}' }
818
+
819
+ let(:events) { [ LogStash::Event.new("foo" => "bar") ] }
820
+
821
+ let(:dlq_writer) { subject.instance_variable_get(:@dlq_writer) }
822
+
823
+ let(:bulk_response) do
824
+ {
825
+ "took"=>1, "ingest_took"=>11, "errors"=>true, "items"=>
826
+ [{
827
+ "index"=>{"_index"=>"bar", "_type"=>"_doc", "_id"=>'bar', "status"=>400,
828
+ "error"=>{"type" => "illegal_argument_exception", "reason" => "TEST" }
829
+ }
830
+ }]
831
+ }
832
+ end
833
+
834
+ before(:each) do
835
+ allow(subject.client).to receive(:bulk_send).and_return(bulk_response)
836
+ end
837
+
838
+ it "should write event to DLQ" do
839
+ expect(dlq_writer).to receive(:write).and_wrap_original do |method, *args|
840
+ expect( args.size ).to eql 2
841
+
842
+ event, reason = *args
843
+ expect( event ).to be_a LogStash::Event
844
+ expect( event ).to be events.first
845
+ expect( reason ).to start_with 'Could not index event to Elasticsearch. status: 400, action: ["index"'
846
+ expect( reason ).to match /_id=>"bar".*"foo"=>"bar".*response:.*"reason"=>"TEST"/
847
+
848
+ method.call(*args) # won't hurt to call LogStash::Util::DummyDeadLetterQueueWriter
849
+ end.once
850
+
851
+ event_action_tuples = subject.map_events(events)
852
+ subject.send(:submit, event_action_tuples)
853
+ end
854
+
855
+ end if LOGSTASH_VERSION > '7.0'
813
856
  end
814
857
 
815
858
  describe "custom headers" do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 11.0.0
4
+ version: 11.0.1
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-04-19 00:00:00.000000000 Z
11
+ date: 2021-04-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement