fluent-plugin-elasticsearch 5.2.2 → 5.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 62c61f26ed869460900c3f05d6c70bc4bbb06dd9747f3a23c92accd4c878756f
4
- data.tar.gz: 1ed0894c48a957b5359739303b8fef7e9df9ce8ec80b1b0101b2b17d40a72967
3
+ metadata.gz: 7e2737dc1ced4c50a3db85d71c83351e77a606d581fbd768261ca6613b1700f8
4
+ data.tar.gz: 49a84ff1ea184c4afd43e69fb6cbb89559926bbd6ac063196bd777281be17d2b
5
5
  SHA512:
6
- metadata.gz: c63a8ea452e6f3691497e4601b46393c851ae188b9583ebfeaacff507e7791206430145db01bea4298d7c0021d610e9e2a229da9cca5394bea731ae36e204b9b
7
- data.tar.gz: 1e4ed30e6d00e372781d8744262bc711937ddaffdfc92f9f0407c7f367f0e4ac7807ba635a119b3287a09d2780c9d7efbe2e1e031330a0aacd527a4ff2a177bd
6
+ metadata.gz: 157dbe3ab067ec279f2051a8b6c6ac25538821903b48e96a09687b547270796d7f6972d0707225c99cd36694e8a8668d32dc9ebba3599d24f08c814712c0849a
7
+ data.tar.gz: 73c611531aa95d5d03d8bd5146ab5f6812c30b3be634f5d1ca139b2ffc6206ccebb8915c4fa1e78d2d53706f6c14b7203342d235bd217a89ac9bf0135e3fec21
@@ -0,0 +1,6 @@
1
+ version: 2
2
+ updates:
3
+ - package-ecosystem: "github-actions"
4
+ directory: "/"
5
+ schedule:
6
+ interval: "weekly"
@@ -2,6 +2,9 @@ name: Testing on Ubuntu
2
2
  on:
3
3
  - push
4
4
  - pull_request
5
+ permissions:
6
+ contents: read
7
+
5
8
  jobs:
6
9
  build:
7
10
  runs-on: ${{ matrix.os }}
@@ -13,7 +16,7 @@ jobs:
13
16
  - ubuntu-latest
14
17
  name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
15
18
  steps:
16
- - uses: actions/checkout@v2
19
+ - uses: actions/checkout@v3
17
20
  - uses: ruby/setup-ruby@v1
18
21
  with:
19
22
  ruby-version: ${{ matrix.ruby }}
@@ -2,6 +2,9 @@ name: Testing on macOS
2
2
  on:
3
3
  - push
4
4
  - pull_request
5
+ permissions:
6
+ contents: read
7
+
5
8
  jobs:
6
9
  build:
7
10
  runs-on: ${{ matrix.os }}
@@ -13,7 +16,7 @@ jobs:
13
16
  - macOS-latest
14
17
  name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
15
18
  steps:
16
- - uses: actions/checkout@v2
19
+ - uses: actions/checkout@v3
17
20
  - uses: ruby/setup-ruby@v1
18
21
  with:
19
22
  ruby-version: ${{ matrix.ruby }}
@@ -2,6 +2,9 @@ name: Testing on Windows
2
2
  on:
3
3
  - push
4
4
  - pull_request
5
+ permissions:
6
+ contents: read
7
+
5
8
  jobs:
6
9
  build:
7
10
  runs-on: ${{ matrix.os }}
@@ -13,7 +16,7 @@ jobs:
13
16
  - windows-latest
14
17
  name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
15
18
  steps:
16
- - uses: actions/checkout@v2
19
+ - uses: actions/checkout@v3
17
20
  - uses: ruby/setup-ruby@v1
18
21
  with:
19
22
  ruby-version: ${{ matrix.ruby }}
data/History.md CHANGED
@@ -2,6 +2,13 @@
2
2
 
3
3
  ### [Unreleased]
4
4
 
5
+ ### 5.2.3
6
+ - Bump actions/checkout from 2 to 3 (#978)
7
+ - chore: Included githubactions in the dependabot config (#977)
8
+ - chore: Set permissions for GitHub actions (#972)
9
+ - Remove nested msgpack\_each in handle\_error (#970)
10
+ - do not overwrite @timestamp in data stream if it already exists in the record (#968)
11
+
5
12
  ### 5.2.2
6
13
  - Add missing top level class markers (#961)
7
14
  - Ensure use_record_as_seed for same records (#960)
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '5.2.2'
6
+ s.version = '5.2.3'
7
7
  s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
9
9
  s.description = %q{Elasticsearch output plugin for Fluent event collector}
@@ -35,7 +35,7 @@ class Fluent::Plugin::ElasticsearchErrorHandler
35
35
  end
36
36
  end
37
37
 
38
- def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
38
+ def handle_error(response, tag, chunk, bulk_message_count, extracted_values, unpacked_msg_arr)
39
39
  items = response['items']
40
40
  if items.nil? || !items.is_a?(Array)
41
41
  raise ElasticsearchVersionMismatch, "The response format was unrecognized: #{response}"
@@ -48,7 +48,11 @@ class Fluent::Plugin::ElasticsearchErrorHandler
48
48
  meta = {}
49
49
  header = {}
50
50
  affinity_target_indices = @plugin.get_affinity_target_indices(chunk)
51
- chunk.msgpack_each do |time, rawrecord|
51
+
52
+ unpacked_msg_arr.each do |msg|
53
+ time = msg[:time]
54
+ rawrecord = msg[:record]
55
+
52
56
  bulk_message = ''
53
57
  next unless rawrecord.is_a? Hash
54
58
  begin
@@ -827,6 +827,7 @@ EOC
827
827
  bulk_message = Hash.new { |h,k| h[k] = '' }
828
828
  header = {}
829
829
  meta = {}
830
+ unpackedMsgArr = {}
830
831
 
831
832
  tag = chunk.metadata.tag
832
833
  chunk_id = dump_unique_id_hex(chunk.unique_id)
@@ -851,9 +852,13 @@ EOC
851
852
  RequestInfo.new(host, nil, meta["_index".freeze], meta.delete("_alias".freeze))
852
853
  end
853
854
 
855
+ unpackedMsgArr[info] = [] if unpackedMsgArr[info].nil?
856
+ unpackedMsgArr[info] << {:time => time, :record => record}
857
+
854
858
  if split_request?(bulk_message, info)
855
859
  bulk_message.each do |info, msgs|
856
- send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info) unless msgs.empty?
860
+ send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info, unpackedMsgArr[info]) unless msgs.empty?
861
+ unpackedMsgArr[info].clear
857
862
  msgs.clear
858
863
  # Clear bulk_message_count for this info.
859
864
  bulk_message_count[info] = 0;
@@ -876,7 +881,9 @@ EOC
876
881
  end
877
882
 
878
883
  bulk_message.each do |info, msgs|
879
- send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info) unless msgs.empty?
884
+ send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info, unpackedMsgArr[info]) unless msgs.empty?
885
+
886
+ unpackedMsgArr[info].clear
880
887
  msgs.clear
881
888
  end
882
889
  end
@@ -1090,7 +1097,7 @@ EOC
1090
1097
 
1091
1098
  # send_bulk given a specific bulk request, the original tag,
1092
1099
  # chunk, and bulk_message_count
1093
- def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
1100
+ def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info, unpacked_msg_arr)
1094
1101
  _logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline, ilm_policy_id = extracted_values
1095
1102
  if deflector_alias
1096
1103
  template_installation(deflector_alias, template_name, customize_template, application_name, index_name, ilm_policy_id, info.host)
@@ -1113,7 +1120,7 @@ EOC
1113
1120
 
1114
1121
  if response['errors']
1115
1122
  error = Fluent::Plugin::ElasticsearchErrorHandler.new(self)
1116
- error.handle_error(response, tag, chunk, bulk_message_count, extracted_values)
1123
+ error.handle_error(response, tag, chunk, bulk_message_count, extracted_values, unpacked_msg_arr)
1117
1124
  end
1118
1125
  rescue RetryStreamError => e
1119
1126
  log.trace "router.emit_stream for retry stream doing..."
@@ -248,7 +248,9 @@ module Fluent::Plugin
248
248
  end
249
249
 
250
250
  begin
251
- record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
251
+ unless record.has_key?("@timestamp")
252
+ record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
253
+ end
252
254
  bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
253
255
  rescue => e
254
256
  router.emit_error_event(tag, time, record, e)
Binary file
@@ -2,6 +2,7 @@ require_relative '../helper'
2
2
  require 'fluent/plugin/out_elasticsearch'
3
3
  require 'fluent/plugin/elasticsearch_error_handler'
4
4
  require 'json'
5
+ require 'msgpack'
5
6
 
6
7
  class TestElasticsearchErrorHandler < Test::Unit::TestCase
7
8
 
@@ -54,6 +55,27 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
54
55
  end
55
56
  end
56
57
 
58
+ class MockMsgpackChunk
59
+ def initialize(chunk)
60
+ @chunk = chunk
61
+ @factory = MessagePack::Factory.new
62
+ @factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)
63
+ end
64
+
65
+ def msgpack_each
66
+ @factory.unpacker(@chunk).each { |time, record| yield(time, record) }
67
+ end
68
+ end
69
+
70
+ class MockUnpackedMsg
71
+ def initialize(records)
72
+ @records = records
73
+ end
74
+ def each
75
+ @records.each { |item| yield({:time => item[:time], :record => item[:record]}) }
76
+ end
77
+ end
78
+
57
79
  def setup
58
80
  Fluent::Test.setup
59
81
  @log_device = Fluent::Test::DummyLogDevice.new
@@ -98,8 +120,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
98
120
  ]
99
121
  }))
100
122
  chunk = MockChunk.new(records)
123
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
101
124
  dummy_extracted_values = []
102
- @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
125
+ @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
103
126
  assert_equal(1, @plugin.error_events.size)
104
127
  expected_log = "failed to parse"
105
128
  exception_message = @plugin.error_events.first[:error].message
@@ -140,8 +163,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
140
163
  ]
141
164
  }))
142
165
  chunk = MockChunk.new(records)
166
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
143
167
  dummy_extracted_values = []
144
- @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
168
+ @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
145
169
  assert_equal(1, @plugin.error_events.size)
146
170
  expected_log = "failed to parse"
147
171
  exception_message = @plugin.error_events.first[:error].message
@@ -159,8 +183,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
159
183
  "items" : [{}]
160
184
  }))
161
185
  chunk = MockChunk.new(records)
186
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
162
187
  dummy_extracted_values = []
163
- @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
188
+ @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
164
189
  assert_equal(0, @plugin.error_events.size)
165
190
  assert_nil(@plugin.error_events[0])
166
191
  end
@@ -181,8 +206,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
181
206
  ]
182
207
  }))
183
208
  chunk = MockChunk.new(records)
209
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
184
210
  dummy_extracted_values = []
185
- @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
211
+ @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
186
212
  assert_equal(1, @plugin.error_events.size)
187
213
  assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
188
214
  end
@@ -204,8 +230,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
204
230
  ]
205
231
  }))
206
232
  chunk = MockChunk.new(records)
233
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
207
234
  dummy_extracted_values = []
208
- @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
235
+ @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
209
236
  assert_equal(1, @plugin.error_events.size)
210
237
  assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
211
238
  end
@@ -230,10 +257,11 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
230
257
  ]
231
258
  }))
232
259
 
233
- chunk = MockChunk.new(records)
234
- dummy_extracted_values = []
260
+ chunk = MockChunk.new(records)
261
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
262
+ dummy_extracted_values = []
235
263
  assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError) do
236
- @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
264
+ @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
237
265
  end
238
266
  end
239
267
 
@@ -257,10 +285,11 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
257
285
  ]
258
286
  }))
259
287
 
260
- chunk = MockChunk.new(records)
261
- dummy_extracted_values = []
288
+ chunk = MockChunk.new(records)
289
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
290
+ dummy_extracted_values = []
262
291
  assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError) do
263
- @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
292
+ @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
264
293
  end
265
294
  end
266
295
 
@@ -290,8 +319,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
290
319
  begin
291
320
  failed = false
292
321
  chunk = MockChunk.new(records)
322
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
293
323
  dummy_extracted_values = []
294
- handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
324
+ handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
295
325
  rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
296
326
  failed = true
297
327
  records = [].tap do |records|
@@ -312,6 +342,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
312
342
  records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
313
343
  end
314
344
  chunk = MockChunk.new(records)
345
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
315
346
 
316
347
  response = parse_response(%({
317
348
  "took" : 1,
@@ -410,7 +441,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
410
441
  begin
411
442
  failed = false
412
443
  dummy_extracted_values = []
413
- @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
444
+ @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
414
445
  rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
415
446
  failed = true
416
447
  records = [].tap do |records|
@@ -439,6 +470,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
439
470
  records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
440
471
  end
441
472
  chunk = MockChunk.new(records)
473
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
442
474
 
443
475
  response = parse_response(%({
444
476
  "took" : 1,
@@ -526,7 +558,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
526
558
  begin
527
559
  failed = false
528
560
  dummy_extracted_values = []
529
- @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
561
+ @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
530
562
  rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
531
563
  failed = true
532
564
  records = [].tap do |records|
@@ -549,6 +581,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
549
581
  records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
550
582
  end
551
583
  chunk = MockChunk.new(records)
584
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
552
585
 
553
586
  response = parse_response(%({
554
587
  "took" : 1,
@@ -639,7 +672,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
639
672
  begin
640
673
  failed = false
641
674
  dummy_extracted_values = []
642
- @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
675
+ @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
643
676
  rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
644
677
  failed = true
645
678
  records = [].tap do |records|
@@ -660,4 +693,61 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
660
693
  end
661
694
  assert_true failed
662
695
  end
696
+
697
+ def test_nested_msgpack_each
698
+ cwd = File.dirname(__FILE__)
699
+ chunk_path = File.join(cwd, 'mock_chunk.dat')
700
+ chunk_file = File.open(chunk_path, 'rb', 0644)
701
+ chunk_file.seek(0, IO::SEEK_SET)
702
+
703
+ chunk = MockMsgpackChunk.new(chunk_file)
704
+
705
+ unpacked_msg_arr = []
706
+ msg_count = 0
707
+ count_to_trigger_error_handle = 0
708
+ chunk.msgpack_each do |time, record|
709
+ next unless record.is_a? Hash
710
+
711
+ unpacked_msg_arr << {:time => time, :record => record}
712
+ msg_count += 1
713
+
714
+ record.each_key do |k|
715
+ if k != 'aaa' && k != 'bbb' && k != 'ccc' && k != 'log_path'
716
+ assert_equal(:impossible, k)
717
+ end
718
+ end
719
+
720
+ if msg_count % 55 == 0
721
+ if count_to_trigger_error_handle == 1
722
+ begin
723
+ response = {}
724
+ response['errors'] = true
725
+ response['items'] = []
726
+ item = {}
727
+ item['index'] = {}
728
+ item['index']['status'] = 429
729
+ item['index']['error'] = {}
730
+ item['index']['error']['type'] = "es_rejected_execution_exception"
731
+ abc = 0
732
+ while abc < unpacked_msg_arr.length
733
+ abc += 1
734
+ response['items'] << item
735
+ end
736
+
737
+ dummy_extracted_values = []
738
+ @handler.handle_error(response, 'atag', chunk, unpacked_msg_arr.length, dummy_extracted_values, unpacked_msg_arr)
739
+ assert_equal(0, @plugin.error_events.size)
740
+ assert_nil(@plugin.error_events[0])
741
+ rescue => e
742
+ # capture ElasticsearchRequestAbortError, beacuse es_rejected_execution_exception is unrecoverable.
743
+ end
744
+ end
745
+
746
+ count_to_trigger_error_handle += 1
747
+ unpacked_msg_arr.clear
748
+ end # end if
749
+ end # end chunk.msgpack_each
750
+
751
+ chunk_file.close
752
+ end
663
753
  end
@@ -70,8 +70,13 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
70
70
  }
71
71
  end
72
72
 
73
+ SAMPLE_RECORD_TIMESTAMP = Time.now.iso8601
73
74
  def sample_record
74
- {'@timestamp' => Time.now.iso8601, 'message' => 'Sample record'}
75
+ {'@timestamp' => SAMPLE_RECORD_TIMESTAMP, 'message' => 'Sample record'}
76
+ end
77
+
78
+ def sample_record_no_timestamp
79
+ {'message' => 'Sample record no timestamp'}
75
80
  end
76
81
 
77
82
  RESPONSE_ACKNOWLEDGED = {"acknowledged": true}
@@ -859,7 +864,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
859
864
  stub_default
860
865
  stub_bulk_feed
861
866
  driver(config)
862
- driver.run(default_tag: 'mytag') do
867
+ driver.run(default_tag: 'mytag') do
863
868
  driver.feed(sample_record)
864
869
  end
865
870
 
@@ -882,7 +887,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
882
887
  stub_default
883
888
  stub_bulk_feed
884
889
  driver(config)
885
- driver.run(default_tag: 'mytag') do
890
+ driver.run(default_tag: 'mytag') do
886
891
  driver.feed(sample_record)
887
892
  end
888
893
 
@@ -890,4 +895,43 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
890
895
  assert(@bulk_records[0].has_key?('custom_tag_key'))
891
896
  assert_equal('mytag', @bulk_records[0]['custom_tag_key'])
892
897
  end
898
+
899
+ def test_use_record_timestamp_if_present
900
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
901
+
902
+ stub_default
903
+ stub_bulk_feed
904
+ conf = config_element(
905
+ 'ROOT', '', {
906
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
907
+ 'data_stream_name' => 'foo',
908
+ 'data_stream_ilm_name' => 'foo_ilm_policy',
909
+ 'data_stream_template_name' => 'foo_tpl'
910
+ })
911
+ driver(conf).run(default_tag: 'test') do
912
+ driver.feed(sample_record)
913
+ end
914
+ assert_equal 1, @bulk_records.length
915
+ assert(@bulk_records[0].has_key?('@timestamp'))
916
+ assert_equal SAMPLE_RECORD_TIMESTAMP, @bulk_records[0]['@timestamp']
917
+ end
918
+
919
+ def test_add_timestamp_if_not_present_in_record
920
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
921
+
922
+ stub_default
923
+ stub_bulk_feed
924
+ conf = config_element(
925
+ 'ROOT', '', {
926
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
927
+ 'data_stream_name' => 'foo',
928
+ 'data_stream_ilm_name' => 'foo_ilm_policy',
929
+ 'data_stream_template_name' => 'foo_tpl'
930
+ })
931
+ driver(conf).run(default_tag: 'test') do
932
+ driver.feed(sample_record_no_timestamp)
933
+ end
934
+ assert_equal 1, @bulk_records.length
935
+ assert(@bulk_records[0].has_key?('@timestamp'))
936
+ end
893
937
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.2.2
4
+ version: 5.2.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - diogo
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2022-04-11 00:00:00.000000000 Z
13
+ date: 2022-06-16 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: fluentd
@@ -151,6 +151,7 @@ files:
151
151
  - ".editorconfig"
152
152
  - ".github/ISSUE_TEMPLATE/bug_report.md"
153
153
  - ".github/ISSUE_TEMPLATE/feature_request.md"
154
+ - ".github/dependabot.yml"
154
155
  - ".github/workflows/issue-auto-closer.yml"
155
156
  - ".github/workflows/linux.yml"
156
157
  - ".github/workflows/macos.yml"
@@ -187,6 +188,7 @@ files:
187
188
  - lib/fluent/plugin/out_elasticsearch_data_stream.rb
188
189
  - lib/fluent/plugin/out_elasticsearch_dynamic.rb
189
190
  - test/helper.rb
191
+ - test/plugin/mock_chunk.dat
190
192
  - test/plugin/test_alias_template.json
191
193
  - test/plugin/test_elasticsearch_error_handler.rb
192
194
  - test/plugin/test_elasticsearch_fallback_selector.rb
@@ -228,6 +230,7 @@ specification_version: 4
228
230
  summary: Elasticsearch output plugin for Fluent event collector
229
231
  test_files:
230
232
  - test/helper.rb
233
+ - test/plugin/mock_chunk.dat
231
234
  - test/plugin/test_alias_template.json
232
235
  - test/plugin/test_elasticsearch_error_handler.rb
233
236
  - test/plugin/test_elasticsearch_fallback_selector.rb