fluent-plugin-elasticsearch 4.1.1 → 5.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +37 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.md +24 -0
  4. data/.github/dependabot.yml +6 -0
  5. data/.github/workflows/issue-auto-closer.yml +2 -2
  6. data/.github/workflows/linux.yml +5 -2
  7. data/.github/workflows/macos.yml +5 -2
  8. data/.github/workflows/windows.yml +5 -2
  9. data/Gemfile +1 -2
  10. data/History.md +146 -0
  11. data/README.ElasticsearchGenID.md +4 -4
  12. data/README.ElasticsearchInput.md +1 -1
  13. data/README.Troubleshooting.md +692 -0
  14. data/README.md +260 -550
  15. data/fluent-plugin-elasticsearch.gemspec +4 -1
  16. data/lib/fluent/plugin/elasticsearch_compat.rb +31 -0
  17. data/lib/fluent/plugin/elasticsearch_error_handler.rb +19 -4
  18. data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +2 -2
  19. data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb +18 -4
  20. data/lib/fluent/plugin/elasticsearch_index_template.rb +65 -21
  21. data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb +2 -1
  22. data/lib/fluent/plugin/filter_elasticsearch_genid.rb +1 -1
  23. data/lib/fluent/plugin/in_elasticsearch.rb +8 -2
  24. data/lib/fluent/plugin/oj_serializer.rb +2 -1
  25. data/lib/fluent/plugin/out_elasticsearch.rb +192 -36
  26. data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +298 -0
  27. data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +3 -1
  28. data/test/helper.rb +0 -4
  29. data/test/plugin/mock_chunk.dat +0 -0
  30. data/test/plugin/test_elasticsearch_error_handler.rb +130 -23
  31. data/test/plugin/test_elasticsearch_fallback_selector.rb +17 -8
  32. data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +57 -18
  33. data/test/plugin/test_elasticsearch_tls.rb +8 -2
  34. data/test/plugin/test_filter_elasticsearch_genid.rb +16 -16
  35. data/test/plugin/test_in_elasticsearch.rb +51 -21
  36. data/test/plugin/test_index_alias_template.json +11 -0
  37. data/test/plugin/test_index_template.json +25 -0
  38. data/test/plugin/test_out_elasticsearch.rb +2118 -704
  39. data/test/plugin/test_out_elasticsearch_data_stream.rb +1199 -0
  40. data/test/plugin/test_out_elasticsearch_dynamic.rb +170 -31
  41. metadata +62 -10
  42. data/.coveralls.yml +0 -2
  43. data/.travis.yml +0 -44
  44. data/appveyor.yml +0 -20
  45. data/gemfiles/Gemfile.without.ilm +0 -10
@@ -0,0 +1,298 @@
1
+
2
+ require_relative 'out_elasticsearch'
3
+
4
+ module Fluent::Plugin
5
+ class ElasticsearchOutputDataStream < ElasticsearchOutput
6
+
7
+ Fluent::Plugin.register_output('elasticsearch_data_stream', self)
8
+
9
+ helpers :event_emitter
10
+
11
+ config_param :data_stream_name, :string
12
+ config_param :data_stream_ilm_name, :string, :default => nil
13
+ config_param :data_stream_template_name, :string, :default => nil
14
+ config_param :data_stream_ilm_policy, :string, :default => nil
15
+ config_param :data_stream_ilm_policy_overwrite, :bool, :default => false
16
+ config_param :data_stream_template_use_index_patterns_wildcard, :bool, :default => true
17
+
18
+ # Elasticsearch 7.9 or later always support new style of index template.
19
+ config_set_default :use_legacy_template, false
20
+
21
+ INVALID_START_CHRACTERS = ["-", "_", "+", "."]
22
+ INVALID_CHARACTERS = ["\\", "/", "*", "?", "\"", "<", ">", "|", " ", ",", "#", ":"]
23
+
24
+ def configure(conf)
25
+ super
26
+
27
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
28
+ begin
29
+ require 'elasticsearch/api'
30
+ require 'elasticsearch/xpack'
31
+ rescue LoadError
32
+ raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
33
+ end
34
+ else
35
+ begin
36
+ require 'elasticsearch/api'
37
+ rescue LoadError
38
+ raise Fluent::ConfigError, "'elasticsearch/api is required for <@elasticsearch_data_stream>."
39
+ end
40
+ end
41
+
42
+ @data_stream_ilm_name = "#{@data_stream_name}_policy" if @data_stream_ilm_name.nil?
43
+ @data_stream_template_name = "#{@data_stream_name}_template" if @data_stream_template_name.nil?
44
+ @data_stream_ilm_policy = File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json")) if @data_stream_ilm_policy.nil?
45
+
46
+ # ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
47
+ unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
48
+ validate_data_stream_parameters
49
+ else
50
+ @use_placeholder = true
51
+ @data_stream_names = []
52
+ end
53
+
54
+ unless @use_placeholder
55
+ begin
56
+ @data_stream_names = [@data_stream_name]
57
+ retry_operate(@max_retry_putting_template,
58
+ @fail_on_putting_template_retry_exceed,
59
+ @catch_transport_exception_on_retry) do
60
+ create_ilm_policy(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
61
+ create_index_template(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
62
+ create_data_stream(@data_stream_name)
63
+ end
64
+ rescue => e
65
+ raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
66
+ end
67
+ end
68
+ end
69
+
70
+ def validate_data_stream_parameters
71
+ {"data_stream_name" => @data_stream_name,
72
+ "data_stream_template_name"=> @data_stream_template_name,
73
+ "data_stream_ilm_name" => @data_stream_ilm_name}.each do |parameter, value|
74
+ unless valid_data_stream_parameters?(value)
75
+ unless start_with_valid_characters?(value)
76
+ if not_dots?(value)
77
+ raise Fluent::ConfigError, "'#{parameter}' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{value}>"
78
+ else
79
+ raise Fluent::ConfigError, "'#{parameter}' must not be . or ..: <#{value}>"
80
+ end
81
+ end
82
+ unless valid_characters?(value)
83
+ raise Fluent::ConfigError, "'#{parameter}' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{value}>"
84
+ end
85
+ unless lowercase_only?(value)
86
+ raise Fluent::ConfigError, "'#{parameter}' must be lowercase only: <#{value}>"
87
+ end
88
+ if value.bytes.size > 255
89
+ raise Fluent::ConfigError, "'#{parameter}' must not be longer than 255 bytes: <#{value}>"
90
+ end
91
+ end
92
+ end
93
+ end
94
+
95
+ def create_ilm_policy(datastream_name, template_name, ilm_name, host = nil)
96
+ unless @data_stream_ilm_policy_overwrite
97
+ return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host) or ilm_policy_exists?(ilm_name, host)
98
+ end
99
+
100
+ params = {
101
+ body: @data_stream_ilm_policy
102
+ }
103
+ retry_operate(@max_retry_putting_template,
104
+ @fail_on_putting_template_retry_exceed,
105
+ @catch_transport_exception_on_retry) do
106
+ if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
107
+ client(host).ilm.put_lifecycle(params.merge(policy: ilm_name))
108
+ else
109
+ client(host).xpack.ilm.put_policy(params.merge(policy_id: ilm_name))
110
+ end
111
+ end
112
+ end
113
+
114
+ def create_index_template(datastream_name, template_name, ilm_name, host = nil)
115
+ return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
116
+ wildcard = @data_stream_template_use_index_patterns_wildcard ? '*' : ''
117
+ body = {
118
+ "index_patterns" => ["#{datastream_name}#{wildcard}"],
119
+ "data_stream" => {},
120
+ "template" => {
121
+ "settings" => {
122
+ "index.lifecycle.name" => "#{ilm_name}"
123
+ }
124
+ }
125
+ }
126
+ params = {
127
+ name: template_name,
128
+ body: body
129
+ }
130
+ retry_operate(@max_retry_putting_template,
131
+ @fail_on_putting_template_retry_exceed,
132
+ @catch_transport_exception_on_retry) do
133
+ client(host).indices.put_index_template(params)
134
+ end
135
+ end
136
+
137
+ def data_stream_exist?(datastream_name, host = nil)
138
+ params = {
139
+ name: datastream_name
140
+ }
141
+ begin
142
+ response = client(host).indices.get_data_stream(params)
143
+ return (not response.is_a?(TRANSPORT_CLASS::Transport::Errors::NotFound))
144
+ rescue TRANSPORT_CLASS::Transport::Errors::NotFound => e
145
+ log.info "Specified data stream does not exist. Will be created: <#{e}>"
146
+ return false
147
+ end
148
+ end
149
+
150
+ def create_data_stream(datastream_name, host = nil)
151
+ return if data_stream_exist?(datastream_name, host)
152
+ params = {
153
+ name: datastream_name
154
+ }
155
+ retry_operate(@max_retry_putting_template,
156
+ @fail_on_putting_template_retry_exceed,
157
+ @catch_transport_exception_on_retry) do
158
+ client(host).indices.create_data_stream(params)
159
+ end
160
+ end
161
+
162
+ def ilm_policy_exists?(policy_id, host = nil)
163
+ begin
164
+ if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
165
+ client(host).ilm.get_lifecycle(policy: policy_id)
166
+ else
167
+ client(host).ilm.get_policy(policy_id: policy_id)
168
+ end
169
+ true
170
+ rescue
171
+ false
172
+ end
173
+ end
174
+
175
+ def template_exists?(name, host = nil)
176
+ if @use_legacy_template
177
+ client(host).indices.get_template(:name => name)
178
+ else
179
+ client(host).indices.get_index_template(:name => name)
180
+ end
181
+ return true
182
+ rescue TRANSPORT_CLASS::Transport::Errors::NotFound
183
+ return false
184
+ end
185
+
186
+ def valid_data_stream_parameters?(data_stream_parameter)
187
+ lowercase_only?(data_stream_parameter) and
188
+ valid_characters?(data_stream_parameter) and
189
+ start_with_valid_characters?(data_stream_parameter) and
190
+ not_dots?(data_stream_parameter) and
191
+ data_stream_parameter.bytes.size <= 255
192
+ end
193
+
194
+ def lowercase_only?(data_stream_parameter)
195
+ data_stream_parameter.downcase == data_stream_parameter
196
+ end
197
+
198
+ def valid_characters?(data_stream_parameter)
199
+ not (INVALID_CHARACTERS.each.any? do |v| data_stream_parameter.include?(v) end)
200
+ end
201
+
202
+ def start_with_valid_characters?(data_stream_parameter)
203
+ not (INVALID_START_CHRACTERS.each.any? do |v| data_stream_parameter.start_with?(v) end)
204
+ end
205
+
206
+ def not_dots?(data_stream_parameter)
207
+ not (data_stream_parameter == "." or data_stream_parameter == "..")
208
+ end
209
+
210
+ def client_library_version
211
+ Elasticsearch::VERSION
212
+ end
213
+
214
+ def multi_workers_ready?
215
+ true
216
+ end
217
+
218
+ def write(chunk)
219
+ data_stream_name = @data_stream_name
220
+ data_stream_template_name = @data_stream_template_name
221
+ data_stream_ilm_name = @data_stream_ilm_name
222
+ host = nil
223
+ if @use_placeholder
224
+ host = if @hosts
225
+ extract_placeholders(@hosts, chunk)
226
+ else
227
+ extract_placeholders(@host, chunk)
228
+ end
229
+ data_stream_name = extract_placeholders(@data_stream_name, chunk)
230
+ data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
231
+ data_stream_ilm_name = extract_placeholders(@data_stream_ilm_name, chunk)
232
+ unless @data_stream_names.include?(data_stream_name)
233
+ begin
234
+ create_ilm_policy(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
235
+ create_index_template(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
236
+ create_data_stream(data_stream_name)
237
+ @data_stream_names << data_stream_name
238
+ rescue => e
239
+ raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
240
+ end
241
+ end
242
+ end
243
+
244
+ bulk_message = ""
245
+ headers = {
246
+ CREATE_OP => {}
247
+ }
248
+ tag = chunk.metadata.tag
249
+ chunk.msgpack_each do |time, record|
250
+ next unless record.is_a? Hash
251
+
252
+ if @include_tag_key
253
+ record[@tag_key] = tag
254
+ end
255
+
256
+ begin
257
+ unless record.has_key?("@timestamp")
258
+ record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
259
+ end
260
+ bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
261
+ rescue => e
262
+ router.emit_error_event(tag, time, record, e)
263
+ end
264
+ end
265
+
266
+ prepared_data = if compression
267
+ gzip(bulk_message)
268
+ else
269
+ bulk_message
270
+ end
271
+
272
+ params = {
273
+ index: data_stream_name,
274
+ body: prepared_data
275
+ }
276
+ begin
277
+ response = client(host, compression).bulk(params)
278
+ if response['errors']
279
+ log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
280
+ @num_errors_metrics.inc
281
+ end
282
+ rescue => e
283
+ raise RecoverableRequestFailure, "could not push logs to Elasticsearch cluster (#{data_stream_name}): #{e.message}"
284
+ end
285
+ end
286
+
287
+ def append_record_to_messages(op, meta, header, record, msgs)
288
+ header[CREATE_OP] = meta
289
+ msgs << @dump_proc.call(header) << BODY_DELIMITER
290
+ msgs << @dump_proc.call(record) << BODY_DELIMITER
291
+ msgs
292
+ end
293
+
294
+ def retry_stream_retryable?
295
+ @buffer.storable?
296
+ end
297
+ end
298
+ end
@@ -28,6 +28,8 @@ module Fluent::Plugin
28
28
  @dynamic_config[key] = value.to_s
29
29
  }
30
30
  # end eval all configs
31
+
32
+ log.warn "Elasticsearch dynamic plugin will be deprecated and removed in the future. Please consider to use normal Elasticsearch plugin"
31
33
  end
32
34
 
33
35
  def create_meta_config_map
@@ -53,7 +55,7 @@ module Fluent::Plugin
53
55
  end
54
56
  headers = { 'Content-Type' => @content_type.to_s, }.merge(gzip_headers)
55
57
  ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
56
- transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
58
+ transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(connection_options.merge(
57
59
  options: {
58
60
  reload_connections: @reload_connections,
59
61
  reload_on_failure: @reload_on_failure,
data/test/helper.rb CHANGED
@@ -5,9 +5,6 @@ SimpleCov.start do
5
5
  end
6
6
  end
7
7
 
8
- require 'coveralls'
9
- Coveralls.wear!
10
-
11
8
  # needs to be after simplecov but before test/unit, because fluentd sets default
12
9
  # encoding to ASCII-8BIT, but coverall might load git data which could contain a
13
10
  # UTF-8 character
@@ -18,7 +15,6 @@ end
18
15
 
19
16
  require 'test/unit'
20
17
  require 'fluent/test'
21
- require 'minitest/pride'
22
18
 
23
19
  require 'webmock/test_unit'
24
20
  WebMock.disable_net_connect!
Binary file
@@ -2,6 +2,7 @@ require_relative '../helper'
2
2
  require 'fluent/plugin/out_elasticsearch'
3
3
  require 'fluent/plugin/elasticsearch_error_handler'
4
4
  require 'json'
5
+ require 'msgpack'
5
6
 
6
7
  class TestElasticsearchErrorHandler < Test::Unit::TestCase
7
8
 
@@ -27,13 +28,18 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
27
28
  @error_events << {:tag => tag, :time=>time, :record=>record, :error=>e}
28
29
  end
29
30
 
30
- def process_message(tag, meta, header, time, record, extracted_values)
31
+ def process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
31
32
  return [meta, header, record]
32
33
  end
33
34
 
35
+ def get_affinity_target_indices(chunk)
36
+ indices = Hash.new
37
+ indices
38
+ end
39
+
34
40
  def append_record_to_messages(op, meta, header, record, msgs)
35
41
  if record.has_key?('raise') && record['raise']
36
- raise Exception('process_message')
42
+ raise 'process_message'
37
43
  end
38
44
  return true
39
45
  end
@@ -49,6 +55,27 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
49
55
  end
50
56
  end
51
57
 
58
+ class MockMsgpackChunk
59
+ def initialize(chunk)
60
+ @chunk = chunk
61
+ @factory = MessagePack::Factory.new
62
+ @factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)
63
+ end
64
+
65
+ def msgpack_each
66
+ @factory.unpacker(@chunk).each { |time, record| yield(time, record) }
67
+ end
68
+ end
69
+
70
+ class MockUnpackedMsg
71
+ def initialize(records)
72
+ @records = records
73
+ end
74
+ def each
75
+ @records.each { |item| yield({:time => item[:time], :record => item[:record]}) }
76
+ end
77
+ end
78
+
52
79
  def setup
53
80
  Fluent::Test.setup
54
81
  @log_device = Fluent::Test::DummyLogDevice.new
@@ -93,8 +120,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
93
120
  ]
94
121
  }))
95
122
  chunk = MockChunk.new(records)
123
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
96
124
  dummy_extracted_values = []
97
- @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
125
+ @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
98
126
  assert_equal(1, @plugin.error_events.size)
99
127
  expected_log = "failed to parse"
100
128
  exception_message = @plugin.error_events.first[:error].message
@@ -135,8 +163,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
135
163
  ]
136
164
  }))
137
165
  chunk = MockChunk.new(records)
166
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
138
167
  dummy_extracted_values = []
139
- @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
168
+ @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
140
169
  assert_equal(1, @plugin.error_events.size)
141
170
  expected_log = "failed to parse"
142
171
  exception_message = @plugin.error_events.first[:error].message
@@ -154,8 +183,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
154
183
  "items" : [{}]
155
184
  }))
156
185
  chunk = MockChunk.new(records)
186
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
157
187
  dummy_extracted_values = []
158
- @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
188
+ @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
159
189
  assert_equal(0, @plugin.error_events.size)
160
190
  assert_nil(@plugin.error_events[0])
161
191
  end
@@ -176,8 +206,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
176
206
  ]
177
207
  }))
178
208
  chunk = MockChunk.new(records)
209
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
179
210
  dummy_extracted_values = []
180
- @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
211
+ @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
181
212
  assert_equal(1, @plugin.error_events.size)
182
213
  assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
183
214
  end
@@ -199,8 +230,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
199
230
  ]
200
231
  }))
201
232
  chunk = MockChunk.new(records)
233
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
202
234
  dummy_extracted_values = []
203
- @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
235
+ @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
204
236
  assert_equal(1, @plugin.error_events.size)
205
237
  assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
206
238
  end
@@ -225,10 +257,11 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
225
257
  ]
226
258
  }))
227
259
 
228
- chunk = MockChunk.new(records)
229
- dummy_extracted_values = []
260
+ chunk = MockChunk.new(records)
261
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
262
+ dummy_extracted_values = []
230
263
  assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError) do
231
- @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
264
+ @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
232
265
  end
233
266
  end
234
267
 
@@ -252,10 +285,11 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
252
285
  ]
253
286
  }))
254
287
 
255
- chunk = MockChunk.new(records)
256
- dummy_extracted_values = []
288
+ chunk = MockChunk.new(records)
289
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
290
+ dummy_extracted_values = []
257
291
  assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError) do
258
- @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
292
+ @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
259
293
  end
260
294
  end
261
295
 
@@ -285,8 +319,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
285
319
  begin
286
320
  failed = false
287
321
  chunk = MockChunk.new(records)
322
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
288
323
  dummy_extracted_values = []
289
- handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
324
+ handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
290
325
  rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
291
326
  failed = true
292
327
  records = [].tap do |records|
@@ -302,11 +337,12 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
302
337
  def test_retry_error
303
338
  records = []
304
339
  error_records = Hash.new(false)
305
- error_records.merge!({0=>true, 4=>true, 9=>true})
340
+ error_records.merge!({0=>true, 4=>true})
306
341
  10.times do |i|
307
342
  records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
308
343
  end
309
344
  chunk = MockChunk.new(records)
345
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
310
346
 
311
347
  response = parse_response(%({
312
348
  "took" : 1,
@@ -386,6 +422,18 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
386
422
  "reason":"unrecognized error"
387
423
  }
388
424
  }
425
+ },
426
+ {
427
+ "create" : {
428
+ "_index" : "foo",
429
+ "_type" : "bar",
430
+ "_id" : "9",
431
+ "status" : 500,
432
+ "error" : {
433
+ "type" : "json_parse_exception",
434
+ "reason":"Invalid UTF-8 start byte 0x92\\n at [Source: org.elasticsearch.transport.netty4.ByteBufStreamInput@204fe9c9; line: 1, column: 81]"
435
+ }
436
+ }
389
437
  }
390
438
  ]
391
439
  }))
@@ -393,19 +441,19 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
393
441
  begin
394
442
  failed = false
395
443
  dummy_extracted_values = []
396
- @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
444
+ @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
397
445
  rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
398
446
  failed = true
399
447
  records = [].tap do |records|
400
448
  next unless e.respond_to?(:retry_stream)
401
449
  e.retry_stream.each {|time, record| records << record}
402
450
  end
403
- assert_equal 2, records.length
404
- assert_equal 2, records[0]['_id']
405
- assert_equal 8, records[1]['_id']
451
+ assert_equal 2, records.length, "Exp. retry_stream to contain records"
452
+ assert_equal 2, records[0]['_id'], "Exp record with given ID to in retry_stream"
453
+ assert_equal 8, records[1]['_id'], "Exp record with given ID to in retry_stream"
406
454
  error_ids = @plugin.error_events.collect {|h| h[:record]['_id']}
407
- assert_equal 3, error_ids.length
408
- assert_equal [5, 6, 7], error_ids
455
+ assert_equal 4, error_ids.length, "Exp. a certain number of records to be dropped from retry_stream"
456
+ assert_equal [5, 6, 7, 9], error_ids, "Exp. specific records to be dropped from retry_stream"
409
457
  @plugin.error_events.collect {|h| h[:error]}.each do |e|
410
458
  assert_true e.respond_to?(:backtrace)
411
459
  end
@@ -422,6 +470,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
422
470
  records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
423
471
  end
424
472
  chunk = MockChunk.new(records)
473
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
425
474
 
426
475
  response = parse_response(%({
427
476
  "took" : 1,
@@ -509,7 +558,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
509
558
  begin
510
559
  failed = false
511
560
  dummy_extracted_values = []
512
- @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
561
+ @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
513
562
  rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
514
563
  failed = true
515
564
  records = [].tap do |records|
@@ -532,6 +581,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
532
581
  records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
533
582
  end
534
583
  chunk = MockChunk.new(records)
584
+ unpacked_msg_arr = MockUnpackedMsg.new(records)
535
585
 
536
586
  response = parse_response(%({
537
587
  "took" : 1,
@@ -622,7 +672,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
622
672
  begin
623
673
  failed = false
624
674
  dummy_extracted_values = []
625
- @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
675
+ @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
626
676
  rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
627
677
  failed = true
628
678
  records = [].tap do |records|
@@ -643,4 +693,61 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
643
693
  end
644
694
  assert_true failed
645
695
  end
696
+
697
+ def test_nested_msgpack_each
698
+ cwd = File.dirname(__FILE__)
699
+ chunk_path = File.join(cwd, 'mock_chunk.dat')
700
+ chunk_file = File.open(chunk_path, 'rb', 0644)
701
+ chunk_file.seek(0, IO::SEEK_SET)
702
+
703
+ chunk = MockMsgpackChunk.new(chunk_file)
704
+
705
+ unpacked_msg_arr = []
706
+ msg_count = 0
707
+ count_to_trigger_error_handle = 0
708
+ chunk.msgpack_each do |time, record|
709
+ next unless record.is_a? Hash
710
+
711
+ unpacked_msg_arr << {:time => time, :record => record}
712
+ msg_count += 1
713
+
714
+ record.each_key do |k|
715
+ if k != 'aaa' && k != 'bbb' && k != 'ccc' && k != 'log_path'
716
+ assert_equal(:impossible, k)
717
+ end
718
+ end
719
+
720
+ if msg_count % 55 == 0
721
+ if count_to_trigger_error_handle == 1
722
+ begin
723
+ response = {}
724
+ response['errors'] = true
725
+ response['items'] = []
726
+ item = {}
727
+ item['index'] = {}
728
+ item['index']['status'] = 429
729
+ item['index']['error'] = {}
730
+ item['index']['error']['type'] = "es_rejected_execution_exception"
731
+ abc = 0
732
+ while abc < unpacked_msg_arr.length
733
+ abc += 1
734
+ response['items'] << item
735
+ end
736
+
737
+ dummy_extracted_values = []
738
+ @handler.handle_error(response, 'atag', chunk, unpacked_msg_arr.length, dummy_extracted_values, unpacked_msg_arr)
739
+ assert_equal(0, @plugin.error_events.size)
740
+ assert_nil(@plugin.error_events[0])
741
+ rescue => e
742
+ # capture ElasticsearchRequestAbortError, beacuse es_rejected_execution_exception is unrecoverable.
743
+ end
744
+ end
745
+
746
+ count_to_trigger_error_handle += 1
747
+ unpacked_msg_arr.clear
748
+ end # end if
749
+ end # end chunk.msgpack_each
750
+
751
+ chunk_file.close
752
+ end
646
753
  end
@@ -15,20 +15,28 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
15
15
  def stub_elastic(url="http://localhost:9200/_bulk")
16
16
  stub_request(:post, url).with do |req|
17
17
  @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
18
+ end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
19
+ end
20
+
21
+ def elasticsearch_version
22
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
23
+ TRANSPORT_CLASS::VERSION
24
+ else
25
+ '6.4.2'.freeze
18
26
  end
19
27
  end
20
28
 
21
- def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
22
- body ="{\"version\":{\"number\":\"#{version}\"}}"
23
- stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
29
+ def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
30
+ body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
31
+ stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
24
32
  end
25
33
 
26
- def stub_elastic_info_not_found(url="http://localhost:9200/", version="6.4.2")
27
- stub_request(:get, url).to_return(:status => [404, "Not Found"])
34
+ def stub_elastic_info_not_found(url="http://localhost:9200/", version=elasticsearch_version)
35
+ stub_request(:get, url).to_return(:status => [404, "Not Found"], headers: {'x-elastic-product' => 'Elasticsearch' })
28
36
  end
29
37
 
30
- def stub_elastic_info_unavailable(url="http://localhost:9200/", version="6.4.2")
31
- stub_request(:get, url).to_return(:status => [503, "Service Unavailable"])
38
+ def stub_elastic_info_unavailable(url="http://localhost:9200/", version=elasticsearch_version)
39
+ stub_request(:get, url).to_return(:status => [503, "Service Unavailable"], headers: {'x-elastic-product' => 'Elasticsearch' })
32
40
  end
33
41
 
34
42
  def sample_record(content={})
@@ -58,8 +66,9 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
58
66
  with_transporter_log true
59
67
  reload_connections true
60
68
  reload_after 10
69
+ catch_transport_exception_on_retry false # For fallback testing
61
70
  ]
62
- assert_raise(Elasticsearch::Transport::Transport::Errors::NotFound) do
71
+ assert_raise(TRANSPORT_CLASS::Transport::Errors::NotFound) do
63
72
  driver(config)
64
73
  end
65
74
  driver.run(default_tag: 'test') do