fluent-plugin-elasticsearch 4.1.1 → 5.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +37 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.md +24 -0
  4. data/.github/dependabot.yml +6 -0
  5. data/.github/workflows/issue-auto-closer.yml +2 -2
  6. data/.github/workflows/linux.yml +5 -2
  7. data/.github/workflows/macos.yml +5 -2
  8. data/.github/workflows/windows.yml +5 -2
  9. data/Gemfile +1 -2
  10. data/History.md +146 -0
  11. data/README.ElasticsearchGenID.md +4 -4
  12. data/README.ElasticsearchInput.md +1 -1
  13. data/README.Troubleshooting.md +692 -0
  14. data/README.md +260 -550
  15. data/fluent-plugin-elasticsearch.gemspec +4 -1
  16. data/lib/fluent/plugin/elasticsearch_compat.rb +31 -0
  17. data/lib/fluent/plugin/elasticsearch_error_handler.rb +19 -4
  18. data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +2 -2
  19. data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb +18 -4
  20. data/lib/fluent/plugin/elasticsearch_index_template.rb +65 -21
  21. data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb +2 -1
  22. data/lib/fluent/plugin/filter_elasticsearch_genid.rb +1 -1
  23. data/lib/fluent/plugin/in_elasticsearch.rb +8 -2
  24. data/lib/fluent/plugin/oj_serializer.rb +2 -1
  25. data/lib/fluent/plugin/out_elasticsearch.rb +192 -36
  26. data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +298 -0
  27. data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +3 -1
  28. data/test/helper.rb +0 -4
  29. data/test/plugin/mock_chunk.dat +0 -0
  30. data/test/plugin/test_elasticsearch_error_handler.rb +130 -23
  31. data/test/plugin/test_elasticsearch_fallback_selector.rb +17 -8
  32. data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +57 -18
  33. data/test/plugin/test_elasticsearch_tls.rb +8 -2
  34. data/test/plugin/test_filter_elasticsearch_genid.rb +16 -16
  35. data/test/plugin/test_in_elasticsearch.rb +51 -21
  36. data/test/plugin/test_index_alias_template.json +11 -0
  37. data/test/plugin/test_index_template.json +25 -0
  38. data/test/plugin/test_out_elasticsearch.rb +2118 -704
  39. data/test/plugin/test_out_elasticsearch_data_stream.rb +1199 -0
  40. data/test/plugin/test_out_elasticsearch_dynamic.rb +170 -31
  41. metadata +62 -10
  42. data/.coveralls.yml +0 -2
  43. data/.travis.yml +0 -44
  44. data/appveyor.yml +0 -20
  45. data/gemfiles/Gemfile.without.ilm +0 -10
@@ -0,0 +1,1199 @@
1
+ require_relative '../helper'
2
+ require 'date'
3
+ require 'fluent/test/helpers'
4
+ require 'fluent/test/driver/output'
5
+ require 'flexmock/test_unit'
6
+ require 'fluent/plugin/out_elasticsearch_data_stream'
7
+
8
+ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
9
+ include FlexMock::TestCase
10
+ include Fluent::Test::Helpers
11
+
12
+ attr_accessor :bulk_records
13
+
14
+ REQUIRED_ELASTIC_MESSAGE = "Elasticsearch 7.9.0 or later is needed."
15
+ ELASTIC_DATA_STREAM_TYPE = "elasticsearch_data_stream"
16
+
17
+ def setup
18
+ Fluent::Test.setup
19
+ @driver = nil
20
+ log = Fluent::Engine.log
21
+ log.out.logs.slice!(0, log.out.logs.length)
22
+ @bulk_records = []
23
+ end
24
+
25
+ def elasticsearch_version
26
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
27
+ TRANSPORT_CLASS::VERSION
28
+ else
29
+ '5.0.0'.freeze
30
+ end
31
+ end
32
+
33
+ def ilm_endpoint
34
+ '_ilm'.freeze
35
+ end
36
+
37
+ def index_template_endpoint
38
+ '_index_template'.freeze
39
+ end
40
+
41
+ def driver(conf='', es_version=elasticsearch_version.to_i, client_version=elasticsearch_version)
42
+ # For request stub to detect compatibility.
43
+ @es_version ||= es_version
44
+ @client_version ||= client_version
45
+ Fluent::Plugin::ElasticsearchOutputDataStream.module_eval(<<-CODE)
46
+ def detect_es_major_version
47
+ #{@es_version}
48
+ end
49
+ CODE
50
+ @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutputDataStream) {
51
+ # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
52
+ if !defined?(Fluent::Plugin::Output)
53
+ def format(tag, time, record)
54
+ [time, record].to_msgpack
55
+ end
56
+ end
57
+ }.configure(conf)
58
+ end
59
+
60
+ def elasticsearch_transport_layer_decoupling?
61
+ Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
62
+ end
63
+
64
+ def elastic_transport_layer?
65
+ Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
66
+ end
67
+
68
+ def sample_data_stream
69
+ {
70
+ 'data_streams': [
71
+ {
72
+ 'name' => 'foo',
73
+ 'timestamp_field' => {
74
+ 'name' => '@timestamp'
75
+ }
76
+ }
77
+ ]
78
+ }
79
+ end
80
+
81
+ SAMPLE_RECORD_TIMESTAMP = Time.now.iso8601
82
+ def sample_record
83
+ {'@timestamp' => SAMPLE_RECORD_TIMESTAMP, 'message' => 'Sample record'}
84
+ end
85
+
86
+ def sample_record_no_timestamp
87
+ {'message' => 'Sample record no timestamp'}
88
+ end
89
+
90
+ RESPONSE_ACKNOWLEDGED = {"acknowledged": true}
91
+ DUPLICATED_DATA_STREAM_EXCEPTION = {"error": {}, "status": 400}
92
+ NONEXISTENT_DATA_STREAM_EXCEPTION = {"error": {}, "status": 404}
93
+
94
+ def stub_ilm_policy(name="foo_ilm_policy", url="http://localhost:9200")
95
+ stub_request(:put, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED], :headers => {'x-elastic-product' => 'Elasticsearch'})
96
+ end
97
+
98
+ def stub_index_template(name="foo_tpl", url="http://localhost:9200")
99
+ stub_request(:put, "#{url}/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED], :headers => {'x-elastic-product' => 'Elasticsearch'})
100
+ end
101
+
102
+ def stub_data_stream(name="foo", url="http://localhost:9200")
103
+ stub_request(:put, "#{url}/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED], :headers => {'x-elastic-product' => 'Elasticsearch'})
104
+ end
105
+
106
+ def stub_existent_data_stream?(name="foo", url="http://localhost:9200")
107
+ stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED], :headers => {'x-elastic-product' => 'Elasticsearch'})
108
+ end
109
+
110
+ def stub_existent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
111
+
112
+ stub_request(:get, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED], :headers => {'x-elastic-product' => 'Elasticsearch'})
113
+ end
114
+
115
+ def stub_existent_template?(name="foo_tpl", url="http://localhost:9200")
116
+ stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED], :headers => {'x-elastic-product' => 'Elasticsearch'})
117
+ end
118
+
119
+ def stub_nonexistent_data_stream?(name="foo", url="http://localhost:9200")
120
+ stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound], :headers => {'x-elastic-product' => 'Elasticsearch'})
121
+ end
122
+
123
+ def stub_nonexistent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
124
+ stub_request(:get, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound], :headers => {'x-elastic-product' => 'Elasticsearch'})
125
+ end
126
+
127
+ def stub_nonexistent_template?(name="foo_tpl", url="http://localhost:9200")
128
+ stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound], :headers => {'x-elastic-product' => 'Elasticsearch'})
129
+ end
130
+
131
+ def push_bulk_request(req_body)
132
+ # bulk data must be pair of OP and records
133
+ # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
134
+ # {"@timestamp": ...}
135
+ ops = req_body.split("\n")
136
+ @bulk_records += ops.values_at(
137
+ * ops.each_index.select {|i| i.odd? }
138
+ ).map{ |i| JSON.parse(i) }
139
+ end
140
+
141
+ def stub_nonexistent_template_retry?(name="foo_tpl", url="http://localhost:9200")
142
+ stub_request(:get, "#{url}/_index_template/#{name}").
143
+ to_return({ status: 500, body: 'Internal Server Error' }, { status: 404, body: '{}', :headers => {'x-elastic-product' => 'Elasticsearch'} })
144
+ end
145
+
146
+ def stub_bulk_feed(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl", url="http://localhost:9200")
147
+ stub_request(:post, "#{url}/#{datastream_name}/_bulk").with do |req|
148
+ # bulk data must be pair of OP and records
149
+ # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
150
+ # {"@timestamp": ...}
151
+ push_bulk_request(req.body)
152
+ end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
153
+ stub_request(:post, "#{url}/#{ilm_name}/_bulk").with do |req|
154
+ # bulk data must be pair of OP and records
155
+ # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
156
+ # {"@timestamp": ...}
157
+ push_bulk_request(req.body)
158
+ end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
159
+ stub_request(:post, "#{url}/#{template_name}/_bulk").with do |req|
160
+ # bulk data must be pair of OP and records
161
+ # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
162
+ # {"@timestamp": ...}
163
+ push_bulk_request(req.body)
164
+ end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
165
+ end
166
+
167
+ def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version, headers={})
168
+ body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
169
+ stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' }.merge(headers) })
170
+ end
171
+
172
+ def stub_default(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl", host="http://localhost:9200")
173
+ stub_elastic_info(host)
174
+ stub_nonexistent_ilm?(ilm_name)
175
+ stub_ilm_policy(ilm_name)
176
+ stub_nonexistent_template?(template_name)
177
+ stub_index_template(template_name)
178
+ stub_nonexistent_data_stream?(datastream_name)
179
+ stub_data_stream(datastream_name)
180
+ end
181
+
182
+ def stub_elastic_with_store_index_command_counts(url="http://localhost:9200/_bulk")
183
+ if @index_command_counts == nil
184
+ @index_command_counts = {}
185
+ @index_command_counts.default = 0
186
+ end
187
+
188
+ stub_request(:post, url).with do |req|
189
+ index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
190
+ @index_command_counts[url] += index_cmds.size
191
+ end.to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
192
+ end
193
+
194
+ def data_stream_supported?
195
+ Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.create("7.9.0")
196
+ end
197
+
198
+ # ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
199
+ class DataStreamNameTest < self
200
+
201
+ def test_missing_data_stream_name
202
+ conf = config_element(
203
+ 'ROOT', '', {
204
+ '@type' => 'elasticsearch_datastream'
205
+ })
206
+ assert_raise Fluent::ConfigError.new("'data_stream_name' parameter is required") do
207
+ driver(conf).run
208
+ end
209
+ end
210
+
211
+ sub_test_case "invalid uppercase" do
212
+ def test_stream_name
213
+ conf = config_element(
214
+ 'ROOT', '', {
215
+ '@type' => 'elasticsearch_datastream',
216
+ 'data_stream_name' => 'TEST',
217
+ 'data_stream_ilm_name' => 'default-policy',
218
+ 'data_stream_template_name' => 'template'
219
+ })
220
+ assert_raise Fluent::ConfigError.new("'data_stream_name' must be lowercase only: <TEST>") do
221
+ driver(conf)
222
+ end
223
+ end
224
+ def test_stream_ilm_name
225
+ conf = config_element(
226
+ 'ROOT', '', {
227
+ '@type' => 'elasticsearch_datastream',
228
+ 'data_stream_name' => 'data_stream',
229
+ 'data_stream_ilm_name' => 'TEST-ILM',
230
+ 'data_stream_template_name' => 'template'
231
+ })
232
+ assert_raise Fluent::ConfigError.new("'data_stream_ilm_name' must be lowercase only: <TEST-ILM>") do
233
+ driver(conf)
234
+ end
235
+ end
236
+ def test_stream_template_name
237
+ conf = config_element(
238
+ 'ROOT', '', {
239
+ '@type' => 'elasticsearch_datastream',
240
+ 'data_stream_name' => 'default',
241
+ 'data_stream_ilm_name' => 'default-policy',
242
+ 'data_stream_template_name' => 'TEST-TPL'
243
+ })
244
+ assert_raise Fluent::ConfigError.new("'data_stream_template_name' must be lowercase only: <TEST-TPL>") do
245
+ driver(conf)
246
+ end
247
+ end
248
+ end
249
+
250
+ sub_test_case "invalid parameters" do
251
+ data("backslash" => "\\",
252
+ "slash" => "/",
253
+ "asterisk" => "*",
254
+ "question" => "?",
255
+ "doublequote" => "\"",
256
+ "lt" => "<",
257
+ "gt" => ">",
258
+ "bar" => "|",
259
+ "space" => " ",
260
+ "comma" => ",",
261
+ "sharp" => "#",
262
+ "colon" => ":")
263
+ def test_stream_name(data)
264
+ c, _ = data
265
+ conf = config_element(
266
+ 'ROOT', '', {
267
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
268
+ 'data_stream_name' => "TEST#{c}",
269
+ 'data_stream_ilm_name' => "default_policy",
270
+ 'data_stream_template_name' => "data_stream"
271
+ })
272
+ label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_CHARACTERS.join(',')
273
+ assert_raise Fluent::ConfigError.new("'data_stream_name' must not contain invalid characters #{label}: <TEST#{c}>") do
274
+ driver(conf)
275
+ end
276
+ end
277
+
278
+ data("backslash" => "\\",
279
+ "slash" => "/",
280
+ "asterisk" => "*",
281
+ "question" => "?",
282
+ "doublequote" => "\"",
283
+ "lt" => "<",
284
+ "gt" => ">",
285
+ "bar" => "|",
286
+ "space" => " ",
287
+ "comma" => ",",
288
+ "sharp" => "#",
289
+ "colon" => ":")
290
+ def test_stream_ilm_name(data)
291
+ c, _ = data
292
+ conf = config_element(
293
+ 'ROOT', '', {
294
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
295
+ 'data_stream_name' => "default",
296
+ 'data_stream_ilm_name' => "TEST#{c}",
297
+ 'data_stream_template_name' => "data_stream"
298
+ })
299
+ label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_CHARACTERS.join(',')
300
+ assert_raise Fluent::ConfigError.new("'data_stream_ilm_name' must not contain invalid characters #{label}: <TEST#{c}>") do
301
+ driver(conf)
302
+ end
303
+ end
304
+
305
+ data("backslash" => "\\",
306
+ "slash" => "/",
307
+ "asterisk" => "*",
308
+ "question" => "?",
309
+ "doublequote" => "\"",
310
+ "lt" => "<",
311
+ "gt" => ">",
312
+ "bar" => "|",
313
+ "space" => " ",
314
+ "comma" => ",",
315
+ "sharp" => "#",
316
+ "colon" => ":")
317
+ def test_stream_template_name(data)
318
+ c, _ = data
319
+ conf = config_element(
320
+ 'ROOT', '', {
321
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
322
+ 'data_stream_name' => "default",
323
+ 'data_stream_ilm_name' => "default_policy",
324
+ 'data_stream_template_name' => "TEST#{c}"
325
+ })
326
+ label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_CHARACTERS.join(',')
327
+ assert_raise Fluent::ConfigError.new("'data_stream_template_name' must not contain invalid characters #{label}: <TEST#{c}>") do
328
+ driver(conf)
329
+ end
330
+ end
331
+ end
332
+
333
+ sub_test_case "invalid start characters" do
334
+ data("hyphen" => "-",
335
+ "underscore" => "_",
336
+ "plus" => "+",
337
+ "period" => ".")
338
+ def test_stream_name(data)
339
+ c, _ = data
340
+ conf = config_element(
341
+ 'ROOT', '', {
342
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
343
+ 'data_stream_name' => "#{c}TEST",
344
+ 'data_stream_ilm_name' => "default-policy",
345
+ 'data_stream_template_name' => "template"
346
+ })
347
+ label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_START_CHRACTERS.join(',')
348
+ assert_raise Fluent::ConfigError.new("'data_stream_name' must not start with #{label}: <#{c}TEST>") do
349
+ driver(conf)
350
+ end
351
+ end
352
+ data("hyphen" => "-",
353
+ "underscore" => "_",
354
+ "plus" => "+",
355
+ "period" => ".")
356
+ def test_stream_ilm_name(data)
357
+ c, _ = data
358
+ conf = config_element(
359
+ 'ROOT', '', {
360
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
361
+ 'data_stream_name' => "default",
362
+ 'data_stream_ilm_name' => "#{c}TEST",
363
+ 'data_stream_template_name' => "template"
364
+ })
365
+ label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_START_CHRACTERS.join(',')
366
+ assert_raise Fluent::ConfigError.new("'data_stream_ilm_name' must not start with #{label}: <#{c}TEST>") do
367
+ driver(conf)
368
+ end
369
+ end
370
+ data("hyphen" => "-",
371
+ "underscore" => "_",
372
+ "plus" => "+",
373
+ "period" => ".")
374
+ def test_stream_template_name(data)
375
+ c, _ = data
376
+ conf = config_element(
377
+ 'ROOT', '', {
378
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
379
+ 'data_stream_name' => "default",
380
+ 'data_stream_ilm_name' => "default-policy",
381
+ 'data_stream_template_name' => "#{c}TEST"
382
+ })
383
+ label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_START_CHRACTERS.join(',')
384
+ assert_raise Fluent::ConfigError.new("'data_stream_template_name' must not start with #{label}: <#{c}TEST>") do
385
+ driver(conf)
386
+ end
387
+ end
388
+ end
389
+
390
+ sub_test_case "invalid dots" do
391
+ data("current" => ".",
392
+ "parents" => "..")
393
+ def test_stream_name
394
+ c, _ = data
395
+ conf = config_element(
396
+ 'ROOT', '', {
397
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
398
+ 'data_stream_name' => "#{c}",
399
+ 'data_stream_ilm_name' => "default-policy",
400
+ 'data_stream_template_name' => "template"
401
+ })
402
+ assert_raise Fluent::ConfigError.new("'data_stream_name' must not be . or ..: <#{c}>") do
403
+ driver(conf)
404
+ end
405
+ end
406
+
407
+ data("current" => ".",
408
+ "parents" => "..")
409
+ def test_stream_ilm_name
410
+ c, _ = data
411
+ conf = config_element(
412
+ 'ROOT', '', {
413
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
414
+ 'data_stream_name' => "default",
415
+ 'data_stream_ilm_name' => "#{c}",
416
+ 'data_stream_template_name' => "template"
417
+ })
418
+ assert_raise Fluent::ConfigError.new("'data_stream_ilm_name' must not be . or ..: <#{c}>") do
419
+ driver(conf)
420
+ end
421
+ end
422
+
423
+ data("current" => ".",
424
+ "parents" => "..")
425
+ def test_stream_template_name
426
+ c, _ = data
427
+ conf = config_element(
428
+ 'ROOT', '', {
429
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
430
+ 'data_stream_name' => "default",
431
+ 'data_stream_ilm_name' => "default-policy",
432
+ 'data_stream_template_name' => "#{c}"
433
+ })
434
+ assert_raise Fluent::ConfigError.new("'data_stream_template_name' must not be . or ..: <#{c}>") do
435
+ driver(conf)
436
+ end
437
+ end
438
+ end
439
+
440
+ sub_test_case "invalid length" do
441
+ def test_stream_name
442
+ c = "a" * 256
443
+ conf = config_element(
444
+ 'ROOT', '', {
445
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
446
+ 'data_stream_name' => "#{c}",
447
+ 'data_stream_ilm_name' => "default-policy",
448
+ 'data_stream_template_name' => "template"
449
+ })
450
+ assert_raise Fluent::ConfigError.new("'data_stream_name' must not be longer than 255 bytes: <#{c}>") do
451
+ driver(conf)
452
+ end
453
+ end
454
+ def test_stream_ilm_name
455
+ c = "a" * 256
456
+ conf = config_element(
457
+ 'ROOT', '', {
458
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
459
+ 'data_stream_name' => "default",
460
+ 'data_stream_ilm_name' => "#{c}",
461
+ 'data_stream_template_name' => "template"
462
+ })
463
+ assert_raise Fluent::ConfigError.new("'data_stream_ilm_name' must not be longer than 255 bytes: <#{c}>") do
464
+ driver(conf)
465
+ end
466
+ end
467
+ def test_stream_template_name
468
+ c = "a" * 256
469
+ conf = config_element(
470
+ 'ROOT', '', {
471
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
472
+ 'data_stream_name' => "default",
473
+ 'data_stream_ilm_name' => "default-policy",
474
+ 'data_stream_template_name' => "#{c}"
475
+ })
476
+ assert_raise Fluent::ConfigError.new("'data_stream_template_name' must not be longer than 255 bytes: <#{c}>") do
477
+ driver(conf)
478
+ end
479
+ end
480
+ end
481
+ end
482
+
483
+ def test_datastream_configure
484
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
485
+
486
+ stub_default
487
+ conf = config_element(
488
+ 'ROOT', '', {
489
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
490
+ 'data_stream_name' => 'foo',
491
+ 'data_stream_ilm_name' => "foo_ilm_policy",
492
+ 'data_stream_template_name' => "foo_tpl",
493
+ 'data_stream_template_use_index_patterns_wildcard' => false
494
+ })
495
+ assert_equal "foo", driver(conf).instance.data_stream_name
496
+ end
497
+
498
+ def test_datastream_configure_retry
499
+ stub_elastic_info
500
+ stub_nonexistent_ilm?
501
+ stub_ilm_policy
502
+ stub_nonexistent_template_retry?
503
+ stub_index_template
504
+ stub_nonexistent_data_stream?
505
+ stub_data_stream
506
+ conf = config_element(
507
+ 'ROOT', '', {
508
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
509
+ 'data_stream_name' => 'foo',
510
+ 'data_stream_ilm_name' => "foo_ilm_policy",
511
+ 'data_stream_template_name' => "foo_tpl"
512
+ })
513
+ assert_equal "foo", driver(conf).instance.data_stream_name
514
+ end
515
+
516
+ def test_hosts_list_configure
517
+ config = %{
518
+ hosts https://john:password@host1:443/elastic/,http://host2
519
+ path /default_path
520
+ user default_user
521
+ password default_password
522
+ data_stream_name default
523
+ }
524
+ stub_elastic_info("https://host1:443/elastic//", elasticsearch_version,
525
+ {'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
526
+ stub_elastic_info("http://host2/default_path/_data_stream/default", elasticsearch_version,
527
+ {'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
528
+ stub_existent_data_stream?("default", "https://host1/elastic/")
529
+ instance = driver(config).instance
530
+
531
+ assert_equal 2, instance.get_connection_options[:hosts].length
532
+ host1, host2 = instance.get_connection_options[:hosts]
533
+
534
+ assert_equal 'host1', host1[:host]
535
+ assert_equal 443, host1[:port]
536
+ assert_equal 'https', host1[:scheme]
537
+ assert_equal 'john', host1[:user]
538
+ assert_equal 'password', host1[:password]
539
+ assert_equal '/elastic/', host1[:path]
540
+
541
+ assert_equal 'host2', host2[:host]
542
+ assert_equal 'http', host2[:scheme]
543
+ assert_equal 'default_user', host2[:user]
544
+ assert_equal 'default_password', host2[:password]
545
+ assert_equal '/default_path', host2[:path]
546
+ end
547
+
548
+ def test_configure_compression
549
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
550
+
551
+ config = %{
552
+ data_stream_name foo
553
+ data_stream_template_name foo_tpl
554
+ data_stream_ilm_name foo_ilm_policy
555
+ compression_level best_compression
556
+ }
557
+
558
+ stub_default
559
+
560
+ assert_equal true, driver(config).instance.compression
561
+ end
562
+
563
+ def test_configure_wildcard_usage
564
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
565
+
566
+ config = %{
567
+ data_stream_name foo
568
+ data_stream_template_name foo_tpl
569
+ data_stream_ilm_name foo_ilm_policy
570
+ data_stream_template_use_index_patterns_wildcard false
571
+ }
572
+
573
+ stub_default
574
+
575
+ assert_equal false, driver(config).instance.data_stream_template_use_index_patterns_wildcard
576
+ end
577
+
578
+ def test_check_compression_strategy
579
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
580
+
581
+ config = %{
582
+ data_stream_name foo
583
+ data_stream_template_name foo_tpl
584
+ data_stream_ilm_name foo_ilm_policy
585
+ compression_level best_speed
586
+ }
587
+
588
+ stub_default
589
+ stub_bulk_feed
590
+
591
+ assert_equal Zlib::BEST_SPEED, driver(config).instance.compression_strategy
592
+ end
593
+
594
+ def test_check_content_encoding_header_with_compression
595
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
596
+
597
+ config = %{
598
+ data_stream_name foo
599
+ data_stream_template_name foo_tpl
600
+ data_stream_ilm_name foo_ilm_policy
601
+ compression_level best_compression
602
+ }
603
+
604
+ stub_default
605
+ stub_request(:post, "http://localhost:9200/foo/_bulk").
606
+ to_return(status: 200, body: "", headers: {'x-elastic-product' => 'Elasticsearch'})
607
+
608
+ instance = driver(config).instance
609
+
610
+ if elastic_transport_layer?
611
+ assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
612
+ elsif elasticsearch_transport_layer_decoupling?
613
+ assert_equal nil, instance.client.transport.transport.options[:transport_options][:headers]["Content-Encoding"]
614
+ else
615
+ assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
616
+ end
617
+
618
+ driver.run(default_tag: 'test') do
619
+ driver.feed(sample_record)
620
+ end
621
+ compressable = instance.compressable_connection
622
+
623
+ if elastic_transport_layer?
624
+ assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
625
+ elsif elasticsearch_transport_layer_decoupling?
626
+ assert_equal "gzip", instance.client(nil, compressable).transport.transport.options[:transport_options][:headers]["Content-Encoding"]
627
+ else
628
+ assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
629
+ end
630
+ end
631
+
632
+ def test_check_compression_option_is_passed_to_transport
633
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
634
+
635
+ config = %{
636
+ data_stream_name foo
637
+ data_stream_template_name foo_tpl
638
+ data_stream_ilm_name foo_ilm_policy
639
+ compression_level best_compression
640
+ }
641
+
642
+ stub_default
643
+ stub_request(:post, "http://localhost:9200/foo/_bulk").
644
+ to_return(status: 200, body: "", headers: {'x-elastic-product' => 'Elasticsearch'})
645
+
646
+ instance = driver(config).instance
647
+
648
+ if elastic_transport_layer?
649
+ assert_equal false, instance.client.transport.options[:compression]
650
+ elsif elasticsearch_transport_layer_decoupling?
651
+ assert_equal false, instance.client.transport.transport.options[:compression]
652
+ else
653
+ assert_equal false, instance.client.transport.options[:compression]
654
+ end
655
+
656
+
657
+ driver.run(default_tag: 'test') do
658
+ driver.feed(sample_record)
659
+ end
660
+ compressable = instance.compressable_connection
661
+
662
+ if elastic_transport_layer?
663
+ assert_equal true, instance.client(nil, compressable).transport.options[:compression]
664
+ elsif elasticsearch_transport_layer_decoupling?
665
+ assert_equal true, instance.client(nil, compressable).transport.transport.options[:compression]
666
+ else
667
+ assert_equal true, instance.client(nil, compressable).transport.options[:compression]
668
+ end
669
+ end
670
+
671
+ def test_existent_data_stream
672
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
673
+
674
+ stub_ilm_policy
675
+ stub_index_template
676
+ stub_existent_data_stream?
677
+ stub_data_stream
678
+ stub_elastic_info
679
+ conf = config_element(
680
+ 'ROOT', '', {
681
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
682
+ 'data_stream_name' => 'foo',
683
+ 'data_stream_ilm_name' => "foo_ilm_policy",
684
+ 'data_stream_template_name' => "foo_tpl"
685
+ })
686
+ assert_equal "foo", driver(conf).instance.data_stream_name
687
+ end
688
+
689
+ def test_template_unset
690
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
691
+
692
+ stub_ilm_policy
693
+ stub_index_template
694
+ stub_existent_data_stream?
695
+ stub_data_stream
696
+ stub_elastic_info
697
+ conf = config_element(
698
+ 'ROOT', '', {
699
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
700
+ 'data_stream_name' => 'foo',
701
+ 'data_stream_ilm_name' => "foo_ilm_policy",
702
+ })
703
+ assert_equal "foo", driver(conf).instance.data_stream_name
704
+ assert_equal "foo_ilm_policy", driver(conf).instance.data_stream_ilm_name
705
+ assert_equal "foo_template", driver(conf).instance.data_stream_template_name
706
+ end
707
+
708
+ def test_ilm_unset
709
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
710
+
711
+ stub_ilm_policy
712
+ stub_index_template
713
+ stub_existent_data_stream?
714
+ stub_data_stream
715
+ stub_elastic_info
716
+ conf = config_element(
717
+ 'ROOT', '', {
718
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
719
+ 'data_stream_name' => 'foo',
720
+ 'data_stream_template_name' => "foo_tpl"
721
+ })
722
+ assert_equal "foo", driver(conf).instance.data_stream_name
723
+ assert_equal "foo_tpl", driver(conf).instance.data_stream_template_name
724
+ end
725
+
726
+ def test_template_and_ilm_unset
727
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
728
+
729
+ stub_ilm_policy
730
+ stub_index_template
731
+ stub_existent_data_stream?
732
+ stub_data_stream
733
+ stub_elastic_info
734
+ conf = config_element(
735
+ 'ROOT', '', {
736
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
737
+ 'data_stream_name' => 'foo',
738
+ })
739
+ assert_equal "foo", driver(conf).instance.data_stream_name
740
+ assert_equal "foo_template", driver(conf).instance.data_stream_template_name
741
+ assert_equal "foo_policy", driver(conf).instance.data_stream_ilm_name
742
+ end
743
+
744
+ def test_template_index_patterns_with_wildcard
745
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
746
+
747
+ stub_existent_ilm?
748
+
749
+ stub_nonexistent_data_stream?
750
+ stub_data_stream
751
+
752
+ stub_nonexistent_template?("foo_template")
753
+ stub_request(:put, "http://localhost:9200/#{index_template_endpoint}/foo_template").with do |req|
754
+ # bulk data must be pair of OP and records
755
+ # {"create": {}}\nhttp://localhost:9200/_index_template//foo_template
756
+ # {"@timestamp": ...}
757
+ push_bulk_request(req.body)
758
+ end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
759
+
760
+ conf = config_element(
761
+ 'ROOT', '', {
762
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
763
+ 'data_stream_name' => 'foo',
764
+ 'data_stream_ilm_name' => 'foo_ilm_policy',
765
+ 'data_stream_template_use_index_patterns_wildcard' => false
766
+ })
767
+
768
+ assert_nothing_raised {
769
+ driver(conf)
770
+ }
771
+
772
+ assert_requested(:put, "http://localhost:9200/#{index_template_endpoint}/foo_template",
773
+ body: '{"index_patterns":["foo"],"data_stream":{},"template":{"settings":{"index.lifecycle.name":"foo_ilm_policy"}}}',
774
+ times: 1)
775
+ end
776
+
777
+ def test_template_index_patterns_without_wildcard
778
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
779
+
780
+ stub_existent_ilm?
781
+
782
+ stub_nonexistent_data_stream?
783
+ stub_data_stream
784
+
785
+ stub_nonexistent_template?("foo_template")
786
+ stub_request(:put, "http://localhost:9200/#{index_template_endpoint}/foo_template").with do |req|
787
+ # bulk data must be pair of OP and records
788
+ # {"create": {}}\nhttp://localhost:9200/_index_template//foo_template
789
+ # {"@timestamp": ...}
790
+ push_bulk_request(req.body)
791
+ end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
792
+
793
+ conf = config_element(
794
+ 'ROOT', '', {
795
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
796
+ 'data_stream_name' => 'foo',
797
+ 'data_stream_ilm_name' => 'foo_ilm_policy',
798
+ 'data_stream_template_use_index_patterns_wildcard' => true
799
+ })
800
+
801
+ assert_nothing_raised {
802
+ driver(conf)
803
+ }
804
+
805
+ assert_requested(:put, "http://localhost:9200/#{index_template_endpoint}/foo_template",
806
+ body: '{"index_patterns":["foo*"],"data_stream":{},"template":{"settings":{"index.lifecycle.name":"foo_ilm_policy"}}}',
807
+ times: 1)
808
+ end
809
+
810
+ def test_placeholder
811
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
812
+
813
+ dsname = "foo_test"
814
+ ilmname = "foo_ilm_test"
815
+ tplname = "foo_tpl_test"
816
+ stub_default(dsname, ilmname, tplname)
817
+ stub_bulk_feed(dsname, ilmname, tplname)
818
+ conf = config_element(
819
+ 'ROOT', '', {
820
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
821
+ 'data_stream_name' => 'foo_${tag}',
822
+ 'data_stream_ilm_name' => "foo_ilm_${tag}",
823
+ 'data_stream_template_name' => "foo_tpl_${tag}"
824
+ })
825
+ driver(conf).run(default_tag: 'test') do
826
+ driver.feed(sample_record)
827
+ end
828
+ assert_equal 1, @bulk_records.length
829
+ end
830
+
831
+ def test_placeholder_params_unset
832
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
833
+
834
+ dsname = "foo_test"
835
+ ilmname = "foo_test_policy"
836
+ tplname = "foo_test_template"
837
+ stub_default(dsname, ilmname, tplname)
838
+ stub_bulk_feed(dsname, ilmname, tplname)
839
+ conf = config_element(
840
+ 'ROOT', '', {
841
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
842
+ 'data_stream_name' => 'foo_${tag}',
843
+ })
844
+ driver(conf).run(default_tag: 'test') do
845
+ driver.feed(sample_record)
846
+ end
847
+ assert_equal 1, @bulk_records.length
848
+ end
849
+
850
+
851
+ def test_time_placeholder
852
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
853
+
854
+ time = Time.now
855
+ dsname = "foo_#{time.strftime("%Y%m%d")}"
856
+ ilmname = "foo_ilm_#{time.strftime("%Y%m%d")}"
857
+ tplname = "foo_tpl_#{time.strftime("%Y%m%d")}"
858
+ stub_default(dsname, ilmname, tplname)
859
+ stub_bulk_feed(dsname, ilmname, tplname)
860
+ conf = config_element(
861
+ 'ROOT', '', {
862
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
863
+ 'data_stream_name' => 'foo_%Y%m%d',
864
+ 'data_stream_ilm_name' => 'foo_ilm_%Y%m%d',
865
+ 'data_stream_template_name' => 'foo_tpl_%Y%m%d'
866
+ }, [config_element('buffer', 'time', {
867
+ 'timekey' => '1d'
868
+ }, [])]
869
+ )
870
+ driver(conf).run(default_tag: 'test') do
871
+ driver.feed(sample_record)
872
+ end
873
+ assert_equal 1, @bulk_records.length
874
+ end
875
+
876
+ def test_custom_record_placeholder
877
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
878
+
879
+ keys = ["bar", "baz"]
880
+ keys.each do |key|
881
+ dsname = "foo_#{key}"
882
+ ilmname = "foo_ilm_#{key}"
883
+ tplname = "foo_tpl_#{key}"
884
+ stub_default(dsname, ilmname, tplname)
885
+ stub_bulk_feed(dsname, ilmname, tplname)
886
+ end
887
+ conf = config_element(
888
+ 'ROOT', '', {
889
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
890
+ 'data_stream_name' => 'foo_${key1}',
891
+ 'data_stream_ilm_name' => 'foo_ilm_${key1}',
892
+ 'data_stream_template_name' => 'foo_tpl_${key1}'
893
+ }, [config_element('buffer', 'tag,key1', {
894
+ 'timekey' => '1d'
895
+ }, [])]
896
+ )
897
+ driver(conf).run(default_tag: 'test') do
898
+ keys.each do |key|
899
+ record = sample_record.merge({"key1" => key})
900
+ driver.feed(record)
901
+ end
902
+ end
903
+ assert_equal keys.count, @bulk_records.length
904
+ end
905
+
906
+ def test_bulk_insert_feed
907
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
908
+
909
+ stub_default
910
+ stub_bulk_feed
911
+ conf = config_element(
912
+ 'ROOT', '', {
913
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
914
+ 'data_stream_name' => 'foo',
915
+ 'data_stream_ilm_name' => 'foo_ilm_policy',
916
+ 'data_stream_template_name' => 'foo_tpl'
917
+ })
918
+ driver(conf).run(default_tag: 'test') do
919
+ driver.feed(sample_record)
920
+ end
921
+ assert_equal 1, @bulk_records.length
922
+ end
923
+
924
+ def test_placeholder_writes_to_multi_hosts
925
+ stub_default("foo_bar", "foo_tpl_bar")
926
+ hosts = [['192.168.33.50', 9201], ['192.168.33.51', 9201], ['192.168.33.52', 9201]]
927
+ hosts_string = hosts.map {|x| "#{x[0]}:#{x[1]}"}.compact.join(',')
928
+ hosts.each do |host_info|
929
+ host, port = host_info
930
+ stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/foo_bar/_bulk")
931
+ stub_elastic_info("http://#{host}:#{port}/")
932
+ stub_request(:get, "http://#{host}:#{port}/_data_stream/foo_bar").
933
+ to_return(status: 200, body: "", headers: {'x-elastic-product' => 'Elasticsearch'})
934
+ end
935
+
936
+ conf = config_element(
937
+ 'ROOT', '', {
938
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
939
+ 'data_stream_name' => 'foo_${key1}',
940
+ 'data_stream_template_name' => 'foo_tpl_${key1}',
941
+ 'hosts' => "#{hosts_string}"
942
+ }, [config_element('buffer', 'tag,key1', {
943
+ 'timekey' => '1d'
944
+ }, [])])
945
+ driver(conf).run(default_tag: 'test') do
946
+ hashes = {
947
+ 'age' => rand(100),
948
+ 'key1' => 'bar'
949
+ }
950
+ 1000.times do
951
+ driver.feed(sample_record.merge(hashes))
952
+ end
953
+ end
954
+
955
+ # @note: we cannot make multi chunks with options (flush_interval, buffer_chunk_limit)
956
+ # it's Fluentd test driver's constraint
957
+ # so @index_command_counts.size is always 1
958
+ assert(@index_command_counts.size > 0, "not working with hosts options")
959
+
960
+ total = 0
961
+ @index_command_counts.each do |_, count|
962
+ total += count
963
+ end
964
+ assert_equal(2000, total)
965
+ end
966
+
967
+ # gzip compress data
968
+ def gzip(string, strategy)
969
+ wio = StringIO.new("w")
970
+ w_gz = Zlib::GzipWriter.new(wio, strategy = strategy)
971
+ w_gz.write(string)
972
+ w_gz.close
973
+ wio.string
974
+ end
975
+
976
+ def test_writes_to_data_stream_with_compression
977
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
978
+
979
+ config = %{
980
+ data_stream_name foo
981
+ data_stream_template_name foo_tpl
982
+ data_stream_ilm_name foo_ilm_policy
983
+ compression_level default_compression
984
+ }
985
+
986
+ bodystr = %({
987
+ "took" : 500,
988
+ "errors" : false,
989
+ "items" : [
990
+ {
991
+ "create": {
992
+ "_index" : "fluentd",
993
+ "_type" : "fluentd"
994
+ }
995
+ }
996
+ ]
997
+ })
998
+
999
+ compressed_body = gzip(bodystr, Zlib::DEFAULT_COMPRESSION)
1000
+
1001
+ stub_default
1002
+ elastic_request = stub_request(:post, "http://localhost:9200/foo/_bulk").
1003
+ to_return(:status => 200, :headers => {'Content-Type' => 'application/json', 'x-elastic-product' => 'Elasticsearch'}, :body => compressed_body)
1004
+
1005
+ driver(config)
1006
+ driver.run(default_tag: 'test') do
1007
+ driver.feed(sample_record)
1008
+ end
1009
+
1010
+ assert_requested(elastic_request)
1011
+ end
1012
+
1013
+ def test_doesnt_update_ilm_policy_if_overwrite_unset
1014
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
1015
+
1016
+ config = %{
1017
+ data_stream_name foo
1018
+ data_stream_ilm_name foo_ilm_policy
1019
+ data_stream_ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
1020
+ }
1021
+
1022
+ stub_elastic_info
1023
+ stub_index_template
1024
+ stub_existent_data_stream?
1025
+ stub_existent_ilm?
1026
+ stub_data_stream
1027
+
1028
+ stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
1029
+ to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
1030
+
1031
+ assert_nothing_raised {
1032
+ driver(config)
1033
+ }
1034
+ assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 0)
1035
+ end
1036
+
1037
+ def test_updates_ilm_policy_if_overwrite_set
1038
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
1039
+
1040
+ config = %{
1041
+ data_stream_name foo
1042
+ data_stream_ilm_name foo_ilm_policy
1043
+ data_stream_ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
1044
+ data_stream_ilm_policy_overwrite true
1045
+ }
1046
+
1047
+ stub_elastic_info
1048
+ stub_index_template
1049
+ stub_existent_data_stream?
1050
+ stub_existent_ilm?
1051
+ stub_data_stream
1052
+
1053
+ stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
1054
+ to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
1055
+
1056
+ assert_nothing_raised {
1057
+ driver(config)
1058
+ }
1059
+
1060
+ assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 1)
1061
+ assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy",
1062
+ body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
1063
+ times: 1)
1064
+ end
1065
+
1066
+ def test_creates_custom_ilm_policy_if_none_exists
1067
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
1068
+
1069
+ config = %{
1070
+ data_stream_name foo
1071
+ data_stream_ilm_name foo_ilm_policy
1072
+ data_stream_ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
1073
+ }
1074
+
1075
+ stub_elastic_info
1076
+ stub_index_template("foo_template")
1077
+ stub_data_stream
1078
+ stub_nonexistent_data_stream?
1079
+ stub_nonexistent_ilm?
1080
+ stub_nonexistent_template?("foo_template")
1081
+
1082
+ stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
1083
+ to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
1084
+
1085
+ assert_nothing_raised {
1086
+ driver(config)
1087
+ }
1088
+
1089
+ assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 1)
1090
+ assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy",
1091
+ body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
1092
+ times: 1)
1093
+ end
1094
+
1095
+ def test_doesnt_add_tag_key_when_not_configured
1096
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
1097
+
1098
+ config = %{
1099
+ data_stream_name foo
1100
+ data_stream_template_name foo_tpl
1101
+ data_stream_ilm_name foo_ilm_policy
1102
+ }
1103
+
1104
+ stub_default
1105
+ stub_bulk_feed
1106
+ driver(config)
1107
+ driver.run(default_tag: 'mytag') do
1108
+ driver.feed(sample_record)
1109
+ end
1110
+
1111
+ assert_equal(1, @bulk_records.length)
1112
+ assert_false(@bulk_records[0].has_key?('tag'))
1113
+ end
1114
+
1115
+
1116
+ def test_adds_tag_key_when_configured
1117
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
1118
+
1119
+ config = %{
1120
+ data_stream_name foo
1121
+ data_stream_template_name foo_tpl
1122
+ data_stream_ilm_name foo_ilm_policy
1123
+ include_tag_key true
1124
+ }
1125
+
1126
+ stub_default
1127
+ stub_bulk_feed
1128
+ driver(config)
1129
+ driver.run(default_tag: 'mytag') do
1130
+ driver.feed(sample_record)
1131
+ end
1132
+
1133
+ assert_equal(1, @bulk_records.length)
1134
+ assert(@bulk_records[0].has_key?('tag'))
1135
+ assert_equal('mytag', @bulk_records[0]['tag'])
1136
+ end
1137
+
1138
+ def test_adds_custom_tag_key_when_configured
1139
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
1140
+
1141
+ config = %{
1142
+ data_stream_name foo
1143
+ data_stream_template_name foo_tpl
1144
+ data_stream_ilm_name foo_ilm_policy
1145
+ include_tag_key true
1146
+ tag_key custom_tag_key
1147
+ }
1148
+
1149
+ stub_default
1150
+ stub_bulk_feed
1151
+ driver(config)
1152
+ driver.run(default_tag: 'mytag') do
1153
+ driver.feed(sample_record)
1154
+ end
1155
+
1156
+ assert_equal(1, @bulk_records.length)
1157
+ assert(@bulk_records[0].has_key?('custom_tag_key'))
1158
+ assert_equal('mytag', @bulk_records[0]['custom_tag_key'])
1159
+ end
1160
+
1161
+ def test_use_record_timestamp_if_present
1162
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
1163
+
1164
+ stub_default
1165
+ stub_bulk_feed
1166
+ conf = config_element(
1167
+ 'ROOT', '', {
1168
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
1169
+ 'data_stream_name' => 'foo',
1170
+ 'data_stream_ilm_name' => 'foo_ilm_policy',
1171
+ 'data_stream_template_name' => 'foo_tpl'
1172
+ })
1173
+ driver(conf).run(default_tag: 'test') do
1174
+ driver.feed(sample_record)
1175
+ end
1176
+ assert_equal 1, @bulk_records.length
1177
+ assert(@bulk_records[0].has_key?('@timestamp'))
1178
+ assert_equal SAMPLE_RECORD_TIMESTAMP, @bulk_records[0]['@timestamp']
1179
+ end
1180
+
1181
+ def test_add_timestamp_if_not_present_in_record
1182
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
1183
+
1184
+ stub_default
1185
+ stub_bulk_feed
1186
+ conf = config_element(
1187
+ 'ROOT', '', {
1188
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
1189
+ 'data_stream_name' => 'foo',
1190
+ 'data_stream_ilm_name' => 'foo_ilm_policy',
1191
+ 'data_stream_template_name' => 'foo_tpl'
1192
+ })
1193
+ driver(conf).run(default_tag: 'test') do
1194
+ driver.feed(sample_record_no_timestamp)
1195
+ end
1196
+ assert_equal 1, @bulk_records.length
1197
+ assert(@bulk_records[0].has_key?('@timestamp'))
1198
+ end
1199
+ end