fluent-plugin-elasticsearch 4.1.1 → 5.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/bug_report.md +37 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +24 -0
- data/.github/dependabot.yml +6 -0
- data/.github/workflows/issue-auto-closer.yml +2 -2
- data/.github/workflows/linux.yml +5 -2
- data/.github/workflows/macos.yml +5 -2
- data/.github/workflows/windows.yml +5 -2
- data/Gemfile +1 -2
- data/History.md +146 -0
- data/README.ElasticsearchGenID.md +4 -4
- data/README.ElasticsearchInput.md +1 -1
- data/README.Troubleshooting.md +692 -0
- data/README.md +260 -550
- data/fluent-plugin-elasticsearch.gemspec +4 -1
- data/lib/fluent/plugin/elasticsearch_compat.rb +31 -0
- data/lib/fluent/plugin/elasticsearch_error_handler.rb +19 -4
- data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +2 -2
- data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb +18 -4
- data/lib/fluent/plugin/elasticsearch_index_template.rb +65 -21
- data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb +2 -1
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +1 -1
- data/lib/fluent/plugin/in_elasticsearch.rb +8 -2
- data/lib/fluent/plugin/oj_serializer.rb +2 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +192 -36
- data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +298 -0
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +3 -1
- data/test/helper.rb +0 -4
- data/test/plugin/mock_chunk.dat +0 -0
- data/test/plugin/test_elasticsearch_error_handler.rb +130 -23
- data/test/plugin/test_elasticsearch_fallback_selector.rb +17 -8
- data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +57 -18
- data/test/plugin/test_elasticsearch_tls.rb +8 -2
- data/test/plugin/test_filter_elasticsearch_genid.rb +16 -16
- data/test/plugin/test_in_elasticsearch.rb +51 -21
- data/test/plugin/test_index_alias_template.json +11 -0
- data/test/plugin/test_index_template.json +25 -0
- data/test/plugin/test_out_elasticsearch.rb +2118 -704
- data/test/plugin/test_out_elasticsearch_data_stream.rb +1199 -0
- data/test/plugin/test_out_elasticsearch_dynamic.rb +170 -31
- metadata +62 -10
- data/.coveralls.yml +0 -2
- data/.travis.yml +0 -44
- data/appveyor.yml +0 -20
- data/gemfiles/Gemfile.without.ilm +0 -10
@@ -16,7 +16,15 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
16
16
|
@driver = nil
|
17
17
|
end
|
18
18
|
|
19
|
-
def
|
19
|
+
def elasticsearch_version
|
20
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
21
|
+
TRANSPORT_CLASS::VERSION
|
22
|
+
else
|
23
|
+
'6.4.2'.freeze
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def driver(conf='', es_version=elasticsearch_version.to_i)
|
20
28
|
# For request stub to detect compatibility.
|
21
29
|
@es_version ||= es_version
|
22
30
|
Fluent::Plugin::ElasticsearchOutputDynamic.module_eval(<<-CODE)
|
@@ -34,6 +42,14 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
34
42
|
}.configure(conf)
|
35
43
|
end
|
36
44
|
|
45
|
+
def elasticsearch_transport_layer_decoupling?
|
46
|
+
Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
47
|
+
end
|
48
|
+
|
49
|
+
def elastic_transport_layer?
|
50
|
+
Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
51
|
+
end
|
52
|
+
|
37
53
|
def default_type_name
|
38
54
|
Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME
|
39
55
|
end
|
@@ -51,7 +67,12 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
51
67
|
def stub_elastic(url="http://localhost:9200/_bulk")
|
52
68
|
stub_request(:post, url).with do |req|
|
53
69
|
@index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
|
54
|
-
end
|
70
|
+
end.to_return({:status => 200, :body => "", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
|
71
|
+
end
|
72
|
+
|
73
|
+
def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
|
74
|
+
body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
|
75
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
|
55
76
|
end
|
56
77
|
|
57
78
|
def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
|
@@ -71,7 +92,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
71
92
|
stub_request(:post, url).with do |req|
|
72
93
|
index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
|
73
94
|
@index_command_counts[url] += index_cmds.size
|
74
|
-
end
|
95
|
+
end.to_return({:status => 200, :body => "", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
|
75
96
|
end
|
76
97
|
|
77
98
|
def assert_logs_include(logs, msg, exp_matches=1)
|
@@ -101,8 +122,13 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
101
122
|
assert_nil instance.ssl_max_version
|
102
123
|
assert_nil instance.ssl_min_version
|
103
124
|
if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
|
104
|
-
|
105
|
-
|
125
|
+
if defined?(OpenSSL::SSL::TLS1_3_VERSION)
|
126
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
|
127
|
+
instance.ssl_version_options)
|
128
|
+
else
|
129
|
+
assert_equal({max_version: nil, min_version: OpenSSL::SSL::TLS1_2_VERSION},
|
130
|
+
instance.ssl_version_options)
|
131
|
+
end
|
106
132
|
else
|
107
133
|
assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
|
108
134
|
instance.ssl_version_options)
|
@@ -118,7 +144,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
118
144
|
end
|
119
145
|
|
120
146
|
test 'configure compression' do
|
121
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
147
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
122
148
|
|
123
149
|
config = %{
|
124
150
|
compression_level best_compression
|
@@ -129,7 +155,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
129
155
|
end
|
130
156
|
|
131
157
|
test 'check compression strategy' do
|
132
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
158
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
133
159
|
|
134
160
|
config = %{
|
135
161
|
compression_level best_speed
|
@@ -140,43 +166,69 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
140
166
|
end
|
141
167
|
|
142
168
|
test 'check content-encoding header with compression' do
|
143
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
169
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
144
170
|
|
145
171
|
config = %{
|
146
172
|
compression_level best_compression
|
147
173
|
}
|
148
174
|
instance = driver(config).instance
|
149
175
|
|
150
|
-
|
176
|
+
if elastic_transport_layer?
|
177
|
+
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
178
|
+
elsif elasticsearch_transport_layer_decoupling?
|
179
|
+
assert_equal nil, instance.client.transport.transport.options[:transport_options][:headers]["Content-Encoding"]
|
180
|
+
else
|
181
|
+
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
182
|
+
end
|
151
183
|
|
152
184
|
stub_request(:post, "http://localhost:9200/_bulk").
|
153
|
-
to_return(status: 200, body: "", headers: {})
|
185
|
+
to_return(status: 200, body: "", headers: {'x-elastic-product' => 'Elasticsearch'})
|
186
|
+
stub_elastic_info
|
154
187
|
driver.run(default_tag: 'test') do
|
155
188
|
driver.feed(sample_record)
|
156
189
|
end
|
157
190
|
compressable = instance.compressable_connection
|
158
191
|
|
159
|
-
|
192
|
+
if elastic_transport_layer?
|
193
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
194
|
+
elsif elasticsearch_transport_layer_decoupling?
|
195
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.transport.options[:transport_options][:headers]["Content-Encoding"]
|
196
|
+
else
|
197
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
198
|
+
end
|
160
199
|
end
|
161
200
|
|
162
201
|
test 'check compression option is passed to transport' do
|
163
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
202
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
164
203
|
|
165
204
|
config = %{
|
166
205
|
compression_level best_compression
|
167
206
|
}
|
168
207
|
instance = driver(config).instance
|
169
208
|
|
170
|
-
|
209
|
+
if elastic_transport_layer?
|
210
|
+
assert_equal false, instance.client.transport.options[:compression]
|
211
|
+
elsif elasticsearch_transport_layer_decoupling?
|
212
|
+
assert_equal false, instance.client.transport.transport.options[:compression]
|
213
|
+
else
|
214
|
+
assert_equal false, instance.client.transport.options[:compression]
|
215
|
+
end
|
171
216
|
|
172
217
|
stub_request(:post, "http://localhost:9200/_bulk").
|
173
|
-
to_return(status: 200, body: "", headers: {})
|
218
|
+
to_return(status: 200, body: "", headers: {'x-elastic-product' => 'Elasticsearch'})
|
219
|
+
stub_elastic_info
|
174
220
|
driver.run(default_tag: 'test') do
|
175
221
|
driver.feed(sample_record)
|
176
222
|
end
|
177
223
|
compressable = instance.compressable_connection
|
178
224
|
|
179
|
-
|
225
|
+
if elastic_transport_layer?
|
226
|
+
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
227
|
+
elsif elasticsearch_transport_layer_decoupling?
|
228
|
+
assert_equal true, instance.client(nil, compressable).transport.transport.options[:compression]
|
229
|
+
else
|
230
|
+
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
231
|
+
end
|
180
232
|
end
|
181
233
|
|
182
234
|
test 'configure Content-Type' do
|
@@ -335,14 +387,18 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
335
387
|
|
336
388
|
def test_content_type_header
|
337
389
|
stub_request(:head, "http://localhost:9200/").
|
338
|
-
to_return(:status => 200, :body => "", :headers => {})
|
390
|
+
to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
|
339
391
|
if Elasticsearch::VERSION >= "6.0.2"
|
340
392
|
elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
|
341
|
-
with(headers: { "Content-Type" => "application/x-ndjson"
|
393
|
+
with(headers: { "Content-Type" => "application/x-ndjson"}).
|
394
|
+
to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
|
342
395
|
else
|
343
396
|
elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
|
344
|
-
with(headers: { "Content-Type" => "application/json"
|
397
|
+
with(headers: { "Content-Type" => "application/json"}).
|
398
|
+
to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
|
345
399
|
end
|
400
|
+
stub_elastic_info('http://localhost:9200')
|
401
|
+
|
346
402
|
driver.run(default_tag: 'test') do
|
347
403
|
driver.feed(sample_record)
|
348
404
|
end
|
@@ -351,6 +407,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
351
407
|
|
352
408
|
def test_writes_to_default_index
|
353
409
|
stub_elastic
|
410
|
+
stub_elastic_info
|
354
411
|
driver.run(default_tag: 'test') do
|
355
412
|
driver.feed(sample_record)
|
356
413
|
end
|
@@ -367,7 +424,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
367
424
|
end
|
368
425
|
|
369
426
|
def test_writes_to_default_index_with_compression
|
370
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
427
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
371
428
|
|
372
429
|
config = %[
|
373
430
|
compression_level default_compression
|
@@ -389,7 +446,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
389
446
|
compressed_body = gzip(bodystr, Zlib::DEFAULT_COMPRESSION)
|
390
447
|
|
391
448
|
elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
|
392
|
-
to_return(:status => 200, :headers => {'Content-Type' => 'Application/json'}, :body => compressed_body)
|
449
|
+
to_return(:status => 200, :headers => {'Content-Type' => 'Application/json', 'x-elastic-product' => 'Elasticsearch'}, :body => compressed_body)
|
450
|
+
stub_elastic_info
|
393
451
|
|
394
452
|
driver(config)
|
395
453
|
driver.run(default_tag: 'test') do
|
@@ -401,15 +459,23 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
401
459
|
|
402
460
|
def test_writes_to_default_type
|
403
461
|
stub_elastic
|
462
|
+
stub_elastic_info
|
404
463
|
driver.run(default_tag: 'test') do
|
405
464
|
driver.feed(sample_record)
|
406
465
|
end
|
407
|
-
|
466
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
467
|
+
assert_nil(index_cmds.first['index']['_type'])
|
468
|
+
elsif Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.0.0")
|
469
|
+
assert_equal("_doc", index_cmds.first['index']['_type'])
|
470
|
+
else
|
471
|
+
assert_equal("fluentd", index_cmds.first['index']['_type'])
|
472
|
+
end
|
408
473
|
end
|
409
474
|
|
410
475
|
def test_writes_to_specified_index
|
411
476
|
driver.configure("index_name myindex\n")
|
412
477
|
stub_elastic
|
478
|
+
stub_elastic_info
|
413
479
|
driver.run(default_tag: 'test') do
|
414
480
|
driver.feed(sample_record)
|
415
481
|
end
|
@@ -419,6 +485,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
419
485
|
def test_writes_to_specified_index_uppercase
|
420
486
|
driver.configure("index_name MyIndex\n")
|
421
487
|
stub_elastic
|
488
|
+
stub_elastic_info
|
422
489
|
driver.run(default_tag: 'test') do
|
423
490
|
driver.feed(sample_record)
|
424
491
|
end
|
@@ -428,15 +495,23 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
428
495
|
def test_writes_to_specified_type
|
429
496
|
driver.configure("type_name mytype\n")
|
430
497
|
stub_elastic
|
498
|
+
stub_elastic_info
|
431
499
|
driver.run(default_tag: 'test') do
|
432
500
|
driver.feed(sample_record)
|
433
501
|
end
|
434
|
-
|
502
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
503
|
+
assert_nil(index_cmds.first['index']['_type'])
|
504
|
+
elsif Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.0.0")
|
505
|
+
assert_equal("_doc", index_cmds.first['index']['_type'])
|
506
|
+
else
|
507
|
+
assert_equal("mytype", index_cmds.first['index']['_type'])
|
508
|
+
end
|
435
509
|
end
|
436
510
|
|
437
511
|
def test_writes_to_specified_host
|
438
512
|
driver.configure("host 192.168.33.50\n")
|
439
513
|
elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
|
514
|
+
stub_elastic_info("http://192.168.33.50:9200/")
|
440
515
|
driver.run(default_tag: 'test') do
|
441
516
|
driver.feed(sample_record)
|
442
517
|
end
|
@@ -446,6 +521,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
446
521
|
def test_writes_to_specified_port
|
447
522
|
driver.configure("port 9201\n")
|
448
523
|
elastic_request = stub_elastic("http://localhost:9201/_bulk")
|
524
|
+
stub_elastic_info("http://localhost:9201/")
|
449
525
|
driver.run(default_tag: 'test') do
|
450
526
|
driver.feed(sample_record)
|
451
527
|
end
|
@@ -461,6 +537,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
461
537
|
hosts.each do |host_info|
|
462
538
|
host, port = host_info
|
463
539
|
stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/_bulk")
|
540
|
+
stub_elastic_info("http://#{host}:#{port}/")
|
464
541
|
end
|
465
542
|
|
466
543
|
driver.run(default_tag: 'test') do
|
@@ -496,6 +573,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
496
573
|
]}
|
497
574
|
|
498
575
|
stub_elastic
|
576
|
+
stub_elastic_info
|
499
577
|
driver.run(default_tag: 'test') do
|
500
578
|
driver.feed(original_hash)
|
501
579
|
end
|
@@ -509,6 +587,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
509
587
|
expected_output = {"foo" => {"bar" => "baz"}}
|
510
588
|
|
511
589
|
stub_elastic
|
590
|
+
stub_elastic_info
|
512
591
|
driver.run(default_tag: 'test') do
|
513
592
|
driver.feed(original_hash)
|
514
593
|
end
|
@@ -517,6 +596,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
517
596
|
|
518
597
|
def test_makes_bulk_request
|
519
598
|
stub_elastic
|
599
|
+
stub_elastic_info
|
520
600
|
driver.run(default_tag: 'test') do
|
521
601
|
driver.feed(sample_record)
|
522
602
|
driver.feed(sample_record.merge('age' => 27))
|
@@ -526,6 +606,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
526
606
|
|
527
607
|
def test_all_records_are_preserved_in_bulk
|
528
608
|
stub_elastic
|
609
|
+
stub_elastic_info
|
529
610
|
driver.run(default_tag: 'test') do
|
530
611
|
driver.feed(sample_record)
|
531
612
|
driver.feed(sample_record.merge('age' => 27))
|
@@ -539,6 +620,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
539
620
|
time = Time.parse Date.today.iso8601
|
540
621
|
logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
|
541
622
|
stub_elastic
|
623
|
+
stub_elastic_info
|
542
624
|
driver.run(default_tag: 'test') do
|
543
625
|
driver.feed(time.to_i, sample_record)
|
544
626
|
end
|
@@ -551,6 +633,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
551
633
|
time = Time.parse Date.today.iso8601
|
552
634
|
utc_index = "logstash-#{time.strftime("%Y.%m.%d")}"
|
553
635
|
stub_elastic
|
636
|
+
stub_elastic_info
|
554
637
|
driver.run(default_tag: 'test') do
|
555
638
|
driver.feed(time.to_i, sample_record)
|
556
639
|
end
|
@@ -563,6 +646,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
563
646
|
time = Time.parse Date.today.iso8601
|
564
647
|
logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
|
565
648
|
stub_elastic
|
649
|
+
stub_elastic_info
|
566
650
|
driver.run(default_tag: 'test') do
|
567
651
|
driver.feed(time.to_i, sample_record)
|
568
652
|
end
|
@@ -577,6 +661,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
577
661
|
time = Time.parse Date.today.iso8601
|
578
662
|
logstash_index = "myprefix#{separator}#{time.getutc.strftime("%Y.%m.%d")}"
|
579
663
|
stub_elastic
|
664
|
+
stub_elastic_info
|
580
665
|
driver.run(default_tag: 'test') do
|
581
666
|
driver.feed(time.to_i, sample_record)
|
582
667
|
end
|
@@ -589,18 +674,20 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
589
674
|
time = Time.parse Date.today.iso8601
|
590
675
|
logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
|
591
676
|
stub_elastic
|
677
|
+
stub_elastic_info
|
592
678
|
driver.run(default_tag: 'test') do
|
593
679
|
driver.feed(time.to_i, sample_record)
|
594
680
|
end
|
595
681
|
assert_equal(logstash_index, index_cmds.first['index']['_index'])
|
596
682
|
end
|
597
683
|
|
598
|
-
|
684
|
+
def test_writes_to_logstash_index_with_specified_dateformat
|
599
685
|
driver.configure("logstash_format true
|
600
686
|
logstash_dateformat %Y.%m")
|
601
687
|
time = Time.parse Date.today.iso8601
|
602
688
|
logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
|
603
689
|
stub_elastic
|
690
|
+
stub_elastic_info
|
604
691
|
driver.run(default_tag: 'test') do
|
605
692
|
driver.feed(time.to_i, sample_record)
|
606
693
|
end
|
@@ -614,6 +701,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
614
701
|
time = Time.parse Date.today.iso8601
|
615
702
|
logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
|
616
703
|
stub_elastic
|
704
|
+
stub_elastic_info
|
617
705
|
driver.run(default_tag: 'test') do
|
618
706
|
driver.feed(time.to_i, sample_record)
|
619
707
|
end
|
@@ -622,6 +710,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
622
710
|
|
623
711
|
def test_doesnt_add_logstash_timestamp_by_default
|
624
712
|
stub_elastic
|
713
|
+
stub_elastic_info
|
625
714
|
driver.run(default_tag: 'test') do
|
626
715
|
driver.feed(sample_record)
|
627
716
|
end
|
@@ -631,6 +720,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
631
720
|
def test_adds_logstash_timestamp_when_configured
|
632
721
|
driver.configure("logstash_format true\n")
|
633
722
|
stub_elastic
|
723
|
+
stub_elastic_info
|
634
724
|
time = Fluent::EventTime.new(Time.now.to_i, 123456789)
|
635
725
|
driver.run(default_tag: 'test') do
|
636
726
|
driver.feed(time, sample_record)
|
@@ -643,6 +733,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
643
733
|
driver.configure("logstash_format true
|
644
734
|
time_precision 3\n")
|
645
735
|
stub_elastic
|
736
|
+
stub_elastic_info
|
646
737
|
time = Fluent::EventTime.new(Time.now.to_i, 123456789)
|
647
738
|
driver.run(default_tag: 'test') do
|
648
739
|
driver.feed(time, sample_record)
|
@@ -654,6 +745,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
654
745
|
def test_uses_custom_timestamp_when_included_in_record
|
655
746
|
driver.configure("include_timestamp true\n")
|
656
747
|
stub_elastic
|
748
|
+
stub_elastic_info
|
657
749
|
ts = DateTime.new(2001,2,3).iso8601
|
658
750
|
driver.run(default_tag: 'test') do
|
659
751
|
driver.feed(sample_record.merge!('@timestamp' => ts))
|
@@ -665,6 +757,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
665
757
|
def test_uses_custom_timestamp_when_included_in_record_logstash
|
666
758
|
driver.configure("logstash_format true\n")
|
667
759
|
stub_elastic
|
760
|
+
stub_elastic_info
|
668
761
|
ts = DateTime.new(2001,2,3).iso8601
|
669
762
|
driver.run(default_tag: 'test') do
|
670
763
|
driver.feed(sample_record.merge!('@timestamp' => ts))
|
@@ -677,6 +770,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
677
770
|
driver.configure("logstash_format true
|
678
771
|
time_key vtm\n")
|
679
772
|
stub_elastic
|
773
|
+
stub_elastic_info
|
680
774
|
ts = DateTime.new(2001,2,3).iso8601
|
681
775
|
driver.run(default_tag: 'test') do
|
682
776
|
driver.feed(sample_record.merge!('vtm' => ts))
|
@@ -689,6 +783,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
689
783
|
driver.configure("include_timestamp true
|
690
784
|
time_key vtm\n")
|
691
785
|
stub_elastic
|
786
|
+
stub_elastic_info
|
692
787
|
ts = DateTime.new(2001,2,3).iso8601
|
693
788
|
driver.run(default_tag: 'test') do
|
694
789
|
driver.feed(sample_record.merge!('vtm' => ts))
|
@@ -702,6 +797,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
702
797
|
index_name test
|
703
798
|
time_key vtm\n")
|
704
799
|
stub_elastic
|
800
|
+
stub_elastic_info
|
705
801
|
ts = DateTime.new(2001,2,3).iso8601
|
706
802
|
driver.run(default_tag: 'test') do
|
707
803
|
driver.feed(sample_record.merge!('vtm' => ts))
|
@@ -716,6 +812,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
716
812
|
time_key vtm
|
717
813
|
time_key_exclude_timestamp true\n")
|
718
814
|
stub_elastic
|
815
|
+
stub_elastic_info
|
719
816
|
ts = DateTime.new(2001,2,3).iso8601
|
720
817
|
driver.run(default_tag: 'test') do
|
721
818
|
driver.feed(sample_record.merge!('vtm' => ts))
|
@@ -728,6 +825,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
728
825
|
time_key vtm
|
729
826
|
time_key_exclude_timestamp true\n")
|
730
827
|
stub_elastic
|
828
|
+
stub_elastic_info
|
731
829
|
ts = DateTime.new(2001,2,3).iso8601
|
732
830
|
driver.run(default_tag: 'test') do
|
733
831
|
driver.feed(sample_record.merge!('vtm' => ts))
|
@@ -737,6 +835,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
737
835
|
|
738
836
|
def test_doesnt_add_tag_key_by_default
|
739
837
|
stub_elastic
|
838
|
+
stub_elastic_info
|
740
839
|
driver.run(default_tag: 'test') do
|
741
840
|
driver.feed(sample_record)
|
742
841
|
end
|
@@ -746,6 +845,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
746
845
|
def test_adds_tag_key_when_configured
|
747
846
|
driver.configure("include_tag_key true\n")
|
748
847
|
stub_elastic
|
848
|
+
stub_elastic_info
|
749
849
|
driver.run(default_tag: 'mytag') do
|
750
850
|
driver.feed(sample_record)
|
751
851
|
end
|
@@ -756,6 +856,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
756
856
|
def test_adds_id_key_when_configured
|
757
857
|
driver.configure("id_key request_id\n")
|
758
858
|
stub_elastic
|
859
|
+
stub_elastic_info
|
759
860
|
driver.run(default_tag: 'test') do
|
760
861
|
driver.feed(sample_record)
|
761
862
|
end
|
@@ -766,6 +867,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
766
867
|
def test_adds_nested_id_key_with_dot
|
767
868
|
driver.configure("id_key nested.request_id\n")
|
768
869
|
stub_elastic
|
870
|
+
stub_elastic_info
|
769
871
|
driver.run(default_tag: 'test') do
|
770
872
|
driver.feed(nested_sample_record)
|
771
873
|
end
|
@@ -775,6 +877,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
775
877
|
def test_adds_nested_id_key_with_dollar_dot
|
776
878
|
driver.configure("id_key $.nested.request_id\n")
|
777
879
|
stub_elastic
|
880
|
+
stub_elastic_info
|
778
881
|
driver.run(default_tag: 'test') do
|
779
882
|
driver.feed(nested_sample_record)
|
780
883
|
end
|
@@ -784,6 +887,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
784
887
|
def test_adds_nested_id_key_with_bracket
|
785
888
|
driver.configure("id_key $['nested']['request_id']\n")
|
786
889
|
stub_elastic
|
890
|
+
stub_elastic_info
|
787
891
|
driver.run(default_tag: 'test') do
|
788
892
|
driver.feed(nested_sample_record)
|
789
893
|
end
|
@@ -794,6 +898,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
794
898
|
def test_doesnt_add_id_key_if_missing_when_configured
|
795
899
|
driver.configure("id_key another_request_id\n")
|
796
900
|
stub_elastic
|
901
|
+
stub_elastic_info
|
797
902
|
driver.run(default_tag: 'test') do
|
798
903
|
driver.feed(sample_record)
|
799
904
|
end
|
@@ -802,6 +907,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
802
907
|
|
803
908
|
def test_adds_id_key_when_not_configured
|
804
909
|
stub_elastic
|
910
|
+
stub_elastic_info
|
805
911
|
driver.run(default_tag: 'test') do
|
806
912
|
driver.feed(sample_record)
|
807
913
|
end
|
@@ -811,6 +917,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
811
917
|
def test_adds_parent_key_when_configured
|
812
918
|
driver.configure("parent_key parent_id\n")
|
813
919
|
stub_elastic
|
920
|
+
stub_elastic_info
|
814
921
|
driver.run(default_tag: 'test') do
|
815
922
|
driver.feed(sample_record)
|
816
923
|
end
|
@@ -821,6 +928,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
821
928
|
def test_adds_nested_parent_key_with_dot
|
822
929
|
driver.configure("parent_key nested.parent_id\n")
|
823
930
|
stub_elastic
|
931
|
+
stub_elastic_info
|
824
932
|
driver.run(default_tag: 'test') do
|
825
933
|
driver.feed(nested_sample_record)
|
826
934
|
end
|
@@ -830,6 +938,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
830
938
|
def test_adds_nested_parent_key_with_dollar_dot
|
831
939
|
driver.configure("parent_key $.nested.parent_id\n")
|
832
940
|
stub_elastic
|
941
|
+
stub_elastic_info
|
833
942
|
driver.run(default_tag: 'test') do
|
834
943
|
driver.feed(nested_sample_record)
|
835
944
|
end
|
@@ -839,6 +948,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
839
948
|
def test_adds_nested_parent_key_with_bracket
|
840
949
|
driver.configure("parent_key $['nested']['parent_id']\n")
|
841
950
|
stub_elastic
|
951
|
+
stub_elastic_info
|
842
952
|
driver.run(default_tag: 'test') do
|
843
953
|
driver.feed(nested_sample_record)
|
844
954
|
end
|
@@ -849,6 +959,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
849
959
|
def test_doesnt_add_parent_key_if_missing_when_configured
|
850
960
|
driver.configure("parent_key another_parent_id\n")
|
851
961
|
stub_elastic
|
962
|
+
stub_elastic_info
|
852
963
|
driver.run(default_tag: 'test') do
|
853
964
|
driver.feed(sample_record)
|
854
965
|
end
|
@@ -857,6 +968,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
857
968
|
|
858
969
|
def test_adds_parent_key_when_not_configured
|
859
970
|
stub_elastic
|
971
|
+
stub_elastic_info
|
860
972
|
driver.run(default_tag: 'test') do
|
861
973
|
driver.feed(sample_record)
|
862
974
|
end
|
@@ -867,6 +979,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
867
979
|
def test_es6
|
868
980
|
driver("routing_key routing_id\n", 6)
|
869
981
|
stub_elastic
|
982
|
+
stub_elastic_info
|
870
983
|
driver.run(default_tag: 'test') do
|
871
984
|
driver.feed(sample_record)
|
872
985
|
end
|
@@ -876,6 +989,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
876
989
|
def test_es7
|
877
990
|
driver("routing_key routing_id\n", 7)
|
878
991
|
stub_elastic
|
992
|
+
stub_elastic_info
|
879
993
|
driver.run(default_tag: 'test') do
|
880
994
|
driver.feed(sample_record)
|
881
995
|
end
|
@@ -887,51 +1001,62 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
887
1001
|
def test_adds_nested_routing_key_with_dot
|
888
1002
|
driver.configure("routing_key nested.routing_id\n")
|
889
1003
|
stub_elastic
|
1004
|
+
stub_elastic_info
|
890
1005
|
driver.run(default_tag: 'test') do
|
891
1006
|
driver.feed(nested_sample_record)
|
892
1007
|
end
|
893
|
-
|
1008
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1009
|
+
assert_equal('routing', index_cmds[0]['index'][routing_key])
|
894
1010
|
end
|
895
1011
|
|
896
1012
|
def test_adds_nested_routing_key_with_dollar_dot
|
897
1013
|
driver.configure("routing_key $.nested.routing_id\n")
|
898
1014
|
stub_elastic
|
1015
|
+
stub_elastic_info
|
899
1016
|
driver.run(default_tag: 'test') do
|
900
1017
|
driver.feed(nested_sample_record)
|
901
1018
|
end
|
902
|
-
|
1019
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1020
|
+
assert_equal('routing', index_cmds[0]['index'][routing_key])
|
903
1021
|
end
|
904
1022
|
|
905
1023
|
def test_adds_nested_routing_key_with_bracket
|
906
1024
|
driver.configure("routing_key $['nested']['routing_id']\n")
|
907
1025
|
stub_elastic
|
1026
|
+
stub_elastic_info
|
908
1027
|
driver.run(default_tag: 'test') do
|
909
1028
|
driver.feed(nested_sample_record)
|
910
1029
|
end
|
911
|
-
|
1030
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1031
|
+
assert_equal('routing', index_cmds[0]['index'][routing_key])
|
912
1032
|
end
|
913
1033
|
end
|
914
1034
|
|
915
1035
|
def test_doesnt_add_routing_key_if_missing_when_configured
|
916
1036
|
driver.configure("routing_key another_routing_id\n")
|
917
1037
|
stub_elastic
|
1038
|
+
stub_elastic_info
|
918
1039
|
driver.run(default_tag: 'test') do
|
919
1040
|
driver.feed(sample_record)
|
920
1041
|
end
|
921
|
-
|
1042
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1043
|
+
assert(!index_cmds[0]['index'].has_key?(routing_key))
|
922
1044
|
end
|
923
1045
|
|
924
1046
|
def test_adds_routing_key_when_not_configured
|
925
1047
|
stub_elastic
|
1048
|
+
stub_elastic_info
|
926
1049
|
driver.run(default_tag: 'test') do
|
927
1050
|
driver.feed(sample_record)
|
928
1051
|
end
|
929
|
-
|
1052
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1053
|
+
assert(!index_cmds[0]['index'].has_key?(routing_key))
|
930
1054
|
end
|
931
1055
|
|
932
1056
|
def test_remove_one_key
|
933
1057
|
driver.configure("remove_keys key1\n")
|
934
1058
|
stub_elastic
|
1059
|
+
stub_elastic_info
|
935
1060
|
driver.run(default_tag: 'test') do
|
936
1061
|
driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
|
937
1062
|
end
|
@@ -942,6 +1067,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
942
1067
|
def test_remove_multi_keys
|
943
1068
|
driver.configure("remove_keys key1, key2\n")
|
944
1069
|
stub_elastic
|
1070
|
+
stub_elastic_info
|
945
1071
|
driver.run(default_tag: 'test') do
|
946
1072
|
driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
|
947
1073
|
end
|
@@ -951,6 +1077,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
951
1077
|
|
952
1078
|
def test_request_error
|
953
1079
|
stub_elastic_unavailable
|
1080
|
+
stub_elastic_info
|
954
1081
|
assert_raise(Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure) {
|
955
1082
|
driver.run(default_tag: 'test', shutdown: false) do
|
956
1083
|
driver.feed(sample_record)
|
@@ -962,6 +1089,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
962
1089
|
omit("retry_forever test is unstable.") if ENV["CI"]
|
963
1090
|
|
964
1091
|
stub_elastic
|
1092
|
+
stub_elastic_info
|
965
1093
|
driver.configure(Fluent::Config::Element.new(
|
966
1094
|
'ROOT', '', {
|
967
1095
|
'@type' => 'elasticsearch',
|
@@ -981,6 +1109,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
981
1109
|
|
982
1110
|
def test_tag_parts_index_error_event
|
983
1111
|
stub_elastic
|
1112
|
+
stub_elastic_info
|
984
1113
|
driver.configure("logstash_prefix ${tag_parts[1]}\n")
|
985
1114
|
flexmock(driver.instance.router).should_receive(:emit_error_event)
|
986
1115
|
.with('test', Fluent::EventTime, Hash, TypeError).once
|
@@ -995,7 +1124,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
995
1124
|
stub_request(:post, "http://localhost:9200/_bulk").with do |req|
|
996
1125
|
connection_resets += 1
|
997
1126
|
raise Faraday::ConnectionFailed, "Test message"
|
998
|
-
end
|
1127
|
+
end.to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
|
1128
|
+
stub_elastic_info
|
999
1129
|
|
1000
1130
|
assert_raise(Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure) {
|
1001
1131
|
driver.run(default_tag: 'test', shutdown: false) do
|
@@ -1011,7 +1141,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
1011
1141
|
stub_request(:post, "http://localhost:9200/_bulk").with do |req|
|
1012
1142
|
connection_resets += 1
|
1013
1143
|
raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
|
1014
|
-
end
|
1144
|
+
end.to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
|
1145
|
+
stub_elastic_info
|
1015
1146
|
|
1016
1147
|
driver.configure("reconnect_on_error true\n")
|
1017
1148
|
|
@@ -1037,7 +1168,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
1037
1168
|
stub_request(:post, "http://localhost:9200/_bulk").with do |req|
|
1038
1169
|
connection_resets += 1
|
1039
1170
|
raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
|
1040
|
-
end
|
1171
|
+
end.to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
|
1172
|
+
stub_elastic_info
|
1041
1173
|
|
1042
1174
|
driver.configure("reconnect_on_error false\n")
|
1043
1175
|
|
@@ -1058,6 +1190,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
1058
1190
|
def test_update_should_not_write_if_theres_no_id
|
1059
1191
|
driver.configure("write_operation update\n")
|
1060
1192
|
stub_elastic
|
1193
|
+
stub_elastic_info
|
1061
1194
|
driver.run(default_tag: 'test') do
|
1062
1195
|
driver.feed(sample_record)
|
1063
1196
|
end
|
@@ -1067,6 +1200,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
1067
1200
|
def test_upsert_should_not_write_if_theres_no_id
|
1068
1201
|
driver.configure("write_operation upsert\n")
|
1069
1202
|
stub_elastic
|
1203
|
+
stub_elastic_info
|
1070
1204
|
driver.run(default_tag: 'test') do
|
1071
1205
|
driver.feed(sample_record)
|
1072
1206
|
end
|
@@ -1076,6 +1210,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
1076
1210
|
def test_create_should_not_write_if_theres_no_id
|
1077
1211
|
driver.configure("write_operation create\n")
|
1078
1212
|
stub_elastic
|
1213
|
+
stub_elastic_info
|
1079
1214
|
driver.run(default_tag: 'test') do
|
1080
1215
|
driver.feed(sample_record)
|
1081
1216
|
end
|
@@ -1086,6 +1221,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
1086
1221
|
driver.configure("write_operation update
|
1087
1222
|
id_key request_id")
|
1088
1223
|
stub_elastic
|
1224
|
+
stub_elastic_info
|
1089
1225
|
driver.run(default_tag: 'test') do
|
1090
1226
|
driver.feed(sample_record)
|
1091
1227
|
end
|
@@ -1097,6 +1233,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
1097
1233
|
driver.configure("write_operation upsert
|
1098
1234
|
id_key request_id")
|
1099
1235
|
stub_elastic
|
1236
|
+
stub_elastic_info
|
1100
1237
|
driver.run(default_tag: 'test') do
|
1101
1238
|
driver.feed(sample_record)
|
1102
1239
|
end
|
@@ -1108,6 +1245,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
1108
1245
|
driver.configure("write_operation create
|
1109
1246
|
id_key request_id")
|
1110
1247
|
stub_elastic
|
1248
|
+
stub_elastic_info
|
1111
1249
|
driver.run(default_tag: 'test') do
|
1112
1250
|
driver.feed(sample_record)
|
1113
1251
|
end
|
@@ -1116,6 +1254,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
1116
1254
|
|
1117
1255
|
def test_include_index_in_url
|
1118
1256
|
stub_elastic('http://localhost:9200/logstash-2018.01.01/_bulk')
|
1257
|
+
stub_elastic_info('http://localhost:9200/')
|
1119
1258
|
|
1120
1259
|
driver.configure("index_name logstash-2018.01.01
|
1121
1260
|
include_index_in_url true")
|