fluent-plugin-elasticsearch 4.1.1 → 5.4.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +37 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.md +24 -0
  4. data/.github/dependabot.yml +6 -0
  5. data/.github/workflows/issue-auto-closer.yml +2 -2
  6. data/.github/workflows/linux.yml +5 -2
  7. data/.github/workflows/macos.yml +5 -2
  8. data/.github/workflows/windows.yml +5 -2
  9. data/Gemfile +1 -2
  10. data/History.md +146 -0
  11. data/README.ElasticsearchGenID.md +4 -4
  12. data/README.ElasticsearchInput.md +1 -1
  13. data/README.Troubleshooting.md +692 -0
  14. data/README.md +260 -550
  15. data/fluent-plugin-elasticsearch.gemspec +4 -1
  16. data/lib/fluent/plugin/elasticsearch_compat.rb +31 -0
  17. data/lib/fluent/plugin/elasticsearch_error_handler.rb +19 -4
  18. data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +2 -2
  19. data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb +18 -4
  20. data/lib/fluent/plugin/elasticsearch_index_template.rb +65 -21
  21. data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb +2 -1
  22. data/lib/fluent/plugin/filter_elasticsearch_genid.rb +1 -1
  23. data/lib/fluent/plugin/in_elasticsearch.rb +8 -2
  24. data/lib/fluent/plugin/oj_serializer.rb +2 -1
  25. data/lib/fluent/plugin/out_elasticsearch.rb +192 -36
  26. data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +298 -0
  27. data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +3 -1
  28. data/test/helper.rb +0 -4
  29. data/test/plugin/mock_chunk.dat +0 -0
  30. data/test/plugin/test_elasticsearch_error_handler.rb +130 -23
  31. data/test/plugin/test_elasticsearch_fallback_selector.rb +17 -8
  32. data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +57 -18
  33. data/test/plugin/test_elasticsearch_tls.rb +8 -2
  34. data/test/plugin/test_filter_elasticsearch_genid.rb +16 -16
  35. data/test/plugin/test_in_elasticsearch.rb +51 -21
  36. data/test/plugin/test_index_alias_template.json +11 -0
  37. data/test/plugin/test_index_template.json +25 -0
  38. data/test/plugin/test_out_elasticsearch.rb +2118 -704
  39. data/test/plugin/test_out_elasticsearch_data_stream.rb +1199 -0
  40. data/test/plugin/test_out_elasticsearch_dynamic.rb +170 -31
  41. metadata +62 -10
  42. data/.coveralls.yml +0 -2
  43. data/.travis.yml +0 -44
  44. data/appveyor.yml +0 -20
  45. data/gemfiles/Gemfile.without.ilm +0 -10
@@ -16,7 +16,15 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
16
16
  @driver = nil
17
17
  end
18
18
 
19
- def driver(conf='', es_version=5)
19
+ def elasticsearch_version
20
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
21
+ TRANSPORT_CLASS::VERSION
22
+ else
23
+ '6.4.2'.freeze
24
+ end
25
+ end
26
+
27
+ def driver(conf='', es_version=elasticsearch_version.to_i)
20
28
  # For request stub to detect compatibility.
21
29
  @es_version ||= es_version
22
30
  Fluent::Plugin::ElasticsearchOutputDynamic.module_eval(<<-CODE)
@@ -34,6 +42,14 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
34
42
  }.configure(conf)
35
43
  end
36
44
 
45
+ def elasticsearch_transport_layer_decoupling?
46
+ Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
47
+ end
48
+
49
+ def elastic_transport_layer?
50
+ Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
51
+ end
52
+
37
53
  def default_type_name
38
54
  Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME
39
55
  end
@@ -51,7 +67,12 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
51
67
  def stub_elastic(url="http://localhost:9200/_bulk")
52
68
  stub_request(:post, url).with do |req|
53
69
  @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
54
- end
70
+ end.to_return({:status => 200, :body => "", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
71
+ end
72
+
73
+ def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
74
+ body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
75
+ stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
55
76
  end
56
77
 
57
78
  def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
@@ -71,7 +92,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
71
92
  stub_request(:post, url).with do |req|
72
93
  index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
73
94
  @index_command_counts[url] += index_cmds.size
74
- end
95
+ end.to_return({:status => 200, :body => "", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
75
96
  end
76
97
 
77
98
  def assert_logs_include(logs, msg, exp_matches=1)
@@ -101,8 +122,13 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
101
122
  assert_nil instance.ssl_max_version
102
123
  assert_nil instance.ssl_min_version
103
124
  if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
104
- assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
105
- instance.ssl_version_options)
125
+ if defined?(OpenSSL::SSL::TLS1_3_VERSION)
126
+ assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
127
+ instance.ssl_version_options)
128
+ else
129
+ assert_equal({max_version: nil, min_version: OpenSSL::SSL::TLS1_2_VERSION},
130
+ instance.ssl_version_options)
131
+ end
106
132
  else
107
133
  assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
108
134
  instance.ssl_version_options)
@@ -118,7 +144,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
118
144
  end
119
145
 
120
146
  test 'configure compression' do
121
- omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
147
+ omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
122
148
 
123
149
  config = %{
124
150
  compression_level best_compression
@@ -129,7 +155,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
129
155
  end
130
156
 
131
157
  test 'check compression strategy' do
132
- omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
158
+ omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
133
159
 
134
160
  config = %{
135
161
  compression_level best_speed
@@ -140,43 +166,69 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
140
166
  end
141
167
 
142
168
  test 'check content-encoding header with compression' do
143
- omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
169
+ omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
144
170
 
145
171
  config = %{
146
172
  compression_level best_compression
147
173
  }
148
174
  instance = driver(config).instance
149
175
 
150
- assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
176
+ if elastic_transport_layer?
177
+ assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
178
+ elsif elasticsearch_transport_layer_decoupling?
179
+ assert_equal nil, instance.client.transport.transport.options[:transport_options][:headers]["Content-Encoding"]
180
+ else
181
+ assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
182
+ end
151
183
 
152
184
  stub_request(:post, "http://localhost:9200/_bulk").
153
- to_return(status: 200, body: "", headers: {})
185
+ to_return(status: 200, body: "", headers: {'x-elastic-product' => 'Elasticsearch'})
186
+ stub_elastic_info
154
187
  driver.run(default_tag: 'test') do
155
188
  driver.feed(sample_record)
156
189
  end
157
190
  compressable = instance.compressable_connection
158
191
 
159
- assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
192
+ if elastic_transport_layer?
193
+ assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
194
+ elsif elasticsearch_transport_layer_decoupling?
195
+ assert_equal "gzip", instance.client(nil, compressable).transport.transport.options[:transport_options][:headers]["Content-Encoding"]
196
+ else
197
+ assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
198
+ end
160
199
  end
161
200
 
162
201
  test 'check compression option is passed to transport' do
163
- omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
202
+ omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
164
203
 
165
204
  config = %{
166
205
  compression_level best_compression
167
206
  }
168
207
  instance = driver(config).instance
169
208
 
170
- assert_equal false, instance.client.transport.options[:compression]
209
+ if elastic_transport_layer?
210
+ assert_equal false, instance.client.transport.options[:compression]
211
+ elsif elasticsearch_transport_layer_decoupling?
212
+ assert_equal false, instance.client.transport.transport.options[:compression]
213
+ else
214
+ assert_equal false, instance.client.transport.options[:compression]
215
+ end
171
216
 
172
217
  stub_request(:post, "http://localhost:9200/_bulk").
173
- to_return(status: 200, body: "", headers: {})
218
+ to_return(status: 200, body: "", headers: {'x-elastic-product' => 'Elasticsearch'})
219
+ stub_elastic_info
174
220
  driver.run(default_tag: 'test') do
175
221
  driver.feed(sample_record)
176
222
  end
177
223
  compressable = instance.compressable_connection
178
224
 
179
- assert_equal true, instance.client(nil, compressable).transport.options[:compression]
225
+ if elastic_transport_layer?
226
+ assert_equal true, instance.client(nil, compressable).transport.options[:compression]
227
+ elsif elasticsearch_transport_layer_decoupling?
228
+ assert_equal true, instance.client(nil, compressable).transport.transport.options[:compression]
229
+ else
230
+ assert_equal true, instance.client(nil, compressable).transport.options[:compression]
231
+ end
180
232
  end
181
233
 
182
234
  test 'configure Content-Type' do
@@ -335,14 +387,18 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
335
387
 
336
388
  def test_content_type_header
337
389
  stub_request(:head, "http://localhost:9200/").
338
- to_return(:status => 200, :body => "", :headers => {})
390
+ to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
339
391
  if Elasticsearch::VERSION >= "6.0.2"
340
392
  elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
341
- with(headers: { "Content-Type" => "application/x-ndjson" })
393
+ with(headers: { "Content-Type" => "application/x-ndjson"}).
394
+ to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
342
395
  else
343
396
  elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
344
- with(headers: { "Content-Type" => "application/json" })
397
+ with(headers: { "Content-Type" => "application/json"}).
398
+ to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
345
399
  end
400
+ stub_elastic_info('http://localhost:9200')
401
+
346
402
  driver.run(default_tag: 'test') do
347
403
  driver.feed(sample_record)
348
404
  end
@@ -351,6 +407,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
351
407
 
352
408
  def test_writes_to_default_index
353
409
  stub_elastic
410
+ stub_elastic_info
354
411
  driver.run(default_tag: 'test') do
355
412
  driver.feed(sample_record)
356
413
  end
@@ -367,7 +424,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
367
424
  end
368
425
 
369
426
  def test_writes_to_default_index_with_compression
370
- omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
427
+ omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
371
428
 
372
429
  config = %[
373
430
  compression_level default_compression
@@ -389,7 +446,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
389
446
  compressed_body = gzip(bodystr, Zlib::DEFAULT_COMPRESSION)
390
447
 
391
448
  elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
392
- to_return(:status => 200, :headers => {'Content-Type' => 'Application/json'}, :body => compressed_body)
449
+ to_return(:status => 200, :headers => {'Content-Type' => 'Application/json', 'x-elastic-product' => 'Elasticsearch'}, :body => compressed_body)
450
+ stub_elastic_info
393
451
 
394
452
  driver(config)
395
453
  driver.run(default_tag: 'test') do
@@ -401,15 +459,23 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
401
459
 
402
460
  def test_writes_to_default_type
403
461
  stub_elastic
462
+ stub_elastic_info
404
463
  driver.run(default_tag: 'test') do
405
464
  driver.feed(sample_record)
406
465
  end
407
- assert_equal(default_type_name, index_cmds.first['index']['_type'])
466
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
467
+ assert_nil(index_cmds.first['index']['_type'])
468
+ elsif Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.0.0")
469
+ assert_equal("_doc", index_cmds.first['index']['_type'])
470
+ else
471
+ assert_equal("fluentd", index_cmds.first['index']['_type'])
472
+ end
408
473
  end
409
474
 
410
475
  def test_writes_to_specified_index
411
476
  driver.configure("index_name myindex\n")
412
477
  stub_elastic
478
+ stub_elastic_info
413
479
  driver.run(default_tag: 'test') do
414
480
  driver.feed(sample_record)
415
481
  end
@@ -419,6 +485,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
419
485
  def test_writes_to_specified_index_uppercase
420
486
  driver.configure("index_name MyIndex\n")
421
487
  stub_elastic
488
+ stub_elastic_info
422
489
  driver.run(default_tag: 'test') do
423
490
  driver.feed(sample_record)
424
491
  end
@@ -428,15 +495,23 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
428
495
  def test_writes_to_specified_type
429
496
  driver.configure("type_name mytype\n")
430
497
  stub_elastic
498
+ stub_elastic_info
431
499
  driver.run(default_tag: 'test') do
432
500
  driver.feed(sample_record)
433
501
  end
434
- assert_equal('mytype', index_cmds.first['index']['_type'])
502
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
503
+ assert_nil(index_cmds.first['index']['_type'])
504
+ elsif Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.0.0")
505
+ assert_equal("_doc", index_cmds.first['index']['_type'])
506
+ else
507
+ assert_equal("mytype", index_cmds.first['index']['_type'])
508
+ end
435
509
  end
436
510
 
437
511
  def test_writes_to_specified_host
438
512
  driver.configure("host 192.168.33.50\n")
439
513
  elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
514
+ stub_elastic_info("http://192.168.33.50:9200/")
440
515
  driver.run(default_tag: 'test') do
441
516
  driver.feed(sample_record)
442
517
  end
@@ -446,6 +521,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
446
521
  def test_writes_to_specified_port
447
522
  driver.configure("port 9201\n")
448
523
  elastic_request = stub_elastic("http://localhost:9201/_bulk")
524
+ stub_elastic_info("http://localhost:9201/")
449
525
  driver.run(default_tag: 'test') do
450
526
  driver.feed(sample_record)
451
527
  end
@@ -461,6 +537,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
461
537
  hosts.each do |host_info|
462
538
  host, port = host_info
463
539
  stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/_bulk")
540
+ stub_elastic_info("http://#{host}:#{port}/")
464
541
  end
465
542
 
466
543
  driver.run(default_tag: 'test') do
@@ -496,6 +573,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
496
573
  ]}
497
574
 
498
575
  stub_elastic
576
+ stub_elastic_info
499
577
  driver.run(default_tag: 'test') do
500
578
  driver.feed(original_hash)
501
579
  end
@@ -509,6 +587,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
509
587
  expected_output = {"foo" => {"bar" => "baz"}}
510
588
 
511
589
  stub_elastic
590
+ stub_elastic_info
512
591
  driver.run(default_tag: 'test') do
513
592
  driver.feed(original_hash)
514
593
  end
@@ -517,6 +596,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
517
596
 
518
597
  def test_makes_bulk_request
519
598
  stub_elastic
599
+ stub_elastic_info
520
600
  driver.run(default_tag: 'test') do
521
601
  driver.feed(sample_record)
522
602
  driver.feed(sample_record.merge('age' => 27))
@@ -526,6 +606,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
526
606
 
527
607
  def test_all_records_are_preserved_in_bulk
528
608
  stub_elastic
609
+ stub_elastic_info
529
610
  driver.run(default_tag: 'test') do
530
611
  driver.feed(sample_record)
531
612
  driver.feed(sample_record.merge('age' => 27))
@@ -539,6 +620,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
539
620
  time = Time.parse Date.today.iso8601
540
621
  logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
541
622
  stub_elastic
623
+ stub_elastic_info
542
624
  driver.run(default_tag: 'test') do
543
625
  driver.feed(time.to_i, sample_record)
544
626
  end
@@ -551,6 +633,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
551
633
  time = Time.parse Date.today.iso8601
552
634
  utc_index = "logstash-#{time.strftime("%Y.%m.%d")}"
553
635
  stub_elastic
636
+ stub_elastic_info
554
637
  driver.run(default_tag: 'test') do
555
638
  driver.feed(time.to_i, sample_record)
556
639
  end
@@ -563,6 +646,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
563
646
  time = Time.parse Date.today.iso8601
564
647
  logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
565
648
  stub_elastic
649
+ stub_elastic_info
566
650
  driver.run(default_tag: 'test') do
567
651
  driver.feed(time.to_i, sample_record)
568
652
  end
@@ -577,6 +661,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
577
661
  time = Time.parse Date.today.iso8601
578
662
  logstash_index = "myprefix#{separator}#{time.getutc.strftime("%Y.%m.%d")}"
579
663
  stub_elastic
664
+ stub_elastic_info
580
665
  driver.run(default_tag: 'test') do
581
666
  driver.feed(time.to_i, sample_record)
582
667
  end
@@ -589,18 +674,20 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
589
674
  time = Time.parse Date.today.iso8601
590
675
  logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
591
676
  stub_elastic
677
+ stub_elastic_info
592
678
  driver.run(default_tag: 'test') do
593
679
  driver.feed(time.to_i, sample_record)
594
680
  end
595
681
  assert_equal(logstash_index, index_cmds.first['index']['_index'])
596
682
  end
597
683
 
598
- def test_writes_to_logstash_index_with_specified_dateformat
684
+ def test_writes_to_logstash_index_with_specified_dateformat
599
685
  driver.configure("logstash_format true
600
686
  logstash_dateformat %Y.%m")
601
687
  time = Time.parse Date.today.iso8601
602
688
  logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
603
689
  stub_elastic
690
+ stub_elastic_info
604
691
  driver.run(default_tag: 'test') do
605
692
  driver.feed(time.to_i, sample_record)
606
693
  end
@@ -614,6 +701,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
614
701
  time = Time.parse Date.today.iso8601
615
702
  logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
616
703
  stub_elastic
704
+ stub_elastic_info
617
705
  driver.run(default_tag: 'test') do
618
706
  driver.feed(time.to_i, sample_record)
619
707
  end
@@ -622,6 +710,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
622
710
 
623
711
  def test_doesnt_add_logstash_timestamp_by_default
624
712
  stub_elastic
713
+ stub_elastic_info
625
714
  driver.run(default_tag: 'test') do
626
715
  driver.feed(sample_record)
627
716
  end
@@ -631,6 +720,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
631
720
  def test_adds_logstash_timestamp_when_configured
632
721
  driver.configure("logstash_format true\n")
633
722
  stub_elastic
723
+ stub_elastic_info
634
724
  time = Fluent::EventTime.new(Time.now.to_i, 123456789)
635
725
  driver.run(default_tag: 'test') do
636
726
  driver.feed(time, sample_record)
@@ -643,6 +733,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
643
733
  driver.configure("logstash_format true
644
734
  time_precision 3\n")
645
735
  stub_elastic
736
+ stub_elastic_info
646
737
  time = Fluent::EventTime.new(Time.now.to_i, 123456789)
647
738
  driver.run(default_tag: 'test') do
648
739
  driver.feed(time, sample_record)
@@ -654,6 +745,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
654
745
  def test_uses_custom_timestamp_when_included_in_record
655
746
  driver.configure("include_timestamp true\n")
656
747
  stub_elastic
748
+ stub_elastic_info
657
749
  ts = DateTime.new(2001,2,3).iso8601
658
750
  driver.run(default_tag: 'test') do
659
751
  driver.feed(sample_record.merge!('@timestamp' => ts))
@@ -665,6 +757,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
665
757
  def test_uses_custom_timestamp_when_included_in_record_logstash
666
758
  driver.configure("logstash_format true\n")
667
759
  stub_elastic
760
+ stub_elastic_info
668
761
  ts = DateTime.new(2001,2,3).iso8601
669
762
  driver.run(default_tag: 'test') do
670
763
  driver.feed(sample_record.merge!('@timestamp' => ts))
@@ -677,6 +770,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
677
770
  driver.configure("logstash_format true
678
771
  time_key vtm\n")
679
772
  stub_elastic
773
+ stub_elastic_info
680
774
  ts = DateTime.new(2001,2,3).iso8601
681
775
  driver.run(default_tag: 'test') do
682
776
  driver.feed(sample_record.merge!('vtm' => ts))
@@ -689,6 +783,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
689
783
  driver.configure("include_timestamp true
690
784
  time_key vtm\n")
691
785
  stub_elastic
786
+ stub_elastic_info
692
787
  ts = DateTime.new(2001,2,3).iso8601
693
788
  driver.run(default_tag: 'test') do
694
789
  driver.feed(sample_record.merge!('vtm' => ts))
@@ -702,6 +797,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
702
797
  index_name test
703
798
  time_key vtm\n")
704
799
  stub_elastic
800
+ stub_elastic_info
705
801
  ts = DateTime.new(2001,2,3).iso8601
706
802
  driver.run(default_tag: 'test') do
707
803
  driver.feed(sample_record.merge!('vtm' => ts))
@@ -716,6 +812,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
716
812
  time_key vtm
717
813
  time_key_exclude_timestamp true\n")
718
814
  stub_elastic
815
+ stub_elastic_info
719
816
  ts = DateTime.new(2001,2,3).iso8601
720
817
  driver.run(default_tag: 'test') do
721
818
  driver.feed(sample_record.merge!('vtm' => ts))
@@ -728,6 +825,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
728
825
  time_key vtm
729
826
  time_key_exclude_timestamp true\n")
730
827
  stub_elastic
828
+ stub_elastic_info
731
829
  ts = DateTime.new(2001,2,3).iso8601
732
830
  driver.run(default_tag: 'test') do
733
831
  driver.feed(sample_record.merge!('vtm' => ts))
@@ -737,6 +835,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
737
835
 
738
836
  def test_doesnt_add_tag_key_by_default
739
837
  stub_elastic
838
+ stub_elastic_info
740
839
  driver.run(default_tag: 'test') do
741
840
  driver.feed(sample_record)
742
841
  end
@@ -746,6 +845,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
746
845
  def test_adds_tag_key_when_configured
747
846
  driver.configure("include_tag_key true\n")
748
847
  stub_elastic
848
+ stub_elastic_info
749
849
  driver.run(default_tag: 'mytag') do
750
850
  driver.feed(sample_record)
751
851
  end
@@ -756,6 +856,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
756
856
  def test_adds_id_key_when_configured
757
857
  driver.configure("id_key request_id\n")
758
858
  stub_elastic
859
+ stub_elastic_info
759
860
  driver.run(default_tag: 'test') do
760
861
  driver.feed(sample_record)
761
862
  end
@@ -766,6 +867,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
766
867
  def test_adds_nested_id_key_with_dot
767
868
  driver.configure("id_key nested.request_id\n")
768
869
  stub_elastic
870
+ stub_elastic_info
769
871
  driver.run(default_tag: 'test') do
770
872
  driver.feed(nested_sample_record)
771
873
  end
@@ -775,6 +877,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
775
877
  def test_adds_nested_id_key_with_dollar_dot
776
878
  driver.configure("id_key $.nested.request_id\n")
777
879
  stub_elastic
880
+ stub_elastic_info
778
881
  driver.run(default_tag: 'test') do
779
882
  driver.feed(nested_sample_record)
780
883
  end
@@ -784,6 +887,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
784
887
  def test_adds_nested_id_key_with_bracket
785
888
  driver.configure("id_key $['nested']['request_id']\n")
786
889
  stub_elastic
890
+ stub_elastic_info
787
891
  driver.run(default_tag: 'test') do
788
892
  driver.feed(nested_sample_record)
789
893
  end
@@ -794,6 +898,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
794
898
  def test_doesnt_add_id_key_if_missing_when_configured
795
899
  driver.configure("id_key another_request_id\n")
796
900
  stub_elastic
901
+ stub_elastic_info
797
902
  driver.run(default_tag: 'test') do
798
903
  driver.feed(sample_record)
799
904
  end
@@ -802,6 +907,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
802
907
 
803
908
  def test_adds_id_key_when_not_configured
804
909
  stub_elastic
910
+ stub_elastic_info
805
911
  driver.run(default_tag: 'test') do
806
912
  driver.feed(sample_record)
807
913
  end
@@ -811,6 +917,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
811
917
  def test_adds_parent_key_when_configured
812
918
  driver.configure("parent_key parent_id\n")
813
919
  stub_elastic
920
+ stub_elastic_info
814
921
  driver.run(default_tag: 'test') do
815
922
  driver.feed(sample_record)
816
923
  end
@@ -821,6 +928,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
821
928
  def test_adds_nested_parent_key_with_dot
822
929
  driver.configure("parent_key nested.parent_id\n")
823
930
  stub_elastic
931
+ stub_elastic_info
824
932
  driver.run(default_tag: 'test') do
825
933
  driver.feed(nested_sample_record)
826
934
  end
@@ -830,6 +938,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
830
938
  def test_adds_nested_parent_key_with_dollar_dot
831
939
  driver.configure("parent_key $.nested.parent_id\n")
832
940
  stub_elastic
941
+ stub_elastic_info
833
942
  driver.run(default_tag: 'test') do
834
943
  driver.feed(nested_sample_record)
835
944
  end
@@ -839,6 +948,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
839
948
  def test_adds_nested_parent_key_with_bracket
840
949
  driver.configure("parent_key $['nested']['parent_id']\n")
841
950
  stub_elastic
951
+ stub_elastic_info
842
952
  driver.run(default_tag: 'test') do
843
953
  driver.feed(nested_sample_record)
844
954
  end
@@ -849,6 +959,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
849
959
  def test_doesnt_add_parent_key_if_missing_when_configured
850
960
  driver.configure("parent_key another_parent_id\n")
851
961
  stub_elastic
962
+ stub_elastic_info
852
963
  driver.run(default_tag: 'test') do
853
964
  driver.feed(sample_record)
854
965
  end
@@ -857,6 +968,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
857
968
 
858
969
  def test_adds_parent_key_when_not_configured
859
970
  stub_elastic
971
+ stub_elastic_info
860
972
  driver.run(default_tag: 'test') do
861
973
  driver.feed(sample_record)
862
974
  end
@@ -867,6 +979,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
867
979
  def test_es6
868
980
  driver("routing_key routing_id\n", 6)
869
981
  stub_elastic
982
+ stub_elastic_info
870
983
  driver.run(default_tag: 'test') do
871
984
  driver.feed(sample_record)
872
985
  end
@@ -876,6 +989,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
876
989
  def test_es7
877
990
  driver("routing_key routing_id\n", 7)
878
991
  stub_elastic
992
+ stub_elastic_info
879
993
  driver.run(default_tag: 'test') do
880
994
  driver.feed(sample_record)
881
995
  end
@@ -887,51 +1001,62 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
887
1001
  def test_adds_nested_routing_key_with_dot
888
1002
  driver.configure("routing_key nested.routing_id\n")
889
1003
  stub_elastic
1004
+ stub_elastic_info
890
1005
  driver.run(default_tag: 'test') do
891
1006
  driver.feed(nested_sample_record)
892
1007
  end
893
- assert_equal('routing', index_cmds[0]['index']['_routing'])
1008
+ routing_key = driver.instance.instance_variable_get(:@routing_key_name)
1009
+ assert_equal('routing', index_cmds[0]['index'][routing_key])
894
1010
  end
895
1011
 
896
1012
  def test_adds_nested_routing_key_with_dollar_dot
897
1013
  driver.configure("routing_key $.nested.routing_id\n")
898
1014
  stub_elastic
1015
+ stub_elastic_info
899
1016
  driver.run(default_tag: 'test') do
900
1017
  driver.feed(nested_sample_record)
901
1018
  end
902
- assert_equal('routing', index_cmds[0]['index']['_routing'])
1019
+ routing_key = driver.instance.instance_variable_get(:@routing_key_name)
1020
+ assert_equal('routing', index_cmds[0]['index'][routing_key])
903
1021
  end
904
1022
 
905
1023
  def test_adds_nested_routing_key_with_bracket
906
1024
  driver.configure("routing_key $['nested']['routing_id']\n")
907
1025
  stub_elastic
1026
+ stub_elastic_info
908
1027
  driver.run(default_tag: 'test') do
909
1028
  driver.feed(nested_sample_record)
910
1029
  end
911
- assert_equal('routing', index_cmds[0]['index']['_routing'])
1030
+ routing_key = driver.instance.instance_variable_get(:@routing_key_name)
1031
+ assert_equal('routing', index_cmds[0]['index'][routing_key])
912
1032
  end
913
1033
  end
914
1034
 
915
1035
  def test_doesnt_add_routing_key_if_missing_when_configured
916
1036
  driver.configure("routing_key another_routing_id\n")
917
1037
  stub_elastic
1038
+ stub_elastic_info
918
1039
  driver.run(default_tag: 'test') do
919
1040
  driver.feed(sample_record)
920
1041
  end
921
- assert(!index_cmds[0]['index'].has_key?('_routing'))
1042
+ routing_key = driver.instance.instance_variable_get(:@routing_key_name)
1043
+ assert(!index_cmds[0]['index'].has_key?(routing_key))
922
1044
  end
923
1045
 
924
1046
  def test_adds_routing_key_when_not_configured
925
1047
  stub_elastic
1048
+ stub_elastic_info
926
1049
  driver.run(default_tag: 'test') do
927
1050
  driver.feed(sample_record)
928
1051
  end
929
- assert(!index_cmds[0]['index'].has_key?('_routing'))
1052
+ routing_key = driver.instance.instance_variable_get(:@routing_key_name)
1053
+ assert(!index_cmds[0]['index'].has_key?(routing_key))
930
1054
  end
931
1055
 
932
1056
  def test_remove_one_key
933
1057
  driver.configure("remove_keys key1\n")
934
1058
  stub_elastic
1059
+ stub_elastic_info
935
1060
  driver.run(default_tag: 'test') do
936
1061
  driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
937
1062
  end
@@ -942,6 +1067,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
942
1067
  def test_remove_multi_keys
943
1068
  driver.configure("remove_keys key1, key2\n")
944
1069
  stub_elastic
1070
+ stub_elastic_info
945
1071
  driver.run(default_tag: 'test') do
946
1072
  driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
947
1073
  end
@@ -951,6 +1077,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
951
1077
 
952
1078
  def test_request_error
953
1079
  stub_elastic_unavailable
1080
+ stub_elastic_info
954
1081
  assert_raise(Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure) {
955
1082
  driver.run(default_tag: 'test', shutdown: false) do
956
1083
  driver.feed(sample_record)
@@ -962,6 +1089,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
962
1089
  omit("retry_forever test is unstable.") if ENV["CI"]
963
1090
 
964
1091
  stub_elastic
1092
+ stub_elastic_info
965
1093
  driver.configure(Fluent::Config::Element.new(
966
1094
  'ROOT', '', {
967
1095
  '@type' => 'elasticsearch',
@@ -981,6 +1109,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
981
1109
 
982
1110
  def test_tag_parts_index_error_event
983
1111
  stub_elastic
1112
+ stub_elastic_info
984
1113
  driver.configure("logstash_prefix ${tag_parts[1]}\n")
985
1114
  flexmock(driver.instance.router).should_receive(:emit_error_event)
986
1115
  .with('test', Fluent::EventTime, Hash, TypeError).once
@@ -995,7 +1124,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
995
1124
  stub_request(:post, "http://localhost:9200/_bulk").with do |req|
996
1125
  connection_resets += 1
997
1126
  raise Faraday::ConnectionFailed, "Test message"
998
- end
1127
+ end.to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
1128
+ stub_elastic_info
999
1129
 
1000
1130
  assert_raise(Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure) {
1001
1131
  driver.run(default_tag: 'test', shutdown: false) do
@@ -1011,7 +1141,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
1011
1141
  stub_request(:post, "http://localhost:9200/_bulk").with do |req|
1012
1142
  connection_resets += 1
1013
1143
  raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
1014
- end
1144
+ end.to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
1145
+ stub_elastic_info
1015
1146
 
1016
1147
  driver.configure("reconnect_on_error true\n")
1017
1148
 
@@ -1037,7 +1168,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
1037
1168
  stub_request(:post, "http://localhost:9200/_bulk").with do |req|
1038
1169
  connection_resets += 1
1039
1170
  raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
1040
- end
1171
+ end.to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
1172
+ stub_elastic_info
1041
1173
 
1042
1174
  driver.configure("reconnect_on_error false\n")
1043
1175
 
@@ -1058,6 +1190,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
1058
1190
  def test_update_should_not_write_if_theres_no_id
1059
1191
  driver.configure("write_operation update\n")
1060
1192
  stub_elastic
1193
+ stub_elastic_info
1061
1194
  driver.run(default_tag: 'test') do
1062
1195
  driver.feed(sample_record)
1063
1196
  end
@@ -1067,6 +1200,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
1067
1200
  def test_upsert_should_not_write_if_theres_no_id
1068
1201
  driver.configure("write_operation upsert\n")
1069
1202
  stub_elastic
1203
+ stub_elastic_info
1070
1204
  driver.run(default_tag: 'test') do
1071
1205
  driver.feed(sample_record)
1072
1206
  end
@@ -1076,6 +1210,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
1076
1210
  def test_create_should_not_write_if_theres_no_id
1077
1211
  driver.configure("write_operation create\n")
1078
1212
  stub_elastic
1213
+ stub_elastic_info
1079
1214
  driver.run(default_tag: 'test') do
1080
1215
  driver.feed(sample_record)
1081
1216
  end
@@ -1086,6 +1221,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
1086
1221
  driver.configure("write_operation update
1087
1222
  id_key request_id")
1088
1223
  stub_elastic
1224
+ stub_elastic_info
1089
1225
  driver.run(default_tag: 'test') do
1090
1226
  driver.feed(sample_record)
1091
1227
  end
@@ -1097,6 +1233,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
1097
1233
  driver.configure("write_operation upsert
1098
1234
  id_key request_id")
1099
1235
  stub_elastic
1236
+ stub_elastic_info
1100
1237
  driver.run(default_tag: 'test') do
1101
1238
  driver.feed(sample_record)
1102
1239
  end
@@ -1108,6 +1245,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
1108
1245
  driver.configure("write_operation create
1109
1246
  id_key request_id")
1110
1247
  stub_elastic
1248
+ stub_elastic_info
1111
1249
  driver.run(default_tag: 'test') do
1112
1250
  driver.feed(sample_record)
1113
1251
  end
@@ -1116,6 +1254,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
1116
1254
 
1117
1255
  def test_include_index_in_url
1118
1256
  stub_elastic('http://localhost:9200/logstash-2018.01.01/_bulk')
1257
+ stub_elastic_info('http://localhost:9200/')
1119
1258
 
1120
1259
  driver.configure("index_name logstash-2018.01.01
1121
1260
  include_index_in_url true")