fluent-plugin-elasticsearch 5.1.2 → 5.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -22,7 +22,23 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
22
22
  @bulk_records = 0
23
23
  end
24
24
 
25
- def driver(conf='', es_version=5, client_version="\"5.0\"")
25
+ def elasticsearch_version
26
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
27
+ TRANSPORT_CLASS::VERSION
28
+ else
29
+ '5.0.0'.freeze
30
+ end
31
+ end
32
+
33
+ def ilm_endpoint
34
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
35
+ '_enrich'.freeze
36
+ else
37
+ '_ilm'.freeze
38
+ end
39
+ end
40
+
41
+ def driver(conf='', es_version=elasticsearch_version.to_i, client_version=elasticsearch_version)
26
42
  # For request stub to detect compatibility.
27
43
  @es_version ||= es_version
28
44
  @client_version ||= client_version
@@ -62,56 +78,62 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
62
78
  DUPLICATED_DATA_STREAM_EXCEPTION = {"error": {}, "status": 400}
63
79
  NONEXISTENT_DATA_STREAM_EXCEPTION = {"error": {}, "status": 404}
64
80
 
65
- def stub_ilm_policy(name="foo_ilm")
66
- stub_request(:put, "http://localhost:9200/_ilm/policy/#{name}_policy").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
81
+ def stub_ilm_policy(name="foo_ilm_policy", url="http://localhost:9200")
82
+ stub_request(:put, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
83
+ end
84
+
85
+ def stub_index_template(name="foo_tpl", url="http://localhost:9200")
86
+ stub_request(:put, "#{url}/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
67
87
  end
68
88
 
69
- def stub_index_template(name="foo_tpl")
70
- stub_request(:put, "http://localhost:9200/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
89
+ def stub_data_stream(name="foo", url="http://localhost:9200")
90
+ stub_request(:put, "#{url}/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
71
91
  end
72
92
 
73
- def stub_data_stream(name="foo")
74
- stub_request(:put, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
93
+ def stub_existent_data_stream?(name="foo", url="http://localhost:9200")
94
+ stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
75
95
  end
76
96
 
77
- def stub_existent_data_stream?(name="foo")
78
- stub_request(:get, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
97
+ def stub_existent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
98
+
99
+ stub_request(:get, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
79
100
  end
80
101
 
81
- def stub_existent_ilm?(name="foo_ilm")
82
- stub_request(:get, "http://localhost:9200/_ilm/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
102
+ def stub_existent_template?(name="foo_tpl", url="http://localhost:9200")
103
+ stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
83
104
  end
84
105
 
85
- def stub_existent_template?(name="foo_tpl")
86
- stub_request(:get, "http://localhost:9200/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
106
+ def stub_nonexistent_data_stream?(name="foo", url="http://localhost:9200")
107
+ stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
87
108
  end
88
109
 
89
- def stub_nonexistent_data_stream?(name="foo")
90
- stub_request(:get, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
110
+ def stub_nonexistent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
111
+ stub_request(:get, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
91
112
  end
92
113
 
93
- def stub_nonexistent_ilm?(name="foo_ilm")
94
- stub_request(:get, "http://localhost:9200/_ilm/policy/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
114
+ def stub_nonexistent_template?(name="foo_tpl", url="http://localhost:9200")
115
+ stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
95
116
  end
96
117
 
97
- def stub_nonexistent_template?(name="foo_tpl")
98
- stub_request(:get, "http://localhost:9200/_index_template/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
118
+ def stub_nonexistent_template_retry?(name="foo_tpl", url="http://localhost:9200")
119
+ stub_request(:get, "#{url}/_index_template/#{name}").
120
+ to_return({ status: 500, body: 'Internal Server Error' }, { status: 404, body: '{}' })
99
121
  end
100
122
 
101
- def stub_bulk_feed(datastream_name="foo", ilm_name="foo_ilm", template_name="foo_tpl")
102
- stub_request(:post, "http://localhost:9200/#{datastream_name}/_bulk").with do |req|
123
+ def stub_bulk_feed(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl", url="http://localhost:9200")
124
+ stub_request(:post, "#{url}/#{datastream_name}/_bulk").with do |req|
103
125
  # bulk data must be pair of OP and records
104
126
  # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
105
127
  # {"@timestamp": ...}
106
128
  @bulk_records += req.body.split("\n").size / 2
107
129
  end
108
- stub_request(:post, "http://localhost:9200/#{ilm_name}/_bulk").with do |req|
130
+ stub_request(:post, "#{url}/#{ilm_name}/_bulk").with do |req|
109
131
  # bulk data must be pair of OP and records
110
132
  # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
111
133
  # {"@timestamp": ...}
112
134
  @bulk_records += req.body.split("\n").size / 2
113
135
  end
114
- stub_request(:post, "http://localhost:9200/#{template_name}/_bulk").with do |req|
136
+ stub_request(:post, "#{url}/#{template_name}/_bulk").with do |req|
115
137
  # bulk data must be pair of OP and records
116
138
  # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
117
139
  # {"@timestamp": ...}
@@ -119,12 +141,12 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
119
141
  end
120
142
  end
121
143
 
122
- def stub_elastic_info(url="http://localhost:9200/", version="7.9.0")
144
+ def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version, headers={})
123
145
  body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
124
- stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
146
+ stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' }.merge(headers) })
125
147
  end
126
148
 
127
- def stub_default(datastream_name="foo", ilm_name="foo_ilm", template_name="foo_tpl", host="http://localhost:9200")
149
+ def stub_default(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl", host="http://localhost:9200")
128
150
  stub_elastic_info(host)
129
151
  stub_nonexistent_ilm?(ilm_name)
130
152
  stub_ilm_policy(ilm_name)
@@ -135,7 +157,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
135
157
  end
136
158
 
137
159
  def data_stream_supported?
138
- Gem::Version.create(::Elasticsearch::Transport::VERSION) >= Gem::Version.create("7.9.0")
160
+ Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.create("7.9.0")
139
161
  end
140
162
 
141
163
  # ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
@@ -431,12 +453,62 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
431
453
  'ROOT', '', {
432
454
  '@type' => ELASTIC_DATA_STREAM_TYPE,
433
455
  'data_stream_name' => 'foo',
434
- 'data_stream_ilm_name' => "foo_ilm",
456
+ 'data_stream_ilm_name' => "foo_ilm_policy",
435
457
  'data_stream_template_name' => "foo_tpl"
436
458
  })
437
459
  assert_equal "foo", driver(conf).instance.data_stream_name
438
460
  end
439
461
 
462
+ def test_datastream_configure_retry
463
+ stub_elastic_info
464
+ stub_nonexistent_ilm?
465
+ stub_ilm_policy
466
+ stub_nonexistent_template_retry?
467
+ stub_index_template
468
+ stub_nonexistent_data_stream?
469
+ stub_data_stream
470
+ conf = config_element(
471
+ 'ROOT', '', {
472
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
473
+ 'data_stream_name' => 'foo',
474
+ 'data_stream_ilm_name' => "foo_ilm_policy",
475
+ 'data_stream_template_name' => "foo_tpl"
476
+ })
477
+ assert_equal "foo", driver(conf).instance.data_stream_name
478
+ end
479
+
480
+ def test_hosts_list_configure
481
+ config = %{
482
+ hosts https://john:password@host1:443/elastic/,http://host2
483
+ path /default_path
484
+ user default_user
485
+ password default_password
486
+ data_stream_name default
487
+ }
488
+ stub_elastic_info("https://host1:443/elastic//", elasticsearch_version,
489
+ {'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
490
+ stub_elastic_info("http://host2/default_path/_data_stream/default", elasticsearch_version,
491
+ {'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
492
+ stub_existent_data_stream?("default", "https://host1/elastic/")
493
+ instance = driver(config).instance
494
+
495
+ assert_equal 2, instance.get_connection_options[:hosts].length
496
+ host1, host2 = instance.get_connection_options[:hosts]
497
+
498
+ assert_equal 'host1', host1[:host]
499
+ assert_equal 443, host1[:port]
500
+ assert_equal 'https', host1[:scheme]
501
+ assert_equal 'john', host1[:user]
502
+ assert_equal 'password', host1[:password]
503
+ assert_equal '/elastic/', host1[:path]
504
+
505
+ assert_equal 'host2', host2[:host]
506
+ assert_equal 'http', host2[:scheme]
507
+ assert_equal 'default_user', host2[:user]
508
+ assert_equal 'default_password', host2[:password]
509
+ assert_equal '/default_path', host2[:path]
510
+ end
511
+
440
512
  def test_existent_data_stream
441
513
  omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
442
514
 
@@ -449,7 +521,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
449
521
  'ROOT', '', {
450
522
  '@type' => ELASTIC_DATA_STREAM_TYPE,
451
523
  'data_stream_name' => 'foo',
452
- 'data_stream_ilm_name' => "foo_ilm",
524
+ 'data_stream_ilm_name' => "foo_ilm_policy",
453
525
  'data_stream_template_name' => "foo_tpl"
454
526
  })
455
527
  assert_equal "foo", driver(conf).instance.data_stream_name
@@ -467,10 +539,10 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
467
539
  'ROOT', '', {
468
540
  '@type' => ELASTIC_DATA_STREAM_TYPE,
469
541
  'data_stream_name' => 'foo',
470
- 'data_stream_ilm_name' => "foo_ilm",
542
+ 'data_stream_ilm_name' => "foo_ilm_policy",
471
543
  })
472
544
  assert_equal "foo", driver(conf).instance.data_stream_name
473
- assert_equal "foo_ilm", driver(conf).instance.data_stream_ilm_name
545
+ assert_equal "foo_ilm_policy", driver(conf).instance.data_stream_ilm_name
474
546
  assert_equal "foo_template", driver(conf).instance.data_stream_template_name
475
547
  end
476
548
 
@@ -615,7 +687,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
615
687
  'ROOT', '', {
616
688
  '@type' => ELASTIC_DATA_STREAM_TYPE,
617
689
  'data_stream_name' => 'foo',
618
- 'data_stream_ilm_name' => 'foo_ilm',
690
+ 'data_stream_ilm_name' => 'foo_ilm_policy',
619
691
  'data_stream_template_name' => 'foo_tpl'
620
692
  })
621
693
  driver(conf).run(default_tag: 'test') do
@@ -635,7 +707,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
635
707
  port 778
636
708
  scheme https
637
709
  data_stream_name foo
638
- data_stream_ilm_name foo_ilm
710
+ data_stream_ilm_name foo_ilm_policy
639
711
  data_stream_template_name foo_tpl
640
712
  user john
641
713
  password doe
@@ -659,4 +731,86 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
659
731
 
660
732
  assert_equal(4, connection_resets)
661
733
  end
734
+
735
+ def test_doesnt_update_ilm_policy_if_overwrite_unset
736
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
737
+
738
+ config = %{
739
+ data_stream_name foo
740
+ data_stream_ilm_name foo_ilm_policy
741
+ data_stream_ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
742
+ }
743
+
744
+ stub_elastic_info
745
+ stub_index_template
746
+ stub_existent_data_stream?
747
+ stub_existent_ilm?
748
+ stub_data_stream
749
+
750
+ stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
751
+ to_return(:status => 200, :body => "", :headers => {})
752
+
753
+ assert_nothing_raised {
754
+ driver(config)
755
+ }
756
+ assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 0)
757
+ end
758
+
759
+ def test_updates_ilm_policy_if_overwrite_set
760
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
761
+
762
+ config = %{
763
+ data_stream_name foo
764
+ data_stream_ilm_name foo_ilm_policy
765
+ data_stream_ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
766
+ data_stream_ilm_policy_overwrite true
767
+ }
768
+
769
+ stub_elastic_info
770
+ stub_index_template
771
+ stub_existent_data_stream?
772
+ stub_existent_ilm?
773
+ stub_data_stream
774
+
775
+ stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
776
+ to_return(:status => 200, :body => "", :headers => {})
777
+
778
+ assert_nothing_raised {
779
+ driver(config)
780
+ }
781
+
782
+ assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 1)
783
+ assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy",
784
+ body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
785
+ times: 1)
786
+ end
787
+
788
+ def test_creates_custom_ilm_policy_if_none_exists
789
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
790
+
791
+ config = %{
792
+ data_stream_name foo
793
+ data_stream_ilm_name foo_ilm_policy
794
+ data_stream_ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
795
+ }
796
+
797
+ stub_elastic_info
798
+ stub_index_template("foo_template")
799
+ stub_data_stream
800
+ stub_nonexistent_data_stream?
801
+ stub_nonexistent_ilm?
802
+ stub_nonexistent_template?("foo_template")
803
+
804
+ stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
805
+ to_return(:status => 200, :body => "", :headers => {})
806
+
807
+ assert_nothing_raised {
808
+ driver(config)
809
+ }
810
+
811
+ assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 1)
812
+ assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy",
813
+ body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
814
+ times: 1)
815
+ end
662
816
  end
@@ -16,7 +16,15 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
16
16
  @driver = nil
17
17
  end
18
18
 
19
- def driver(conf='', es_version=5)
19
+ def elasticsearch_version
20
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
21
+ TRANSPORT_CLASS::VERSION
22
+ else
23
+ '6.4.2'.freeze
24
+ end
25
+ end
26
+
27
+ def driver(conf='', es_version=elasticsearch_version.to_i)
20
28
  # For request stub to detect compatibility.
21
29
  @es_version ||= es_version
22
30
  Fluent::Plugin::ElasticsearchOutputDynamic.module_eval(<<-CODE)
@@ -35,7 +43,11 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
35
43
  end
36
44
 
37
45
  def elasticsearch_transport_layer_decoupling?
38
- Gem::Version.create(::Elasticsearch::Transport::VERSION) >= Gem::Version.new("7.14.0")
46
+ Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
47
+ end
48
+
49
+ def elastic_transport_layer?
50
+ Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
39
51
  end
40
52
 
41
53
  def default_type_name
@@ -58,9 +70,9 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
58
70
  end
59
71
  end
60
72
 
61
- def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
73
+ def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
62
74
  body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
63
- stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
75
+ stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
64
76
  end
65
77
 
66
78
  def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
@@ -132,7 +144,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
132
144
  end
133
145
 
134
146
  test 'configure compression' do
135
- omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
147
+ omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
136
148
 
137
149
  config = %{
138
150
  compression_level best_compression
@@ -143,7 +155,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
143
155
  end
144
156
 
145
157
  test 'check compression strategy' do
146
- omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
158
+ omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
147
159
 
148
160
  config = %{
149
161
  compression_level best_speed
@@ -154,14 +166,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
154
166
  end
155
167
 
156
168
  test 'check content-encoding header with compression' do
157
- omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
169
+ omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
158
170
 
159
171
  config = %{
160
172
  compression_level best_compression
161
173
  }
162
174
  instance = driver(config).instance
163
175
 
164
- if elasticsearch_transport_layer_decoupling?
176
+ if elastic_transport_layer?
177
+ assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
178
+ elsif elasticsearch_transport_layer_decoupling?
165
179
  assert_equal nil, instance.client.transport.transport.options[:transport_options][:headers]["Content-Encoding"]
166
180
  else
167
181
  assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
@@ -175,7 +189,9 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
175
189
  end
176
190
  compressable = instance.compressable_connection
177
191
 
178
- if elasticsearch_transport_layer_decoupling?
192
+ if elastic_transport_layer?
193
+ assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
194
+ elsif elasticsearch_transport_layer_decoupling?
179
195
  assert_equal "gzip", instance.client(nil, compressable).transport.transport.options[:transport_options][:headers]["Content-Encoding"]
180
196
  else
181
197
  assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
@@ -183,14 +199,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
183
199
  end
184
200
 
185
201
  test 'check compression option is passed to transport' do
186
- omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
202
+ omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
187
203
 
188
204
  config = %{
189
205
  compression_level best_compression
190
206
  }
191
207
  instance = driver(config).instance
192
208
 
193
- if elasticsearch_transport_layer_decoupling?
209
+ if elastic_transport_layer?
210
+ assert_equal false, instance.client.transport.options[:compression]
211
+ elsif elasticsearch_transport_layer_decoupling?
194
212
  assert_equal false, instance.client.transport.transport.options[:compression]
195
213
  else
196
214
  assert_equal false, instance.client.transport.options[:compression]
@@ -204,7 +222,9 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
204
222
  end
205
223
  compressable = instance.compressable_connection
206
224
 
207
- if elasticsearch_transport_layer_decoupling?
225
+ if elastic_transport_layer?
226
+ assert_equal true, instance.client(nil, compressable).transport.options[:compression]
227
+ elsif elasticsearch_transport_layer_decoupling?
208
228
  assert_equal true, instance.client(nil, compressable).transport.transport.options[:compression]
209
229
  else
210
230
  assert_equal true, instance.client(nil, compressable).transport.options[:compression]
@@ -402,7 +422,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
402
422
  end
403
423
 
404
424
  def test_writes_to_default_index_with_compression
405
- omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
425
+ omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
406
426
 
407
427
  config = %[
408
428
  compression_level default_compression
@@ -441,7 +461,13 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
441
461
  driver.run(default_tag: 'test') do
442
462
  driver.feed(sample_record)
443
463
  end
444
- assert_equal(default_type_name, index_cmds.first['index']['_type'])
464
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
465
+ assert_nil(index_cmds.first['index']['_type'])
466
+ elsif Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.0.0")
467
+ assert_equal("_doc", index_cmds.first['index']['_type'])
468
+ else
469
+ assert_equal("fluentd", index_cmds.first['index']['_type'])
470
+ end
445
471
  end
446
472
 
447
473
  def test_writes_to_specified_index
@@ -471,7 +497,13 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
471
497
  driver.run(default_tag: 'test') do
472
498
  driver.feed(sample_record)
473
499
  end
474
- assert_equal('mytype', index_cmds.first['index']['_type'])
500
+ if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
501
+ assert_nil(index_cmds.first['index']['_type'])
502
+ elsif Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.0.0")
503
+ assert_equal("_doc", index_cmds.first['index']['_type'])
504
+ else
505
+ assert_equal("mytype", index_cmds.first['index']['_type'])
506
+ end
475
507
  end
476
508
 
477
509
  def test_writes_to_specified_host
@@ -971,7 +1003,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
971
1003
  driver.run(default_tag: 'test') do
972
1004
  driver.feed(nested_sample_record)
973
1005
  end
974
- assert_equal('routing', index_cmds[0]['index']['_routing'])
1006
+ routing_key = driver.instance.instance_variable_get(:@routing_key_name)
1007
+ assert_equal('routing', index_cmds[0]['index'][routing_key])
975
1008
  end
976
1009
 
977
1010
  def test_adds_nested_routing_key_with_dollar_dot
@@ -981,7 +1014,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
981
1014
  driver.run(default_tag: 'test') do
982
1015
  driver.feed(nested_sample_record)
983
1016
  end
984
- assert_equal('routing', index_cmds[0]['index']['_routing'])
1017
+ routing_key = driver.instance.instance_variable_get(:@routing_key_name)
1018
+ assert_equal('routing', index_cmds[0]['index'][routing_key])
985
1019
  end
986
1020
 
987
1021
  def test_adds_nested_routing_key_with_bracket
@@ -991,7 +1025,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
991
1025
  driver.run(default_tag: 'test') do
992
1026
  driver.feed(nested_sample_record)
993
1027
  end
994
- assert_equal('routing', index_cmds[0]['index']['_routing'])
1028
+ routing_key = driver.instance.instance_variable_get(:@routing_key_name)
1029
+ assert_equal('routing', index_cmds[0]['index'][routing_key])
995
1030
  end
996
1031
  end
997
1032
 
@@ -1002,7 +1037,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
1002
1037
  driver.run(default_tag: 'test') do
1003
1038
  driver.feed(sample_record)
1004
1039
  end
1005
- assert(!index_cmds[0]['index'].has_key?('_routing'))
1040
+ routing_key = driver.instance.instance_variable_get(:@routing_key_name)
1041
+ assert(!index_cmds[0]['index'].has_key?(routing_key))
1006
1042
  end
1007
1043
 
1008
1044
  def test_adds_routing_key_when_not_configured
@@ -1011,7 +1047,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
1011
1047
  driver.run(default_tag: 'test') do
1012
1048
  driver.feed(sample_record)
1013
1049
  end
1014
- assert(!index_cmds[0]['index'].has_key?('_routing'))
1050
+ routing_key = driver.instance.instance_variable_get(:@routing_key_name)
1051
+ assert(!index_cmds[0]['index'].has_key?(routing_key))
1015
1052
  end
1016
1053
 
1017
1054
  def test_remove_one_key
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.1.2
4
+ version: 5.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - diogo
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2021-11-04 00:00:00.000000000 Z
13
+ date: 2022-02-21 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: fluentd
@@ -171,6 +171,7 @@ files:
171
171
  - gemfiles/Gemfile.elasticsearch.v6
172
172
  - lib/fluent/log-ext.rb
173
173
  - lib/fluent/plugin/default-ilm-policy.json
174
+ - lib/fluent/plugin/elasticsearch_compat.rb
174
175
  - lib/fluent/plugin/elasticsearch_constants.rb
175
176
  - lib/fluent/plugin/elasticsearch_error.rb
176
177
  - lib/fluent/plugin/elasticsearch_error_handler.rb
@@ -221,7 +222,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
221
222
  - !ruby/object:Gem::Version
222
223
  version: '0'
223
224
  requirements: []
224
- rubygems_version: 3.2.22
225
+ rubygems_version: 3.2.30
225
226
  signing_key:
226
227
  specification_version: 4
227
228
  summary: Elasticsearch output plugin for Fluent event collector