fluent-plugin-elasticsearch 5.1.2 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/History.md +15 -0
- data/README.md +16 -1
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_compat.rb +27 -0
- data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +2 -2
- data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb +18 -4
- data/lib/fluent/plugin/elasticsearch_index_template.rb +8 -4
- data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb +2 -1
- data/lib/fluent/plugin/in_elasticsearch.rb +2 -1
- data/lib/fluent/plugin/oj_serializer.rb +2 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +6 -9
- data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +56 -30
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +1 -1
- data/test/plugin/test_elasticsearch_fallback_selector.rb +15 -7
- data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +47 -17
- data/test/plugin/test_in_elasticsearch.rb +10 -2
- data/test/plugin/test_out_elasticsearch.rb +220 -122
- data/test/plugin/test_out_elasticsearch_data_stream.rb +187 -33
- data/test/plugin/test_out_elasticsearch_dynamic.rb +57 -20
- metadata +4 -3
@@ -22,7 +22,23 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
22
22
|
@bulk_records = 0
|
23
23
|
end
|
24
24
|
|
25
|
-
def
|
25
|
+
def elasticsearch_version
|
26
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
27
|
+
TRANSPORT_CLASS::VERSION
|
28
|
+
else
|
29
|
+
'5.0.0'.freeze
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def ilm_endpoint
|
34
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
35
|
+
'_enrich'.freeze
|
36
|
+
else
|
37
|
+
'_ilm'.freeze
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def driver(conf='', es_version=elasticsearch_version.to_i, client_version=elasticsearch_version)
|
26
42
|
# For request stub to detect compatibility.
|
27
43
|
@es_version ||= es_version
|
28
44
|
@client_version ||= client_version
|
@@ -62,56 +78,62 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
62
78
|
DUPLICATED_DATA_STREAM_EXCEPTION = {"error": {}, "status": 400}
|
63
79
|
NONEXISTENT_DATA_STREAM_EXCEPTION = {"error": {}, "status": 404}
|
64
80
|
|
65
|
-
def stub_ilm_policy(name="
|
66
|
-
stub_request(:put, "
|
81
|
+
def stub_ilm_policy(name="foo_ilm_policy", url="http://localhost:9200")
|
82
|
+
stub_request(:put, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
|
83
|
+
end
|
84
|
+
|
85
|
+
def stub_index_template(name="foo_tpl", url="http://localhost:9200")
|
86
|
+
stub_request(:put, "#{url}/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
|
67
87
|
end
|
68
88
|
|
69
|
-
def
|
70
|
-
stub_request(:put, "
|
89
|
+
def stub_data_stream(name="foo", url="http://localhost:9200")
|
90
|
+
stub_request(:put, "#{url}/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
|
71
91
|
end
|
72
92
|
|
73
|
-
def
|
74
|
-
stub_request(:
|
93
|
+
def stub_existent_data_stream?(name="foo", url="http://localhost:9200")
|
94
|
+
stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
|
75
95
|
end
|
76
96
|
|
77
|
-
def
|
78
|
-
|
97
|
+
def stub_existent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
|
98
|
+
|
99
|
+
stub_request(:get, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
|
79
100
|
end
|
80
101
|
|
81
|
-
def
|
82
|
-
stub_request(:get, "
|
102
|
+
def stub_existent_template?(name="foo_tpl", url="http://localhost:9200")
|
103
|
+
stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
|
83
104
|
end
|
84
105
|
|
85
|
-
def
|
86
|
-
stub_request(:get, "
|
106
|
+
def stub_nonexistent_data_stream?(name="foo", url="http://localhost:9200")
|
107
|
+
stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
|
87
108
|
end
|
88
109
|
|
89
|
-
def
|
90
|
-
stub_request(:get, "
|
110
|
+
def stub_nonexistent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
|
111
|
+
stub_request(:get, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
|
91
112
|
end
|
92
113
|
|
93
|
-
def
|
94
|
-
stub_request(:get, "
|
114
|
+
def stub_nonexistent_template?(name="foo_tpl", url="http://localhost:9200")
|
115
|
+
stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
|
95
116
|
end
|
96
117
|
|
97
|
-
def
|
98
|
-
stub_request(:get, "
|
118
|
+
def stub_nonexistent_template_retry?(name="foo_tpl", url="http://localhost:9200")
|
119
|
+
stub_request(:get, "#{url}/_index_template/#{name}").
|
120
|
+
to_return({ status: 500, body: 'Internal Server Error' }, { status: 404, body: '{}' })
|
99
121
|
end
|
100
122
|
|
101
|
-
def stub_bulk_feed(datastream_name="foo", ilm_name="
|
102
|
-
stub_request(:post, "
|
123
|
+
def stub_bulk_feed(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl", url="http://localhost:9200")
|
124
|
+
stub_request(:post, "#{url}/#{datastream_name}/_bulk").with do |req|
|
103
125
|
# bulk data must be pair of OP and records
|
104
126
|
# {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
|
105
127
|
# {"@timestamp": ...}
|
106
128
|
@bulk_records += req.body.split("\n").size / 2
|
107
129
|
end
|
108
|
-
stub_request(:post, "
|
130
|
+
stub_request(:post, "#{url}/#{ilm_name}/_bulk").with do |req|
|
109
131
|
# bulk data must be pair of OP and records
|
110
132
|
# {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
|
111
133
|
# {"@timestamp": ...}
|
112
134
|
@bulk_records += req.body.split("\n").size / 2
|
113
135
|
end
|
114
|
-
stub_request(:post, "
|
136
|
+
stub_request(:post, "#{url}/#{template_name}/_bulk").with do |req|
|
115
137
|
# bulk data must be pair of OP and records
|
116
138
|
# {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
|
117
139
|
# {"@timestamp": ...}
|
@@ -119,12 +141,12 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
119
141
|
end
|
120
142
|
end
|
121
143
|
|
122
|
-
def stub_elastic_info(url="http://localhost:9200/", version=
|
144
|
+
def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version, headers={})
|
123
145
|
body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
|
124
|
-
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
|
146
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' }.merge(headers) })
|
125
147
|
end
|
126
148
|
|
127
|
-
def stub_default(datastream_name="foo", ilm_name="
|
149
|
+
def stub_default(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl", host="http://localhost:9200")
|
128
150
|
stub_elastic_info(host)
|
129
151
|
stub_nonexistent_ilm?(ilm_name)
|
130
152
|
stub_ilm_policy(ilm_name)
|
@@ -135,7 +157,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
135
157
|
end
|
136
158
|
|
137
159
|
def data_stream_supported?
|
138
|
-
Gem::Version.create(::
|
160
|
+
Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.create("7.9.0")
|
139
161
|
end
|
140
162
|
|
141
163
|
# ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
|
@@ -431,12 +453,62 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
431
453
|
'ROOT', '', {
|
432
454
|
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
433
455
|
'data_stream_name' => 'foo',
|
434
|
-
'data_stream_ilm_name' => "
|
456
|
+
'data_stream_ilm_name' => "foo_ilm_policy",
|
435
457
|
'data_stream_template_name' => "foo_tpl"
|
436
458
|
})
|
437
459
|
assert_equal "foo", driver(conf).instance.data_stream_name
|
438
460
|
end
|
439
461
|
|
462
|
+
def test_datastream_configure_retry
|
463
|
+
stub_elastic_info
|
464
|
+
stub_nonexistent_ilm?
|
465
|
+
stub_ilm_policy
|
466
|
+
stub_nonexistent_template_retry?
|
467
|
+
stub_index_template
|
468
|
+
stub_nonexistent_data_stream?
|
469
|
+
stub_data_stream
|
470
|
+
conf = config_element(
|
471
|
+
'ROOT', '', {
|
472
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
473
|
+
'data_stream_name' => 'foo',
|
474
|
+
'data_stream_ilm_name' => "foo_ilm_policy",
|
475
|
+
'data_stream_template_name' => "foo_tpl"
|
476
|
+
})
|
477
|
+
assert_equal "foo", driver(conf).instance.data_stream_name
|
478
|
+
end
|
479
|
+
|
480
|
+
def test_hosts_list_configure
|
481
|
+
config = %{
|
482
|
+
hosts https://john:password@host1:443/elastic/,http://host2
|
483
|
+
path /default_path
|
484
|
+
user default_user
|
485
|
+
password default_password
|
486
|
+
data_stream_name default
|
487
|
+
}
|
488
|
+
stub_elastic_info("https://host1:443/elastic//", elasticsearch_version,
|
489
|
+
{'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
|
490
|
+
stub_elastic_info("http://host2/default_path/_data_stream/default", elasticsearch_version,
|
491
|
+
{'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
|
492
|
+
stub_existent_data_stream?("default", "https://host1/elastic/")
|
493
|
+
instance = driver(config).instance
|
494
|
+
|
495
|
+
assert_equal 2, instance.get_connection_options[:hosts].length
|
496
|
+
host1, host2 = instance.get_connection_options[:hosts]
|
497
|
+
|
498
|
+
assert_equal 'host1', host1[:host]
|
499
|
+
assert_equal 443, host1[:port]
|
500
|
+
assert_equal 'https', host1[:scheme]
|
501
|
+
assert_equal 'john', host1[:user]
|
502
|
+
assert_equal 'password', host1[:password]
|
503
|
+
assert_equal '/elastic/', host1[:path]
|
504
|
+
|
505
|
+
assert_equal 'host2', host2[:host]
|
506
|
+
assert_equal 'http', host2[:scheme]
|
507
|
+
assert_equal 'default_user', host2[:user]
|
508
|
+
assert_equal 'default_password', host2[:password]
|
509
|
+
assert_equal '/default_path', host2[:path]
|
510
|
+
end
|
511
|
+
|
440
512
|
def test_existent_data_stream
|
441
513
|
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
442
514
|
|
@@ -449,7 +521,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
449
521
|
'ROOT', '', {
|
450
522
|
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
451
523
|
'data_stream_name' => 'foo',
|
452
|
-
'data_stream_ilm_name' => "
|
524
|
+
'data_stream_ilm_name' => "foo_ilm_policy",
|
453
525
|
'data_stream_template_name' => "foo_tpl"
|
454
526
|
})
|
455
527
|
assert_equal "foo", driver(conf).instance.data_stream_name
|
@@ -467,10 +539,10 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
467
539
|
'ROOT', '', {
|
468
540
|
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
469
541
|
'data_stream_name' => 'foo',
|
470
|
-
'data_stream_ilm_name' => "
|
542
|
+
'data_stream_ilm_name' => "foo_ilm_policy",
|
471
543
|
})
|
472
544
|
assert_equal "foo", driver(conf).instance.data_stream_name
|
473
|
-
assert_equal "
|
545
|
+
assert_equal "foo_ilm_policy", driver(conf).instance.data_stream_ilm_name
|
474
546
|
assert_equal "foo_template", driver(conf).instance.data_stream_template_name
|
475
547
|
end
|
476
548
|
|
@@ -615,7 +687,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
615
687
|
'ROOT', '', {
|
616
688
|
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
617
689
|
'data_stream_name' => 'foo',
|
618
|
-
'data_stream_ilm_name' => '
|
690
|
+
'data_stream_ilm_name' => 'foo_ilm_policy',
|
619
691
|
'data_stream_template_name' => 'foo_tpl'
|
620
692
|
})
|
621
693
|
driver(conf).run(default_tag: 'test') do
|
@@ -635,7 +707,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
635
707
|
port 778
|
636
708
|
scheme https
|
637
709
|
data_stream_name foo
|
638
|
-
data_stream_ilm_name
|
710
|
+
data_stream_ilm_name foo_ilm_policy
|
639
711
|
data_stream_template_name foo_tpl
|
640
712
|
user john
|
641
713
|
password doe
|
@@ -659,4 +731,86 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
659
731
|
|
660
732
|
assert_equal(4, connection_resets)
|
661
733
|
end
|
734
|
+
|
735
|
+
def test_doesnt_update_ilm_policy_if_overwrite_unset
|
736
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
737
|
+
|
738
|
+
config = %{
|
739
|
+
data_stream_name foo
|
740
|
+
data_stream_ilm_name foo_ilm_policy
|
741
|
+
data_stream_ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
|
742
|
+
}
|
743
|
+
|
744
|
+
stub_elastic_info
|
745
|
+
stub_index_template
|
746
|
+
stub_existent_data_stream?
|
747
|
+
stub_existent_ilm?
|
748
|
+
stub_data_stream
|
749
|
+
|
750
|
+
stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
|
751
|
+
to_return(:status => 200, :body => "", :headers => {})
|
752
|
+
|
753
|
+
assert_nothing_raised {
|
754
|
+
driver(config)
|
755
|
+
}
|
756
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 0)
|
757
|
+
end
|
758
|
+
|
759
|
+
def test_updates_ilm_policy_if_overwrite_set
|
760
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
761
|
+
|
762
|
+
config = %{
|
763
|
+
data_stream_name foo
|
764
|
+
data_stream_ilm_name foo_ilm_policy
|
765
|
+
data_stream_ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
|
766
|
+
data_stream_ilm_policy_overwrite true
|
767
|
+
}
|
768
|
+
|
769
|
+
stub_elastic_info
|
770
|
+
stub_index_template
|
771
|
+
stub_existent_data_stream?
|
772
|
+
stub_existent_ilm?
|
773
|
+
stub_data_stream
|
774
|
+
|
775
|
+
stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
|
776
|
+
to_return(:status => 200, :body => "", :headers => {})
|
777
|
+
|
778
|
+
assert_nothing_raised {
|
779
|
+
driver(config)
|
780
|
+
}
|
781
|
+
|
782
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 1)
|
783
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy",
|
784
|
+
body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
|
785
|
+
times: 1)
|
786
|
+
end
|
787
|
+
|
788
|
+
def test_creates_custom_ilm_policy_if_none_exists
|
789
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
790
|
+
|
791
|
+
config = %{
|
792
|
+
data_stream_name foo
|
793
|
+
data_stream_ilm_name foo_ilm_policy
|
794
|
+
data_stream_ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
|
795
|
+
}
|
796
|
+
|
797
|
+
stub_elastic_info
|
798
|
+
stub_index_template("foo_template")
|
799
|
+
stub_data_stream
|
800
|
+
stub_nonexistent_data_stream?
|
801
|
+
stub_nonexistent_ilm?
|
802
|
+
stub_nonexistent_template?("foo_template")
|
803
|
+
|
804
|
+
stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
|
805
|
+
to_return(:status => 200, :body => "", :headers => {})
|
806
|
+
|
807
|
+
assert_nothing_raised {
|
808
|
+
driver(config)
|
809
|
+
}
|
810
|
+
|
811
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 1)
|
812
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy",
|
813
|
+
body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
|
814
|
+
times: 1)
|
815
|
+
end
|
662
816
|
end
|
@@ -16,7 +16,15 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
16
16
|
@driver = nil
|
17
17
|
end
|
18
18
|
|
19
|
-
def
|
19
|
+
def elasticsearch_version
|
20
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
21
|
+
TRANSPORT_CLASS::VERSION
|
22
|
+
else
|
23
|
+
'6.4.2'.freeze
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def driver(conf='', es_version=elasticsearch_version.to_i)
|
20
28
|
# For request stub to detect compatibility.
|
21
29
|
@es_version ||= es_version
|
22
30
|
Fluent::Plugin::ElasticsearchOutputDynamic.module_eval(<<-CODE)
|
@@ -35,7 +43,11 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
35
43
|
end
|
36
44
|
|
37
45
|
def elasticsearch_transport_layer_decoupling?
|
38
|
-
Gem::Version.create(::
|
46
|
+
Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
47
|
+
end
|
48
|
+
|
49
|
+
def elastic_transport_layer?
|
50
|
+
Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
39
51
|
end
|
40
52
|
|
41
53
|
def default_type_name
|
@@ -58,9 +70,9 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
58
70
|
end
|
59
71
|
end
|
60
72
|
|
61
|
-
def stub_elastic_info(url="http://localhost:9200/", version=
|
73
|
+
def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
|
62
74
|
body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
|
63
|
-
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
|
75
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
|
64
76
|
end
|
65
77
|
|
66
78
|
def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
|
@@ -132,7 +144,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
132
144
|
end
|
133
145
|
|
134
146
|
test 'configure compression' do
|
135
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
147
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
136
148
|
|
137
149
|
config = %{
|
138
150
|
compression_level best_compression
|
@@ -143,7 +155,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
143
155
|
end
|
144
156
|
|
145
157
|
test 'check compression strategy' do
|
146
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
158
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
147
159
|
|
148
160
|
config = %{
|
149
161
|
compression_level best_speed
|
@@ -154,14 +166,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
154
166
|
end
|
155
167
|
|
156
168
|
test 'check content-encoding header with compression' do
|
157
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
169
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
158
170
|
|
159
171
|
config = %{
|
160
172
|
compression_level best_compression
|
161
173
|
}
|
162
174
|
instance = driver(config).instance
|
163
175
|
|
164
|
-
if
|
176
|
+
if elastic_transport_layer?
|
177
|
+
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
178
|
+
elsif elasticsearch_transport_layer_decoupling?
|
165
179
|
assert_equal nil, instance.client.transport.transport.options[:transport_options][:headers]["Content-Encoding"]
|
166
180
|
else
|
167
181
|
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
@@ -175,7 +189,9 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
175
189
|
end
|
176
190
|
compressable = instance.compressable_connection
|
177
191
|
|
178
|
-
if
|
192
|
+
if elastic_transport_layer?
|
193
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
194
|
+
elsif elasticsearch_transport_layer_decoupling?
|
179
195
|
assert_equal "gzip", instance.client(nil, compressable).transport.transport.options[:transport_options][:headers]["Content-Encoding"]
|
180
196
|
else
|
181
197
|
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
@@ -183,14 +199,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
183
199
|
end
|
184
200
|
|
185
201
|
test 'check compression option is passed to transport' do
|
186
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
202
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
187
203
|
|
188
204
|
config = %{
|
189
205
|
compression_level best_compression
|
190
206
|
}
|
191
207
|
instance = driver(config).instance
|
192
208
|
|
193
|
-
if
|
209
|
+
if elastic_transport_layer?
|
210
|
+
assert_equal false, instance.client.transport.options[:compression]
|
211
|
+
elsif elasticsearch_transport_layer_decoupling?
|
194
212
|
assert_equal false, instance.client.transport.transport.options[:compression]
|
195
213
|
else
|
196
214
|
assert_equal false, instance.client.transport.options[:compression]
|
@@ -204,7 +222,9 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
204
222
|
end
|
205
223
|
compressable = instance.compressable_connection
|
206
224
|
|
207
|
-
if
|
225
|
+
if elastic_transport_layer?
|
226
|
+
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
227
|
+
elsif elasticsearch_transport_layer_decoupling?
|
208
228
|
assert_equal true, instance.client(nil, compressable).transport.transport.options[:compression]
|
209
229
|
else
|
210
230
|
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
@@ -402,7 +422,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
402
422
|
end
|
403
423
|
|
404
424
|
def test_writes_to_default_index_with_compression
|
405
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
425
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
406
426
|
|
407
427
|
config = %[
|
408
428
|
compression_level default_compression
|
@@ -441,7 +461,13 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
441
461
|
driver.run(default_tag: 'test') do
|
442
462
|
driver.feed(sample_record)
|
443
463
|
end
|
444
|
-
|
464
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
465
|
+
assert_nil(index_cmds.first['index']['_type'])
|
466
|
+
elsif Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.0.0")
|
467
|
+
assert_equal("_doc", index_cmds.first['index']['_type'])
|
468
|
+
else
|
469
|
+
assert_equal("fluentd", index_cmds.first['index']['_type'])
|
470
|
+
end
|
445
471
|
end
|
446
472
|
|
447
473
|
def test_writes_to_specified_index
|
@@ -471,7 +497,13 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
471
497
|
driver.run(default_tag: 'test') do
|
472
498
|
driver.feed(sample_record)
|
473
499
|
end
|
474
|
-
|
500
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
501
|
+
assert_nil(index_cmds.first['index']['_type'])
|
502
|
+
elsif Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.0.0")
|
503
|
+
assert_equal("_doc", index_cmds.first['index']['_type'])
|
504
|
+
else
|
505
|
+
assert_equal("mytype", index_cmds.first['index']['_type'])
|
506
|
+
end
|
475
507
|
end
|
476
508
|
|
477
509
|
def test_writes_to_specified_host
|
@@ -971,7 +1003,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
971
1003
|
driver.run(default_tag: 'test') do
|
972
1004
|
driver.feed(nested_sample_record)
|
973
1005
|
end
|
974
|
-
|
1006
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1007
|
+
assert_equal('routing', index_cmds[0]['index'][routing_key])
|
975
1008
|
end
|
976
1009
|
|
977
1010
|
def test_adds_nested_routing_key_with_dollar_dot
|
@@ -981,7 +1014,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
981
1014
|
driver.run(default_tag: 'test') do
|
982
1015
|
driver.feed(nested_sample_record)
|
983
1016
|
end
|
984
|
-
|
1017
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1018
|
+
assert_equal('routing', index_cmds[0]['index'][routing_key])
|
985
1019
|
end
|
986
1020
|
|
987
1021
|
def test_adds_nested_routing_key_with_bracket
|
@@ -991,7 +1025,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
991
1025
|
driver.run(default_tag: 'test') do
|
992
1026
|
driver.feed(nested_sample_record)
|
993
1027
|
end
|
994
|
-
|
1028
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1029
|
+
assert_equal('routing', index_cmds[0]['index'][routing_key])
|
995
1030
|
end
|
996
1031
|
end
|
997
1032
|
|
@@ -1002,7 +1037,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
1002
1037
|
driver.run(default_tag: 'test') do
|
1003
1038
|
driver.feed(sample_record)
|
1004
1039
|
end
|
1005
|
-
|
1040
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1041
|
+
assert(!index_cmds[0]['index'].has_key?(routing_key))
|
1006
1042
|
end
|
1007
1043
|
|
1008
1044
|
def test_adds_routing_key_when_not_configured
|
@@ -1011,7 +1047,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
1011
1047
|
driver.run(default_tag: 'test') do
|
1012
1048
|
driver.feed(sample_record)
|
1013
1049
|
end
|
1014
|
-
|
1050
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1051
|
+
assert(!index_cmds[0]['index'].has_key?(routing_key))
|
1015
1052
|
end
|
1016
1053
|
|
1017
1054
|
def test_remove_one_key
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.
|
4
|
+
version: 5.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2022-02-21 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|
@@ -171,6 +171,7 @@ files:
|
|
171
171
|
- gemfiles/Gemfile.elasticsearch.v6
|
172
172
|
- lib/fluent/log-ext.rb
|
173
173
|
- lib/fluent/plugin/default-ilm-policy.json
|
174
|
+
- lib/fluent/plugin/elasticsearch_compat.rb
|
174
175
|
- lib/fluent/plugin/elasticsearch_constants.rb
|
175
176
|
- lib/fluent/plugin/elasticsearch_error.rb
|
176
177
|
- lib/fluent/plugin/elasticsearch_error_handler.rb
|
@@ -221,7 +222,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
221
222
|
- !ruby/object:Gem::Version
|
222
223
|
version: '0'
|
223
224
|
requirements: []
|
224
|
-
rubygems_version: 3.2.
|
225
|
+
rubygems_version: 3.2.30
|
225
226
|
signing_key:
|
226
227
|
specification_version: 4
|
227
228
|
summary: Elasticsearch output plugin for Fluent event collector
|