fluent-plugin-elasticsearch 5.1.5 → 5.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/History.md +12 -0
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_compat.rb +30 -0
- data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +2 -2
- data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb +18 -4
- data/lib/fluent/plugin/elasticsearch_index_template.rb +8 -4
- data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb +2 -1
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +1 -1
- data/lib/fluent/plugin/in_elasticsearch.rb +2 -1
- data/lib/fluent/plugin/oj_serializer.rb +2 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +9 -8
- data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +30 -11
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +3 -1
- data/test/plugin/test_elasticsearch_fallback_selector.rb +15 -7
- data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +47 -17
- data/test/plugin/test_filter_elasticsearch_genid.rb +16 -16
- data/test/plugin/test_in_elasticsearch.rb +10 -2
- data/test/plugin/test_out_elasticsearch.rb +281 -122
- data/test/plugin/test_out_elasticsearch_data_stream.rb +122 -28
- data/test/plugin/test_out_elasticsearch_dynamic.rb +57 -20
- metadata +4 -3
@@ -19,10 +19,26 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
19
19
|
@driver = nil
|
20
20
|
log = Fluent::Engine.log
|
21
21
|
log.out.logs.slice!(0, log.out.logs.length)
|
22
|
-
@bulk_records =
|
22
|
+
@bulk_records = []
|
23
23
|
end
|
24
24
|
|
25
|
-
def
|
25
|
+
def elasticsearch_version
|
26
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
27
|
+
TRANSPORT_CLASS::VERSION
|
28
|
+
else
|
29
|
+
'5.0.0'.freeze
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def ilm_endpoint
|
34
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
35
|
+
'_enrich'.freeze
|
36
|
+
else
|
37
|
+
'_ilm'.freeze
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def driver(conf='', es_version=elasticsearch_version.to_i, client_version=elasticsearch_version)
|
26
42
|
# For request stub to detect compatibility.
|
27
43
|
@es_version ||= es_version
|
28
44
|
@client_version ||= client_version
|
@@ -63,7 +79,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
63
79
|
NONEXISTENT_DATA_STREAM_EXCEPTION = {"error": {}, "status": 404}
|
64
80
|
|
65
81
|
def stub_ilm_policy(name="foo_ilm_policy", url="http://localhost:9200")
|
66
|
-
stub_request(:put, "#{url}/
|
82
|
+
stub_request(:put, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
|
67
83
|
end
|
68
84
|
|
69
85
|
def stub_index_template(name="foo_tpl", url="http://localhost:9200")
|
@@ -79,7 +95,8 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
79
95
|
end
|
80
96
|
|
81
97
|
def stub_existent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
|
82
|
-
|
98
|
+
|
99
|
+
stub_request(:get, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
|
83
100
|
end
|
84
101
|
|
85
102
|
def stub_existent_template?(name="foo_tpl", url="http://localhost:9200")
|
@@ -87,15 +104,26 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
87
104
|
end
|
88
105
|
|
89
106
|
def stub_nonexistent_data_stream?(name="foo", url="http://localhost:9200")
|
90
|
-
stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [404,
|
107
|
+
stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
|
91
108
|
end
|
92
109
|
|
93
110
|
def stub_nonexistent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
|
94
|
-
stub_request(:get, "#{url}/
|
111
|
+
stub_request(:get, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
|
95
112
|
end
|
96
113
|
|
97
114
|
def stub_nonexistent_template?(name="foo_tpl", url="http://localhost:9200")
|
98
|
-
stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404,
|
115
|
+
stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
|
116
|
+
end
|
117
|
+
|
118
|
+
|
119
|
+
def push_bulk_request(req_body)
|
120
|
+
# bulk data must be pair of OP and records
|
121
|
+
# {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
|
122
|
+
# {"@timestamp": ...}
|
123
|
+
ops = req_body.split("\n")
|
124
|
+
@bulk_records += ops.values_at(
|
125
|
+
* ops.each_index.select {|i| i.odd? }
|
126
|
+
).map{ |i| JSON.parse(i) }
|
99
127
|
end
|
100
128
|
|
101
129
|
def stub_nonexistent_template_retry?(name="foo_tpl", url="http://localhost:9200")
|
@@ -108,25 +136,25 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
108
136
|
# bulk data must be pair of OP and records
|
109
137
|
# {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
|
110
138
|
# {"@timestamp": ...}
|
111
|
-
|
139
|
+
push_bulk_request(req.body)
|
112
140
|
end
|
113
141
|
stub_request(:post, "#{url}/#{ilm_name}/_bulk").with do |req|
|
114
142
|
# bulk data must be pair of OP and records
|
115
143
|
# {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
|
116
144
|
# {"@timestamp": ...}
|
117
|
-
|
145
|
+
push_bulk_request(req.body)
|
118
146
|
end
|
119
147
|
stub_request(:post, "#{url}/#{template_name}/_bulk").with do |req|
|
120
148
|
# bulk data must be pair of OP and records
|
121
149
|
# {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
|
122
150
|
# {"@timestamp": ...}
|
123
|
-
|
151
|
+
push_bulk_request(req.body)
|
124
152
|
end
|
125
153
|
end
|
126
154
|
|
127
|
-
def stub_elastic_info(url="http://localhost:9200/", version=
|
155
|
+
def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version, headers={})
|
128
156
|
body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
|
129
|
-
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' }.merge(headers) })
|
157
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' }.merge(headers) })
|
130
158
|
end
|
131
159
|
|
132
160
|
def stub_default(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl", host="http://localhost:9200")
|
@@ -140,7 +168,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
140
168
|
end
|
141
169
|
|
142
170
|
def data_stream_supported?
|
143
|
-
Gem::Version.create(::
|
171
|
+
Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.create("7.9.0")
|
144
172
|
end
|
145
173
|
|
146
174
|
# ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
|
@@ -468,9 +496,9 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
468
496
|
password default_password
|
469
497
|
data_stream_name default
|
470
498
|
}
|
471
|
-
stub_elastic_info("https://host1:443/elastic//",
|
499
|
+
stub_elastic_info("https://host1:443/elastic//", elasticsearch_version,
|
472
500
|
{'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
|
473
|
-
stub_elastic_info("http://host2/default_path/_data_stream/default",
|
501
|
+
stub_elastic_info("http://host2/default_path/_data_stream/default", elasticsearch_version,
|
474
502
|
{'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
|
475
503
|
stub_existent_data_stream?("default", "https://host1/elastic/")
|
476
504
|
instance = driver(config).instance
|
@@ -583,7 +611,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
583
611
|
driver(conf).run(default_tag: 'test') do
|
584
612
|
driver.feed(sample_record)
|
585
613
|
end
|
586
|
-
assert_equal 1, @bulk_records
|
614
|
+
assert_equal 1, @bulk_records.length
|
587
615
|
end
|
588
616
|
|
589
617
|
def test_placeholder_params_unset
|
@@ -602,7 +630,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
602
630
|
driver(conf).run(default_tag: 'test') do
|
603
631
|
driver.feed(sample_record)
|
604
632
|
end
|
605
|
-
assert_equal 1, @bulk_records
|
633
|
+
assert_equal 1, @bulk_records.length
|
606
634
|
end
|
607
635
|
|
608
636
|
|
@@ -628,7 +656,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
628
656
|
driver(conf).run(default_tag: 'test') do
|
629
657
|
driver.feed(sample_record)
|
630
658
|
end
|
631
|
-
assert_equal 1, @bulk_records
|
659
|
+
assert_equal 1, @bulk_records.length
|
632
660
|
end
|
633
661
|
|
634
662
|
def test_custom_record_placeholder
|
@@ -658,7 +686,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
658
686
|
driver.feed(record)
|
659
687
|
end
|
660
688
|
end
|
661
|
-
assert_equal keys.count, @bulk_records
|
689
|
+
assert_equal keys.count, @bulk_records.length
|
662
690
|
end
|
663
691
|
|
664
692
|
def test_bulk_insert_feed
|
@@ -676,7 +704,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
676
704
|
driver(conf).run(default_tag: 'test') do
|
677
705
|
driver.feed(sample_record)
|
678
706
|
end
|
679
|
-
assert_equal 1, @bulk_records
|
707
|
+
assert_equal 1, @bulk_records.length
|
680
708
|
end
|
681
709
|
|
682
710
|
def test_template_retry_install_fails
|
@@ -730,13 +758,13 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
730
758
|
stub_existent_ilm?
|
731
759
|
stub_data_stream
|
732
760
|
|
733
|
-
stub_request(:put, "http://localhost:9200/
|
761
|
+
stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
|
734
762
|
to_return(:status => 200, :body => "", :headers => {})
|
735
763
|
|
736
764
|
assert_nothing_raised {
|
737
765
|
driver(config)
|
738
766
|
}
|
739
|
-
assert_requested(:put, "http://localhost:9200/
|
767
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 0)
|
740
768
|
end
|
741
769
|
|
742
770
|
def test_updates_ilm_policy_if_overwrite_set
|
@@ -755,15 +783,15 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
755
783
|
stub_existent_ilm?
|
756
784
|
stub_data_stream
|
757
785
|
|
758
|
-
stub_request(:put, "http://localhost:9200/
|
786
|
+
stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
|
759
787
|
to_return(:status => 200, :body => "", :headers => {})
|
760
788
|
|
761
789
|
assert_nothing_raised {
|
762
790
|
driver(config)
|
763
791
|
}
|
764
792
|
|
765
|
-
assert_requested(:put, "http://localhost:9200/
|
766
|
-
assert_requested(:put, "http://localhost:9200/
|
793
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 1)
|
794
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy",
|
767
795
|
body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
|
768
796
|
times: 1)
|
769
797
|
end
|
@@ -784,16 +812,82 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
784
812
|
stub_nonexistent_ilm?
|
785
813
|
stub_nonexistent_template?("foo_template")
|
786
814
|
|
787
|
-
stub_request(:put, "http://localhost:9200/
|
815
|
+
stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
|
788
816
|
to_return(:status => 200, :body => "", :headers => {})
|
789
817
|
|
790
818
|
assert_nothing_raised {
|
791
819
|
driver(config)
|
792
820
|
}
|
793
821
|
|
794
|
-
assert_requested(:put, "http://localhost:9200/
|
795
|
-
assert_requested(:put, "http://localhost:9200/
|
822
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 1)
|
823
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy",
|
796
824
|
body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
|
797
825
|
times: 1)
|
798
826
|
end
|
827
|
+
|
828
|
+
def test_doesnt_add_tag_key_when_not_configured
|
829
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
830
|
+
|
831
|
+
config = %{
|
832
|
+
data_stream_name foo
|
833
|
+
data_stream_template_name foo_tpl
|
834
|
+
data_stream_ilm_name foo_ilm_policy
|
835
|
+
}
|
836
|
+
|
837
|
+
stub_default
|
838
|
+
stub_bulk_feed
|
839
|
+
driver(config)
|
840
|
+
driver.run(default_tag: 'mytag') do
|
841
|
+
driver.feed(sample_record)
|
842
|
+
end
|
843
|
+
|
844
|
+
assert_equal(1, @bulk_records.length)
|
845
|
+
assert_false(@bulk_records[0].has_key?('tag'))
|
846
|
+
end
|
847
|
+
|
848
|
+
|
849
|
+
def test_adds_tag_key_when_configured
|
850
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
851
|
+
|
852
|
+
config = %{
|
853
|
+
data_stream_name foo
|
854
|
+
data_stream_template_name foo_tpl
|
855
|
+
data_stream_ilm_name foo_ilm_policy
|
856
|
+
include_tag_key true
|
857
|
+
}
|
858
|
+
|
859
|
+
stub_default
|
860
|
+
stub_bulk_feed
|
861
|
+
driver(config)
|
862
|
+
driver.run(default_tag: 'mytag') do
|
863
|
+
driver.feed(sample_record)
|
864
|
+
end
|
865
|
+
|
866
|
+
assert_equal(1, @bulk_records.length)
|
867
|
+
assert(@bulk_records[0].has_key?('tag'))
|
868
|
+
assert_equal('mytag', @bulk_records[0]['tag'])
|
869
|
+
end
|
870
|
+
|
871
|
+
def test_adds_custom_tag_key_when_configured
|
872
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
873
|
+
|
874
|
+
config = %{
|
875
|
+
data_stream_name foo
|
876
|
+
data_stream_template_name foo_tpl
|
877
|
+
data_stream_ilm_name foo_ilm_policy
|
878
|
+
include_tag_key true
|
879
|
+
tag_key custom_tag_key
|
880
|
+
}
|
881
|
+
|
882
|
+
stub_default
|
883
|
+
stub_bulk_feed
|
884
|
+
driver(config)
|
885
|
+
driver.run(default_tag: 'mytag') do
|
886
|
+
driver.feed(sample_record)
|
887
|
+
end
|
888
|
+
|
889
|
+
assert_equal(1, @bulk_records.length)
|
890
|
+
assert(@bulk_records[0].has_key?('custom_tag_key'))
|
891
|
+
assert_equal('mytag', @bulk_records[0]['custom_tag_key'])
|
892
|
+
end
|
799
893
|
end
|
@@ -16,7 +16,15 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
16
16
|
@driver = nil
|
17
17
|
end
|
18
18
|
|
19
|
-
def
|
19
|
+
def elasticsearch_version
|
20
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
21
|
+
TRANSPORT_CLASS::VERSION
|
22
|
+
else
|
23
|
+
'6.4.2'.freeze
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def driver(conf='', es_version=elasticsearch_version.to_i)
|
20
28
|
# For request stub to detect compatibility.
|
21
29
|
@es_version ||= es_version
|
22
30
|
Fluent::Plugin::ElasticsearchOutputDynamic.module_eval(<<-CODE)
|
@@ -35,7 +43,11 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
35
43
|
end
|
36
44
|
|
37
45
|
def elasticsearch_transport_layer_decoupling?
|
38
|
-
Gem::Version.create(::
|
46
|
+
Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
47
|
+
end
|
48
|
+
|
49
|
+
def elastic_transport_layer?
|
50
|
+
Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
39
51
|
end
|
40
52
|
|
41
53
|
def default_type_name
|
@@ -58,9 +70,9 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
58
70
|
end
|
59
71
|
end
|
60
72
|
|
61
|
-
def stub_elastic_info(url="http://localhost:9200/", version=
|
73
|
+
def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
|
62
74
|
body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
|
63
|
-
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
|
75
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
|
64
76
|
end
|
65
77
|
|
66
78
|
def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
|
@@ -132,7 +144,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
132
144
|
end
|
133
145
|
|
134
146
|
test 'configure compression' do
|
135
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
147
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
136
148
|
|
137
149
|
config = %{
|
138
150
|
compression_level best_compression
|
@@ -143,7 +155,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
143
155
|
end
|
144
156
|
|
145
157
|
test 'check compression strategy' do
|
146
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
158
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
147
159
|
|
148
160
|
config = %{
|
149
161
|
compression_level best_speed
|
@@ -154,14 +166,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
154
166
|
end
|
155
167
|
|
156
168
|
test 'check content-encoding header with compression' do
|
157
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
169
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
158
170
|
|
159
171
|
config = %{
|
160
172
|
compression_level best_compression
|
161
173
|
}
|
162
174
|
instance = driver(config).instance
|
163
175
|
|
164
|
-
if
|
176
|
+
if elastic_transport_layer?
|
177
|
+
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
178
|
+
elsif elasticsearch_transport_layer_decoupling?
|
165
179
|
assert_equal nil, instance.client.transport.transport.options[:transport_options][:headers]["Content-Encoding"]
|
166
180
|
else
|
167
181
|
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
@@ -175,7 +189,9 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
175
189
|
end
|
176
190
|
compressable = instance.compressable_connection
|
177
191
|
|
178
|
-
if
|
192
|
+
if elastic_transport_layer?
|
193
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
194
|
+
elsif elasticsearch_transport_layer_decoupling?
|
179
195
|
assert_equal "gzip", instance.client(nil, compressable).transport.transport.options[:transport_options][:headers]["Content-Encoding"]
|
180
196
|
else
|
181
197
|
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
@@ -183,14 +199,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
183
199
|
end
|
184
200
|
|
185
201
|
test 'check compression option is passed to transport' do
|
186
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
202
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
187
203
|
|
188
204
|
config = %{
|
189
205
|
compression_level best_compression
|
190
206
|
}
|
191
207
|
instance = driver(config).instance
|
192
208
|
|
193
|
-
if
|
209
|
+
if elastic_transport_layer?
|
210
|
+
assert_equal false, instance.client.transport.options[:compression]
|
211
|
+
elsif elasticsearch_transport_layer_decoupling?
|
194
212
|
assert_equal false, instance.client.transport.transport.options[:compression]
|
195
213
|
else
|
196
214
|
assert_equal false, instance.client.transport.options[:compression]
|
@@ -204,7 +222,9 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
204
222
|
end
|
205
223
|
compressable = instance.compressable_connection
|
206
224
|
|
207
|
-
if
|
225
|
+
if elastic_transport_layer?
|
226
|
+
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
227
|
+
elsif elasticsearch_transport_layer_decoupling?
|
208
228
|
assert_equal true, instance.client(nil, compressable).transport.transport.options[:compression]
|
209
229
|
else
|
210
230
|
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
@@ -402,7 +422,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
402
422
|
end
|
403
423
|
|
404
424
|
def test_writes_to_default_index_with_compression
|
405
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
425
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
406
426
|
|
407
427
|
config = %[
|
408
428
|
compression_level default_compression
|
@@ -441,7 +461,13 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
441
461
|
driver.run(default_tag: 'test') do
|
442
462
|
driver.feed(sample_record)
|
443
463
|
end
|
444
|
-
|
464
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
465
|
+
assert_nil(index_cmds.first['index']['_type'])
|
466
|
+
elsif Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.0.0")
|
467
|
+
assert_equal("_doc", index_cmds.first['index']['_type'])
|
468
|
+
else
|
469
|
+
assert_equal("fluentd", index_cmds.first['index']['_type'])
|
470
|
+
end
|
445
471
|
end
|
446
472
|
|
447
473
|
def test_writes_to_specified_index
|
@@ -471,7 +497,13 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
471
497
|
driver.run(default_tag: 'test') do
|
472
498
|
driver.feed(sample_record)
|
473
499
|
end
|
474
|
-
|
500
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
501
|
+
assert_nil(index_cmds.first['index']['_type'])
|
502
|
+
elsif Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.0.0")
|
503
|
+
assert_equal("_doc", index_cmds.first['index']['_type'])
|
504
|
+
else
|
505
|
+
assert_equal("mytype", index_cmds.first['index']['_type'])
|
506
|
+
end
|
475
507
|
end
|
476
508
|
|
477
509
|
def test_writes_to_specified_host
|
@@ -971,7 +1003,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
971
1003
|
driver.run(default_tag: 'test') do
|
972
1004
|
driver.feed(nested_sample_record)
|
973
1005
|
end
|
974
|
-
|
1006
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1007
|
+
assert_equal('routing', index_cmds[0]['index'][routing_key])
|
975
1008
|
end
|
976
1009
|
|
977
1010
|
def test_adds_nested_routing_key_with_dollar_dot
|
@@ -981,7 +1014,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
981
1014
|
driver.run(default_tag: 'test') do
|
982
1015
|
driver.feed(nested_sample_record)
|
983
1016
|
end
|
984
|
-
|
1017
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1018
|
+
assert_equal('routing', index_cmds[0]['index'][routing_key])
|
985
1019
|
end
|
986
1020
|
|
987
1021
|
def test_adds_nested_routing_key_with_bracket
|
@@ -991,7 +1025,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
991
1025
|
driver.run(default_tag: 'test') do
|
992
1026
|
driver.feed(nested_sample_record)
|
993
1027
|
end
|
994
|
-
|
1028
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1029
|
+
assert_equal('routing', index_cmds[0]['index'][routing_key])
|
995
1030
|
end
|
996
1031
|
end
|
997
1032
|
|
@@ -1002,7 +1037,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
1002
1037
|
driver.run(default_tag: 'test') do
|
1003
1038
|
driver.feed(sample_record)
|
1004
1039
|
end
|
1005
|
-
|
1040
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1041
|
+
assert(!index_cmds[0]['index'].has_key?(routing_key))
|
1006
1042
|
end
|
1007
1043
|
|
1008
1044
|
def test_adds_routing_key_when_not_configured
|
@@ -1011,7 +1047,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
1011
1047
|
driver.run(default_tag: 'test') do
|
1012
1048
|
driver.feed(sample_record)
|
1013
1049
|
end
|
1014
|
-
|
1050
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1051
|
+
assert(!index_cmds[0]['index'].has_key?(routing_key))
|
1015
1052
|
end
|
1016
1053
|
|
1017
1054
|
def test_remove_one_key
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.
|
4
|
+
version: 5.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2022-
|
13
|
+
date: 2022-04-11 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|
@@ -171,6 +171,7 @@ files:
|
|
171
171
|
- gemfiles/Gemfile.elasticsearch.v6
|
172
172
|
- lib/fluent/log-ext.rb
|
173
173
|
- lib/fluent/plugin/default-ilm-policy.json
|
174
|
+
- lib/fluent/plugin/elasticsearch_compat.rb
|
174
175
|
- lib/fluent/plugin/elasticsearch_constants.rb
|
175
176
|
- lib/fluent/plugin/elasticsearch_error.rb
|
176
177
|
- lib/fluent/plugin/elasticsearch_error_handler.rb
|
@@ -221,7 +222,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
221
222
|
- !ruby/object:Gem::Version
|
222
223
|
version: '0'
|
223
224
|
requirements: []
|
224
|
-
rubygems_version: 3.2.
|
225
|
+
rubygems_version: 3.2.32
|
225
226
|
signing_key:
|
226
227
|
specification_version: 4
|
227
228
|
summary: Elasticsearch output plugin for Fluent event collector
|