fluent-plugin-elasticsearch 5.1.5 → 5.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/History.md +12 -0
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_compat.rb +30 -0
- data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +2 -2
- data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb +18 -4
- data/lib/fluent/plugin/elasticsearch_index_template.rb +8 -4
- data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb +2 -1
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +1 -1
- data/lib/fluent/plugin/in_elasticsearch.rb +2 -1
- data/lib/fluent/plugin/oj_serializer.rb +2 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +9 -8
- data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +30 -11
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +3 -1
- data/test/plugin/test_elasticsearch_fallback_selector.rb +15 -7
- data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +47 -17
- data/test/plugin/test_filter_elasticsearch_genid.rb +16 -16
- data/test/plugin/test_in_elasticsearch.rb +10 -2
- data/test/plugin/test_out_elasticsearch.rb +281 -122
- data/test/plugin/test_out_elasticsearch_data_stream.rb +122 -28
- data/test/plugin/test_out_elasticsearch_dynamic.rb +57 -20
- metadata +4 -3
@@ -19,10 +19,26 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
19
19
|
@driver = nil
|
20
20
|
log = Fluent::Engine.log
|
21
21
|
log.out.logs.slice!(0, log.out.logs.length)
|
22
|
-
@bulk_records =
|
22
|
+
@bulk_records = []
|
23
23
|
end
|
24
24
|
|
25
|
-
def
|
25
|
+
def elasticsearch_version
|
26
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
27
|
+
TRANSPORT_CLASS::VERSION
|
28
|
+
else
|
29
|
+
'5.0.0'.freeze
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def ilm_endpoint
|
34
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
35
|
+
'_enrich'.freeze
|
36
|
+
else
|
37
|
+
'_ilm'.freeze
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def driver(conf='', es_version=elasticsearch_version.to_i, client_version=elasticsearch_version)
|
26
42
|
# For request stub to detect compatibility.
|
27
43
|
@es_version ||= es_version
|
28
44
|
@client_version ||= client_version
|
@@ -63,7 +79,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
63
79
|
NONEXISTENT_DATA_STREAM_EXCEPTION = {"error": {}, "status": 404}
|
64
80
|
|
65
81
|
def stub_ilm_policy(name="foo_ilm_policy", url="http://localhost:9200")
|
66
|
-
stub_request(:put, "#{url}/
|
82
|
+
stub_request(:put, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
|
67
83
|
end
|
68
84
|
|
69
85
|
def stub_index_template(name="foo_tpl", url="http://localhost:9200")
|
@@ -79,7 +95,8 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
79
95
|
end
|
80
96
|
|
81
97
|
def stub_existent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
|
82
|
-
|
98
|
+
|
99
|
+
stub_request(:get, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
|
83
100
|
end
|
84
101
|
|
85
102
|
def stub_existent_template?(name="foo_tpl", url="http://localhost:9200")
|
@@ -87,15 +104,26 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
87
104
|
end
|
88
105
|
|
89
106
|
def stub_nonexistent_data_stream?(name="foo", url="http://localhost:9200")
|
90
|
-
stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [404,
|
107
|
+
stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
|
91
108
|
end
|
92
109
|
|
93
110
|
def stub_nonexistent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
|
94
|
-
stub_request(:get, "#{url}/
|
111
|
+
stub_request(:get, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
|
95
112
|
end
|
96
113
|
|
97
114
|
def stub_nonexistent_template?(name="foo_tpl", url="http://localhost:9200")
|
98
|
-
stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404,
|
115
|
+
stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
|
116
|
+
end
|
117
|
+
|
118
|
+
|
119
|
+
def push_bulk_request(req_body)
|
120
|
+
# bulk data must be pair of OP and records
|
121
|
+
# {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
|
122
|
+
# {"@timestamp": ...}
|
123
|
+
ops = req_body.split("\n")
|
124
|
+
@bulk_records += ops.values_at(
|
125
|
+
* ops.each_index.select {|i| i.odd? }
|
126
|
+
).map{ |i| JSON.parse(i) }
|
99
127
|
end
|
100
128
|
|
101
129
|
def stub_nonexistent_template_retry?(name="foo_tpl", url="http://localhost:9200")
|
@@ -108,25 +136,25 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
108
136
|
# bulk data must be pair of OP and records
|
109
137
|
# {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
|
110
138
|
# {"@timestamp": ...}
|
111
|
-
|
139
|
+
push_bulk_request(req.body)
|
112
140
|
end
|
113
141
|
stub_request(:post, "#{url}/#{ilm_name}/_bulk").with do |req|
|
114
142
|
# bulk data must be pair of OP and records
|
115
143
|
# {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
|
116
144
|
# {"@timestamp": ...}
|
117
|
-
|
145
|
+
push_bulk_request(req.body)
|
118
146
|
end
|
119
147
|
stub_request(:post, "#{url}/#{template_name}/_bulk").with do |req|
|
120
148
|
# bulk data must be pair of OP and records
|
121
149
|
# {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
|
122
150
|
# {"@timestamp": ...}
|
123
|
-
|
151
|
+
push_bulk_request(req.body)
|
124
152
|
end
|
125
153
|
end
|
126
154
|
|
127
|
-
def stub_elastic_info(url="http://localhost:9200/", version=
|
155
|
+
def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version, headers={})
|
128
156
|
body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
|
129
|
-
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' }.merge(headers) })
|
157
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' }.merge(headers) })
|
130
158
|
end
|
131
159
|
|
132
160
|
def stub_default(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl", host="http://localhost:9200")
|
@@ -140,7 +168,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
140
168
|
end
|
141
169
|
|
142
170
|
def data_stream_supported?
|
143
|
-
Gem::Version.create(::
|
171
|
+
Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.create("7.9.0")
|
144
172
|
end
|
145
173
|
|
146
174
|
# ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
|
@@ -468,9 +496,9 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
468
496
|
password default_password
|
469
497
|
data_stream_name default
|
470
498
|
}
|
471
|
-
stub_elastic_info("https://host1:443/elastic//",
|
499
|
+
stub_elastic_info("https://host1:443/elastic//", elasticsearch_version,
|
472
500
|
{'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
|
473
|
-
stub_elastic_info("http://host2/default_path/_data_stream/default",
|
501
|
+
stub_elastic_info("http://host2/default_path/_data_stream/default", elasticsearch_version,
|
474
502
|
{'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
|
475
503
|
stub_existent_data_stream?("default", "https://host1/elastic/")
|
476
504
|
instance = driver(config).instance
|
@@ -583,7 +611,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
583
611
|
driver(conf).run(default_tag: 'test') do
|
584
612
|
driver.feed(sample_record)
|
585
613
|
end
|
586
|
-
assert_equal 1, @bulk_records
|
614
|
+
assert_equal 1, @bulk_records.length
|
587
615
|
end
|
588
616
|
|
589
617
|
def test_placeholder_params_unset
|
@@ -602,7 +630,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
602
630
|
driver(conf).run(default_tag: 'test') do
|
603
631
|
driver.feed(sample_record)
|
604
632
|
end
|
605
|
-
assert_equal 1, @bulk_records
|
633
|
+
assert_equal 1, @bulk_records.length
|
606
634
|
end
|
607
635
|
|
608
636
|
|
@@ -628,7 +656,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
628
656
|
driver(conf).run(default_tag: 'test') do
|
629
657
|
driver.feed(sample_record)
|
630
658
|
end
|
631
|
-
assert_equal 1, @bulk_records
|
659
|
+
assert_equal 1, @bulk_records.length
|
632
660
|
end
|
633
661
|
|
634
662
|
def test_custom_record_placeholder
|
@@ -658,7 +686,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
658
686
|
driver.feed(record)
|
659
687
|
end
|
660
688
|
end
|
661
|
-
assert_equal keys.count, @bulk_records
|
689
|
+
assert_equal keys.count, @bulk_records.length
|
662
690
|
end
|
663
691
|
|
664
692
|
def test_bulk_insert_feed
|
@@ -676,7 +704,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
676
704
|
driver(conf).run(default_tag: 'test') do
|
677
705
|
driver.feed(sample_record)
|
678
706
|
end
|
679
|
-
assert_equal 1, @bulk_records
|
707
|
+
assert_equal 1, @bulk_records.length
|
680
708
|
end
|
681
709
|
|
682
710
|
def test_template_retry_install_fails
|
@@ -730,13 +758,13 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
730
758
|
stub_existent_ilm?
|
731
759
|
stub_data_stream
|
732
760
|
|
733
|
-
stub_request(:put, "http://localhost:9200/
|
761
|
+
stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
|
734
762
|
to_return(:status => 200, :body => "", :headers => {})
|
735
763
|
|
736
764
|
assert_nothing_raised {
|
737
765
|
driver(config)
|
738
766
|
}
|
739
|
-
assert_requested(:put, "http://localhost:9200/
|
767
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 0)
|
740
768
|
end
|
741
769
|
|
742
770
|
def test_updates_ilm_policy_if_overwrite_set
|
@@ -755,15 +783,15 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
755
783
|
stub_existent_ilm?
|
756
784
|
stub_data_stream
|
757
785
|
|
758
|
-
stub_request(:put, "http://localhost:9200/
|
786
|
+
stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
|
759
787
|
to_return(:status => 200, :body => "", :headers => {})
|
760
788
|
|
761
789
|
assert_nothing_raised {
|
762
790
|
driver(config)
|
763
791
|
}
|
764
792
|
|
765
|
-
assert_requested(:put, "http://localhost:9200/
|
766
|
-
assert_requested(:put, "http://localhost:9200/
|
793
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 1)
|
794
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy",
|
767
795
|
body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
|
768
796
|
times: 1)
|
769
797
|
end
|
@@ -784,16 +812,82 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
784
812
|
stub_nonexistent_ilm?
|
785
813
|
stub_nonexistent_template?("foo_template")
|
786
814
|
|
787
|
-
stub_request(:put, "http://localhost:9200/
|
815
|
+
stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
|
788
816
|
to_return(:status => 200, :body => "", :headers => {})
|
789
817
|
|
790
818
|
assert_nothing_raised {
|
791
819
|
driver(config)
|
792
820
|
}
|
793
821
|
|
794
|
-
assert_requested(:put, "http://localhost:9200/
|
795
|
-
assert_requested(:put, "http://localhost:9200/
|
822
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 1)
|
823
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy",
|
796
824
|
body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
|
797
825
|
times: 1)
|
798
826
|
end
|
827
|
+
|
828
|
+
def test_doesnt_add_tag_key_when_not_configured
|
829
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
830
|
+
|
831
|
+
config = %{
|
832
|
+
data_stream_name foo
|
833
|
+
data_stream_template_name foo_tpl
|
834
|
+
data_stream_ilm_name foo_ilm_policy
|
835
|
+
}
|
836
|
+
|
837
|
+
stub_default
|
838
|
+
stub_bulk_feed
|
839
|
+
driver(config)
|
840
|
+
driver.run(default_tag: 'mytag') do
|
841
|
+
driver.feed(sample_record)
|
842
|
+
end
|
843
|
+
|
844
|
+
assert_equal(1, @bulk_records.length)
|
845
|
+
assert_false(@bulk_records[0].has_key?('tag'))
|
846
|
+
end
|
847
|
+
|
848
|
+
|
849
|
+
def test_adds_tag_key_when_configured
|
850
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
851
|
+
|
852
|
+
config = %{
|
853
|
+
data_stream_name foo
|
854
|
+
data_stream_template_name foo_tpl
|
855
|
+
data_stream_ilm_name foo_ilm_policy
|
856
|
+
include_tag_key true
|
857
|
+
}
|
858
|
+
|
859
|
+
stub_default
|
860
|
+
stub_bulk_feed
|
861
|
+
driver(config)
|
862
|
+
driver.run(default_tag: 'mytag') do
|
863
|
+
driver.feed(sample_record)
|
864
|
+
end
|
865
|
+
|
866
|
+
assert_equal(1, @bulk_records.length)
|
867
|
+
assert(@bulk_records[0].has_key?('tag'))
|
868
|
+
assert_equal('mytag', @bulk_records[0]['tag'])
|
869
|
+
end
|
870
|
+
|
871
|
+
def test_adds_custom_tag_key_when_configured
|
872
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
873
|
+
|
874
|
+
config = %{
|
875
|
+
data_stream_name foo
|
876
|
+
data_stream_template_name foo_tpl
|
877
|
+
data_stream_ilm_name foo_ilm_policy
|
878
|
+
include_tag_key true
|
879
|
+
tag_key custom_tag_key
|
880
|
+
}
|
881
|
+
|
882
|
+
stub_default
|
883
|
+
stub_bulk_feed
|
884
|
+
driver(config)
|
885
|
+
driver.run(default_tag: 'mytag') do
|
886
|
+
driver.feed(sample_record)
|
887
|
+
end
|
888
|
+
|
889
|
+
assert_equal(1, @bulk_records.length)
|
890
|
+
assert(@bulk_records[0].has_key?('custom_tag_key'))
|
891
|
+
assert_equal('mytag', @bulk_records[0]['custom_tag_key'])
|
892
|
+
end
|
799
893
|
end
|
@@ -16,7 +16,15 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
16
16
|
@driver = nil
|
17
17
|
end
|
18
18
|
|
19
|
-
def
|
19
|
+
def elasticsearch_version
|
20
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
21
|
+
TRANSPORT_CLASS::VERSION
|
22
|
+
else
|
23
|
+
'6.4.2'.freeze
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def driver(conf='', es_version=elasticsearch_version.to_i)
|
20
28
|
# For request stub to detect compatibility.
|
21
29
|
@es_version ||= es_version
|
22
30
|
Fluent::Plugin::ElasticsearchOutputDynamic.module_eval(<<-CODE)
|
@@ -35,7 +43,11 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
35
43
|
end
|
36
44
|
|
37
45
|
def elasticsearch_transport_layer_decoupling?
|
38
|
-
Gem::Version.create(::
|
46
|
+
Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
47
|
+
end
|
48
|
+
|
49
|
+
def elastic_transport_layer?
|
50
|
+
Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
39
51
|
end
|
40
52
|
|
41
53
|
def default_type_name
|
@@ -58,9 +70,9 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
58
70
|
end
|
59
71
|
end
|
60
72
|
|
61
|
-
def stub_elastic_info(url="http://localhost:9200/", version=
|
73
|
+
def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
|
62
74
|
body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
|
63
|
-
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
|
75
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
|
64
76
|
end
|
65
77
|
|
66
78
|
def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
|
@@ -132,7 +144,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
132
144
|
end
|
133
145
|
|
134
146
|
test 'configure compression' do
|
135
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
147
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
136
148
|
|
137
149
|
config = %{
|
138
150
|
compression_level best_compression
|
@@ -143,7 +155,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
143
155
|
end
|
144
156
|
|
145
157
|
test 'check compression strategy' do
|
146
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
158
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
147
159
|
|
148
160
|
config = %{
|
149
161
|
compression_level best_speed
|
@@ -154,14 +166,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
154
166
|
end
|
155
167
|
|
156
168
|
test 'check content-encoding header with compression' do
|
157
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
169
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
158
170
|
|
159
171
|
config = %{
|
160
172
|
compression_level best_compression
|
161
173
|
}
|
162
174
|
instance = driver(config).instance
|
163
175
|
|
164
|
-
if
|
176
|
+
if elastic_transport_layer?
|
177
|
+
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
178
|
+
elsif elasticsearch_transport_layer_decoupling?
|
165
179
|
assert_equal nil, instance.client.transport.transport.options[:transport_options][:headers]["Content-Encoding"]
|
166
180
|
else
|
167
181
|
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
@@ -175,7 +189,9 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
175
189
|
end
|
176
190
|
compressable = instance.compressable_connection
|
177
191
|
|
178
|
-
if
|
192
|
+
if elastic_transport_layer?
|
193
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
194
|
+
elsif elasticsearch_transport_layer_decoupling?
|
179
195
|
assert_equal "gzip", instance.client(nil, compressable).transport.transport.options[:transport_options][:headers]["Content-Encoding"]
|
180
196
|
else
|
181
197
|
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
@@ -183,14 +199,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
183
199
|
end
|
184
200
|
|
185
201
|
test 'check compression option is passed to transport' do
|
186
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
202
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
187
203
|
|
188
204
|
config = %{
|
189
205
|
compression_level best_compression
|
190
206
|
}
|
191
207
|
instance = driver(config).instance
|
192
208
|
|
193
|
-
if
|
209
|
+
if elastic_transport_layer?
|
210
|
+
assert_equal false, instance.client.transport.options[:compression]
|
211
|
+
elsif elasticsearch_transport_layer_decoupling?
|
194
212
|
assert_equal false, instance.client.transport.transport.options[:compression]
|
195
213
|
else
|
196
214
|
assert_equal false, instance.client.transport.options[:compression]
|
@@ -204,7 +222,9 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
204
222
|
end
|
205
223
|
compressable = instance.compressable_connection
|
206
224
|
|
207
|
-
if
|
225
|
+
if elastic_transport_layer?
|
226
|
+
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
227
|
+
elsif elasticsearch_transport_layer_decoupling?
|
208
228
|
assert_equal true, instance.client(nil, compressable).transport.transport.options[:compression]
|
209
229
|
else
|
210
230
|
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
@@ -402,7 +422,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
402
422
|
end
|
403
423
|
|
404
424
|
def test_writes_to_default_index_with_compression
|
405
|
-
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::
|
425
|
+
omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
|
406
426
|
|
407
427
|
config = %[
|
408
428
|
compression_level default_compression
|
@@ -441,7 +461,13 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
441
461
|
driver.run(default_tag: 'test') do
|
442
462
|
driver.feed(sample_record)
|
443
463
|
end
|
444
|
-
|
464
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
465
|
+
assert_nil(index_cmds.first['index']['_type'])
|
466
|
+
elsif Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.0.0")
|
467
|
+
assert_equal("_doc", index_cmds.first['index']['_type'])
|
468
|
+
else
|
469
|
+
assert_equal("fluentd", index_cmds.first['index']['_type'])
|
470
|
+
end
|
445
471
|
end
|
446
472
|
|
447
473
|
def test_writes_to_specified_index
|
@@ -471,7 +497,13 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
471
497
|
driver.run(default_tag: 'test') do
|
472
498
|
driver.feed(sample_record)
|
473
499
|
end
|
474
|
-
|
500
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
501
|
+
assert_nil(index_cmds.first['index']['_type'])
|
502
|
+
elsif Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.0.0")
|
503
|
+
assert_equal("_doc", index_cmds.first['index']['_type'])
|
504
|
+
else
|
505
|
+
assert_equal("mytype", index_cmds.first['index']['_type'])
|
506
|
+
end
|
475
507
|
end
|
476
508
|
|
477
509
|
def test_writes_to_specified_host
|
@@ -971,7 +1003,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
971
1003
|
driver.run(default_tag: 'test') do
|
972
1004
|
driver.feed(nested_sample_record)
|
973
1005
|
end
|
974
|
-
|
1006
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1007
|
+
assert_equal('routing', index_cmds[0]['index'][routing_key])
|
975
1008
|
end
|
976
1009
|
|
977
1010
|
def test_adds_nested_routing_key_with_dollar_dot
|
@@ -981,7 +1014,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
981
1014
|
driver.run(default_tag: 'test') do
|
982
1015
|
driver.feed(nested_sample_record)
|
983
1016
|
end
|
984
|
-
|
1017
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1018
|
+
assert_equal('routing', index_cmds[0]['index'][routing_key])
|
985
1019
|
end
|
986
1020
|
|
987
1021
|
def test_adds_nested_routing_key_with_bracket
|
@@ -991,7 +1025,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
991
1025
|
driver.run(default_tag: 'test') do
|
992
1026
|
driver.feed(nested_sample_record)
|
993
1027
|
end
|
994
|
-
|
1028
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1029
|
+
assert_equal('routing', index_cmds[0]['index'][routing_key])
|
995
1030
|
end
|
996
1031
|
end
|
997
1032
|
|
@@ -1002,7 +1037,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
1002
1037
|
driver.run(default_tag: 'test') do
|
1003
1038
|
driver.feed(sample_record)
|
1004
1039
|
end
|
1005
|
-
|
1040
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1041
|
+
assert(!index_cmds[0]['index'].has_key?(routing_key))
|
1006
1042
|
end
|
1007
1043
|
|
1008
1044
|
def test_adds_routing_key_when_not_configured
|
@@ -1011,7 +1047,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
1011
1047
|
driver.run(default_tag: 'test') do
|
1012
1048
|
driver.feed(sample_record)
|
1013
1049
|
end
|
1014
|
-
|
1050
|
+
routing_key = driver.instance.instance_variable_get(:@routing_key_name)
|
1051
|
+
assert(!index_cmds[0]['index'].has_key?(routing_key))
|
1015
1052
|
end
|
1016
1053
|
|
1017
1054
|
def test_remove_one_key
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.
|
4
|
+
version: 5.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2022-
|
13
|
+
date: 2022-04-11 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|
@@ -171,6 +171,7 @@ files:
|
|
171
171
|
- gemfiles/Gemfile.elasticsearch.v6
|
172
172
|
- lib/fluent/log-ext.rb
|
173
173
|
- lib/fluent/plugin/default-ilm-policy.json
|
174
|
+
- lib/fluent/plugin/elasticsearch_compat.rb
|
174
175
|
- lib/fluent/plugin/elasticsearch_constants.rb
|
175
176
|
- lib/fluent/plugin/elasticsearch_error.rb
|
176
177
|
- lib/fluent/plugin/elasticsearch_error_handler.rb
|
@@ -221,7 +222,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
221
222
|
- !ruby/object:Gem::Version
|
222
223
|
version: '0'
|
223
224
|
requirements: []
|
224
|
-
rubygems_version: 3.2.
|
225
|
+
rubygems_version: 3.2.32
|
225
226
|
signing_key:
|
226
227
|
specification_version: 4
|
227
228
|
summary: Elasticsearch output plugin for Fluent event collector
|