fluent-plugin-elasticsearch 4.0.5 → 4.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/.github/workflows/linux.yml +26 -0
- data/.github/workflows/macos.yml +26 -0
- data/.github/workflows/windows.yml +26 -0
- data/.travis.yml +1 -1
- data/Gemfile +2 -1
- data/History.md +27 -0
- data/README.ElasticsearchGenID.md +116 -0
- data/README.md +99 -28
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/gemfiles/Gemfile.elasticsearch.v6 +1 -1
- data/gemfiles/{Gemfile.ilm → Gemfile.without.ilm} +1 -2
- data/lib/fluent/plugin/elasticsearch_index_template.rb +1 -1
- data/lib/fluent/plugin/elasticsearch_tls.rb +3 -3
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +52 -0
- data/lib/fluent/plugin/out_elasticsearch.rb +76 -36
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +6 -4
- data/test/plugin/test_elasticsearch_error_handler.rb +1 -1
- data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +1 -1
- data/test/plugin/test_elasticsearch_tls.rb +2 -2
- data/test/plugin/test_filter_elasticsearch_genid.rb +172 -1
- data/test/plugin/test_in_elasticsearch.rb +1 -1
- data/test/plugin/test_out_elasticsearch.rb +329 -6
- data/test/plugin/test_out_elasticsearch_dynamic.rb +23 -5
- metadata +8 -3
@@ -62,8 +62,8 @@ class TestElasticsearchTLS < Test::Unit::TestCase
|
|
62
62
|
d = driver('')
|
63
63
|
ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
|
64
64
|
if @use_tls_minmax_version
|
65
|
-
assert_equal({max_version: OpenSSL::SSL::
|
66
|
-
min_version: OpenSSL::SSL::
|
65
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
|
66
|
+
min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
|
67
67
|
else
|
68
68
|
assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
|
69
69
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
|
1
|
+
require_relative '../helper'
|
2
2
|
require 'date'
|
3
3
|
require 'fluent/test/helpers'
|
4
4
|
require 'json'
|
@@ -18,6 +18,12 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
|
|
18
18
|
Fluent::Test::Driver::Filter.new(Fluent::Plugin::ElasticsearchGenidFilter).configure(conf)
|
19
19
|
end
|
20
20
|
|
21
|
+
test "invalid configuration" do
|
22
|
+
assert_raise(Fluent::ConfigError) do
|
23
|
+
create_driver("use_record_as_seed true")
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
21
27
|
def sample_record
|
22
28
|
{'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
|
23
29
|
end
|
@@ -41,4 +47,169 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
|
|
41
47
|
assert_equal(Base64.strict_encode64(SecureRandom.uuid),
|
42
48
|
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
43
49
|
end
|
50
|
+
|
51
|
+
class UseRecordAsSeedTest < self
|
52
|
+
data("md5" => ["md5", "PPg+zmH1ASUCpNzMUcTzqw=="],
|
53
|
+
"sha1" => ["sha1", "JKfCrEAxeAyRSdcKqkw4unC9xZ8="],
|
54
|
+
"sha256" => ["sha256", "9Z9i+897bGivSItD/6i0vye9uRwq/sLwWkxOwydtTJY="],
|
55
|
+
"sha512" => ["sha512", "KWI5OdZPaCFW9/CEY3NoGrvueMtjZJdmGdqIVGJP8vgI4uW+0gHExZVaHerw+RhbtIdLCtVZ43xBgMKH+KliQg=="],
|
56
|
+
)
|
57
|
+
def test_simple(data)
|
58
|
+
hash_type, expected = data
|
59
|
+
d = create_driver(%[
|
60
|
+
use_record_as_seed true
|
61
|
+
record_keys age,parent_id,routing_id,custom_key
|
62
|
+
hash_type #{hash_type}
|
63
|
+
])
|
64
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
65
|
+
d.run(default_tag: 'test') do
|
66
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
67
|
+
end
|
68
|
+
assert_equal(expected,
|
69
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
70
|
+
end
|
71
|
+
|
72
|
+
data("md5" => ["md5", "qUO/xqWiOJq4D0ApdoHVEQ=="],
|
73
|
+
"sha1" => ["sha1", "v3UWYr90zIH2veGQBVwUH586TuI="],
|
74
|
+
"sha256" => ["sha256", "4hwh10qfw9B24NtNFoEFF8wCiImvgIy1Vk4gzcKt5Pw="],
|
75
|
+
"sha512" => ["sha512", "TY3arcmC8mhYClDIjQxH8ePRLnHK01Cj5QQL8FxbwNtPQBY3IZ4qJY9CpOusmdWBYwm1golRVQCmURiAhlnWIQ=="],)
|
76
|
+
def test_record_with_tag(data)
|
77
|
+
hash_type, expected = data
|
78
|
+
d = create_driver(%[
|
79
|
+
use_record_as_seed true
|
80
|
+
record_keys age,parent_id,routing_id,custom_key
|
81
|
+
hash_type #{hash_type}
|
82
|
+
include_tag_in_seed true
|
83
|
+
])
|
84
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
85
|
+
d.run(default_tag: 'test.fluentd') do
|
86
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
87
|
+
end
|
88
|
+
assert_equal(expected,
|
89
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
90
|
+
end
|
91
|
+
|
92
|
+
data("md5" => ["md5", "oHo+PoC5I4KC+XCfXvyf9w=="],
|
93
|
+
"sha1" => ["sha1", "50Nwarm2225gLy1ka8d9i+W6cKA="],
|
94
|
+
"sha256" => ["sha256", "ReX1XgizcrHjBc0sQwx9Sjuf2QBFll2njYf4ee+XSIc="],
|
95
|
+
"sha512" => ["sha512", "8bcpZrqNUQIz6opdoVZz0MwxP8r9SCqOEPkWF6xGLlFwPCJVqk2SQp99m8rPufr0xPIgvZyOMejA5slBV9xrdg=="],)
|
96
|
+
def test_record_with_time(data)
|
97
|
+
hash_type, expected = data
|
98
|
+
d = create_driver(%[
|
99
|
+
use_record_as_seed true
|
100
|
+
record_keys age,parent_id,routing_id,custom_key
|
101
|
+
hash_type #{hash_type}
|
102
|
+
include_time_in_seed true
|
103
|
+
])
|
104
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
105
|
+
d.run(default_tag: 'test.fluentd') do
|
106
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
107
|
+
end
|
108
|
+
assert_equal(expected,
|
109
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
110
|
+
end
|
111
|
+
|
112
|
+
data("md5" => ["md5", "u7/hr09gDC9CM5DI7tLc2Q=="],
|
113
|
+
"sha1" => ["sha1", "1WgptcTnVSHtTAlNUwNcoiaY3oM="],
|
114
|
+
"sha256" => ["sha256", "1iWZHI19m/A1VH8iFK7H2KFoyLdszpJRiVeKBv1Ndis="],
|
115
|
+
"sha512" => ["sha512", "NM+ui0lUmeDaEJsT7c9EyTc+lQBbRf1x6MQXXYdxp21CX3jZvHy3IT8Xp9ZdIKevZwhoo3Suo/tIBlfyLFXJXw=="],)
|
116
|
+
def test_record_with_tag_and_time
|
117
|
+
hash_type, expected = data
|
118
|
+
d = create_driver(%[
|
119
|
+
use_record_as_seed true
|
120
|
+
record_keys age,parent_id,routing_id,custom_key
|
121
|
+
hash_type #{hash_type}
|
122
|
+
include_tag_in_seed true
|
123
|
+
include_time_in_seed true
|
124
|
+
])
|
125
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
126
|
+
d.run(default_tag: 'test.fluentd') do
|
127
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
128
|
+
end
|
129
|
+
assert_equal(expected,
|
130
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
class UseEntireRecordAsSeedTest < self
|
135
|
+
data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
|
136
|
+
"sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
|
137
|
+
"sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
|
138
|
+
"sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
|
139
|
+
def test_record
|
140
|
+
hash_type, expected = data
|
141
|
+
d = create_driver(%[
|
142
|
+
use_record_as_seed true
|
143
|
+
use_entire_record true
|
144
|
+
hash_type #{hash_type}
|
145
|
+
])
|
146
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
147
|
+
d.run(default_tag: 'test.fluentd') do
|
148
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
149
|
+
end
|
150
|
+
assert_equal(expected,
|
151
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
152
|
+
end
|
153
|
+
|
154
|
+
data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
|
155
|
+
"sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
|
156
|
+
"sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
|
157
|
+
"sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
|
158
|
+
def test_record_with_tag
|
159
|
+
hash_type, expected = data
|
160
|
+
d = create_driver(%[
|
161
|
+
use_record_as_seed true
|
162
|
+
use_entire_record true
|
163
|
+
hash_type #{hash_type}
|
164
|
+
include_tag_in_seed true
|
165
|
+
])
|
166
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
167
|
+
d.run(default_tag: 'test.fluentd') do
|
168
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
169
|
+
end
|
170
|
+
assert_equal(expected,
|
171
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
172
|
+
end
|
173
|
+
|
174
|
+
data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
|
175
|
+
"sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
|
176
|
+
"sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
|
177
|
+
"sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
|
178
|
+
def test_record_with_time
|
179
|
+
hash_type, expected = data
|
180
|
+
d = create_driver(%[
|
181
|
+
use_record_as_seed true
|
182
|
+
use_entire_record true
|
183
|
+
hash_type #{hash_type}
|
184
|
+
include_time_in_seed true
|
185
|
+
])
|
186
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
187
|
+
d.run(default_tag: 'test.fluentd') do
|
188
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
189
|
+
end
|
190
|
+
assert_equal(expected,
|
191
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
192
|
+
end
|
193
|
+
|
194
|
+
data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
|
195
|
+
"sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
|
196
|
+
"sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
|
197
|
+
"sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
|
198
|
+
def test_record_with_tag_and_time
|
199
|
+
hash_type, expected = data
|
200
|
+
d = create_driver(%[
|
201
|
+
use_record_as_seed true
|
202
|
+
use_entire_record true
|
203
|
+
hash_type #{hash_type}
|
204
|
+
include_tag_in_seed true
|
205
|
+
include_time_in_seed true
|
206
|
+
])
|
207
|
+
time = event_time("2017-10-15 15:00:23.34567890 UTC")
|
208
|
+
d.run(default_tag: 'test.fluentd') do
|
209
|
+
d.feed(time, sample_record.merge("custom_key" => "This is also encoded value."))
|
210
|
+
end
|
211
|
+
assert_equal(expected,
|
212
|
+
d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
|
213
|
+
end
|
214
|
+
end
|
44
215
|
end
|
@@ -4,6 +4,7 @@ require 'fluent/test/helpers'
|
|
4
4
|
require 'json'
|
5
5
|
require 'fluent/test/driver/input'
|
6
6
|
require 'flexmock/test_unit'
|
7
|
+
require 'fluent/plugin/in_elasticsearch'
|
7
8
|
|
8
9
|
class ElasticsearchInputTest < Test::Unit::TestCase
|
9
10
|
include FlexMock::TestCase
|
@@ -16,7 +17,6 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
16
17
|
|
17
18
|
def setup
|
18
19
|
Fluent::Test.setup
|
19
|
-
require 'fluent/plugin/in_elasticsearch'
|
20
20
|
@driver = nil
|
21
21
|
log = Fluent::Engine.log
|
22
22
|
log.out.logs.slice!(0, log.out.logs.length)
|
@@ -1,11 +1,12 @@
|
|
1
|
-
|
1
|
+
require_relative '../helper'
|
2
2
|
require 'date'
|
3
3
|
require 'fluent/test/helpers'
|
4
4
|
require 'json'
|
5
5
|
require 'fluent/test/driver/output'
|
6
6
|
require 'flexmock/test_unit'
|
7
|
+
require 'fluent/plugin/out_elasticsearch'
|
7
8
|
|
8
|
-
class
|
9
|
+
class ElasticsearchOutputTest < Test::Unit::TestCase
|
9
10
|
include FlexMock::TestCase
|
10
11
|
include Fluent::Test::Helpers
|
11
12
|
|
@@ -13,7 +14,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
13
14
|
|
14
15
|
def setup
|
15
16
|
Fluent::Test.setup
|
16
|
-
require 'fluent/plugin/out_elasticsearch'
|
17
17
|
@driver = nil
|
18
18
|
log = Fluent::Engine.log
|
19
19
|
log.out.logs.slice!(0, log.out.logs.length)
|
@@ -230,7 +230,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
230
230
|
assert_nil instance.ssl_max_version
|
231
231
|
assert_nil instance.ssl_min_version
|
232
232
|
if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
|
233
|
-
assert_equal({max_version: OpenSSL::SSL::
|
233
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
|
234
234
|
instance.ssl_version_options)
|
235
235
|
else
|
236
236
|
assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
|
@@ -251,6 +251,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
251
251
|
assert_equal 20 * 1024 * 1024, Fluent::Plugin::ElasticsearchOutput::TARGET_BULK_BYTES
|
252
252
|
assert_false instance.compression
|
253
253
|
assert_equal :no_compression, instance.compression_level
|
254
|
+
assert_true instance.http_backend_excon_nonblock
|
254
255
|
end
|
255
256
|
|
256
257
|
test 'configure compression' do
|
@@ -283,7 +284,16 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
283
284
|
}
|
284
285
|
instance = driver(config).instance
|
285
286
|
|
286
|
-
assert_equal
|
287
|
+
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
288
|
+
|
289
|
+
stub_request(:post, "http://localhost:9200/_bulk").
|
290
|
+
to_return(status: 200, body: "", headers: {})
|
291
|
+
driver.run(default_tag: 'test') do
|
292
|
+
driver.feed(sample_record)
|
293
|
+
end
|
294
|
+
compressable = instance.compressable_connection
|
295
|
+
|
296
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
287
297
|
end
|
288
298
|
|
289
299
|
test 'check compression option is passed to transport' do
|
@@ -294,7 +304,16 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
294
304
|
}
|
295
305
|
instance = driver(config).instance
|
296
306
|
|
297
|
-
assert_equal
|
307
|
+
assert_equal false, instance.client.transport.options[:compression]
|
308
|
+
|
309
|
+
stub_request(:post, "http://localhost:9200/_bulk").
|
310
|
+
to_return(status: 200, body: "", headers: {})
|
311
|
+
driver.run(default_tag: 'test') do
|
312
|
+
driver.feed(sample_record)
|
313
|
+
end
|
314
|
+
compressable = instance.compressable_connection
|
315
|
+
|
316
|
+
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
298
317
|
end
|
299
318
|
|
300
319
|
test 'configure Content-Type' do
|
@@ -332,6 +351,20 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
332
351
|
}
|
333
352
|
end
|
334
353
|
|
354
|
+
sub_test_case 'Check TLS handshake stuck warning log' do
|
355
|
+
test 'warning TLS log' do
|
356
|
+
config = %{
|
357
|
+
scheme https
|
358
|
+
http_backend_excon_nonblock false
|
359
|
+
ssl_version TLSv1_2
|
360
|
+
@log_level info
|
361
|
+
}
|
362
|
+
driver(config)
|
363
|
+
logs = driver.logs
|
364
|
+
assert_logs_include(logs, /TLS handshake will be stucked with block connection.\n Consider to set `http_backend_excon_nonblock` as true\n/)
|
365
|
+
end
|
366
|
+
end
|
367
|
+
|
335
368
|
sub_test_case 'ILM default config' do
|
336
369
|
setup do
|
337
370
|
begin
|
@@ -535,6 +568,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
535
568
|
"template_name_placeholder" => ["template_name", "logstash-${mykey}"],
|
536
569
|
"customize_template" => ["customize_template", '{"<<TAG>>":"${mykey}"}'],
|
537
570
|
"logstash_prefix_placeholder" => ["logstash_prefix", "fluentd-${mykey}"],
|
571
|
+
"logstash_dateformat_placeholder" => ["logstash_dateformat", "${mykey}"],
|
538
572
|
"deflector_alias_placeholder" => ["deflector_alias", "fluentd-${mykey}"],
|
539
573
|
"application_name_placeholder" => ["application_name", "fluentd-${mykey}"],
|
540
574
|
)
|
@@ -560,6 +594,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
560
594
|
"template_name_placeholder" => ["template_name", "logstash-${tag}-%Y%m%d"],
|
561
595
|
"customize_template" => ["customize_template", '{"<<TAG>>":"${es_index}"}'],
|
562
596
|
"logstash_prefix_placeholder" => ["logstash_prefix", "fluentd-${es_index}-%Y%m%d"],
|
597
|
+
"logstash_dataformat_placeholder" => ["logstash_dateformat", "${es_index}"],
|
563
598
|
"deflector_alias_placeholder" => ["deflector_alias", "fluentd-%Y%m%d"],
|
564
599
|
"application_name_placeholder" => ["application_name", "fluentd-${tag}-${es_index}-%Y%m%d"],
|
565
600
|
)
|
@@ -710,6 +745,60 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
710
745
|
end
|
711
746
|
end
|
712
747
|
|
748
|
+
class GetElasticsearchVersionWithFallbackTest < self
|
749
|
+
def create_driver(conf='', client_version="\"5.0\"")
|
750
|
+
# For request stub to detect compatibility.
|
751
|
+
@client_version ||= client_version
|
752
|
+
# Ensure original implementation existence.
|
753
|
+
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
754
|
+
def detect_es_major_version
|
755
|
+
@_es_info ||= client.info
|
756
|
+
@_es_info["version"]["number"].to_i
|
757
|
+
end
|
758
|
+
CODE
|
759
|
+
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
760
|
+
def client_library_version
|
761
|
+
#{@client_version}
|
762
|
+
end
|
763
|
+
CODE
|
764
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput).configure(conf)
|
765
|
+
end
|
766
|
+
|
767
|
+
data("Elasticsearch 5" => ["5.0", 5],
|
768
|
+
"Elasticsearch 6" => ["6.0", 6],
|
769
|
+
"Elasticsearch 7" => ["7.0", 7],
|
770
|
+
"Elasticsearch 8" => ["8.0", 8])
|
771
|
+
def test_retry_get_es_version_without_fail_on_detecting_es_version_retry_exceeded(data)
|
772
|
+
client_version, es_major_version = data
|
773
|
+
config = %{
|
774
|
+
host logs.google.com
|
775
|
+
port 778
|
776
|
+
scheme https
|
777
|
+
path /es/
|
778
|
+
user john
|
779
|
+
password doe
|
780
|
+
verify_es_version_at_startup true
|
781
|
+
max_retry_get_es_version 2
|
782
|
+
fail_on_detecting_es_version_retry_exceed false
|
783
|
+
default_elasticsearch_version #{es_major_version}
|
784
|
+
@log_level info
|
785
|
+
}
|
786
|
+
|
787
|
+
connection_resets = 0
|
788
|
+
stub_request(:get, "https://logs.google.com:778/es//").
|
789
|
+
with(basic_auth: ['john', 'doe']) do |req|
|
790
|
+
connection_resets += 1
|
791
|
+
raise Faraday::ConnectionFailed, "Test message"
|
792
|
+
end
|
793
|
+
|
794
|
+
d = create_driver(config, client_version)
|
795
|
+
|
796
|
+
assert_equal es_major_version, d.instance.default_elasticsearch_version
|
797
|
+
assert_equal 3, connection_resets
|
798
|
+
assert_equal es_major_version, d.instance.instance_variable_get(:@last_seen_major_version)
|
799
|
+
end
|
800
|
+
end
|
801
|
+
|
713
802
|
def test_template_already_present
|
714
803
|
config = %{
|
715
804
|
host logs.google.com
|
@@ -1228,6 +1317,224 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
1228
1317
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1229
1318
|
end
|
1230
1319
|
|
1320
|
+
def test_template_create_with_rollover_index_and_ilm_policies_and_placeholders
|
1321
|
+
cwd = File.dirname(__FILE__)
|
1322
|
+
template_file = File.join(cwd, 'test_template.json')
|
1323
|
+
|
1324
|
+
config = %{
|
1325
|
+
host logs.google.com
|
1326
|
+
port 777
|
1327
|
+
scheme https
|
1328
|
+
path /es/
|
1329
|
+
user john
|
1330
|
+
password doe
|
1331
|
+
template_name logstash
|
1332
|
+
template_file #{template_file}
|
1333
|
+
index_date_pattern now/w{xxxx.ww}
|
1334
|
+
ilm_policy_id fluentd-policy
|
1335
|
+
enable_ilm true
|
1336
|
+
index_name logstash
|
1337
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
1338
|
+
}
|
1339
|
+
|
1340
|
+
# connection start
|
1341
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
1342
|
+
with(basic_auth: ['john', 'doe']).
|
1343
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1344
|
+
# check if template exists
|
1345
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1346
|
+
with(basic_auth: ['john', 'doe']).
|
1347
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1348
|
+
# creation
|
1349
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1350
|
+
with(basic_auth: ['john', 'doe']).
|
1351
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1352
|
+
# check if alias exists
|
1353
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1354
|
+
with(basic_auth: ['john', 'doe']).
|
1355
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1356
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1357
|
+
with(basic_auth: ['john', 'doe']).
|
1358
|
+
to_return(status: 404, body: "", headers: {})
|
1359
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1360
|
+
with(basic_auth: ['john', 'doe'],
|
1361
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1362
|
+
to_return(status: 200, body: "", headers: {})
|
1363
|
+
# put the alias for the index
|
1364
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1365
|
+
with(basic_auth: ['john', 'doe']).
|
1366
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1367
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
1368
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
1369
|
+
to_return(status: 200, body: "", headers: {})
|
1370
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1371
|
+
with(basic_auth: ['john', 'doe']).
|
1372
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1373
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
1374
|
+
with(basic_auth: ['john', 'doe']).
|
1375
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1376
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
1377
|
+
with(basic_auth: ['john', 'doe'],
|
1378
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
1379
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1380
|
+
|
1381
|
+
driver(config)
|
1382
|
+
|
1383
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1384
|
+
driver.run(default_tag: 'test') do
|
1385
|
+
driver.feed(sample_record)
|
1386
|
+
end
|
1387
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1388
|
+
|
1389
|
+
assert_requested(elastic_request)
|
1390
|
+
end
|
1391
|
+
|
1392
|
+
class TemplateCreateWithRolloverIndexAndILMPoliciesWithPlaceholdersTest < self
|
1393
|
+
def test_tag_placeholder
|
1394
|
+
cwd = File.dirname(__FILE__)
|
1395
|
+
template_file = File.join(cwd, 'test_template.json')
|
1396
|
+
|
1397
|
+
config = %{
|
1398
|
+
host logs.google.com
|
1399
|
+
port 777
|
1400
|
+
scheme https
|
1401
|
+
path /es/
|
1402
|
+
user john
|
1403
|
+
password doe
|
1404
|
+
template_name logstash
|
1405
|
+
template_file #{template_file}
|
1406
|
+
index_date_pattern now/w{xxxx.ww}
|
1407
|
+
ilm_policy_id ${tag}
|
1408
|
+
enable_ilm true
|
1409
|
+
index_name logstash
|
1410
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
1411
|
+
}
|
1412
|
+
|
1413
|
+
# connection start
|
1414
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
1415
|
+
with(basic_auth: ['john', 'doe']).
|
1416
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1417
|
+
# check if template exists
|
1418
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1419
|
+
with(basic_auth: ['john', 'doe']).
|
1420
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1421
|
+
# creation
|
1422
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1423
|
+
with(basic_auth: ['john', 'doe']).
|
1424
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1425
|
+
# check if alias exists
|
1426
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1427
|
+
with(basic_auth: ['john', 'doe']).
|
1428
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1429
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1430
|
+
with(basic_auth: ['john', 'doe']).
|
1431
|
+
to_return(status: 404, body: "", headers: {})
|
1432
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1433
|
+
with(basic_auth: ['john', 'doe'],
|
1434
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1435
|
+
to_return(status: 200, body: "", headers: {})
|
1436
|
+
# put the alias for the index
|
1437
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1438
|
+
with(basic_auth: ['john', 'doe']).
|
1439
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1440
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
1441
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
1442
|
+
to_return(status: 200, body: "", headers: {})
|
1443
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1444
|
+
with(basic_auth: ['john', 'doe']).
|
1445
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1446
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
1447
|
+
with(basic_auth: ['john', 'doe']).
|
1448
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1449
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
1450
|
+
with(basic_auth: ['john', 'doe'],
|
1451
|
+
body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
1452
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1453
|
+
|
1454
|
+
driver(config)
|
1455
|
+
|
1456
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1457
|
+
driver.run(default_tag: 'fluentd-policy') do
|
1458
|
+
driver.feed(sample_record)
|
1459
|
+
end
|
1460
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1461
|
+
|
1462
|
+
assert_requested(elastic_request)
|
1463
|
+
end
|
1464
|
+
|
1465
|
+
def test_tag_placeholder_with_multiple_policies
|
1466
|
+
cwd = File.dirname(__FILE__)
|
1467
|
+
template_file = File.join(cwd, 'test_template.json')
|
1468
|
+
|
1469
|
+
config = %{
|
1470
|
+
host logs.google.com
|
1471
|
+
port 777
|
1472
|
+
scheme https
|
1473
|
+
path /es/
|
1474
|
+
user john
|
1475
|
+
password doe
|
1476
|
+
template_name logstash
|
1477
|
+
template_file #{template_file}
|
1478
|
+
index_date_pattern now/w{xxxx.ww}
|
1479
|
+
ilm_policy_id ${tag}
|
1480
|
+
enable_ilm true
|
1481
|
+
index_name logstash
|
1482
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}, "fluentd-policy2":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"80gb", "max_age":"20d"}}}}}}}
|
1483
|
+
}
|
1484
|
+
|
1485
|
+
# connection start
|
1486
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
1487
|
+
with(basic_auth: ['john', 'doe']).
|
1488
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1489
|
+
# check if template exists
|
1490
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1491
|
+
with(basic_auth: ['john', 'doe']).
|
1492
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1493
|
+
# creation
|
1494
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1495
|
+
with(basic_auth: ['john', 'doe']).
|
1496
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1497
|
+
# check if alias exists
|
1498
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1499
|
+
with(basic_auth: ['john', 'doe']).
|
1500
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1501
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1502
|
+
with(basic_auth: ['john', 'doe']).
|
1503
|
+
to_return(status: 404, body: "", headers: {})
|
1504
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1505
|
+
with(basic_auth: ['john', 'doe'],
|
1506
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1507
|
+
to_return(status: 200, body: "", headers: {})
|
1508
|
+
# put the alias for the index
|
1509
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1510
|
+
with(basic_auth: ['john', 'doe']).
|
1511
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1512
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
1513
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
1514
|
+
to_return(status: 200, body: "", headers: {})
|
1515
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1516
|
+
with(basic_auth: ['john', 'doe']).
|
1517
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1518
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
|
1519
|
+
with(basic_auth: ['john', 'doe']).
|
1520
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1521
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
|
1522
|
+
with(basic_auth: ['john', 'doe'],
|
1523
|
+
body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"80gb\",\"max_age\":\"20d\"}}}}}}").
|
1524
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1525
|
+
|
1526
|
+
driver(config)
|
1527
|
+
|
1528
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1529
|
+
driver.run(default_tag: 'fluentd-policy2') do
|
1530
|
+
driver.feed(sample_record)
|
1531
|
+
end
|
1532
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1533
|
+
|
1534
|
+
assert_requested(elastic_request)
|
1535
|
+
end
|
1536
|
+
end
|
1537
|
+
|
1231
1538
|
def test_template_create_with_rollover_index_and_default_ilm_and_placeholders
|
1232
1539
|
cwd = File.dirname(__FILE__)
|
1233
1540
|
template_file = File.join(cwd, 'test_template.json')
|
@@ -2840,6 +3147,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
2840
3147
|
|
2841
3148
|
data("border" => {"es_version" => 6, "_type" => "mytype"},
|
2842
3149
|
"fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
|
3150
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
2843
3151
|
)
|
2844
3152
|
def test_writes_to_speficied_type(data)
|
2845
3153
|
driver('', data["es_version"]).configure("type_name mytype\n")
|
@@ -2852,6 +3160,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
2852
3160
|
|
2853
3161
|
data("border" => {"es_version" => 6, "_type" => "mytype.test"},
|
2854
3162
|
"fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
|
3163
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
2855
3164
|
)
|
2856
3165
|
def test_writes_to_speficied_type_with_placeholders(data)
|
2857
3166
|
driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
|
@@ -2862,6 +3171,20 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
2862
3171
|
assert_equal(data['_type'], index_cmds.first['index']['_type'])
|
2863
3172
|
end
|
2864
3173
|
|
3174
|
+
data("border" => {"es_version" => 6, "_type" => "mytype.test"},
|
3175
|
+
"fixed_behavior"=> {"es_version" => 7, "_type" => nil},
|
3176
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
3177
|
+
)
|
3178
|
+
def test_writes_to_speficied_type_with_suppress_type_name(data)
|
3179
|
+
driver('', data["es_version"])
|
3180
|
+
.configure("type_name mytype.${tag}\nsuppress_type_name true")
|
3181
|
+
stub_elastic
|
3182
|
+
driver.run(default_tag: 'test') do
|
3183
|
+
driver.feed(sample_record)
|
3184
|
+
end
|
3185
|
+
assert_equal(data['_type'], index_cmds.first['index']['_type'])
|
3186
|
+
end
|
3187
|
+
|
2865
3188
|
data("old" => {"es_version" => 2, "_type" => "local-override"},
|
2866
3189
|
"old_behavior" => {"es_version" => 5, "_type" => "local-override"},
|
2867
3190
|
"border" => {"es_version" => 6, "_type" => "fluentd"},
|