fluent-plugin-elasticsearch 4.0.6 → 4.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/.github/workflows/linux.yml +26 -0
- data/.github/workflows/macos.yml +26 -0
- data/.github/workflows/windows.yml +26 -0
- data/History.md +25 -0
- data/README.ElasticsearchGenID.md +116 -0
- data/README.md +81 -3
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_tls.rb +3 -3
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +52 -0
- data/lib/fluent/plugin/out_elasticsearch.rb +67 -32
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +6 -4
- data/test/plugin/test_elasticsearch_tls.rb +2 -2
- data/test/plugin/test_filter_elasticsearch_genid.rb +171 -0
- data/test/plugin/test_out_elasticsearch.rb +423 -5
- data/test/plugin/test_out_elasticsearch_dynamic.rb +21 -3
- metadata +7 -2
@@ -230,7 +230,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
230
230
|
assert_nil instance.ssl_max_version
|
231
231
|
assert_nil instance.ssl_min_version
|
232
232
|
if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
|
233
|
-
assert_equal({max_version: OpenSSL::SSL::
|
233
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
|
234
234
|
instance.ssl_version_options)
|
235
235
|
else
|
236
236
|
assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
|
@@ -251,6 +251,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
251
251
|
assert_equal 20 * 1024 * 1024, Fluent::Plugin::ElasticsearchOutput::TARGET_BULK_BYTES
|
252
252
|
assert_false instance.compression
|
253
253
|
assert_equal :no_compression, instance.compression_level
|
254
|
+
assert_true instance.http_backend_excon_nonblock
|
254
255
|
end
|
255
256
|
|
256
257
|
test 'configure compression' do
|
@@ -283,7 +284,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
283
284
|
}
|
284
285
|
instance = driver(config).instance
|
285
286
|
|
286
|
-
assert_equal
|
287
|
+
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
288
|
+
|
289
|
+
stub_request(:post, "http://localhost:9200/_bulk").
|
290
|
+
to_return(status: 200, body: "", headers: {})
|
291
|
+
driver.run(default_tag: 'test') do
|
292
|
+
driver.feed(sample_record)
|
293
|
+
end
|
294
|
+
compressable = instance.compressable_connection
|
295
|
+
|
296
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
287
297
|
end
|
288
298
|
|
289
299
|
test 'check compression option is passed to transport' do
|
@@ -294,7 +304,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
294
304
|
}
|
295
305
|
instance = driver(config).instance
|
296
306
|
|
297
|
-
assert_equal
|
307
|
+
assert_equal false, instance.client.transport.options[:compression]
|
308
|
+
|
309
|
+
stub_request(:post, "http://localhost:9200/_bulk").
|
310
|
+
to_return(status: 200, body: "", headers: {})
|
311
|
+
driver.run(default_tag: 'test') do
|
312
|
+
driver.feed(sample_record)
|
313
|
+
end
|
314
|
+
compressable = instance.compressable_connection
|
315
|
+
|
316
|
+
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
298
317
|
end
|
299
318
|
|
300
319
|
test 'configure Content-Type' do
|
@@ -332,6 +351,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
332
351
|
}
|
333
352
|
end
|
334
353
|
|
354
|
+
sub_test_case 'Check TLS handshake stuck warning log' do
|
355
|
+
test 'warning TLS log' do
|
356
|
+
config = %{
|
357
|
+
scheme https
|
358
|
+
http_backend_excon_nonblock false
|
359
|
+
ssl_version TLSv1_2
|
360
|
+
@log_level info
|
361
|
+
}
|
362
|
+
driver(config)
|
363
|
+
logs = driver.logs
|
364
|
+
assert_logs_include(logs, /TLS handshake will be stucked with block connection.\n Consider to set `http_backend_excon_nonblock` as true\n/)
|
365
|
+
end
|
366
|
+
end
|
367
|
+
|
335
368
|
sub_test_case 'ILM default config' do
|
336
369
|
setup do
|
337
370
|
begin
|
@@ -1020,6 +1053,80 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1020
1053
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1021
1054
|
end
|
1022
1055
|
|
1056
|
+
def test_template_create_with_rollover_index_and_default_ilm_on_logstash_format
|
1057
|
+
cwd = File.dirname(__FILE__)
|
1058
|
+
template_file = File.join(cwd, 'test_template.json')
|
1059
|
+
|
1060
|
+
config = %{
|
1061
|
+
host logs.google.com
|
1062
|
+
port 777
|
1063
|
+
scheme https
|
1064
|
+
path /es/
|
1065
|
+
user john
|
1066
|
+
password doe
|
1067
|
+
template_name logstash
|
1068
|
+
template_file #{template_file}
|
1069
|
+
index_date_pattern now/w{xxxx.ww}
|
1070
|
+
enable_ilm true
|
1071
|
+
logstash_format true
|
1072
|
+
application_name log
|
1073
|
+
}
|
1074
|
+
|
1075
|
+
date_str = Time.now.strftime("%Y.%m.%d")
|
1076
|
+
# connection start
|
1077
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
1078
|
+
with(basic_auth: ['john', 'doe']).
|
1079
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1080
|
+
# check if template exists
|
1081
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
|
1082
|
+
with(basic_auth: ['john', 'doe']).
|
1083
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1084
|
+
# creation
|
1085
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
|
1086
|
+
with(basic_auth: ['john', 'doe']).
|
1087
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1088
|
+
# check if alias exists
|
1089
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-log-#{date_str}").
|
1090
|
+
with(basic_auth: ['john', 'doe']).
|
1091
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1092
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
|
1093
|
+
with(basic_auth: ['john', 'doe']).
|
1094
|
+
to_return(status: 404, body: "", headers: {})
|
1095
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
|
1096
|
+
with(basic_auth: ['john', 'doe'],
|
1097
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-log-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-log-#{date_str}-*\",\"order\":53}").
|
1098
|
+
to_return(status: 200, body: "", headers: {})
|
1099
|
+
# put the alias for the index
|
1100
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1101
|
+
with(basic_auth: ['john', 'doe']).
|
1102
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1103
|
+
|
1104
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash-log-#{date_str}").
|
1105
|
+
with(basic_auth: ['john', 'doe'],
|
1106
|
+
body: "{\"aliases\":{\"logstash-log-#{date_str}\":{\"is_write_index\":true}}}").
|
1107
|
+
to_return(status: 200, body: "", headers: {})
|
1108
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1109
|
+
with(basic_auth: ['john', 'doe']).
|
1110
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1111
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
1112
|
+
with(basic_auth: ['john', 'doe']).
|
1113
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1114
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
1115
|
+
with(basic_auth: ['john', 'doe'],
|
1116
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
1117
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1118
|
+
|
1119
|
+
driver(config)
|
1120
|
+
|
1121
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1122
|
+
driver.run(default_tag: 'test') do
|
1123
|
+
driver.feed(sample_record)
|
1124
|
+
end
|
1125
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}", times: 1)
|
1126
|
+
|
1127
|
+
assert_requested(elastic_request)
|
1128
|
+
end
|
1129
|
+
|
1023
1130
|
def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite
|
1024
1131
|
cwd = File.dirname(__FILE__)
|
1025
1132
|
template_file = File.join(cwd, 'test_template.json')
|
@@ -1284,6 +1391,224 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1284
1391
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1285
1392
|
end
|
1286
1393
|
|
1394
|
+
def test_template_create_with_rollover_index_and_ilm_policies_and_placeholders
|
1395
|
+
cwd = File.dirname(__FILE__)
|
1396
|
+
template_file = File.join(cwd, 'test_template.json')
|
1397
|
+
|
1398
|
+
config = %{
|
1399
|
+
host logs.google.com
|
1400
|
+
port 777
|
1401
|
+
scheme https
|
1402
|
+
path /es/
|
1403
|
+
user john
|
1404
|
+
password doe
|
1405
|
+
template_name logstash
|
1406
|
+
template_file #{template_file}
|
1407
|
+
index_date_pattern now/w{xxxx.ww}
|
1408
|
+
ilm_policy_id fluentd-policy
|
1409
|
+
enable_ilm true
|
1410
|
+
index_name logstash
|
1411
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
1412
|
+
}
|
1413
|
+
|
1414
|
+
# connection start
|
1415
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
1416
|
+
with(basic_auth: ['john', 'doe']).
|
1417
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1418
|
+
# check if template exists
|
1419
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1420
|
+
with(basic_auth: ['john', 'doe']).
|
1421
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1422
|
+
# creation
|
1423
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1424
|
+
with(basic_auth: ['john', 'doe']).
|
1425
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1426
|
+
# check if alias exists
|
1427
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1428
|
+
with(basic_auth: ['john', 'doe']).
|
1429
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1430
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1431
|
+
with(basic_auth: ['john', 'doe']).
|
1432
|
+
to_return(status: 404, body: "", headers: {})
|
1433
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1434
|
+
with(basic_auth: ['john', 'doe'],
|
1435
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1436
|
+
to_return(status: 200, body: "", headers: {})
|
1437
|
+
# put the alias for the index
|
1438
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1439
|
+
with(basic_auth: ['john', 'doe']).
|
1440
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1441
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
1442
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
1443
|
+
to_return(status: 200, body: "", headers: {})
|
1444
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1445
|
+
with(basic_auth: ['john', 'doe']).
|
1446
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1447
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
1448
|
+
with(basic_auth: ['john', 'doe']).
|
1449
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1450
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
1451
|
+
with(basic_auth: ['john', 'doe'],
|
1452
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
1453
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1454
|
+
|
1455
|
+
driver(config)
|
1456
|
+
|
1457
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1458
|
+
driver.run(default_tag: 'test') do
|
1459
|
+
driver.feed(sample_record)
|
1460
|
+
end
|
1461
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1462
|
+
|
1463
|
+
assert_requested(elastic_request)
|
1464
|
+
end
|
1465
|
+
|
1466
|
+
class TemplateCreateWithRolloverIndexAndILMPoliciesWithPlaceholdersTest < self
|
1467
|
+
def test_tag_placeholder
|
1468
|
+
cwd = File.dirname(__FILE__)
|
1469
|
+
template_file = File.join(cwd, 'test_template.json')
|
1470
|
+
|
1471
|
+
config = %{
|
1472
|
+
host logs.google.com
|
1473
|
+
port 777
|
1474
|
+
scheme https
|
1475
|
+
path /es/
|
1476
|
+
user john
|
1477
|
+
password doe
|
1478
|
+
template_name logstash
|
1479
|
+
template_file #{template_file}
|
1480
|
+
index_date_pattern now/w{xxxx.ww}
|
1481
|
+
ilm_policy_id ${tag}
|
1482
|
+
enable_ilm true
|
1483
|
+
index_name logstash
|
1484
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
1485
|
+
}
|
1486
|
+
|
1487
|
+
# connection start
|
1488
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
1489
|
+
with(basic_auth: ['john', 'doe']).
|
1490
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1491
|
+
# check if template exists
|
1492
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1493
|
+
with(basic_auth: ['john', 'doe']).
|
1494
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1495
|
+
# creation
|
1496
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1497
|
+
with(basic_auth: ['john', 'doe']).
|
1498
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1499
|
+
# check if alias exists
|
1500
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1501
|
+
with(basic_auth: ['john', 'doe']).
|
1502
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1503
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1504
|
+
with(basic_auth: ['john', 'doe']).
|
1505
|
+
to_return(status: 404, body: "", headers: {})
|
1506
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1507
|
+
with(basic_auth: ['john', 'doe'],
|
1508
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1509
|
+
to_return(status: 200, body: "", headers: {})
|
1510
|
+
# put the alias for the index
|
1511
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1512
|
+
with(basic_auth: ['john', 'doe']).
|
1513
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1514
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
1515
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
1516
|
+
to_return(status: 200, body: "", headers: {})
|
1517
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1518
|
+
with(basic_auth: ['john', 'doe']).
|
1519
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1520
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
1521
|
+
with(basic_auth: ['john', 'doe']).
|
1522
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1523
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
1524
|
+
with(basic_auth: ['john', 'doe'],
|
1525
|
+
body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
1526
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1527
|
+
|
1528
|
+
driver(config)
|
1529
|
+
|
1530
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1531
|
+
driver.run(default_tag: 'fluentd-policy') do
|
1532
|
+
driver.feed(sample_record)
|
1533
|
+
end
|
1534
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1535
|
+
|
1536
|
+
assert_requested(elastic_request)
|
1537
|
+
end
|
1538
|
+
|
1539
|
+
def test_tag_placeholder_with_multiple_policies
|
1540
|
+
cwd = File.dirname(__FILE__)
|
1541
|
+
template_file = File.join(cwd, 'test_template.json')
|
1542
|
+
|
1543
|
+
config = %{
|
1544
|
+
host logs.google.com
|
1545
|
+
port 777
|
1546
|
+
scheme https
|
1547
|
+
path /es/
|
1548
|
+
user john
|
1549
|
+
password doe
|
1550
|
+
template_name logstash
|
1551
|
+
template_file #{template_file}
|
1552
|
+
index_date_pattern now/w{xxxx.ww}
|
1553
|
+
ilm_policy_id ${tag}
|
1554
|
+
enable_ilm true
|
1555
|
+
index_name logstash
|
1556
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}, "fluentd-policy2":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"80gb", "max_age":"20d"}}}}}}}
|
1557
|
+
}
|
1558
|
+
|
1559
|
+
# connection start
|
1560
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
1561
|
+
with(basic_auth: ['john', 'doe']).
|
1562
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1563
|
+
# check if template exists
|
1564
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1565
|
+
with(basic_auth: ['john', 'doe']).
|
1566
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1567
|
+
# creation
|
1568
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1569
|
+
with(basic_auth: ['john', 'doe']).
|
1570
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1571
|
+
# check if alias exists
|
1572
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1573
|
+
with(basic_auth: ['john', 'doe']).
|
1574
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1575
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1576
|
+
with(basic_auth: ['john', 'doe']).
|
1577
|
+
to_return(status: 404, body: "", headers: {})
|
1578
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1579
|
+
with(basic_auth: ['john', 'doe'],
|
1580
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1581
|
+
to_return(status: 200, body: "", headers: {})
|
1582
|
+
# put the alias for the index
|
1583
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1584
|
+
with(basic_auth: ['john', 'doe']).
|
1585
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1586
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
1587
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
1588
|
+
to_return(status: 200, body: "", headers: {})
|
1589
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1590
|
+
with(basic_auth: ['john', 'doe']).
|
1591
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1592
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
|
1593
|
+
with(basic_auth: ['john', 'doe']).
|
1594
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1595
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
|
1596
|
+
with(basic_auth: ['john', 'doe'],
|
1597
|
+
body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"80gb\",\"max_age\":\"20d\"}}}}}}").
|
1598
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1599
|
+
|
1600
|
+
driver(config)
|
1601
|
+
|
1602
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1603
|
+
driver.run(default_tag: 'fluentd-policy2') do
|
1604
|
+
driver.feed(sample_record)
|
1605
|
+
end
|
1606
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1607
|
+
|
1608
|
+
assert_requested(elastic_request)
|
1609
|
+
end
|
1610
|
+
end
|
1611
|
+
|
1287
1612
|
def test_template_create_with_rollover_index_and_default_ilm_and_placeholders
|
1288
1613
|
cwd = File.dirname(__FILE__)
|
1289
1614
|
template_file = File.join(cwd, 'test_template.json')
|
@@ -1356,6 +1681,83 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1356
1681
|
|
1357
1682
|
assert_requested(elastic_request)
|
1358
1683
|
end
|
1684
|
+
|
1685
|
+
def test_template_create_with_rollover_index_and_default_ilm_and_custom_and_time_placeholders
|
1686
|
+
cwd = File.dirname(__FILE__)
|
1687
|
+
template_file = File.join(cwd, 'test_template.json')
|
1688
|
+
|
1689
|
+
config = Fluent::Config::Element.new(
|
1690
|
+
'ROOT', '', {
|
1691
|
+
'@type' => 'elasticsearch',
|
1692
|
+
'host' => 'logs.google.com',
|
1693
|
+
'port' => 777,
|
1694
|
+
'scheme' => "https",
|
1695
|
+
'path' => "/es/",
|
1696
|
+
'user' => 'john',
|
1697
|
+
'password' => 'doe',
|
1698
|
+
'template_name' => 'logstash',
|
1699
|
+
'template_file' => "#{template_file}",
|
1700
|
+
'index_date_pattern' => 'now/w{xxxx.ww}',
|
1701
|
+
'index_name' => "${taskDef}-%Y.%m",
|
1702
|
+
'enable_ilm' => true,
|
1703
|
+
}, [
|
1704
|
+
Fluent::Config::Element.new('buffer', 'tag, time, taskDef', {
|
1705
|
+
'chunk_keys' => ['tag', 'time', 'taskDef'],
|
1706
|
+
'timekey' => 3600,
|
1707
|
+
}, [])
|
1708
|
+
]
|
1709
|
+
)
|
1710
|
+
|
1711
|
+
task_def_value = "task_definition"
|
1712
|
+
date_str = Time.now.strftime("%Y.%m")
|
1713
|
+
# connection start
|
1714
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
1715
|
+
with(basic_auth: ['john', 'doe']).
|
1716
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1717
|
+
# check if template exists
|
1718
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
|
1719
|
+
with(basic_auth: ['john', 'doe']).
|
1720
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1721
|
+
# creation
|
1722
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
|
1723
|
+
with(basic_auth: ['john', 'doe'],
|
1724
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"#{task_def_value}-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"#{task_def_value}-#{date_str}-*\",\"order\":52}").
|
1725
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1726
|
+
# check if alias exists
|
1727
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/#{task_def_value}-#{date_str}").
|
1728
|
+
with(basic_auth: ['john', 'doe']).
|
1729
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1730
|
+
# put the alias for the index
|
1731
|
+
stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1732
|
+
with(basic_auth: ['john', 'doe']).
|
1733
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1734
|
+
stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/#{task_def_value}-#{date_str}").
|
1735
|
+
with(basic_auth: ['john', 'doe'],
|
1736
|
+
:body => "{\"aliases\":{\"#{task_def_value}-#{date_str}\":{\"is_write_index\":true}}}").
|
1737
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1738
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1739
|
+
with(basic_auth: ['john', 'doe']).
|
1740
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1741
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
1742
|
+
with(basic_auth: ['john', 'doe']).
|
1743
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1744
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
1745
|
+
with(basic_auth: ['john', 'doe'],
|
1746
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
1747
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1748
|
+
|
1749
|
+
driver(config)
|
1750
|
+
|
1751
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1752
|
+
driver.run(default_tag: 'test') do
|
1753
|
+
driver.feed(sample_record.merge("taskDef" => task_def_value))
|
1754
|
+
end
|
1755
|
+
assert_equal("#{task_def_value}-#{date_str}", index_cmds.first['index']['_index'])
|
1756
|
+
|
1757
|
+
assert_equal ["#{task_def_value}-#{date_str}"], driver.instance.alias_indexes
|
1758
|
+
|
1759
|
+
assert_requested(elastic_request)
|
1760
|
+
end
|
1359
1761
|
end
|
1360
1762
|
|
1361
1763
|
def test_custom_template_create
|
@@ -1612,11 +2014,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1612
2014
|
to_return(:status => 200, :body => "", :headers => {})
|
1613
2015
|
# check if alias exists
|
1614
2016
|
timestr = Time.now.strftime("%Y.%m.%d")
|
1615
|
-
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-#{timestr}").
|
2017
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-myapp-#{timestr}").
|
1616
2018
|
with(basic_auth: ['john', 'doe']).
|
1617
2019
|
to_return(:status => 404, :body => "", :headers => {})
|
1618
2020
|
# put the alias for the index
|
1619
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-#{timestr}").
|
2021
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-myapp-#{timestr}").
|
1620
2022
|
with(basic_auth: ['john', 'doe']).
|
1621
2023
|
to_return(:status => 200, :body => "", :headers => {})
|
1622
2024
|
|
@@ -2896,6 +3298,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2896
3298
|
|
2897
3299
|
data("border" => {"es_version" => 6, "_type" => "mytype"},
|
2898
3300
|
"fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
|
3301
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
2899
3302
|
)
|
2900
3303
|
def test_writes_to_speficied_type(data)
|
2901
3304
|
driver('', data["es_version"]).configure("type_name mytype\n")
|
@@ -2908,6 +3311,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2908
3311
|
|
2909
3312
|
data("border" => {"es_version" => 6, "_type" => "mytype.test"},
|
2910
3313
|
"fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
|
3314
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
2911
3315
|
)
|
2912
3316
|
def test_writes_to_speficied_type_with_placeholders(data)
|
2913
3317
|
driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
|
@@ -2918,6 +3322,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2918
3322
|
assert_equal(data['_type'], index_cmds.first['index']['_type'])
|
2919
3323
|
end
|
2920
3324
|
|
3325
|
+
data("border" => {"es_version" => 6, "_type" => "mytype.test"},
|
3326
|
+
"fixed_behavior"=> {"es_version" => 7, "_type" => nil},
|
3327
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
3328
|
+
)
|
3329
|
+
def test_writes_to_speficied_type_with_suppress_type_name(data)
|
3330
|
+
driver('', data["es_version"])
|
3331
|
+
.configure("type_name mytype.${tag}\nsuppress_type_name true")
|
3332
|
+
stub_elastic
|
3333
|
+
driver.run(default_tag: 'test') do
|
3334
|
+
driver.feed(sample_record)
|
3335
|
+
end
|
3336
|
+
assert_equal(data['_type'], index_cmds.first['index']['_type'])
|
3337
|
+
end
|
3338
|
+
|
2921
3339
|
data("old" => {"es_version" => 2, "_type" => "local-override"},
|
2922
3340
|
"old_behavior" => {"es_version" => 5, "_type" => "local-override"},
|
2923
3341
|
"border" => {"es_version" => 6, "_type" => "fluentd"},
|