fluent-plugin-elasticsearch 4.0.6 → 4.0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -230,7 +230,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
230
230
  assert_nil instance.ssl_max_version
231
231
  assert_nil instance.ssl_min_version
232
232
  if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
233
- assert_equal({max_version: OpenSSL::SSL::TLS1_VERSION, min_version: OpenSSL::SSL::TLS1_VERSION},
233
+ assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
234
234
  instance.ssl_version_options)
235
235
  else
236
236
  assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
@@ -251,6 +251,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
251
251
  assert_equal 20 * 1024 * 1024, Fluent::Plugin::ElasticsearchOutput::TARGET_BULK_BYTES
252
252
  assert_false instance.compression
253
253
  assert_equal :no_compression, instance.compression_level
254
+ assert_true instance.http_backend_excon_nonblock
254
255
  end
255
256
 
256
257
  test 'configure compression' do
@@ -283,7 +284,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
283
284
  }
284
285
  instance = driver(config).instance
285
286
 
286
- assert_equal "gzip", instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
287
+ assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
288
+
289
+ stub_request(:post, "http://localhost:9200/_bulk").
290
+ to_return(status: 200, body: "", headers: {})
291
+ driver.run(default_tag: 'test') do
292
+ driver.feed(sample_record)
293
+ end
294
+ compressable = instance.compressable_connection
295
+
296
+ assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
287
297
  end
288
298
 
289
299
  test 'check compression option is passed to transport' do
@@ -294,7 +304,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
294
304
  }
295
305
  instance = driver(config).instance
296
306
 
297
- assert_equal true, instance.client.transport.options[:compression]
307
+ assert_equal false, instance.client.transport.options[:compression]
308
+
309
+ stub_request(:post, "http://localhost:9200/_bulk").
310
+ to_return(status: 200, body: "", headers: {})
311
+ driver.run(default_tag: 'test') do
312
+ driver.feed(sample_record)
313
+ end
314
+ compressable = instance.compressable_connection
315
+
316
+ assert_equal true, instance.client(nil, compressable).transport.options[:compression]
298
317
  end
299
318
 
300
319
  test 'configure Content-Type' do
@@ -332,6 +351,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
332
351
  }
333
352
  end
334
353
 
354
+ sub_test_case 'Check TLS handshake stuck warning log' do
355
+ test 'warning TLS log' do
356
+ config = %{
357
+ scheme https
358
+ http_backend_excon_nonblock false
359
+ ssl_version TLSv1_2
360
+ @log_level info
361
+ }
362
+ driver(config)
363
+ logs = driver.logs
364
+ assert_logs_include(logs, /TLS handshake will be stucked with block connection.\n Consider to set `http_backend_excon_nonblock` as true\n/)
365
+ end
366
+ end
367
+
335
368
  sub_test_case 'ILM default config' do
336
369
  setup do
337
370
  begin
@@ -1020,6 +1053,80 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1020
1053
  assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1021
1054
  end
1022
1055
 
1056
+ def test_template_create_with_rollover_index_and_default_ilm_on_logstash_format
1057
+ cwd = File.dirname(__FILE__)
1058
+ template_file = File.join(cwd, 'test_template.json')
1059
+
1060
+ config = %{
1061
+ host logs.google.com
1062
+ port 777
1063
+ scheme https
1064
+ path /es/
1065
+ user john
1066
+ password doe
1067
+ template_name logstash
1068
+ template_file #{template_file}
1069
+ index_date_pattern now/w{xxxx.ww}
1070
+ enable_ilm true
1071
+ logstash_format true
1072
+ application_name log
1073
+ }
1074
+
1075
+ date_str = Time.now.strftime("%Y.%m.%d")
1076
+ # connection start
1077
+ stub_request(:head, "https://logs.google.com:777/es//").
1078
+ with(basic_auth: ['john', 'doe']).
1079
+ to_return(:status => 200, :body => "", :headers => {})
1080
+ # check if template exists
1081
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
1082
+ with(basic_auth: ['john', 'doe']).
1083
+ to_return(:status => 404, :body => "", :headers => {})
1084
+ # creation
1085
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
1086
+ with(basic_auth: ['john', 'doe']).
1087
+ to_return(:status => 200, :body => "", :headers => {})
1088
+ # check if alias exists
1089
+ stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-log-#{date_str}").
1090
+ with(basic_auth: ['john', 'doe']).
1091
+ to_return(:status => 404, :body => "", :headers => {})
1092
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
1093
+ with(basic_auth: ['john', 'doe']).
1094
+ to_return(status: 404, body: "", headers: {})
1095
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}").
1096
+ with(basic_auth: ['john', 'doe'],
1097
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-log-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-log-#{date_str}-*\",\"order\":53}").
1098
+ to_return(status: 200, body: "", headers: {})
1099
+ # put the alias for the index
1100
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1101
+ with(basic_auth: ['john', 'doe']).
1102
+ to_return(:status => 200, :body => "", :headers => {})
1103
+
1104
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash-log-#{date_str}").
1105
+ with(basic_auth: ['john', 'doe'],
1106
+ body: "{\"aliases\":{\"logstash-log-#{date_str}\":{\"is_write_index\":true}}}").
1107
+ to_return(status: 200, body: "", headers: {})
1108
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1109
+ with(basic_auth: ['john', 'doe']).
1110
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1111
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
1112
+ with(basic_auth: ['john', 'doe']).
1113
+ to_return(:status => 404, :body => "", :headers => {})
1114
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
1115
+ with(basic_auth: ['john', 'doe'],
1116
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
1117
+ to_return(:status => 200, :body => "", :headers => {})
1118
+
1119
+ driver(config)
1120
+
1121
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1122
+ driver.run(default_tag: 'test') do
1123
+ driver.feed(sample_record)
1124
+ end
1125
+ assert_requested(:put, "https://logs.google.com:777/es//_template/logstash-log-#{date_str}", times: 1)
1126
+
1127
+ assert_requested(elastic_request)
1128
+ end
1129
+
1023
1130
  def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite
1024
1131
  cwd = File.dirname(__FILE__)
1025
1132
  template_file = File.join(cwd, 'test_template.json')
@@ -1284,6 +1391,224 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1284
1391
  assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1285
1392
  end
1286
1393
 
1394
+ def test_template_create_with_rollover_index_and_ilm_policies_and_placeholders
1395
+ cwd = File.dirname(__FILE__)
1396
+ template_file = File.join(cwd, 'test_template.json')
1397
+
1398
+ config = %{
1399
+ host logs.google.com
1400
+ port 777
1401
+ scheme https
1402
+ path /es/
1403
+ user john
1404
+ password doe
1405
+ template_name logstash
1406
+ template_file #{template_file}
1407
+ index_date_pattern now/w{xxxx.ww}
1408
+ ilm_policy_id fluentd-policy
1409
+ enable_ilm true
1410
+ index_name logstash
1411
+ ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
1412
+ }
1413
+
1414
+ # connection start
1415
+ stub_request(:head, "https://logs.google.com:777/es//").
1416
+ with(basic_auth: ['john', 'doe']).
1417
+ to_return(:status => 200, :body => "", :headers => {})
1418
+ # check if template exists
1419
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1420
+ with(basic_auth: ['john', 'doe']).
1421
+ to_return(:status => 404, :body => "", :headers => {})
1422
+ # creation
1423
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1424
+ with(basic_auth: ['john', 'doe']).
1425
+ to_return(:status => 200, :body => "", :headers => {})
1426
+ # check if alias exists
1427
+ stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1428
+ with(basic_auth: ['john', 'doe']).
1429
+ to_return(:status => 404, :body => "", :headers => {})
1430
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1431
+ with(basic_auth: ['john', 'doe']).
1432
+ to_return(status: 404, body: "", headers: {})
1433
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1434
+ with(basic_auth: ['john', 'doe'],
1435
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1436
+ to_return(status: 200, body: "", headers: {})
1437
+ # put the alias for the index
1438
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1439
+ with(basic_auth: ['john', 'doe']).
1440
+ to_return(:status => 200, :body => "", :headers => {})
1441
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
1442
+ with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
1443
+ to_return(status: 200, body: "", headers: {})
1444
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1445
+ with(basic_auth: ['john', 'doe']).
1446
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1447
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1448
+ with(basic_auth: ['john', 'doe']).
1449
+ to_return(:status => 404, :body => "", :headers => {})
1450
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1451
+ with(basic_auth: ['john', 'doe'],
1452
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
1453
+ to_return(:status => 200, :body => "", :headers => {})
1454
+
1455
+ driver(config)
1456
+
1457
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1458
+ driver.run(default_tag: 'test') do
1459
+ driver.feed(sample_record)
1460
+ end
1461
+ assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1462
+
1463
+ assert_requested(elastic_request)
1464
+ end
1465
+
1466
+ class TemplateCreateWithRolloverIndexAndILMPoliciesWithPlaceholdersTest < self
1467
+ def test_tag_placeholder
1468
+ cwd = File.dirname(__FILE__)
1469
+ template_file = File.join(cwd, 'test_template.json')
1470
+
1471
+ config = %{
1472
+ host logs.google.com
1473
+ port 777
1474
+ scheme https
1475
+ path /es/
1476
+ user john
1477
+ password doe
1478
+ template_name logstash
1479
+ template_file #{template_file}
1480
+ index_date_pattern now/w{xxxx.ww}
1481
+ ilm_policy_id ${tag}
1482
+ enable_ilm true
1483
+ index_name logstash
1484
+ ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
1485
+ }
1486
+
1487
+ # connection start
1488
+ stub_request(:head, "https://logs.google.com:777/es//").
1489
+ with(basic_auth: ['john', 'doe']).
1490
+ to_return(:status => 200, :body => "", :headers => {})
1491
+ # check if template exists
1492
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1493
+ with(basic_auth: ['john', 'doe']).
1494
+ to_return(:status => 404, :body => "", :headers => {})
1495
+ # creation
1496
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1497
+ with(basic_auth: ['john', 'doe']).
1498
+ to_return(:status => 200, :body => "", :headers => {})
1499
+ # check if alias exists
1500
+ stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1501
+ with(basic_auth: ['john', 'doe']).
1502
+ to_return(:status => 404, :body => "", :headers => {})
1503
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1504
+ with(basic_auth: ['john', 'doe']).
1505
+ to_return(status: 404, body: "", headers: {})
1506
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1507
+ with(basic_auth: ['john', 'doe'],
1508
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1509
+ to_return(status: 200, body: "", headers: {})
1510
+ # put the alias for the index
1511
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1512
+ with(basic_auth: ['john', 'doe']).
1513
+ to_return(:status => 200, :body => "", :headers => {})
1514
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
1515
+ with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
1516
+ to_return(status: 200, body: "", headers: {})
1517
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1518
+ with(basic_auth: ['john', 'doe']).
1519
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1520
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1521
+ with(basic_auth: ['john', 'doe']).
1522
+ to_return(:status => 404, :body => "", :headers => {})
1523
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1524
+ with(basic_auth: ['john', 'doe'],
1525
+ body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
1526
+ to_return(:status => 200, :body => "", :headers => {})
1527
+
1528
+ driver(config)
1529
+
1530
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1531
+ driver.run(default_tag: 'fluentd-policy') do
1532
+ driver.feed(sample_record)
1533
+ end
1534
+ assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1535
+
1536
+ assert_requested(elastic_request)
1537
+ end
1538
+
1539
+ def test_tag_placeholder_with_multiple_policies
1540
+ cwd = File.dirname(__FILE__)
1541
+ template_file = File.join(cwd, 'test_template.json')
1542
+
1543
+ config = %{
1544
+ host logs.google.com
1545
+ port 777
1546
+ scheme https
1547
+ path /es/
1548
+ user john
1549
+ password doe
1550
+ template_name logstash
1551
+ template_file #{template_file}
1552
+ index_date_pattern now/w{xxxx.ww}
1553
+ ilm_policy_id ${tag}
1554
+ enable_ilm true
1555
+ index_name logstash
1556
+ ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}, "fluentd-policy2":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"80gb", "max_age":"20d"}}}}}}}
1557
+ }
1558
+
1559
+ # connection start
1560
+ stub_request(:head, "https://logs.google.com:777/es//").
1561
+ with(basic_auth: ['john', 'doe']).
1562
+ to_return(:status => 200, :body => "", :headers => {})
1563
+ # check if template exists
1564
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1565
+ with(basic_auth: ['john', 'doe']).
1566
+ to_return(:status => 404, :body => "", :headers => {})
1567
+ # creation
1568
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1569
+ with(basic_auth: ['john', 'doe']).
1570
+ to_return(:status => 200, :body => "", :headers => {})
1571
+ # check if alias exists
1572
+ stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1573
+ with(basic_auth: ['john', 'doe']).
1574
+ to_return(:status => 404, :body => "", :headers => {})
1575
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1576
+ with(basic_auth: ['john', 'doe']).
1577
+ to_return(status: 404, body: "", headers: {})
1578
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1579
+ with(basic_auth: ['john', 'doe'],
1580
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1581
+ to_return(status: 200, body: "", headers: {})
1582
+ # put the alias for the index
1583
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1584
+ with(basic_auth: ['john', 'doe']).
1585
+ to_return(:status => 200, :body => "", :headers => {})
1586
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
1587
+ with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
1588
+ to_return(status: 200, body: "", headers: {})
1589
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1590
+ with(basic_auth: ['john', 'doe']).
1591
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1592
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
1593
+ with(basic_auth: ['john', 'doe']).
1594
+ to_return(:status => 404, :body => "", :headers => {})
1595
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
1596
+ with(basic_auth: ['john', 'doe'],
1597
+ body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"80gb\",\"max_age\":\"20d\"}}}}}}").
1598
+ to_return(:status => 200, :body => "", :headers => {})
1599
+
1600
+ driver(config)
1601
+
1602
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1603
+ driver.run(default_tag: 'fluentd-policy2') do
1604
+ driver.feed(sample_record)
1605
+ end
1606
+ assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1607
+
1608
+ assert_requested(elastic_request)
1609
+ end
1610
+ end
1611
+
1287
1612
  def test_template_create_with_rollover_index_and_default_ilm_and_placeholders
1288
1613
  cwd = File.dirname(__FILE__)
1289
1614
  template_file = File.join(cwd, 'test_template.json')
@@ -1356,6 +1681,83 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1356
1681
 
1357
1682
  assert_requested(elastic_request)
1358
1683
  end
1684
+
1685
+ def test_template_create_with_rollover_index_and_default_ilm_and_custom_and_time_placeholders
1686
+ cwd = File.dirname(__FILE__)
1687
+ template_file = File.join(cwd, 'test_template.json')
1688
+
1689
+ config = Fluent::Config::Element.new(
1690
+ 'ROOT', '', {
1691
+ '@type' => 'elasticsearch',
1692
+ 'host' => 'logs.google.com',
1693
+ 'port' => 777,
1694
+ 'scheme' => "https",
1695
+ 'path' => "/es/",
1696
+ 'user' => 'john',
1697
+ 'password' => 'doe',
1698
+ 'template_name' => 'logstash',
1699
+ 'template_file' => "#{template_file}",
1700
+ 'index_date_pattern' => 'now/w{xxxx.ww}',
1701
+ 'index_name' => "${taskDef}-%Y.%m",
1702
+ 'enable_ilm' => true,
1703
+ }, [
1704
+ Fluent::Config::Element.new('buffer', 'tag, time, taskDef', {
1705
+ 'chunk_keys' => ['tag', 'time', 'taskDef'],
1706
+ 'timekey' => 3600,
1707
+ }, [])
1708
+ ]
1709
+ )
1710
+
1711
+ task_def_value = "task_definition"
1712
+ date_str = Time.now.strftime("%Y.%m")
1713
+ # connection start
1714
+ stub_request(:head, "https://logs.google.com:777/es//").
1715
+ with(basic_auth: ['john', 'doe']).
1716
+ to_return(:status => 200, :body => "", :headers => {})
1717
+ # check if template exists
1718
+ stub_request(:get, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
1719
+ with(basic_auth: ['john', 'doe']).
1720
+ to_return(:status => 404, :body => "", :headers => {})
1721
+ # creation
1722
+ stub_request(:put, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
1723
+ with(basic_auth: ['john', 'doe'],
1724
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"#{task_def_value}-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"#{task_def_value}-#{date_str}-*\",\"order\":52}").
1725
+ to_return(:status => 200, :body => "", :headers => {})
1726
+ # check if alias exists
1727
+ stub_request(:head, "https://logs.google.com:777/es//_alias/#{task_def_value}-#{date_str}").
1728
+ with(basic_auth: ['john', 'doe']).
1729
+ to_return(:status => 404, :body => "", :headers => {})
1730
+ # put the alias for the index
1731
+ stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1732
+ with(basic_auth: ['john', 'doe']).
1733
+ to_return(:status => 200, :body => "", :headers => {})
1734
+ stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/#{task_def_value}-#{date_str}").
1735
+ with(basic_auth: ['john', 'doe'],
1736
+ :body => "{\"aliases\":{\"#{task_def_value}-#{date_str}\":{\"is_write_index\":true}}}").
1737
+ to_return(:status => 200, :body => "", :headers => {})
1738
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1739
+ with(basic_auth: ['john', 'doe']).
1740
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1741
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
1742
+ with(basic_auth: ['john', 'doe']).
1743
+ to_return(:status => 404, :body => "", :headers => {})
1744
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
1745
+ with(basic_auth: ['john', 'doe'],
1746
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
1747
+ to_return(:status => 200, :body => "", :headers => {})
1748
+
1749
+ driver(config)
1750
+
1751
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1752
+ driver.run(default_tag: 'test') do
1753
+ driver.feed(sample_record.merge("taskDef" => task_def_value))
1754
+ end
1755
+ assert_equal("#{task_def_value}-#{date_str}", index_cmds.first['index']['_index'])
1756
+
1757
+ assert_equal ["#{task_def_value}-#{date_str}"], driver.instance.alias_indexes
1758
+
1759
+ assert_requested(elastic_request)
1760
+ end
1359
1761
  end
1360
1762
 
1361
1763
  def test_custom_template_create
@@ -1612,11 +2014,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1612
2014
  to_return(:status => 200, :body => "", :headers => {})
1613
2015
  # check if alias exists
1614
2016
  timestr = Time.now.strftime("%Y.%m.%d")
1615
- stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-#{timestr}").
2017
+ stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-myapp-#{timestr}").
1616
2018
  with(basic_auth: ['john', 'doe']).
1617
2019
  to_return(:status => 404, :body => "", :headers => {})
1618
2020
  # put the alias for the index
1619
- stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-#{timestr}").
2021
+ stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-myapp-#{timestr}").
1620
2022
  with(basic_auth: ['john', 'doe']).
1621
2023
  to_return(:status => 200, :body => "", :headers => {})
1622
2024
 
@@ -2896,6 +3298,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2896
3298
 
2897
3299
  data("border" => {"es_version" => 6, "_type" => "mytype"},
2898
3300
  "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
3301
+ "to be nil" => {"es_version" => 8, "_type" => nil},
2899
3302
  )
2900
3303
  def test_writes_to_speficied_type(data)
2901
3304
  driver('', data["es_version"]).configure("type_name mytype\n")
@@ -2908,6 +3311,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2908
3311
 
2909
3312
  data("border" => {"es_version" => 6, "_type" => "mytype.test"},
2910
3313
  "fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
3314
+ "to be nil" => {"es_version" => 8, "_type" => nil},
2911
3315
  )
2912
3316
  def test_writes_to_speficied_type_with_placeholders(data)
2913
3317
  driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
@@ -2918,6 +3322,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2918
3322
  assert_equal(data['_type'], index_cmds.first['index']['_type'])
2919
3323
  end
2920
3324
 
3325
+ data("border" => {"es_version" => 6, "_type" => "mytype.test"},
3326
+ "fixed_behavior"=> {"es_version" => 7, "_type" => nil},
3327
+ "to be nil" => {"es_version" => 8, "_type" => nil},
3328
+ )
3329
+ def test_writes_to_speficied_type_with_suppress_type_name(data)
3330
+ driver('', data["es_version"])
3331
+ .configure("type_name mytype.${tag}\nsuppress_type_name true")
3332
+ stub_elastic
3333
+ driver.run(default_tag: 'test') do
3334
+ driver.feed(sample_record)
3335
+ end
3336
+ assert_equal(data['_type'], index_cmds.first['index']['_type'])
3337
+ end
3338
+
2921
3339
  data("old" => {"es_version" => 2, "_type" => "local-override"},
2922
3340
  "old_behavior" => {"es_version" => 5, "_type" => "local-override"},
2923
3341
  "border" => {"es_version" => 6, "_type" => "fluentd"},