fluent-plugin-elasticsearch 4.0.8 → 4.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/History.md +23 -0
- data/README.ElasticsearchGenID.md +116 -0
- data/README.md +104 -5
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +9 -0
- data/lib/fluent/plugin/elasticsearch_index_template.rb +19 -11
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +52 -0
- data/lib/fluent/plugin/out_elasticsearch.rb +64 -31
- data/test/plugin/test_elasticsearch_fallback_selector.rb +73 -0
- data/test/plugin/test_filter_elasticsearch_genid.rb +171 -0
- data/test/plugin/test_out_elasticsearch.rb +375 -72
- metadata +7 -2
|
@@ -1053,6 +1053,80 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1053
1053
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1054
1054
|
end
|
|
1055
1055
|
|
|
1056
|
+
def test_template_create_with_rollover_index_and_default_ilm_on_logstash_format
|
|
1057
|
+
cwd = File.dirname(__FILE__)
|
|
1058
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1059
|
+
|
|
1060
|
+
config = %{
|
|
1061
|
+
host logs.google.com
|
|
1062
|
+
port 777
|
|
1063
|
+
scheme https
|
|
1064
|
+
path /es/
|
|
1065
|
+
user john
|
|
1066
|
+
password doe
|
|
1067
|
+
template_name logstash
|
|
1068
|
+
template_file #{template_file}
|
|
1069
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1070
|
+
enable_ilm true
|
|
1071
|
+
logstash_format true
|
|
1072
|
+
application_name log
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
date_str = Time.now.strftime("%Y.%m.%d")
|
|
1076
|
+
# connection start
|
|
1077
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1078
|
+
with(basic_auth: ['john', 'doe']).
|
|
1079
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1080
|
+
# check if template exists
|
|
1081
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
|
|
1082
|
+
with(basic_auth: ['john', 'doe']).
|
|
1083
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1084
|
+
# creation
|
|
1085
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
|
|
1086
|
+
with(basic_auth: ['john', 'doe']).
|
|
1087
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1088
|
+
# check if alias exists
|
|
1089
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-#{date_str}").
|
|
1090
|
+
with(basic_auth: ['john', 'doe']).
|
|
1091
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1092
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
|
|
1093
|
+
with(basic_auth: ['john', 'doe']).
|
|
1094
|
+
to_return(status: 404, body: "", headers: {})
|
|
1095
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
|
|
1096
|
+
with(basic_auth: ['john', 'doe'],
|
|
1097
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-log-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-log-#{date_str}-*\",\"order\":53}").
|
|
1098
|
+
to_return(status: 200, body: "", headers: {})
|
|
1099
|
+
# put the alias for the index
|
|
1100
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-#{date_str}-000001%3E").
|
|
1101
|
+
with(basic_auth: ['john', 'doe']).
|
|
1102
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1103
|
+
|
|
1104
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-#{date_str}-000001%3E/#{alias_endpoint}/logstash-#{date_str}").
|
|
1105
|
+
with(basic_auth: ['john', 'doe'],
|
|
1106
|
+
body: "{\"aliases\":{\"logstash-#{date_str}\":{\"is_write_index\":true}}}").
|
|
1107
|
+
to_return(status: 200, body: "", headers: {})
|
|
1108
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1109
|
+
with(basic_auth: ['john', 'doe']).
|
|
1110
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1111
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1112
|
+
with(basic_auth: ['john', 'doe']).
|
|
1113
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1114
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1115
|
+
with(basic_auth: ['john', 'doe'],
|
|
1116
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1117
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1118
|
+
|
|
1119
|
+
driver(config)
|
|
1120
|
+
|
|
1121
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1122
|
+
driver.run(default_tag: 'test') do
|
|
1123
|
+
driver.feed(sample_record)
|
|
1124
|
+
end
|
|
1125
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash-#{date_str}", times: 1)
|
|
1126
|
+
|
|
1127
|
+
assert_requested(elastic_request)
|
|
1128
|
+
end
|
|
1129
|
+
|
|
1056
1130
|
def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite
|
|
1057
1131
|
cwd = File.dirname(__FILE__)
|
|
1058
1132
|
template_file = File.join(cwd, 'test_template.json')
|
|
@@ -1139,6 +1213,30 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1139
1213
|
enable_ilm true
|
|
1140
1214
|
}
|
|
1141
1215
|
|
|
1216
|
+
# Should raise error because multiple alias indices IllegalArgument Error on executing ILM feature
|
|
1217
|
+
assert_raise(Fluent::ConfigError) do
|
|
1218
|
+
driver(config)
|
|
1219
|
+
end
|
|
1220
|
+
end
|
|
1221
|
+
|
|
1222
|
+
def test_template_create_with_rollover_index_and_default_ilm_with_empty_index_date_pattern
|
|
1223
|
+
cwd = File.dirname(__FILE__)
|
|
1224
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1225
|
+
|
|
1226
|
+
config = %{
|
|
1227
|
+
host logs.google.com
|
|
1228
|
+
port 777
|
|
1229
|
+
scheme https
|
|
1230
|
+
path /es/
|
|
1231
|
+
user john
|
|
1232
|
+
password doe
|
|
1233
|
+
template_name logstash
|
|
1234
|
+
template_file #{template_file}
|
|
1235
|
+
index_date_pattern ""
|
|
1236
|
+
index_name logstash
|
|
1237
|
+
enable_ilm true
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1142
1240
|
# connection start
|
|
1143
1241
|
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1144
1242
|
with(basic_auth: ['john', 'doe']).
|
|
@@ -1152,23 +1250,23 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1152
1250
|
with(basic_auth: ['john', 'doe']).
|
|
1153
1251
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1154
1252
|
# check if alias exists
|
|
1155
|
-
stub_request(:head, "https://logs.google.com:777/es//_alias/
|
|
1253
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
|
1156
1254
|
with(basic_auth: ['john', 'doe']).
|
|
1157
1255
|
to_return(:status => 404, :body => "", :headers => {})
|
|
1158
|
-
stub_request(:get, "https://logs.google.com:777/es//_template/
|
|
1256
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1159
1257
|
with(basic_auth: ['john', 'doe']).
|
|
1160
1258
|
to_return(status: 404, body: "", headers: {})
|
|
1161
1259
|
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
|
|
1162
1260
|
with(basic_auth: ['john', 'doe'],
|
|
1163
|
-
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"
|
|
1261
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
|
|
1164
1262
|
to_return(status: 200, body: "", headers: {})
|
|
1165
1263
|
# put the alias for the index
|
|
1166
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default
|
|
1264
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E").
|
|
1167
1265
|
with(basic_auth: ['john', 'doe']).
|
|
1168
1266
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1169
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default
|
|
1267
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E/#{alias_endpoint}/logstash").
|
|
1170
1268
|
with(basic_auth: ['john', 'doe'],
|
|
1171
|
-
:body => "{\"aliases\":{\"
|
|
1269
|
+
:body => "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
|
1172
1270
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1173
1271
|
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1174
1272
|
with(basic_auth: ['john', 'doe']).
|
|
@@ -1183,10 +1281,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1183
1281
|
|
|
1184
1282
|
driver(config)
|
|
1185
1283
|
|
|
1186
|
-
assert_requested(:put, "https://logs.google.com:777/es//_template/
|
|
1284
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1187
1285
|
end
|
|
1188
1286
|
|
|
1189
|
-
def
|
|
1287
|
+
def test_template_create_with_rollover_index_and_custom_ilm
|
|
1190
1288
|
cwd = File.dirname(__FILE__)
|
|
1191
1289
|
template_file = File.join(cwd, 'test_template.json')
|
|
1192
1290
|
|
|
@@ -1199,9 +1297,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1199
1297
|
password doe
|
|
1200
1298
|
template_name logstash
|
|
1201
1299
|
template_file #{template_file}
|
|
1202
|
-
index_date_pattern
|
|
1203
|
-
|
|
1300
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1301
|
+
ilm_policy_id fluentd-policy
|
|
1204
1302
|
enable_ilm true
|
|
1303
|
+
index_name logstash
|
|
1304
|
+
ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}
|
|
1205
1305
|
}
|
|
1206
1306
|
|
|
1207
1307
|
# connection start
|
|
@@ -1223,27 +1323,26 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1223
1323
|
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1224
1324
|
with(basic_auth: ['john', 'doe']).
|
|
1225
1325
|
to_return(status: 404, body: "", headers: {})
|
|
1226
|
-
stub_request(:put, "https://logs.google.com:777/es//_template/
|
|
1326
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1227
1327
|
with(basic_auth: ['john', 'doe'],
|
|
1228
|
-
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"
|
|
1328
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
|
1229
1329
|
to_return(status: 200, body: "", headers: {})
|
|
1230
1330
|
# put the alias for the index
|
|
1231
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E").
|
|
1331
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1232
1332
|
with(basic_auth: ['john', 'doe']).
|
|
1233
1333
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1234
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E/#{alias_endpoint}/logstash").
|
|
1235
|
-
with(
|
|
1236
|
-
|
|
1237
|
-
to_return(:status => 200, :body => "", :headers => {})
|
|
1334
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
|
1335
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
|
1336
|
+
to_return(status: 200, body: "", headers: {})
|
|
1238
1337
|
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1239
1338
|
with(basic_auth: ['john', 'doe']).
|
|
1240
1339
|
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1241
|
-
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/
|
|
1340
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1242
1341
|
with(basic_auth: ['john', 'doe']).
|
|
1243
1342
|
to_return(:status => 404, :body => "", :headers => {})
|
|
1244
|
-
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/
|
|
1343
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1245
1344
|
with(basic_auth: ['john', 'doe'],
|
|
1246
|
-
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"
|
|
1345
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1247
1346
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1248
1347
|
|
|
1249
1348
|
driver(config)
|
|
@@ -1251,7 +1350,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1251
1350
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1252
1351
|
end
|
|
1253
1352
|
|
|
1254
|
-
def
|
|
1353
|
+
def test_template_create_with_rollover_index_and_ilm_policies_and_placeholders
|
|
1255
1354
|
cwd = File.dirname(__FILE__)
|
|
1256
1355
|
template_file = File.join(cwd, 'test_template.json')
|
|
1257
1356
|
|
|
@@ -1268,7 +1367,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1268
1367
|
ilm_policy_id fluentd-policy
|
|
1269
1368
|
enable_ilm true
|
|
1270
1369
|
index_name logstash
|
|
1271
|
-
|
|
1370
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
|
1272
1371
|
}
|
|
1273
1372
|
|
|
1274
1373
|
# connection start
|
|
@@ -1314,7 +1413,159 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1314
1413
|
|
|
1315
1414
|
driver(config)
|
|
1316
1415
|
|
|
1416
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1417
|
+
driver.run(default_tag: 'test') do
|
|
1418
|
+
driver.feed(sample_record)
|
|
1419
|
+
end
|
|
1317
1420
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1421
|
+
|
|
1422
|
+
assert_requested(elastic_request)
|
|
1423
|
+
end
|
|
1424
|
+
|
|
1425
|
+
class TemplateCreateWithRolloverIndexAndILMPoliciesWithPlaceholdersTest < self
|
|
1426
|
+
def test_tag_placeholder
|
|
1427
|
+
cwd = File.dirname(__FILE__)
|
|
1428
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1429
|
+
|
|
1430
|
+
config = %{
|
|
1431
|
+
host logs.google.com
|
|
1432
|
+
port 777
|
|
1433
|
+
scheme https
|
|
1434
|
+
path /es/
|
|
1435
|
+
user john
|
|
1436
|
+
password doe
|
|
1437
|
+
template_name logstash
|
|
1438
|
+
template_file #{template_file}
|
|
1439
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1440
|
+
ilm_policy_id ${tag}
|
|
1441
|
+
enable_ilm true
|
|
1442
|
+
index_name logstash
|
|
1443
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
# connection start
|
|
1447
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1448
|
+
with(basic_auth: ['john', 'doe']).
|
|
1449
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1450
|
+
# check if template exists
|
|
1451
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1452
|
+
with(basic_auth: ['john', 'doe']).
|
|
1453
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1454
|
+
# creation
|
|
1455
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1456
|
+
with(basic_auth: ['john', 'doe']).
|
|
1457
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1458
|
+
# check if alias exists
|
|
1459
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
|
1460
|
+
with(basic_auth: ['john', 'doe']).
|
|
1461
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1462
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1463
|
+
with(basic_auth: ['john', 'doe']).
|
|
1464
|
+
to_return(status: 404, body: "", headers: {})
|
|
1465
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1466
|
+
with(basic_auth: ['john', 'doe'],
|
|
1467
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
|
1468
|
+
to_return(status: 200, body: "", headers: {})
|
|
1469
|
+
# put the alias for the index
|
|
1470
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1471
|
+
with(basic_auth: ['john', 'doe']).
|
|
1472
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1473
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
|
1474
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
|
1475
|
+
to_return(status: 200, body: "", headers: {})
|
|
1476
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1477
|
+
with(basic_auth: ['john', 'doe']).
|
|
1478
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1479
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1480
|
+
with(basic_auth: ['john', 'doe']).
|
|
1481
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1482
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1483
|
+
with(basic_auth: ['john', 'doe'],
|
|
1484
|
+
body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1485
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1486
|
+
|
|
1487
|
+
driver(config)
|
|
1488
|
+
|
|
1489
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1490
|
+
driver.run(default_tag: 'fluentd-policy') do
|
|
1491
|
+
driver.feed(sample_record)
|
|
1492
|
+
end
|
|
1493
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1494
|
+
|
|
1495
|
+
assert_requested(elastic_request)
|
|
1496
|
+
end
|
|
1497
|
+
|
|
1498
|
+
def test_tag_placeholder_with_multiple_policies
|
|
1499
|
+
cwd = File.dirname(__FILE__)
|
|
1500
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1501
|
+
|
|
1502
|
+
config = %{
|
|
1503
|
+
host logs.google.com
|
|
1504
|
+
port 777
|
|
1505
|
+
scheme https
|
|
1506
|
+
path /es/
|
|
1507
|
+
user john
|
|
1508
|
+
password doe
|
|
1509
|
+
template_name logstash
|
|
1510
|
+
template_file #{template_file}
|
|
1511
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1512
|
+
ilm_policy_id ${tag}
|
|
1513
|
+
enable_ilm true
|
|
1514
|
+
index_name logstash
|
|
1515
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}, "fluentd-policy2":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"80gb", "max_age":"20d"}}}}}}}
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
# connection start
|
|
1519
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1520
|
+
with(basic_auth: ['john', 'doe']).
|
|
1521
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1522
|
+
# check if template exists
|
|
1523
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1524
|
+
with(basic_auth: ['john', 'doe']).
|
|
1525
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1526
|
+
# creation
|
|
1527
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1528
|
+
with(basic_auth: ['john', 'doe']).
|
|
1529
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1530
|
+
# check if alias exists
|
|
1531
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
|
1532
|
+
with(basic_auth: ['john', 'doe']).
|
|
1533
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1534
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1535
|
+
with(basic_auth: ['john', 'doe']).
|
|
1536
|
+
to_return(status: 404, body: "", headers: {})
|
|
1537
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1538
|
+
with(basic_auth: ['john', 'doe'],
|
|
1539
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
|
1540
|
+
to_return(status: 200, body: "", headers: {})
|
|
1541
|
+
# put the alias for the index
|
|
1542
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1543
|
+
with(basic_auth: ['john', 'doe']).
|
|
1544
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1545
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
|
1546
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
|
1547
|
+
to_return(status: 200, body: "", headers: {})
|
|
1548
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1549
|
+
with(basic_auth: ['john', 'doe']).
|
|
1550
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1551
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
|
|
1552
|
+
with(basic_auth: ['john', 'doe']).
|
|
1553
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1554
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
|
|
1555
|
+
with(basic_auth: ['john', 'doe'],
|
|
1556
|
+
body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"80gb\",\"max_age\":\"20d\"}}}}}}").
|
|
1557
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1558
|
+
|
|
1559
|
+
driver(config)
|
|
1560
|
+
|
|
1561
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1562
|
+
driver.run(default_tag: 'fluentd-policy2') do
|
|
1563
|
+
driver.feed(sample_record)
|
|
1564
|
+
end
|
|
1565
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1566
|
+
|
|
1567
|
+
assert_requested(elastic_request)
|
|
1568
|
+
end
|
|
1318
1569
|
end
|
|
1319
1570
|
|
|
1320
1571
|
def test_template_create_with_rollover_index_and_default_ilm_and_placeholders
|
|
@@ -1389,6 +1640,83 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1389
1640
|
|
|
1390
1641
|
assert_requested(elastic_request)
|
|
1391
1642
|
end
|
|
1643
|
+
|
|
1644
|
+
def test_template_create_with_rollover_index_and_default_ilm_and_custom_and_time_placeholders
|
|
1645
|
+
cwd = File.dirname(__FILE__)
|
|
1646
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1647
|
+
|
|
1648
|
+
config = Fluent::Config::Element.new(
|
|
1649
|
+
'ROOT', '', {
|
|
1650
|
+
'@type' => 'elasticsearch',
|
|
1651
|
+
'host' => 'logs.google.com',
|
|
1652
|
+
'port' => 777,
|
|
1653
|
+
'scheme' => "https",
|
|
1654
|
+
'path' => "/es/",
|
|
1655
|
+
'user' => 'john',
|
|
1656
|
+
'password' => 'doe',
|
|
1657
|
+
'template_name' => 'logstash',
|
|
1658
|
+
'template_file' => "#{template_file}",
|
|
1659
|
+
'index_date_pattern' => 'now/w{xxxx.ww}',
|
|
1660
|
+
'index_name' => "${taskDef}-%Y.%m",
|
|
1661
|
+
'enable_ilm' => true,
|
|
1662
|
+
}, [
|
|
1663
|
+
Fluent::Config::Element.new('buffer', 'tag, time, taskDef', {
|
|
1664
|
+
'chunk_keys' => ['tag', 'time', 'taskDef'],
|
|
1665
|
+
'timekey' => 3600,
|
|
1666
|
+
}, [])
|
|
1667
|
+
]
|
|
1668
|
+
)
|
|
1669
|
+
|
|
1670
|
+
task_def_value = "task_definition"
|
|
1671
|
+
date_str = Time.now.strftime("%Y.%m")
|
|
1672
|
+
# connection start
|
|
1673
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1674
|
+
with(basic_auth: ['john', 'doe']).
|
|
1675
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1676
|
+
# check if template exists
|
|
1677
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
|
|
1678
|
+
with(basic_auth: ['john', 'doe']).
|
|
1679
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1680
|
+
# creation
|
|
1681
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
|
|
1682
|
+
with(basic_auth: ['john', 'doe'],
|
|
1683
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"#{task_def_value}-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"#{task_def_value}-#{date_str}-*\",\"order\":52}").
|
|
1684
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1685
|
+
# check if alias exists
|
|
1686
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/#{task_def_value}-#{date_str}").
|
|
1687
|
+
with(basic_auth: ['john', 'doe']).
|
|
1688
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1689
|
+
# put the alias for the index
|
|
1690
|
+
stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1691
|
+
with(basic_auth: ['john', 'doe']).
|
|
1692
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1693
|
+
stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/#{task_def_value}-#{date_str}").
|
|
1694
|
+
with(basic_auth: ['john', 'doe'],
|
|
1695
|
+
:body => "{\"aliases\":{\"#{task_def_value}-#{date_str}\":{\"is_write_index\":true}}}").
|
|
1696
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1697
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1698
|
+
with(basic_auth: ['john', 'doe']).
|
|
1699
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1700
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1701
|
+
with(basic_auth: ['john', 'doe']).
|
|
1702
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1703
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1704
|
+
with(basic_auth: ['john', 'doe'],
|
|
1705
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1706
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1707
|
+
|
|
1708
|
+
driver(config)
|
|
1709
|
+
|
|
1710
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1711
|
+
driver.run(default_tag: 'test') do
|
|
1712
|
+
driver.feed(sample_record.merge("taskDef" => task_def_value))
|
|
1713
|
+
end
|
|
1714
|
+
assert_equal("#{task_def_value}-#{date_str}", index_cmds.first['index']['_index'])
|
|
1715
|
+
|
|
1716
|
+
assert_equal ["#{task_def_value}-#{date_str}"], driver.instance.alias_indexes
|
|
1717
|
+
|
|
1718
|
+
assert_requested(elastic_request)
|
|
1719
|
+
end
|
|
1392
1720
|
end
|
|
1393
1721
|
|
|
1394
1722
|
def test_custom_template_create
|
|
@@ -1627,6 +1955,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1627
1955
|
application_name myapp
|
|
1628
1956
|
}
|
|
1629
1957
|
|
|
1958
|
+
timestr = Time.now.strftime("%Y.%m.%d")
|
|
1630
1959
|
# connection start
|
|
1631
1960
|
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1632
1961
|
with(basic_auth: ['john', 'doe']).
|
|
@@ -1640,17 +1969,17 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1640
1969
|
with(basic_auth: ['john', 'doe']).
|
|
1641
1970
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1642
1971
|
# creation of index which can rollover
|
|
1643
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp
|
|
1972
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-#{timestr}-000001%3E").
|
|
1644
1973
|
with(basic_auth: ['john', 'doe']).
|
|
1645
1974
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1646
1975
|
# check if alias exists
|
|
1647
|
-
timestr = Time.now.strftime("%Y.%m.%d")
|
|
1648
1976
|
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-#{timestr}").
|
|
1649
1977
|
with(basic_auth: ['john', 'doe']).
|
|
1650
1978
|
to_return(:status => 404, :body => "", :headers => {})
|
|
1651
1979
|
# put the alias for the index
|
|
1652
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp
|
|
1653
|
-
with(basic_auth: ['john', 'doe']
|
|
1980
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-#{timestr}-000001%3E/#{alias_endpoint}/mylogs-#{timestr}").
|
|
1981
|
+
with(basic_auth: ['john', 'doe'],
|
|
1982
|
+
body: "{\"aliases\":{\"mylogs-#{timestr}\":{\"is_write_index\":true}}}").
|
|
1654
1983
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1655
1984
|
|
|
1656
1985
|
driver(config)
|
|
@@ -1833,52 +2162,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1833
2162
|
enable_ilm true
|
|
1834
2163
|
}
|
|
1835
2164
|
|
|
1836
|
-
#
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
# check if template exists
|
|
1841
|
-
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
|
|
1842
|
-
with(basic_auth: ['john', 'doe']).
|
|
1843
|
-
to_return(:status => 404, :body => "", :headers => {})
|
|
1844
|
-
# creation
|
|
1845
|
-
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
|
|
1846
|
-
with(basic_auth: ['john', 'doe']).
|
|
1847
|
-
to_return(:status => 200, :body => "", :headers => {})
|
|
1848
|
-
# creation of index which can rollover
|
|
1849
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1850
|
-
with(basic_auth: ['john', 'doe']).
|
|
1851
|
-
to_return(:status => 200, :body => "", :headers => {})
|
|
1852
|
-
# check if alias exists
|
|
1853
|
-
stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
|
|
1854
|
-
with(basic_auth: ['john', 'doe']).
|
|
1855
|
-
to_return(:status => 404, :body => "", :headers => {})
|
|
1856
|
-
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
|
|
1857
|
-
with(basic_auth: ['john', 'doe']).
|
|
1858
|
-
to_return(status: 404, body: "", headers: {})
|
|
1859
|
-
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
|
|
1860
|
-
with(basic_auth: ['john', 'doe'],
|
|
1861
|
-
body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"myapp_deflector-*\"}").
|
|
1862
|
-
to_return(status: 200, body: "", headers: {})
|
|
1863
|
-
# put the alias for the index
|
|
1864
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/myapp_deflector").
|
|
1865
|
-
with(basic_auth: ['john', 'doe'],
|
|
1866
|
-
:body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
|
|
1867
|
-
to_return(:status => 200, :body => "", :headers => {})
|
|
1868
|
-
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1869
|
-
with(basic_auth: ['john', 'doe']).
|
|
1870
|
-
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1871
|
-
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1872
|
-
with(basic_auth: ['john', 'doe']).
|
|
1873
|
-
to_return(:status => 404, :body => "", :headers => {})
|
|
1874
|
-
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1875
|
-
with(basic_auth: ['john', 'doe'],
|
|
1876
|
-
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1877
|
-
to_return(:status => 200, :body => "", :headers => {})
|
|
1878
|
-
|
|
1879
|
-
driver(config)
|
|
1880
|
-
|
|
1881
|
-
assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_deflector", times: 1)
|
|
2165
|
+
# Should raise error because multiple alias indices IllegalArgument Error on executing ILM feature
|
|
2166
|
+
assert_raise(Fluent::ConfigError) do
|
|
2167
|
+
driver(config)
|
|
2168
|
+
end
|
|
1882
2169
|
end
|
|
1883
2170
|
|
|
1884
2171
|
def test_custom_template_with_rollover_index_create_and_default_ilm_and_placeholders
|
|
@@ -2929,6 +3216,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
2929
3216
|
|
|
2930
3217
|
data("border" => {"es_version" => 6, "_type" => "mytype"},
|
|
2931
3218
|
"fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
|
|
3219
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
|
2932
3220
|
)
|
|
2933
3221
|
def test_writes_to_speficied_type(data)
|
|
2934
3222
|
driver('', data["es_version"]).configure("type_name mytype\n")
|
|
@@ -2941,6 +3229,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
2941
3229
|
|
|
2942
3230
|
data("border" => {"es_version" => 6, "_type" => "mytype.test"},
|
|
2943
3231
|
"fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
|
|
3232
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
|
2944
3233
|
)
|
|
2945
3234
|
def test_writes_to_speficied_type_with_placeholders(data)
|
|
2946
3235
|
driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
|
|
@@ -2951,6 +3240,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
2951
3240
|
assert_equal(data['_type'], index_cmds.first['index']['_type'])
|
|
2952
3241
|
end
|
|
2953
3242
|
|
|
3243
|
+
data("border" => {"es_version" => 6, "_type" => "mytype.test"},
|
|
3244
|
+
"fixed_behavior"=> {"es_version" => 7, "_type" => nil},
|
|
3245
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
|
3246
|
+
)
|
|
3247
|
+
def test_writes_to_speficied_type_with_suppress_type_name(data)
|
|
3248
|
+
driver('', data["es_version"])
|
|
3249
|
+
.configure("type_name mytype.${tag}\nsuppress_type_name true")
|
|
3250
|
+
stub_elastic
|
|
3251
|
+
driver.run(default_tag: 'test') do
|
|
3252
|
+
driver.feed(sample_record)
|
|
3253
|
+
end
|
|
3254
|
+
assert_equal(data['_type'], index_cmds.first['index']['_type'])
|
|
3255
|
+
end
|
|
3256
|
+
|
|
2954
3257
|
data("old" => {"es_version" => 2, "_type" => "local-override"},
|
|
2955
3258
|
"old_behavior" => {"es_version" => 5, "_type" => "local-override"},
|
|
2956
3259
|
"border" => {"es_version" => 6, "_type" => "fluentd"},
|