fluent-plugin-elasticsearch 4.0.8 → 4.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/issue-auto-closer.yml +12 -0
- data/History.md +23 -0
- data/README.ElasticsearchGenID.md +116 -0
- data/README.md +104 -5
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +9 -0
- data/lib/fluent/plugin/elasticsearch_index_template.rb +19 -11
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +52 -0
- data/lib/fluent/plugin/out_elasticsearch.rb +64 -31
- data/test/plugin/test_elasticsearch_fallback_selector.rb +73 -0
- data/test/plugin/test_filter_elasticsearch_genid.rb +171 -0
- data/test/plugin/test_out_elasticsearch.rb +375 -72
- metadata +7 -2
|
@@ -1053,6 +1053,80 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1053
1053
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1054
1054
|
end
|
|
1055
1055
|
|
|
1056
|
+
def test_template_create_with_rollover_index_and_default_ilm_on_logstash_format
|
|
1057
|
+
cwd = File.dirname(__FILE__)
|
|
1058
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1059
|
+
|
|
1060
|
+
config = %{
|
|
1061
|
+
host logs.google.com
|
|
1062
|
+
port 777
|
|
1063
|
+
scheme https
|
|
1064
|
+
path /es/
|
|
1065
|
+
user john
|
|
1066
|
+
password doe
|
|
1067
|
+
template_name logstash
|
|
1068
|
+
template_file #{template_file}
|
|
1069
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1070
|
+
enable_ilm true
|
|
1071
|
+
logstash_format true
|
|
1072
|
+
application_name log
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
date_str = Time.now.strftime("%Y.%m.%d")
|
|
1076
|
+
# connection start
|
|
1077
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1078
|
+
with(basic_auth: ['john', 'doe']).
|
|
1079
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1080
|
+
# check if template exists
|
|
1081
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
|
|
1082
|
+
with(basic_auth: ['john', 'doe']).
|
|
1083
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1084
|
+
# creation
|
|
1085
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
|
|
1086
|
+
with(basic_auth: ['john', 'doe']).
|
|
1087
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1088
|
+
# check if alias exists
|
|
1089
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-#{date_str}").
|
|
1090
|
+
with(basic_auth: ['john', 'doe']).
|
|
1091
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1092
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
|
|
1093
|
+
with(basic_auth: ['john', 'doe']).
|
|
1094
|
+
to_return(status: 404, body: "", headers: {})
|
|
1095
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
|
|
1096
|
+
with(basic_auth: ['john', 'doe'],
|
|
1097
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-log-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-log-#{date_str}-*\",\"order\":53}").
|
|
1098
|
+
to_return(status: 200, body: "", headers: {})
|
|
1099
|
+
# put the alias for the index
|
|
1100
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-#{date_str}-000001%3E").
|
|
1101
|
+
with(basic_auth: ['john', 'doe']).
|
|
1102
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1103
|
+
|
|
1104
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-#{date_str}-000001%3E/#{alias_endpoint}/logstash-#{date_str}").
|
|
1105
|
+
with(basic_auth: ['john', 'doe'],
|
|
1106
|
+
body: "{\"aliases\":{\"logstash-#{date_str}\":{\"is_write_index\":true}}}").
|
|
1107
|
+
to_return(status: 200, body: "", headers: {})
|
|
1108
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1109
|
+
with(basic_auth: ['john', 'doe']).
|
|
1110
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1111
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1112
|
+
with(basic_auth: ['john', 'doe']).
|
|
1113
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1114
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1115
|
+
with(basic_auth: ['john', 'doe'],
|
|
1116
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1117
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1118
|
+
|
|
1119
|
+
driver(config)
|
|
1120
|
+
|
|
1121
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1122
|
+
driver.run(default_tag: 'test') do
|
|
1123
|
+
driver.feed(sample_record)
|
|
1124
|
+
end
|
|
1125
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash-#{date_str}", times: 1)
|
|
1126
|
+
|
|
1127
|
+
assert_requested(elastic_request)
|
|
1128
|
+
end
|
|
1129
|
+
|
|
1056
1130
|
def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite
|
|
1057
1131
|
cwd = File.dirname(__FILE__)
|
|
1058
1132
|
template_file = File.join(cwd, 'test_template.json')
|
|
@@ -1139,6 +1213,30 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1139
1213
|
enable_ilm true
|
|
1140
1214
|
}
|
|
1141
1215
|
|
|
1216
|
+
# Should raise error because multiple alias indices IllegalArgument Error on executing ILM feature
|
|
1217
|
+
assert_raise(Fluent::ConfigError) do
|
|
1218
|
+
driver(config)
|
|
1219
|
+
end
|
|
1220
|
+
end
|
|
1221
|
+
|
|
1222
|
+
def test_template_create_with_rollover_index_and_default_ilm_with_empty_index_date_pattern
|
|
1223
|
+
cwd = File.dirname(__FILE__)
|
|
1224
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1225
|
+
|
|
1226
|
+
config = %{
|
|
1227
|
+
host logs.google.com
|
|
1228
|
+
port 777
|
|
1229
|
+
scheme https
|
|
1230
|
+
path /es/
|
|
1231
|
+
user john
|
|
1232
|
+
password doe
|
|
1233
|
+
template_name logstash
|
|
1234
|
+
template_file #{template_file}
|
|
1235
|
+
index_date_pattern ""
|
|
1236
|
+
index_name logstash
|
|
1237
|
+
enable_ilm true
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1142
1240
|
# connection start
|
|
1143
1241
|
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1144
1242
|
with(basic_auth: ['john', 'doe']).
|
|
@@ -1152,23 +1250,23 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1152
1250
|
with(basic_auth: ['john', 'doe']).
|
|
1153
1251
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1154
1252
|
# check if alias exists
|
|
1155
|
-
stub_request(:head, "https://logs.google.com:777/es//_alias/
|
|
1253
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
|
1156
1254
|
with(basic_auth: ['john', 'doe']).
|
|
1157
1255
|
to_return(:status => 404, :body => "", :headers => {})
|
|
1158
|
-
stub_request(:get, "https://logs.google.com:777/es//_template/
|
|
1256
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1159
1257
|
with(basic_auth: ['john', 'doe']).
|
|
1160
1258
|
to_return(status: 404, body: "", headers: {})
|
|
1161
1259
|
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
|
|
1162
1260
|
with(basic_auth: ['john', 'doe'],
|
|
1163
|
-
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"
|
|
1261
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
|
|
1164
1262
|
to_return(status: 200, body: "", headers: {})
|
|
1165
1263
|
# put the alias for the index
|
|
1166
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default
|
|
1264
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E").
|
|
1167
1265
|
with(basic_auth: ['john', 'doe']).
|
|
1168
1266
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1169
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default
|
|
1267
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E/#{alias_endpoint}/logstash").
|
|
1170
1268
|
with(basic_auth: ['john', 'doe'],
|
|
1171
|
-
:body => "{\"aliases\":{\"
|
|
1269
|
+
:body => "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
|
1172
1270
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1173
1271
|
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1174
1272
|
with(basic_auth: ['john', 'doe']).
|
|
@@ -1183,10 +1281,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1183
1281
|
|
|
1184
1282
|
driver(config)
|
|
1185
1283
|
|
|
1186
|
-
assert_requested(:put, "https://logs.google.com:777/es//_template/
|
|
1284
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1187
1285
|
end
|
|
1188
1286
|
|
|
1189
|
-
def
|
|
1287
|
+
def test_template_create_with_rollover_index_and_custom_ilm
|
|
1190
1288
|
cwd = File.dirname(__FILE__)
|
|
1191
1289
|
template_file = File.join(cwd, 'test_template.json')
|
|
1192
1290
|
|
|
@@ -1199,9 +1297,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1199
1297
|
password doe
|
|
1200
1298
|
template_name logstash
|
|
1201
1299
|
template_file #{template_file}
|
|
1202
|
-
index_date_pattern
|
|
1203
|
-
|
|
1300
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1301
|
+
ilm_policy_id fluentd-policy
|
|
1204
1302
|
enable_ilm true
|
|
1303
|
+
index_name logstash
|
|
1304
|
+
ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}
|
|
1205
1305
|
}
|
|
1206
1306
|
|
|
1207
1307
|
# connection start
|
|
@@ -1223,27 +1323,26 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1223
1323
|
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1224
1324
|
with(basic_auth: ['john', 'doe']).
|
|
1225
1325
|
to_return(status: 404, body: "", headers: {})
|
|
1226
|
-
stub_request(:put, "https://logs.google.com:777/es//_template/
|
|
1326
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1227
1327
|
with(basic_auth: ['john', 'doe'],
|
|
1228
|
-
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"
|
|
1328
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
|
1229
1329
|
to_return(status: 200, body: "", headers: {})
|
|
1230
1330
|
# put the alias for the index
|
|
1231
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E").
|
|
1331
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1232
1332
|
with(basic_auth: ['john', 'doe']).
|
|
1233
1333
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1234
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E/#{alias_endpoint}/logstash").
|
|
1235
|
-
with(
|
|
1236
|
-
|
|
1237
|
-
to_return(:status => 200, :body => "", :headers => {})
|
|
1334
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
|
1335
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
|
1336
|
+
to_return(status: 200, body: "", headers: {})
|
|
1238
1337
|
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1239
1338
|
with(basic_auth: ['john', 'doe']).
|
|
1240
1339
|
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1241
|
-
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/
|
|
1340
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1242
1341
|
with(basic_auth: ['john', 'doe']).
|
|
1243
1342
|
to_return(:status => 404, :body => "", :headers => {})
|
|
1244
|
-
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/
|
|
1343
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1245
1344
|
with(basic_auth: ['john', 'doe'],
|
|
1246
|
-
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"
|
|
1345
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1247
1346
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1248
1347
|
|
|
1249
1348
|
driver(config)
|
|
@@ -1251,7 +1350,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1251
1350
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1252
1351
|
end
|
|
1253
1352
|
|
|
1254
|
-
def
|
|
1353
|
+
def test_template_create_with_rollover_index_and_ilm_policies_and_placeholders
|
|
1255
1354
|
cwd = File.dirname(__FILE__)
|
|
1256
1355
|
template_file = File.join(cwd, 'test_template.json')
|
|
1257
1356
|
|
|
@@ -1268,7 +1367,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1268
1367
|
ilm_policy_id fluentd-policy
|
|
1269
1368
|
enable_ilm true
|
|
1270
1369
|
index_name logstash
|
|
1271
|
-
|
|
1370
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
|
1272
1371
|
}
|
|
1273
1372
|
|
|
1274
1373
|
# connection start
|
|
@@ -1314,7 +1413,159 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1314
1413
|
|
|
1315
1414
|
driver(config)
|
|
1316
1415
|
|
|
1416
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1417
|
+
driver.run(default_tag: 'test') do
|
|
1418
|
+
driver.feed(sample_record)
|
|
1419
|
+
end
|
|
1317
1420
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1421
|
+
|
|
1422
|
+
assert_requested(elastic_request)
|
|
1423
|
+
end
|
|
1424
|
+
|
|
1425
|
+
class TemplateCreateWithRolloverIndexAndILMPoliciesWithPlaceholdersTest < self
|
|
1426
|
+
def test_tag_placeholder
|
|
1427
|
+
cwd = File.dirname(__FILE__)
|
|
1428
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1429
|
+
|
|
1430
|
+
config = %{
|
|
1431
|
+
host logs.google.com
|
|
1432
|
+
port 777
|
|
1433
|
+
scheme https
|
|
1434
|
+
path /es/
|
|
1435
|
+
user john
|
|
1436
|
+
password doe
|
|
1437
|
+
template_name logstash
|
|
1438
|
+
template_file #{template_file}
|
|
1439
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1440
|
+
ilm_policy_id ${tag}
|
|
1441
|
+
enable_ilm true
|
|
1442
|
+
index_name logstash
|
|
1443
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
# connection start
|
|
1447
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1448
|
+
with(basic_auth: ['john', 'doe']).
|
|
1449
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1450
|
+
# check if template exists
|
|
1451
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1452
|
+
with(basic_auth: ['john', 'doe']).
|
|
1453
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1454
|
+
# creation
|
|
1455
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1456
|
+
with(basic_auth: ['john', 'doe']).
|
|
1457
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1458
|
+
# check if alias exists
|
|
1459
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
|
1460
|
+
with(basic_auth: ['john', 'doe']).
|
|
1461
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1462
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1463
|
+
with(basic_auth: ['john', 'doe']).
|
|
1464
|
+
to_return(status: 404, body: "", headers: {})
|
|
1465
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1466
|
+
with(basic_auth: ['john', 'doe'],
|
|
1467
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
|
1468
|
+
to_return(status: 200, body: "", headers: {})
|
|
1469
|
+
# put the alias for the index
|
|
1470
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1471
|
+
with(basic_auth: ['john', 'doe']).
|
|
1472
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1473
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
|
1474
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
|
1475
|
+
to_return(status: 200, body: "", headers: {})
|
|
1476
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1477
|
+
with(basic_auth: ['john', 'doe']).
|
|
1478
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1479
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1480
|
+
with(basic_auth: ['john', 'doe']).
|
|
1481
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1482
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1483
|
+
with(basic_auth: ['john', 'doe'],
|
|
1484
|
+
body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1485
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1486
|
+
|
|
1487
|
+
driver(config)
|
|
1488
|
+
|
|
1489
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1490
|
+
driver.run(default_tag: 'fluentd-policy') do
|
|
1491
|
+
driver.feed(sample_record)
|
|
1492
|
+
end
|
|
1493
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1494
|
+
|
|
1495
|
+
assert_requested(elastic_request)
|
|
1496
|
+
end
|
|
1497
|
+
|
|
1498
|
+
def test_tag_placeholder_with_multiple_policies
|
|
1499
|
+
cwd = File.dirname(__FILE__)
|
|
1500
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1501
|
+
|
|
1502
|
+
config = %{
|
|
1503
|
+
host logs.google.com
|
|
1504
|
+
port 777
|
|
1505
|
+
scheme https
|
|
1506
|
+
path /es/
|
|
1507
|
+
user john
|
|
1508
|
+
password doe
|
|
1509
|
+
template_name logstash
|
|
1510
|
+
template_file #{template_file}
|
|
1511
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1512
|
+
ilm_policy_id ${tag}
|
|
1513
|
+
enable_ilm true
|
|
1514
|
+
index_name logstash
|
|
1515
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}, "fluentd-policy2":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"80gb", "max_age":"20d"}}}}}}}
|
|
1516
|
+
}
|
|
1517
|
+
|
|
1518
|
+
# connection start
|
|
1519
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1520
|
+
with(basic_auth: ['john', 'doe']).
|
|
1521
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1522
|
+
# check if template exists
|
|
1523
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1524
|
+
with(basic_auth: ['john', 'doe']).
|
|
1525
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1526
|
+
# creation
|
|
1527
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1528
|
+
with(basic_auth: ['john', 'doe']).
|
|
1529
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1530
|
+
# check if alias exists
|
|
1531
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
|
1532
|
+
with(basic_auth: ['john', 'doe']).
|
|
1533
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1534
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
1535
|
+
with(basic_auth: ['john', 'doe']).
|
|
1536
|
+
to_return(status: 404, body: "", headers: {})
|
|
1537
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
1538
|
+
with(basic_auth: ['john', 'doe'],
|
|
1539
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
|
1540
|
+
to_return(status: 200, body: "", headers: {})
|
|
1541
|
+
# put the alias for the index
|
|
1542
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1543
|
+
with(basic_auth: ['john', 'doe']).
|
|
1544
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1545
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
|
1546
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
|
1547
|
+
to_return(status: 200, body: "", headers: {})
|
|
1548
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1549
|
+
with(basic_auth: ['john', 'doe']).
|
|
1550
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1551
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
|
|
1552
|
+
with(basic_auth: ['john', 'doe']).
|
|
1553
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1554
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
|
|
1555
|
+
with(basic_auth: ['john', 'doe'],
|
|
1556
|
+
body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"80gb\",\"max_age\":\"20d\"}}}}}}").
|
|
1557
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1558
|
+
|
|
1559
|
+
driver(config)
|
|
1560
|
+
|
|
1561
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1562
|
+
driver.run(default_tag: 'fluentd-policy2') do
|
|
1563
|
+
driver.feed(sample_record)
|
|
1564
|
+
end
|
|
1565
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
1566
|
+
|
|
1567
|
+
assert_requested(elastic_request)
|
|
1568
|
+
end
|
|
1318
1569
|
end
|
|
1319
1570
|
|
|
1320
1571
|
def test_template_create_with_rollover_index_and_default_ilm_and_placeholders
|
|
@@ -1389,6 +1640,83 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1389
1640
|
|
|
1390
1641
|
assert_requested(elastic_request)
|
|
1391
1642
|
end
|
|
1643
|
+
|
|
1644
|
+
def test_template_create_with_rollover_index_and_default_ilm_and_custom_and_time_placeholders
|
|
1645
|
+
cwd = File.dirname(__FILE__)
|
|
1646
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
1647
|
+
|
|
1648
|
+
config = Fluent::Config::Element.new(
|
|
1649
|
+
'ROOT', '', {
|
|
1650
|
+
'@type' => 'elasticsearch',
|
|
1651
|
+
'host' => 'logs.google.com',
|
|
1652
|
+
'port' => 777,
|
|
1653
|
+
'scheme' => "https",
|
|
1654
|
+
'path' => "/es/",
|
|
1655
|
+
'user' => 'john',
|
|
1656
|
+
'password' => 'doe',
|
|
1657
|
+
'template_name' => 'logstash',
|
|
1658
|
+
'template_file' => "#{template_file}",
|
|
1659
|
+
'index_date_pattern' => 'now/w{xxxx.ww}',
|
|
1660
|
+
'index_name' => "${taskDef}-%Y.%m",
|
|
1661
|
+
'enable_ilm' => true,
|
|
1662
|
+
}, [
|
|
1663
|
+
Fluent::Config::Element.new('buffer', 'tag, time, taskDef', {
|
|
1664
|
+
'chunk_keys' => ['tag', 'time', 'taskDef'],
|
|
1665
|
+
'timekey' => 3600,
|
|
1666
|
+
}, [])
|
|
1667
|
+
]
|
|
1668
|
+
)
|
|
1669
|
+
|
|
1670
|
+
task_def_value = "task_definition"
|
|
1671
|
+
date_str = Time.now.strftime("%Y.%m")
|
|
1672
|
+
# connection start
|
|
1673
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1674
|
+
with(basic_auth: ['john', 'doe']).
|
|
1675
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1676
|
+
# check if template exists
|
|
1677
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
|
|
1678
|
+
with(basic_auth: ['john', 'doe']).
|
|
1679
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1680
|
+
# creation
|
|
1681
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
|
|
1682
|
+
with(basic_auth: ['john', 'doe'],
|
|
1683
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"#{task_def_value}-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"#{task_def_value}-#{date_str}-*\",\"order\":52}").
|
|
1684
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1685
|
+
# check if alias exists
|
|
1686
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/#{task_def_value}-#{date_str}").
|
|
1687
|
+
with(basic_auth: ['john', 'doe']).
|
|
1688
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1689
|
+
# put the alias for the index
|
|
1690
|
+
stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1691
|
+
with(basic_auth: ['john', 'doe']).
|
|
1692
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1693
|
+
stub_request(:put, "https://logs.google.com:777/es//%3C#{task_def_value}-#{date_str}-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/#{task_def_value}-#{date_str}").
|
|
1694
|
+
with(basic_auth: ['john', 'doe'],
|
|
1695
|
+
:body => "{\"aliases\":{\"#{task_def_value}-#{date_str}\":{\"is_write_index\":true}}}").
|
|
1696
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1697
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1698
|
+
with(basic_auth: ['john', 'doe']).
|
|
1699
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1700
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1701
|
+
with(basic_auth: ['john', 'doe']).
|
|
1702
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1703
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
1704
|
+
with(basic_auth: ['john', 'doe'],
|
|
1705
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1706
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1707
|
+
|
|
1708
|
+
driver(config)
|
|
1709
|
+
|
|
1710
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1711
|
+
driver.run(default_tag: 'test') do
|
|
1712
|
+
driver.feed(sample_record.merge("taskDef" => task_def_value))
|
|
1713
|
+
end
|
|
1714
|
+
assert_equal("#{task_def_value}-#{date_str}", index_cmds.first['index']['_index'])
|
|
1715
|
+
|
|
1716
|
+
assert_equal ["#{task_def_value}-#{date_str}"], driver.instance.alias_indexes
|
|
1717
|
+
|
|
1718
|
+
assert_requested(elastic_request)
|
|
1719
|
+
end
|
|
1392
1720
|
end
|
|
1393
1721
|
|
|
1394
1722
|
def test_custom_template_create
|
|
@@ -1627,6 +1955,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1627
1955
|
application_name myapp
|
|
1628
1956
|
}
|
|
1629
1957
|
|
|
1958
|
+
timestr = Time.now.strftime("%Y.%m.%d")
|
|
1630
1959
|
# connection start
|
|
1631
1960
|
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1632
1961
|
with(basic_auth: ['john', 'doe']).
|
|
@@ -1640,17 +1969,17 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1640
1969
|
with(basic_auth: ['john', 'doe']).
|
|
1641
1970
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1642
1971
|
# creation of index which can rollover
|
|
1643
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp
|
|
1972
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-#{timestr}-000001%3E").
|
|
1644
1973
|
with(basic_auth: ['john', 'doe']).
|
|
1645
1974
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1646
1975
|
# check if alias exists
|
|
1647
|
-
timestr = Time.now.strftime("%Y.%m.%d")
|
|
1648
1976
|
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-#{timestr}").
|
|
1649
1977
|
with(basic_auth: ['john', 'doe']).
|
|
1650
1978
|
to_return(:status => 404, :body => "", :headers => {})
|
|
1651
1979
|
# put the alias for the index
|
|
1652
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp
|
|
1653
|
-
with(basic_auth: ['john', 'doe']
|
|
1980
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-#{timestr}-000001%3E/#{alias_endpoint}/mylogs-#{timestr}").
|
|
1981
|
+
with(basic_auth: ['john', 'doe'],
|
|
1982
|
+
body: "{\"aliases\":{\"mylogs-#{timestr}\":{\"is_write_index\":true}}}").
|
|
1654
1983
|
to_return(:status => 200, :body => "", :headers => {})
|
|
1655
1984
|
|
|
1656
1985
|
driver(config)
|
|
@@ -1833,52 +2162,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
1833
2162
|
enable_ilm true
|
|
1834
2163
|
}
|
|
1835
2164
|
|
|
1836
|
-
#
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
# check if template exists
|
|
1841
|
-
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
|
|
1842
|
-
with(basic_auth: ['john', 'doe']).
|
|
1843
|
-
to_return(:status => 404, :body => "", :headers => {})
|
|
1844
|
-
# creation
|
|
1845
|
-
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
|
|
1846
|
-
with(basic_auth: ['john', 'doe']).
|
|
1847
|
-
to_return(:status => 200, :body => "", :headers => {})
|
|
1848
|
-
# creation of index which can rollover
|
|
1849
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1850
|
-
with(basic_auth: ['john', 'doe']).
|
|
1851
|
-
to_return(:status => 200, :body => "", :headers => {})
|
|
1852
|
-
# check if alias exists
|
|
1853
|
-
stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
|
|
1854
|
-
with(basic_auth: ['john', 'doe']).
|
|
1855
|
-
to_return(:status => 404, :body => "", :headers => {})
|
|
1856
|
-
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
|
|
1857
|
-
with(basic_auth: ['john', 'doe']).
|
|
1858
|
-
to_return(status: 404, body: "", headers: {})
|
|
1859
|
-
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
|
|
1860
|
-
with(basic_auth: ['john', 'doe'],
|
|
1861
|
-
body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"myapp_deflector-*\"}").
|
|
1862
|
-
to_return(status: 200, body: "", headers: {})
|
|
1863
|
-
# put the alias for the index
|
|
1864
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/myapp_deflector").
|
|
1865
|
-
with(basic_auth: ['john', 'doe'],
|
|
1866
|
-
:body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
|
|
1867
|
-
to_return(:status => 200, :body => "", :headers => {})
|
|
1868
|
-
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1869
|
-
with(basic_auth: ['john', 'doe']).
|
|
1870
|
-
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1871
|
-
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1872
|
-
with(basic_auth: ['john', 'doe']).
|
|
1873
|
-
to_return(:status => 404, :body => "", :headers => {})
|
|
1874
|
-
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1875
|
-
with(basic_auth: ['john', 'doe'],
|
|
1876
|
-
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1877
|
-
to_return(:status => 200, :body => "", :headers => {})
|
|
1878
|
-
|
|
1879
|
-
driver(config)
|
|
1880
|
-
|
|
1881
|
-
assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_deflector", times: 1)
|
|
2165
|
+
# Should raise error because multiple alias indices IllegalArgument Error on executing ILM feature
|
|
2166
|
+
assert_raise(Fluent::ConfigError) do
|
|
2167
|
+
driver(config)
|
|
2168
|
+
end
|
|
1882
2169
|
end
|
|
1883
2170
|
|
|
1884
2171
|
def test_custom_template_with_rollover_index_create_and_default_ilm_and_placeholders
|
|
@@ -2929,6 +3216,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
2929
3216
|
|
|
2930
3217
|
data("border" => {"es_version" => 6, "_type" => "mytype"},
|
|
2931
3218
|
"fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
|
|
3219
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
|
2932
3220
|
)
|
|
2933
3221
|
def test_writes_to_speficied_type(data)
|
|
2934
3222
|
driver('', data["es_version"]).configure("type_name mytype\n")
|
|
@@ -2941,6 +3229,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
2941
3229
|
|
|
2942
3230
|
data("border" => {"es_version" => 6, "_type" => "mytype.test"},
|
|
2943
3231
|
"fixed_behavior"=> {"es_version" => 7, "_type" => "_doc"},
|
|
3232
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
|
2944
3233
|
)
|
|
2945
3234
|
def test_writes_to_speficied_type_with_placeholders(data)
|
|
2946
3235
|
driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
|
|
@@ -2951,6 +3240,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
|
2951
3240
|
assert_equal(data['_type'], index_cmds.first['index']['_type'])
|
|
2952
3241
|
end
|
|
2953
3242
|
|
|
3243
|
+
data("border" => {"es_version" => 6, "_type" => "mytype.test"},
|
|
3244
|
+
"fixed_behavior"=> {"es_version" => 7, "_type" => nil},
|
|
3245
|
+
"to be nil" => {"es_version" => 8, "_type" => nil},
|
|
3246
|
+
)
|
|
3247
|
+
def test_writes_to_speficied_type_with_suppress_type_name(data)
|
|
3248
|
+
driver('', data["es_version"])
|
|
3249
|
+
.configure("type_name mytype.${tag}\nsuppress_type_name true")
|
|
3250
|
+
stub_elastic
|
|
3251
|
+
driver.run(default_tag: 'test') do
|
|
3252
|
+
driver.feed(sample_record)
|
|
3253
|
+
end
|
|
3254
|
+
assert_equal(data['_type'], index_cmds.first['index']['_type'])
|
|
3255
|
+
end
|
|
3256
|
+
|
|
2954
3257
|
data("old" => {"es_version" => 2, "_type" => "local-override"},
|
|
2955
3258
|
"old_behavior" => {"es_version" => 5, "_type" => "local-override"},
|
|
2956
3259
|
"border" => {"es_version" => 6, "_type" => "fluentd"},
|