fluent-plugin-elasticsearch 4.0.8 → 4.0.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/History.md +4 -0
- data/README.md +36 -0
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +25 -12
- data/test/plugin/test_out_elasticsearch.rb +218 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f76e6d69d85cace43250e57e272d26e492e4ae5d48856306687efc487279dd4d
|
4
|
+
data.tar.gz: c6f583de5857aba3c9a01bdde72e16c851d52cdc07b1151052155356954155ec
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 58003e22128dc44d615351ca0b36bd6bc2495d21b146f08713db6f8cfc77323192c38c841df446f30ec6d0edde89cced9c51a676f523e44d7488444ec9f2e737
|
7
|
+
data.tar.gz: 9b34567bea973a765fb2dbf699dadd26d80fd50b45b7b1179469d60425c67602e3d288f8da25305c089c20eb6daf236b900931ca0abd503d2665a5d6ab153c12
|
data/History.md
CHANGED
data/README.md
CHANGED
@@ -93,9 +93,11 @@ Current maintainers: @cosmo0920
|
|
93
93
|
+ [enable_ilm](#enable_ilm)
|
94
94
|
+ [ilm_policy_id](#ilm_policy_id)
|
95
95
|
+ [ilm_policy](#ilm_policy)
|
96
|
+
+ [ilm_policies](#ilm_policies)
|
96
97
|
+ [ilm_policy_overwrite](#ilm_policy_overwrite)
|
97
98
|
+ [truncate_caches_interval](#truncate_caches_interval)
|
98
99
|
* [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
|
100
|
+
* [Elasticsearch permissions](#elasticsearch-permissions)
|
99
101
|
* [Troubleshooting](#troubleshooting)
|
100
102
|
+ [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
|
101
103
|
+ [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
|
@@ -1213,6 +1215,14 @@ Default value is `{}`.
|
|
1213
1215
|
|
1214
1216
|
**NOTE:** This parameter requests to install elasticsearch-xpack gem.
|
1215
1217
|
|
1218
|
+
## ilm_policies
|
1219
|
+
|
1220
|
+
A hash in the format `{"ilm_policy_id1":{ <ILM policy 1 hash> }, "ilm_policy_id2": { <ILM policy 2 hash> }}`.
|
1221
|
+
|
1222
|
+
Default value is `{}`.
|
1223
|
+
|
1224
|
+
**NOTE:** This parameter requests to install elasticsearch-xpack gem.
|
1225
|
+
|
1216
1226
|
## ilm_policy_overwrite
|
1217
1227
|
|
1218
1228
|
Specify whether overwriting ilm policy or not.
|
@@ -1233,6 +1243,32 @@ Default value is `nil`.
|
|
1233
1243
|
|
1234
1244
|
See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)
|
1235
1245
|
|
1246
|
+
## Elasticsearch permissions
|
1247
|
+
|
1248
|
+
If the target Elasticsearch requires authentication, a user holding the necessary permissions needs to be provided.
|
1249
|
+
|
1250
|
+
The set of required permissions are the following:
|
1251
|
+
|
1252
|
+
```json
|
1253
|
+
"cluster": ["manage_index_templates", "monitor", "manage_ilm"],
|
1254
|
+
"indices": [
|
1255
|
+
{
|
1256
|
+
"names": [ "*" ],
|
1257
|
+
"privileges": ["write","create","delete","create_index","manage","manage_ilm"]
|
1258
|
+
}
|
1259
|
+
]
|
1260
|
+
```
|
1261
|
+
|
1262
|
+
These permissions can be narrowed down by:
|
1263
|
+
|
1264
|
+
- Setting a more specific pattern for indices under the `names` field
|
1265
|
+
- Removing the `manage_index_templates` cluster permission when not using the feature within your plugin configuration
|
1266
|
+
- Removing the `manage_ilm` cluster permission and the `manage` and `manage_ilm` indices privileges when not using ilm
|
1267
|
+
features in the plugin configuration
|
1268
|
+
|
1269
|
+
The list of privileges along with their description can be found in
|
1270
|
+
[security privileges](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-privileges.html).
|
1271
|
+
|
1236
1272
|
## Troubleshooting
|
1237
1273
|
|
1238
1274
|
### Cannot send events to Elasticsearch
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '4.0.
|
6
|
+
s.version = '4.0.9'
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -160,6 +160,7 @@ EOC
|
|
160
160
|
config_param :enable_ilm, :bool, :default => false
|
161
161
|
config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
|
162
162
|
config_param :ilm_policy, :hash, :default => {}
|
163
|
+
config_param :ilm_policies, :hash, :default => {}
|
163
164
|
config_param :ilm_policy_overwrite, :bool, :default => false
|
164
165
|
config_param :truncate_caches_interval, :time, :default => nil
|
165
166
|
|
@@ -220,6 +221,11 @@ EOC
|
|
220
221
|
log.info "host placeholder and template installation makes your Elasticsearch cluster a bit slow down(beta)."
|
221
222
|
end
|
222
223
|
|
224
|
+
raise Fluent::ConfigError, "You can't specify ilm_policy and ilm_policies at the same time" unless @ilm_policy.empty? or @ilm_policies.empty?
|
225
|
+
|
226
|
+
unless @ilm_policy.empty?
|
227
|
+
@ilm_policies = { @ilm_policy_id => @ilm_policy }
|
228
|
+
end
|
223
229
|
@alias_indexes = []
|
224
230
|
@template_names = []
|
225
231
|
if !dry_run?
|
@@ -235,7 +241,7 @@ EOC
|
|
235
241
|
alias_method :template_installation, :template_installation_actual
|
236
242
|
end
|
237
243
|
else
|
238
|
-
template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name)
|
244
|
+
template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name, @ilm_policy_id)
|
239
245
|
end
|
240
246
|
verify_ilm_working if @enable_ilm
|
241
247
|
elsif @templates
|
@@ -719,7 +725,12 @@ EOC
|
|
719
725
|
else
|
720
726
|
pipeline = nil
|
721
727
|
end
|
722
|
-
|
728
|
+
if @ilm_policy_id
|
729
|
+
ilm_policy_id = extract_placeholders(@ilm_policy_id, chunk)
|
730
|
+
else
|
731
|
+
ilm_policy_id = nil
|
732
|
+
end
|
733
|
+
return logstash_prefix, logstash_dateformat, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline, ilm_policy_id
|
723
734
|
end
|
724
735
|
|
725
736
|
def multi_workers_ready?
|
@@ -793,7 +804,7 @@ EOC
|
|
793
804
|
end
|
794
805
|
|
795
806
|
def process_message(tag, meta, header, time, record, extracted_values)
|
796
|
-
logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline = extracted_values
|
807
|
+
logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline, _ilm_policy_id = extracted_values
|
797
808
|
|
798
809
|
if @flatten_hashes
|
799
810
|
record = flatten_record(record)
|
@@ -901,16 +912,17 @@ EOC
|
|
901
912
|
placeholder?(:logstash_prefix, @logstash_prefix.to_s) ||
|
902
913
|
placeholder?(:logstash_dateformat, @logstash_dateformat.to_s) ||
|
903
914
|
placeholder?(:deflector_alias, @deflector_alias.to_s) ||
|
904
|
-
placeholder?(:application_name, @application_name.to_s)
|
915
|
+
placeholder?(:application_name, @application_name.to_s) ||
|
916
|
+
placeholder?(:ilm_policy_id, @ilm_policy_id.to_s)
|
905
917
|
log.debug("Need substitution: #{need_substitution}")
|
906
918
|
need_substitution
|
907
919
|
end
|
908
920
|
|
909
|
-
def template_installation(deflector_alias, template_name, customize_template, application_name, target_index, host)
|
921
|
+
def template_installation(deflector_alias, template_name, customize_template, application_name, ilm_policy_id, target_index, host)
|
910
922
|
# for safety.
|
911
923
|
end
|
912
924
|
|
913
|
-
def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, host=nil)
|
925
|
+
def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, ilm_policy_id, host=nil)
|
914
926
|
if template_name && @template_file
|
915
927
|
if @alias_indexes.include? deflector_alias
|
916
928
|
log.debug("Index alias #{deflector_alias} already exists (cached)")
|
@@ -919,11 +931,12 @@ EOC
|
|
919
931
|
else
|
920
932
|
retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
|
921
933
|
if customize_template
|
922
|
-
template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias,
|
934
|
+
template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host)
|
923
935
|
else
|
924
|
-
template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias,
|
936
|
+
template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host)
|
925
937
|
end
|
926
|
-
|
938
|
+
ilm_policy = @ilm_policies[ilm_policy_id] || {}
|
939
|
+
create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, ilm_policy_id, ilm_policy, @ilm_policy_overwrite, host)
|
927
940
|
end
|
928
941
|
@alias_indexes << deflector_alias unless deflector_alias.nil?
|
929
942
|
@template_names << template_name unless template_name.nil?
|
@@ -934,11 +947,11 @@ EOC
|
|
934
947
|
# send_bulk given a specific bulk request, the original tag,
|
935
948
|
# chunk, and bulk_message_count
|
936
949
|
def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
|
937
|
-
logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline = extracted_values
|
950
|
+
logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline, ilm_policy_id = extracted_values
|
938
951
|
if deflector_alias
|
939
|
-
template_installation(deflector_alias, template_name, customize_template, application_name, index_name, info.host)
|
952
|
+
template_installation(deflector_alias, template_name, customize_template, application_name, index_name, ilm_policy_id, info.host)
|
940
953
|
else
|
941
|
-
template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, info.host)
|
954
|
+
template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, ilm_policy_id, info.host)
|
942
955
|
end
|
943
956
|
|
944
957
|
begin
|
@@ -1317,6 +1317,224 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1317
1317
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1318
1318
|
end
|
1319
1319
|
|
1320
|
+
def test_template_create_with_rollover_index_and_ilm_policies_and_placeholders
|
1321
|
+
cwd = File.dirname(__FILE__)
|
1322
|
+
template_file = File.join(cwd, 'test_template.json')
|
1323
|
+
|
1324
|
+
config = %{
|
1325
|
+
host logs.google.com
|
1326
|
+
port 777
|
1327
|
+
scheme https
|
1328
|
+
path /es/
|
1329
|
+
user john
|
1330
|
+
password doe
|
1331
|
+
template_name logstash
|
1332
|
+
template_file #{template_file}
|
1333
|
+
index_date_pattern now/w{xxxx.ww}
|
1334
|
+
ilm_policy_id fluentd-policy
|
1335
|
+
enable_ilm true
|
1336
|
+
index_name logstash
|
1337
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
1338
|
+
}
|
1339
|
+
|
1340
|
+
# connection start
|
1341
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
1342
|
+
with(basic_auth: ['john', 'doe']).
|
1343
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1344
|
+
# check if template exists
|
1345
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1346
|
+
with(basic_auth: ['john', 'doe']).
|
1347
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1348
|
+
# creation
|
1349
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1350
|
+
with(basic_auth: ['john', 'doe']).
|
1351
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1352
|
+
# check if alias exists
|
1353
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1354
|
+
with(basic_auth: ['john', 'doe']).
|
1355
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1356
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1357
|
+
with(basic_auth: ['john', 'doe']).
|
1358
|
+
to_return(status: 404, body: "", headers: {})
|
1359
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1360
|
+
with(basic_auth: ['john', 'doe'],
|
1361
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1362
|
+
to_return(status: 200, body: "", headers: {})
|
1363
|
+
# put the alias for the index
|
1364
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1365
|
+
with(basic_auth: ['john', 'doe']).
|
1366
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1367
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
1368
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
1369
|
+
to_return(status: 200, body: "", headers: {})
|
1370
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1371
|
+
with(basic_auth: ['john', 'doe']).
|
1372
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1373
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
1374
|
+
with(basic_auth: ['john', 'doe']).
|
1375
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1376
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
1377
|
+
with(basic_auth: ['john', 'doe'],
|
1378
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
1379
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1380
|
+
|
1381
|
+
driver(config)
|
1382
|
+
|
1383
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1384
|
+
driver.run(default_tag: 'test') do
|
1385
|
+
driver.feed(sample_record)
|
1386
|
+
end
|
1387
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1388
|
+
|
1389
|
+
assert_requested(elastic_request)
|
1390
|
+
end
|
1391
|
+
|
1392
|
+
class TemplateCreateWithRolloverIndexAndILMPoliciesWithPlaceholdersTest < self
|
1393
|
+
def test_tag_placeholder
|
1394
|
+
cwd = File.dirname(__FILE__)
|
1395
|
+
template_file = File.join(cwd, 'test_template.json')
|
1396
|
+
|
1397
|
+
config = %{
|
1398
|
+
host logs.google.com
|
1399
|
+
port 777
|
1400
|
+
scheme https
|
1401
|
+
path /es/
|
1402
|
+
user john
|
1403
|
+
password doe
|
1404
|
+
template_name logstash
|
1405
|
+
template_file #{template_file}
|
1406
|
+
index_date_pattern now/w{xxxx.ww}
|
1407
|
+
ilm_policy_id ${tag}
|
1408
|
+
enable_ilm true
|
1409
|
+
index_name logstash
|
1410
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
1411
|
+
}
|
1412
|
+
|
1413
|
+
# connection start
|
1414
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
1415
|
+
with(basic_auth: ['john', 'doe']).
|
1416
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1417
|
+
# check if template exists
|
1418
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1419
|
+
with(basic_auth: ['john', 'doe']).
|
1420
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1421
|
+
# creation
|
1422
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1423
|
+
with(basic_auth: ['john', 'doe']).
|
1424
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1425
|
+
# check if alias exists
|
1426
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1427
|
+
with(basic_auth: ['john', 'doe']).
|
1428
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1429
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1430
|
+
with(basic_auth: ['john', 'doe']).
|
1431
|
+
to_return(status: 404, body: "", headers: {})
|
1432
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1433
|
+
with(basic_auth: ['john', 'doe'],
|
1434
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1435
|
+
to_return(status: 200, body: "", headers: {})
|
1436
|
+
# put the alias for the index
|
1437
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1438
|
+
with(basic_auth: ['john', 'doe']).
|
1439
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1440
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
1441
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
1442
|
+
to_return(status: 200, body: "", headers: {})
|
1443
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1444
|
+
with(basic_auth: ['john', 'doe']).
|
1445
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1446
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
1447
|
+
with(basic_auth: ['john', 'doe']).
|
1448
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1449
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
1450
|
+
with(basic_auth: ['john', 'doe'],
|
1451
|
+
body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
1452
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1453
|
+
|
1454
|
+
driver(config)
|
1455
|
+
|
1456
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1457
|
+
driver.run(default_tag: 'fluentd-policy') do
|
1458
|
+
driver.feed(sample_record)
|
1459
|
+
end
|
1460
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1461
|
+
|
1462
|
+
assert_requested(elastic_request)
|
1463
|
+
end
|
1464
|
+
|
1465
|
+
def test_tag_placeholder_with_multiple_policies
|
1466
|
+
cwd = File.dirname(__FILE__)
|
1467
|
+
template_file = File.join(cwd, 'test_template.json')
|
1468
|
+
|
1469
|
+
config = %{
|
1470
|
+
host logs.google.com
|
1471
|
+
port 777
|
1472
|
+
scheme https
|
1473
|
+
path /es/
|
1474
|
+
user john
|
1475
|
+
password doe
|
1476
|
+
template_name logstash
|
1477
|
+
template_file #{template_file}
|
1478
|
+
index_date_pattern now/w{xxxx.ww}
|
1479
|
+
ilm_policy_id ${tag}
|
1480
|
+
enable_ilm true
|
1481
|
+
index_name logstash
|
1482
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}, "fluentd-policy2":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"80gb", "max_age":"20d"}}}}}}}
|
1483
|
+
}
|
1484
|
+
|
1485
|
+
# connection start
|
1486
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
1487
|
+
with(basic_auth: ['john', 'doe']).
|
1488
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1489
|
+
# check if template exists
|
1490
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1491
|
+
with(basic_auth: ['john', 'doe']).
|
1492
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1493
|
+
# creation
|
1494
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1495
|
+
with(basic_auth: ['john', 'doe']).
|
1496
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1497
|
+
# check if alias exists
|
1498
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1499
|
+
with(basic_auth: ['john', 'doe']).
|
1500
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1501
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1502
|
+
with(basic_auth: ['john', 'doe']).
|
1503
|
+
to_return(status: 404, body: "", headers: {})
|
1504
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1505
|
+
with(basic_auth: ['john', 'doe'],
|
1506
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1507
|
+
to_return(status: 200, body: "", headers: {})
|
1508
|
+
# put the alias for the index
|
1509
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1510
|
+
with(basic_auth: ['john', 'doe']).
|
1511
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1512
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
1513
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
1514
|
+
to_return(status: 200, body: "", headers: {})
|
1515
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1516
|
+
with(basic_auth: ['john', 'doe']).
|
1517
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1518
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
|
1519
|
+
with(basic_auth: ['john', 'doe']).
|
1520
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1521
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
|
1522
|
+
with(basic_auth: ['john', 'doe'],
|
1523
|
+
body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"80gb\",\"max_age\":\"20d\"}}}}}}").
|
1524
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1525
|
+
|
1526
|
+
driver(config)
|
1527
|
+
|
1528
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1529
|
+
driver.run(default_tag: 'fluentd-policy2') do
|
1530
|
+
driver.feed(sample_record)
|
1531
|
+
end
|
1532
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1533
|
+
|
1534
|
+
assert_requested(elastic_request)
|
1535
|
+
end
|
1536
|
+
end
|
1537
|
+
|
1320
1538
|
def test_template_create_with_rollover_index_and_default_ilm_and_placeholders
|
1321
1539
|
cwd = File.dirname(__FILE__)
|
1322
1540
|
template_file = File.join(cwd, 'test_template.json')
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.0.
|
4
|
+
version: 4.0.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2020-05-
|
13
|
+
date: 2020-05-27 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|