fluent-plugin-elasticsearch 4.0.8 → 4.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/History.md +4 -0
- data/README.md +36 -0
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +25 -12
- data/test/plugin/test_out_elasticsearch.rb +218 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f76e6d69d85cace43250e57e272d26e492e4ae5d48856306687efc487279dd4d
|
4
|
+
data.tar.gz: c6f583de5857aba3c9a01bdde72e16c851d52cdc07b1151052155356954155ec
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 58003e22128dc44d615351ca0b36bd6bc2495d21b146f08713db6f8cfc77323192c38c841df446f30ec6d0edde89cced9c51a676f523e44d7488444ec9f2e737
|
7
|
+
data.tar.gz: 9b34567bea973a765fb2dbf699dadd26d80fd50b45b7b1179469d60425c67602e3d288f8da25305c089c20eb6daf236b900931ca0abd503d2665a5d6ab153c12
|
data/History.md
CHANGED
data/README.md
CHANGED
@@ -93,9 +93,11 @@ Current maintainers: @cosmo0920
|
|
93
93
|
+ [enable_ilm](#enable_ilm)
|
94
94
|
+ [ilm_policy_id](#ilm_policy_id)
|
95
95
|
+ [ilm_policy](#ilm_policy)
|
96
|
+
+ [ilm_policies](#ilm_policies)
|
96
97
|
+ [ilm_policy_overwrite](#ilm_policy_overwrite)
|
97
98
|
+ [truncate_caches_interval](#truncate_caches_interval)
|
98
99
|
* [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
|
100
|
+
* [Elasticsearch permissions](#elasticsearch-permissions)
|
99
101
|
* [Troubleshooting](#troubleshooting)
|
100
102
|
+ [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
|
101
103
|
+ [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
|
@@ -1213,6 +1215,14 @@ Default value is `{}`.
|
|
1213
1215
|
|
1214
1216
|
**NOTE:** This parameter requests to install elasticsearch-xpack gem.
|
1215
1217
|
|
1218
|
+
## ilm_policies
|
1219
|
+
|
1220
|
+
A hash in the format `{"ilm_policy_id1":{ <ILM policy 1 hash> }, "ilm_policy_id2": { <ILM policy 2 hash> }}`.
|
1221
|
+
|
1222
|
+
Default value is `{}`.
|
1223
|
+
|
1224
|
+
**NOTE:** This parameter requests to install elasticsearch-xpack gem.
|
1225
|
+
|
1216
1226
|
## ilm_policy_overwrite
|
1217
1227
|
|
1218
1228
|
Specify whether overwriting ilm policy or not.
|
@@ -1233,6 +1243,32 @@ Default value is `nil`.
|
|
1233
1243
|
|
1234
1244
|
See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)
|
1235
1245
|
|
1246
|
+
## Elasticsearch permissions
|
1247
|
+
|
1248
|
+
If the target Elasticsearch requires authentication, a user holding the necessary permissions needs to be provided.
|
1249
|
+
|
1250
|
+
The set of required permissions are the following:
|
1251
|
+
|
1252
|
+
```json
|
1253
|
+
"cluster": ["manage_index_templates", "monitor", "manage_ilm"],
|
1254
|
+
"indices": [
|
1255
|
+
{
|
1256
|
+
"names": [ "*" ],
|
1257
|
+
"privileges": ["write","create","delete","create_index","manage","manage_ilm"]
|
1258
|
+
}
|
1259
|
+
]
|
1260
|
+
```
|
1261
|
+
|
1262
|
+
These permissions can be narrowed down by:
|
1263
|
+
|
1264
|
+
- Setting a more specific pattern for indices under the `names` field
|
1265
|
+
- Removing the `manage_index_templates` cluster permission when not using the feature within your plugin configuration
|
1266
|
+
- Removing the `manage_ilm` cluster permission and the `manage` and `manage_ilm` indices privileges when not using ilm
|
1267
|
+
features in the plugin configuration
|
1268
|
+
|
1269
|
+
The list of privileges along with their description can be found in
|
1270
|
+
[security privileges](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-privileges.html).
|
1271
|
+
|
1236
1272
|
## Troubleshooting
|
1237
1273
|
|
1238
1274
|
### Cannot send events to Elasticsearch
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '4.0.
|
6
|
+
s.version = '4.0.9'
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -160,6 +160,7 @@ EOC
|
|
160
160
|
config_param :enable_ilm, :bool, :default => false
|
161
161
|
config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
|
162
162
|
config_param :ilm_policy, :hash, :default => {}
|
163
|
+
config_param :ilm_policies, :hash, :default => {}
|
163
164
|
config_param :ilm_policy_overwrite, :bool, :default => false
|
164
165
|
config_param :truncate_caches_interval, :time, :default => nil
|
165
166
|
|
@@ -220,6 +221,11 @@ EOC
|
|
220
221
|
log.info "host placeholder and template installation makes your Elasticsearch cluster a bit slow down(beta)."
|
221
222
|
end
|
222
223
|
|
224
|
+
raise Fluent::ConfigError, "You can't specify ilm_policy and ilm_policies at the same time" unless @ilm_policy.empty? or @ilm_policies.empty?
|
225
|
+
|
226
|
+
unless @ilm_policy.empty?
|
227
|
+
@ilm_policies = { @ilm_policy_id => @ilm_policy }
|
228
|
+
end
|
223
229
|
@alias_indexes = []
|
224
230
|
@template_names = []
|
225
231
|
if !dry_run?
|
@@ -235,7 +241,7 @@ EOC
|
|
235
241
|
alias_method :template_installation, :template_installation_actual
|
236
242
|
end
|
237
243
|
else
|
238
|
-
template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name)
|
244
|
+
template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name, @ilm_policy_id)
|
239
245
|
end
|
240
246
|
verify_ilm_working if @enable_ilm
|
241
247
|
elsif @templates
|
@@ -719,7 +725,12 @@ EOC
|
|
719
725
|
else
|
720
726
|
pipeline = nil
|
721
727
|
end
|
722
|
-
|
728
|
+
if @ilm_policy_id
|
729
|
+
ilm_policy_id = extract_placeholders(@ilm_policy_id, chunk)
|
730
|
+
else
|
731
|
+
ilm_policy_id = nil
|
732
|
+
end
|
733
|
+
return logstash_prefix, logstash_dateformat, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline, ilm_policy_id
|
723
734
|
end
|
724
735
|
|
725
736
|
def multi_workers_ready?
|
@@ -793,7 +804,7 @@ EOC
|
|
793
804
|
end
|
794
805
|
|
795
806
|
def process_message(tag, meta, header, time, record, extracted_values)
|
796
|
-
logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline = extracted_values
|
807
|
+
logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline, _ilm_policy_id = extracted_values
|
797
808
|
|
798
809
|
if @flatten_hashes
|
799
810
|
record = flatten_record(record)
|
@@ -901,16 +912,17 @@ EOC
|
|
901
912
|
placeholder?(:logstash_prefix, @logstash_prefix.to_s) ||
|
902
913
|
placeholder?(:logstash_dateformat, @logstash_dateformat.to_s) ||
|
903
914
|
placeholder?(:deflector_alias, @deflector_alias.to_s) ||
|
904
|
-
placeholder?(:application_name, @application_name.to_s)
|
915
|
+
placeholder?(:application_name, @application_name.to_s) ||
|
916
|
+
placeholder?(:ilm_policy_id, @ilm_policy_id.to_s)
|
905
917
|
log.debug("Need substitution: #{need_substitution}")
|
906
918
|
need_substitution
|
907
919
|
end
|
908
920
|
|
909
|
-
def template_installation(deflector_alias, template_name, customize_template, application_name, target_index, host)
|
921
|
+
def template_installation(deflector_alias, template_name, customize_template, application_name, ilm_policy_id, target_index, host)
|
910
922
|
# for safety.
|
911
923
|
end
|
912
924
|
|
913
|
-
def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, host=nil)
|
925
|
+
def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, ilm_policy_id, host=nil)
|
914
926
|
if template_name && @template_file
|
915
927
|
if @alias_indexes.include? deflector_alias
|
916
928
|
log.debug("Index alias #{deflector_alias} already exists (cached)")
|
@@ -919,11 +931,12 @@ EOC
|
|
919
931
|
else
|
920
932
|
retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
|
921
933
|
if customize_template
|
922
|
-
template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias,
|
934
|
+
template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host)
|
923
935
|
else
|
924
|
-
template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias,
|
936
|
+
template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host)
|
925
937
|
end
|
926
|
-
|
938
|
+
ilm_policy = @ilm_policies[ilm_policy_id] || {}
|
939
|
+
create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, ilm_policy_id, ilm_policy, @ilm_policy_overwrite, host)
|
927
940
|
end
|
928
941
|
@alias_indexes << deflector_alias unless deflector_alias.nil?
|
929
942
|
@template_names << template_name unless template_name.nil?
|
@@ -934,11 +947,11 @@ EOC
|
|
934
947
|
# send_bulk given a specific bulk request, the original tag,
|
935
948
|
# chunk, and bulk_message_count
|
936
949
|
def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
|
937
|
-
logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline = extracted_values
|
950
|
+
logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline, ilm_policy_id = extracted_values
|
938
951
|
if deflector_alias
|
939
|
-
template_installation(deflector_alias, template_name, customize_template, application_name, index_name, info.host)
|
952
|
+
template_installation(deflector_alias, template_name, customize_template, application_name, index_name, ilm_policy_id, info.host)
|
940
953
|
else
|
941
|
-
template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, info.host)
|
954
|
+
template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, ilm_policy_id, info.host)
|
942
955
|
end
|
943
956
|
|
944
957
|
begin
|
@@ -1317,6 +1317,224 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1317
1317
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1318
1318
|
end
|
1319
1319
|
|
1320
|
+
def test_template_create_with_rollover_index_and_ilm_policies_and_placeholders
|
1321
|
+
cwd = File.dirname(__FILE__)
|
1322
|
+
template_file = File.join(cwd, 'test_template.json')
|
1323
|
+
|
1324
|
+
config = %{
|
1325
|
+
host logs.google.com
|
1326
|
+
port 777
|
1327
|
+
scheme https
|
1328
|
+
path /es/
|
1329
|
+
user john
|
1330
|
+
password doe
|
1331
|
+
template_name logstash
|
1332
|
+
template_file #{template_file}
|
1333
|
+
index_date_pattern now/w{xxxx.ww}
|
1334
|
+
ilm_policy_id fluentd-policy
|
1335
|
+
enable_ilm true
|
1336
|
+
index_name logstash
|
1337
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
1338
|
+
}
|
1339
|
+
|
1340
|
+
# connection start
|
1341
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
1342
|
+
with(basic_auth: ['john', 'doe']).
|
1343
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1344
|
+
# check if template exists
|
1345
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1346
|
+
with(basic_auth: ['john', 'doe']).
|
1347
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1348
|
+
# creation
|
1349
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1350
|
+
with(basic_auth: ['john', 'doe']).
|
1351
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1352
|
+
# check if alias exists
|
1353
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1354
|
+
with(basic_auth: ['john', 'doe']).
|
1355
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1356
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1357
|
+
with(basic_auth: ['john', 'doe']).
|
1358
|
+
to_return(status: 404, body: "", headers: {})
|
1359
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1360
|
+
with(basic_auth: ['john', 'doe'],
|
1361
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1362
|
+
to_return(status: 200, body: "", headers: {})
|
1363
|
+
# put the alias for the index
|
1364
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1365
|
+
with(basic_auth: ['john', 'doe']).
|
1366
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1367
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
1368
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
1369
|
+
to_return(status: 200, body: "", headers: {})
|
1370
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1371
|
+
with(basic_auth: ['john', 'doe']).
|
1372
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1373
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
1374
|
+
with(basic_auth: ['john', 'doe']).
|
1375
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1376
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
1377
|
+
with(basic_auth: ['john', 'doe'],
|
1378
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
1379
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1380
|
+
|
1381
|
+
driver(config)
|
1382
|
+
|
1383
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1384
|
+
driver.run(default_tag: 'test') do
|
1385
|
+
driver.feed(sample_record)
|
1386
|
+
end
|
1387
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1388
|
+
|
1389
|
+
assert_requested(elastic_request)
|
1390
|
+
end
|
1391
|
+
|
1392
|
+
class TemplateCreateWithRolloverIndexAndILMPoliciesWithPlaceholdersTest < self
|
1393
|
+
def test_tag_placeholder
|
1394
|
+
cwd = File.dirname(__FILE__)
|
1395
|
+
template_file = File.join(cwd, 'test_template.json')
|
1396
|
+
|
1397
|
+
config = %{
|
1398
|
+
host logs.google.com
|
1399
|
+
port 777
|
1400
|
+
scheme https
|
1401
|
+
path /es/
|
1402
|
+
user john
|
1403
|
+
password doe
|
1404
|
+
template_name logstash
|
1405
|
+
template_file #{template_file}
|
1406
|
+
index_date_pattern now/w{xxxx.ww}
|
1407
|
+
ilm_policy_id ${tag}
|
1408
|
+
enable_ilm true
|
1409
|
+
index_name logstash
|
1410
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
1411
|
+
}
|
1412
|
+
|
1413
|
+
# connection start
|
1414
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
1415
|
+
with(basic_auth: ['john', 'doe']).
|
1416
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1417
|
+
# check if template exists
|
1418
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1419
|
+
with(basic_auth: ['john', 'doe']).
|
1420
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1421
|
+
# creation
|
1422
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1423
|
+
with(basic_auth: ['john', 'doe']).
|
1424
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1425
|
+
# check if alias exists
|
1426
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1427
|
+
with(basic_auth: ['john', 'doe']).
|
1428
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1429
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1430
|
+
with(basic_auth: ['john', 'doe']).
|
1431
|
+
to_return(status: 404, body: "", headers: {})
|
1432
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1433
|
+
with(basic_auth: ['john', 'doe'],
|
1434
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1435
|
+
to_return(status: 200, body: "", headers: {})
|
1436
|
+
# put the alias for the index
|
1437
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1438
|
+
with(basic_auth: ['john', 'doe']).
|
1439
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1440
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
1441
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
1442
|
+
to_return(status: 200, body: "", headers: {})
|
1443
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1444
|
+
with(basic_auth: ['john', 'doe']).
|
1445
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1446
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
1447
|
+
with(basic_auth: ['john', 'doe']).
|
1448
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1449
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
1450
|
+
with(basic_auth: ['john', 'doe'],
|
1451
|
+
body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
1452
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1453
|
+
|
1454
|
+
driver(config)
|
1455
|
+
|
1456
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1457
|
+
driver.run(default_tag: 'fluentd-policy') do
|
1458
|
+
driver.feed(sample_record)
|
1459
|
+
end
|
1460
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1461
|
+
|
1462
|
+
assert_requested(elastic_request)
|
1463
|
+
end
|
1464
|
+
|
1465
|
+
def test_tag_placeholder_with_multiple_policies
|
1466
|
+
cwd = File.dirname(__FILE__)
|
1467
|
+
template_file = File.join(cwd, 'test_template.json')
|
1468
|
+
|
1469
|
+
config = %{
|
1470
|
+
host logs.google.com
|
1471
|
+
port 777
|
1472
|
+
scheme https
|
1473
|
+
path /es/
|
1474
|
+
user john
|
1475
|
+
password doe
|
1476
|
+
template_name logstash
|
1477
|
+
template_file #{template_file}
|
1478
|
+
index_date_pattern now/w{xxxx.ww}
|
1479
|
+
ilm_policy_id ${tag}
|
1480
|
+
enable_ilm true
|
1481
|
+
index_name logstash
|
1482
|
+
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}, "fluentd-policy2":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"80gb", "max_age":"20d"}}}}}}}
|
1483
|
+
}
|
1484
|
+
|
1485
|
+
# connection start
|
1486
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
1487
|
+
with(basic_auth: ['john', 'doe']).
|
1488
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1489
|
+
# check if template exists
|
1490
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1491
|
+
with(basic_auth: ['john', 'doe']).
|
1492
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1493
|
+
# creation
|
1494
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1495
|
+
with(basic_auth: ['john', 'doe']).
|
1496
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1497
|
+
# check if alias exists
|
1498
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1499
|
+
with(basic_auth: ['john', 'doe']).
|
1500
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1501
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1502
|
+
with(basic_auth: ['john', 'doe']).
|
1503
|
+
to_return(status: 404, body: "", headers: {})
|
1504
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1505
|
+
with(basic_auth: ['john', 'doe'],
|
1506
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1507
|
+
to_return(status: 200, body: "", headers: {})
|
1508
|
+
# put the alias for the index
|
1509
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1510
|
+
with(basic_auth: ['john', 'doe']).
|
1511
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1512
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
|
1513
|
+
with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
|
1514
|
+
to_return(status: 200, body: "", headers: {})
|
1515
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1516
|
+
with(basic_auth: ['john', 'doe']).
|
1517
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1518
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
|
1519
|
+
with(basic_auth: ['john', 'doe']).
|
1520
|
+
to_return(:status => 404, :body => "", :headers => {})
|
1521
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
|
1522
|
+
with(basic_auth: ['john', 'doe'],
|
1523
|
+
body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"80gb\",\"max_age\":\"20d\"}}}}}}").
|
1524
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1525
|
+
|
1526
|
+
driver(config)
|
1527
|
+
|
1528
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
1529
|
+
driver.run(default_tag: 'fluentd-policy2') do
|
1530
|
+
driver.feed(sample_record)
|
1531
|
+
end
|
1532
|
+
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
1533
|
+
|
1534
|
+
assert_requested(elastic_request)
|
1535
|
+
end
|
1536
|
+
end
|
1537
|
+
|
1320
1538
|
def test_template_create_with_rollover_index_and_default_ilm_and_placeholders
|
1321
1539
|
cwd = File.dirname(__FILE__)
|
1322
1540
|
template_file = File.join(cwd, 'test_template.json')
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.0.
|
4
|
+
version: 4.0.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2020-05-
|
13
|
+
date: 2020-05-27 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|