fluent-plugin-elasticsearch 4.0.8 → 4.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1979708479afe32ade449552ade837bc9cfc8acff343cd248008abc579b794d3
4
- data.tar.gz: 6ed1a94f6e8553ca72fb75e319ee954f5a8a676ef4ccc0a6a65ca4d041c101c2
3
+ metadata.gz: f76e6d69d85cace43250e57e272d26e492e4ae5d48856306687efc487279dd4d
4
+ data.tar.gz: c6f583de5857aba3c9a01bdde72e16c851d52cdc07b1151052155356954155ec
5
5
  SHA512:
6
- metadata.gz: e9957264a1ade9aa7e3b7cac60157c0adc9d9b2cd9e6d3d72ce73ca480e6ee32415bb94ad83c95ee7419e6b02b6695c28c39e2696945906e35e25c43421f70ca
7
- data.tar.gz: 0efced0623b3312f1d767c82bb3ac9647e0082389bc6457052a4c74f5063a57ba21ff4effe881a18996c0a3f256ade017ec68b9df7423cb5c63869b3e49c1b07
6
+ metadata.gz: 58003e22128dc44d615351ca0b36bd6bc2495d21b146f08713db6f8cfc77323192c38c841df446f30ec6d0edde89cced9c51a676f523e44d7488444ec9f2e737
7
+ data.tar.gz: 9b34567bea973a765fb2dbf699dadd26d80fd50b45b7b1179469d60425c67602e3d288f8da25305c089c20eb6daf236b900931ca0abd503d2665a5d6ab153c12
data/History.md CHANGED
@@ -2,6 +2,10 @@
2
2
 
3
3
  ### [Unreleased]
4
4
 
5
+ ### 4.0.9
6
+ - Add possibility to configure multiple ILM policies (#753)
7
+ - Document required permissions (#757)
8
+
5
9
  ### 4.0.8
6
10
  - Handle compressable connection usable state (#743)
7
11
  - Use newer tls protocol versions (#739)
data/README.md CHANGED
@@ -93,9 +93,11 @@ Current maintainers: @cosmo0920
93
93
  + [enable_ilm](#enable_ilm)
94
94
  + [ilm_policy_id](#ilm_policy_id)
95
95
  + [ilm_policy](#ilm_policy)
96
+ + [ilm_policies](#ilm_policies)
96
97
  + [ilm_policy_overwrite](#ilm_policy_overwrite)
97
98
  + [truncate_caches_interval](#truncate_caches_interval)
98
99
  * [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
100
+ * [Elasticsearch permissions](#elasticsearch-permissions)
99
101
  * [Troubleshooting](#troubleshooting)
100
102
  + [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
101
103
  + [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
@@ -1213,6 +1215,14 @@ Default value is `{}`.
1213
1215
 
1214
1216
  **NOTE:** This parameter requests to install elasticsearch-xpack gem.
1215
1217
 
1218
+ ## ilm_policies
1219
+
1220
+ A hash in the format `{"ilm_policy_id1":{ <ILM policy 1 hash> }, "ilm_policy_id2": { <ILM policy 2 hash> }}`.
1221
+
1222
+ Default value is `{}`.
1223
+
1224
+ **NOTE:** This parameter requests to install elasticsearch-xpack gem.
1225
+
1216
1226
  ## ilm_policy_overwrite
1217
1227
 
1218
1228
  Specify whether overwriting ilm policy or not.
@@ -1233,6 +1243,32 @@ Default value is `nil`.
1233
1243
 
1234
1244
  See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)
1235
1245
 
1246
+ ## Elasticsearch permissions
1247
+
1248
+ If the target Elasticsearch requires authentication, a user holding the necessary permissions needs to be provided.
1249
+
1250
+ The set of required permissions are the following:
1251
+
1252
+ ```json
1253
+ "cluster": ["manage_index_templates", "monitor", "manage_ilm"],
1254
+ "indices": [
1255
+ {
1256
+ "names": [ "*" ],
1257
+ "privileges": ["write","create","delete","create_index","manage","manage_ilm"]
1258
+ }
1259
+ ]
1260
+ ```
1261
+
1262
+ These permissions can be narrowed down by:
1263
+
1264
+ - Setting a more specific pattern for indices under the `names` field
1265
+ - Removing the `manage_index_templates` cluster permission when not using the feature within your plugin configuration
1266
+ - Removing the `manage_ilm` cluster permission and the `manage` and `manage_ilm` indices privileges when not using ilm
1267
+ features in the plugin configuration
1268
+
1269
+ The list of privileges along with their description can be found in
1270
+ [security privileges](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-privileges.html).
1271
+
1236
1272
  ## Troubleshooting
1237
1273
 
1238
1274
  ### Cannot send events to Elasticsearch
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '4.0.8'
6
+ s.version = '4.0.9'
7
7
  s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
9
9
  s.description = %q{Elasticsearch output plugin for Fluent event collector}
@@ -160,6 +160,7 @@ EOC
160
160
  config_param :enable_ilm, :bool, :default => false
161
161
  config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
162
162
  config_param :ilm_policy, :hash, :default => {}
163
+ config_param :ilm_policies, :hash, :default => {}
163
164
  config_param :ilm_policy_overwrite, :bool, :default => false
164
165
  config_param :truncate_caches_interval, :time, :default => nil
165
166
 
@@ -220,6 +221,11 @@ EOC
220
221
  log.info "host placeholder and template installation makes your Elasticsearch cluster a bit slow down(beta)."
221
222
  end
222
223
 
224
+ raise Fluent::ConfigError, "You can't specify ilm_policy and ilm_policies at the same time" unless @ilm_policy.empty? or @ilm_policies.empty?
225
+
226
+ unless @ilm_policy.empty?
227
+ @ilm_policies = { @ilm_policy_id => @ilm_policy }
228
+ end
223
229
  @alias_indexes = []
224
230
  @template_names = []
225
231
  if !dry_run?
@@ -235,7 +241,7 @@ EOC
235
241
  alias_method :template_installation, :template_installation_actual
236
242
  end
237
243
  else
238
- template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name)
244
+ template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name, @ilm_policy_id)
239
245
  end
240
246
  verify_ilm_working if @enable_ilm
241
247
  elsif @templates
@@ -719,7 +725,12 @@ EOC
719
725
  else
720
726
  pipeline = nil
721
727
  end
722
- return logstash_prefix, logstash_dateformat, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline
728
+ if @ilm_policy_id
729
+ ilm_policy_id = extract_placeholders(@ilm_policy_id, chunk)
730
+ else
731
+ ilm_policy_id = nil
732
+ end
733
+ return logstash_prefix, logstash_dateformat, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline, ilm_policy_id
723
734
  end
724
735
 
725
736
  def multi_workers_ready?
@@ -793,7 +804,7 @@ EOC
793
804
  end
794
805
 
795
806
  def process_message(tag, meta, header, time, record, extracted_values)
796
- logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline = extracted_values
807
+ logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline, _ilm_policy_id = extracted_values
797
808
 
798
809
  if @flatten_hashes
799
810
  record = flatten_record(record)
@@ -901,16 +912,17 @@ EOC
901
912
  placeholder?(:logstash_prefix, @logstash_prefix.to_s) ||
902
913
  placeholder?(:logstash_dateformat, @logstash_dateformat.to_s) ||
903
914
  placeholder?(:deflector_alias, @deflector_alias.to_s) ||
904
- placeholder?(:application_name, @application_name.to_s)
915
+ placeholder?(:application_name, @application_name.to_s) ||
916
+ placeholder?(:ilm_policy_id, @ilm_policy_id.to_s)
905
917
  log.debug("Need substitution: #{need_substitution}")
906
918
  need_substitution
907
919
  end
908
920
 
909
- def template_installation(deflector_alias, template_name, customize_template, application_name, target_index, host)
921
+ def template_installation(deflector_alias, template_name, customize_template, application_name, ilm_policy_id, target_index, host)
910
922
  # for safety.
911
923
  end
912
924
 
913
- def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, host=nil)
925
+ def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, ilm_policy_id, host=nil)
914
926
  if template_name && @template_file
915
927
  if @alias_indexes.include? deflector_alias
916
928
  log.debug("Index alias #{deflector_alias} already exists (cached)")
@@ -919,11 +931,12 @@ EOC
919
931
  else
920
932
  retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
921
933
  if customize_template
922
- template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, @ilm_policy_id, host)
934
+ template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host)
923
935
  else
924
- template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, @ilm_policy_id, host)
936
+ template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host)
925
937
  end
926
- create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, @ilm_policy_id, @ilm_policy, @ilm_policy_overwrite, host)
938
+ ilm_policy = @ilm_policies[ilm_policy_id] || {}
939
+ create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, ilm_policy_id, ilm_policy, @ilm_policy_overwrite, host)
927
940
  end
928
941
  @alias_indexes << deflector_alias unless deflector_alias.nil?
929
942
  @template_names << template_name unless template_name.nil?
@@ -934,11 +947,11 @@ EOC
934
947
  # send_bulk given a specific bulk request, the original tag,
935
948
  # chunk, and bulk_message_count
936
949
  def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
937
- logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline = extracted_values
950
+ logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline, ilm_policy_id = extracted_values
938
951
  if deflector_alias
939
- template_installation(deflector_alias, template_name, customize_template, application_name, index_name, info.host)
952
+ template_installation(deflector_alias, template_name, customize_template, application_name, index_name, ilm_policy_id, info.host)
940
953
  else
941
- template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, info.host)
954
+ template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, ilm_policy_id, info.host)
942
955
  end
943
956
 
944
957
  begin
@@ -1317,6 +1317,224 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1317
1317
  assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1318
1318
  end
1319
1319
 
1320
+ def test_template_create_with_rollover_index_and_ilm_policies_and_placeholders
1321
+ cwd = File.dirname(__FILE__)
1322
+ template_file = File.join(cwd, 'test_template.json')
1323
+
1324
+ config = %{
1325
+ host logs.google.com
1326
+ port 777
1327
+ scheme https
1328
+ path /es/
1329
+ user john
1330
+ password doe
1331
+ template_name logstash
1332
+ template_file #{template_file}
1333
+ index_date_pattern now/w{xxxx.ww}
1334
+ ilm_policy_id fluentd-policy
1335
+ enable_ilm true
1336
+ index_name logstash
1337
+ ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
1338
+ }
1339
+
1340
+ # connection start
1341
+ stub_request(:head, "https://logs.google.com:777/es//").
1342
+ with(basic_auth: ['john', 'doe']).
1343
+ to_return(:status => 200, :body => "", :headers => {})
1344
+ # check if template exists
1345
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1346
+ with(basic_auth: ['john', 'doe']).
1347
+ to_return(:status => 404, :body => "", :headers => {})
1348
+ # creation
1349
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1350
+ with(basic_auth: ['john', 'doe']).
1351
+ to_return(:status => 200, :body => "", :headers => {})
1352
+ # check if alias exists
1353
+ stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1354
+ with(basic_auth: ['john', 'doe']).
1355
+ to_return(:status => 404, :body => "", :headers => {})
1356
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1357
+ with(basic_auth: ['john', 'doe']).
1358
+ to_return(status: 404, body: "", headers: {})
1359
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1360
+ with(basic_auth: ['john', 'doe'],
1361
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1362
+ to_return(status: 200, body: "", headers: {})
1363
+ # put the alias for the index
1364
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1365
+ with(basic_auth: ['john', 'doe']).
1366
+ to_return(:status => 200, :body => "", :headers => {})
1367
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
1368
+ with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
1369
+ to_return(status: 200, body: "", headers: {})
1370
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1371
+ with(basic_auth: ['john', 'doe']).
1372
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1373
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1374
+ with(basic_auth: ['john', 'doe']).
1375
+ to_return(:status => 404, :body => "", :headers => {})
1376
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1377
+ with(basic_auth: ['john', 'doe'],
1378
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
1379
+ to_return(:status => 200, :body => "", :headers => {})
1380
+
1381
+ driver(config)
1382
+
1383
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1384
+ driver.run(default_tag: 'test') do
1385
+ driver.feed(sample_record)
1386
+ end
1387
+ assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1388
+
1389
+ assert_requested(elastic_request)
1390
+ end
1391
+
1392
+ class TemplateCreateWithRolloverIndexAndILMPoliciesWithPlaceholdersTest < self
1393
+ def test_tag_placeholder
1394
+ cwd = File.dirname(__FILE__)
1395
+ template_file = File.join(cwd, 'test_template.json')
1396
+
1397
+ config = %{
1398
+ host logs.google.com
1399
+ port 777
1400
+ scheme https
1401
+ path /es/
1402
+ user john
1403
+ password doe
1404
+ template_name logstash
1405
+ template_file #{template_file}
1406
+ index_date_pattern now/w{xxxx.ww}
1407
+ ilm_policy_id ${tag}
1408
+ enable_ilm true
1409
+ index_name logstash
1410
+ ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
1411
+ }
1412
+
1413
+ # connection start
1414
+ stub_request(:head, "https://logs.google.com:777/es//").
1415
+ with(basic_auth: ['john', 'doe']).
1416
+ to_return(:status => 200, :body => "", :headers => {})
1417
+ # check if template exists
1418
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1419
+ with(basic_auth: ['john', 'doe']).
1420
+ to_return(:status => 404, :body => "", :headers => {})
1421
+ # creation
1422
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1423
+ with(basic_auth: ['john', 'doe']).
1424
+ to_return(:status => 200, :body => "", :headers => {})
1425
+ # check if alias exists
1426
+ stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1427
+ with(basic_auth: ['john', 'doe']).
1428
+ to_return(:status => 404, :body => "", :headers => {})
1429
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1430
+ with(basic_auth: ['john', 'doe']).
1431
+ to_return(status: 404, body: "", headers: {})
1432
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1433
+ with(basic_auth: ['john', 'doe'],
1434
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1435
+ to_return(status: 200, body: "", headers: {})
1436
+ # put the alias for the index
1437
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1438
+ with(basic_auth: ['john', 'doe']).
1439
+ to_return(:status => 200, :body => "", :headers => {})
1440
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
1441
+ with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
1442
+ to_return(status: 200, body: "", headers: {})
1443
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1444
+ with(basic_auth: ['john', 'doe']).
1445
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1446
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1447
+ with(basic_auth: ['john', 'doe']).
1448
+ to_return(:status => 404, :body => "", :headers => {})
1449
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1450
+ with(basic_auth: ['john', 'doe'],
1451
+ body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
1452
+ to_return(:status => 200, :body => "", :headers => {})
1453
+
1454
+ driver(config)
1455
+
1456
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1457
+ driver.run(default_tag: 'fluentd-policy') do
1458
+ driver.feed(sample_record)
1459
+ end
1460
+ assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1461
+
1462
+ assert_requested(elastic_request)
1463
+ end
1464
+
1465
+ def test_tag_placeholder_with_multiple_policies
1466
+ cwd = File.dirname(__FILE__)
1467
+ template_file = File.join(cwd, 'test_template.json')
1468
+
1469
+ config = %{
1470
+ host logs.google.com
1471
+ port 777
1472
+ scheme https
1473
+ path /es/
1474
+ user john
1475
+ password doe
1476
+ template_name logstash
1477
+ template_file #{template_file}
1478
+ index_date_pattern now/w{xxxx.ww}
1479
+ ilm_policy_id ${tag}
1480
+ enable_ilm true
1481
+ index_name logstash
1482
+ ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}, "fluentd-policy2":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"80gb", "max_age":"20d"}}}}}}}
1483
+ }
1484
+
1485
+ # connection start
1486
+ stub_request(:head, "https://logs.google.com:777/es//").
1487
+ with(basic_auth: ['john', 'doe']).
1488
+ to_return(:status => 200, :body => "", :headers => {})
1489
+ # check if template exists
1490
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1491
+ with(basic_auth: ['john', 'doe']).
1492
+ to_return(:status => 404, :body => "", :headers => {})
1493
+ # creation
1494
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1495
+ with(basic_auth: ['john', 'doe']).
1496
+ to_return(:status => 200, :body => "", :headers => {})
1497
+ # check if alias exists
1498
+ stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1499
+ with(basic_auth: ['john', 'doe']).
1500
+ to_return(:status => 404, :body => "", :headers => {})
1501
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1502
+ with(basic_auth: ['john', 'doe']).
1503
+ to_return(status: 404, body: "", headers: {})
1504
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1505
+ with(basic_auth: ['john', 'doe'],
1506
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1507
+ to_return(status: 200, body: "", headers: {})
1508
+ # put the alias for the index
1509
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1510
+ with(basic_auth: ['john', 'doe']).
1511
+ to_return(:status => 200, :body => "", :headers => {})
1512
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
1513
+ with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
1514
+ to_return(status: 200, body: "", headers: {})
1515
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1516
+ with(basic_auth: ['john', 'doe']).
1517
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1518
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
1519
+ with(basic_auth: ['john', 'doe']).
1520
+ to_return(:status => 404, :body => "", :headers => {})
1521
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
1522
+ with(basic_auth: ['john', 'doe'],
1523
+ body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"80gb\",\"max_age\":\"20d\"}}}}}}").
1524
+ to_return(:status => 200, :body => "", :headers => {})
1525
+
1526
+ driver(config)
1527
+
1528
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1529
+ driver.run(default_tag: 'fluentd-policy2') do
1530
+ driver.feed(sample_record)
1531
+ end
1532
+ assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1533
+
1534
+ assert_requested(elastic_request)
1535
+ end
1536
+ end
1537
+
1320
1538
  def test_template_create_with_rollover_index_and_default_ilm_and_placeholders
1321
1539
  cwd = File.dirname(__FILE__)
1322
1540
  template_file = File.join(cwd, 'test_template.json')
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.0.8
4
+ version: 4.0.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - diogo
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2020-05-07 00:00:00.000000000 Z
13
+ date: 2020-05-27 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: fluentd