fluent-plugin-elasticsearch 4.0.8 → 4.0.9

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 1979708479afe32ade449552ade837bc9cfc8acff343cd248008abc579b794d3
4
- data.tar.gz: 6ed1a94f6e8553ca72fb75e319ee954f5a8a676ef4ccc0a6a65ca4d041c101c2
3
+ metadata.gz: f76e6d69d85cace43250e57e272d26e492e4ae5d48856306687efc487279dd4d
4
+ data.tar.gz: c6f583de5857aba3c9a01bdde72e16c851d52cdc07b1151052155356954155ec
5
5
  SHA512:
6
- metadata.gz: e9957264a1ade9aa7e3b7cac60157c0adc9d9b2cd9e6d3d72ce73ca480e6ee32415bb94ad83c95ee7419e6b02b6695c28c39e2696945906e35e25c43421f70ca
7
- data.tar.gz: 0efced0623b3312f1d767c82bb3ac9647e0082389bc6457052a4c74f5063a57ba21ff4effe881a18996c0a3f256ade017ec68b9df7423cb5c63869b3e49c1b07
6
+ metadata.gz: 58003e22128dc44d615351ca0b36bd6bc2495d21b146f08713db6f8cfc77323192c38c841df446f30ec6d0edde89cced9c51a676f523e44d7488444ec9f2e737
7
+ data.tar.gz: 9b34567bea973a765fb2dbf699dadd26d80fd50b45b7b1179469d60425c67602e3d288f8da25305c089c20eb6daf236b900931ca0abd503d2665a5d6ab153c12
data/History.md CHANGED
@@ -2,6 +2,10 @@
2
2
 
3
3
  ### [Unreleased]
4
4
 
5
+ ### 4.0.9
6
+ - Add possibility to configure multiple ILM policies (#753)
7
+ - Document required permissions (#757)
8
+
5
9
  ### 4.0.8
6
10
  - Handle compressable connection usable state (#743)
7
11
  - Use newer tls protocol versions (#739)
data/README.md CHANGED
@@ -93,9 +93,11 @@ Current maintainers: @cosmo0920
93
93
  + [enable_ilm](#enable_ilm)
94
94
  + [ilm_policy_id](#ilm_policy_id)
95
95
  + [ilm_policy](#ilm_policy)
96
+ + [ilm_policies](#ilm_policies)
96
97
  + [ilm_policy_overwrite](#ilm_policy_overwrite)
97
98
  + [truncate_caches_interval](#truncate_caches_interval)
98
99
  * [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
100
+ * [Elasticsearch permissions](#elasticsearch-permissions)
99
101
  * [Troubleshooting](#troubleshooting)
100
102
  + [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
101
103
  + [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
@@ -1213,6 +1215,14 @@ Default value is `{}`.
1213
1215
 
1214
1216
  **NOTE:** This parameter requests to install elasticsearch-xpack gem.
1215
1217
 
1218
+ ## ilm_policies
1219
+
1220
+ A hash in the format `{"ilm_policy_id1":{ <ILM policy 1 hash> }, "ilm_policy_id2": { <ILM policy 2 hash> }}`.
1221
+
1222
+ Default value is `{}`.
1223
+
1224
+ **NOTE:** This parameter requests to install elasticsearch-xpack gem.
1225
+
1216
1226
  ## ilm_policy_overwrite
1217
1227
 
1218
1228
  Specify whether overwriting ilm policy or not.
@@ -1233,6 +1243,32 @@ Default value is `nil`.
1233
1243
 
1234
1244
  See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)
1235
1245
 
1246
+ ## Elasticsearch permissions
1247
+
1248
+ If the target Elasticsearch requires authentication, a user holding the necessary permissions needs to be provided.
1249
+
1250
+ The set of required permissions are the following:
1251
+
1252
+ ```json
1253
+ "cluster": ["manage_index_templates", "monitor", "manage_ilm"],
1254
+ "indices": [
1255
+ {
1256
+ "names": [ "*" ],
1257
+ "privileges": ["write","create","delete","create_index","manage","manage_ilm"]
1258
+ }
1259
+ ]
1260
+ ```
1261
+
1262
+ These permissions can be narrowed down by:
1263
+
1264
+ - Setting a more specific pattern for indices under the `names` field
1265
+ - Removing the `manage_index_templates` cluster permission when not using the feature within your plugin configuration
1266
+ - Removing the `manage_ilm` cluster permission and the `manage` and `manage_ilm` indices privileges when not using ilm
1267
+ features in the plugin configuration
1268
+
1269
+ The list of privileges along with their description can be found in
1270
+ [security privileges](https://www.elastic.co/guide/en/elasticsearch/reference/current/security-privileges.html).
1271
+
1236
1272
  ## Troubleshooting
1237
1273
 
1238
1274
  ### Cannot send events to Elasticsearch
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '4.0.8'
6
+ s.version = '4.0.9'
7
7
  s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
9
9
  s.description = %q{Elasticsearch output plugin for Fluent event collector}
@@ -160,6 +160,7 @@ EOC
160
160
  config_param :enable_ilm, :bool, :default => false
161
161
  config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
162
162
  config_param :ilm_policy, :hash, :default => {}
163
+ config_param :ilm_policies, :hash, :default => {}
163
164
  config_param :ilm_policy_overwrite, :bool, :default => false
164
165
  config_param :truncate_caches_interval, :time, :default => nil
165
166
 
@@ -220,6 +221,11 @@ EOC
220
221
  log.info "host placeholder and template installation makes your Elasticsearch cluster a bit slow down(beta)."
221
222
  end
222
223
 
224
+ raise Fluent::ConfigError, "You can't specify ilm_policy and ilm_policies at the same time" unless @ilm_policy.empty? or @ilm_policies.empty?
225
+
226
+ unless @ilm_policy.empty?
227
+ @ilm_policies = { @ilm_policy_id => @ilm_policy }
228
+ end
223
229
  @alias_indexes = []
224
230
  @template_names = []
225
231
  if !dry_run?
@@ -235,7 +241,7 @@ EOC
235
241
  alias_method :template_installation, :template_installation_actual
236
242
  end
237
243
  else
238
- template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name)
244
+ template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name, @ilm_policy_id)
239
245
  end
240
246
  verify_ilm_working if @enable_ilm
241
247
  elsif @templates
@@ -719,7 +725,12 @@ EOC
719
725
  else
720
726
  pipeline = nil
721
727
  end
722
- return logstash_prefix, logstash_dateformat, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline
728
+ if @ilm_policy_id
729
+ ilm_policy_id = extract_placeholders(@ilm_policy_id, chunk)
730
+ else
731
+ ilm_policy_id = nil
732
+ end
733
+ return logstash_prefix, logstash_dateformat, index_name, type_name, template_name, customize_template, deflector_alias, application_name, pipeline, ilm_policy_id
723
734
  end
724
735
 
725
736
  def multi_workers_ready?
@@ -793,7 +804,7 @@ EOC
793
804
  end
794
805
 
795
806
  def process_message(tag, meta, header, time, record, extracted_values)
796
- logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline = extracted_values
807
+ logstash_prefix, logstash_dateformat, index_name, type_name, _template_name, _customize_template, _deflector_alias, _application_name, pipeline, _ilm_policy_id = extracted_values
797
808
 
798
809
  if @flatten_hashes
799
810
  record = flatten_record(record)
@@ -901,16 +912,17 @@ EOC
901
912
  placeholder?(:logstash_prefix, @logstash_prefix.to_s) ||
902
913
  placeholder?(:logstash_dateformat, @logstash_dateformat.to_s) ||
903
914
  placeholder?(:deflector_alias, @deflector_alias.to_s) ||
904
- placeholder?(:application_name, @application_name.to_s)
915
+ placeholder?(:application_name, @application_name.to_s) ||
916
+ placeholder?(:ilm_policy_id, @ilm_policy_id.to_s)
905
917
  log.debug("Need substitution: #{need_substitution}")
906
918
  need_substitution
907
919
  end
908
920
 
909
- def template_installation(deflector_alias, template_name, customize_template, application_name, target_index, host)
921
+ def template_installation(deflector_alias, template_name, customize_template, application_name, ilm_policy_id, target_index, host)
910
922
  # for safety.
911
923
  end
912
924
 
913
- def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, host=nil)
925
+ def template_installation_actual(deflector_alias, template_name, customize_template, application_name, target_index, ilm_policy_id, host=nil)
914
926
  if template_name && @template_file
915
927
  if @alias_indexes.include? deflector_alias
916
928
  log.debug("Index alias #{deflector_alias} already exists (cached)")
@@ -919,11 +931,12 @@ EOC
919
931
  else
920
932
  retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
921
933
  if customize_template
922
- template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, @ilm_policy_id, host)
934
+ template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host)
923
935
  else
924
- template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, @ilm_policy_id, host)
936
+ template_install(template_name, @template_file, @template_overwrite, @enable_ilm, deflector_alias, ilm_policy_id, host)
925
937
  end
926
- create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, @ilm_policy_id, @ilm_policy, @ilm_policy_overwrite, host)
938
+ ilm_policy = @ilm_policies[ilm_policy_id] || {}
939
+ create_rollover_alias(target_index, @rollover_index, deflector_alias, application_name, @index_date_pattern, @index_separator, @enable_ilm, ilm_policy_id, ilm_policy, @ilm_policy_overwrite, host)
927
940
  end
928
941
  @alias_indexes << deflector_alias unless deflector_alias.nil?
929
942
  @template_names << template_name unless template_name.nil?
@@ -934,11 +947,11 @@ EOC
934
947
  # send_bulk given a specific bulk request, the original tag,
935
948
  # chunk, and bulk_message_count
936
949
  def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
937
- logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline = extracted_values
950
+ logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline, ilm_policy_id = extracted_values
938
951
  if deflector_alias
939
- template_installation(deflector_alias, template_name, customize_template, application_name, index_name, info.host)
952
+ template_installation(deflector_alias, template_name, customize_template, application_name, index_name, ilm_policy_id, info.host)
940
953
  else
941
- template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, info.host)
954
+ template_installation(info.ilm_index, template_name, customize_template, application_name, @logstash_format ? logstash_prefix : index_name, ilm_policy_id, info.host)
942
955
  end
943
956
 
944
957
  begin
@@ -1317,6 +1317,224 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1317
1317
  assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1318
1318
  end
1319
1319
 
1320
+ def test_template_create_with_rollover_index_and_ilm_policies_and_placeholders
1321
+ cwd = File.dirname(__FILE__)
1322
+ template_file = File.join(cwd, 'test_template.json')
1323
+
1324
+ config = %{
1325
+ host logs.google.com
1326
+ port 777
1327
+ scheme https
1328
+ path /es/
1329
+ user john
1330
+ password doe
1331
+ template_name logstash
1332
+ template_file #{template_file}
1333
+ index_date_pattern now/w{xxxx.ww}
1334
+ ilm_policy_id fluentd-policy
1335
+ enable_ilm true
1336
+ index_name logstash
1337
+ ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
1338
+ }
1339
+
1340
+ # connection start
1341
+ stub_request(:head, "https://logs.google.com:777/es//").
1342
+ with(basic_auth: ['john', 'doe']).
1343
+ to_return(:status => 200, :body => "", :headers => {})
1344
+ # check if template exists
1345
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1346
+ with(basic_auth: ['john', 'doe']).
1347
+ to_return(:status => 404, :body => "", :headers => {})
1348
+ # creation
1349
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1350
+ with(basic_auth: ['john', 'doe']).
1351
+ to_return(:status => 200, :body => "", :headers => {})
1352
+ # check if alias exists
1353
+ stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1354
+ with(basic_auth: ['john', 'doe']).
1355
+ to_return(:status => 404, :body => "", :headers => {})
1356
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1357
+ with(basic_auth: ['john', 'doe']).
1358
+ to_return(status: 404, body: "", headers: {})
1359
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1360
+ with(basic_auth: ['john', 'doe'],
1361
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1362
+ to_return(status: 200, body: "", headers: {})
1363
+ # put the alias for the index
1364
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1365
+ with(basic_auth: ['john', 'doe']).
1366
+ to_return(:status => 200, :body => "", :headers => {})
1367
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
1368
+ with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
1369
+ to_return(status: 200, body: "", headers: {})
1370
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1371
+ with(basic_auth: ['john', 'doe']).
1372
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1373
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1374
+ with(basic_auth: ['john', 'doe']).
1375
+ to_return(:status => 404, :body => "", :headers => {})
1376
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1377
+ with(basic_auth: ['john', 'doe'],
1378
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
1379
+ to_return(:status => 200, :body => "", :headers => {})
1380
+
1381
+ driver(config)
1382
+
1383
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1384
+ driver.run(default_tag: 'test') do
1385
+ driver.feed(sample_record)
1386
+ end
1387
+ assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1388
+
1389
+ assert_requested(elastic_request)
1390
+ end
1391
+
1392
+ class TemplateCreateWithRolloverIndexAndILMPoliciesWithPlaceholdersTest < self
1393
+ def test_tag_placeholder
1394
+ cwd = File.dirname(__FILE__)
1395
+ template_file = File.join(cwd, 'test_template.json')
1396
+
1397
+ config = %{
1398
+ host logs.google.com
1399
+ port 777
1400
+ scheme https
1401
+ path /es/
1402
+ user john
1403
+ password doe
1404
+ template_name logstash
1405
+ template_file #{template_file}
1406
+ index_date_pattern now/w{xxxx.ww}
1407
+ ilm_policy_id ${tag}
1408
+ enable_ilm true
1409
+ index_name logstash
1410
+ ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
1411
+ }
1412
+
1413
+ # connection start
1414
+ stub_request(:head, "https://logs.google.com:777/es//").
1415
+ with(basic_auth: ['john', 'doe']).
1416
+ to_return(:status => 200, :body => "", :headers => {})
1417
+ # check if template exists
1418
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1419
+ with(basic_auth: ['john', 'doe']).
1420
+ to_return(:status => 404, :body => "", :headers => {})
1421
+ # creation
1422
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1423
+ with(basic_auth: ['john', 'doe']).
1424
+ to_return(:status => 200, :body => "", :headers => {})
1425
+ # check if alias exists
1426
+ stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1427
+ with(basic_auth: ['john', 'doe']).
1428
+ to_return(:status => 404, :body => "", :headers => {})
1429
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1430
+ with(basic_auth: ['john', 'doe']).
1431
+ to_return(status: 404, body: "", headers: {})
1432
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1433
+ with(basic_auth: ['john', 'doe'],
1434
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1435
+ to_return(status: 200, body: "", headers: {})
1436
+ # put the alias for the index
1437
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1438
+ with(basic_auth: ['john', 'doe']).
1439
+ to_return(:status => 200, :body => "", :headers => {})
1440
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
1441
+ with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
1442
+ to_return(status: 200, body: "", headers: {})
1443
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1444
+ with(basic_auth: ['john', 'doe']).
1445
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1446
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1447
+ with(basic_auth: ['john', 'doe']).
1448
+ to_return(:status => 404, :body => "", :headers => {})
1449
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1450
+ with(basic_auth: ['john', 'doe'],
1451
+ body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
1452
+ to_return(:status => 200, :body => "", :headers => {})
1453
+
1454
+ driver(config)
1455
+
1456
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1457
+ driver.run(default_tag: 'fluentd-policy') do
1458
+ driver.feed(sample_record)
1459
+ end
1460
+ assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1461
+
1462
+ assert_requested(elastic_request)
1463
+ end
1464
+
1465
+ def test_tag_placeholder_with_multiple_policies
1466
+ cwd = File.dirname(__FILE__)
1467
+ template_file = File.join(cwd, 'test_template.json')
1468
+
1469
+ config = %{
1470
+ host logs.google.com
1471
+ port 777
1472
+ scheme https
1473
+ path /es/
1474
+ user john
1475
+ password doe
1476
+ template_name logstash
1477
+ template_file #{template_file}
1478
+ index_date_pattern now/w{xxxx.ww}
1479
+ ilm_policy_id ${tag}
1480
+ enable_ilm true
1481
+ index_name logstash
1482
+ ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}, "fluentd-policy2":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"80gb", "max_age":"20d"}}}}}}}
1483
+ }
1484
+
1485
+ # connection start
1486
+ stub_request(:head, "https://logs.google.com:777/es//").
1487
+ with(basic_auth: ['john', 'doe']).
1488
+ to_return(:status => 200, :body => "", :headers => {})
1489
+ # check if template exists
1490
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1491
+ with(basic_auth: ['john', 'doe']).
1492
+ to_return(:status => 404, :body => "", :headers => {})
1493
+ # creation
1494
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1495
+ with(basic_auth: ['john', 'doe']).
1496
+ to_return(:status => 200, :body => "", :headers => {})
1497
+ # check if alias exists
1498
+ stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1499
+ with(basic_auth: ['john', 'doe']).
1500
+ to_return(:status => 404, :body => "", :headers => {})
1501
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1502
+ with(basic_auth: ['john', 'doe']).
1503
+ to_return(status: 404, body: "", headers: {})
1504
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1505
+ with(basic_auth: ['john', 'doe'],
1506
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1507
+ to_return(status: 200, body: "", headers: {})
1508
+ # put the alias for the index
1509
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1510
+ with(basic_auth: ['john', 'doe']).
1511
+ to_return(:status => 200, :body => "", :headers => {})
1512
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/logstash").
1513
+ with(body: "{\"aliases\":{\"logstash\":{\"is_write_index\":true}}}").
1514
+ to_return(status: 200, body: "", headers: {})
1515
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1516
+ with(basic_auth: ['john', 'doe']).
1517
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1518
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
1519
+ with(basic_auth: ['john', 'doe']).
1520
+ to_return(:status => 404, :body => "", :headers => {})
1521
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy2").
1522
+ with(basic_auth: ['john', 'doe'],
1523
+ body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"80gb\",\"max_age\":\"20d\"}}}}}}").
1524
+ to_return(:status => 200, :body => "", :headers => {})
1525
+
1526
+ driver(config)
1527
+
1528
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1529
+ driver.run(default_tag: 'fluentd-policy2') do
1530
+ driver.feed(sample_record)
1531
+ end
1532
+ assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1533
+
1534
+ assert_requested(elastic_request)
1535
+ end
1536
+ end
1537
+
1320
1538
  def test_template_create_with_rollover_index_and_default_ilm_and_placeholders
1321
1539
  cwd = File.dirname(__FILE__)
1322
1540
  template_file = File.join(cwd, 'test_template.json')
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.0.8
4
+ version: 4.0.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - diogo
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2020-05-07 00:00:00.000000000 Z
13
+ date: 2020-05-27 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: fluentd