fluent-plugin-elasticsearch 4.1.4 → 4.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c7923225803a37c0076c7cb4c4d6e94c1ec9ccf120ce4067b45db8c0fc19f17f
4
- data.tar.gz: f1ce4098fb1e4e563e00aae9fe8d30b2bc9cfcb08100616f0ff00b3dfa210667
3
+ metadata.gz: 46901e5f885a46c37ea29dca2968d53fd4026dec637f007936cabdd3ee070c5c
4
+ data.tar.gz: 21f3d3e9dd22752bcab060ea46ba12e8309147c2ddbb3be4aea656b527aff103
5
5
  SHA512:
6
- metadata.gz: '0579350a7eba583c7c4784f5500d9faa6df4e398b4b0d0e4aacf230502751a39fb9591d189217c869a82f6da689ffb435445aea7bd5920332787e787170b37d7'
7
- data.tar.gz: 8d150716862d82b313a31b104842b368aac3f945bece0a11074bf43a25830c91e6b5ee0556a23e93fdc2159ab20185e08847b4c94bb10033fe7b259741b332b6
6
+ metadata.gz: ed30b7f203f9c102fdedb0a28c88210f172e9bccab3f153de81ba88b821a065e1cfdb8caccc097fbb83745069c7c5cd052e70879501ebd83b472f55c835b6116
7
+ data.tar.gz: 3ded3c8f6041224546a71df34878148a14b0f2e47970b86efb11d8b4fe4008764680534149b1c56c1651bc7dbfffc0df26ad7a459c87e739c91f391d8da83f42
@@ -8,7 +8,7 @@ jobs:
8
8
  strategy:
9
9
  fail-fast: false
10
10
  matrix:
11
- ruby: [ '2.4', '2.5', '2.6' ]
11
+ ruby: [ '2.4', '2.5', '2.6', '2.7' ]
12
12
  os:
13
13
  - ubuntu-latest
14
14
  name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
@@ -8,7 +8,7 @@ jobs:
8
8
  strategy:
9
9
  fail-fast: false
10
10
  matrix:
11
- ruby: [ '2.4', '2.5', '2.6' ]
11
+ ruby: [ '2.4', '2.5', '2.6', '2.7' ]
12
12
  os:
13
13
  - macOS-latest
14
14
  name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
@@ -8,7 +8,7 @@ jobs:
8
8
  strategy:
9
9
  fail-fast: false
10
10
  matrix:
11
- ruby: [ '2.4', '2.5', '2.6' ]
11
+ ruby: [ '2.4', '2.5', '2.6', '2.7' ]
12
12
  os:
13
13
  - windows-latest
14
14
  name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
data/History.md CHANGED
@@ -2,6 +2,10 @@
2
2
 
3
3
  ### [Unreleased]
4
4
 
5
+ ### 4.2.0
6
+ - ci: Add Ruby 2.7 jobs (#812)
7
+ - Support Elasticsearch new style template (#810)
8
+
5
9
  ### 4.1.4
6
10
  - ElasticsearchGenID update docs for hash_type (#809)
7
11
  - Handle api key header (#808)
data/README.md CHANGED
@@ -101,6 +101,7 @@ Current maintainers: @cosmo0920
101
101
  + [ilm_policies](#ilm_policies)
102
102
  + [ilm_policy_overwrite](#ilm_policy_overwrite)
103
103
  + [truncate_caches_interval](#truncate_caches_interval)
104
+ + [use_legacy_template](#use_legacy_template)
104
105
  * [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
105
106
  * [Configuration - Elasticsearch Filter GenID](#configuration---elasticsearch-filter-genid)
106
107
  * [Elasticsearch permissions](#elasticsearch-permissions)
@@ -1303,6 +1304,18 @@ If it is set, timer for clearing `alias_indexes` and `template_names` caches wil
1303
1304
 
1304
1305
  Default value is `nil`.
1305
1306
 
1307
+ ## use_legacy_template
1308
+
1309
+ Use legacy template or not.
1310
+
1311
+ Elasticsearch 7.8 or later supports the brand new composable templates.
1312
+
1313
+ For Elasticsearch 7.7 or older, users should specify this parameter as `false`.
1314
+
1315
+ Composable template documentation is [Put Index Template API | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-templates.html) and legacy template documentation is [Index Templates | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates-v1.html).
1316
+
1317
+ Please confirm that whether the using Elasticsearch cluster(s) support the composable template feature or not when turn on the brand new feature with this parameter.
1318
+
1306
1319
  ## Configuration - Elasticsearch Input
1307
1320
 
1308
1321
  See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '4.1.4'
6
+ s.version = '4.2.0'
7
7
  s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
9
9
  s.description = %q{Elasticsearch output plugin for Fluent event collector}
@@ -22,7 +22,11 @@ module Fluent::ElasticsearchIndexTemplate
22
22
  end
23
23
 
24
24
  def template_exists?(name, host = nil)
25
- client(host).indices.get_template(:name => name)
25
+ if @use_legacy_template
26
+ client(host).indices.get_template(:name => name)
27
+ else
28
+ client(host).indices.get_index_template(:name => name)
29
+ end
26
30
  return true
27
31
  rescue Elasticsearch::Transport::Transport::Errors::NotFound
28
32
  return false
@@ -52,7 +56,11 @@ module Fluent::ElasticsearchIndexTemplate
52
56
  end
53
57
 
54
58
  def template_put(name, template, host = nil)
55
- client(host).indices.put_template(:name => name, :body => template)
59
+ if @use_legacy_template
60
+ client(host).indices.put_template(:name => name, :body => template)
61
+ else
62
+ client(host).indices.put_index_template(:name => name, :body => template)
63
+ end
56
64
  end
57
65
 
58
66
  def indexcreation(index_name, host = nil)
@@ -121,13 +129,29 @@ module Fluent::ElasticsearchIndexTemplate
121
129
 
122
130
  def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template)
123
131
  log.debug("Overwriting index patterns when Index Lifecycle Management is enabled.")
124
- template.delete('template') if template.include?('template')
125
132
  template['index_patterns'] = "#{target_index}-*"
126
- template['order'] = template['order'] ? template['order'] + target_index.split('-').length : 50 + target_index.split('-').length
127
- if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
128
- log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
133
+ if @use_legacy_template
134
+ template.delete('template') if template.include?('template')
135
+ # Prepare settings Hash
136
+ if !template.key?('settings')
137
+ template['settings'] = {}
138
+ end
139
+ if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
140
+ log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
141
+ end
142
+ template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
143
+ template['order'] = template['order'] ? template['order'] + target_index.split('-').length : 50 + target_index.split('-').length
144
+ else
145
+ # Prepare template.settings Hash
146
+ if !template['template'].key?('settings')
147
+ template['template']['settings'] = {}
148
+ end
149
+ if template['template']['settings'] && (template['template']['settings']['index.lifecycle.name'] || template['template']['settings']['index.lifecycle.rollover_alias'])
150
+ log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
151
+ end
152
+ template['template']['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
153
+ template['priority'] = template['priority'] ? template['priority'] + target_index.split('-').length : 100 + target_index.split('-').length
129
154
  end
130
- template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
131
155
  template
132
156
  end
133
157
 
@@ -169,6 +169,7 @@ EOC
169
169
  config_param :ilm_policies, :hash, :default => {}
170
170
  config_param :ilm_policy_overwrite, :bool, :default => false
171
171
  config_param :truncate_caches_interval, :time, :default => nil
172
+ config_param :use_legacy_template, :bool, :default => true
172
173
 
173
174
  config_section :buffer do
174
175
  config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -0,0 +1,11 @@
1
+ {
2
+ "priority": 105,
3
+ "index_patterns": "--index_prefix-----appid---*",
4
+ "template" : {
5
+ "settings": {},
6
+ "mappings": {},
7
+ "aliases": {
8
+ "--appid---alias": {}
9
+ }
10
+ }
11
+ }
@@ -0,0 +1,25 @@
1
+ {
2
+ "index_patterns": "te*",
3
+ "template": {
4
+ "settings": {
5
+ "number_of_shards": 1
6
+ },
7
+ "mappings": {
8
+ "type1": {
9
+ "_source": {
10
+ "enabled": false
11
+ },
12
+ "properties": {
13
+ "host_name": {
14
+ "type": "string",
15
+ "index": "not_analyzed"
16
+ },
17
+ "created_at": {
18
+ "type": "date",
19
+ "format": "EEE MMM dd HH:mm:ss Z YYYY"
20
+ }
21
+ }
22
+ }
23
+ }
24
+ }
25
+ }
@@ -375,7 +375,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
375
375
  end
376
376
  end
377
377
 
378
- test 'valid configuration of index lifecycle management' do
378
+ data("legacy_template" => [true, "_template"],
379
+ "new_template" => [false, "_index_template"])
380
+ test 'valid configuration of index lifecycle management' do |data|
381
+ use_legacy_template_flag, endpoint = data
379
382
  cwd = File.dirname(__FILE__)
380
383
  template_file = File.join(cwd, 'test_template.json')
381
384
 
@@ -383,8 +386,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
383
386
  enable_ilm true
384
387
  template_name logstash
385
388
  template_file #{template_file}
389
+ use_legacy_template #{use_legacy_template_flag}
386
390
  }
387
- stub_request(:get, "http://localhost:9200/_template/fluentd").
391
+ stub_request(:get, "http://localhost:9200/#{endpoint}/fluentd").
388
392
  to_return(status: 200, body: "", headers: {})
389
393
  stub_request(:head, "http://localhost:9200/_alias/fluentd").
390
394
  to_return(status: 404, body: "", headers: {})
@@ -407,7 +411,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
407
411
  }
408
412
  end
409
413
 
410
- test 'valid configuration of overwriting ilm_policy' do
414
+ data("legacy_template" => [true, "_template"],
415
+ "new_template" => [false, "_index_template"])
416
+ test 'valid configuration of overwriting ilm_policy' do |data|
417
+ use_legacy_template_flag, endpoint = data
411
418
  cwd = File.dirname(__FILE__)
412
419
  template_file = File.join(cwd, 'test_template.json')
413
420
 
@@ -417,8 +424,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
417
424
  template_file #{template_file}
418
425
  ilm_policy_overwrite true
419
426
  ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"75gb","max_age": "50d"}}}}}}
427
+ use_legacy_template #{use_legacy_template_flag}
420
428
  }
421
- stub_request(:get, "http://localhost:9200/_template/fluentd").
429
+ stub_request(:get, "http://localhost:9200/#{endpoint}/fluentd").
422
430
  to_return(status: 200, body: "", headers: {})
423
431
  stub_request(:head, "http://localhost:9200/_alias/fluentd").
424
432
  to_return(status: 404, body: "", headers: {})
@@ -806,7 +814,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
806
814
  end
807
815
  end
808
816
 
809
- def test_template_already_present
817
+ data("legacy_template" => [true, "_template"],
818
+ "new_template" => [false, "_index_template"])
819
+ def test_template_already_present(data)
820
+ use_legacy_template_flag, endpoint = data
810
821
  config = %{
811
822
  host logs.google.com
812
823
  port 777
@@ -816,6 +827,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
816
827
  password doe
817
828
  template_name logstash
818
829
  template_file /abc123
830
+ use_legacy_template #{use_legacy_template_flag}
819
831
  }
820
832
 
821
833
  # connection start
@@ -823,18 +835,25 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
823
835
  with(basic_auth: ['john', 'doe']).
824
836
  to_return(:status => 200, :body => "", :headers => {})
825
837
  # check if template exists
826
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
838
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
827
839
  with(basic_auth: ['john', 'doe']).
828
840
  to_return(:status => 200, :body => "", :headers => {})
829
841
 
830
842
  driver(config)
831
843
 
832
- assert_not_requested(:put, "https://logs.google.com:777/es//_template/logstash")
844
+ assert_not_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash")
833
845
  end
834
846
 
835
- def test_template_create
847
+ data("legacy_template" => [true, "_template"],
848
+ "new_template" => [false, "_index_template"])
849
+ def test_template_create(data)
850
+ use_legacy_template_flag, endpoint = data
836
851
  cwd = File.dirname(__FILE__)
837
- template_file = File.join(cwd, 'test_template.json')
852
+ template_file = if use_legacy_template_flag
853
+ File.join(cwd, 'test_template.json')
854
+ else
855
+ File.join(cwd, 'test_index_template.json')
856
+ end
838
857
 
839
858
  config = %{
840
859
  host logs.google.com
@@ -845,6 +864,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
845
864
  password doe
846
865
  template_name logstash
847
866
  template_file #{template_file}
867
+ use_legacy_template #{use_legacy_template_flag}
848
868
  }
849
869
 
850
870
  # connection start
@@ -852,22 +872,29 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
852
872
  with(basic_auth: ['john', 'doe']).
853
873
  to_return(:status => 200, :body => "", :headers => {})
854
874
  # check if template exists
855
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
875
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
856
876
  with(basic_auth: ['john', 'doe']).
857
877
  to_return(:status => 404, :body => "", :headers => {})
858
878
  # creation
859
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
879
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
860
880
  with(basic_auth: ['john', 'doe']).
861
881
  to_return(:status => 200, :body => "", :headers => {})
862
882
 
863
883
  driver(config)
864
884
 
865
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
885
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
866
886
  end
867
887
 
868
- def test_template_create_with_rollover_index_and_template_related_placeholders
888
+ data("legacy_template" => [true, "_template"],
889
+ "new_template" => [false, "_index_template"])
890
+ def test_template_create_with_rollover_index_and_template_related_placeholders(data)
891
+ use_legacy_template_flag, endpoint = data
869
892
  cwd = File.dirname(__FILE__)
870
- template_file = File.join(cwd, 'test_template.json')
893
+ template_file = if use_legacy_template_flag
894
+ File.join(cwd, 'test_template.json')
895
+ else
896
+ File.join(cwd, 'test_index_template.json')
897
+ end
871
898
  config = %{
872
899
  host logs.google.com
873
900
  port 777
@@ -881,6 +908,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
881
908
  index_date_pattern ""
882
909
  index_name fluentd-${tag}
883
910
  deflector_alias myapp_deflector-${tag}
911
+ use_legacy_template #{use_legacy_template_flag}
884
912
  }
885
913
 
886
914
  # connection start
@@ -888,11 +916,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
888
916
  with(basic_auth: ['john', 'doe']).
889
917
  to_return(:status => 200, :body => "", :headers => {})
890
918
  # check if template exists
891
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash-test.template").
919
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-test.template").
892
920
  with(basic_auth: ['john', 'doe']).
893
921
  to_return(:status => 404, :body => "", :headers => {})
894
922
  # create template
895
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash-test.template").
923
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-test.template").
896
924
  with(basic_auth: ['john', 'doe']).
897
925
  to_return(:status => 200, :body => "", :headers => {})
898
926
  # check if alias exists
@@ -922,9 +950,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
922
950
  assert_requested(elastic_request)
923
951
  end
924
952
 
925
- def test_template_create_with_rollover_index_and_template_related_placeholders_with_truncating_caches
953
+ data("legacy_template" => [true, "_template"],
954
+ "new_template" => [false, "_index_template"])
955
+ def test_template_create_with_rollover_index_and_template_related_placeholders_with_truncating_caches(data)
956
+ use_legacy_template_flag, endpoint = data
926
957
  cwd = File.dirname(__FILE__)
927
- template_file = File.join(cwd, 'test_template.json')
958
+ template_file = if use_legacy_template_flag
959
+ File.join(cwd, 'test_template.json')
960
+ else
961
+ File.join(cwd, 'test_index_template.json')
962
+ end
928
963
  config = %{
929
964
  host logs.google.com
930
965
  port 777
@@ -939,6 +974,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
939
974
  index_name fluentd-${tag}
940
975
  deflector_alias myapp_deflector-${tag}
941
976
  truncate_caches_interval 2s
977
+ use_legacy_template #{use_legacy_template_flag}
942
978
  }
943
979
 
944
980
  # connection start
@@ -946,11 +982,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
946
982
  with(basic_auth: ['john', 'doe']).
947
983
  to_return(:status => 200, :body => "", :headers => {})
948
984
  # check if template exists
949
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash-test.template").
985
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-test.template").
950
986
  with(basic_auth: ['john', 'doe']).
951
987
  to_return(:status => 404, :body => "", :headers => {})
952
988
  # create template
953
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash-test.template").
989
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-test.template").
954
990
  with(basic_auth: ['john', 'doe']).
955
991
  to_return(:status => 200, :body => "", :headers => {})
956
992
  # check if alias exists
@@ -995,9 +1031,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
995
1031
  end
996
1032
  end
997
1033
 
998
- def test_template_create_with_rollover_index_and_default_ilm
1034
+ data("legacy_template" => [true, "_template"],
1035
+ "new_template" => [false, "_index_template"])
1036
+ def test_template_create_with_rollover_index_and_default_ilm(data)
1037
+ use_legacy_template_flag, endpoint = data
999
1038
  cwd = File.dirname(__FILE__)
1000
- template_file = File.join(cwd, 'test_template.json')
1039
+ template_file = if use_legacy_template_flag
1040
+ File.join(cwd, 'test_template.json')
1041
+ else
1042
+ File.join(cwd, 'test_index_template.json')
1043
+ end
1001
1044
 
1002
1045
  config = %{
1003
1046
  host logs.google.com
@@ -1011,6 +1054,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1011
1054
  index_date_pattern now/w{xxxx.ww}
1012
1055
  index_name logstash
1013
1056
  enable_ilm true
1057
+ use_legacy_template #{use_legacy_template_flag}
1014
1058
  }
1015
1059
 
1016
1060
  # connection start
@@ -1018,24 +1062,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1018
1062
  with(basic_auth: ['john', 'doe']).
1019
1063
  to_return(:status => 200, :body => "", :headers => {})
1020
1064
  # check if template exists
1021
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1065
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
1022
1066
  with(basic_auth: ['john', 'doe']).
1023
1067
  to_return(:status => 404, :body => "", :headers => {})
1024
1068
  # creation
1025
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1069
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1026
1070
  with(basic_auth: ['john', 'doe']).
1027
1071
  to_return(:status => 200, :body => "", :headers => {})
1028
1072
  # check if alias exists
1029
1073
  stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1030
1074
  with(basic_auth: ['john', 'doe']).
1031
1075
  to_return(:status => 404, :body => "", :headers => {})
1032
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1076
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
1033
1077
  with(basic_auth: ['john', 'doe']).
1034
1078
  to_return(status: 404, body: "", headers: {})
1035
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1036
- with(basic_auth: ['john', 'doe'],
1037
- body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
1038
- to_return(status: 200, body: "", headers: {})
1079
+ if use_legacy_template_flag
1080
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1081
+ with(basic_auth: ['john', 'doe'],
1082
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
1083
+ to_return(status: 200, body: "", headers: {})
1084
+ else
1085
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1086
+ with(basic_auth: ['john', 'doe'],
1087
+ body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
1088
+ to_return(status: 200, body: "", headers: {})
1089
+ end
1039
1090
  # put the alias for the index
1040
1091
  stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1041
1092
  with(basic_auth: ['john', 'doe']).
@@ -1057,12 +1108,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1057
1108
 
1058
1109
  driver(config)
1059
1110
 
1060
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1111
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
1061
1112
  end
1062
1113
 
1063
- def test_template_create_with_rollover_index_and_default_ilm_on_logstash_format
1114
+ data("legacy_template" => [true, "_template"],
1115
+ "new_template" => [false, "_index_template"])
1116
+ def test_template_create_with_rollover_index_and_default_ilm_on_logstash_format(data)
1117
+ use_legacy_template_flag, endpoint = data
1064
1118
  cwd = File.dirname(__FILE__)
1065
- template_file = File.join(cwd, 'test_template.json')
1119
+ template_file = if use_legacy_template_flag
1120
+ File.join(cwd, 'test_template.json')
1121
+ else
1122
+ File.join(cwd, 'test_index_template.json')
1123
+ end
1066
1124
 
1067
1125
  config = %{
1068
1126
  host logs.google.com
@@ -1077,6 +1135,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1077
1135
  enable_ilm true
1078
1136
  logstash_format true
1079
1137
  application_name log
1138
+ use_legacy_template #{use_legacy_template_flag}
1080
1139
  }
1081
1140
 
1082
1141
  date_str = Time.now.strftime("%Y.%m.%d")
@@ -1085,24 +1144,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1085
1144
  with(basic_auth: ['john', 'doe']).
1086
1145
  to_return(:status => 200, :body => "", :headers => {})
1087
1146
  # check if template exists
1088
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
1147
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}").
1089
1148
  with(basic_auth: ['john', 'doe']).
1090
1149
  to_return(:status => 404, :body => "", :headers => {})
1091
1150
  # creation
1092
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
1151
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}").
1093
1152
  with(basic_auth: ['john', 'doe']).
1094
1153
  to_return(:status => 200, :body => "", :headers => {})
1095
1154
  # check if alias exists
1096
1155
  stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-#{date_str}").
1097
1156
  with(basic_auth: ['john', 'doe']).
1098
1157
  to_return(:status => 404, :body => "", :headers => {})
1099
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
1158
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}").
1100
1159
  with(basic_auth: ['john', 'doe']).
1101
1160
  to_return(status: 404, body: "", headers: {})
1102
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash-#{date_str}").
1103
- with(basic_auth: ['john', 'doe'],
1104
- body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-log-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-log-#{date_str}-*\",\"order\":53}").
1105
- to_return(status: 200, body: "", headers: {})
1161
+ if use_legacy_template_flag
1162
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}").
1163
+ with(basic_auth: ['john', 'doe'],
1164
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-log-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-log-#{date_str}-*\",\"order\":53}").
1165
+ to_return(status: 200, body: "", headers: {})
1166
+ else
1167
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}").
1168
+ with(basic_auth: ['john', 'doe'],
1169
+ body: "{\"index_patterns\":\"logstash-log-2020.09.21-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-2020.09.21\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":103}").
1170
+ to_return(status: 200, body: "", headers: {})
1171
+ end
1106
1172
  # put the alias for the index
1107
1173
  stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-#{date_str}-000001%3E").
1108
1174
  with(basic_auth: ['john', 'doe']).
@@ -1129,14 +1195,21 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1129
1195
  driver.run(default_tag: 'test') do
1130
1196
  driver.feed(sample_record)
1131
1197
  end
1132
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash-#{date_str}", times: 1)
1198
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}", times: 1)
1133
1199
 
1134
1200
  assert_requested(elastic_request)
1135
1201
  end
1136
1202
 
1137
- def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite
1203
+ data("legacy_template" => [true, "_template"],
1204
+ "new_template" => [false, "_index_template"])
1205
+ def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite(data)
1206
+ use_legacy_template_flag, endpoint = data
1138
1207
  cwd = File.dirname(__FILE__)
1139
- template_file = File.join(cwd, 'test_template.json')
1208
+ template_file = if use_legacy_template_flag
1209
+ File.join(cwd, 'test_template.json')
1210
+ else
1211
+ File.join(cwd, 'test_index_template.json')
1212
+ end
1140
1213
 
1141
1214
  config = %{
1142
1215
  host logs.google.com
@@ -1152,6 +1225,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1152
1225
  enable_ilm true
1153
1226
  ilm_policy_overwrite true
1154
1227
  ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"60gb","max_age": "45d"}}}}}}
1228
+ use_legacy_template #{use_legacy_template_flag}
1155
1229
  }
1156
1230
 
1157
1231
  # connection start
@@ -1159,24 +1233,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1159
1233
  with(basic_auth: ['john', 'doe']).
1160
1234
  to_return(:status => 200, :body => "", :headers => {})
1161
1235
  # check if template exists
1162
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1236
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
1163
1237
  with(basic_auth: ['john', 'doe']).
1164
1238
  to_return(:status => 404, :body => "", :headers => {})
1165
1239
  # creation
1166
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1240
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1167
1241
  with(basic_auth: ['john', 'doe']).
1168
1242
  to_return(:status => 200, :body => "", :headers => {})
1169
1243
  # check if alias exists
1170
1244
  stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1171
1245
  with(basic_auth: ['john', 'doe']).
1172
1246
  to_return(:status => 404, :body => "", :headers => {})
1173
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1247
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
1174
1248
  with(basic_auth: ['john', 'doe']).
1175
1249
  to_return(status: 404, body: "", headers: {})
1176
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1177
- with(basic_auth: ['john', 'doe'],
1178
- body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
1179
- to_return(status: 200, body: "", headers: {})
1250
+ if use_legacy_template_flag
1251
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1252
+ with(basic_auth: ['john', 'doe'],
1253
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
1254
+ to_return(status: 200, body: "", headers: {})
1255
+ else
1256
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1257
+ with(basic_auth: ['john', 'doe'],
1258
+ body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
1259
+ to_return(status: 200, body: "", headers: {})
1260
+ end
1180
1261
  # put the alias for the index
1181
1262
  stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1182
1263
  with(basic_auth: ['john', 'doe']).
@@ -1198,7 +1279,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1198
1279
 
1199
1280
  driver(config)
1200
1281
 
1201
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1282
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
1202
1283
  end
1203
1284
 
1204
1285
  def test_template_create_with_rollover_index_and_default_ilm_with_deflector_alias
@@ -1226,9 +1307,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1226
1307
  end
1227
1308
  end
1228
1309
 
1229
- def test_template_create_with_rollover_index_and_default_ilm_with_empty_index_date_pattern
1310
+ data("legacy_template" => [true, "_template"],
1311
+ "new_template" => [false, "_index_template"])
1312
+ def test_template_create_with_rollover_index_and_default_ilm_with_empty_index_date_pattern(data)
1313
+ use_legacy_template_flag, endpoint = data
1230
1314
  cwd = File.dirname(__FILE__)
1231
- template_file = File.join(cwd, 'test_template.json')
1315
+ template_file = if use_legacy_template_flag
1316
+ File.join(cwd, 'test_template.json')
1317
+ else
1318
+ File.join(cwd, 'test_index_template.json')
1319
+ end
1232
1320
 
1233
1321
  config = %{
1234
1322
  host logs.google.com
@@ -1242,6 +1330,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1242
1330
  index_date_pattern ""
1243
1331
  index_name logstash
1244
1332
  enable_ilm true
1333
+ use_legacy_template #{use_legacy_template_flag}
1245
1334
  }
1246
1335
 
1247
1336
  # connection start
@@ -1249,24 +1338,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1249
1338
  with(basic_auth: ['john', 'doe']).
1250
1339
  to_return(:status => 200, :body => "", :headers => {})
1251
1340
  # check if template exists
1252
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1341
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
1253
1342
  with(basic_auth: ['john', 'doe']).
1254
1343
  to_return(:status => 404, :body => "", :headers => {})
1255
1344
  # creation
1256
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1345
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1257
1346
  with(basic_auth: ['john', 'doe']).
1258
1347
  to_return(:status => 200, :body => "", :headers => {})
1259
1348
  # check if alias exists
1260
1349
  stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1261
1350
  with(basic_auth: ['john', 'doe']).
1262
1351
  to_return(:status => 404, :body => "", :headers => {})
1263
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1352
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
1264
1353
  with(basic_auth: ['john', 'doe']).
1265
1354
  to_return(status: 404, body: "", headers: {})
1266
- stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
1267
- with(basic_auth: ['john', 'doe'],
1268
- body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
1269
- to_return(status: 200, body: "", headers: {})
1355
+ if use_legacy_template_flag
1356
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_deflector").
1357
+ with(basic_auth: ['john', 'doe'],
1358
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
1359
+ to_return(status: 200, body: "", headers: {})
1360
+ else
1361
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_deflector").
1362
+ with(basic_auth: ['john', 'doe'],
1363
+ body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
1364
+ to_return(status: 200, body: "", headers: {})
1365
+ end
1270
1366
  # put the alias for the index
1271
1367
  stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E").
1272
1368
  with(basic_auth: ['john', 'doe']).
@@ -1288,12 +1384,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1288
1384
 
1289
1385
  driver(config)
1290
1386
 
1291
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1387
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
1292
1388
  end
1293
1389
 
1294
- def test_template_create_with_rollover_index_and_custom_ilm
1390
+ data("legacy_template" => [true, "_template"],
1391
+ "new_template" => [false, "_index_template"])
1392
+ def test_template_create_with_rollover_index_and_custom_ilm(data)
1393
+ use_legacy_template_flag, endpoint = data
1295
1394
  cwd = File.dirname(__FILE__)
1296
- template_file = File.join(cwd, 'test_template.json')
1395
+ template_file = if use_legacy_template_flag
1396
+ File.join(cwd, 'test_template.json')
1397
+ else
1398
+ File.join(cwd, 'test_index_template.json')
1399
+ end
1297
1400
 
1298
1401
  config = %{
1299
1402
  host logs.google.com
@@ -1309,6 +1412,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1309
1412
  enable_ilm true
1310
1413
  index_name logstash
1311
1414
  ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}
1415
+ use_legacy_template #{use_legacy_template_flag}
1312
1416
  }
1313
1417
 
1314
1418
  # connection start
@@ -1316,24 +1420,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1316
1420
  with(basic_auth: ['john', 'doe']).
1317
1421
  to_return(:status => 200, :body => "", :headers => {})
1318
1422
  # check if template exists
1319
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1423
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
1320
1424
  with(basic_auth: ['john', 'doe']).
1321
1425
  to_return(:status => 404, :body => "", :headers => {})
1322
1426
  # creation
1323
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1427
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1324
1428
  with(basic_auth: ['john', 'doe']).
1325
1429
  to_return(:status => 200, :body => "", :headers => {})
1326
1430
  # check if alias exists
1327
1431
  stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1328
1432
  with(basic_auth: ['john', 'doe']).
1329
1433
  to_return(:status => 404, :body => "", :headers => {})
1330
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1434
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
1331
1435
  with(basic_auth: ['john', 'doe']).
1332
1436
  to_return(status: 404, body: "", headers: {})
1333
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1334
- with(basic_auth: ['john', 'doe'],
1335
- body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1336
- to_return(status: 200, body: "", headers: {})
1437
+ if use_legacy_template_flag
1438
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1439
+ with(basic_auth: ['john', 'doe'],
1440
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1441
+ to_return(status: 200, body: "", headers: {})
1442
+ else
1443
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1444
+ with(basic_auth: ['john', 'doe'],
1445
+ body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
1446
+ to_return(status: 200, body: "", headers: {})
1447
+ end
1337
1448
  # put the alias for the index
1338
1449
  stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1339
1450
  with(basic_auth: ['john', 'doe']).
@@ -1354,12 +1465,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1354
1465
 
1355
1466
  driver(config)
1356
1467
 
1357
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1468
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
1358
1469
  end
1359
1470
 
1360
- def test_template_create_with_rollover_index_and_ilm_policies_and_placeholders
1471
+ data("legacy_template" => [true, "_template"],
1472
+ "new_template" => [false, "_index_template"])
1473
+ def test_template_create_with_rollover_index_and_ilm_policies_and_placeholderstest_template_create_with_rollover_index_and_ilm_policies_and_placeholders(data)
1474
+ use_legacy_template_flag, endpoint = data
1361
1475
  cwd = File.dirname(__FILE__)
1362
- template_file = File.join(cwd, 'test_template.json')
1476
+ template_file = if use_legacy_template_flag
1477
+ File.join(cwd, 'test_template.json')
1478
+ else
1479
+ File.join(cwd, 'test_index_template.json')
1480
+ end
1363
1481
 
1364
1482
  config = %{
1365
1483
  host logs.google.com
@@ -1375,6 +1493,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1375
1493
  enable_ilm true
1376
1494
  index_name logstash
1377
1495
  ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
1496
+ use_legacy_template #{use_legacy_template_flag}
1378
1497
  }
1379
1498
 
1380
1499
  # connection start
@@ -1382,24 +1501,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1382
1501
  with(basic_auth: ['john', 'doe']).
1383
1502
  to_return(:status => 200, :body => "", :headers => {})
1384
1503
  # check if template exists
1385
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1504
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
1386
1505
  with(basic_auth: ['john', 'doe']).
1387
1506
  to_return(:status => 404, :body => "", :headers => {})
1388
1507
  # creation
1389
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1508
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1390
1509
  with(basic_auth: ['john', 'doe']).
1391
1510
  to_return(:status => 200, :body => "", :headers => {})
1392
1511
  # check if alias exists
1393
1512
  stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1394
1513
  with(basic_auth: ['john', 'doe']).
1395
1514
  to_return(:status => 404, :body => "", :headers => {})
1396
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1515
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
1397
1516
  with(basic_auth: ['john', 'doe']).
1398
1517
  to_return(status: 404, body: "", headers: {})
1399
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1400
- with(basic_auth: ['john', 'doe'],
1401
- body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1402
- to_return(status: 200, body: "", headers: {})
1518
+ if use_legacy_template_flag
1519
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1520
+ with(basic_auth: ['john', 'doe'],
1521
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1522
+ to_return(status: 200, body: "", headers: {})
1523
+ else
1524
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1525
+ with(basic_auth: ['john', 'doe'],
1526
+ body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
1527
+ to_return(status: 200, body: "", headers: {})
1528
+ end
1403
1529
  # put the alias for the index
1404
1530
  stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1405
1531
  with(basic_auth: ['john', 'doe']).
@@ -1424,15 +1550,22 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1424
1550
  driver.run(default_tag: 'test') do
1425
1551
  driver.feed(sample_record)
1426
1552
  end
1427
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1553
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
1428
1554
 
1429
1555
  assert_requested(elastic_request)
1430
1556
  end
1431
1557
 
1432
1558
  class TemplateCreateWithRolloverIndexAndILMPoliciesWithPlaceholdersTest < self
1433
- def test_tag_placeholder
1559
+ data("legacy_template" => [true, "_template"],
1560
+ "new_template" => [false, "_index_template"])
1561
+ def test_tag_placeholder(data)
1562
+ use_legacy_template_flag, endpoint = data
1434
1563
  cwd = File.dirname(__FILE__)
1435
- template_file = File.join(cwd, 'test_template.json')
1564
+ template_file = if use_legacy_template_flag
1565
+ File.join(cwd, 'test_template.json')
1566
+ else
1567
+ File.join(cwd, 'test_index_template.json')
1568
+ end
1436
1569
 
1437
1570
  config = %{
1438
1571
  host logs.google.com
@@ -1448,6 +1581,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1448
1581
  enable_ilm true
1449
1582
  index_name logstash
1450
1583
  ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
1584
+ use_legacy_template #{use_legacy_template_flag}
1451
1585
  }
1452
1586
 
1453
1587
  # connection start
@@ -1455,24 +1589,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1455
1589
  with(basic_auth: ['john', 'doe']).
1456
1590
  to_return(:status => 200, :body => "", :headers => {})
1457
1591
  # check if template exists
1458
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1592
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
1459
1593
  with(basic_auth: ['john', 'doe']).
1460
1594
  to_return(:status => 404, :body => "", :headers => {})
1461
1595
  # creation
1462
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1596
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1463
1597
  with(basic_auth: ['john', 'doe']).
1464
1598
  to_return(:status => 200, :body => "", :headers => {})
1465
1599
  # check if alias exists
1466
1600
  stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1467
1601
  with(basic_auth: ['john', 'doe']).
1468
1602
  to_return(:status => 404, :body => "", :headers => {})
1469
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1603
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
1470
1604
  with(basic_auth: ['john', 'doe']).
1471
1605
  to_return(status: 404, body: "", headers: {})
1472
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1473
- with(basic_auth: ['john', 'doe'],
1474
- body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1475
- to_return(status: 200, body: "", headers: {})
1606
+ if use_legacy_template_flag
1607
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1608
+ with(basic_auth: ['john', 'doe'],
1609
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1610
+ to_return(status: 200, body: "", headers: {})
1611
+ else
1612
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1613
+ with(basic_auth: ['john', 'doe'],
1614
+ body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
1615
+ to_return(status: 200, body: "", headers: {})
1616
+ end
1476
1617
  # put the alias for the index
1477
1618
  stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1478
1619
  with(basic_auth: ['john', 'doe']).
@@ -1497,14 +1638,21 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1497
1638
  driver.run(default_tag: 'fluentd-policy') do
1498
1639
  driver.feed(sample_record)
1499
1640
  end
1500
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1641
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
1501
1642
 
1502
1643
  assert_requested(elastic_request)
1503
1644
  end
1504
1645
 
1505
- def test_tag_placeholder_with_multiple_policies
1646
+ data("legacy_template" => [true, "_template"],
1647
+ "new_template" => [false, "_index_template"])
1648
+ def test_tag_placeholder_with_multiple_policies(data)
1649
+ use_legacy_template_flag, endpoint = data
1506
1650
  cwd = File.dirname(__FILE__)
1507
- template_file = File.join(cwd, 'test_template.json')
1651
+ template_file = if use_legacy_template_flag
1652
+ File.join(cwd, 'test_template.json')
1653
+ else
1654
+ File.join(cwd, 'test_index_template.json')
1655
+ end
1508
1656
 
1509
1657
  config = %{
1510
1658
  host logs.google.com
@@ -1520,6 +1668,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1520
1668
  enable_ilm true
1521
1669
  index_name logstash
1522
1670
  ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}, "fluentd-policy2":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"80gb", "max_age":"20d"}}}}}}}
1671
+ use_legacy_template #{use_legacy_template_flag}
1523
1672
  }
1524
1673
 
1525
1674
  # connection start
@@ -1527,24 +1676,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1527
1676
  with(basic_auth: ['john', 'doe']).
1528
1677
  to_return(:status => 200, :body => "", :headers => {})
1529
1678
  # check if template exists
1530
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1679
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
1531
1680
  with(basic_auth: ['john', 'doe']).
1532
1681
  to_return(:status => 404, :body => "", :headers => {})
1533
1682
  # creation
1534
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1683
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1535
1684
  with(basic_auth: ['john', 'doe']).
1536
1685
  to_return(:status => 200, :body => "", :headers => {})
1537
1686
  # check if alias exists
1538
1687
  stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
1539
1688
  with(basic_auth: ['john', 'doe']).
1540
1689
  to_return(:status => 404, :body => "", :headers => {})
1541
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1690
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
1542
1691
  with(basic_auth: ['john', 'doe']).
1543
1692
  to_return(status: 404, body: "", headers: {})
1544
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1545
- with(basic_auth: ['john', 'doe'],
1546
- body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
1547
- to_return(status: 200, body: "", headers: {})
1693
+ if use_legacy_template_flag
1694
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1695
+ with(basic_auth: ['john', 'doe'],
1696
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy2\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
1697
+ to_return(status: 200, body: "", headers: {})
1698
+ else
1699
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1700
+ with(basic_auth: ['john', 'doe'],
1701
+ body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy2\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
1702
+ to_return(status: 200, body: "", headers: {})
1703
+ end
1548
1704
  # put the alias for the index
1549
1705
  stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1550
1706
  with(basic_auth: ['john', 'doe']).
@@ -1569,15 +1725,22 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1569
1725
  driver.run(default_tag: 'fluentd-policy2') do
1570
1726
  driver.feed(sample_record)
1571
1727
  end
1572
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
1728
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
1573
1729
 
1574
1730
  assert_requested(elastic_request)
1575
1731
  end
1576
1732
  end
1577
1733
 
1578
- def test_template_create_with_rollover_index_and_default_ilm_and_placeholders
1734
+ data("legacy_template" => [true, "_template"],
1735
+ "new_template" => [false, "_index_template"])
1736
+ def test_template_create_with_rollover_index_and_default_ilm_and_placeholders(data)
1737
+ use_legacy_template_flag, endpoint = data
1579
1738
  cwd = File.dirname(__FILE__)
1580
- template_file = File.join(cwd, 'test_template.json')
1739
+ template_file = if use_legacy_template_flag
1740
+ File.join(cwd, 'test_template.json')
1741
+ else
1742
+ File.join(cwd, 'test_index_template.json')
1743
+ end
1581
1744
 
1582
1745
  config = %{
1583
1746
  host logs.google.com
@@ -1591,6 +1754,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1591
1754
  index_date_pattern now/w{xxxx.ww}
1592
1755
  index_name logstash-${tag}
1593
1756
  enable_ilm true
1757
+ use_legacy_template #{use_legacy_template_flag}
1594
1758
  }
1595
1759
 
1596
1760
  # connection start
@@ -1598,24 +1762,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1598
1762
  with(basic_auth: ['john', 'doe']).
1599
1763
  to_return(:status => 200, :body => "", :headers => {})
1600
1764
  # check if template exists
1601
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
1765
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
1602
1766
  with(basic_auth: ['john', 'doe']).
1603
1767
  to_return(:status => 404, :body => "", :headers => {})
1604
1768
  # creation
1605
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
1769
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
1606
1770
  with(basic_auth: ['john', 'doe']).
1607
1771
  to_return(:status => 200, :body => "", :headers => {})
1608
1772
  # check if alias exists
1609
1773
  stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-test").
1610
1774
  with(basic_auth: ['john', 'doe']).
1611
1775
  to_return(:status => 404, :body => "", :headers => {})
1612
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash-test").
1776
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-test").
1613
1777
  with(basic_auth: ['john', 'doe']).
1614
1778
  to_return(status: 404, body: "", headers: {})
1615
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash-test").
1616
- with(basic_auth: ['john', 'doe'],
1617
- body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-test\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-test-*\",\"order\":52}").
1618
- to_return(status: 200, body: "", headers: {})
1779
+ if use_legacy_template_flag
1780
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-test").
1781
+ with(basic_auth: ['john', 'doe'],
1782
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-test\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-test-*\",\"order\":52}").
1783
+ to_return(status: 200, body: "", headers: {})
1784
+ else
1785
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-test").
1786
+ with(basic_auth: ['john', 'doe'],
1787
+ body: "{\"index_patterns\":\"logstash-test-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-test\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
1788
+ to_return(status: 200, body: "", headers: {})
1789
+ end
1619
1790
  # put the alias for the index
1620
1791
  stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-test-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1621
1792
  with(basic_auth: ['john', 'doe']).
@@ -1648,9 +1819,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1648
1819
  assert_requested(elastic_request)
1649
1820
  end
1650
1821
 
1651
- def test_template_create_with_rollover_index_and_default_ilm_and_custom_and_time_placeholders
1822
+ data("legacy_template" => [true, "_template"],
1823
+ "new_template" => [false, "_index_template"])
1824
+ def test_template_create_with_rollover_index_and_default_ilm_and_custom_and_time_placeholders(data)
1825
+ use_legacy_template_flag, endpoint = data
1652
1826
  cwd = File.dirname(__FILE__)
1653
- template_file = File.join(cwd, 'test_template.json')
1827
+ template_file = if use_legacy_template_flag
1828
+ File.join(cwd, 'test_template.json')
1829
+ else
1830
+ File.join(cwd, 'test_index_template.json')
1831
+ end
1654
1832
 
1655
1833
  config = Fluent::Config::Element.new(
1656
1834
  'ROOT', '', {
@@ -1666,6 +1844,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1666
1844
  'index_date_pattern' => 'now/w{xxxx.ww}',
1667
1845
  'index_name' => "${taskDef}-%Y.%m",
1668
1846
  'enable_ilm' => true,
1847
+ 'use_legacy_template' => use_legacy_template_flag,
1848
+
1669
1849
  }, [
1670
1850
  Fluent::Config::Element.new('buffer', 'tag, time, taskDef', {
1671
1851
  'chunk_keys' => ['tag', 'time', 'taskDef'],
@@ -1681,14 +1861,21 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1681
1861
  with(basic_auth: ['john', 'doe']).
1682
1862
  to_return(:status => 200, :body => "", :headers => {})
1683
1863
  # check if template exists
1684
- stub_request(:get, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
1864
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/#{task_def_value}-#{date_str}").
1685
1865
  with(basic_auth: ['john', 'doe']).
1686
1866
  to_return(:status => 404, :body => "", :headers => {})
1687
1867
  # creation
1688
- stub_request(:put, "https://logs.google.com:777/es//_template/#{task_def_value}-#{date_str}").
1689
- with(basic_auth: ['john', 'doe'],
1690
- body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"#{task_def_value}-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"#{task_def_value}-#{date_str}-*\",\"order\":52}").
1691
- to_return(:status => 200, :body => "", :headers => {})
1868
+ if use_legacy_template_flag
1869
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/#{task_def_value}-#{date_str}").
1870
+ with(basic_auth: ['john', 'doe'],
1871
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"#{task_def_value}-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"#{task_def_value}-#{date_str}-*\",\"order\":52}").
1872
+ to_return(:status => 200, :body => "", :headers => {})
1873
+ else
1874
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/#{task_def_value}-#{date_str}").
1875
+ with(basic_auth: ['john', 'doe'],
1876
+ body: "{\"index_patterns\":\"task_definition-2020.09-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"task_definition-2020.09\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
1877
+ to_return(:status => 200, :body => "", :headers => {})
1878
+ end
1692
1879
  # check if alias exists
1693
1880
  stub_request(:head, "https://logs.google.com:777/es//_alias/#{task_def_value}-#{date_str}").
1694
1881
  with(basic_auth: ['john', 'doe']).
@@ -1726,9 +1913,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1726
1913
  end
1727
1914
  end
1728
1915
 
1729
- def test_custom_template_create
1916
+ data("legacy_template" => [true, "_template"],
1917
+ "new_template" => [false, "_index_template"])
1918
+ def test_custom_template_create(data)
1919
+ use_legacy_template_flag, endpoint = data
1730
1920
  cwd = File.dirname(__FILE__)
1731
- template_file = File.join(cwd, 'test_alias_template.json')
1921
+ template_file = if use_legacy_template_flag
1922
+ File.join(cwd, 'test_alias_template.json')
1923
+ else
1924
+ File.join(cwd, 'test_index_alias_template.json')
1925
+ end
1732
1926
 
1733
1927
  config = %{
1734
1928
  host logs.google.com
@@ -1740,6 +1934,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1740
1934
  template_name myapp_alias_template
1741
1935
  template_file #{template_file}
1742
1936
  customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
1937
+ use_legacy_template #{use_legacy_template_flag}
1743
1938
  }
1744
1939
 
1745
1940
  # connection start
@@ -1747,22 +1942,29 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1747
1942
  with(basic_auth: ['john', 'doe']).
1748
1943
  to_return(:status => 200, :body => "", :headers => {})
1749
1944
  # check if template exists
1750
- stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
1945
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
1751
1946
  with(basic_auth: ['john', 'doe']).
1752
1947
  to_return(:status => 404, :body => "", :headers => {})
1753
1948
  # creation
1754
- stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
1949
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
1755
1950
  with(basic_auth: ['john', 'doe']).
1756
1951
  to_return(:status => 200, :body => "", :headers => {})
1757
1952
 
1758
1953
  driver(config)
1759
1954
 
1760
- assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
1955
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template", times: 1)
1761
1956
  end
1762
1957
 
1763
- def test_custom_template_create_with_customize_template_related_placeholders
1958
+ data("legacy_template" => [true, "_template"],
1959
+ "new_template" => [false, "_index_template"])
1960
+ def test_custom_template_create_with_customize_template_related_placeholders(data)
1961
+ use_legacy_template_flag, endpoint = data
1764
1962
  cwd = File.dirname(__FILE__)
1765
- template_file = File.join(cwd, 'test_alias_template.json')
1963
+ template_file = if use_legacy_template_flag
1964
+ File.join(cwd, 'test_alias_template.json')
1965
+ else
1966
+ File.join(cwd, 'test_index_alias_template.json')
1967
+ end
1766
1968
 
1767
1969
  config = %{
1768
1970
  host logs.google.com
@@ -1774,6 +1976,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1774
1976
  template_name myapp_alias_template-${tag}
1775
1977
  template_file #{template_file}
1776
1978
  customize_template {"--appid--": "${tag}-logs","--index_prefix--":"${tag}"}
1979
+ use_legacy_template #{use_legacy_template_flag}
1777
1980
  }
1778
1981
 
1779
1982
  # connection start
@@ -1781,11 +1984,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1781
1984
  with(basic_auth: ['john', 'doe']).
1782
1985
  to_return(:status => 200, :body => "", :headers => {})
1783
1986
  # check if template exists
1784
- stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template-test.template").
1987
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template-test.template").
1785
1988
  with(basic_auth: ['john', 'doe']).
1786
1989
  to_return(:status => 404, :body => "", :headers => {})
1787
1990
  # creation
1788
- stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template-test.template").
1991
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template-test.template").
1789
1992
  with(basic_auth: ['john', 'doe']).
1790
1993
  to_return(:status => 200, :body => "", :headers => {})
1791
1994
 
@@ -1799,12 +2002,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1799
2002
  driver.feed(sample_record)
1800
2003
  end
1801
2004
 
1802
- assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template-test.template", times: 1)
2005
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template-test.template", times: 1)
1803
2006
  end
1804
2007
 
1805
- def test_custom_template_installation_for_host_placeholder
2008
+ data("legacy_template" => [true, "_template"],
2009
+ "new_template" => [false, "_index_template"])
2010
+ def test_custom_template_installation_for_host_placeholder(data)
2011
+ use_legacy_template_flag, endpoint = data
1806
2012
  cwd = File.dirname(__FILE__)
1807
- template_file = File.join(cwd, 'test_template.json')
2013
+ template_file = if use_legacy_template_flag
2014
+ File.join(cwd, 'test_template.json')
2015
+ else
2016
+ File.join(cwd, 'test_index_template.json')
2017
+ end
1808
2018
 
1809
2019
  config = %{
1810
2020
  host logs-${tag}.google.com
@@ -1818,6 +2028,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1818
2028
  verify_es_version_at_startup false
1819
2029
  default_elasticsearch_version 6
1820
2030
  customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
2031
+ use_legacy_template #{use_legacy_template_flag}
1821
2032
  }
1822
2033
 
1823
2034
  # connection start
@@ -1825,10 +2036,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1825
2036
  with(basic_auth: ['john', 'doe']).
1826
2037
  to_return(:status => 200, :body => "", :headers => {})
1827
2038
  # check if template exists
1828
- stub_request(:get, "https://logs-test.google.com:777/es//_template/logstash").
2039
+ stub_request(:get, "https://logs-test.google.com:777/es//#{endpoint}/logstash").
1829
2040
  with(basic_auth: ['john', 'doe']).
1830
2041
  to_return(:status => 404, :body => "", :headers => {})
1831
- stub_request(:put, "https://logs-test.google.com:777/es//_template/logstash").
2042
+ stub_request(:put, "https://logs-test.google.com:777/es//#{endpoint}/logstash").
1832
2043
  with(basic_auth: ['john', 'doe']).
1833
2044
  to_return(status: 200, body: "", headers: {})
1834
2045
 
@@ -1840,9 +2051,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1840
2051
  end
1841
2052
  end
1842
2053
 
1843
- def test_custom_template_with_rollover_index_create
2054
+ data("legacy_template" => [true, "_template"],
2055
+ "new_template" => [false, "_index_template"])
2056
+ def test_custom_template_with_rollover_index_create(data)
2057
+ use_legacy_template_flag, endpoint = data
1844
2058
  cwd = File.dirname(__FILE__)
1845
- template_file = File.join(cwd, 'test_alias_template.json')
2059
+ template_file = if use_legacy_template_flag
2060
+ File.join(cwd, 'test_alias_template.json')
2061
+ else
2062
+ File.join(cwd, 'test_index_alias_template.json')
2063
+ end
1846
2064
 
1847
2065
  config = %{
1848
2066
  host logs.google.com
@@ -1858,6 +2076,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1858
2076
  index_date_pattern now/w{xxxx.ww}
1859
2077
  index_name mylogs
1860
2078
  application_name myapp
2079
+ use_legacy_template #{use_legacy_template_flag}
1861
2080
  }
1862
2081
 
1863
2082
  # connection start
@@ -1865,11 +2084,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1865
2084
  with(basic_auth: ['john', 'doe']).
1866
2085
  to_return(:status => 200, :body => "", :headers => {})
1867
2086
  # check if template exists
1868
- stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
2087
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
1869
2088
  with(basic_auth: ['john', 'doe']).
1870
2089
  to_return(:status => 404, :body => "", :headers => {})
1871
2090
  # creation
1872
- stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
2091
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
1873
2092
  with(basic_auth: ['john', 'doe']).
1874
2093
  to_return(:status => 200, :body => "", :headers => {})
1875
2094
  # creation of index which can rollover
@@ -1887,12 +2106,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1887
2106
 
1888
2107
  driver(config)
1889
2108
 
1890
- assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
2109
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template", times: 1)
1891
2110
  end
1892
2111
 
1893
- def test_custom_template_with_rollover_index_create_and_deflector_alias
2112
+ data("legacy_template" => [true, "_template"],
2113
+ "new_template" => [false, "_index_template"])
2114
+ def test_custom_template_with_rollover_index_create_and_deflector_alias(data)
2115
+ use_legacy_template_flag, endpoint = data
1894
2116
  cwd = File.dirname(__FILE__)
1895
- template_file = File.join(cwd, 'test_alias_template.json')
2117
+ template_file = if use_legacy_template_flag
2118
+ File.join(cwd, 'test_alias_template.json')
2119
+ else
2120
+ File.join(cwd, 'test_index_alias_template.json')
2121
+ end
1896
2122
 
1897
2123
  config = %{
1898
2124
  host logs.google.com
@@ -1909,6 +2135,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1909
2135
  deflector_alias myapp_deflector
1910
2136
  index_name mylogs
1911
2137
  application_name myapp
2138
+ use_legacy_template #{use_legacy_template_flag}
1912
2139
  }
1913
2140
 
1914
2141
  # connection start
@@ -1916,11 +2143,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1916
2143
  with(basic_auth: ['john', 'doe']).
1917
2144
  to_return(:status => 200, :body => "", :headers => {})
1918
2145
  # check if template exists
1919
- stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
2146
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
1920
2147
  with(basic_auth: ['john', 'doe']).
1921
2148
  to_return(:status => 404, :body => "", :headers => {})
1922
2149
  # creation
1923
- stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
2150
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
1924
2151
  with(basic_auth: ['john', 'doe']).
1925
2152
  to_return(:status => 200, :body => "", :headers => {})
1926
2153
  # creation of index which can rollover
@@ -1938,12 +2165,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1938
2165
 
1939
2166
  driver(config)
1940
2167
 
1941
- assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
2168
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template", times: 1)
1942
2169
  end
1943
2170
 
1944
- def test_custom_template_with_rollover_index_create_with_logstash_format
2171
+ data("legacy_template" => [true, "_template"],
2172
+ "new_template" => [false, "_index_template"])
2173
+ def test_custom_template_with_rollover_index_create_with_logstash_format(data)
2174
+ use_legacy_template_flag, endpoint = data
1945
2175
  cwd = File.dirname(__FILE__)
1946
- template_file = File.join(cwd, 'test_alias_template.json')
2176
+ template_file = if use_legacy_template_flag
2177
+ File.join(cwd, 'test_alias_template.json')
2178
+ else
2179
+ File.join(cwd, 'test_index_alias_template.json')
2180
+ end
1947
2181
 
1948
2182
  config = %{
1949
2183
  host logs.google.com
@@ -1960,6 +2194,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1960
2194
  logstash_format true
1961
2195
  logstash_prefix mylogs
1962
2196
  application_name myapp
2197
+ use_legacy_template #{use_legacy_template_flag}
1963
2198
  }
1964
2199
 
1965
2200
  timestr = Time.now.strftime("%Y.%m.%d")
@@ -1968,11 +2203,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
1968
2203
  with(basic_auth: ['john', 'doe']).
1969
2204
  to_return(:status => 200, :body => "", :headers => {})
1970
2205
  # check if template exists
1971
- stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
2206
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
1972
2207
  with(basic_auth: ['john', 'doe']).
1973
2208
  to_return(:status => 404, :body => "", :headers => {})
1974
2209
  # creation
1975
- stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
2210
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
1976
2211
  with(basic_auth: ['john', 'doe']).
1977
2212
  to_return(:status => 200, :body => "", :headers => {})
1978
2213
  # creation of index which can rollover
@@ -2007,9 +2242,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2007
2242
  end
2008
2243
  end
2009
2244
 
2010
- def test_custom_template_with_rollover_index_create_and_default_ilm
2245
+ data("legacy_template" => [true, "_template"],
2246
+ "new_template" => [false, "_index_template"])
2247
+ def test_custom_template_with_rollover_index_create_and_default_ilm(data)
2248
+ use_legacy_template_flag, endpoint = data
2011
2249
  cwd = File.dirname(__FILE__)
2012
- template_file = File.join(cwd, 'test_alias_template.json')
2250
+ template_file = if use_legacy_template_flag
2251
+ File.join(cwd, 'test_alias_template.json')
2252
+ else
2253
+ File.join(cwd, 'test_index_alias_template.json')
2254
+ end
2013
2255
 
2014
2256
  config = %{
2015
2257
  host logs.google.com
@@ -2026,6 +2268,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2026
2268
  application_name myapp
2027
2269
  ilm_policy_id fluentd-policy
2028
2270
  enable_ilm true
2271
+ use_legacy_template #{use_legacy_template_flag}
2029
2272
  }
2030
2273
 
2031
2274
  # connection start
@@ -2033,11 +2276,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2033
2276
  with(basic_auth: ['john', 'doe']).
2034
2277
  to_return(:status => 200, :body => "", :headers => {})
2035
2278
  # check if template exists
2036
- stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
2279
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
2037
2280
  with(basic_auth: ['john', 'doe']).
2038
2281
  to_return(:status => 404, :body => "", :headers => {})
2039
2282
  # creation
2040
- stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
2283
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
2041
2284
  with(basic_auth: ['john', 'doe']).
2042
2285
  to_return(:status => 200, :body => "", :headers => {})
2043
2286
  # creation of index which can rollover
@@ -2048,13 +2291,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2048
2291
  stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs").
2049
2292
  with(basic_auth: ['john', 'doe']).
2050
2293
  to_return(:status => 404, :body => "", :headers => {})
2051
- stub_request(:get, "https://logs.google.com:777/es//_template/mylogs").
2294
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs").
2052
2295
  with(basic_auth: ['john', 'doe']).
2053
2296
  to_return(status: 404, body: "", headers: {})
2054
- stub_request(:put, "https://logs.google.com:777/es//_template/mylogs").
2055
- with(basic_auth: ['john', 'doe'],
2056
- body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-*\"}").
2057
- to_return(status: 200, body: "", headers: {})
2297
+ if use_legacy_template_flag
2298
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs").
2299
+ with(basic_auth: ['john', 'doe'],
2300
+ body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-*\"}").
2301
+ to_return(status: 200, body: "", headers: {})
2302
+ else
2303
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs").
2304
+ with(basic_auth: ['john', 'doe'],
2305
+ body: "{\"priority\":106,\"index_patterns\":\"mylogs-*\",\"template\":{\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}}}}").
2306
+ to_return(status: 200, body: "", headers: {})
2307
+ end
2058
2308
  # put the alias for the index
2059
2309
  stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs").
2060
2310
  with(basic_auth: ['john', 'doe'],
@@ -2073,12 +2323,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2073
2323
 
2074
2324
  driver(config)
2075
2325
 
2076
- assert_requested(:put, "https://logs.google.com:777/es//_template/mylogs", times: 1)
2326
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs", times: 1)
2077
2327
  end
2078
2328
 
2079
- def test_custom_template_with_rollover_index_create_and_default_ilm_and_ilm_policy_overwrite
2329
+ data("legacy_template" => [true, "_template"],
2330
+ "new_template" => [false, "_index_template"])
2331
+ def test_custom_template_with_rollover_index_create_and_default_ilm_and_ilm_policy_overwrite(data)
2332
+ use_legacy_template_flag, endpoint = data
2080
2333
  cwd = File.dirname(__FILE__)
2081
- template_file = File.join(cwd, 'test_alias_template.json')
2334
+ template_file = if use_legacy_template_flag
2335
+ File.join(cwd, 'test_alias_template.json')
2336
+ else
2337
+ File.join(cwd, 'test_index_alias_template.json')
2338
+ end
2082
2339
 
2083
2340
  config = %{
2084
2341
  host logs.google.com
@@ -2097,6 +2354,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2097
2354
  enable_ilm true
2098
2355
  ilm_policy_overwrite true
2099
2356
  ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"60gb","max_age": "45d"}}}}}}
2357
+ use_legacy_template #{use_legacy_template_flag}
2100
2358
  }
2101
2359
 
2102
2360
  # connection start
@@ -2104,11 +2362,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2104
2362
  with(basic_auth: ['john', 'doe']).
2105
2363
  to_return(:status => 200, :body => "", :headers => {})
2106
2364
  # check if template exists
2107
- stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
2365
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
2108
2366
  with(basic_auth: ['john', 'doe']).
2109
2367
  to_return(:status => 404, :body => "", :headers => {})
2110
2368
  # creation
2111
- stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
2369
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
2112
2370
  with(basic_auth: ['john', 'doe']).
2113
2371
  to_return(:status => 200, :body => "", :headers => {})
2114
2372
  # creation of index which can rollover
@@ -2119,13 +2377,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2119
2377
  stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs").
2120
2378
  with(basic_auth: ['john', 'doe']).
2121
2379
  to_return(:status => 404, :body => "", :headers => {})
2122
- stub_request(:get, "https://logs.google.com:777/es//_template/mylogs").
2380
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs").
2123
2381
  with(basic_auth: ['john', 'doe']).
2124
2382
  to_return(status: 404, body: "", headers: {})
2125
- stub_request(:put, "https://logs.google.com:777/es//_template/mylogs").
2126
- with(basic_auth: ['john', 'doe'],
2127
- body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-*\"}").
2128
- to_return(status: 200, body: "", headers: {})
2383
+ if use_legacy_template_flag
2384
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs").
2385
+ with(basic_auth: ['john', 'doe'],
2386
+ body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-*\"}").
2387
+ to_return(status: 200, body: "", headers: {})
2388
+ else
2389
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs").
2390
+ with(basic_auth: ['john', 'doe'],
2391
+ body: "{\"priority\":106,\"index_patterns\":\"mylogs-*\",\"template\":{\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}}}}").
2392
+ to_return(status: 200, body: "", headers: {})
2393
+ end
2129
2394
  # put the alias for the index
2130
2395
  stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs").
2131
2396
  with(basic_auth: ['john', 'doe'],
@@ -2144,7 +2409,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2144
2409
 
2145
2410
  driver(config)
2146
2411
 
2147
- assert_requested(:put, "https://logs.google.com:777/es//_template/mylogs", times: 1)
2412
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs", times: 1)
2148
2413
  end
2149
2414
 
2150
2415
  def test_custom_template_with_rollover_index_create_and_default_ilm_with_deflector_alias
@@ -2175,9 +2440,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2175
2440
  end
2176
2441
  end
2177
2442
 
2178
- def test_custom_template_with_rollover_index_create_and_default_ilm_and_placeholders
2443
+ data("legacy_template" => [true, "_template"],
2444
+ "new_template" => [false, "_index_template"])
2445
+ def test_custom_template_with_rollover_index_create_and_default_ilm_and_placeholders(data)
2446
+ use_legacy_template_flag, endpoint = data
2179
2447
  cwd = File.dirname(__FILE__)
2180
- template_file = File.join(cwd, 'test_alias_template.json')
2448
+ template_file = if use_legacy_template_flag
2449
+ File.join(cwd, 'test_alias_template.json')
2450
+ else
2451
+ File.join(cwd, 'test_index_alias_template.json')
2452
+ end
2181
2453
 
2182
2454
  config = %{
2183
2455
  host logs.google.com
@@ -2194,6 +2466,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2194
2466
  application_name myapp
2195
2467
  ilm_policy_id fluentd-policy
2196
2468
  enable_ilm true
2469
+ use_legacy_template #{use_legacy_template_flag}
2197
2470
  }
2198
2471
 
2199
2472
  # connection start
@@ -2201,11 +2474,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2201
2474
  with(basic_auth: ['john', 'doe']).
2202
2475
  to_return(:status => 200, :body => "", :headers => {})
2203
2476
  # check if template exists
2204
- stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
2477
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
2205
2478
  with(basic_auth: ['john', 'doe']).
2206
2479
  to_return(:status => 404, :body => "", :headers => {})
2207
2480
  # creation
2208
- stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
2481
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
2209
2482
  with(basic_auth: ['john', 'doe']).
2210
2483
  to_return(:status => 200, :body => "", :headers => {})
2211
2484
  # creation of index which can rollover
@@ -2216,13 +2489,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2216
2489
  stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-custom-test").
2217
2490
  with(basic_auth: ['john', 'doe']).
2218
2491
  to_return(:status => 404, :body => "", :headers => {})
2219
- stub_request(:get, "https://logs.google.com:777/es//_template/mylogs-custom-test").
2492
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs-custom-test").
2220
2493
  with(basic_auth: ['john', 'doe']).
2221
2494
  to_return(status: 404, body: "", headers: {})
2222
- stub_request(:put, "https://logs.google.com:777/es//_template/mylogs-custom-test").
2223
- with(basic_auth: ['john', 'doe'],
2224
- body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-custom-test\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-custom-test-*\"}").
2225
- to_return(status: 200, body: "", headers: {})
2495
+ if use_legacy_template_flag
2496
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-custom-test").
2497
+ with(basic_auth: ['john', 'doe'],
2498
+ body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-custom-test\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-custom-test-*\"}").
2499
+ to_return(status: 200, body: "", headers: {})
2500
+ else
2501
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-custom-test").
2502
+ with(basic_auth: ['john', 'doe'],
2503
+ body: "{\"priority\":108,\"index_patterns\":\"mylogs-custom-test-*\",\"template\":{\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-custom-test\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}}}}").
2504
+ to_return(status: 200, body: "", headers: {})
2505
+ end
2226
2506
  # put the alias for the index
2227
2507
  stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-custom-test-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-custom-test").
2228
2508
  with(basic_auth: ['john', 'doe'],
@@ -2252,9 +2532,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2252
2532
  assert_requested(elastic_request)
2253
2533
  end
2254
2534
 
2255
- def test_custom_template_with_rollover_index_create_and_custom_ilm
2535
+ data("legacy_template" => [true, "_template"],
2536
+ "new_template" => [false, "_index_template"])
2537
+ def test_custom_template_with_rollover_index_create_and_custom_ilm(data)
2538
+ use_legacy_template_flag, endpoint = data
2256
2539
  cwd = File.dirname(__FILE__)
2257
- template_file = File.join(cwd, 'test_alias_template.json')
2540
+ template_file = if use_legacy_template_flag
2541
+ File.join(cwd, 'test_alias_template.json')
2542
+ else
2543
+ File.join(cwd, 'test_index_alias_template.json')
2544
+ end
2258
2545
 
2259
2546
  config = %{
2260
2547
  host logs.google.com
@@ -2272,6 +2559,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2272
2559
  ilm_policy_id fluentd-policy
2273
2560
  enable_ilm true
2274
2561
  ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}
2562
+ use_legacy_template #{use_legacy_template_flag}
2275
2563
  }
2276
2564
 
2277
2565
  # connection start
@@ -2279,11 +2567,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2279
2567
  with(basic_auth: ['john', 'doe']).
2280
2568
  to_return(:status => 200, :body => "", :headers => {})
2281
2569
  # check if template exists
2282
- stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
2570
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
2283
2571
  with(basic_auth: ['john', 'doe']).
2284
2572
  to_return(:status => 404, :body => "", :headers => {})
2285
2573
  # creation
2286
- stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
2574
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
2287
2575
  with(basic_auth: ['john', 'doe']).
2288
2576
  to_return(:status => 200, :body => "", :headers => {})
2289
2577
  # creation of index which can rollover
@@ -2294,10 +2582,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2294
2582
  stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs").
2295
2583
  with(basic_auth: ['john', 'doe']).
2296
2584
  to_return(:status => 404, :body => "", :headers => {})
2297
- stub_request(:get, "https://logs.google.com:777/es//_template/mylogs").
2585
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs").
2298
2586
  with(basic_auth: ['john', 'doe']).
2299
2587
  to_return(status: 404, body: "", headers: {})
2300
- stub_request(:put, "https://logs.google.com:777/es//_template/mylogs").
2588
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs").
2301
2589
  with(basic_auth: ['john', 'doe']).
2302
2590
  to_return(status: 200, body: "", headers: {})
2303
2591
  # put the alias for the index
@@ -2318,13 +2606,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2318
2606
 
2319
2607
  driver(config)
2320
2608
 
2321
- assert_requested(:put, "https://logs.google.com:777/es//_template/mylogs", times: 1)
2609
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs", times: 1)
2322
2610
  end
2323
2611
  end
2324
2612
 
2325
- def test_template_overwrite
2613
+ data("legacy_template" => [true, "_template"],
2614
+ "new_template" => [false, "_index_template"])
2615
+ def test_template_overwrite(data)
2616
+ use_legacy_template_flag, endpoint = data
2326
2617
  cwd = File.dirname(__FILE__)
2327
- template_file = File.join(cwd, 'test_template.json')
2618
+ template_file = if use_legacy_template_flag
2619
+ File.join(cwd, 'test_template.json')
2620
+ else
2621
+ File.join(cwd, 'test_index_template.json')
2622
+ end
2328
2623
 
2329
2624
  config = %{
2330
2625
  host logs.google.com
@@ -2336,6 +2631,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2336
2631
  template_name logstash
2337
2632
  template_file #{template_file}
2338
2633
  template_overwrite true
2634
+ use_legacy_template #{use_legacy_template_flag}
2339
2635
  }
2340
2636
 
2341
2637
  # connection start
@@ -2343,22 +2639,29 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2343
2639
  with(basic_auth: ['john', 'doe']).
2344
2640
  to_return(:status => 200, :body => "", :headers => {})
2345
2641
  # check if template exists
2346
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
2642
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
2347
2643
  with(basic_auth: ['john', 'doe']).
2348
2644
  to_return(:status => 200, :body => "", :headers => {})
2349
2645
  # creation
2350
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
2646
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
2351
2647
  with(basic_auth: ['john', 'doe']).
2352
2648
  to_return(:status => 200, :body => "", :headers => {})
2353
2649
 
2354
2650
  driver(config)
2355
2651
 
2356
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
2652
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
2357
2653
  end
2358
2654
 
2359
- def test_custom_template_overwrite
2655
+ data("legacy_template" => [true, "_template"],
2656
+ "new_template" => [false, "_index_template"])
2657
+ def test_custom_template_overwrite(data)
2658
+ use_legacy_template_flag, endpoint = data
2360
2659
  cwd = File.dirname(__FILE__)
2361
- template_file = File.join(cwd, 'test_template.json')
2660
+ template_file = if use_legacy_template_flag
2661
+ File.join(cwd, 'test_template.json')
2662
+ else
2663
+ File.join(cwd, 'test_index_template.json')
2664
+ end
2362
2665
 
2363
2666
  config = %{
2364
2667
  host logs.google.com
@@ -2371,6 +2674,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2371
2674
  template_file #{template_file}
2372
2675
  template_overwrite true
2373
2676
  customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
2677
+ use_legacy_template #{use_legacy_template_flag}
2374
2678
  }
2375
2679
 
2376
2680
  # connection start
@@ -2378,22 +2682,29 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2378
2682
  with(basic_auth: ['john', 'doe']).
2379
2683
  to_return(:status => 200, :body => "", :headers => {})
2380
2684
  # check if template exists
2381
- stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
2685
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
2382
2686
  with(basic_auth: ['john', 'doe']).
2383
2687
  to_return(:status => 200, :body => "", :headers => {})
2384
2688
  # creation
2385
- stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
2689
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
2386
2690
  with(basic_auth: ['john', 'doe']).
2387
2691
  to_return(:status => 200, :body => "", :headers => {})
2388
2692
 
2389
2693
  driver(config)
2390
2694
 
2391
- assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
2695
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template", times: 1)
2392
2696
  end
2393
2697
 
2394
- def test_custom_template_with_rollover_index_overwrite
2698
+ data("legacy_template" => [true, "_template"],
2699
+ "new_template" => [false, "_index_template"])
2700
+ def test_custom_template_with_rollover_index_overwrite(data)
2701
+ use_legacy_template_flag, endpoint = data
2395
2702
  cwd = File.dirname(__FILE__)
2396
- template_file = File.join(cwd, 'test_template.json')
2703
+ template_file = if use_legacy_template_flag
2704
+ File.join(cwd, 'test_template.json')
2705
+ else
2706
+ File.join(cwd, 'test_index_template.json')
2707
+ end
2397
2708
 
2398
2709
  config = %{
2399
2710
  host logs.google.com
@@ -2410,6 +2721,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2410
2721
  rollover_index true
2411
2722
  index_name mylogs
2412
2723
  application_name myapp
2724
+ use_legacy_template #{use_legacy_template_flag}
2413
2725
  }
2414
2726
 
2415
2727
  # connection start
@@ -2417,11 +2729,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2417
2729
  with(basic_auth: ['john', 'doe']).
2418
2730
  to_return(:status => 200, :body => "", :headers => {})
2419
2731
  # check if template exists
2420
- stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
2732
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
2421
2733
  with(basic_auth: ['john', 'doe']).
2422
2734
  to_return(:status => 200, :body => "", :headers => {})
2423
2735
  # creation
2424
- stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
2736
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
2425
2737
  with(basic_auth: ['john', 'doe']).
2426
2738
  to_return(:status => 200, :body => "", :headers => {})
2427
2739
  # creation of index which can rollover
@@ -2439,7 +2751,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2439
2751
 
2440
2752
  driver(config)
2441
2753
 
2442
- assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
2754
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template", times: 1)
2443
2755
  end
2444
2756
 
2445
2757
  def test_template_create_invalid_filename
@@ -2469,9 +2781,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2469
2781
  }
2470
2782
  end
2471
2783
 
2472
- def test_template_create_for_host_placeholder
2784
+ data("legacy_template" => [true, "_template"],
2785
+ "new_template" => [false, "_index_template"])
2786
+ def test_template_create_for_host_placeholder(data)
2787
+ use_legacy_template_flag, endpoint = data
2473
2788
  cwd = File.dirname(__FILE__)
2474
- template_file = File.join(cwd, 'test_template.json')
2789
+ template_file = if use_legacy_template_flag
2790
+ File.join(cwd, 'test_template.json')
2791
+ else
2792
+ File.join(cwd, 'test_index_template.json')
2793
+ end
2475
2794
 
2476
2795
  config = %{
2477
2796
  host logs-${tag}.google.com
@@ -2484,6 +2803,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2484
2803
  template_file #{template_file}
2485
2804
  verify_es_version_at_startup false
2486
2805
  default_elasticsearch_version 6
2806
+ use_legacy_template #{use_legacy_template_flag}
2487
2807
  }
2488
2808
 
2489
2809
  # connection start
@@ -2491,10 +2811,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2491
2811
  with(basic_auth: ['john', 'doe']).
2492
2812
  to_return(:status => 200, :body => "", :headers => {})
2493
2813
  # check if template exists
2494
- stub_request(:get, "https://logs-test.google.com:777/es//_template/logstash").
2814
+ stub_request(:get, "https://logs-test.google.com:777/es//#{endpoint}/logstash").
2495
2815
  with(basic_auth: ['john', 'doe']).
2496
2816
  to_return(:status => 404, :body => "", :headers => {})
2497
- stub_request(:put, "https://logs-test.google.com:777/es//_template/logstash").
2817
+ stub_request(:put, "https://logs-test.google.com:777/es//#{endpoint}/logstash").
2498
2818
  with(basic_auth: ['john', 'doe']).
2499
2819
  to_return(status: 200, body: "", headers: {})
2500
2820
  stub_request(:post, "https://logs-test.google.com:777/es//_bulk").
@@ -2509,9 +2829,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2509
2829
  end
2510
2830
  end
2511
2831
 
2512
- def test_template_retry_install_fails
2832
+ data("legacy_template" => [true, "_template"],
2833
+ "new_template" => [false, "_index_template"])
2834
+ def test_template_retry_install_fails(data)
2835
+ use_legacy_template_flag, endpoint = data
2513
2836
  cwd = File.dirname(__FILE__)
2514
- template_file = File.join(cwd, 'test_template.json')
2837
+ template_file = if use_legacy_template_flag
2838
+ File.join(cwd, 'test_template.json')
2839
+ else
2840
+ File.join(cwd, 'test_index_template.json')
2841
+ end
2515
2842
 
2516
2843
  config = %{
2517
2844
  host logs.google.com
@@ -2523,11 +2850,12 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2523
2850
  template_name logstash
2524
2851
  template_file #{template_file}
2525
2852
  max_retry_putting_template 3
2853
+ use_legacy_template #{use_legacy_template_flag}
2526
2854
  }
2527
2855
 
2528
2856
  connection_resets = 0
2529
2857
  # check if template exists
2530
- stub_request(:get, "https://logs.google.com:778/es//_template/logstash")
2858
+ stub_request(:get, "https://logs.google.com:778/es//#{endpoint}/logstash")
2531
2859
  .with(basic_auth: ['john', 'doe']) do |req|
2532
2860
  connection_resets += 1
2533
2861
  raise Faraday::ConnectionFailed, "Test message"
@@ -2540,9 +2868,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2540
2868
  assert_equal(4, connection_resets)
2541
2869
  end
2542
2870
 
2543
- def test_template_retry_install_does_not_fail
2871
+ data("legacy_template" => [true, "_template"],
2872
+ "new_template" => [false, "_index_template"])
2873
+ def test_template_retry_install_does_not_fail(data)
2874
+ use_legacy_template_flag, endpoint = data
2544
2875
  cwd = File.dirname(__FILE__)
2545
- template_file = File.join(cwd, 'test_template.json')
2876
+ template_file = if use_legacy_template_flag
2877
+ File.join(cwd, 'test_template.json')
2878
+ else
2879
+ File.join(cwd, 'test_index_template.json')
2880
+ end
2546
2881
 
2547
2882
  config = %{
2548
2883
  host logs.google.com
@@ -2555,11 +2890,12 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2555
2890
  template_file #{template_file}
2556
2891
  max_retry_putting_template 3
2557
2892
  fail_on_putting_template_retry_exceed false
2893
+ use_legacy_template #{use_legacy_template_flag}
2558
2894
  }
2559
2895
 
2560
2896
  connection_resets = 0
2561
2897
  # check if template exists
2562
- stub_request(:get, "https://logs.google.com:778/es//_template/logstash")
2898
+ stub_request(:get, "https://logs.google.com:778/es//#{endpoint}/logstash")
2563
2899
  .with(basic_auth: ['john', 'doe']) do |req|
2564
2900
  connection_resets += 1
2565
2901
  raise Faraday::ConnectionFailed, "Test message"
@@ -2570,9 +2906,17 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2570
2906
  assert_equal(4, connection_resets)
2571
2907
  end
2572
2908
 
2573
- def test_templates_create
2909
+ data("legacy_template" => [true, "_template"],
2910
+ "new_template" => [false, "_index_template"])
2911
+ def test_templates_create(data)
2912
+ use_legacy_template_flag, endpoint = data
2574
2913
  cwd = File.dirname(__FILE__)
2575
- template_file = File.join(cwd, 'test_template.json')
2914
+ template_file = if use_legacy_template_flag
2915
+ File.join(cwd, 'test_template.json')
2916
+ else
2917
+ File.join(cwd, 'test_index_template.json')
2918
+ end
2919
+
2576
2920
  config = %{
2577
2921
  host logs.google.com
2578
2922
  port 777
@@ -2581,43 +2925,52 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2581
2925
  user john
2582
2926
  password doe
2583
2927
  templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
2928
+ use_legacy_template #{use_legacy_template_flag}
2584
2929
  }
2585
2930
 
2586
2931
  stub_request(:head, "https://logs.google.com:777/es//").
2587
2932
  with(basic_auth: ['john', 'doe']).
2588
2933
  to_return(:status => 200, :body => "", :headers => {})
2589
2934
  # check if template exists
2590
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash1").
2935
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash1").
2591
2936
  with(basic_auth: ['john', 'doe']).
2592
2937
  to_return(:status => 404, :body => "", :headers => {})
2593
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash2").
2938
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash2").
2594
2939
  with(basic_auth: ['john', 'doe']).
2595
2940
  to_return(:status => 404, :body => "", :headers => {})
2596
2941
 
2597
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash3").
2942
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash3").
2598
2943
  with(basic_auth: ['john', 'doe']).
2599
2944
  to_return(:status => 200, :body => "", :headers => {}) #exists
2600
2945
 
2601
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash1").
2946
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1").
2602
2947
  with(basic_auth: ['john', 'doe']).
2603
2948
  to_return(:status => 200, :body => "", :headers => {})
2604
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash2").
2949
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2").
2605
2950
  with(basic_auth: ['john', 'doe']).
2606
2951
  to_return(:status => 200, :body => "", :headers => {})
2607
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash3").
2952
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash3").
2608
2953
  with(basic_auth: ['john', 'doe']).
2609
2954
  to_return(:status => 200, :body => "", :headers => {})
2610
2955
 
2611
2956
  driver(config)
2612
2957
 
2613
- assert_requested( :put, "https://logs.google.com:777/es//_template/logstash1", times: 1)
2614
- assert_requested( :put, "https://logs.google.com:777/es//_template/logstash2", times: 1)
2615
- assert_not_requested(:put, "https://logs.google.com:777/es//_template/logstash3") #exists
2958
+ assert_requested( :put, "https://logs.google.com:777/es//#{endpoint}/logstash1", times: 1)
2959
+ assert_requested( :put, "https://logs.google.com:777/es//#{endpoint}/logstash2", times: 1)
2960
+ assert_not_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash3") #exists
2616
2961
  end
2617
2962
 
2618
- def test_templates_overwrite
2963
+ data("legacy_template" => [true, "_template"],
2964
+ "new_template" => [false, "_index_template"])
2965
+ def test_templates_overwrite(data)
2966
+ use_legacy_template_flag, endpoint = data
2619
2967
  cwd = File.dirname(__FILE__)
2620
- template_file = File.join(cwd, 'test_template.json')
2968
+ template_file = if use_legacy_template_flag
2969
+ File.join(cwd, 'test_template.json')
2970
+ else
2971
+ File.join(cwd, 'test_index_template.json')
2972
+ end
2973
+
2621
2974
  config = %{
2622
2975
  host logs.google.com
2623
2976
  port 777
@@ -2627,42 +2980,50 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2627
2980
  password doe
2628
2981
  templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
2629
2982
  template_overwrite true
2983
+ use_legacy_template #{use_legacy_template_flag}
2630
2984
  }
2631
2985
 
2632
2986
  stub_request(:head, "https://logs.google.com:777/es//").
2633
2987
  with(basic_auth: ['john', 'doe']).
2634
2988
  to_return(:status => 200, :body => "", :headers => {})
2635
2989
  # check if template exists
2636
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash1").
2990
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash1").
2637
2991
  with(basic_auth: ['john', 'doe']).
2638
2992
  to_return(:status => 200, :body => "", :headers => {})
2639
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash2").
2993
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash2").
2640
2994
  with(basic_auth: ['john', 'doe']).
2641
2995
  to_return(:status => 200, :body => "", :headers => {})
2642
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash3").
2996
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash3").
2643
2997
  with(basic_auth: ['john', 'doe']).
2644
2998
  to_return(:status => 200, :body => "", :headers => {}) #exists
2645
2999
 
2646
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash1").
3000
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1").
2647
3001
  with(basic_auth: ['john', 'doe']).
2648
3002
  to_return(:status => 200, :body => "", :headers => {})
2649
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash2").
3003
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2").
2650
3004
  with(basic_auth: ['john', 'doe']).
2651
3005
  to_return(:status => 200, :body => "", :headers => {})
2652
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash3").
3006
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash3").
2653
3007
  with(basic_auth: ['john', 'doe']).
2654
3008
  to_return(:status => 200, :body => "", :headers => {})
2655
3009
 
2656
3010
  driver(config)
2657
3011
 
2658
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash1", times: 1)
2659
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash2", times: 1)
2660
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash3", times: 1)
3012
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1", times: 1)
3013
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2", times: 1)
3014
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash3", times: 1)
2661
3015
  end
2662
3016
 
2663
- def test_templates_are_also_used
3017
+ data("legacy_template" => [true, "_template"],
3018
+ "new_template" => [false, "_index_template"])
3019
+ def test_templates_are_also_used(data)
3020
+ use_legacy_template_flag, endpoint = data
2664
3021
  cwd = File.dirname(__FILE__)
2665
- template_file = File.join(cwd, 'test_template.json')
3022
+ template_file = if use_legacy_template_flag
3023
+ File.join(cwd, 'test_template.json')
3024
+ else
3025
+ File.join(cwd, 'test_index_template.json')
3026
+ end
2666
3027
 
2667
3028
  config = %{
2668
3029
  host logs.google.com
@@ -2674,43 +3035,52 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2674
3035
  template_name logstash
2675
3036
  template_file #{template_file}
2676
3037
  templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}" }
3038
+ use_legacy_template #{use_legacy_template_flag}
2677
3039
  }
2678
3040
  # connection start
2679
3041
  stub_request(:head, "https://logs.google.com:777/es//").
2680
3042
  with(basic_auth: ['john', 'doe']).
2681
3043
  to_return(:status => 200, :body => "", :headers => {})
2682
3044
  # check if template exists
2683
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
3045
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
2684
3046
  with(basic_auth: ['john', 'doe']).
2685
3047
  to_return(:status => 404, :body => "", :headers => {})
2686
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash1").
3048
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash1").
2687
3049
  with(basic_auth: ['john', 'doe']).
2688
3050
  to_return(:status => 404, :body => "", :headers => {})
2689
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash2").
3051
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash2").
2690
3052
  with(basic_auth: ['john', 'doe']).
2691
3053
  to_return(:status => 404, :body => "", :headers => {})
2692
3054
  #creation
2693
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
3055
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
2694
3056
  with(basic_auth: ['john', 'doe']).
2695
3057
  to_return(:status => 200, :body => "", :headers => {})
2696
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash1").
3058
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1").
2697
3059
  with(basic_auth: ['john', 'doe']).
2698
3060
  to_return(:status => 200, :body => "", :headers => {})
2699
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash2").
3061
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2").
2700
3062
  with(basic_auth: ['john', 'doe']).
2701
3063
  to_return(:status => 200, :body => "", :headers => {})
2702
3064
 
2703
3065
  driver(config)
2704
3066
 
2705
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
3067
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
2706
3068
 
2707
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash1")
2708
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash2")
3069
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1")
3070
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2")
2709
3071
  end
2710
3072
 
2711
- def test_templates_can_be_partially_created_if_error_occurs
3073
+ data("legacy_template" => [true, "_template"],
3074
+ "new_template" => [false, "_index_template"])
3075
+ def test_templates_can_be_partially_created_if_error_occurs(data)
3076
+ use_legacy_template_flag, endpoint = data
2712
3077
  cwd = File.dirname(__FILE__)
2713
- template_file = File.join(cwd, 'test_template.json')
3078
+ template_file = if use_legacy_template_flag
3079
+ File.join(cwd, 'test_template.json')
3080
+ else
3081
+ File.join(cwd, 'test_index_template.json')
3082
+ end
3083
+
2714
3084
  config = %{
2715
3085
  host logs.google.com
2716
3086
  port 777
@@ -2719,22 +3089,23 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2719
3089
  user john
2720
3090
  password doe
2721
3091
  templates {"logstash1":"#{template_file}", "logstash2":"/abc" }
3092
+ use_legacy_template #{use_legacy_template_flag}
2722
3093
  }
2723
3094
  stub_request(:head, "https://logs.google.com:777/es//").
2724
3095
  with(basic_auth: ['john', 'doe']).
2725
3096
  to_return(:status => 200, :body => "", :headers => {})
2726
3097
  # check if template exists
2727
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash1").
3098
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash1").
2728
3099
  with(basic_auth: ['john', 'doe']).
2729
3100
  to_return(:status => 404, :body => "", :headers => {})
2730
- stub_request(:get, "https://logs.google.com:777/es//_template/logstash2").
3101
+ stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash2").
2731
3102
  with(basic_auth: ['john', 'doe']).
2732
3103
  to_return(:status => 404, :body => "", :headers => {})
2733
3104
 
2734
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash1").
3105
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1").
2735
3106
  with(basic_auth: ['john', 'doe']).
2736
3107
  to_return(:status => 200, :body => "", :headers => {})
2737
- stub_request(:put, "https://logs.google.com:777/es//_template/logstash2").
3108
+ stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2").
2738
3109
  with(basic_auth: ['john', 'doe']).
2739
3110
  to_return(:status => 200, :body => "", :headers => {})
2740
3111
 
@@ -2742,8 +3113,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
2742
3113
  driver(config)
2743
3114
  }
2744
3115
 
2745
- assert_requested(:put, "https://logs.google.com:777/es//_template/logstash1", times: 1)
2746
- assert_not_requested(:put, "https://logs.google.com:777/es//_template/logstash2")
3116
+ assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1", times: 1)
3117
+ assert_not_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2")
2747
3118
  end
2748
3119
 
2749
3120
  def test_legacy_hosts_list