fluent-plugin-elasticsearch 4.1.4 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/linux.yml +1 -1
- data/.github/workflows/macos.yml +1 -1
- data/.github/workflows/windows.yml +1 -1
- data/History.md +4 -0
- data/README.md +13 -0
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_index_template.rb +31 -7
- data/lib/fluent/plugin/out_elasticsearch.rb +1 -0
- data/test/plugin/test_index_alias_template.json +11 -0
- data/test/plugin/test_index_template.json +25 -0
- data/test/plugin/test_out_elasticsearch.rb +617 -246
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 46901e5f885a46c37ea29dca2968d53fd4026dec637f007936cabdd3ee070c5c
|
4
|
+
data.tar.gz: 21f3d3e9dd22752bcab060ea46ba12e8309147c2ddbb3be4aea656b527aff103
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ed30b7f203f9c102fdedb0a28c88210f172e9bccab3f153de81ba88b821a065e1cfdb8caccc097fbb83745069c7c5cd052e70879501ebd83b472f55c835b6116
|
7
|
+
data.tar.gz: 3ded3c8f6041224546a71df34878148a14b0f2e47970b86efb11d8b4fe4008764680534149b1c56c1651bc7dbfffc0df26ad7a459c87e739c91f391d8da83f42
|
data/.github/workflows/linux.yml
CHANGED
data/.github/workflows/macos.yml
CHANGED
data/History.md
CHANGED
data/README.md
CHANGED
@@ -101,6 +101,7 @@ Current maintainers: @cosmo0920
|
|
101
101
|
+ [ilm_policies](#ilm_policies)
|
102
102
|
+ [ilm_policy_overwrite](#ilm_policy_overwrite)
|
103
103
|
+ [truncate_caches_interval](#truncate_caches_interval)
|
104
|
+
+ [use_legacy_template](#use_legacy_template)
|
104
105
|
* [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
|
105
106
|
* [Configuration - Elasticsearch Filter GenID](#configuration---elasticsearch-filter-genid)
|
106
107
|
* [Elasticsearch permissions](#elasticsearch-permissions)
|
@@ -1303,6 +1304,18 @@ If it is set, timer for clearing `alias_indexes` and `template_names` caches wil
|
|
1303
1304
|
|
1304
1305
|
Default value is `nil`.
|
1305
1306
|
|
1307
|
+
## use_legacy_template
|
1308
|
+
|
1309
|
+
Use legacy template or not.
|
1310
|
+
|
1311
|
+
Elasticsearch 7.8 or later supports the brand new composable templates.
|
1312
|
+
|
1313
|
+
For Elasticsearch 7.7 or older, users should specify this parameter as `false`.
|
1314
|
+
|
1315
|
+
Composable template documentation is [Put Index Template API | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-templates.html) and legacy template documentation is [Index Templates | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates-v1.html).
|
1316
|
+
|
1317
|
+
Please confirm that whether the using Elasticsearch cluster(s) support the composable template feature or not when turn on the brand new feature with this parameter.
|
1318
|
+
|
1306
1319
|
## Configuration - Elasticsearch Input
|
1307
1320
|
|
1308
1321
|
See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '4.
|
6
|
+
s.version = '4.2.0'
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -22,7 +22,11 @@ module Fluent::ElasticsearchIndexTemplate
|
|
22
22
|
end
|
23
23
|
|
24
24
|
def template_exists?(name, host = nil)
|
25
|
-
|
25
|
+
if @use_legacy_template
|
26
|
+
client(host).indices.get_template(:name => name)
|
27
|
+
else
|
28
|
+
client(host).indices.get_index_template(:name => name)
|
29
|
+
end
|
26
30
|
return true
|
27
31
|
rescue Elasticsearch::Transport::Transport::Errors::NotFound
|
28
32
|
return false
|
@@ -52,7 +56,11 @@ module Fluent::ElasticsearchIndexTemplate
|
|
52
56
|
end
|
53
57
|
|
54
58
|
def template_put(name, template, host = nil)
|
55
|
-
|
59
|
+
if @use_legacy_template
|
60
|
+
client(host).indices.put_template(:name => name, :body => template)
|
61
|
+
else
|
62
|
+
client(host).indices.put_index_template(:name => name, :body => template)
|
63
|
+
end
|
56
64
|
end
|
57
65
|
|
58
66
|
def indexcreation(index_name, host = nil)
|
@@ -121,13 +129,29 @@ module Fluent::ElasticsearchIndexTemplate
|
|
121
129
|
|
122
130
|
def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template)
|
123
131
|
log.debug("Overwriting index patterns when Index Lifecycle Management is enabled.")
|
124
|
-
template.delete('template') if template.include?('template')
|
125
132
|
template['index_patterns'] = "#{target_index}-*"
|
126
|
-
|
127
|
-
|
128
|
-
|
133
|
+
if @use_legacy_template
|
134
|
+
template.delete('template') if template.include?('template')
|
135
|
+
# Prepare settings Hash
|
136
|
+
if !template.key?('settings')
|
137
|
+
template['settings'] = {}
|
138
|
+
end
|
139
|
+
if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
|
140
|
+
log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
|
141
|
+
end
|
142
|
+
template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
|
143
|
+
template['order'] = template['order'] ? template['order'] + target_index.split('-').length : 50 + target_index.split('-').length
|
144
|
+
else
|
145
|
+
# Prepare template.settings Hash
|
146
|
+
if !template['template'].key?('settings')
|
147
|
+
template['template']['settings'] = {}
|
148
|
+
end
|
149
|
+
if template['template']['settings'] && (template['template']['settings']['index.lifecycle.name'] || template['template']['settings']['index.lifecycle.rollover_alias'])
|
150
|
+
log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
|
151
|
+
end
|
152
|
+
template['template']['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
|
153
|
+
template['priority'] = template['priority'] ? template['priority'] + target_index.split('-').length : 100 + target_index.split('-').length
|
129
154
|
end
|
130
|
-
template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
|
131
155
|
template
|
132
156
|
end
|
133
157
|
|
@@ -169,6 +169,7 @@ EOC
|
|
169
169
|
config_param :ilm_policies, :hash, :default => {}
|
170
170
|
config_param :ilm_policy_overwrite, :bool, :default => false
|
171
171
|
config_param :truncate_caches_interval, :time, :default => nil
|
172
|
+
config_param :use_legacy_template, :bool, :default => true
|
172
173
|
|
173
174
|
config_section :buffer do
|
174
175
|
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
@@ -0,0 +1,25 @@
|
|
1
|
+
{
|
2
|
+
"index_patterns": "te*",
|
3
|
+
"template": {
|
4
|
+
"settings": {
|
5
|
+
"number_of_shards": 1
|
6
|
+
},
|
7
|
+
"mappings": {
|
8
|
+
"type1": {
|
9
|
+
"_source": {
|
10
|
+
"enabled": false
|
11
|
+
},
|
12
|
+
"properties": {
|
13
|
+
"host_name": {
|
14
|
+
"type": "string",
|
15
|
+
"index": "not_analyzed"
|
16
|
+
},
|
17
|
+
"created_at": {
|
18
|
+
"type": "date",
|
19
|
+
"format": "EEE MMM dd HH:mm:ss Z YYYY"
|
20
|
+
}
|
21
|
+
}
|
22
|
+
}
|
23
|
+
}
|
24
|
+
}
|
25
|
+
}
|
@@ -375,7 +375,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
375
375
|
end
|
376
376
|
end
|
377
377
|
|
378
|
-
|
378
|
+
data("legacy_template" => [true, "_template"],
|
379
|
+
"new_template" => [false, "_index_template"])
|
380
|
+
test 'valid configuration of index lifecycle management' do |data|
|
381
|
+
use_legacy_template_flag, endpoint = data
|
379
382
|
cwd = File.dirname(__FILE__)
|
380
383
|
template_file = File.join(cwd, 'test_template.json')
|
381
384
|
|
@@ -383,8 +386,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
383
386
|
enable_ilm true
|
384
387
|
template_name logstash
|
385
388
|
template_file #{template_file}
|
389
|
+
use_legacy_template #{use_legacy_template_flag}
|
386
390
|
}
|
387
|
-
stub_request(:get, "http://localhost:9200/
|
391
|
+
stub_request(:get, "http://localhost:9200/#{endpoint}/fluentd").
|
388
392
|
to_return(status: 200, body: "", headers: {})
|
389
393
|
stub_request(:head, "http://localhost:9200/_alias/fluentd").
|
390
394
|
to_return(status: 404, body: "", headers: {})
|
@@ -407,7 +411,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
407
411
|
}
|
408
412
|
end
|
409
413
|
|
410
|
-
|
414
|
+
data("legacy_template" => [true, "_template"],
|
415
|
+
"new_template" => [false, "_index_template"])
|
416
|
+
test 'valid configuration of overwriting ilm_policy' do |data|
|
417
|
+
use_legacy_template_flag, endpoint = data
|
411
418
|
cwd = File.dirname(__FILE__)
|
412
419
|
template_file = File.join(cwd, 'test_template.json')
|
413
420
|
|
@@ -417,8 +424,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
417
424
|
template_file #{template_file}
|
418
425
|
ilm_policy_overwrite true
|
419
426
|
ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"75gb","max_age": "50d"}}}}}}
|
427
|
+
use_legacy_template #{use_legacy_template_flag}
|
420
428
|
}
|
421
|
-
stub_request(:get, "http://localhost:9200/
|
429
|
+
stub_request(:get, "http://localhost:9200/#{endpoint}/fluentd").
|
422
430
|
to_return(status: 200, body: "", headers: {})
|
423
431
|
stub_request(:head, "http://localhost:9200/_alias/fluentd").
|
424
432
|
to_return(status: 404, body: "", headers: {})
|
@@ -806,7 +814,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
806
814
|
end
|
807
815
|
end
|
808
816
|
|
809
|
-
|
817
|
+
data("legacy_template" => [true, "_template"],
|
818
|
+
"new_template" => [false, "_index_template"])
|
819
|
+
def test_template_already_present(data)
|
820
|
+
use_legacy_template_flag, endpoint = data
|
810
821
|
config = %{
|
811
822
|
host logs.google.com
|
812
823
|
port 777
|
@@ -816,6 +827,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
816
827
|
password doe
|
817
828
|
template_name logstash
|
818
829
|
template_file /abc123
|
830
|
+
use_legacy_template #{use_legacy_template_flag}
|
819
831
|
}
|
820
832
|
|
821
833
|
# connection start
|
@@ -823,18 +835,25 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
823
835
|
with(basic_auth: ['john', 'doe']).
|
824
836
|
to_return(:status => 200, :body => "", :headers => {})
|
825
837
|
# check if template exists
|
826
|
-
stub_request(:get, "https://logs.google.com:777/es
|
838
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
827
839
|
with(basic_auth: ['john', 'doe']).
|
828
840
|
to_return(:status => 200, :body => "", :headers => {})
|
829
841
|
|
830
842
|
driver(config)
|
831
843
|
|
832
|
-
assert_not_requested(:put, "https://logs.google.com:777/es
|
844
|
+
assert_not_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash")
|
833
845
|
end
|
834
846
|
|
835
|
-
|
847
|
+
data("legacy_template" => [true, "_template"],
|
848
|
+
"new_template" => [false, "_index_template"])
|
849
|
+
def test_template_create(data)
|
850
|
+
use_legacy_template_flag, endpoint = data
|
836
851
|
cwd = File.dirname(__FILE__)
|
837
|
-
template_file =
|
852
|
+
template_file = if use_legacy_template_flag
|
853
|
+
File.join(cwd, 'test_template.json')
|
854
|
+
else
|
855
|
+
File.join(cwd, 'test_index_template.json')
|
856
|
+
end
|
838
857
|
|
839
858
|
config = %{
|
840
859
|
host logs.google.com
|
@@ -845,6 +864,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
845
864
|
password doe
|
846
865
|
template_name logstash
|
847
866
|
template_file #{template_file}
|
867
|
+
use_legacy_template #{use_legacy_template_flag}
|
848
868
|
}
|
849
869
|
|
850
870
|
# connection start
|
@@ -852,22 +872,29 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
852
872
|
with(basic_auth: ['john', 'doe']).
|
853
873
|
to_return(:status => 200, :body => "", :headers => {})
|
854
874
|
# check if template exists
|
855
|
-
stub_request(:get, "https://logs.google.com:777/es
|
875
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
856
876
|
with(basic_auth: ['john', 'doe']).
|
857
877
|
to_return(:status => 404, :body => "", :headers => {})
|
858
878
|
# creation
|
859
|
-
stub_request(:put, "https://logs.google.com:777/es
|
879
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
860
880
|
with(basic_auth: ['john', 'doe']).
|
861
881
|
to_return(:status => 200, :body => "", :headers => {})
|
862
882
|
|
863
883
|
driver(config)
|
864
884
|
|
865
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
885
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
866
886
|
end
|
867
887
|
|
868
|
-
|
888
|
+
data("legacy_template" => [true, "_template"],
|
889
|
+
"new_template" => [false, "_index_template"])
|
890
|
+
def test_template_create_with_rollover_index_and_template_related_placeholders(data)
|
891
|
+
use_legacy_template_flag, endpoint = data
|
869
892
|
cwd = File.dirname(__FILE__)
|
870
|
-
template_file =
|
893
|
+
template_file = if use_legacy_template_flag
|
894
|
+
File.join(cwd, 'test_template.json')
|
895
|
+
else
|
896
|
+
File.join(cwd, 'test_index_template.json')
|
897
|
+
end
|
871
898
|
config = %{
|
872
899
|
host logs.google.com
|
873
900
|
port 777
|
@@ -881,6 +908,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
881
908
|
index_date_pattern ""
|
882
909
|
index_name fluentd-${tag}
|
883
910
|
deflector_alias myapp_deflector-${tag}
|
911
|
+
use_legacy_template #{use_legacy_template_flag}
|
884
912
|
}
|
885
913
|
|
886
914
|
# connection start
|
@@ -888,11 +916,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
888
916
|
with(basic_auth: ['john', 'doe']).
|
889
917
|
to_return(:status => 200, :body => "", :headers => {})
|
890
918
|
# check if template exists
|
891
|
-
stub_request(:get, "https://logs.google.com:777/es
|
919
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-test.template").
|
892
920
|
with(basic_auth: ['john', 'doe']).
|
893
921
|
to_return(:status => 404, :body => "", :headers => {})
|
894
922
|
# create template
|
895
|
-
stub_request(:put, "https://logs.google.com:777/es
|
923
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-test.template").
|
896
924
|
with(basic_auth: ['john', 'doe']).
|
897
925
|
to_return(:status => 200, :body => "", :headers => {})
|
898
926
|
# check if alias exists
|
@@ -922,9 +950,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
922
950
|
assert_requested(elastic_request)
|
923
951
|
end
|
924
952
|
|
925
|
-
|
953
|
+
data("legacy_template" => [true, "_template"],
|
954
|
+
"new_template" => [false, "_index_template"])
|
955
|
+
def test_template_create_with_rollover_index_and_template_related_placeholders_with_truncating_caches(data)
|
956
|
+
use_legacy_template_flag, endpoint = data
|
926
957
|
cwd = File.dirname(__FILE__)
|
927
|
-
template_file =
|
958
|
+
template_file = if use_legacy_template_flag
|
959
|
+
File.join(cwd, 'test_template.json')
|
960
|
+
else
|
961
|
+
File.join(cwd, 'test_index_template.json')
|
962
|
+
end
|
928
963
|
config = %{
|
929
964
|
host logs.google.com
|
930
965
|
port 777
|
@@ -939,6 +974,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
939
974
|
index_name fluentd-${tag}
|
940
975
|
deflector_alias myapp_deflector-${tag}
|
941
976
|
truncate_caches_interval 2s
|
977
|
+
use_legacy_template #{use_legacy_template_flag}
|
942
978
|
}
|
943
979
|
|
944
980
|
# connection start
|
@@ -946,11 +982,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
946
982
|
with(basic_auth: ['john', 'doe']).
|
947
983
|
to_return(:status => 200, :body => "", :headers => {})
|
948
984
|
# check if template exists
|
949
|
-
stub_request(:get, "https://logs.google.com:777/es
|
985
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-test.template").
|
950
986
|
with(basic_auth: ['john', 'doe']).
|
951
987
|
to_return(:status => 404, :body => "", :headers => {})
|
952
988
|
# create template
|
953
|
-
stub_request(:put, "https://logs.google.com:777/es
|
989
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-test.template").
|
954
990
|
with(basic_auth: ['john', 'doe']).
|
955
991
|
to_return(:status => 200, :body => "", :headers => {})
|
956
992
|
# check if alias exists
|
@@ -995,9 +1031,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
995
1031
|
end
|
996
1032
|
end
|
997
1033
|
|
998
|
-
|
1034
|
+
data("legacy_template" => [true, "_template"],
|
1035
|
+
"new_template" => [false, "_index_template"])
|
1036
|
+
def test_template_create_with_rollover_index_and_default_ilm(data)
|
1037
|
+
use_legacy_template_flag, endpoint = data
|
999
1038
|
cwd = File.dirname(__FILE__)
|
1000
|
-
template_file =
|
1039
|
+
template_file = if use_legacy_template_flag
|
1040
|
+
File.join(cwd, 'test_template.json')
|
1041
|
+
else
|
1042
|
+
File.join(cwd, 'test_index_template.json')
|
1043
|
+
end
|
1001
1044
|
|
1002
1045
|
config = %{
|
1003
1046
|
host logs.google.com
|
@@ -1011,6 +1054,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1011
1054
|
index_date_pattern now/w{xxxx.ww}
|
1012
1055
|
index_name logstash
|
1013
1056
|
enable_ilm true
|
1057
|
+
use_legacy_template #{use_legacy_template_flag}
|
1014
1058
|
}
|
1015
1059
|
|
1016
1060
|
# connection start
|
@@ -1018,24 +1062,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1018
1062
|
with(basic_auth: ['john', 'doe']).
|
1019
1063
|
to_return(:status => 200, :body => "", :headers => {})
|
1020
1064
|
# check if template exists
|
1021
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1065
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1022
1066
|
with(basic_auth: ['john', 'doe']).
|
1023
1067
|
to_return(:status => 404, :body => "", :headers => {})
|
1024
1068
|
# creation
|
1025
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1069
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1026
1070
|
with(basic_auth: ['john', 'doe']).
|
1027
1071
|
to_return(:status => 200, :body => "", :headers => {})
|
1028
1072
|
# check if alias exists
|
1029
1073
|
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1030
1074
|
with(basic_auth: ['john', 'doe']).
|
1031
1075
|
to_return(:status => 404, :body => "", :headers => {})
|
1032
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1076
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1033
1077
|
with(basic_auth: ['john', 'doe']).
|
1034
1078
|
to_return(status: 404, body: "", headers: {})
|
1035
|
-
|
1036
|
-
|
1037
|
-
|
1038
|
-
|
1079
|
+
if use_legacy_template_flag
|
1080
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1081
|
+
with(basic_auth: ['john', 'doe'],
|
1082
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
|
1083
|
+
to_return(status: 200, body: "", headers: {})
|
1084
|
+
else
|
1085
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1086
|
+
with(basic_auth: ['john', 'doe'],
|
1087
|
+
body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
|
1088
|
+
to_return(status: 200, body: "", headers: {})
|
1089
|
+
end
|
1039
1090
|
# put the alias for the index
|
1040
1091
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1041
1092
|
with(basic_auth: ['john', 'doe']).
|
@@ -1057,12 +1108,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1057
1108
|
|
1058
1109
|
driver(config)
|
1059
1110
|
|
1060
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1111
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
1061
1112
|
end
|
1062
1113
|
|
1063
|
-
|
1114
|
+
data("legacy_template" => [true, "_template"],
|
1115
|
+
"new_template" => [false, "_index_template"])
|
1116
|
+
def test_template_create_with_rollover_index_and_default_ilm_on_logstash_format(data)
|
1117
|
+
use_legacy_template_flag, endpoint = data
|
1064
1118
|
cwd = File.dirname(__FILE__)
|
1065
|
-
template_file =
|
1119
|
+
template_file = if use_legacy_template_flag
|
1120
|
+
File.join(cwd, 'test_template.json')
|
1121
|
+
else
|
1122
|
+
File.join(cwd, 'test_index_template.json')
|
1123
|
+
end
|
1066
1124
|
|
1067
1125
|
config = %{
|
1068
1126
|
host logs.google.com
|
@@ -1077,6 +1135,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1077
1135
|
enable_ilm true
|
1078
1136
|
logstash_format true
|
1079
1137
|
application_name log
|
1138
|
+
use_legacy_template #{use_legacy_template_flag}
|
1080
1139
|
}
|
1081
1140
|
|
1082
1141
|
date_str = Time.now.strftime("%Y.%m.%d")
|
@@ -1085,24 +1144,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1085
1144
|
with(basic_auth: ['john', 'doe']).
|
1086
1145
|
to_return(:status => 200, :body => "", :headers => {})
|
1087
1146
|
# check if template exists
|
1088
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1147
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}").
|
1089
1148
|
with(basic_auth: ['john', 'doe']).
|
1090
1149
|
to_return(:status => 404, :body => "", :headers => {})
|
1091
1150
|
# creation
|
1092
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1151
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}").
|
1093
1152
|
with(basic_auth: ['john', 'doe']).
|
1094
1153
|
to_return(:status => 200, :body => "", :headers => {})
|
1095
1154
|
# check if alias exists
|
1096
1155
|
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-#{date_str}").
|
1097
1156
|
with(basic_auth: ['john', 'doe']).
|
1098
1157
|
to_return(:status => 404, :body => "", :headers => {})
|
1099
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1158
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}").
|
1100
1159
|
with(basic_auth: ['john', 'doe']).
|
1101
1160
|
to_return(status: 404, body: "", headers: {})
|
1102
|
-
|
1103
|
-
|
1104
|
-
|
1105
|
-
|
1161
|
+
if use_legacy_template_flag
|
1162
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}").
|
1163
|
+
with(basic_auth: ['john', 'doe'],
|
1164
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-log-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-log-#{date_str}-*\",\"order\":53}").
|
1165
|
+
to_return(status: 200, body: "", headers: {})
|
1166
|
+
else
|
1167
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}").
|
1168
|
+
with(basic_auth: ['john', 'doe'],
|
1169
|
+
body: "{\"index_patterns\":\"logstash-log-2020.09.21-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-2020.09.21\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":103}").
|
1170
|
+
to_return(status: 200, body: "", headers: {})
|
1171
|
+
end
|
1106
1172
|
# put the alias for the index
|
1107
1173
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-#{date_str}-000001%3E").
|
1108
1174
|
with(basic_auth: ['john', 'doe']).
|
@@ -1129,14 +1195,21 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1129
1195
|
driver.run(default_tag: 'test') do
|
1130
1196
|
driver.feed(sample_record)
|
1131
1197
|
end
|
1132
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1198
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}", times: 1)
|
1133
1199
|
|
1134
1200
|
assert_requested(elastic_request)
|
1135
1201
|
end
|
1136
1202
|
|
1137
|
-
|
1203
|
+
data("legacy_template" => [true, "_template"],
|
1204
|
+
"new_template" => [false, "_index_template"])
|
1205
|
+
def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite(data)
|
1206
|
+
use_legacy_template_flag, endpoint = data
|
1138
1207
|
cwd = File.dirname(__FILE__)
|
1139
|
-
template_file =
|
1208
|
+
template_file = if use_legacy_template_flag
|
1209
|
+
File.join(cwd, 'test_template.json')
|
1210
|
+
else
|
1211
|
+
File.join(cwd, 'test_index_template.json')
|
1212
|
+
end
|
1140
1213
|
|
1141
1214
|
config = %{
|
1142
1215
|
host logs.google.com
|
@@ -1152,6 +1225,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1152
1225
|
enable_ilm true
|
1153
1226
|
ilm_policy_overwrite true
|
1154
1227
|
ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"60gb","max_age": "45d"}}}}}}
|
1228
|
+
use_legacy_template #{use_legacy_template_flag}
|
1155
1229
|
}
|
1156
1230
|
|
1157
1231
|
# connection start
|
@@ -1159,24 +1233,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1159
1233
|
with(basic_auth: ['john', 'doe']).
|
1160
1234
|
to_return(:status => 200, :body => "", :headers => {})
|
1161
1235
|
# check if template exists
|
1162
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1236
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1163
1237
|
with(basic_auth: ['john', 'doe']).
|
1164
1238
|
to_return(:status => 404, :body => "", :headers => {})
|
1165
1239
|
# creation
|
1166
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1240
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1167
1241
|
with(basic_auth: ['john', 'doe']).
|
1168
1242
|
to_return(:status => 200, :body => "", :headers => {})
|
1169
1243
|
# check if alias exists
|
1170
1244
|
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1171
1245
|
with(basic_auth: ['john', 'doe']).
|
1172
1246
|
to_return(:status => 404, :body => "", :headers => {})
|
1173
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1247
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1174
1248
|
with(basic_auth: ['john', 'doe']).
|
1175
1249
|
to_return(status: 404, body: "", headers: {})
|
1176
|
-
|
1177
|
-
|
1178
|
-
|
1179
|
-
|
1250
|
+
if use_legacy_template_flag
|
1251
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1252
|
+
with(basic_auth: ['john', 'doe'],
|
1253
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
|
1254
|
+
to_return(status: 200, body: "", headers: {})
|
1255
|
+
else
|
1256
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1257
|
+
with(basic_auth: ['john', 'doe'],
|
1258
|
+
body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
|
1259
|
+
to_return(status: 200, body: "", headers: {})
|
1260
|
+
end
|
1180
1261
|
# put the alias for the index
|
1181
1262
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1182
1263
|
with(basic_auth: ['john', 'doe']).
|
@@ -1198,7 +1279,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1198
1279
|
|
1199
1280
|
driver(config)
|
1200
1281
|
|
1201
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1282
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
1202
1283
|
end
|
1203
1284
|
|
1204
1285
|
def test_template_create_with_rollover_index_and_default_ilm_with_deflector_alias
|
@@ -1226,9 +1307,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1226
1307
|
end
|
1227
1308
|
end
|
1228
1309
|
|
1229
|
-
|
1310
|
+
data("legacy_template" => [true, "_template"],
|
1311
|
+
"new_template" => [false, "_index_template"])
|
1312
|
+
def test_template_create_with_rollover_index_and_default_ilm_with_empty_index_date_pattern(data)
|
1313
|
+
use_legacy_template_flag, endpoint = data
|
1230
1314
|
cwd = File.dirname(__FILE__)
|
1231
|
-
template_file =
|
1315
|
+
template_file = if use_legacy_template_flag
|
1316
|
+
File.join(cwd, 'test_template.json')
|
1317
|
+
else
|
1318
|
+
File.join(cwd, 'test_index_template.json')
|
1319
|
+
end
|
1232
1320
|
|
1233
1321
|
config = %{
|
1234
1322
|
host logs.google.com
|
@@ -1242,6 +1330,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1242
1330
|
index_date_pattern ""
|
1243
1331
|
index_name logstash
|
1244
1332
|
enable_ilm true
|
1333
|
+
use_legacy_template #{use_legacy_template_flag}
|
1245
1334
|
}
|
1246
1335
|
|
1247
1336
|
# connection start
|
@@ -1249,24 +1338,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1249
1338
|
with(basic_auth: ['john', 'doe']).
|
1250
1339
|
to_return(:status => 200, :body => "", :headers => {})
|
1251
1340
|
# check if template exists
|
1252
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1341
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1253
1342
|
with(basic_auth: ['john', 'doe']).
|
1254
1343
|
to_return(:status => 404, :body => "", :headers => {})
|
1255
1344
|
# creation
|
1256
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1345
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1257
1346
|
with(basic_auth: ['john', 'doe']).
|
1258
1347
|
to_return(:status => 200, :body => "", :headers => {})
|
1259
1348
|
# check if alias exists
|
1260
1349
|
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1261
1350
|
with(basic_auth: ['john', 'doe']).
|
1262
1351
|
to_return(:status => 404, :body => "", :headers => {})
|
1263
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1352
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1264
1353
|
with(basic_auth: ['john', 'doe']).
|
1265
1354
|
to_return(status: 404, body: "", headers: {})
|
1266
|
-
|
1267
|
-
|
1268
|
-
|
1269
|
-
|
1355
|
+
if use_legacy_template_flag
|
1356
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_deflector").
|
1357
|
+
with(basic_auth: ['john', 'doe'],
|
1358
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
|
1359
|
+
to_return(status: 200, body: "", headers: {})
|
1360
|
+
else
|
1361
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_deflector").
|
1362
|
+
with(basic_auth: ['john', 'doe'],
|
1363
|
+
body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
|
1364
|
+
to_return(status: 200, body: "", headers: {})
|
1365
|
+
end
|
1270
1366
|
# put the alias for the index
|
1271
1367
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E").
|
1272
1368
|
with(basic_auth: ['john', 'doe']).
|
@@ -1288,12 +1384,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1288
1384
|
|
1289
1385
|
driver(config)
|
1290
1386
|
|
1291
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1387
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
1292
1388
|
end
|
1293
1389
|
|
1294
|
-
|
1390
|
+
data("legacy_template" => [true, "_template"],
|
1391
|
+
"new_template" => [false, "_index_template"])
|
1392
|
+
def test_template_create_with_rollover_index_and_custom_ilm(data)
|
1393
|
+
use_legacy_template_flag, endpoint = data
|
1295
1394
|
cwd = File.dirname(__FILE__)
|
1296
|
-
template_file =
|
1395
|
+
template_file = if use_legacy_template_flag
|
1396
|
+
File.join(cwd, 'test_template.json')
|
1397
|
+
else
|
1398
|
+
File.join(cwd, 'test_index_template.json')
|
1399
|
+
end
|
1297
1400
|
|
1298
1401
|
config = %{
|
1299
1402
|
host logs.google.com
|
@@ -1309,6 +1412,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1309
1412
|
enable_ilm true
|
1310
1413
|
index_name logstash
|
1311
1414
|
ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}
|
1415
|
+
use_legacy_template #{use_legacy_template_flag}
|
1312
1416
|
}
|
1313
1417
|
|
1314
1418
|
# connection start
|
@@ -1316,24 +1420,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1316
1420
|
with(basic_auth: ['john', 'doe']).
|
1317
1421
|
to_return(:status => 200, :body => "", :headers => {})
|
1318
1422
|
# check if template exists
|
1319
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1423
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1320
1424
|
with(basic_auth: ['john', 'doe']).
|
1321
1425
|
to_return(:status => 404, :body => "", :headers => {})
|
1322
1426
|
# creation
|
1323
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1427
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1324
1428
|
with(basic_auth: ['john', 'doe']).
|
1325
1429
|
to_return(:status => 200, :body => "", :headers => {})
|
1326
1430
|
# check if alias exists
|
1327
1431
|
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1328
1432
|
with(basic_auth: ['john', 'doe']).
|
1329
1433
|
to_return(:status => 404, :body => "", :headers => {})
|
1330
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1434
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1331
1435
|
with(basic_auth: ['john', 'doe']).
|
1332
1436
|
to_return(status: 404, body: "", headers: {})
|
1333
|
-
|
1334
|
-
|
1335
|
-
|
1336
|
-
|
1437
|
+
if use_legacy_template_flag
|
1438
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1439
|
+
with(basic_auth: ['john', 'doe'],
|
1440
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1441
|
+
to_return(status: 200, body: "", headers: {})
|
1442
|
+
else
|
1443
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1444
|
+
with(basic_auth: ['john', 'doe'],
|
1445
|
+
body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
|
1446
|
+
to_return(status: 200, body: "", headers: {})
|
1447
|
+
end
|
1337
1448
|
# put the alias for the index
|
1338
1449
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1339
1450
|
with(basic_auth: ['john', 'doe']).
|
@@ -1354,12 +1465,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1354
1465
|
|
1355
1466
|
driver(config)
|
1356
1467
|
|
1357
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1468
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
1358
1469
|
end
|
1359
1470
|
|
1360
|
-
|
1471
|
+
data("legacy_template" => [true, "_template"],
|
1472
|
+
"new_template" => [false, "_index_template"])
|
1473
|
+
def test_template_create_with_rollover_index_and_ilm_policies_and_placeholderstest_template_create_with_rollover_index_and_ilm_policies_and_placeholders(data)
|
1474
|
+
use_legacy_template_flag, endpoint = data
|
1361
1475
|
cwd = File.dirname(__FILE__)
|
1362
|
-
template_file =
|
1476
|
+
template_file = if use_legacy_template_flag
|
1477
|
+
File.join(cwd, 'test_template.json')
|
1478
|
+
else
|
1479
|
+
File.join(cwd, 'test_index_template.json')
|
1480
|
+
end
|
1363
1481
|
|
1364
1482
|
config = %{
|
1365
1483
|
host logs.google.com
|
@@ -1375,6 +1493,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1375
1493
|
enable_ilm true
|
1376
1494
|
index_name logstash
|
1377
1495
|
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
1496
|
+
use_legacy_template #{use_legacy_template_flag}
|
1378
1497
|
}
|
1379
1498
|
|
1380
1499
|
# connection start
|
@@ -1382,24 +1501,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1382
1501
|
with(basic_auth: ['john', 'doe']).
|
1383
1502
|
to_return(:status => 200, :body => "", :headers => {})
|
1384
1503
|
# check if template exists
|
1385
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1504
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1386
1505
|
with(basic_auth: ['john', 'doe']).
|
1387
1506
|
to_return(:status => 404, :body => "", :headers => {})
|
1388
1507
|
# creation
|
1389
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1508
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1390
1509
|
with(basic_auth: ['john', 'doe']).
|
1391
1510
|
to_return(:status => 200, :body => "", :headers => {})
|
1392
1511
|
# check if alias exists
|
1393
1512
|
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1394
1513
|
with(basic_auth: ['john', 'doe']).
|
1395
1514
|
to_return(:status => 404, :body => "", :headers => {})
|
1396
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1515
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1397
1516
|
with(basic_auth: ['john', 'doe']).
|
1398
1517
|
to_return(status: 404, body: "", headers: {})
|
1399
|
-
|
1400
|
-
|
1401
|
-
|
1402
|
-
|
1518
|
+
if use_legacy_template_flag
|
1519
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1520
|
+
with(basic_auth: ['john', 'doe'],
|
1521
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1522
|
+
to_return(status: 200, body: "", headers: {})
|
1523
|
+
else
|
1524
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1525
|
+
with(basic_auth: ['john', 'doe'],
|
1526
|
+
body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
|
1527
|
+
to_return(status: 200, body: "", headers: {})
|
1528
|
+
end
|
1403
1529
|
# put the alias for the index
|
1404
1530
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1405
1531
|
with(basic_auth: ['john', 'doe']).
|
@@ -1424,15 +1550,22 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1424
1550
|
driver.run(default_tag: 'test') do
|
1425
1551
|
driver.feed(sample_record)
|
1426
1552
|
end
|
1427
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1553
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
1428
1554
|
|
1429
1555
|
assert_requested(elastic_request)
|
1430
1556
|
end
|
1431
1557
|
|
1432
1558
|
class TemplateCreateWithRolloverIndexAndILMPoliciesWithPlaceholdersTest < self
|
1433
|
-
|
1559
|
+
data("legacy_template" => [true, "_template"],
|
1560
|
+
"new_template" => [false, "_index_template"])
|
1561
|
+
def test_tag_placeholder(data)
|
1562
|
+
use_legacy_template_flag, endpoint = data
|
1434
1563
|
cwd = File.dirname(__FILE__)
|
1435
|
-
template_file =
|
1564
|
+
template_file = if use_legacy_template_flag
|
1565
|
+
File.join(cwd, 'test_template.json')
|
1566
|
+
else
|
1567
|
+
File.join(cwd, 'test_index_template.json')
|
1568
|
+
end
|
1436
1569
|
|
1437
1570
|
config = %{
|
1438
1571
|
host logs.google.com
|
@@ -1448,6 +1581,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1448
1581
|
enable_ilm true
|
1449
1582
|
index_name logstash
|
1450
1583
|
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
1584
|
+
use_legacy_template #{use_legacy_template_flag}
|
1451
1585
|
}
|
1452
1586
|
|
1453
1587
|
# connection start
|
@@ -1455,24 +1589,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1455
1589
|
with(basic_auth: ['john', 'doe']).
|
1456
1590
|
to_return(:status => 200, :body => "", :headers => {})
|
1457
1591
|
# check if template exists
|
1458
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1592
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1459
1593
|
with(basic_auth: ['john', 'doe']).
|
1460
1594
|
to_return(:status => 404, :body => "", :headers => {})
|
1461
1595
|
# creation
|
1462
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1596
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1463
1597
|
with(basic_auth: ['john', 'doe']).
|
1464
1598
|
to_return(:status => 200, :body => "", :headers => {})
|
1465
1599
|
# check if alias exists
|
1466
1600
|
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1467
1601
|
with(basic_auth: ['john', 'doe']).
|
1468
1602
|
to_return(:status => 404, :body => "", :headers => {})
|
1469
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1603
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1470
1604
|
with(basic_auth: ['john', 'doe']).
|
1471
1605
|
to_return(status: 404, body: "", headers: {})
|
1472
|
-
|
1473
|
-
|
1474
|
-
|
1475
|
-
|
1606
|
+
if use_legacy_template_flag
|
1607
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1608
|
+
with(basic_auth: ['john', 'doe'],
|
1609
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1610
|
+
to_return(status: 200, body: "", headers: {})
|
1611
|
+
else
|
1612
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1613
|
+
with(basic_auth: ['john', 'doe'],
|
1614
|
+
body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
|
1615
|
+
to_return(status: 200, body: "", headers: {})
|
1616
|
+
end
|
1476
1617
|
# put the alias for the index
|
1477
1618
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1478
1619
|
with(basic_auth: ['john', 'doe']).
|
@@ -1497,14 +1638,21 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1497
1638
|
driver.run(default_tag: 'fluentd-policy') do
|
1498
1639
|
driver.feed(sample_record)
|
1499
1640
|
end
|
1500
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1641
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
1501
1642
|
|
1502
1643
|
assert_requested(elastic_request)
|
1503
1644
|
end
|
1504
1645
|
|
1505
|
-
|
1646
|
+
data("legacy_template" => [true, "_template"],
|
1647
|
+
"new_template" => [false, "_index_template"])
|
1648
|
+
def test_tag_placeholder_with_multiple_policies(data)
|
1649
|
+
use_legacy_template_flag, endpoint = data
|
1506
1650
|
cwd = File.dirname(__FILE__)
|
1507
|
-
template_file =
|
1651
|
+
template_file = if use_legacy_template_flag
|
1652
|
+
File.join(cwd, 'test_template.json')
|
1653
|
+
else
|
1654
|
+
File.join(cwd, 'test_index_template.json')
|
1655
|
+
end
|
1508
1656
|
|
1509
1657
|
config = %{
|
1510
1658
|
host logs.google.com
|
@@ -1520,6 +1668,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1520
1668
|
enable_ilm true
|
1521
1669
|
index_name logstash
|
1522
1670
|
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}, "fluentd-policy2":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"80gb", "max_age":"20d"}}}}}}}
|
1671
|
+
use_legacy_template #{use_legacy_template_flag}
|
1523
1672
|
}
|
1524
1673
|
|
1525
1674
|
# connection start
|
@@ -1527,24 +1676,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1527
1676
|
with(basic_auth: ['john', 'doe']).
|
1528
1677
|
to_return(:status => 200, :body => "", :headers => {})
|
1529
1678
|
# check if template exists
|
1530
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1679
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1531
1680
|
with(basic_auth: ['john', 'doe']).
|
1532
1681
|
to_return(:status => 404, :body => "", :headers => {})
|
1533
1682
|
# creation
|
1534
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1683
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1535
1684
|
with(basic_auth: ['john', 'doe']).
|
1536
1685
|
to_return(:status => 200, :body => "", :headers => {})
|
1537
1686
|
# check if alias exists
|
1538
1687
|
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1539
1688
|
with(basic_auth: ['john', 'doe']).
|
1540
1689
|
to_return(:status => 404, :body => "", :headers => {})
|
1541
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1690
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1542
1691
|
with(basic_auth: ['john', 'doe']).
|
1543
1692
|
to_return(status: 404, body: "", headers: {})
|
1544
|
-
|
1545
|
-
|
1546
|
-
|
1547
|
-
|
1693
|
+
if use_legacy_template_flag
|
1694
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1695
|
+
with(basic_auth: ['john', 'doe'],
|
1696
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy2\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
|
1697
|
+
to_return(status: 200, body: "", headers: {})
|
1698
|
+
else
|
1699
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1700
|
+
with(basic_auth: ['john', 'doe'],
|
1701
|
+
body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy2\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
|
1702
|
+
to_return(status: 200, body: "", headers: {})
|
1703
|
+
end
|
1548
1704
|
# put the alias for the index
|
1549
1705
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1550
1706
|
with(basic_auth: ['john', 'doe']).
|
@@ -1569,15 +1725,22 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1569
1725
|
driver.run(default_tag: 'fluentd-policy2') do
|
1570
1726
|
driver.feed(sample_record)
|
1571
1727
|
end
|
1572
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1728
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
1573
1729
|
|
1574
1730
|
assert_requested(elastic_request)
|
1575
1731
|
end
|
1576
1732
|
end
|
1577
1733
|
|
1578
|
-
|
1734
|
+
data("legacy_template" => [true, "_template"],
|
1735
|
+
"new_template" => [false, "_index_template"])
|
1736
|
+
def test_template_create_with_rollover_index_and_default_ilm_and_placeholders(data)
|
1737
|
+
use_legacy_template_flag, endpoint = data
|
1579
1738
|
cwd = File.dirname(__FILE__)
|
1580
|
-
template_file =
|
1739
|
+
template_file = if use_legacy_template_flag
|
1740
|
+
File.join(cwd, 'test_template.json')
|
1741
|
+
else
|
1742
|
+
File.join(cwd, 'test_index_template.json')
|
1743
|
+
end
|
1581
1744
|
|
1582
1745
|
config = %{
|
1583
1746
|
host logs.google.com
|
@@ -1591,6 +1754,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1591
1754
|
index_date_pattern now/w{xxxx.ww}
|
1592
1755
|
index_name logstash-${tag}
|
1593
1756
|
enable_ilm true
|
1757
|
+
use_legacy_template #{use_legacy_template_flag}
|
1594
1758
|
}
|
1595
1759
|
|
1596
1760
|
# connection start
|
@@ -1598,24 +1762,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1598
1762
|
with(basic_auth: ['john', 'doe']).
|
1599
1763
|
to_return(:status => 200, :body => "", :headers => {})
|
1600
1764
|
# check if template exists
|
1601
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1765
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1602
1766
|
with(basic_auth: ['john', 'doe']).
|
1603
1767
|
to_return(:status => 404, :body => "", :headers => {})
|
1604
1768
|
# creation
|
1605
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1769
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1606
1770
|
with(basic_auth: ['john', 'doe']).
|
1607
1771
|
to_return(:status => 200, :body => "", :headers => {})
|
1608
1772
|
# check if alias exists
|
1609
1773
|
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-test").
|
1610
1774
|
with(basic_auth: ['john', 'doe']).
|
1611
1775
|
to_return(:status => 404, :body => "", :headers => {})
|
1612
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1776
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-test").
|
1613
1777
|
with(basic_auth: ['john', 'doe']).
|
1614
1778
|
to_return(status: 404, body: "", headers: {})
|
1615
|
-
|
1616
|
-
|
1617
|
-
|
1618
|
-
|
1779
|
+
if use_legacy_template_flag
|
1780
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-test").
|
1781
|
+
with(basic_auth: ['john', 'doe'],
|
1782
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-test\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-test-*\",\"order\":52}").
|
1783
|
+
to_return(status: 200, body: "", headers: {})
|
1784
|
+
else
|
1785
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-test").
|
1786
|
+
with(basic_auth: ['john', 'doe'],
|
1787
|
+
body: "{\"index_patterns\":\"logstash-test-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-test\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
|
1788
|
+
to_return(status: 200, body: "", headers: {})
|
1789
|
+
end
|
1619
1790
|
# put the alias for the index
|
1620
1791
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-test-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1621
1792
|
with(basic_auth: ['john', 'doe']).
|
@@ -1648,9 +1819,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1648
1819
|
assert_requested(elastic_request)
|
1649
1820
|
end
|
1650
1821
|
|
1651
|
-
|
1822
|
+
data("legacy_template" => [true, "_template"],
|
1823
|
+
"new_template" => [false, "_index_template"])
|
1824
|
+
def test_template_create_with_rollover_index_and_default_ilm_and_custom_and_time_placeholders(data)
|
1825
|
+
use_legacy_template_flag, endpoint = data
|
1652
1826
|
cwd = File.dirname(__FILE__)
|
1653
|
-
template_file =
|
1827
|
+
template_file = if use_legacy_template_flag
|
1828
|
+
File.join(cwd, 'test_template.json')
|
1829
|
+
else
|
1830
|
+
File.join(cwd, 'test_index_template.json')
|
1831
|
+
end
|
1654
1832
|
|
1655
1833
|
config = Fluent::Config::Element.new(
|
1656
1834
|
'ROOT', '', {
|
@@ -1666,6 +1844,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1666
1844
|
'index_date_pattern' => 'now/w{xxxx.ww}',
|
1667
1845
|
'index_name' => "${taskDef}-%Y.%m",
|
1668
1846
|
'enable_ilm' => true,
|
1847
|
+
'use_legacy_template' => use_legacy_template_flag,
|
1848
|
+
|
1669
1849
|
}, [
|
1670
1850
|
Fluent::Config::Element.new('buffer', 'tag, time, taskDef', {
|
1671
1851
|
'chunk_keys' => ['tag', 'time', 'taskDef'],
|
@@ -1681,14 +1861,21 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1681
1861
|
with(basic_auth: ['john', 'doe']).
|
1682
1862
|
to_return(:status => 200, :body => "", :headers => {})
|
1683
1863
|
# check if template exists
|
1684
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1864
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/#{task_def_value}-#{date_str}").
|
1685
1865
|
with(basic_auth: ['john', 'doe']).
|
1686
1866
|
to_return(:status => 404, :body => "", :headers => {})
|
1687
1867
|
# creation
|
1688
|
-
|
1689
|
-
|
1690
|
-
|
1691
|
-
|
1868
|
+
if use_legacy_template_flag
|
1869
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/#{task_def_value}-#{date_str}").
|
1870
|
+
with(basic_auth: ['john', 'doe'],
|
1871
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"#{task_def_value}-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"#{task_def_value}-#{date_str}-*\",\"order\":52}").
|
1872
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1873
|
+
else
|
1874
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/#{task_def_value}-#{date_str}").
|
1875
|
+
with(basic_auth: ['john', 'doe'],
|
1876
|
+
body: "{\"index_patterns\":\"task_definition-2020.09-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"task_definition-2020.09\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
|
1877
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1878
|
+
end
|
1692
1879
|
# check if alias exists
|
1693
1880
|
stub_request(:head, "https://logs.google.com:777/es//_alias/#{task_def_value}-#{date_str}").
|
1694
1881
|
with(basic_auth: ['john', 'doe']).
|
@@ -1726,9 +1913,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1726
1913
|
end
|
1727
1914
|
end
|
1728
1915
|
|
1729
|
-
|
1916
|
+
data("legacy_template" => [true, "_template"],
|
1917
|
+
"new_template" => [false, "_index_template"])
|
1918
|
+
def test_custom_template_create(data)
|
1919
|
+
use_legacy_template_flag, endpoint = data
|
1730
1920
|
cwd = File.dirname(__FILE__)
|
1731
|
-
template_file =
|
1921
|
+
template_file = if use_legacy_template_flag
|
1922
|
+
File.join(cwd, 'test_alias_template.json')
|
1923
|
+
else
|
1924
|
+
File.join(cwd, 'test_index_alias_template.json')
|
1925
|
+
end
|
1732
1926
|
|
1733
1927
|
config = %{
|
1734
1928
|
host logs.google.com
|
@@ -1740,6 +1934,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1740
1934
|
template_name myapp_alias_template
|
1741
1935
|
template_file #{template_file}
|
1742
1936
|
customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
|
1937
|
+
use_legacy_template #{use_legacy_template_flag}
|
1743
1938
|
}
|
1744
1939
|
|
1745
1940
|
# connection start
|
@@ -1747,22 +1942,29 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1747
1942
|
with(basic_auth: ['john', 'doe']).
|
1748
1943
|
to_return(:status => 200, :body => "", :headers => {})
|
1749
1944
|
# check if template exists
|
1750
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1945
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
1751
1946
|
with(basic_auth: ['john', 'doe']).
|
1752
1947
|
to_return(:status => 404, :body => "", :headers => {})
|
1753
1948
|
# creation
|
1754
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1949
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
1755
1950
|
with(basic_auth: ['john', 'doe']).
|
1756
1951
|
to_return(:status => 200, :body => "", :headers => {})
|
1757
1952
|
|
1758
1953
|
driver(config)
|
1759
1954
|
|
1760
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1955
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template", times: 1)
|
1761
1956
|
end
|
1762
1957
|
|
1763
|
-
|
1958
|
+
data("legacy_template" => [true, "_template"],
|
1959
|
+
"new_template" => [false, "_index_template"])
|
1960
|
+
def test_custom_template_create_with_customize_template_related_placeholders(data)
|
1961
|
+
use_legacy_template_flag, endpoint = data
|
1764
1962
|
cwd = File.dirname(__FILE__)
|
1765
|
-
template_file =
|
1963
|
+
template_file = if use_legacy_template_flag
|
1964
|
+
File.join(cwd, 'test_alias_template.json')
|
1965
|
+
else
|
1966
|
+
File.join(cwd, 'test_index_alias_template.json')
|
1967
|
+
end
|
1766
1968
|
|
1767
1969
|
config = %{
|
1768
1970
|
host logs.google.com
|
@@ -1774,6 +1976,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1774
1976
|
template_name myapp_alias_template-${tag}
|
1775
1977
|
template_file #{template_file}
|
1776
1978
|
customize_template {"--appid--": "${tag}-logs","--index_prefix--":"${tag}"}
|
1979
|
+
use_legacy_template #{use_legacy_template_flag}
|
1777
1980
|
}
|
1778
1981
|
|
1779
1982
|
# connection start
|
@@ -1781,11 +1984,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1781
1984
|
with(basic_auth: ['john', 'doe']).
|
1782
1985
|
to_return(:status => 200, :body => "", :headers => {})
|
1783
1986
|
# check if template exists
|
1784
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1987
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template-test.template").
|
1785
1988
|
with(basic_auth: ['john', 'doe']).
|
1786
1989
|
to_return(:status => 404, :body => "", :headers => {})
|
1787
1990
|
# creation
|
1788
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1991
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template-test.template").
|
1789
1992
|
with(basic_auth: ['john', 'doe']).
|
1790
1993
|
to_return(:status => 200, :body => "", :headers => {})
|
1791
1994
|
|
@@ -1799,12 +2002,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1799
2002
|
driver.feed(sample_record)
|
1800
2003
|
end
|
1801
2004
|
|
1802
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2005
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template-test.template", times: 1)
|
1803
2006
|
end
|
1804
2007
|
|
1805
|
-
|
2008
|
+
data("legacy_template" => [true, "_template"],
|
2009
|
+
"new_template" => [false, "_index_template"])
|
2010
|
+
def test_custom_template_installation_for_host_placeholder(data)
|
2011
|
+
use_legacy_template_flag, endpoint = data
|
1806
2012
|
cwd = File.dirname(__FILE__)
|
1807
|
-
template_file =
|
2013
|
+
template_file = if use_legacy_template_flag
|
2014
|
+
File.join(cwd, 'test_template.json')
|
2015
|
+
else
|
2016
|
+
File.join(cwd, 'test_index_template.json')
|
2017
|
+
end
|
1808
2018
|
|
1809
2019
|
config = %{
|
1810
2020
|
host logs-${tag}.google.com
|
@@ -1818,6 +2028,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1818
2028
|
verify_es_version_at_startup false
|
1819
2029
|
default_elasticsearch_version 6
|
1820
2030
|
customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
|
2031
|
+
use_legacy_template #{use_legacy_template_flag}
|
1821
2032
|
}
|
1822
2033
|
|
1823
2034
|
# connection start
|
@@ -1825,10 +2036,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1825
2036
|
with(basic_auth: ['john', 'doe']).
|
1826
2037
|
to_return(:status => 200, :body => "", :headers => {})
|
1827
2038
|
# check if template exists
|
1828
|
-
stub_request(:get, "https://logs-test.google.com:777/es
|
2039
|
+
stub_request(:get, "https://logs-test.google.com:777/es//#{endpoint}/logstash").
|
1829
2040
|
with(basic_auth: ['john', 'doe']).
|
1830
2041
|
to_return(:status => 404, :body => "", :headers => {})
|
1831
|
-
stub_request(:put, "https://logs-test.google.com:777/es
|
2042
|
+
stub_request(:put, "https://logs-test.google.com:777/es//#{endpoint}/logstash").
|
1832
2043
|
with(basic_auth: ['john', 'doe']).
|
1833
2044
|
to_return(status: 200, body: "", headers: {})
|
1834
2045
|
|
@@ -1840,9 +2051,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1840
2051
|
end
|
1841
2052
|
end
|
1842
2053
|
|
1843
|
-
|
2054
|
+
data("legacy_template" => [true, "_template"],
|
2055
|
+
"new_template" => [false, "_index_template"])
|
2056
|
+
def test_custom_template_with_rollover_index_create(data)
|
2057
|
+
use_legacy_template_flag, endpoint = data
|
1844
2058
|
cwd = File.dirname(__FILE__)
|
1845
|
-
template_file =
|
2059
|
+
template_file = if use_legacy_template_flag
|
2060
|
+
File.join(cwd, 'test_alias_template.json')
|
2061
|
+
else
|
2062
|
+
File.join(cwd, 'test_index_alias_template.json')
|
2063
|
+
end
|
1846
2064
|
|
1847
2065
|
config = %{
|
1848
2066
|
host logs.google.com
|
@@ -1858,6 +2076,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1858
2076
|
index_date_pattern now/w{xxxx.ww}
|
1859
2077
|
index_name mylogs
|
1860
2078
|
application_name myapp
|
2079
|
+
use_legacy_template #{use_legacy_template_flag}
|
1861
2080
|
}
|
1862
2081
|
|
1863
2082
|
# connection start
|
@@ -1865,11 +2084,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1865
2084
|
with(basic_auth: ['john', 'doe']).
|
1866
2085
|
to_return(:status => 200, :body => "", :headers => {})
|
1867
2086
|
# check if template exists
|
1868
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2087
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
1869
2088
|
with(basic_auth: ['john', 'doe']).
|
1870
2089
|
to_return(:status => 404, :body => "", :headers => {})
|
1871
2090
|
# creation
|
1872
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2091
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
1873
2092
|
with(basic_auth: ['john', 'doe']).
|
1874
2093
|
to_return(:status => 200, :body => "", :headers => {})
|
1875
2094
|
# creation of index which can rollover
|
@@ -1887,12 +2106,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1887
2106
|
|
1888
2107
|
driver(config)
|
1889
2108
|
|
1890
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2109
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template", times: 1)
|
1891
2110
|
end
|
1892
2111
|
|
1893
|
-
|
2112
|
+
data("legacy_template" => [true, "_template"],
|
2113
|
+
"new_template" => [false, "_index_template"])
|
2114
|
+
def test_custom_template_with_rollover_index_create_and_deflector_alias(data)
|
2115
|
+
use_legacy_template_flag, endpoint = data
|
1894
2116
|
cwd = File.dirname(__FILE__)
|
1895
|
-
template_file =
|
2117
|
+
template_file = if use_legacy_template_flag
|
2118
|
+
File.join(cwd, 'test_alias_template.json')
|
2119
|
+
else
|
2120
|
+
File.join(cwd, 'test_index_alias_template.json')
|
2121
|
+
end
|
1896
2122
|
|
1897
2123
|
config = %{
|
1898
2124
|
host logs.google.com
|
@@ -1909,6 +2135,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1909
2135
|
deflector_alias myapp_deflector
|
1910
2136
|
index_name mylogs
|
1911
2137
|
application_name myapp
|
2138
|
+
use_legacy_template #{use_legacy_template_flag}
|
1912
2139
|
}
|
1913
2140
|
|
1914
2141
|
# connection start
|
@@ -1916,11 +2143,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1916
2143
|
with(basic_auth: ['john', 'doe']).
|
1917
2144
|
to_return(:status => 200, :body => "", :headers => {})
|
1918
2145
|
# check if template exists
|
1919
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2146
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
1920
2147
|
with(basic_auth: ['john', 'doe']).
|
1921
2148
|
to_return(:status => 404, :body => "", :headers => {})
|
1922
2149
|
# creation
|
1923
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2150
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
1924
2151
|
with(basic_auth: ['john', 'doe']).
|
1925
2152
|
to_return(:status => 200, :body => "", :headers => {})
|
1926
2153
|
# creation of index which can rollover
|
@@ -1938,12 +2165,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1938
2165
|
|
1939
2166
|
driver(config)
|
1940
2167
|
|
1941
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2168
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template", times: 1)
|
1942
2169
|
end
|
1943
2170
|
|
1944
|
-
|
2171
|
+
data("legacy_template" => [true, "_template"],
|
2172
|
+
"new_template" => [false, "_index_template"])
|
2173
|
+
def test_custom_template_with_rollover_index_create_with_logstash_format(data)
|
2174
|
+
use_legacy_template_flag, endpoint = data
|
1945
2175
|
cwd = File.dirname(__FILE__)
|
1946
|
-
template_file =
|
2176
|
+
template_file = if use_legacy_template_flag
|
2177
|
+
File.join(cwd, 'test_alias_template.json')
|
2178
|
+
else
|
2179
|
+
File.join(cwd, 'test_index_alias_template.json')
|
2180
|
+
end
|
1947
2181
|
|
1948
2182
|
config = %{
|
1949
2183
|
host logs.google.com
|
@@ -1960,6 +2194,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1960
2194
|
logstash_format true
|
1961
2195
|
logstash_prefix mylogs
|
1962
2196
|
application_name myapp
|
2197
|
+
use_legacy_template #{use_legacy_template_flag}
|
1963
2198
|
}
|
1964
2199
|
|
1965
2200
|
timestr = Time.now.strftime("%Y.%m.%d")
|
@@ -1968,11 +2203,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1968
2203
|
with(basic_auth: ['john', 'doe']).
|
1969
2204
|
to_return(:status => 200, :body => "", :headers => {})
|
1970
2205
|
# check if template exists
|
1971
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2206
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
1972
2207
|
with(basic_auth: ['john', 'doe']).
|
1973
2208
|
to_return(:status => 404, :body => "", :headers => {})
|
1974
2209
|
# creation
|
1975
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2210
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
1976
2211
|
with(basic_auth: ['john', 'doe']).
|
1977
2212
|
to_return(:status => 200, :body => "", :headers => {})
|
1978
2213
|
# creation of index which can rollover
|
@@ -2007,9 +2242,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2007
2242
|
end
|
2008
2243
|
end
|
2009
2244
|
|
2010
|
-
|
2245
|
+
data("legacy_template" => [true, "_template"],
|
2246
|
+
"new_template" => [false, "_index_template"])
|
2247
|
+
def test_custom_template_with_rollover_index_create_and_default_ilm(data)
|
2248
|
+
use_legacy_template_flag, endpoint = data
|
2011
2249
|
cwd = File.dirname(__FILE__)
|
2012
|
-
template_file =
|
2250
|
+
template_file = if use_legacy_template_flag
|
2251
|
+
File.join(cwd, 'test_alias_template.json')
|
2252
|
+
else
|
2253
|
+
File.join(cwd, 'test_index_alias_template.json')
|
2254
|
+
end
|
2013
2255
|
|
2014
2256
|
config = %{
|
2015
2257
|
host logs.google.com
|
@@ -2026,6 +2268,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2026
2268
|
application_name myapp
|
2027
2269
|
ilm_policy_id fluentd-policy
|
2028
2270
|
enable_ilm true
|
2271
|
+
use_legacy_template #{use_legacy_template_flag}
|
2029
2272
|
}
|
2030
2273
|
|
2031
2274
|
# connection start
|
@@ -2033,11 +2276,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2033
2276
|
with(basic_auth: ['john', 'doe']).
|
2034
2277
|
to_return(:status => 200, :body => "", :headers => {})
|
2035
2278
|
# check if template exists
|
2036
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2279
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2037
2280
|
with(basic_auth: ['john', 'doe']).
|
2038
2281
|
to_return(:status => 404, :body => "", :headers => {})
|
2039
2282
|
# creation
|
2040
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2283
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2041
2284
|
with(basic_auth: ['john', 'doe']).
|
2042
2285
|
to_return(:status => 200, :body => "", :headers => {})
|
2043
2286
|
# creation of index which can rollover
|
@@ -2048,13 +2291,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2048
2291
|
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs").
|
2049
2292
|
with(basic_auth: ['john', 'doe']).
|
2050
2293
|
to_return(:status => 404, :body => "", :headers => {})
|
2051
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2294
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs").
|
2052
2295
|
with(basic_auth: ['john', 'doe']).
|
2053
2296
|
to_return(status: 404, body: "", headers: {})
|
2054
|
-
|
2055
|
-
|
2056
|
-
|
2057
|
-
|
2297
|
+
if use_legacy_template_flag
|
2298
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs").
|
2299
|
+
with(basic_auth: ['john', 'doe'],
|
2300
|
+
body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-*\"}").
|
2301
|
+
to_return(status: 200, body: "", headers: {})
|
2302
|
+
else
|
2303
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs").
|
2304
|
+
with(basic_auth: ['john', 'doe'],
|
2305
|
+
body: "{\"priority\":106,\"index_patterns\":\"mylogs-*\",\"template\":{\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}}}}").
|
2306
|
+
to_return(status: 200, body: "", headers: {})
|
2307
|
+
end
|
2058
2308
|
# put the alias for the index
|
2059
2309
|
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs").
|
2060
2310
|
with(basic_auth: ['john', 'doe'],
|
@@ -2073,12 +2323,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2073
2323
|
|
2074
2324
|
driver(config)
|
2075
2325
|
|
2076
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2326
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs", times: 1)
|
2077
2327
|
end
|
2078
2328
|
|
2079
|
-
|
2329
|
+
data("legacy_template" => [true, "_template"],
|
2330
|
+
"new_template" => [false, "_index_template"])
|
2331
|
+
def test_custom_template_with_rollover_index_create_and_default_ilm_and_ilm_policy_overwrite(data)
|
2332
|
+
use_legacy_template_flag, endpoint = data
|
2080
2333
|
cwd = File.dirname(__FILE__)
|
2081
|
-
template_file =
|
2334
|
+
template_file = if use_legacy_template_flag
|
2335
|
+
File.join(cwd, 'test_alias_template.json')
|
2336
|
+
else
|
2337
|
+
File.join(cwd, 'test_index_alias_template.json')
|
2338
|
+
end
|
2082
2339
|
|
2083
2340
|
config = %{
|
2084
2341
|
host logs.google.com
|
@@ -2097,6 +2354,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2097
2354
|
enable_ilm true
|
2098
2355
|
ilm_policy_overwrite true
|
2099
2356
|
ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"60gb","max_age": "45d"}}}}}}
|
2357
|
+
use_legacy_template #{use_legacy_template_flag}
|
2100
2358
|
}
|
2101
2359
|
|
2102
2360
|
# connection start
|
@@ -2104,11 +2362,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2104
2362
|
with(basic_auth: ['john', 'doe']).
|
2105
2363
|
to_return(:status => 200, :body => "", :headers => {})
|
2106
2364
|
# check if template exists
|
2107
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2365
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2108
2366
|
with(basic_auth: ['john', 'doe']).
|
2109
2367
|
to_return(:status => 404, :body => "", :headers => {})
|
2110
2368
|
# creation
|
2111
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2369
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2112
2370
|
with(basic_auth: ['john', 'doe']).
|
2113
2371
|
to_return(:status => 200, :body => "", :headers => {})
|
2114
2372
|
# creation of index which can rollover
|
@@ -2119,13 +2377,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2119
2377
|
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs").
|
2120
2378
|
with(basic_auth: ['john', 'doe']).
|
2121
2379
|
to_return(:status => 404, :body => "", :headers => {})
|
2122
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2380
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs").
|
2123
2381
|
with(basic_auth: ['john', 'doe']).
|
2124
2382
|
to_return(status: 404, body: "", headers: {})
|
2125
|
-
|
2126
|
-
|
2127
|
-
|
2128
|
-
|
2383
|
+
if use_legacy_template_flag
|
2384
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs").
|
2385
|
+
with(basic_auth: ['john', 'doe'],
|
2386
|
+
body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-*\"}").
|
2387
|
+
to_return(status: 200, body: "", headers: {})
|
2388
|
+
else
|
2389
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs").
|
2390
|
+
with(basic_auth: ['john', 'doe'],
|
2391
|
+
body: "{\"priority\":106,\"index_patterns\":\"mylogs-*\",\"template\":{\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}}}}").
|
2392
|
+
to_return(status: 200, body: "", headers: {})
|
2393
|
+
end
|
2129
2394
|
# put the alias for the index
|
2130
2395
|
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs").
|
2131
2396
|
with(basic_auth: ['john', 'doe'],
|
@@ -2144,7 +2409,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2144
2409
|
|
2145
2410
|
driver(config)
|
2146
2411
|
|
2147
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2412
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs", times: 1)
|
2148
2413
|
end
|
2149
2414
|
|
2150
2415
|
def test_custom_template_with_rollover_index_create_and_default_ilm_with_deflector_alias
|
@@ -2175,9 +2440,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2175
2440
|
end
|
2176
2441
|
end
|
2177
2442
|
|
2178
|
-
|
2443
|
+
data("legacy_template" => [true, "_template"],
|
2444
|
+
"new_template" => [false, "_index_template"])
|
2445
|
+
def test_custom_template_with_rollover_index_create_and_default_ilm_and_placeholders(data)
|
2446
|
+
use_legacy_template_flag, endpoint = data
|
2179
2447
|
cwd = File.dirname(__FILE__)
|
2180
|
-
template_file =
|
2448
|
+
template_file = if use_legacy_template_flag
|
2449
|
+
File.join(cwd, 'test_alias_template.json')
|
2450
|
+
else
|
2451
|
+
File.join(cwd, 'test_index_alias_template.json')
|
2452
|
+
end
|
2181
2453
|
|
2182
2454
|
config = %{
|
2183
2455
|
host logs.google.com
|
@@ -2194,6 +2466,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2194
2466
|
application_name myapp
|
2195
2467
|
ilm_policy_id fluentd-policy
|
2196
2468
|
enable_ilm true
|
2469
|
+
use_legacy_template #{use_legacy_template_flag}
|
2197
2470
|
}
|
2198
2471
|
|
2199
2472
|
# connection start
|
@@ -2201,11 +2474,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2201
2474
|
with(basic_auth: ['john', 'doe']).
|
2202
2475
|
to_return(:status => 200, :body => "", :headers => {})
|
2203
2476
|
# check if template exists
|
2204
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2477
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2205
2478
|
with(basic_auth: ['john', 'doe']).
|
2206
2479
|
to_return(:status => 404, :body => "", :headers => {})
|
2207
2480
|
# creation
|
2208
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2481
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2209
2482
|
with(basic_auth: ['john', 'doe']).
|
2210
2483
|
to_return(:status => 200, :body => "", :headers => {})
|
2211
2484
|
# creation of index which can rollover
|
@@ -2216,13 +2489,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2216
2489
|
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-custom-test").
|
2217
2490
|
with(basic_auth: ['john', 'doe']).
|
2218
2491
|
to_return(:status => 404, :body => "", :headers => {})
|
2219
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2492
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs-custom-test").
|
2220
2493
|
with(basic_auth: ['john', 'doe']).
|
2221
2494
|
to_return(status: 404, body: "", headers: {})
|
2222
|
-
|
2223
|
-
|
2224
|
-
|
2225
|
-
|
2495
|
+
if use_legacy_template_flag
|
2496
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-custom-test").
|
2497
|
+
with(basic_auth: ['john', 'doe'],
|
2498
|
+
body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-custom-test\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-custom-test-*\"}").
|
2499
|
+
to_return(status: 200, body: "", headers: {})
|
2500
|
+
else
|
2501
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-custom-test").
|
2502
|
+
with(basic_auth: ['john', 'doe'],
|
2503
|
+
body: "{\"priority\":108,\"index_patterns\":\"mylogs-custom-test-*\",\"template\":{\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-custom-test\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}}}}").
|
2504
|
+
to_return(status: 200, body: "", headers: {})
|
2505
|
+
end
|
2226
2506
|
# put the alias for the index
|
2227
2507
|
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-custom-test-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-custom-test").
|
2228
2508
|
with(basic_auth: ['john', 'doe'],
|
@@ -2252,9 +2532,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2252
2532
|
assert_requested(elastic_request)
|
2253
2533
|
end
|
2254
2534
|
|
2255
|
-
|
2535
|
+
data("legacy_template" => [true, "_template"],
|
2536
|
+
"new_template" => [false, "_index_template"])
|
2537
|
+
def test_custom_template_with_rollover_index_create_and_custom_ilm(data)
|
2538
|
+
use_legacy_template_flag, endpoint = data
|
2256
2539
|
cwd = File.dirname(__FILE__)
|
2257
|
-
template_file =
|
2540
|
+
template_file = if use_legacy_template_flag
|
2541
|
+
File.join(cwd, 'test_alias_template.json')
|
2542
|
+
else
|
2543
|
+
File.join(cwd, 'test_index_alias_template.json')
|
2544
|
+
end
|
2258
2545
|
|
2259
2546
|
config = %{
|
2260
2547
|
host logs.google.com
|
@@ -2272,6 +2559,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2272
2559
|
ilm_policy_id fluentd-policy
|
2273
2560
|
enable_ilm true
|
2274
2561
|
ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}
|
2562
|
+
use_legacy_template #{use_legacy_template_flag}
|
2275
2563
|
}
|
2276
2564
|
|
2277
2565
|
# connection start
|
@@ -2279,11 +2567,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2279
2567
|
with(basic_auth: ['john', 'doe']).
|
2280
2568
|
to_return(:status => 200, :body => "", :headers => {})
|
2281
2569
|
# check if template exists
|
2282
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2570
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2283
2571
|
with(basic_auth: ['john', 'doe']).
|
2284
2572
|
to_return(:status => 404, :body => "", :headers => {})
|
2285
2573
|
# creation
|
2286
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2574
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2287
2575
|
with(basic_auth: ['john', 'doe']).
|
2288
2576
|
to_return(:status => 200, :body => "", :headers => {})
|
2289
2577
|
# creation of index which can rollover
|
@@ -2294,10 +2582,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2294
2582
|
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs").
|
2295
2583
|
with(basic_auth: ['john', 'doe']).
|
2296
2584
|
to_return(:status => 404, :body => "", :headers => {})
|
2297
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2585
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs").
|
2298
2586
|
with(basic_auth: ['john', 'doe']).
|
2299
2587
|
to_return(status: 404, body: "", headers: {})
|
2300
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2588
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs").
|
2301
2589
|
with(basic_auth: ['john', 'doe']).
|
2302
2590
|
to_return(status: 200, body: "", headers: {})
|
2303
2591
|
# put the alias for the index
|
@@ -2318,13 +2606,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2318
2606
|
|
2319
2607
|
driver(config)
|
2320
2608
|
|
2321
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2609
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs", times: 1)
|
2322
2610
|
end
|
2323
2611
|
end
|
2324
2612
|
|
2325
|
-
|
2613
|
+
data("legacy_template" => [true, "_template"],
|
2614
|
+
"new_template" => [false, "_index_template"])
|
2615
|
+
def test_template_overwrite(data)
|
2616
|
+
use_legacy_template_flag, endpoint = data
|
2326
2617
|
cwd = File.dirname(__FILE__)
|
2327
|
-
template_file =
|
2618
|
+
template_file = if use_legacy_template_flag
|
2619
|
+
File.join(cwd, 'test_template.json')
|
2620
|
+
else
|
2621
|
+
File.join(cwd, 'test_index_template.json')
|
2622
|
+
end
|
2328
2623
|
|
2329
2624
|
config = %{
|
2330
2625
|
host logs.google.com
|
@@ -2336,6 +2631,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2336
2631
|
template_name logstash
|
2337
2632
|
template_file #{template_file}
|
2338
2633
|
template_overwrite true
|
2634
|
+
use_legacy_template #{use_legacy_template_flag}
|
2339
2635
|
}
|
2340
2636
|
|
2341
2637
|
# connection start
|
@@ -2343,22 +2639,29 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2343
2639
|
with(basic_auth: ['john', 'doe']).
|
2344
2640
|
to_return(:status => 200, :body => "", :headers => {})
|
2345
2641
|
# check if template exists
|
2346
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2642
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
2347
2643
|
with(basic_auth: ['john', 'doe']).
|
2348
2644
|
to_return(:status => 200, :body => "", :headers => {})
|
2349
2645
|
# creation
|
2350
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2646
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
2351
2647
|
with(basic_auth: ['john', 'doe']).
|
2352
2648
|
to_return(:status => 200, :body => "", :headers => {})
|
2353
2649
|
|
2354
2650
|
driver(config)
|
2355
2651
|
|
2356
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2652
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
2357
2653
|
end
|
2358
2654
|
|
2359
|
-
|
2655
|
+
data("legacy_template" => [true, "_template"],
|
2656
|
+
"new_template" => [false, "_index_template"])
|
2657
|
+
def test_custom_template_overwrite(data)
|
2658
|
+
use_legacy_template_flag, endpoint = data
|
2360
2659
|
cwd = File.dirname(__FILE__)
|
2361
|
-
template_file =
|
2660
|
+
template_file = if use_legacy_template_flag
|
2661
|
+
File.join(cwd, 'test_template.json')
|
2662
|
+
else
|
2663
|
+
File.join(cwd, 'test_index_template.json')
|
2664
|
+
end
|
2362
2665
|
|
2363
2666
|
config = %{
|
2364
2667
|
host logs.google.com
|
@@ -2371,6 +2674,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2371
2674
|
template_file #{template_file}
|
2372
2675
|
template_overwrite true
|
2373
2676
|
customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
|
2677
|
+
use_legacy_template #{use_legacy_template_flag}
|
2374
2678
|
}
|
2375
2679
|
|
2376
2680
|
# connection start
|
@@ -2378,22 +2682,29 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2378
2682
|
with(basic_auth: ['john', 'doe']).
|
2379
2683
|
to_return(:status => 200, :body => "", :headers => {})
|
2380
2684
|
# check if template exists
|
2381
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2685
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2382
2686
|
with(basic_auth: ['john', 'doe']).
|
2383
2687
|
to_return(:status => 200, :body => "", :headers => {})
|
2384
2688
|
# creation
|
2385
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2689
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2386
2690
|
with(basic_auth: ['john', 'doe']).
|
2387
2691
|
to_return(:status => 200, :body => "", :headers => {})
|
2388
2692
|
|
2389
2693
|
driver(config)
|
2390
2694
|
|
2391
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2695
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template", times: 1)
|
2392
2696
|
end
|
2393
2697
|
|
2394
|
-
|
2698
|
+
data("legacy_template" => [true, "_template"],
|
2699
|
+
"new_template" => [false, "_index_template"])
|
2700
|
+
def test_custom_template_with_rollover_index_overwrite(data)
|
2701
|
+
use_legacy_template_flag, endpoint = data
|
2395
2702
|
cwd = File.dirname(__FILE__)
|
2396
|
-
template_file =
|
2703
|
+
template_file = if use_legacy_template_flag
|
2704
|
+
File.join(cwd, 'test_template.json')
|
2705
|
+
else
|
2706
|
+
File.join(cwd, 'test_index_template.json')
|
2707
|
+
end
|
2397
2708
|
|
2398
2709
|
config = %{
|
2399
2710
|
host logs.google.com
|
@@ -2410,6 +2721,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2410
2721
|
rollover_index true
|
2411
2722
|
index_name mylogs
|
2412
2723
|
application_name myapp
|
2724
|
+
use_legacy_template #{use_legacy_template_flag}
|
2413
2725
|
}
|
2414
2726
|
|
2415
2727
|
# connection start
|
@@ -2417,11 +2729,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2417
2729
|
with(basic_auth: ['john', 'doe']).
|
2418
2730
|
to_return(:status => 200, :body => "", :headers => {})
|
2419
2731
|
# check if template exists
|
2420
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2732
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2421
2733
|
with(basic_auth: ['john', 'doe']).
|
2422
2734
|
to_return(:status => 200, :body => "", :headers => {})
|
2423
2735
|
# creation
|
2424
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2736
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2425
2737
|
with(basic_auth: ['john', 'doe']).
|
2426
2738
|
to_return(:status => 200, :body => "", :headers => {})
|
2427
2739
|
# creation of index which can rollover
|
@@ -2439,7 +2751,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2439
2751
|
|
2440
2752
|
driver(config)
|
2441
2753
|
|
2442
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2754
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template", times: 1)
|
2443
2755
|
end
|
2444
2756
|
|
2445
2757
|
def test_template_create_invalid_filename
|
@@ -2469,9 +2781,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2469
2781
|
}
|
2470
2782
|
end
|
2471
2783
|
|
2472
|
-
|
2784
|
+
data("legacy_template" => [true, "_template"],
|
2785
|
+
"new_template" => [false, "_index_template"])
|
2786
|
+
def test_template_create_for_host_placeholder(data)
|
2787
|
+
use_legacy_template_flag, endpoint = data
|
2473
2788
|
cwd = File.dirname(__FILE__)
|
2474
|
-
template_file =
|
2789
|
+
template_file = if use_legacy_template_flag
|
2790
|
+
File.join(cwd, 'test_template.json')
|
2791
|
+
else
|
2792
|
+
File.join(cwd, 'test_index_template.json')
|
2793
|
+
end
|
2475
2794
|
|
2476
2795
|
config = %{
|
2477
2796
|
host logs-${tag}.google.com
|
@@ -2484,6 +2803,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2484
2803
|
template_file #{template_file}
|
2485
2804
|
verify_es_version_at_startup false
|
2486
2805
|
default_elasticsearch_version 6
|
2806
|
+
use_legacy_template #{use_legacy_template_flag}
|
2487
2807
|
}
|
2488
2808
|
|
2489
2809
|
# connection start
|
@@ -2491,10 +2811,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2491
2811
|
with(basic_auth: ['john', 'doe']).
|
2492
2812
|
to_return(:status => 200, :body => "", :headers => {})
|
2493
2813
|
# check if template exists
|
2494
|
-
stub_request(:get, "https://logs-test.google.com:777/es
|
2814
|
+
stub_request(:get, "https://logs-test.google.com:777/es//#{endpoint}/logstash").
|
2495
2815
|
with(basic_auth: ['john', 'doe']).
|
2496
2816
|
to_return(:status => 404, :body => "", :headers => {})
|
2497
|
-
stub_request(:put, "https://logs-test.google.com:777/es
|
2817
|
+
stub_request(:put, "https://logs-test.google.com:777/es//#{endpoint}/logstash").
|
2498
2818
|
with(basic_auth: ['john', 'doe']).
|
2499
2819
|
to_return(status: 200, body: "", headers: {})
|
2500
2820
|
stub_request(:post, "https://logs-test.google.com:777/es//_bulk").
|
@@ -2509,9 +2829,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2509
2829
|
end
|
2510
2830
|
end
|
2511
2831
|
|
2512
|
-
|
2832
|
+
data("legacy_template" => [true, "_template"],
|
2833
|
+
"new_template" => [false, "_index_template"])
|
2834
|
+
def test_template_retry_install_fails(data)
|
2835
|
+
use_legacy_template_flag, endpoint = data
|
2513
2836
|
cwd = File.dirname(__FILE__)
|
2514
|
-
template_file =
|
2837
|
+
template_file = if use_legacy_template_flag
|
2838
|
+
File.join(cwd, 'test_template.json')
|
2839
|
+
else
|
2840
|
+
File.join(cwd, 'test_index_template.json')
|
2841
|
+
end
|
2515
2842
|
|
2516
2843
|
config = %{
|
2517
2844
|
host logs.google.com
|
@@ -2523,11 +2850,12 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2523
2850
|
template_name logstash
|
2524
2851
|
template_file #{template_file}
|
2525
2852
|
max_retry_putting_template 3
|
2853
|
+
use_legacy_template #{use_legacy_template_flag}
|
2526
2854
|
}
|
2527
2855
|
|
2528
2856
|
connection_resets = 0
|
2529
2857
|
# check if template exists
|
2530
|
-
stub_request(:get, "https://logs.google.com:778/es
|
2858
|
+
stub_request(:get, "https://logs.google.com:778/es//#{endpoint}/logstash")
|
2531
2859
|
.with(basic_auth: ['john', 'doe']) do |req|
|
2532
2860
|
connection_resets += 1
|
2533
2861
|
raise Faraday::ConnectionFailed, "Test message"
|
@@ -2540,9 +2868,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2540
2868
|
assert_equal(4, connection_resets)
|
2541
2869
|
end
|
2542
2870
|
|
2543
|
-
|
2871
|
+
data("legacy_template" => [true, "_template"],
|
2872
|
+
"new_template" => [false, "_index_template"])
|
2873
|
+
def test_template_retry_install_does_not_fail(data)
|
2874
|
+
use_legacy_template_flag, endpoint = data
|
2544
2875
|
cwd = File.dirname(__FILE__)
|
2545
|
-
template_file =
|
2876
|
+
template_file = if use_legacy_template_flag
|
2877
|
+
File.join(cwd, 'test_template.json')
|
2878
|
+
else
|
2879
|
+
File.join(cwd, 'test_index_template.json')
|
2880
|
+
end
|
2546
2881
|
|
2547
2882
|
config = %{
|
2548
2883
|
host logs.google.com
|
@@ -2555,11 +2890,12 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2555
2890
|
template_file #{template_file}
|
2556
2891
|
max_retry_putting_template 3
|
2557
2892
|
fail_on_putting_template_retry_exceed false
|
2893
|
+
use_legacy_template #{use_legacy_template_flag}
|
2558
2894
|
}
|
2559
2895
|
|
2560
2896
|
connection_resets = 0
|
2561
2897
|
# check if template exists
|
2562
|
-
stub_request(:get, "https://logs.google.com:778/es
|
2898
|
+
stub_request(:get, "https://logs.google.com:778/es//#{endpoint}/logstash")
|
2563
2899
|
.with(basic_auth: ['john', 'doe']) do |req|
|
2564
2900
|
connection_resets += 1
|
2565
2901
|
raise Faraday::ConnectionFailed, "Test message"
|
@@ -2570,9 +2906,17 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2570
2906
|
assert_equal(4, connection_resets)
|
2571
2907
|
end
|
2572
2908
|
|
2573
|
-
|
2909
|
+
data("legacy_template" => [true, "_template"],
|
2910
|
+
"new_template" => [false, "_index_template"])
|
2911
|
+
def test_templates_create(data)
|
2912
|
+
use_legacy_template_flag, endpoint = data
|
2574
2913
|
cwd = File.dirname(__FILE__)
|
2575
|
-
template_file =
|
2914
|
+
template_file = if use_legacy_template_flag
|
2915
|
+
File.join(cwd, 'test_template.json')
|
2916
|
+
else
|
2917
|
+
File.join(cwd, 'test_index_template.json')
|
2918
|
+
end
|
2919
|
+
|
2576
2920
|
config = %{
|
2577
2921
|
host logs.google.com
|
2578
2922
|
port 777
|
@@ -2581,43 +2925,52 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2581
2925
|
user john
|
2582
2926
|
password doe
|
2583
2927
|
templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
|
2928
|
+
use_legacy_template #{use_legacy_template_flag}
|
2584
2929
|
}
|
2585
2930
|
|
2586
2931
|
stub_request(:head, "https://logs.google.com:777/es//").
|
2587
2932
|
with(basic_auth: ['john', 'doe']).
|
2588
2933
|
to_return(:status => 200, :body => "", :headers => {})
|
2589
2934
|
# check if template exists
|
2590
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2935
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash1").
|
2591
2936
|
with(basic_auth: ['john', 'doe']).
|
2592
2937
|
to_return(:status => 404, :body => "", :headers => {})
|
2593
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2938
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash2").
|
2594
2939
|
with(basic_auth: ['john', 'doe']).
|
2595
2940
|
to_return(:status => 404, :body => "", :headers => {})
|
2596
2941
|
|
2597
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2942
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash3").
|
2598
2943
|
with(basic_auth: ['john', 'doe']).
|
2599
2944
|
to_return(:status => 200, :body => "", :headers => {}) #exists
|
2600
2945
|
|
2601
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2946
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1").
|
2602
2947
|
with(basic_auth: ['john', 'doe']).
|
2603
2948
|
to_return(:status => 200, :body => "", :headers => {})
|
2604
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2949
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2").
|
2605
2950
|
with(basic_auth: ['john', 'doe']).
|
2606
2951
|
to_return(:status => 200, :body => "", :headers => {})
|
2607
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2952
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash3").
|
2608
2953
|
with(basic_auth: ['john', 'doe']).
|
2609
2954
|
to_return(:status => 200, :body => "", :headers => {})
|
2610
2955
|
|
2611
2956
|
driver(config)
|
2612
2957
|
|
2613
|
-
assert_requested( :put, "https://logs.google.com:777/es
|
2614
|
-
assert_requested( :put, "https://logs.google.com:777/es
|
2615
|
-
assert_not_requested(:put, "https://logs.google.com:777/es
|
2958
|
+
assert_requested( :put, "https://logs.google.com:777/es//#{endpoint}/logstash1", times: 1)
|
2959
|
+
assert_requested( :put, "https://logs.google.com:777/es//#{endpoint}/logstash2", times: 1)
|
2960
|
+
assert_not_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash3") #exists
|
2616
2961
|
end
|
2617
2962
|
|
2618
|
-
|
2963
|
+
data("legacy_template" => [true, "_template"],
|
2964
|
+
"new_template" => [false, "_index_template"])
|
2965
|
+
def test_templates_overwrite(data)
|
2966
|
+
use_legacy_template_flag, endpoint = data
|
2619
2967
|
cwd = File.dirname(__FILE__)
|
2620
|
-
template_file =
|
2968
|
+
template_file = if use_legacy_template_flag
|
2969
|
+
File.join(cwd, 'test_template.json')
|
2970
|
+
else
|
2971
|
+
File.join(cwd, 'test_index_template.json')
|
2972
|
+
end
|
2973
|
+
|
2621
2974
|
config = %{
|
2622
2975
|
host logs.google.com
|
2623
2976
|
port 777
|
@@ -2627,42 +2980,50 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2627
2980
|
password doe
|
2628
2981
|
templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
|
2629
2982
|
template_overwrite true
|
2983
|
+
use_legacy_template #{use_legacy_template_flag}
|
2630
2984
|
}
|
2631
2985
|
|
2632
2986
|
stub_request(:head, "https://logs.google.com:777/es//").
|
2633
2987
|
with(basic_auth: ['john', 'doe']).
|
2634
2988
|
to_return(:status => 200, :body => "", :headers => {})
|
2635
2989
|
# check if template exists
|
2636
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2990
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash1").
|
2637
2991
|
with(basic_auth: ['john', 'doe']).
|
2638
2992
|
to_return(:status => 200, :body => "", :headers => {})
|
2639
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2993
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash2").
|
2640
2994
|
with(basic_auth: ['john', 'doe']).
|
2641
2995
|
to_return(:status => 200, :body => "", :headers => {})
|
2642
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2996
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash3").
|
2643
2997
|
with(basic_auth: ['john', 'doe']).
|
2644
2998
|
to_return(:status => 200, :body => "", :headers => {}) #exists
|
2645
2999
|
|
2646
|
-
stub_request(:put, "https://logs.google.com:777/es
|
3000
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1").
|
2647
3001
|
with(basic_auth: ['john', 'doe']).
|
2648
3002
|
to_return(:status => 200, :body => "", :headers => {})
|
2649
|
-
stub_request(:put, "https://logs.google.com:777/es
|
3003
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2").
|
2650
3004
|
with(basic_auth: ['john', 'doe']).
|
2651
3005
|
to_return(:status => 200, :body => "", :headers => {})
|
2652
|
-
stub_request(:put, "https://logs.google.com:777/es
|
3006
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash3").
|
2653
3007
|
with(basic_auth: ['john', 'doe']).
|
2654
3008
|
to_return(:status => 200, :body => "", :headers => {})
|
2655
3009
|
|
2656
3010
|
driver(config)
|
2657
3011
|
|
2658
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2659
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2660
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
3012
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1", times: 1)
|
3013
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2", times: 1)
|
3014
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash3", times: 1)
|
2661
3015
|
end
|
2662
3016
|
|
2663
|
-
|
3017
|
+
data("legacy_template" => [true, "_template"],
|
3018
|
+
"new_template" => [false, "_index_template"])
|
3019
|
+
def test_templates_are_also_used(data)
|
3020
|
+
use_legacy_template_flag, endpoint = data
|
2664
3021
|
cwd = File.dirname(__FILE__)
|
2665
|
-
template_file =
|
3022
|
+
template_file = if use_legacy_template_flag
|
3023
|
+
File.join(cwd, 'test_template.json')
|
3024
|
+
else
|
3025
|
+
File.join(cwd, 'test_index_template.json')
|
3026
|
+
end
|
2666
3027
|
|
2667
3028
|
config = %{
|
2668
3029
|
host logs.google.com
|
@@ -2674,43 +3035,52 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2674
3035
|
template_name logstash
|
2675
3036
|
template_file #{template_file}
|
2676
3037
|
templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}" }
|
3038
|
+
use_legacy_template #{use_legacy_template_flag}
|
2677
3039
|
}
|
2678
3040
|
# connection start
|
2679
3041
|
stub_request(:head, "https://logs.google.com:777/es//").
|
2680
3042
|
with(basic_auth: ['john', 'doe']).
|
2681
3043
|
to_return(:status => 200, :body => "", :headers => {})
|
2682
3044
|
# check if template exists
|
2683
|
-
stub_request(:get, "https://logs.google.com:777/es
|
3045
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
2684
3046
|
with(basic_auth: ['john', 'doe']).
|
2685
3047
|
to_return(:status => 404, :body => "", :headers => {})
|
2686
|
-
stub_request(:get, "https://logs.google.com:777/es
|
3048
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash1").
|
2687
3049
|
with(basic_auth: ['john', 'doe']).
|
2688
3050
|
to_return(:status => 404, :body => "", :headers => {})
|
2689
|
-
stub_request(:get, "https://logs.google.com:777/es
|
3051
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash2").
|
2690
3052
|
with(basic_auth: ['john', 'doe']).
|
2691
3053
|
to_return(:status => 404, :body => "", :headers => {})
|
2692
3054
|
#creation
|
2693
|
-
stub_request(:put, "https://logs.google.com:777/es
|
3055
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
2694
3056
|
with(basic_auth: ['john', 'doe']).
|
2695
3057
|
to_return(:status => 200, :body => "", :headers => {})
|
2696
|
-
stub_request(:put, "https://logs.google.com:777/es
|
3058
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1").
|
2697
3059
|
with(basic_auth: ['john', 'doe']).
|
2698
3060
|
to_return(:status => 200, :body => "", :headers => {})
|
2699
|
-
stub_request(:put, "https://logs.google.com:777/es
|
3061
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2").
|
2700
3062
|
with(basic_auth: ['john', 'doe']).
|
2701
3063
|
to_return(:status => 200, :body => "", :headers => {})
|
2702
3064
|
|
2703
3065
|
driver(config)
|
2704
3066
|
|
2705
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
3067
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
2706
3068
|
|
2707
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2708
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
3069
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1")
|
3070
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2")
|
2709
3071
|
end
|
2710
3072
|
|
2711
|
-
|
3073
|
+
data("legacy_template" => [true, "_template"],
|
3074
|
+
"new_template" => [false, "_index_template"])
|
3075
|
+
def test_templates_can_be_partially_created_if_error_occurs(data)
|
3076
|
+
use_legacy_template_flag, endpoint = data
|
2712
3077
|
cwd = File.dirname(__FILE__)
|
2713
|
-
template_file =
|
3078
|
+
template_file = if use_legacy_template_flag
|
3079
|
+
File.join(cwd, 'test_template.json')
|
3080
|
+
else
|
3081
|
+
File.join(cwd, 'test_index_template.json')
|
3082
|
+
end
|
3083
|
+
|
2714
3084
|
config = %{
|
2715
3085
|
host logs.google.com
|
2716
3086
|
port 777
|
@@ -2719,22 +3089,23 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2719
3089
|
user john
|
2720
3090
|
password doe
|
2721
3091
|
templates {"logstash1":"#{template_file}", "logstash2":"/abc" }
|
3092
|
+
use_legacy_template #{use_legacy_template_flag}
|
2722
3093
|
}
|
2723
3094
|
stub_request(:head, "https://logs.google.com:777/es//").
|
2724
3095
|
with(basic_auth: ['john', 'doe']).
|
2725
3096
|
to_return(:status => 200, :body => "", :headers => {})
|
2726
3097
|
# check if template exists
|
2727
|
-
stub_request(:get, "https://logs.google.com:777/es
|
3098
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash1").
|
2728
3099
|
with(basic_auth: ['john', 'doe']).
|
2729
3100
|
to_return(:status => 404, :body => "", :headers => {})
|
2730
|
-
stub_request(:get, "https://logs.google.com:777/es
|
3101
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash2").
|
2731
3102
|
with(basic_auth: ['john', 'doe']).
|
2732
3103
|
to_return(:status => 404, :body => "", :headers => {})
|
2733
3104
|
|
2734
|
-
stub_request(:put, "https://logs.google.com:777/es
|
3105
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1").
|
2735
3106
|
with(basic_auth: ['john', 'doe']).
|
2736
3107
|
to_return(:status => 200, :body => "", :headers => {})
|
2737
|
-
stub_request(:put, "https://logs.google.com:777/es
|
3108
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2").
|
2738
3109
|
with(basic_auth: ['john', 'doe']).
|
2739
3110
|
to_return(:status => 200, :body => "", :headers => {})
|
2740
3111
|
|
@@ -2742,8 +3113,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2742
3113
|
driver(config)
|
2743
3114
|
}
|
2744
3115
|
|
2745
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2746
|
-
assert_not_requested(:put, "https://logs.google.com:777/es
|
3116
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1", times: 1)
|
3117
|
+
assert_not_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2")
|
2747
3118
|
end
|
2748
3119
|
|
2749
3120
|
def test_legacy_hosts_list
|