fluent-plugin-elasticsearch 4.1.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/linux.yml +1 -1
- data/.github/workflows/macos.yml +1 -1
- data/.github/workflows/windows.yml +1 -1
- data/History.md +19 -0
- data/README.ElasticsearchGenID.md +2 -2
- data/README.md +66 -14
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_index_template.rb +49 -17
- data/lib/fluent/plugin/out_elasticsearch.rb +32 -13
- data/test/plugin/test_in_elasticsearch.rb +14 -9
- data/test/plugin/test_index_alias_template.json +11 -0
- data/test/plugin/test_index_template.json +25 -0
- data/test/plugin/test_out_elasticsearch.rb +656 -348
- metadata +9 -5
@@ -20,6 +20,11 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
20
20
|
@driver = nil
|
21
21
|
log = Fluent::Engine.log
|
22
22
|
log.out.logs.slice!(0, log.out.logs.length)
|
23
|
+
@http_method = if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("7.9.0")
|
24
|
+
:post
|
25
|
+
else
|
26
|
+
:get
|
27
|
+
end
|
23
28
|
end
|
24
29
|
|
25
30
|
def driver(conf='')
|
@@ -313,7 +318,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
313
318
|
end
|
314
319
|
|
315
320
|
def test_emit
|
316
|
-
stub_request(
|
321
|
+
stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
|
317
322
|
with(body: "{\"sort\":[\"_doc\"]}").
|
318
323
|
to_return(status: 200, body: sample_response.to_s,
|
319
324
|
headers: {'Content-Type' => 'application/json'})
|
@@ -328,7 +333,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
328
333
|
|
329
334
|
def test_emit_with_custom_index_name
|
330
335
|
index_name = "logstash"
|
331
|
-
stub_request(
|
336
|
+
stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
|
332
337
|
with(body: "{\"sort\":[\"_doc\"]}").
|
333
338
|
to_return(status: 200, body: sample_response(index_name).to_s,
|
334
339
|
headers: {'Content-Type' => 'application/json'})
|
@@ -343,7 +348,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
343
348
|
|
344
349
|
def test_emit_with_parse_timestamp
|
345
350
|
index_name = "fluentd"
|
346
|
-
stub_request(
|
351
|
+
stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
|
347
352
|
with(body: "{\"sort\":[\"_doc\"]}").
|
348
353
|
to_return(status: 200, body: sample_response(index_name).to_s,
|
349
354
|
headers: {'Content-Type' => 'application/json'})
|
@@ -361,7 +366,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
361
366
|
|
362
367
|
def test_emit_with_parse_timestamp_and_timstamp_format
|
363
368
|
index_name = "fluentd"
|
364
|
-
stub_request(
|
369
|
+
stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
|
365
370
|
with(body: "{\"sort\":[\"_doc\"]}").
|
366
371
|
to_return(status: 200, body: sample_response(index_name).to_s,
|
367
372
|
headers: {'Content-Type' => 'application/json'})
|
@@ -380,7 +385,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
380
385
|
end
|
381
386
|
|
382
387
|
def test_emit_with_docinfo
|
383
|
-
stub_request(
|
388
|
+
stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
|
384
389
|
with(body: "{\"sort\":[\"_doc\"]}").
|
385
390
|
to_return(status: 200, body: sample_response.to_s,
|
386
391
|
headers: {'Content-Type' => 'application/json'})
|
@@ -399,11 +404,11 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
399
404
|
end
|
400
405
|
|
401
406
|
def test_emit_with_slices
|
402
|
-
stub_request(
|
407
|
+
stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
|
403
408
|
with(body: "{\"sort\":[\"_doc\"],\"slice\":{\"id\":0,\"max\":2}}").
|
404
409
|
to_return(status: 200, body: sample_response.to_s,
|
405
410
|
headers: {'Content-Type' => 'application/json'})
|
406
|
-
stub_request(
|
411
|
+
stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
|
407
412
|
with(body: "{\"sort\":[\"_doc\"],\"slice\":{\"id\":1,\"max\":2}}").
|
408
413
|
to_return(status: 200, body: sample_response.to_s,
|
409
414
|
headers: {'Content-Type' => 'application/json'})
|
@@ -419,12 +424,12 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
419
424
|
end
|
420
425
|
|
421
426
|
def test_emit_with_size
|
422
|
-
stub_request(
|
427
|
+
stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1").
|
423
428
|
with(body: "{\"sort\":[\"_doc\"]}").
|
424
429
|
to_return(status: 200, body: sample_scroll_response.to_s,
|
425
430
|
headers: {'Content-Type' => 'application/json'})
|
426
431
|
connection = 0
|
427
|
-
scroll_request = stub_request(
|
432
|
+
scroll_request = stub_request(@http_method, "http://localhost:9200/_search/scroll?scroll=1m").
|
428
433
|
with(
|
429
434
|
body: "{\"scroll_id\":\"WomkoUKG0QPB679Ulo6TqQgh3pIGRUmrl9qXXGK3EeiQh9rbYNasTkspZQcJ01uz\"}") do
|
430
435
|
connection += 1
|
@@ -0,0 +1,25 @@
|
|
1
|
+
{
|
2
|
+
"index_patterns": "te*",
|
3
|
+
"template": {
|
4
|
+
"settings": {
|
5
|
+
"number_of_shards": 1
|
6
|
+
},
|
7
|
+
"mappings": {
|
8
|
+
"type1": {
|
9
|
+
"_source": {
|
10
|
+
"enabled": false
|
11
|
+
},
|
12
|
+
"properties": {
|
13
|
+
"host_name": {
|
14
|
+
"type": "string",
|
15
|
+
"index": "not_analyzed"
|
16
|
+
},
|
17
|
+
"created_at": {
|
18
|
+
"type": "date",
|
19
|
+
"format": "EEE MMM dd HH:mm:ss Z YYYY"
|
20
|
+
}
|
21
|
+
}
|
22
|
+
}
|
23
|
+
}
|
24
|
+
}
|
25
|
+
}
|
@@ -252,6 +252,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
252
252
|
assert_false instance.compression
|
253
253
|
assert_equal :no_compression, instance.compression_level
|
254
254
|
assert_true instance.http_backend_excon_nonblock
|
255
|
+
assert_equal({}, instance.api_key_header)
|
255
256
|
end
|
256
257
|
|
257
258
|
test 'configure compression' do
|
@@ -374,7 +375,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
374
375
|
end
|
375
376
|
end
|
376
377
|
|
377
|
-
|
378
|
+
data("legacy_template" => [true, "_template"],
|
379
|
+
"new_template" => [false, "_index_template"])
|
380
|
+
test 'valid configuration of index lifecycle management' do |data|
|
381
|
+
use_legacy_template_flag, endpoint = data
|
378
382
|
cwd = File.dirname(__FILE__)
|
379
383
|
template_file = File.join(cwd, 'test_template.json')
|
380
384
|
|
@@ -382,8 +386,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
382
386
|
enable_ilm true
|
383
387
|
template_name logstash
|
384
388
|
template_file #{template_file}
|
389
|
+
use_legacy_template #{use_legacy_template_flag}
|
385
390
|
}
|
386
|
-
stub_request(:get, "http://localhost:9200/
|
391
|
+
stub_request(:get, "http://localhost:9200/#{endpoint}/fluentd").
|
387
392
|
to_return(status: 200, body: "", headers: {})
|
388
393
|
stub_request(:head, "http://localhost:9200/_alias/fluentd").
|
389
394
|
to_return(status: 404, body: "", headers: {})
|
@@ -406,7 +411,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
406
411
|
}
|
407
412
|
end
|
408
413
|
|
409
|
-
|
414
|
+
data("legacy_template" => [true, "_template"],
|
415
|
+
"new_template" => [false, "_index_template"])
|
416
|
+
test 'valid configuration of overwriting ilm_policy' do |data|
|
417
|
+
use_legacy_template_flag, endpoint = data
|
410
418
|
cwd = File.dirname(__FILE__)
|
411
419
|
template_file = File.join(cwd, 'test_template.json')
|
412
420
|
|
@@ -416,8 +424,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
416
424
|
template_file #{template_file}
|
417
425
|
ilm_policy_overwrite true
|
418
426
|
ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"75gb","max_age": "50d"}}}}}}
|
427
|
+
use_legacy_template #{use_legacy_template_flag}
|
419
428
|
}
|
420
|
-
stub_request(:get, "http://localhost:9200/
|
429
|
+
stub_request(:get, "http://localhost:9200/#{endpoint}/fluentd").
|
421
430
|
to_return(status: 200, body: "", headers: {})
|
422
431
|
stub_request(:head, "http://localhost:9200/_alias/fluentd").
|
423
432
|
to_return(status: 404, body: "", headers: {})
|
@@ -707,7 +716,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
707
716
|
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
708
717
|
def detect_es_major_version
|
709
718
|
@_es_info ||= client.info
|
710
|
-
@_es_info
|
719
|
+
unless version = @_es_info.dig("version", "number")
|
720
|
+
version = @default_elasticsearch_version
|
721
|
+
end
|
722
|
+
version.to_i
|
711
723
|
end
|
712
724
|
CODE
|
713
725
|
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
@@ -753,7 +765,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
753
765
|
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
754
766
|
def detect_es_major_version
|
755
767
|
@_es_info ||= client.info
|
756
|
-
@_es_info
|
768
|
+
unless version = @_es_info.dig("version", "number")
|
769
|
+
version = @default_elasticsearch_version
|
770
|
+
end
|
771
|
+
version.to_i
|
757
772
|
end
|
758
773
|
CODE
|
759
774
|
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
@@ -799,7 +814,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
799
814
|
end
|
800
815
|
end
|
801
816
|
|
802
|
-
|
817
|
+
data("legacy_template" => [true, "_template"],
|
818
|
+
"new_template" => [false, "_index_template"])
|
819
|
+
def test_template_already_present(data)
|
820
|
+
use_legacy_template_flag, endpoint = data
|
803
821
|
config = %{
|
804
822
|
host logs.google.com
|
805
823
|
port 777
|
@@ -809,6 +827,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
809
827
|
password doe
|
810
828
|
template_name logstash
|
811
829
|
template_file /abc123
|
830
|
+
use_legacy_template #{use_legacy_template_flag}
|
812
831
|
}
|
813
832
|
|
814
833
|
# connection start
|
@@ -816,18 +835,25 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
816
835
|
with(basic_auth: ['john', 'doe']).
|
817
836
|
to_return(:status => 200, :body => "", :headers => {})
|
818
837
|
# check if template exists
|
819
|
-
stub_request(:get, "https://logs.google.com:777/es
|
838
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
820
839
|
with(basic_auth: ['john', 'doe']).
|
821
840
|
to_return(:status => 200, :body => "", :headers => {})
|
822
841
|
|
823
842
|
driver(config)
|
824
843
|
|
825
|
-
assert_not_requested(:put, "https://logs.google.com:777/es
|
844
|
+
assert_not_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash")
|
826
845
|
end
|
827
846
|
|
828
|
-
|
847
|
+
data("legacy_template" => [true, "_template"],
|
848
|
+
"new_template" => [false, "_index_template"])
|
849
|
+
def test_template_create(data)
|
850
|
+
use_legacy_template_flag, endpoint = data
|
829
851
|
cwd = File.dirname(__FILE__)
|
830
|
-
template_file =
|
852
|
+
template_file = if use_legacy_template_flag
|
853
|
+
File.join(cwd, 'test_template.json')
|
854
|
+
else
|
855
|
+
File.join(cwd, 'test_index_template.json')
|
856
|
+
end
|
831
857
|
|
832
858
|
config = %{
|
833
859
|
host logs.google.com
|
@@ -838,6 +864,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
838
864
|
password doe
|
839
865
|
template_name logstash
|
840
866
|
template_file #{template_file}
|
867
|
+
use_legacy_template #{use_legacy_template_flag}
|
841
868
|
}
|
842
869
|
|
843
870
|
# connection start
|
@@ -845,22 +872,29 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
845
872
|
with(basic_auth: ['john', 'doe']).
|
846
873
|
to_return(:status => 200, :body => "", :headers => {})
|
847
874
|
# check if template exists
|
848
|
-
stub_request(:get, "https://logs.google.com:777/es
|
875
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
849
876
|
with(basic_auth: ['john', 'doe']).
|
850
877
|
to_return(:status => 404, :body => "", :headers => {})
|
851
878
|
# creation
|
852
|
-
stub_request(:put, "https://logs.google.com:777/es
|
879
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
853
880
|
with(basic_auth: ['john', 'doe']).
|
854
881
|
to_return(:status => 200, :body => "", :headers => {})
|
855
882
|
|
856
883
|
driver(config)
|
857
884
|
|
858
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
885
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
859
886
|
end
|
860
887
|
|
861
|
-
|
888
|
+
data("legacy_template" => [true, "_template"],
|
889
|
+
"new_template" => [false, "_index_template"])
|
890
|
+
def test_template_create_with_rollover_index_and_template_related_placeholders(data)
|
891
|
+
use_legacy_template_flag, endpoint = data
|
862
892
|
cwd = File.dirname(__FILE__)
|
863
|
-
template_file =
|
893
|
+
template_file = if use_legacy_template_flag
|
894
|
+
File.join(cwd, 'test_template.json')
|
895
|
+
else
|
896
|
+
File.join(cwd, 'test_index_template.json')
|
897
|
+
end
|
864
898
|
config = %{
|
865
899
|
host logs.google.com
|
866
900
|
port 777
|
@@ -874,6 +908,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
874
908
|
index_date_pattern ""
|
875
909
|
index_name fluentd-${tag}
|
876
910
|
deflector_alias myapp_deflector-${tag}
|
911
|
+
use_legacy_template #{use_legacy_template_flag}
|
877
912
|
}
|
878
913
|
|
879
914
|
# connection start
|
@@ -881,11 +916,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
881
916
|
with(basic_auth: ['john', 'doe']).
|
882
917
|
to_return(:status => 200, :body => "", :headers => {})
|
883
918
|
# check if template exists
|
884
|
-
stub_request(:get, "https://logs.google.com:777/es
|
919
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-test.template").
|
885
920
|
with(basic_auth: ['john', 'doe']).
|
886
921
|
to_return(:status => 404, :body => "", :headers => {})
|
887
922
|
# create template
|
888
|
-
stub_request(:put, "https://logs.google.com:777/es
|
923
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-test.template").
|
889
924
|
with(basic_auth: ['john', 'doe']).
|
890
925
|
to_return(:status => 200, :body => "", :headers => {})
|
891
926
|
# check if alias exists
|
@@ -915,9 +950,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
915
950
|
assert_requested(elastic_request)
|
916
951
|
end
|
917
952
|
|
918
|
-
|
953
|
+
data("legacy_template" => [true, "_template"],
|
954
|
+
"new_template" => [false, "_index_template"])
|
955
|
+
def test_template_create_with_rollover_index_and_template_related_placeholders_with_truncating_caches(data)
|
956
|
+
use_legacy_template_flag, endpoint = data
|
919
957
|
cwd = File.dirname(__FILE__)
|
920
|
-
template_file =
|
958
|
+
template_file = if use_legacy_template_flag
|
959
|
+
File.join(cwd, 'test_template.json')
|
960
|
+
else
|
961
|
+
File.join(cwd, 'test_index_template.json')
|
962
|
+
end
|
921
963
|
config = %{
|
922
964
|
host logs.google.com
|
923
965
|
port 777
|
@@ -932,6 +974,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
932
974
|
index_name fluentd-${tag}
|
933
975
|
deflector_alias myapp_deflector-${tag}
|
934
976
|
truncate_caches_interval 2s
|
977
|
+
use_legacy_template #{use_legacy_template_flag}
|
935
978
|
}
|
936
979
|
|
937
980
|
# connection start
|
@@ -939,11 +982,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
939
982
|
with(basic_auth: ['john', 'doe']).
|
940
983
|
to_return(:status => 200, :body => "", :headers => {})
|
941
984
|
# check if template exists
|
942
|
-
stub_request(:get, "https://logs.google.com:777/es
|
985
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-test.template").
|
943
986
|
with(basic_auth: ['john', 'doe']).
|
944
987
|
to_return(:status => 404, :body => "", :headers => {})
|
945
988
|
# create template
|
946
|
-
stub_request(:put, "https://logs.google.com:777/es
|
989
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-test.template").
|
947
990
|
with(basic_auth: ['john', 'doe']).
|
948
991
|
to_return(:status => 200, :body => "", :headers => {})
|
949
992
|
# check if alias exists
|
@@ -988,9 +1031,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
988
1031
|
end
|
989
1032
|
end
|
990
1033
|
|
991
|
-
|
1034
|
+
data("legacy_template" => [true, "_template"],
|
1035
|
+
"new_template" => [false, "_index_template"])
|
1036
|
+
def test_template_create_with_rollover_index_and_default_ilm(data)
|
1037
|
+
use_legacy_template_flag, endpoint = data
|
992
1038
|
cwd = File.dirname(__FILE__)
|
993
|
-
template_file =
|
1039
|
+
template_file = if use_legacy_template_flag
|
1040
|
+
File.join(cwd, 'test_template.json')
|
1041
|
+
else
|
1042
|
+
File.join(cwd, 'test_index_template.json')
|
1043
|
+
end
|
994
1044
|
|
995
1045
|
config = %{
|
996
1046
|
host logs.google.com
|
@@ -1004,6 +1054,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1004
1054
|
index_date_pattern now/w{xxxx.ww}
|
1005
1055
|
index_name logstash
|
1006
1056
|
enable_ilm true
|
1057
|
+
use_legacy_template #{use_legacy_template_flag}
|
1007
1058
|
}
|
1008
1059
|
|
1009
1060
|
# connection start
|
@@ -1011,24 +1062,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1011
1062
|
with(basic_auth: ['john', 'doe']).
|
1012
1063
|
to_return(:status => 200, :body => "", :headers => {})
|
1013
1064
|
# check if template exists
|
1014
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1065
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1015
1066
|
with(basic_auth: ['john', 'doe']).
|
1016
1067
|
to_return(:status => 404, :body => "", :headers => {})
|
1017
1068
|
# creation
|
1018
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1069
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1019
1070
|
with(basic_auth: ['john', 'doe']).
|
1020
1071
|
to_return(:status => 200, :body => "", :headers => {})
|
1021
1072
|
# check if alias exists
|
1022
1073
|
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1023
1074
|
with(basic_auth: ['john', 'doe']).
|
1024
1075
|
to_return(:status => 404, :body => "", :headers => {})
|
1025
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1076
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1026
1077
|
with(basic_auth: ['john', 'doe']).
|
1027
1078
|
to_return(status: 404, body: "", headers: {})
|
1028
|
-
|
1029
|
-
|
1030
|
-
|
1031
|
-
|
1079
|
+
if use_legacy_template_flag
|
1080
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1081
|
+
with(basic_auth: ['john', 'doe'],
|
1082
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
|
1083
|
+
to_return(status: 200, body: "", headers: {})
|
1084
|
+
else
|
1085
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1086
|
+
with(basic_auth: ['john', 'doe'],
|
1087
|
+
body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
|
1088
|
+
to_return(status: 200, body: "", headers: {})
|
1089
|
+
end
|
1032
1090
|
# put the alias for the index
|
1033
1091
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1034
1092
|
with(basic_auth: ['john', 'doe']).
|
@@ -1050,12 +1108,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1050
1108
|
|
1051
1109
|
driver(config)
|
1052
1110
|
|
1053
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1111
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
1054
1112
|
end
|
1055
1113
|
|
1056
|
-
|
1114
|
+
data("legacy_template" => [true, "_template"],
|
1115
|
+
"new_template" => [false, "_index_template"])
|
1116
|
+
def test_template_create_with_rollover_index_and_default_ilm_on_logstash_format(data)
|
1117
|
+
use_legacy_template_flag, endpoint = data
|
1057
1118
|
cwd = File.dirname(__FILE__)
|
1058
|
-
template_file =
|
1119
|
+
template_file = if use_legacy_template_flag
|
1120
|
+
File.join(cwd, 'test_template.json')
|
1121
|
+
else
|
1122
|
+
File.join(cwd, 'test_index_template.json')
|
1123
|
+
end
|
1059
1124
|
|
1060
1125
|
config = %{
|
1061
1126
|
host logs.google.com
|
@@ -1070,6 +1135,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1070
1135
|
enable_ilm true
|
1071
1136
|
logstash_format true
|
1072
1137
|
application_name log
|
1138
|
+
use_legacy_template #{use_legacy_template_flag}
|
1073
1139
|
}
|
1074
1140
|
|
1075
1141
|
date_str = Time.now.strftime("%Y.%m.%d")
|
@@ -1078,32 +1144,39 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1078
1144
|
with(basic_auth: ['john', 'doe']).
|
1079
1145
|
to_return(:status => 200, :body => "", :headers => {})
|
1080
1146
|
# check if template exists
|
1081
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1147
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}").
|
1082
1148
|
with(basic_auth: ['john', 'doe']).
|
1083
1149
|
to_return(:status => 404, :body => "", :headers => {})
|
1084
1150
|
# creation
|
1085
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1151
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}").
|
1086
1152
|
with(basic_auth: ['john', 'doe']).
|
1087
1153
|
to_return(:status => 200, :body => "", :headers => {})
|
1088
1154
|
# check if alias exists
|
1089
|
-
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash
|
1155
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-#{date_str}").
|
1090
1156
|
with(basic_auth: ['john', 'doe']).
|
1091
1157
|
to_return(:status => 404, :body => "", :headers => {})
|
1092
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1158
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}").
|
1093
1159
|
with(basic_auth: ['john', 'doe']).
|
1094
1160
|
to_return(status: 404, body: "", headers: {})
|
1095
|
-
|
1096
|
-
|
1097
|
-
|
1098
|
-
|
1161
|
+
if use_legacy_template_flag
|
1162
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}").
|
1163
|
+
with(basic_auth: ['john', 'doe'],
|
1164
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-log-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-log-#{date_str}-*\",\"order\":53}").
|
1165
|
+
to_return(status: 200, body: "", headers: {})
|
1166
|
+
else
|
1167
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}").
|
1168
|
+
with(basic_auth: ['john', 'doe'],
|
1169
|
+
body: "{\"index_patterns\":\"logstash-log-2020.09.21-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-2020.09.21\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":103}").
|
1170
|
+
to_return(status: 200, body: "", headers: {})
|
1171
|
+
end
|
1099
1172
|
# put the alias for the index
|
1100
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log
|
1173
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-#{date_str}-000001%3E").
|
1101
1174
|
with(basic_auth: ['john', 'doe']).
|
1102
1175
|
to_return(:status => 200, :body => "", :headers => {})
|
1103
1176
|
|
1104
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log
|
1177
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-log-#{date_str}-000001%3E/#{alias_endpoint}/logstash-#{date_str}").
|
1105
1178
|
with(basic_auth: ['john', 'doe'],
|
1106
|
-
body: "{\"aliases\":{\"logstash
|
1179
|
+
body: "{\"aliases\":{\"logstash-#{date_str}\":{\"is_write_index\":true}}}").
|
1107
1180
|
to_return(status: 200, body: "", headers: {})
|
1108
1181
|
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1109
1182
|
with(basic_auth: ['john', 'doe']).
|
@@ -1122,14 +1195,21 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1122
1195
|
driver.run(default_tag: 'test') do
|
1123
1196
|
driver.feed(sample_record)
|
1124
1197
|
end
|
1125
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1198
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-#{date_str}", times: 1)
|
1126
1199
|
|
1127
1200
|
assert_requested(elastic_request)
|
1128
1201
|
end
|
1129
1202
|
|
1130
|
-
|
1203
|
+
data("legacy_template" => [true, "_template"],
|
1204
|
+
"new_template" => [false, "_index_template"])
|
1205
|
+
def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite(data)
|
1206
|
+
use_legacy_template_flag, endpoint = data
|
1131
1207
|
cwd = File.dirname(__FILE__)
|
1132
|
-
template_file =
|
1208
|
+
template_file = if use_legacy_template_flag
|
1209
|
+
File.join(cwd, 'test_template.json')
|
1210
|
+
else
|
1211
|
+
File.join(cwd, 'test_index_template.json')
|
1212
|
+
end
|
1133
1213
|
|
1134
1214
|
config = %{
|
1135
1215
|
host logs.google.com
|
@@ -1145,6 +1225,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1145
1225
|
enable_ilm true
|
1146
1226
|
ilm_policy_overwrite true
|
1147
1227
|
ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"60gb","max_age": "45d"}}}}}}
|
1228
|
+
use_legacy_template #{use_legacy_template_flag}
|
1148
1229
|
}
|
1149
1230
|
|
1150
1231
|
# connection start
|
@@ -1152,24 +1233,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1152
1233
|
with(basic_auth: ['john', 'doe']).
|
1153
1234
|
to_return(:status => 200, :body => "", :headers => {})
|
1154
1235
|
# check if template exists
|
1155
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1236
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1156
1237
|
with(basic_auth: ['john', 'doe']).
|
1157
1238
|
to_return(:status => 404, :body => "", :headers => {})
|
1158
1239
|
# creation
|
1159
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1240
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1160
1241
|
with(basic_auth: ['john', 'doe']).
|
1161
1242
|
to_return(:status => 200, :body => "", :headers => {})
|
1162
1243
|
# check if alias exists
|
1163
1244
|
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1164
1245
|
with(basic_auth: ['john', 'doe']).
|
1165
1246
|
to_return(:status => 404, :body => "", :headers => {})
|
1166
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1247
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1167
1248
|
with(basic_auth: ['john', 'doe']).
|
1168
1249
|
to_return(status: 404, body: "", headers: {})
|
1169
|
-
|
1170
|
-
|
1171
|
-
|
1172
|
-
|
1250
|
+
if use_legacy_template_flag
|
1251
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1252
|
+
with(basic_auth: ['john', 'doe'],
|
1253
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
|
1254
|
+
to_return(status: 200, body: "", headers: {})
|
1255
|
+
else
|
1256
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1257
|
+
with(basic_auth: ['john', 'doe'],
|
1258
|
+
body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
|
1259
|
+
to_return(status: 200, body: "", headers: {})
|
1260
|
+
end
|
1173
1261
|
# put the alias for the index
|
1174
1262
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1175
1263
|
with(basic_auth: ['john', 'doe']).
|
@@ -1191,7 +1279,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1191
1279
|
|
1192
1280
|
driver(config)
|
1193
1281
|
|
1194
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1282
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
1195
1283
|
end
|
1196
1284
|
|
1197
1285
|
def test_template_create_with_rollover_index_and_default_ilm_with_deflector_alias
|
@@ -1213,56 +1301,22 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1213
1301
|
enable_ilm true
|
1214
1302
|
}
|
1215
1303
|
|
1216
|
-
#
|
1217
|
-
|
1218
|
-
|
1219
|
-
|
1220
|
-
# check if template exists
|
1221
|
-
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
1222
|
-
with(basic_auth: ['john', 'doe']).
|
1223
|
-
to_return(:status => 404, :body => "", :headers => {})
|
1224
|
-
# creation
|
1225
|
-
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
1226
|
-
with(basic_auth: ['john', 'doe']).
|
1227
|
-
to_return(:status => 200, :body => "", :headers => {})
|
1228
|
-
# check if alias exists
|
1229
|
-
stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
|
1230
|
-
with(basic_auth: ['john', 'doe']).
|
1231
|
-
to_return(:status => 404, :body => "", :headers => {})
|
1232
|
-
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
|
1233
|
-
with(basic_auth: ['john', 'doe']).
|
1234
|
-
to_return(status: 404, body: "", headers: {})
|
1235
|
-
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
|
1236
|
-
with(basic_auth: ['john', 'doe'],
|
1237
|
-
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"myapp_deflector-*\",\"order\":51}").
|
1238
|
-
to_return(status: 200, body: "", headers: {})
|
1239
|
-
# put the alias for the index
|
1240
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1241
|
-
with(basic_auth: ['john', 'doe']).
|
1242
|
-
to_return(:status => 200, :body => "", :headers => {})
|
1243
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/myapp_deflector").
|
1244
|
-
with(basic_auth: ['john', 'doe'],
|
1245
|
-
:body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
|
1246
|
-
to_return(:status => 200, :body => "", :headers => {})
|
1247
|
-
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
1248
|
-
with(basic_auth: ['john', 'doe']).
|
1249
|
-
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
1250
|
-
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
1251
|
-
with(basic_auth: ['john', 'doe']).
|
1252
|
-
to_return(:status => 404, :body => "", :headers => {})
|
1253
|
-
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
1254
|
-
with(basic_auth: ['john', 'doe'],
|
1255
|
-
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
1256
|
-
to_return(:status => 200, :body => "", :headers => {})
|
1257
|
-
|
1258
|
-
driver(config)
|
1259
|
-
|
1260
|
-
assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_deflector", times: 1)
|
1304
|
+
# Should raise error because multiple alias indices IllegalArgument Error on executing ILM feature
|
1305
|
+
assert_raise(Fluent::ConfigError) do
|
1306
|
+
driver(config)
|
1307
|
+
end
|
1261
1308
|
end
|
1262
1309
|
|
1263
|
-
|
1310
|
+
data("legacy_template" => [true, "_template"],
|
1311
|
+
"new_template" => [false, "_index_template"])
|
1312
|
+
def test_template_create_with_rollover_index_and_default_ilm_with_empty_index_date_pattern(data)
|
1313
|
+
use_legacy_template_flag, endpoint = data
|
1264
1314
|
cwd = File.dirname(__FILE__)
|
1265
|
-
template_file =
|
1315
|
+
template_file = if use_legacy_template_flag
|
1316
|
+
File.join(cwd, 'test_template.json')
|
1317
|
+
else
|
1318
|
+
File.join(cwd, 'test_index_template.json')
|
1319
|
+
end
|
1266
1320
|
|
1267
1321
|
config = %{
|
1268
1322
|
host logs.google.com
|
@@ -1276,6 +1330,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1276
1330
|
index_date_pattern ""
|
1277
1331
|
index_name logstash
|
1278
1332
|
enable_ilm true
|
1333
|
+
use_legacy_template #{use_legacy_template_flag}
|
1279
1334
|
}
|
1280
1335
|
|
1281
1336
|
# connection start
|
@@ -1283,24 +1338,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1283
1338
|
with(basic_auth: ['john', 'doe']).
|
1284
1339
|
to_return(:status => 200, :body => "", :headers => {})
|
1285
1340
|
# check if template exists
|
1286
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1341
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1287
1342
|
with(basic_auth: ['john', 'doe']).
|
1288
1343
|
to_return(:status => 404, :body => "", :headers => {})
|
1289
1344
|
# creation
|
1290
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1345
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1291
1346
|
with(basic_auth: ['john', 'doe']).
|
1292
1347
|
to_return(:status => 200, :body => "", :headers => {})
|
1293
1348
|
# check if alias exists
|
1294
1349
|
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1295
1350
|
with(basic_auth: ['john', 'doe']).
|
1296
1351
|
to_return(:status => 404, :body => "", :headers => {})
|
1297
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1352
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1298
1353
|
with(basic_auth: ['john', 'doe']).
|
1299
1354
|
to_return(status: 404, body: "", headers: {})
|
1300
|
-
|
1301
|
-
|
1302
|
-
|
1303
|
-
|
1355
|
+
if use_legacy_template_flag
|
1356
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_deflector").
|
1357
|
+
with(basic_auth: ['john', 'doe'],
|
1358
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
|
1359
|
+
to_return(status: 200, body: "", headers: {})
|
1360
|
+
else
|
1361
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_deflector").
|
1362
|
+
with(basic_auth: ['john', 'doe'],
|
1363
|
+
body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
|
1364
|
+
to_return(status: 200, body: "", headers: {})
|
1365
|
+
end
|
1304
1366
|
# put the alias for the index
|
1305
1367
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-000001%3E").
|
1306
1368
|
with(basic_auth: ['john', 'doe']).
|
@@ -1322,12 +1384,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1322
1384
|
|
1323
1385
|
driver(config)
|
1324
1386
|
|
1325
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1387
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
1326
1388
|
end
|
1327
1389
|
|
1328
|
-
|
1390
|
+
data("legacy_template" => [true, "_template"],
|
1391
|
+
"new_template" => [false, "_index_template"])
|
1392
|
+
def test_template_create_with_rollover_index_and_custom_ilm(data)
|
1393
|
+
use_legacy_template_flag, endpoint = data
|
1329
1394
|
cwd = File.dirname(__FILE__)
|
1330
|
-
template_file =
|
1395
|
+
template_file = if use_legacy_template_flag
|
1396
|
+
File.join(cwd, 'test_template.json')
|
1397
|
+
else
|
1398
|
+
File.join(cwd, 'test_index_template.json')
|
1399
|
+
end
|
1331
1400
|
|
1332
1401
|
config = %{
|
1333
1402
|
host logs.google.com
|
@@ -1343,6 +1412,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1343
1412
|
enable_ilm true
|
1344
1413
|
index_name logstash
|
1345
1414
|
ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}
|
1415
|
+
use_legacy_template #{use_legacy_template_flag}
|
1346
1416
|
}
|
1347
1417
|
|
1348
1418
|
# connection start
|
@@ -1350,24 +1420,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1350
1420
|
with(basic_auth: ['john', 'doe']).
|
1351
1421
|
to_return(:status => 200, :body => "", :headers => {})
|
1352
1422
|
# check if template exists
|
1353
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1423
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1354
1424
|
with(basic_auth: ['john', 'doe']).
|
1355
1425
|
to_return(:status => 404, :body => "", :headers => {})
|
1356
1426
|
# creation
|
1357
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1427
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1358
1428
|
with(basic_auth: ['john', 'doe']).
|
1359
1429
|
to_return(:status => 200, :body => "", :headers => {})
|
1360
1430
|
# check if alias exists
|
1361
1431
|
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1362
1432
|
with(basic_auth: ['john', 'doe']).
|
1363
1433
|
to_return(:status => 404, :body => "", :headers => {})
|
1364
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1434
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1365
1435
|
with(basic_auth: ['john', 'doe']).
|
1366
1436
|
to_return(status: 404, body: "", headers: {})
|
1367
|
-
|
1368
|
-
|
1369
|
-
|
1370
|
-
|
1437
|
+
if use_legacy_template_flag
|
1438
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1439
|
+
with(basic_auth: ['john', 'doe'],
|
1440
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1441
|
+
to_return(status: 200, body: "", headers: {})
|
1442
|
+
else
|
1443
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1444
|
+
with(basic_auth: ['john', 'doe'],
|
1445
|
+
body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
|
1446
|
+
to_return(status: 200, body: "", headers: {})
|
1447
|
+
end
|
1371
1448
|
# put the alias for the index
|
1372
1449
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1373
1450
|
with(basic_auth: ['john', 'doe']).
|
@@ -1388,12 +1465,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1388
1465
|
|
1389
1466
|
driver(config)
|
1390
1467
|
|
1391
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1468
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
1392
1469
|
end
|
1393
1470
|
|
1394
|
-
|
1471
|
+
data("legacy_template" => [true, "_template"],
|
1472
|
+
"new_template" => [false, "_index_template"])
|
1473
|
+
def test_template_create_with_rollover_index_and_ilm_policies_and_placeholderstest_template_create_with_rollover_index_and_ilm_policies_and_placeholders(data)
|
1474
|
+
use_legacy_template_flag, endpoint = data
|
1395
1475
|
cwd = File.dirname(__FILE__)
|
1396
|
-
template_file =
|
1476
|
+
template_file = if use_legacy_template_flag
|
1477
|
+
File.join(cwd, 'test_template.json')
|
1478
|
+
else
|
1479
|
+
File.join(cwd, 'test_index_template.json')
|
1480
|
+
end
|
1397
1481
|
|
1398
1482
|
config = %{
|
1399
1483
|
host logs.google.com
|
@@ -1409,6 +1493,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1409
1493
|
enable_ilm true
|
1410
1494
|
index_name logstash
|
1411
1495
|
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
1496
|
+
use_legacy_template #{use_legacy_template_flag}
|
1412
1497
|
}
|
1413
1498
|
|
1414
1499
|
# connection start
|
@@ -1416,24 +1501,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1416
1501
|
with(basic_auth: ['john', 'doe']).
|
1417
1502
|
to_return(:status => 200, :body => "", :headers => {})
|
1418
1503
|
# check if template exists
|
1419
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1504
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1420
1505
|
with(basic_auth: ['john', 'doe']).
|
1421
1506
|
to_return(:status => 404, :body => "", :headers => {})
|
1422
1507
|
# creation
|
1423
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1508
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1424
1509
|
with(basic_auth: ['john', 'doe']).
|
1425
1510
|
to_return(:status => 200, :body => "", :headers => {})
|
1426
1511
|
# check if alias exists
|
1427
1512
|
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1428
1513
|
with(basic_auth: ['john', 'doe']).
|
1429
1514
|
to_return(:status => 404, :body => "", :headers => {})
|
1430
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1515
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1431
1516
|
with(basic_auth: ['john', 'doe']).
|
1432
1517
|
to_return(status: 404, body: "", headers: {})
|
1433
|
-
|
1434
|
-
|
1435
|
-
|
1436
|
-
|
1518
|
+
if use_legacy_template_flag
|
1519
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1520
|
+
with(basic_auth: ['john', 'doe'],
|
1521
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1522
|
+
to_return(status: 200, body: "", headers: {})
|
1523
|
+
else
|
1524
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1525
|
+
with(basic_auth: ['john', 'doe'],
|
1526
|
+
body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
|
1527
|
+
to_return(status: 200, body: "", headers: {})
|
1528
|
+
end
|
1437
1529
|
# put the alias for the index
|
1438
1530
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1439
1531
|
with(basic_auth: ['john', 'doe']).
|
@@ -1458,15 +1550,22 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1458
1550
|
driver.run(default_tag: 'test') do
|
1459
1551
|
driver.feed(sample_record)
|
1460
1552
|
end
|
1461
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1553
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
1462
1554
|
|
1463
1555
|
assert_requested(elastic_request)
|
1464
1556
|
end
|
1465
1557
|
|
1466
1558
|
class TemplateCreateWithRolloverIndexAndILMPoliciesWithPlaceholdersTest < self
|
1467
|
-
|
1559
|
+
data("legacy_template" => [true, "_template"],
|
1560
|
+
"new_template" => [false, "_index_template"])
|
1561
|
+
def test_tag_placeholder(data)
|
1562
|
+
use_legacy_template_flag, endpoint = data
|
1468
1563
|
cwd = File.dirname(__FILE__)
|
1469
|
-
template_file =
|
1564
|
+
template_file = if use_legacy_template_flag
|
1565
|
+
File.join(cwd, 'test_template.json')
|
1566
|
+
else
|
1567
|
+
File.join(cwd, 'test_index_template.json')
|
1568
|
+
end
|
1470
1569
|
|
1471
1570
|
config = %{
|
1472
1571
|
host logs.google.com
|
@@ -1482,6 +1581,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1482
1581
|
enable_ilm true
|
1483
1582
|
index_name logstash
|
1484
1583
|
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}}
|
1584
|
+
use_legacy_template #{use_legacy_template_flag}
|
1485
1585
|
}
|
1486
1586
|
|
1487
1587
|
# connection start
|
@@ -1489,24 +1589,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1489
1589
|
with(basic_auth: ['john', 'doe']).
|
1490
1590
|
to_return(:status => 200, :body => "", :headers => {})
|
1491
1591
|
# check if template exists
|
1492
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1592
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1493
1593
|
with(basic_auth: ['john', 'doe']).
|
1494
1594
|
to_return(:status => 404, :body => "", :headers => {})
|
1495
1595
|
# creation
|
1496
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1596
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1497
1597
|
with(basic_auth: ['john', 'doe']).
|
1498
1598
|
to_return(:status => 200, :body => "", :headers => {})
|
1499
1599
|
# check if alias exists
|
1500
1600
|
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1501
1601
|
with(basic_auth: ['john', 'doe']).
|
1502
1602
|
to_return(:status => 404, :body => "", :headers => {})
|
1503
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1603
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1504
1604
|
with(basic_auth: ['john', 'doe']).
|
1505
1605
|
to_return(status: 404, body: "", headers: {})
|
1506
|
-
|
1507
|
-
|
1508
|
-
|
1509
|
-
|
1606
|
+
if use_legacy_template_flag
|
1607
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1608
|
+
with(basic_auth: ['john', 'doe'],
|
1609
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myalogs\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"mylogs-*\",\"order\":51}").
|
1610
|
+
to_return(status: 200, body: "", headers: {})
|
1611
|
+
else
|
1612
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1613
|
+
with(basic_auth: ['john', 'doe'],
|
1614
|
+
body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
|
1615
|
+
to_return(status: 200, body: "", headers: {})
|
1616
|
+
end
|
1510
1617
|
# put the alias for the index
|
1511
1618
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1512
1619
|
with(basic_auth: ['john', 'doe']).
|
@@ -1531,14 +1638,21 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1531
1638
|
driver.run(default_tag: 'fluentd-policy') do
|
1532
1639
|
driver.feed(sample_record)
|
1533
1640
|
end
|
1534
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1641
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
1535
1642
|
|
1536
1643
|
assert_requested(elastic_request)
|
1537
1644
|
end
|
1538
1645
|
|
1539
|
-
|
1646
|
+
data("legacy_template" => [true, "_template"],
|
1647
|
+
"new_template" => [false, "_index_template"])
|
1648
|
+
def test_tag_placeholder_with_multiple_policies(data)
|
1649
|
+
use_legacy_template_flag, endpoint = data
|
1540
1650
|
cwd = File.dirname(__FILE__)
|
1541
|
-
template_file =
|
1651
|
+
template_file = if use_legacy_template_flag
|
1652
|
+
File.join(cwd, 'test_template.json')
|
1653
|
+
else
|
1654
|
+
File.join(cwd, 'test_index_template.json')
|
1655
|
+
end
|
1542
1656
|
|
1543
1657
|
config = %{
|
1544
1658
|
host logs.google.com
|
@@ -1554,6 +1668,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1554
1668
|
enable_ilm true
|
1555
1669
|
index_name logstash
|
1556
1670
|
ilm_policies {"fluentd-policy":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}, "fluentd-policy2":{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"80gb", "max_age":"20d"}}}}}}}
|
1671
|
+
use_legacy_template #{use_legacy_template_flag}
|
1557
1672
|
}
|
1558
1673
|
|
1559
1674
|
# connection start
|
@@ -1561,24 +1676,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1561
1676
|
with(basic_auth: ['john', 'doe']).
|
1562
1677
|
to_return(:status => 200, :body => "", :headers => {})
|
1563
1678
|
# check if template exists
|
1564
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1679
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1565
1680
|
with(basic_auth: ['john', 'doe']).
|
1566
1681
|
to_return(:status => 404, :body => "", :headers => {})
|
1567
1682
|
# creation
|
1568
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1683
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1569
1684
|
with(basic_auth: ['john', 'doe']).
|
1570
1685
|
to_return(:status => 200, :body => "", :headers => {})
|
1571
1686
|
# check if alias exists
|
1572
1687
|
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash").
|
1573
1688
|
with(basic_auth: ['john', 'doe']).
|
1574
1689
|
to_return(:status => 404, :body => "", :headers => {})
|
1575
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1690
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1576
1691
|
with(basic_auth: ['john', 'doe']).
|
1577
1692
|
to_return(status: 404, body: "", headers: {})
|
1578
|
-
|
1579
|
-
|
1580
|
-
|
1581
|
-
|
1693
|
+
if use_legacy_template_flag
|
1694
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1695
|
+
with(basic_auth: ['john', 'doe'],
|
1696
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy2\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-*\",\"order\":51}").
|
1697
|
+
to_return(status: 200, body: "", headers: {})
|
1698
|
+
else
|
1699
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1700
|
+
with(basic_auth: ['john', 'doe'],
|
1701
|
+
body: "{\"index_patterns\":\"logstash-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy2\",\"index.lifecycle.rollover_alias\":\"logstash\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":101}").
|
1702
|
+
to_return(status: 200, body: "", headers: {})
|
1703
|
+
end
|
1582
1704
|
# put the alias for the index
|
1583
1705
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1584
1706
|
with(basic_auth: ['john', 'doe']).
|
@@ -1603,15 +1725,22 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1603
1725
|
driver.run(default_tag: 'fluentd-policy2') do
|
1604
1726
|
driver.feed(sample_record)
|
1605
1727
|
end
|
1606
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1728
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
1607
1729
|
|
1608
1730
|
assert_requested(elastic_request)
|
1609
1731
|
end
|
1610
1732
|
end
|
1611
1733
|
|
1612
|
-
|
1734
|
+
data("legacy_template" => [true, "_template"],
|
1735
|
+
"new_template" => [false, "_index_template"])
|
1736
|
+
def test_template_create_with_rollover_index_and_default_ilm_and_placeholders(data)
|
1737
|
+
use_legacy_template_flag, endpoint = data
|
1613
1738
|
cwd = File.dirname(__FILE__)
|
1614
|
-
template_file =
|
1739
|
+
template_file = if use_legacy_template_flag
|
1740
|
+
File.join(cwd, 'test_template.json')
|
1741
|
+
else
|
1742
|
+
File.join(cwd, 'test_index_template.json')
|
1743
|
+
end
|
1615
1744
|
|
1616
1745
|
config = %{
|
1617
1746
|
host logs.google.com
|
@@ -1625,6 +1754,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1625
1754
|
index_date_pattern now/w{xxxx.ww}
|
1626
1755
|
index_name logstash-${tag}
|
1627
1756
|
enable_ilm true
|
1757
|
+
use_legacy_template #{use_legacy_template_flag}
|
1628
1758
|
}
|
1629
1759
|
|
1630
1760
|
# connection start
|
@@ -1632,24 +1762,31 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1632
1762
|
with(basic_auth: ['john', 'doe']).
|
1633
1763
|
to_return(:status => 200, :body => "", :headers => {})
|
1634
1764
|
# check if template exists
|
1635
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1765
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1636
1766
|
with(basic_auth: ['john', 'doe']).
|
1637
1767
|
to_return(:status => 404, :body => "", :headers => {})
|
1638
1768
|
# creation
|
1639
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1769
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
1640
1770
|
with(basic_auth: ['john', 'doe']).
|
1641
1771
|
to_return(:status => 200, :body => "", :headers => {})
|
1642
1772
|
# check if alias exists
|
1643
1773
|
stub_request(:head, "https://logs.google.com:777/es//_alias/logstash-test").
|
1644
1774
|
with(basic_auth: ['john', 'doe']).
|
1645
1775
|
to_return(:status => 404, :body => "", :headers => {})
|
1646
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1776
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash-test").
|
1647
1777
|
with(basic_auth: ['john', 'doe']).
|
1648
1778
|
to_return(status: 404, body: "", headers: {})
|
1649
|
-
|
1650
|
-
|
1651
|
-
|
1652
|
-
|
1779
|
+
if use_legacy_template_flag
|
1780
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-test").
|
1781
|
+
with(basic_auth: ['john', 'doe'],
|
1782
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-test\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"logstash-test-*\",\"order\":52}").
|
1783
|
+
to_return(status: 200, body: "", headers: {})
|
1784
|
+
else
|
1785
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash-test").
|
1786
|
+
with(basic_auth: ['john', 'doe'],
|
1787
|
+
body: "{\"index_patterns\":\"logstash-test-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"logstash-test\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
|
1788
|
+
to_return(status: 200, body: "", headers: {})
|
1789
|
+
end
|
1653
1790
|
# put the alias for the index
|
1654
1791
|
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-test-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
1655
1792
|
with(basic_auth: ['john', 'doe']).
|
@@ -1682,9 +1819,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1682
1819
|
assert_requested(elastic_request)
|
1683
1820
|
end
|
1684
1821
|
|
1685
|
-
|
1822
|
+
data("legacy_template" => [true, "_template"],
|
1823
|
+
"new_template" => [false, "_index_template"])
|
1824
|
+
def test_template_create_with_rollover_index_and_default_ilm_and_custom_and_time_placeholders(data)
|
1825
|
+
use_legacy_template_flag, endpoint = data
|
1686
1826
|
cwd = File.dirname(__FILE__)
|
1687
|
-
template_file =
|
1827
|
+
template_file = if use_legacy_template_flag
|
1828
|
+
File.join(cwd, 'test_template.json')
|
1829
|
+
else
|
1830
|
+
File.join(cwd, 'test_index_template.json')
|
1831
|
+
end
|
1688
1832
|
|
1689
1833
|
config = Fluent::Config::Element.new(
|
1690
1834
|
'ROOT', '', {
|
@@ -1700,6 +1844,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1700
1844
|
'index_date_pattern' => 'now/w{xxxx.ww}',
|
1701
1845
|
'index_name' => "${taskDef}-%Y.%m",
|
1702
1846
|
'enable_ilm' => true,
|
1847
|
+
'use_legacy_template' => use_legacy_template_flag,
|
1848
|
+
|
1703
1849
|
}, [
|
1704
1850
|
Fluent::Config::Element.new('buffer', 'tag, time, taskDef', {
|
1705
1851
|
'chunk_keys' => ['tag', 'time', 'taskDef'],
|
@@ -1715,14 +1861,21 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1715
1861
|
with(basic_auth: ['john', 'doe']).
|
1716
1862
|
to_return(:status => 200, :body => "", :headers => {})
|
1717
1863
|
# check if template exists
|
1718
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1864
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/#{task_def_value}-#{date_str}").
|
1719
1865
|
with(basic_auth: ['john', 'doe']).
|
1720
1866
|
to_return(:status => 404, :body => "", :headers => {})
|
1721
1867
|
# creation
|
1722
|
-
|
1723
|
-
|
1724
|
-
|
1725
|
-
|
1868
|
+
if use_legacy_template_flag
|
1869
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/#{task_def_value}-#{date_str}").
|
1870
|
+
with(basic_auth: ['john', 'doe'],
|
1871
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"#{task_def_value}-#{date_str}\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"#{task_def_value}-#{date_str}-*\",\"order\":52}").
|
1872
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1873
|
+
else
|
1874
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/#{task_def_value}-#{date_str}").
|
1875
|
+
with(basic_auth: ['john', 'doe'],
|
1876
|
+
body: "{\"index_patterns\":\"task_definition-2020.09-*\",\"template\":{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"task_definition-2020.09\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}}},\"priority\":102}").
|
1877
|
+
to_return(:status => 200, :body => "", :headers => {})
|
1878
|
+
end
|
1726
1879
|
# check if alias exists
|
1727
1880
|
stub_request(:head, "https://logs.google.com:777/es//_alias/#{task_def_value}-#{date_str}").
|
1728
1881
|
with(basic_auth: ['john', 'doe']).
|
@@ -1760,9 +1913,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1760
1913
|
end
|
1761
1914
|
end
|
1762
1915
|
|
1763
|
-
|
1916
|
+
data("legacy_template" => [true, "_template"],
|
1917
|
+
"new_template" => [false, "_index_template"])
|
1918
|
+
def test_custom_template_create(data)
|
1919
|
+
use_legacy_template_flag, endpoint = data
|
1764
1920
|
cwd = File.dirname(__FILE__)
|
1765
|
-
template_file =
|
1921
|
+
template_file = if use_legacy_template_flag
|
1922
|
+
File.join(cwd, 'test_alias_template.json')
|
1923
|
+
else
|
1924
|
+
File.join(cwd, 'test_index_alias_template.json')
|
1925
|
+
end
|
1766
1926
|
|
1767
1927
|
config = %{
|
1768
1928
|
host logs.google.com
|
@@ -1774,6 +1934,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1774
1934
|
template_name myapp_alias_template
|
1775
1935
|
template_file #{template_file}
|
1776
1936
|
customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
|
1937
|
+
use_legacy_template #{use_legacy_template_flag}
|
1777
1938
|
}
|
1778
1939
|
|
1779
1940
|
# connection start
|
@@ -1781,22 +1942,29 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1781
1942
|
with(basic_auth: ['john', 'doe']).
|
1782
1943
|
to_return(:status => 200, :body => "", :headers => {})
|
1783
1944
|
# check if template exists
|
1784
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1945
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
1785
1946
|
with(basic_auth: ['john', 'doe']).
|
1786
1947
|
to_return(:status => 404, :body => "", :headers => {})
|
1787
1948
|
# creation
|
1788
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1949
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
1789
1950
|
with(basic_auth: ['john', 'doe']).
|
1790
1951
|
to_return(:status => 200, :body => "", :headers => {})
|
1791
1952
|
|
1792
1953
|
driver(config)
|
1793
1954
|
|
1794
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
1955
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template", times: 1)
|
1795
1956
|
end
|
1796
1957
|
|
1797
|
-
|
1958
|
+
data("legacy_template" => [true, "_template"],
|
1959
|
+
"new_template" => [false, "_index_template"])
|
1960
|
+
def test_custom_template_create_with_customize_template_related_placeholders(data)
|
1961
|
+
use_legacy_template_flag, endpoint = data
|
1798
1962
|
cwd = File.dirname(__FILE__)
|
1799
|
-
template_file =
|
1963
|
+
template_file = if use_legacy_template_flag
|
1964
|
+
File.join(cwd, 'test_alias_template.json')
|
1965
|
+
else
|
1966
|
+
File.join(cwd, 'test_index_alias_template.json')
|
1967
|
+
end
|
1800
1968
|
|
1801
1969
|
config = %{
|
1802
1970
|
host logs.google.com
|
@@ -1808,6 +1976,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1808
1976
|
template_name myapp_alias_template-${tag}
|
1809
1977
|
template_file #{template_file}
|
1810
1978
|
customize_template {"--appid--": "${tag}-logs","--index_prefix--":"${tag}"}
|
1979
|
+
use_legacy_template #{use_legacy_template_flag}
|
1811
1980
|
}
|
1812
1981
|
|
1813
1982
|
# connection start
|
@@ -1815,11 +1984,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1815
1984
|
with(basic_auth: ['john', 'doe']).
|
1816
1985
|
to_return(:status => 200, :body => "", :headers => {})
|
1817
1986
|
# check if template exists
|
1818
|
-
stub_request(:get, "https://logs.google.com:777/es
|
1987
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template-test.template").
|
1819
1988
|
with(basic_auth: ['john', 'doe']).
|
1820
1989
|
to_return(:status => 404, :body => "", :headers => {})
|
1821
1990
|
# creation
|
1822
|
-
stub_request(:put, "https://logs.google.com:777/es
|
1991
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template-test.template").
|
1823
1992
|
with(basic_auth: ['john', 'doe']).
|
1824
1993
|
to_return(:status => 200, :body => "", :headers => {})
|
1825
1994
|
|
@@ -1833,12 +2002,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1833
2002
|
driver.feed(sample_record)
|
1834
2003
|
end
|
1835
2004
|
|
1836
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2005
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template-test.template", times: 1)
|
1837
2006
|
end
|
1838
2007
|
|
1839
|
-
|
2008
|
+
data("legacy_template" => [true, "_template"],
|
2009
|
+
"new_template" => [false, "_index_template"])
|
2010
|
+
def test_custom_template_installation_for_host_placeholder(data)
|
2011
|
+
use_legacy_template_flag, endpoint = data
|
1840
2012
|
cwd = File.dirname(__FILE__)
|
1841
|
-
template_file =
|
2013
|
+
template_file = if use_legacy_template_flag
|
2014
|
+
File.join(cwd, 'test_template.json')
|
2015
|
+
else
|
2016
|
+
File.join(cwd, 'test_index_template.json')
|
2017
|
+
end
|
1842
2018
|
|
1843
2019
|
config = %{
|
1844
2020
|
host logs-${tag}.google.com
|
@@ -1852,6 +2028,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1852
2028
|
verify_es_version_at_startup false
|
1853
2029
|
default_elasticsearch_version 6
|
1854
2030
|
customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
|
2031
|
+
use_legacy_template #{use_legacy_template_flag}
|
1855
2032
|
}
|
1856
2033
|
|
1857
2034
|
# connection start
|
@@ -1859,10 +2036,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1859
2036
|
with(basic_auth: ['john', 'doe']).
|
1860
2037
|
to_return(:status => 200, :body => "", :headers => {})
|
1861
2038
|
# check if template exists
|
1862
|
-
stub_request(:get, "https://logs-test.google.com:777/es
|
2039
|
+
stub_request(:get, "https://logs-test.google.com:777/es//#{endpoint}/logstash").
|
1863
2040
|
with(basic_auth: ['john', 'doe']).
|
1864
2041
|
to_return(:status => 404, :body => "", :headers => {})
|
1865
|
-
stub_request(:put, "https://logs-test.google.com:777/es
|
2042
|
+
stub_request(:put, "https://logs-test.google.com:777/es//#{endpoint}/logstash").
|
1866
2043
|
with(basic_auth: ['john', 'doe']).
|
1867
2044
|
to_return(status: 200, body: "", headers: {})
|
1868
2045
|
|
@@ -1874,9 +2051,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1874
2051
|
end
|
1875
2052
|
end
|
1876
2053
|
|
1877
|
-
|
2054
|
+
data("legacy_template" => [true, "_template"],
|
2055
|
+
"new_template" => [false, "_index_template"])
|
2056
|
+
def test_custom_template_with_rollover_index_create(data)
|
2057
|
+
use_legacy_template_flag, endpoint = data
|
1878
2058
|
cwd = File.dirname(__FILE__)
|
1879
|
-
template_file =
|
2059
|
+
template_file = if use_legacy_template_flag
|
2060
|
+
File.join(cwd, 'test_alias_template.json')
|
2061
|
+
else
|
2062
|
+
File.join(cwd, 'test_index_alias_template.json')
|
2063
|
+
end
|
1880
2064
|
|
1881
2065
|
config = %{
|
1882
2066
|
host logs.google.com
|
@@ -1892,6 +2076,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1892
2076
|
index_date_pattern now/w{xxxx.ww}
|
1893
2077
|
index_name mylogs
|
1894
2078
|
application_name myapp
|
2079
|
+
use_legacy_template #{use_legacy_template_flag}
|
1895
2080
|
}
|
1896
2081
|
|
1897
2082
|
# connection start
|
@@ -1899,11 +2084,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1899
2084
|
with(basic_auth: ['john', 'doe']).
|
1900
2085
|
to_return(:status => 200, :body => "", :headers => {})
|
1901
2086
|
# check if template exists
|
1902
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2087
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
1903
2088
|
with(basic_auth: ['john', 'doe']).
|
1904
2089
|
to_return(:status => 404, :body => "", :headers => {})
|
1905
2090
|
# creation
|
1906
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2091
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
1907
2092
|
with(basic_auth: ['john', 'doe']).
|
1908
2093
|
to_return(:status => 200, :body => "", :headers => {})
|
1909
2094
|
# creation of index which can rollover
|
@@ -1921,12 +2106,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1921
2106
|
|
1922
2107
|
driver(config)
|
1923
2108
|
|
1924
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2109
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template", times: 1)
|
1925
2110
|
end
|
1926
2111
|
|
1927
|
-
|
2112
|
+
data("legacy_template" => [true, "_template"],
|
2113
|
+
"new_template" => [false, "_index_template"])
|
2114
|
+
def test_custom_template_with_rollover_index_create_and_deflector_alias(data)
|
2115
|
+
use_legacy_template_flag, endpoint = data
|
1928
2116
|
cwd = File.dirname(__FILE__)
|
1929
|
-
template_file =
|
2117
|
+
template_file = if use_legacy_template_flag
|
2118
|
+
File.join(cwd, 'test_alias_template.json')
|
2119
|
+
else
|
2120
|
+
File.join(cwd, 'test_index_alias_template.json')
|
2121
|
+
end
|
1930
2122
|
|
1931
2123
|
config = %{
|
1932
2124
|
host logs.google.com
|
@@ -1943,6 +2135,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1943
2135
|
deflector_alias myapp_deflector
|
1944
2136
|
index_name mylogs
|
1945
2137
|
application_name myapp
|
2138
|
+
use_legacy_template #{use_legacy_template_flag}
|
1946
2139
|
}
|
1947
2140
|
|
1948
2141
|
# connection start
|
@@ -1950,11 +2143,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1950
2143
|
with(basic_auth: ['john', 'doe']).
|
1951
2144
|
to_return(:status => 200, :body => "", :headers => {})
|
1952
2145
|
# check if template exists
|
1953
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2146
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
1954
2147
|
with(basic_auth: ['john', 'doe']).
|
1955
2148
|
to_return(:status => 404, :body => "", :headers => {})
|
1956
2149
|
# creation
|
1957
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2150
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
1958
2151
|
with(basic_auth: ['john', 'doe']).
|
1959
2152
|
to_return(:status => 200, :body => "", :headers => {})
|
1960
2153
|
# creation of index which can rollover
|
@@ -1972,12 +2165,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1972
2165
|
|
1973
2166
|
driver(config)
|
1974
2167
|
|
1975
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2168
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template", times: 1)
|
1976
2169
|
end
|
1977
2170
|
|
1978
|
-
|
2171
|
+
data("legacy_template" => [true, "_template"],
|
2172
|
+
"new_template" => [false, "_index_template"])
|
2173
|
+
def test_custom_template_with_rollover_index_create_with_logstash_format(data)
|
2174
|
+
use_legacy_template_flag, endpoint = data
|
1979
2175
|
cwd = File.dirname(__FILE__)
|
1980
|
-
template_file =
|
2176
|
+
template_file = if use_legacy_template_flag
|
2177
|
+
File.join(cwd, 'test_alias_template.json')
|
2178
|
+
else
|
2179
|
+
File.join(cwd, 'test_index_alias_template.json')
|
2180
|
+
end
|
1981
2181
|
|
1982
2182
|
config = %{
|
1983
2183
|
host logs.google.com
|
@@ -1994,32 +2194,34 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
1994
2194
|
logstash_format true
|
1995
2195
|
logstash_prefix mylogs
|
1996
2196
|
application_name myapp
|
2197
|
+
use_legacy_template #{use_legacy_template_flag}
|
1997
2198
|
}
|
1998
2199
|
|
2200
|
+
timestr = Time.now.strftime("%Y.%m.%d")
|
1999
2201
|
# connection start
|
2000
2202
|
stub_request(:head, "https://logs.google.com:777/es//").
|
2001
2203
|
with(basic_auth: ['john', 'doe']).
|
2002
2204
|
to_return(:status => 200, :body => "", :headers => {})
|
2003
2205
|
# check if template exists
|
2004
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2206
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2005
2207
|
with(basic_auth: ['john', 'doe']).
|
2006
2208
|
to_return(:status => 404, :body => "", :headers => {})
|
2007
2209
|
# creation
|
2008
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2210
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2009
2211
|
with(basic_auth: ['john', 'doe']).
|
2010
2212
|
to_return(:status => 200, :body => "", :headers => {})
|
2011
2213
|
# creation of index which can rollover
|
2012
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp
|
2214
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-#{timestr}-000001%3E").
|
2013
2215
|
with(basic_auth: ['john', 'doe']).
|
2014
2216
|
to_return(:status => 200, :body => "", :headers => {})
|
2015
2217
|
# check if alias exists
|
2016
|
-
|
2017
|
-
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-myapp-#{timestr}").
|
2218
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-#{timestr}").
|
2018
2219
|
with(basic_auth: ['john', 'doe']).
|
2019
2220
|
to_return(:status => 404, :body => "", :headers => {})
|
2020
2221
|
# put the alias for the index
|
2021
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp
|
2022
|
-
with(basic_auth: ['john', 'doe']
|
2222
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-#{timestr}-000001%3E/#{alias_endpoint}/mylogs-#{timestr}").
|
2223
|
+
with(basic_auth: ['john', 'doe'],
|
2224
|
+
body: "{\"aliases\":{\"mylogs-#{timestr}\":{\"is_write_index\":true}}}").
|
2023
2225
|
to_return(:status => 200, :body => "", :headers => {})
|
2024
2226
|
|
2025
2227
|
driver(config)
|
@@ -2040,9 +2242,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2040
2242
|
end
|
2041
2243
|
end
|
2042
2244
|
|
2043
|
-
|
2245
|
+
data("legacy_template" => [true, "_template"],
|
2246
|
+
"new_template" => [false, "_index_template"])
|
2247
|
+
def test_custom_template_with_rollover_index_create_and_default_ilm(data)
|
2248
|
+
use_legacy_template_flag, endpoint = data
|
2044
2249
|
cwd = File.dirname(__FILE__)
|
2045
|
-
template_file =
|
2250
|
+
template_file = if use_legacy_template_flag
|
2251
|
+
File.join(cwd, 'test_alias_template.json')
|
2252
|
+
else
|
2253
|
+
File.join(cwd, 'test_index_alias_template.json')
|
2254
|
+
end
|
2046
2255
|
|
2047
2256
|
config = %{
|
2048
2257
|
host logs.google.com
|
@@ -2059,6 +2268,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2059
2268
|
application_name myapp
|
2060
2269
|
ilm_policy_id fluentd-policy
|
2061
2270
|
enable_ilm true
|
2271
|
+
use_legacy_template #{use_legacy_template_flag}
|
2062
2272
|
}
|
2063
2273
|
|
2064
2274
|
# connection start
|
@@ -2066,11 +2276,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2066
2276
|
with(basic_auth: ['john', 'doe']).
|
2067
2277
|
to_return(:status => 200, :body => "", :headers => {})
|
2068
2278
|
# check if template exists
|
2069
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2279
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2070
2280
|
with(basic_auth: ['john', 'doe']).
|
2071
2281
|
to_return(:status => 404, :body => "", :headers => {})
|
2072
2282
|
# creation
|
2073
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2283
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2074
2284
|
with(basic_auth: ['john', 'doe']).
|
2075
2285
|
to_return(:status => 200, :body => "", :headers => {})
|
2076
2286
|
# creation of index which can rollover
|
@@ -2081,13 +2291,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2081
2291
|
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs").
|
2082
2292
|
with(basic_auth: ['john', 'doe']).
|
2083
2293
|
to_return(:status => 404, :body => "", :headers => {})
|
2084
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2294
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs").
|
2085
2295
|
with(basic_auth: ['john', 'doe']).
|
2086
2296
|
to_return(status: 404, body: "", headers: {})
|
2087
|
-
|
2088
|
-
|
2089
|
-
|
2090
|
-
|
2297
|
+
if use_legacy_template_flag
|
2298
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs").
|
2299
|
+
with(basic_auth: ['john', 'doe'],
|
2300
|
+
body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-*\"}").
|
2301
|
+
to_return(status: 200, body: "", headers: {})
|
2302
|
+
else
|
2303
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs").
|
2304
|
+
with(basic_auth: ['john', 'doe'],
|
2305
|
+
body: "{\"priority\":106,\"index_patterns\":\"mylogs-*\",\"template\":{\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}}}}").
|
2306
|
+
to_return(status: 200, body: "", headers: {})
|
2307
|
+
end
|
2091
2308
|
# put the alias for the index
|
2092
2309
|
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs").
|
2093
2310
|
with(basic_auth: ['john', 'doe'],
|
@@ -2106,12 +2323,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2106
2323
|
|
2107
2324
|
driver(config)
|
2108
2325
|
|
2109
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2326
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs", times: 1)
|
2110
2327
|
end
|
2111
2328
|
|
2112
|
-
|
2329
|
+
data("legacy_template" => [true, "_template"],
|
2330
|
+
"new_template" => [false, "_index_template"])
|
2331
|
+
def test_custom_template_with_rollover_index_create_and_default_ilm_and_ilm_policy_overwrite(data)
|
2332
|
+
use_legacy_template_flag, endpoint = data
|
2113
2333
|
cwd = File.dirname(__FILE__)
|
2114
|
-
template_file =
|
2334
|
+
template_file = if use_legacy_template_flag
|
2335
|
+
File.join(cwd, 'test_alias_template.json')
|
2336
|
+
else
|
2337
|
+
File.join(cwd, 'test_index_alias_template.json')
|
2338
|
+
end
|
2115
2339
|
|
2116
2340
|
config = %{
|
2117
2341
|
host logs.google.com
|
@@ -2130,6 +2354,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2130
2354
|
enable_ilm true
|
2131
2355
|
ilm_policy_overwrite true
|
2132
2356
|
ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"60gb","max_age": "45d"}}}}}}
|
2357
|
+
use_legacy_template #{use_legacy_template_flag}
|
2133
2358
|
}
|
2134
2359
|
|
2135
2360
|
# connection start
|
@@ -2137,11 +2362,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2137
2362
|
with(basic_auth: ['john', 'doe']).
|
2138
2363
|
to_return(:status => 200, :body => "", :headers => {})
|
2139
2364
|
# check if template exists
|
2140
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2365
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2141
2366
|
with(basic_auth: ['john', 'doe']).
|
2142
2367
|
to_return(:status => 404, :body => "", :headers => {})
|
2143
2368
|
# creation
|
2144
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2369
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2145
2370
|
with(basic_auth: ['john', 'doe']).
|
2146
2371
|
to_return(:status => 200, :body => "", :headers => {})
|
2147
2372
|
# creation of index which can rollover
|
@@ -2152,13 +2377,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2152
2377
|
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs").
|
2153
2378
|
with(basic_auth: ['john', 'doe']).
|
2154
2379
|
to_return(:status => 404, :body => "", :headers => {})
|
2155
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2380
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs").
|
2156
2381
|
with(basic_auth: ['john', 'doe']).
|
2157
2382
|
to_return(status: 404, body: "", headers: {})
|
2158
|
-
|
2159
|
-
|
2160
|
-
|
2161
|
-
|
2383
|
+
if use_legacy_template_flag
|
2384
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs").
|
2385
|
+
with(basic_auth: ['john', 'doe'],
|
2386
|
+
body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-*\"}").
|
2387
|
+
to_return(status: 200, body: "", headers: {})
|
2388
|
+
else
|
2389
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs").
|
2390
|
+
with(basic_auth: ['john', 'doe'],
|
2391
|
+
body: "{\"priority\":106,\"index_patterns\":\"mylogs-*\",\"template\":{\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}}}}").
|
2392
|
+
to_return(status: 200, body: "", headers: {})
|
2393
|
+
end
|
2162
2394
|
# put the alias for the index
|
2163
2395
|
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs").
|
2164
2396
|
with(basic_auth: ['john', 'doe'],
|
@@ -2177,7 +2409,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2177
2409
|
|
2178
2410
|
driver(config)
|
2179
2411
|
|
2180
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2412
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs", times: 1)
|
2181
2413
|
end
|
2182
2414
|
|
2183
2415
|
def test_custom_template_with_rollover_index_create_and_default_ilm_with_deflector_alias
|
@@ -2202,57 +2434,22 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2202
2434
|
enable_ilm true
|
2203
2435
|
}
|
2204
2436
|
|
2205
|
-
#
|
2206
|
-
|
2207
|
-
|
2208
|
-
|
2209
|
-
# check if template exists
|
2210
|
-
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
|
2211
|
-
with(basic_auth: ['john', 'doe']).
|
2212
|
-
to_return(:status => 404, :body => "", :headers => {})
|
2213
|
-
# creation
|
2214
|
-
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
|
2215
|
-
with(basic_auth: ['john', 'doe']).
|
2216
|
-
to_return(:status => 200, :body => "", :headers => {})
|
2217
|
-
# creation of index which can rollover
|
2218
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
2219
|
-
with(basic_auth: ['john', 'doe']).
|
2220
|
-
to_return(:status => 200, :body => "", :headers => {})
|
2221
|
-
# check if alias exists
|
2222
|
-
stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
|
2223
|
-
with(basic_auth: ['john', 'doe']).
|
2224
|
-
to_return(:status => 404, :body => "", :headers => {})
|
2225
|
-
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
|
2226
|
-
with(basic_auth: ['john', 'doe']).
|
2227
|
-
to_return(status: 404, body: "", headers: {})
|
2228
|
-
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
|
2229
|
-
with(basic_auth: ['john', 'doe'],
|
2230
|
-
body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"myapp_deflector-*\"}").
|
2231
|
-
to_return(status: 200, body: "", headers: {})
|
2232
|
-
# put the alias for the index
|
2233
|
-
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/myapp_deflector").
|
2234
|
-
with(basic_auth: ['john', 'doe'],
|
2235
|
-
:body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
|
2236
|
-
to_return(:status => 200, :body => "", :headers => {})
|
2237
|
-
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
2238
|
-
with(basic_auth: ['john', 'doe']).
|
2239
|
-
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
2240
|
-
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
2241
|
-
with(basic_auth: ['john', 'doe']).
|
2242
|
-
to_return(:status => 404, :body => "", :headers => {})
|
2243
|
-
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
2244
|
-
with(basic_auth: ['john', 'doe'],
|
2245
|
-
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
2246
|
-
to_return(:status => 200, :body => "", :headers => {})
|
2247
|
-
|
2248
|
-
driver(config)
|
2249
|
-
|
2250
|
-
assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_deflector", times: 1)
|
2437
|
+
# Should raise error because multiple alias indices IllegalArgument Error on executing ILM feature
|
2438
|
+
assert_raise(Fluent::ConfigError) do
|
2439
|
+
driver(config)
|
2440
|
+
end
|
2251
2441
|
end
|
2252
2442
|
|
2253
|
-
|
2443
|
+
data("legacy_template" => [true, "_template"],
|
2444
|
+
"new_template" => [false, "_index_template"])
|
2445
|
+
def test_custom_template_with_rollover_index_create_and_default_ilm_and_placeholders(data)
|
2446
|
+
use_legacy_template_flag, endpoint = data
|
2254
2447
|
cwd = File.dirname(__FILE__)
|
2255
|
-
template_file =
|
2448
|
+
template_file = if use_legacy_template_flag
|
2449
|
+
File.join(cwd, 'test_alias_template.json')
|
2450
|
+
else
|
2451
|
+
File.join(cwd, 'test_index_alias_template.json')
|
2452
|
+
end
|
2256
2453
|
|
2257
2454
|
config = %{
|
2258
2455
|
host logs.google.com
|
@@ -2269,6 +2466,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2269
2466
|
application_name myapp
|
2270
2467
|
ilm_policy_id fluentd-policy
|
2271
2468
|
enable_ilm true
|
2469
|
+
use_legacy_template #{use_legacy_template_flag}
|
2272
2470
|
}
|
2273
2471
|
|
2274
2472
|
# connection start
|
@@ -2276,11 +2474,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2276
2474
|
with(basic_auth: ['john', 'doe']).
|
2277
2475
|
to_return(:status => 200, :body => "", :headers => {})
|
2278
2476
|
# check if template exists
|
2279
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2477
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2280
2478
|
with(basic_auth: ['john', 'doe']).
|
2281
2479
|
to_return(:status => 404, :body => "", :headers => {})
|
2282
2480
|
# creation
|
2283
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2481
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2284
2482
|
with(basic_auth: ['john', 'doe']).
|
2285
2483
|
to_return(:status => 200, :body => "", :headers => {})
|
2286
2484
|
# creation of index which can rollover
|
@@ -2291,13 +2489,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2291
2489
|
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs-custom-test").
|
2292
2490
|
with(basic_auth: ['john', 'doe']).
|
2293
2491
|
to_return(:status => 404, :body => "", :headers => {})
|
2294
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2492
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs-custom-test").
|
2295
2493
|
with(basic_auth: ['john', 'doe']).
|
2296
2494
|
to_return(status: 404, body: "", headers: {})
|
2297
|
-
|
2298
|
-
|
2299
|
-
|
2300
|
-
|
2495
|
+
if use_legacy_template_flag
|
2496
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-custom-test").
|
2497
|
+
with(basic_auth: ['john', 'doe'],
|
2498
|
+
body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-custom-test\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"mylogs-custom-test-*\"}").
|
2499
|
+
to_return(status: 200, body: "", headers: {})
|
2500
|
+
else
|
2501
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs-custom-test").
|
2502
|
+
with(basic_auth: ['john', 'doe'],
|
2503
|
+
body: "{\"priority\":108,\"index_patterns\":\"mylogs-custom-test-*\",\"template\":{\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"mylogs-custom-test\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}}}}").
|
2504
|
+
to_return(status: 200, body: "", headers: {})
|
2505
|
+
end
|
2301
2506
|
# put the alias for the index
|
2302
2507
|
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-custom-test-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/#{alias_endpoint}/mylogs-custom-test").
|
2303
2508
|
with(basic_auth: ['john', 'doe'],
|
@@ -2327,9 +2532,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2327
2532
|
assert_requested(elastic_request)
|
2328
2533
|
end
|
2329
2534
|
|
2330
|
-
|
2535
|
+
data("legacy_template" => [true, "_template"],
|
2536
|
+
"new_template" => [false, "_index_template"])
|
2537
|
+
def test_custom_template_with_rollover_index_create_and_custom_ilm(data)
|
2538
|
+
use_legacy_template_flag, endpoint = data
|
2331
2539
|
cwd = File.dirname(__FILE__)
|
2332
|
-
template_file =
|
2540
|
+
template_file = if use_legacy_template_flag
|
2541
|
+
File.join(cwd, 'test_alias_template.json')
|
2542
|
+
else
|
2543
|
+
File.join(cwd, 'test_index_alias_template.json')
|
2544
|
+
end
|
2333
2545
|
|
2334
2546
|
config = %{
|
2335
2547
|
host logs.google.com
|
@@ -2347,6 +2559,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2347
2559
|
ilm_policy_id fluentd-policy
|
2348
2560
|
enable_ilm true
|
2349
2561
|
ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}
|
2562
|
+
use_legacy_template #{use_legacy_template_flag}
|
2350
2563
|
}
|
2351
2564
|
|
2352
2565
|
# connection start
|
@@ -2354,11 +2567,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2354
2567
|
with(basic_auth: ['john', 'doe']).
|
2355
2568
|
to_return(:status => 200, :body => "", :headers => {})
|
2356
2569
|
# check if template exists
|
2357
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2570
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2358
2571
|
with(basic_auth: ['john', 'doe']).
|
2359
2572
|
to_return(:status => 404, :body => "", :headers => {})
|
2360
2573
|
# creation
|
2361
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2574
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2362
2575
|
with(basic_auth: ['john', 'doe']).
|
2363
2576
|
to_return(:status => 200, :body => "", :headers => {})
|
2364
2577
|
# creation of index which can rollover
|
@@ -2369,10 +2582,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2369
2582
|
stub_request(:head, "https://logs.google.com:777/es//_alias/mylogs").
|
2370
2583
|
with(basic_auth: ['john', 'doe']).
|
2371
2584
|
to_return(:status => 404, :body => "", :headers => {})
|
2372
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2585
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/mylogs").
|
2373
2586
|
with(basic_auth: ['john', 'doe']).
|
2374
2587
|
to_return(status: 404, body: "", headers: {})
|
2375
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2588
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs").
|
2376
2589
|
with(basic_auth: ['john', 'doe']).
|
2377
2590
|
to_return(status: 200, body: "", headers: {})
|
2378
2591
|
# put the alias for the index
|
@@ -2393,13 +2606,20 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2393
2606
|
|
2394
2607
|
driver(config)
|
2395
2608
|
|
2396
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2609
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/mylogs", times: 1)
|
2397
2610
|
end
|
2398
2611
|
end
|
2399
2612
|
|
2400
|
-
|
2613
|
+
data("legacy_template" => [true, "_template"],
|
2614
|
+
"new_template" => [false, "_index_template"])
|
2615
|
+
def test_template_overwrite(data)
|
2616
|
+
use_legacy_template_flag, endpoint = data
|
2401
2617
|
cwd = File.dirname(__FILE__)
|
2402
|
-
template_file =
|
2618
|
+
template_file = if use_legacy_template_flag
|
2619
|
+
File.join(cwd, 'test_template.json')
|
2620
|
+
else
|
2621
|
+
File.join(cwd, 'test_index_template.json')
|
2622
|
+
end
|
2403
2623
|
|
2404
2624
|
config = %{
|
2405
2625
|
host logs.google.com
|
@@ -2411,6 +2631,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2411
2631
|
template_name logstash
|
2412
2632
|
template_file #{template_file}
|
2413
2633
|
template_overwrite true
|
2634
|
+
use_legacy_template #{use_legacy_template_flag}
|
2414
2635
|
}
|
2415
2636
|
|
2416
2637
|
# connection start
|
@@ -2418,22 +2639,29 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2418
2639
|
with(basic_auth: ['john', 'doe']).
|
2419
2640
|
to_return(:status => 200, :body => "", :headers => {})
|
2420
2641
|
# check if template exists
|
2421
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2642
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
2422
2643
|
with(basic_auth: ['john', 'doe']).
|
2423
2644
|
to_return(:status => 200, :body => "", :headers => {})
|
2424
2645
|
# creation
|
2425
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2646
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
2426
2647
|
with(basic_auth: ['john', 'doe']).
|
2427
2648
|
to_return(:status => 200, :body => "", :headers => {})
|
2428
2649
|
|
2429
2650
|
driver(config)
|
2430
2651
|
|
2431
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2652
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
2432
2653
|
end
|
2433
2654
|
|
2434
|
-
|
2655
|
+
data("legacy_template" => [true, "_template"],
|
2656
|
+
"new_template" => [false, "_index_template"])
|
2657
|
+
def test_custom_template_overwrite(data)
|
2658
|
+
use_legacy_template_flag, endpoint = data
|
2435
2659
|
cwd = File.dirname(__FILE__)
|
2436
|
-
template_file =
|
2660
|
+
template_file = if use_legacy_template_flag
|
2661
|
+
File.join(cwd, 'test_template.json')
|
2662
|
+
else
|
2663
|
+
File.join(cwd, 'test_index_template.json')
|
2664
|
+
end
|
2437
2665
|
|
2438
2666
|
config = %{
|
2439
2667
|
host logs.google.com
|
@@ -2446,6 +2674,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2446
2674
|
template_file #{template_file}
|
2447
2675
|
template_overwrite true
|
2448
2676
|
customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
|
2677
|
+
use_legacy_template #{use_legacy_template_flag}
|
2449
2678
|
}
|
2450
2679
|
|
2451
2680
|
# connection start
|
@@ -2453,22 +2682,29 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2453
2682
|
with(basic_auth: ['john', 'doe']).
|
2454
2683
|
to_return(:status => 200, :body => "", :headers => {})
|
2455
2684
|
# check if template exists
|
2456
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2685
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2457
2686
|
with(basic_auth: ['john', 'doe']).
|
2458
2687
|
to_return(:status => 200, :body => "", :headers => {})
|
2459
2688
|
# creation
|
2460
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2689
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2461
2690
|
with(basic_auth: ['john', 'doe']).
|
2462
2691
|
to_return(:status => 200, :body => "", :headers => {})
|
2463
2692
|
|
2464
2693
|
driver(config)
|
2465
2694
|
|
2466
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2695
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template", times: 1)
|
2467
2696
|
end
|
2468
2697
|
|
2469
|
-
|
2698
|
+
data("legacy_template" => [true, "_template"],
|
2699
|
+
"new_template" => [false, "_index_template"])
|
2700
|
+
def test_custom_template_with_rollover_index_overwrite(data)
|
2701
|
+
use_legacy_template_flag, endpoint = data
|
2470
2702
|
cwd = File.dirname(__FILE__)
|
2471
|
-
template_file =
|
2703
|
+
template_file = if use_legacy_template_flag
|
2704
|
+
File.join(cwd, 'test_template.json')
|
2705
|
+
else
|
2706
|
+
File.join(cwd, 'test_index_template.json')
|
2707
|
+
end
|
2472
2708
|
|
2473
2709
|
config = %{
|
2474
2710
|
host logs.google.com
|
@@ -2485,6 +2721,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2485
2721
|
rollover_index true
|
2486
2722
|
index_name mylogs
|
2487
2723
|
application_name myapp
|
2724
|
+
use_legacy_template #{use_legacy_template_flag}
|
2488
2725
|
}
|
2489
2726
|
|
2490
2727
|
# connection start
|
@@ -2492,11 +2729,11 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2492
2729
|
with(basic_auth: ['john', 'doe']).
|
2493
2730
|
to_return(:status => 200, :body => "", :headers => {})
|
2494
2731
|
# check if template exists
|
2495
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2732
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2496
2733
|
with(basic_auth: ['john', 'doe']).
|
2497
2734
|
to_return(:status => 200, :body => "", :headers => {})
|
2498
2735
|
# creation
|
2499
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2736
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template").
|
2500
2737
|
with(basic_auth: ['john', 'doe']).
|
2501
2738
|
to_return(:status => 200, :body => "", :headers => {})
|
2502
2739
|
# creation of index which can rollover
|
@@ -2514,7 +2751,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2514
2751
|
|
2515
2752
|
driver(config)
|
2516
2753
|
|
2517
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2754
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/myapp_alias_template", times: 1)
|
2518
2755
|
end
|
2519
2756
|
|
2520
2757
|
def test_template_create_invalid_filename
|
@@ -2544,9 +2781,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2544
2781
|
}
|
2545
2782
|
end
|
2546
2783
|
|
2547
|
-
|
2784
|
+
data("legacy_template" => [true, "_template"],
|
2785
|
+
"new_template" => [false, "_index_template"])
|
2786
|
+
def test_template_create_for_host_placeholder(data)
|
2787
|
+
use_legacy_template_flag, endpoint = data
|
2548
2788
|
cwd = File.dirname(__FILE__)
|
2549
|
-
template_file =
|
2789
|
+
template_file = if use_legacy_template_flag
|
2790
|
+
File.join(cwd, 'test_template.json')
|
2791
|
+
else
|
2792
|
+
File.join(cwd, 'test_index_template.json')
|
2793
|
+
end
|
2550
2794
|
|
2551
2795
|
config = %{
|
2552
2796
|
host logs-${tag}.google.com
|
@@ -2559,6 +2803,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2559
2803
|
template_file #{template_file}
|
2560
2804
|
verify_es_version_at_startup false
|
2561
2805
|
default_elasticsearch_version 6
|
2806
|
+
use_legacy_template #{use_legacy_template_flag}
|
2562
2807
|
}
|
2563
2808
|
|
2564
2809
|
# connection start
|
@@ -2566,10 +2811,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2566
2811
|
with(basic_auth: ['john', 'doe']).
|
2567
2812
|
to_return(:status => 200, :body => "", :headers => {})
|
2568
2813
|
# check if template exists
|
2569
|
-
stub_request(:get, "https://logs-test.google.com:777/es
|
2814
|
+
stub_request(:get, "https://logs-test.google.com:777/es//#{endpoint}/logstash").
|
2570
2815
|
with(basic_auth: ['john', 'doe']).
|
2571
2816
|
to_return(:status => 404, :body => "", :headers => {})
|
2572
|
-
stub_request(:put, "https://logs-test.google.com:777/es
|
2817
|
+
stub_request(:put, "https://logs-test.google.com:777/es//#{endpoint}/logstash").
|
2573
2818
|
with(basic_auth: ['john', 'doe']).
|
2574
2819
|
to_return(status: 200, body: "", headers: {})
|
2575
2820
|
stub_request(:post, "https://logs-test.google.com:777/es//_bulk").
|
@@ -2584,9 +2829,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2584
2829
|
end
|
2585
2830
|
end
|
2586
2831
|
|
2587
|
-
|
2832
|
+
data("legacy_template" => [true, "_template"],
|
2833
|
+
"new_template" => [false, "_index_template"])
|
2834
|
+
def test_template_retry_install_fails(data)
|
2835
|
+
use_legacy_template_flag, endpoint = data
|
2588
2836
|
cwd = File.dirname(__FILE__)
|
2589
|
-
template_file =
|
2837
|
+
template_file = if use_legacy_template_flag
|
2838
|
+
File.join(cwd, 'test_template.json')
|
2839
|
+
else
|
2840
|
+
File.join(cwd, 'test_index_template.json')
|
2841
|
+
end
|
2590
2842
|
|
2591
2843
|
config = %{
|
2592
2844
|
host logs.google.com
|
@@ -2598,11 +2850,12 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2598
2850
|
template_name logstash
|
2599
2851
|
template_file #{template_file}
|
2600
2852
|
max_retry_putting_template 3
|
2853
|
+
use_legacy_template #{use_legacy_template_flag}
|
2601
2854
|
}
|
2602
2855
|
|
2603
2856
|
connection_resets = 0
|
2604
2857
|
# check if template exists
|
2605
|
-
stub_request(:get, "https://logs.google.com:778/es
|
2858
|
+
stub_request(:get, "https://logs.google.com:778/es//#{endpoint}/logstash")
|
2606
2859
|
.with(basic_auth: ['john', 'doe']) do |req|
|
2607
2860
|
connection_resets += 1
|
2608
2861
|
raise Faraday::ConnectionFailed, "Test message"
|
@@ -2615,9 +2868,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2615
2868
|
assert_equal(4, connection_resets)
|
2616
2869
|
end
|
2617
2870
|
|
2618
|
-
|
2871
|
+
data("legacy_template" => [true, "_template"],
|
2872
|
+
"new_template" => [false, "_index_template"])
|
2873
|
+
def test_template_retry_install_does_not_fail(data)
|
2874
|
+
use_legacy_template_flag, endpoint = data
|
2619
2875
|
cwd = File.dirname(__FILE__)
|
2620
|
-
template_file =
|
2876
|
+
template_file = if use_legacy_template_flag
|
2877
|
+
File.join(cwd, 'test_template.json')
|
2878
|
+
else
|
2879
|
+
File.join(cwd, 'test_index_template.json')
|
2880
|
+
end
|
2621
2881
|
|
2622
2882
|
config = %{
|
2623
2883
|
host logs.google.com
|
@@ -2630,11 +2890,12 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2630
2890
|
template_file #{template_file}
|
2631
2891
|
max_retry_putting_template 3
|
2632
2892
|
fail_on_putting_template_retry_exceed false
|
2893
|
+
use_legacy_template #{use_legacy_template_flag}
|
2633
2894
|
}
|
2634
2895
|
|
2635
2896
|
connection_resets = 0
|
2636
2897
|
# check if template exists
|
2637
|
-
stub_request(:get, "https://logs.google.com:778/es
|
2898
|
+
stub_request(:get, "https://logs.google.com:778/es//#{endpoint}/logstash")
|
2638
2899
|
.with(basic_auth: ['john', 'doe']) do |req|
|
2639
2900
|
connection_resets += 1
|
2640
2901
|
raise Faraday::ConnectionFailed, "Test message"
|
@@ -2645,9 +2906,17 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2645
2906
|
assert_equal(4, connection_resets)
|
2646
2907
|
end
|
2647
2908
|
|
2648
|
-
|
2909
|
+
data("legacy_template" => [true, "_template"],
|
2910
|
+
"new_template" => [false, "_index_template"])
|
2911
|
+
def test_templates_create(data)
|
2912
|
+
use_legacy_template_flag, endpoint = data
|
2649
2913
|
cwd = File.dirname(__FILE__)
|
2650
|
-
template_file =
|
2914
|
+
template_file = if use_legacy_template_flag
|
2915
|
+
File.join(cwd, 'test_template.json')
|
2916
|
+
else
|
2917
|
+
File.join(cwd, 'test_index_template.json')
|
2918
|
+
end
|
2919
|
+
|
2651
2920
|
config = %{
|
2652
2921
|
host logs.google.com
|
2653
2922
|
port 777
|
@@ -2656,43 +2925,52 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2656
2925
|
user john
|
2657
2926
|
password doe
|
2658
2927
|
templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
|
2928
|
+
use_legacy_template #{use_legacy_template_flag}
|
2659
2929
|
}
|
2660
2930
|
|
2661
2931
|
stub_request(:head, "https://logs.google.com:777/es//").
|
2662
2932
|
with(basic_auth: ['john', 'doe']).
|
2663
2933
|
to_return(:status => 200, :body => "", :headers => {})
|
2664
2934
|
# check if template exists
|
2665
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2935
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash1").
|
2666
2936
|
with(basic_auth: ['john', 'doe']).
|
2667
2937
|
to_return(:status => 404, :body => "", :headers => {})
|
2668
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2938
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash2").
|
2669
2939
|
with(basic_auth: ['john', 'doe']).
|
2670
2940
|
to_return(:status => 404, :body => "", :headers => {})
|
2671
2941
|
|
2672
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2942
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash3").
|
2673
2943
|
with(basic_auth: ['john', 'doe']).
|
2674
2944
|
to_return(:status => 200, :body => "", :headers => {}) #exists
|
2675
2945
|
|
2676
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2946
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1").
|
2677
2947
|
with(basic_auth: ['john', 'doe']).
|
2678
2948
|
to_return(:status => 200, :body => "", :headers => {})
|
2679
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2949
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2").
|
2680
2950
|
with(basic_auth: ['john', 'doe']).
|
2681
2951
|
to_return(:status => 200, :body => "", :headers => {})
|
2682
|
-
stub_request(:put, "https://logs.google.com:777/es
|
2952
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash3").
|
2683
2953
|
with(basic_auth: ['john', 'doe']).
|
2684
2954
|
to_return(:status => 200, :body => "", :headers => {})
|
2685
2955
|
|
2686
2956
|
driver(config)
|
2687
2957
|
|
2688
|
-
assert_requested( :put, "https://logs.google.com:777/es
|
2689
|
-
assert_requested( :put, "https://logs.google.com:777/es
|
2690
|
-
assert_not_requested(:put, "https://logs.google.com:777/es
|
2958
|
+
assert_requested( :put, "https://logs.google.com:777/es//#{endpoint}/logstash1", times: 1)
|
2959
|
+
assert_requested( :put, "https://logs.google.com:777/es//#{endpoint}/logstash2", times: 1)
|
2960
|
+
assert_not_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash3") #exists
|
2691
2961
|
end
|
2692
2962
|
|
2693
|
-
|
2963
|
+
data("legacy_template" => [true, "_template"],
|
2964
|
+
"new_template" => [false, "_index_template"])
|
2965
|
+
def test_templates_overwrite(data)
|
2966
|
+
use_legacy_template_flag, endpoint = data
|
2694
2967
|
cwd = File.dirname(__FILE__)
|
2695
|
-
template_file =
|
2968
|
+
template_file = if use_legacy_template_flag
|
2969
|
+
File.join(cwd, 'test_template.json')
|
2970
|
+
else
|
2971
|
+
File.join(cwd, 'test_index_template.json')
|
2972
|
+
end
|
2973
|
+
|
2696
2974
|
config = %{
|
2697
2975
|
host logs.google.com
|
2698
2976
|
port 777
|
@@ -2702,42 +2980,50 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2702
2980
|
password doe
|
2703
2981
|
templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
|
2704
2982
|
template_overwrite true
|
2983
|
+
use_legacy_template #{use_legacy_template_flag}
|
2705
2984
|
}
|
2706
2985
|
|
2707
2986
|
stub_request(:head, "https://logs.google.com:777/es//").
|
2708
2987
|
with(basic_auth: ['john', 'doe']).
|
2709
2988
|
to_return(:status => 200, :body => "", :headers => {})
|
2710
2989
|
# check if template exists
|
2711
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2990
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash1").
|
2712
2991
|
with(basic_auth: ['john', 'doe']).
|
2713
2992
|
to_return(:status => 200, :body => "", :headers => {})
|
2714
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2993
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash2").
|
2715
2994
|
with(basic_auth: ['john', 'doe']).
|
2716
2995
|
to_return(:status => 200, :body => "", :headers => {})
|
2717
|
-
stub_request(:get, "https://logs.google.com:777/es
|
2996
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash3").
|
2718
2997
|
with(basic_auth: ['john', 'doe']).
|
2719
2998
|
to_return(:status => 200, :body => "", :headers => {}) #exists
|
2720
2999
|
|
2721
|
-
stub_request(:put, "https://logs.google.com:777/es
|
3000
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1").
|
2722
3001
|
with(basic_auth: ['john', 'doe']).
|
2723
3002
|
to_return(:status => 200, :body => "", :headers => {})
|
2724
|
-
stub_request(:put, "https://logs.google.com:777/es
|
3003
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2").
|
2725
3004
|
with(basic_auth: ['john', 'doe']).
|
2726
3005
|
to_return(:status => 200, :body => "", :headers => {})
|
2727
|
-
stub_request(:put, "https://logs.google.com:777/es
|
3006
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash3").
|
2728
3007
|
with(basic_auth: ['john', 'doe']).
|
2729
3008
|
to_return(:status => 200, :body => "", :headers => {})
|
2730
3009
|
|
2731
3010
|
driver(config)
|
2732
3011
|
|
2733
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2734
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2735
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
3012
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1", times: 1)
|
3013
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2", times: 1)
|
3014
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash3", times: 1)
|
2736
3015
|
end
|
2737
3016
|
|
2738
|
-
|
3017
|
+
data("legacy_template" => [true, "_template"],
|
3018
|
+
"new_template" => [false, "_index_template"])
|
3019
|
+
def test_templates_are_also_used(data)
|
3020
|
+
use_legacy_template_flag, endpoint = data
|
2739
3021
|
cwd = File.dirname(__FILE__)
|
2740
|
-
template_file =
|
3022
|
+
template_file = if use_legacy_template_flag
|
3023
|
+
File.join(cwd, 'test_template.json')
|
3024
|
+
else
|
3025
|
+
File.join(cwd, 'test_index_template.json')
|
3026
|
+
end
|
2741
3027
|
|
2742
3028
|
config = %{
|
2743
3029
|
host logs.google.com
|
@@ -2749,43 +3035,52 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2749
3035
|
template_name logstash
|
2750
3036
|
template_file #{template_file}
|
2751
3037
|
templates {"logstash1":"#{template_file}", "logstash2":"#{template_file}" }
|
3038
|
+
use_legacy_template #{use_legacy_template_flag}
|
2752
3039
|
}
|
2753
3040
|
# connection start
|
2754
3041
|
stub_request(:head, "https://logs.google.com:777/es//").
|
2755
3042
|
with(basic_auth: ['john', 'doe']).
|
2756
3043
|
to_return(:status => 200, :body => "", :headers => {})
|
2757
3044
|
# check if template exists
|
2758
|
-
stub_request(:get, "https://logs.google.com:777/es
|
3045
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
2759
3046
|
with(basic_auth: ['john', 'doe']).
|
2760
3047
|
to_return(:status => 404, :body => "", :headers => {})
|
2761
|
-
stub_request(:get, "https://logs.google.com:777/es
|
3048
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash1").
|
2762
3049
|
with(basic_auth: ['john', 'doe']).
|
2763
3050
|
to_return(:status => 404, :body => "", :headers => {})
|
2764
|
-
stub_request(:get, "https://logs.google.com:777/es
|
3051
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash2").
|
2765
3052
|
with(basic_auth: ['john', 'doe']).
|
2766
3053
|
to_return(:status => 404, :body => "", :headers => {})
|
2767
3054
|
#creation
|
2768
|
-
stub_request(:put, "https://logs.google.com:777/es
|
3055
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash").
|
2769
3056
|
with(basic_auth: ['john', 'doe']).
|
2770
3057
|
to_return(:status => 200, :body => "", :headers => {})
|
2771
|
-
stub_request(:put, "https://logs.google.com:777/es
|
3058
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1").
|
2772
3059
|
with(basic_auth: ['john', 'doe']).
|
2773
3060
|
to_return(:status => 200, :body => "", :headers => {})
|
2774
|
-
stub_request(:put, "https://logs.google.com:777/es
|
3061
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2").
|
2775
3062
|
with(basic_auth: ['john', 'doe']).
|
2776
3063
|
to_return(:status => 200, :body => "", :headers => {})
|
2777
3064
|
|
2778
3065
|
driver(config)
|
2779
3066
|
|
2780
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
3067
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash", times: 1)
|
2781
3068
|
|
2782
|
-
|
2783
|
-
|
3069
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1")
|
3070
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2")
|
2784
3071
|
end
|
2785
3072
|
|
2786
|
-
|
3073
|
+
data("legacy_template" => [true, "_template"],
|
3074
|
+
"new_template" => [false, "_index_template"])
|
3075
|
+
def test_templates_can_be_partially_created_if_error_occurs(data)
|
3076
|
+
use_legacy_template_flag, endpoint = data
|
2787
3077
|
cwd = File.dirname(__FILE__)
|
2788
|
-
template_file =
|
3078
|
+
template_file = if use_legacy_template_flag
|
3079
|
+
File.join(cwd, 'test_template.json')
|
3080
|
+
else
|
3081
|
+
File.join(cwd, 'test_index_template.json')
|
3082
|
+
end
|
3083
|
+
|
2789
3084
|
config = %{
|
2790
3085
|
host logs.google.com
|
2791
3086
|
port 777
|
@@ -2794,22 +3089,23 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2794
3089
|
user john
|
2795
3090
|
password doe
|
2796
3091
|
templates {"logstash1":"#{template_file}", "logstash2":"/abc" }
|
3092
|
+
use_legacy_template #{use_legacy_template_flag}
|
2797
3093
|
}
|
2798
3094
|
stub_request(:head, "https://logs.google.com:777/es//").
|
2799
3095
|
with(basic_auth: ['john', 'doe']).
|
2800
3096
|
to_return(:status => 200, :body => "", :headers => {})
|
2801
3097
|
# check if template exists
|
2802
|
-
stub_request(:get, "https://logs.google.com:777/es
|
3098
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash1").
|
2803
3099
|
with(basic_auth: ['john', 'doe']).
|
2804
3100
|
to_return(:status => 404, :body => "", :headers => {})
|
2805
|
-
stub_request(:get, "https://logs.google.com:777/es
|
3101
|
+
stub_request(:get, "https://logs.google.com:777/es//#{endpoint}/logstash2").
|
2806
3102
|
with(basic_auth: ['john', 'doe']).
|
2807
3103
|
to_return(:status => 404, :body => "", :headers => {})
|
2808
3104
|
|
2809
|
-
stub_request(:put, "https://logs.google.com:777/es
|
3105
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1").
|
2810
3106
|
with(basic_auth: ['john', 'doe']).
|
2811
3107
|
to_return(:status => 200, :body => "", :headers => {})
|
2812
|
-
stub_request(:put, "https://logs.google.com:777/es
|
3108
|
+
stub_request(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2").
|
2813
3109
|
with(basic_auth: ['john', 'doe']).
|
2814
3110
|
to_return(:status => 200, :body => "", :headers => {})
|
2815
3111
|
|
@@ -2817,8 +3113,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2817
3113
|
driver(config)
|
2818
3114
|
}
|
2819
3115
|
|
2820
|
-
assert_requested(:put, "https://logs.google.com:777/es
|
2821
|
-
assert_not_requested(:put, "https://logs.google.com:777/es
|
3116
|
+
assert_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash1", times: 1)
|
3117
|
+
assert_not_requested(:put, "https://logs.google.com:777/es//#{endpoint}/logstash2")
|
2822
3118
|
end
|
2823
3119
|
|
2824
3120
|
def test_legacy_hosts_list
|
@@ -2986,6 +3282,18 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2986
3282
|
assert_requested(elastic_request)
|
2987
3283
|
end
|
2988
3284
|
|
3285
|
+
def test_api_key_header
|
3286
|
+
stub_request(:head, "http://localhost:9200/").
|
3287
|
+
to_return(:status => 200, :body => "", :headers => {})
|
3288
|
+
elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
|
3289
|
+
with(headers: {'Authorization'=>'ApiKey dGVzdGF1dGhoZWFkZXI='})
|
3290
|
+
driver.configure(%[api_key testauthheader])
|
3291
|
+
driver.run(default_tag: 'test') do
|
3292
|
+
driver.feed(sample_record)
|
3293
|
+
end
|
3294
|
+
assert_requested(elastic_request)
|
3295
|
+
end
|
3296
|
+
|
2989
3297
|
def test_write_message_with_bad_chunk
|
2990
3298
|
driver.configure("target_index_key bad_value\n@log_level debug\n")
|
2991
3299
|
stub_elastic
|