fluent-plugin-elasticsearch 3.5.6 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +16 -7
- data/History.md +6 -0
- data/README.md +66 -0
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/gemfiles/Gemfile.ilm +10 -0
- data/lib/fluent/plugin/default-ilm-policy.json +14 -0
- data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb +67 -0
- data/lib/fluent/plugin/elasticsearch_index_template.rb +81 -23
- data/lib/fluent/plugin/out_elasticsearch.rb +54 -13
- data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +63 -0
- data/test/plugin/test_out_elasticsearch.rb +617 -7
- metadata +7 -2
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
require 'helper'
|
|
2
|
+
require 'elasticsearch'
|
|
3
|
+
require 'fluent/plugin/elasticsearch_index_lifecycle_management'
|
|
4
|
+
|
|
5
|
+
class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
|
|
6
|
+
include Fluent::Plugin::ElasticsearchIndexLifecycleManagement
|
|
7
|
+
|
|
8
|
+
def setup
|
|
9
|
+
begin
|
|
10
|
+
require "elasticsearch/xpack"
|
|
11
|
+
rescue LoadError
|
|
12
|
+
omit "ILM testcase needs elasticsearch-xpack gem."
|
|
13
|
+
end
|
|
14
|
+
Fluent::Plugin::ElasticsearchIndexLifecycleManagement.module_eval(<<-CODE)
|
|
15
|
+
def client
|
|
16
|
+
Elasticsearch::Client.new url: 'localhost:9200'
|
|
17
|
+
end
|
|
18
|
+
def log
|
|
19
|
+
log_device = Fluent::Test::DummyLogDevice.new
|
|
20
|
+
dl_opts = {:log_level => ServerEngine::DaemonLogger::INFO}
|
|
21
|
+
logger = ServerEngine::DaemonLogger.new(log_device, dl_opts)
|
|
22
|
+
Fluent::Log.new(logger)
|
|
23
|
+
end
|
|
24
|
+
CODE
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def test_xpack_info
|
|
28
|
+
stub_request(:get, "http://localhost:9200/_xpack").
|
|
29
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
30
|
+
expected = {"features"=>{"ilm"=>{"available"=>true, "enabled"=>true}}}
|
|
31
|
+
assert_equal(expected, xpack_info)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def test_verify_ilm_working
|
|
35
|
+
stub_request(:get, "http://localhost:9200/_xpack").
|
|
36
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
37
|
+
assert_nothing_raised { verify_ilm_working }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def test_ilm_policy_doesnt_exists
|
|
41
|
+
stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluentd-policy%22%7D").
|
|
42
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
43
|
+
assert_false(ilm_policy_exists?(policy_id: "fluentd-policy"))
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def test_ilm_policy_exists
|
|
47
|
+
stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluent-policy%22%7D").
|
|
48
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
49
|
+
assert_true(ilm_policy_exists?(policy_id: "fluent-policy"))
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def test_create_ilm_policy
|
|
53
|
+
stub_request(:get, "http://localhost:9200/_ilm/policy/fluent-policy").
|
|
54
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
55
|
+
stub_request(:put, "http://localhost:9200/_ilm/policy/fluent-policy").
|
|
56
|
+
with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
|
|
57
|
+
:headers => {'Content-Type'=>'application/json'}).
|
|
58
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
59
|
+
create_ilm_policy("fluent-policy")
|
|
60
|
+
|
|
61
|
+
assert_requested(:put, "http://localhost:9200/_ilm/policy/fluent-policy", times: 1)
|
|
62
|
+
end
|
|
63
|
+
end
|
|
@@ -269,6 +269,20 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
|
269
269
|
}
|
|
270
270
|
end
|
|
271
271
|
|
|
272
|
+
test 'invalid configuration of index lifecycle management' do
|
|
273
|
+
cwd = File.dirname(__FILE__)
|
|
274
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
275
|
+
|
|
276
|
+
config = %{
|
|
277
|
+
enable_ilm true
|
|
278
|
+
template_name logstash
|
|
279
|
+
template_file #{template_file}
|
|
280
|
+
}
|
|
281
|
+
assert_raise(Fluent::ConfigError) {
|
|
282
|
+
driver(config)
|
|
283
|
+
}
|
|
284
|
+
end
|
|
285
|
+
|
|
272
286
|
test 'Detected Elasticsearch 7' do
|
|
273
287
|
config = %{
|
|
274
288
|
type_name changed
|
|
@@ -347,6 +361,27 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
|
347
361
|
end
|
|
348
362
|
end
|
|
349
363
|
|
|
364
|
+
test 'Detected exclusive features which are host placeholder, template installation, and verify Elasticsearch version at startup' do
|
|
365
|
+
cwd = File.dirname(__FILE__)
|
|
366
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
367
|
+
|
|
368
|
+
assert_raise_message(/host placeholder, template installation, and verify Elasticsearch version at startup are exclusive feature at same time./) do
|
|
369
|
+
config = %{
|
|
370
|
+
host logs-${tag}.google.com
|
|
371
|
+
port 777
|
|
372
|
+
scheme https
|
|
373
|
+
path /es/
|
|
374
|
+
user john
|
|
375
|
+
password doe
|
|
376
|
+
template_name logstash
|
|
377
|
+
template_file #{template_file}
|
|
378
|
+
verify_es_version_at_startup true
|
|
379
|
+
default_elasticsearch_version 6
|
|
380
|
+
}
|
|
381
|
+
driver(config)
|
|
382
|
+
end
|
|
383
|
+
end
|
|
384
|
+
|
|
350
385
|
sub_test_case 'connection exceptions' do
|
|
351
386
|
test 'default connection exception' do
|
|
352
387
|
driver(Fluent::Config::Element.new(
|
|
@@ -469,10 +504,243 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
|
469
504
|
to_return(:status => 200, :body => "", :headers => {})
|
|
470
505
|
|
|
471
506
|
driver(config)
|
|
472
|
-
|
|
507
|
+
stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
508
|
+
driver.run(default_tag: 'test') do
|
|
509
|
+
driver.feed(sample_record)
|
|
510
|
+
end
|
|
473
511
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
474
512
|
end
|
|
475
513
|
|
|
514
|
+
class TemplateIndexLifecycleManagementTest < self
|
|
515
|
+
def setup
|
|
516
|
+
begin
|
|
517
|
+
require "elasticsearch/xpack"
|
|
518
|
+
rescue LoadError
|
|
519
|
+
omit "ILM testcase needs elasticsearch-xpack gem."
|
|
520
|
+
end
|
|
521
|
+
end
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def test_template_create_with_rollover_index_and_default_ilm
|
|
525
|
+
cwd = File.dirname(__FILE__)
|
|
526
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
527
|
+
|
|
528
|
+
config = %{
|
|
529
|
+
host logs.google.com
|
|
530
|
+
port 777
|
|
531
|
+
scheme https
|
|
532
|
+
path /es/
|
|
533
|
+
user john
|
|
534
|
+
password doe
|
|
535
|
+
template_name logstash
|
|
536
|
+
template_file #{template_file}
|
|
537
|
+
rollover_index true
|
|
538
|
+
index_date_pattern now/w{xxxx.ww}
|
|
539
|
+
deflector_alias myapp_deflector
|
|
540
|
+
enable_ilm true
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
# connection start
|
|
544
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
545
|
+
with(basic_auth: ['john', 'doe']).
|
|
546
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
547
|
+
# check if template exists
|
|
548
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
549
|
+
with(basic_auth: ['john', 'doe']).
|
|
550
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
551
|
+
# creation
|
|
552
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
553
|
+
with(basic_auth: ['john', 'doe']).
|
|
554
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
555
|
+
# check if alias exists
|
|
556
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
|
|
557
|
+
with(basic_auth: ['john', 'doe']).
|
|
558
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
559
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
|
|
560
|
+
with(basic_auth: ['john', 'doe']).
|
|
561
|
+
to_return(status: 404, body: "", headers: {})
|
|
562
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
|
|
563
|
+
with(basic_auth: ['john', 'doe'],
|
|
564
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"myapp_deflector-*\",\"order\":51}").
|
|
565
|
+
to_return(status: 200, body: "", headers: {})
|
|
566
|
+
# put the alias for the index
|
|
567
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
568
|
+
with(basic_auth: ['john', 'doe']).
|
|
569
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
570
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector").
|
|
571
|
+
with(basic_auth: ['john', 'doe'],
|
|
572
|
+
:body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
|
|
573
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
574
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
575
|
+
with(basic_auth: ['john', 'doe']).
|
|
576
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
577
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
578
|
+
with(basic_auth: ['john', 'doe']).
|
|
579
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
580
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
581
|
+
with(basic_auth: ['john', 'doe'],
|
|
582
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
|
583
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
584
|
+
|
|
585
|
+
driver(config)
|
|
586
|
+
|
|
587
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
588
|
+
driver.run(default_tag: 'test') do
|
|
589
|
+
driver.feed(sample_record)
|
|
590
|
+
end
|
|
591
|
+
|
|
592
|
+
assert_requested(elastic_request)
|
|
593
|
+
end
|
|
594
|
+
|
|
595
|
+
def test_template_create_with_rollover_index_and_custom_ilm
|
|
596
|
+
cwd = File.dirname(__FILE__)
|
|
597
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
598
|
+
|
|
599
|
+
config = %{
|
|
600
|
+
host logs.google.com
|
|
601
|
+
port 777
|
|
602
|
+
scheme https
|
|
603
|
+
path /es/
|
|
604
|
+
user john
|
|
605
|
+
password doe
|
|
606
|
+
template_name logstash
|
|
607
|
+
template_file #{template_file}
|
|
608
|
+
rollover_index true
|
|
609
|
+
index_date_pattern now/w{xxxx.ww}
|
|
610
|
+
deflector_alias myapp_deflector
|
|
611
|
+
ilm_policy_id fluentd-policy
|
|
612
|
+
enable_ilm true
|
|
613
|
+
ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
# connection start
|
|
617
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
618
|
+
with(basic_auth: ['john', 'doe']).
|
|
619
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
620
|
+
# check if template exists
|
|
621
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
622
|
+
with(basic_auth: ['john', 'doe']).
|
|
623
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
624
|
+
# creation
|
|
625
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
626
|
+
with(basic_auth: ['john', 'doe']).
|
|
627
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
628
|
+
# check if alias exists
|
|
629
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
|
|
630
|
+
with(basic_auth: ['john', 'doe']).
|
|
631
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
632
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
|
|
633
|
+
with(basic_auth: ['john', 'doe']).
|
|
634
|
+
to_return(status: 404, body: "", headers: {})
|
|
635
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
|
|
636
|
+
with(basic_auth: ['john', 'doe'],
|
|
637
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"myapp_deflector-*\",\"order\":51}").
|
|
638
|
+
to_return(status: 200, body: "", headers: {})
|
|
639
|
+
# put the alias for the index
|
|
640
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
641
|
+
with(basic_auth: ['john', 'doe']).
|
|
642
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
643
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector").
|
|
644
|
+
with(basic_auth: ['john', 'doe'],
|
|
645
|
+
:body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
|
|
646
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
647
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
648
|
+
with(basic_auth: ['john', 'doe']).
|
|
649
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
650
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
651
|
+
with(basic_auth: ['john', 'doe']).
|
|
652
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
653
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
654
|
+
with(basic_auth: ['john', 'doe'],
|
|
655
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
|
656
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
657
|
+
|
|
658
|
+
driver(config)
|
|
659
|
+
|
|
660
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
661
|
+
driver.run(default_tag: 'test') do
|
|
662
|
+
driver.feed(sample_record)
|
|
663
|
+
end
|
|
664
|
+
|
|
665
|
+
assert_requested(elastic_request)
|
|
666
|
+
end
|
|
667
|
+
|
|
668
|
+
def test_template_create_with_rollover_index_and_default_ilm_and_placeholders
|
|
669
|
+
cwd = File.dirname(__FILE__)
|
|
670
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
671
|
+
|
|
672
|
+
config = %{
|
|
673
|
+
host logs.google.com
|
|
674
|
+
port 777
|
|
675
|
+
scheme https
|
|
676
|
+
path /es/
|
|
677
|
+
user john
|
|
678
|
+
password doe
|
|
679
|
+
template_name logstash
|
|
680
|
+
template_file #{template_file}
|
|
681
|
+
rollover_index true
|
|
682
|
+
index_date_pattern now/w{xxxx.ww}
|
|
683
|
+
index_name fluentd-${tag}
|
|
684
|
+
deflector_alias myapp_deflector-${tag}
|
|
685
|
+
enable_ilm true
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
# connection start
|
|
689
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
690
|
+
with(basic_auth: ['john', 'doe']).
|
|
691
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
692
|
+
# check if template exists
|
|
693
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
|
|
694
|
+
with(basic_auth: ['john', 'doe']).
|
|
695
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
696
|
+
# creation
|
|
697
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
|
|
698
|
+
with(basic_auth: ['john', 'doe']).
|
|
699
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
700
|
+
# check if alias exists
|
|
701
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector-test").
|
|
702
|
+
with(basic_auth: ['john', 'doe']).
|
|
703
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
704
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector-test").
|
|
705
|
+
with(basic_auth: ['john', 'doe']).
|
|
706
|
+
to_return(status: 404, body: "", headers: {})
|
|
707
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector-test").
|
|
708
|
+
with(basic_auth: ['john', 'doe'],
|
|
709
|
+
body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector-test\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"myapp_deflector-test-*\",\"order\":52}").
|
|
710
|
+
to_return(status: 200, body: "", headers: {})
|
|
711
|
+
# put the alias for the index
|
|
712
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
713
|
+
with(basic_auth: ['john', 'doe']).
|
|
714
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
715
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector-test").
|
|
716
|
+
with(basic_auth: ['john', 'doe'],
|
|
717
|
+
:body => "{\"aliases\":{\"myapp_deflector-test\":{\"is_write_index\":true}}}").
|
|
718
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
719
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
720
|
+
with(basic_auth: ['john', 'doe']).
|
|
721
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
722
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
723
|
+
with(basic_auth: ['john', 'doe']).
|
|
724
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
725
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
|
|
726
|
+
with(basic_auth: ['john', 'doe'],
|
|
727
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
|
728
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
729
|
+
|
|
730
|
+
driver(config)
|
|
731
|
+
|
|
732
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
733
|
+
driver.run(default_tag: 'test') do
|
|
734
|
+
driver.feed(sample_record)
|
|
735
|
+
end
|
|
736
|
+
assert_equal('fluentd-test', index_cmds.first['index']['_index'])
|
|
737
|
+
|
|
738
|
+
assert_equal ["myapp_deflector-test"], driver.instance.alias_indexes
|
|
739
|
+
|
|
740
|
+
assert_requested(elastic_request)
|
|
741
|
+
end
|
|
742
|
+
end
|
|
743
|
+
|
|
476
744
|
def test_custom_template_create
|
|
477
745
|
cwd = File.dirname(__FILE__)
|
|
478
746
|
template_file = File.join(cwd, 'test_alias_template.json')
|
|
@@ -504,9 +772,52 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
|
504
772
|
|
|
505
773
|
driver(config)
|
|
506
774
|
|
|
775
|
+
stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
776
|
+
driver.run(default_tag: 'test') do
|
|
777
|
+
driver.feed(sample_record)
|
|
778
|
+
end
|
|
779
|
+
|
|
507
780
|
assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
|
|
508
781
|
end
|
|
509
782
|
|
|
783
|
+
def test_custom_template_installation_for_host_placeholder
|
|
784
|
+
cwd = File.dirname(__FILE__)
|
|
785
|
+
template_file = File.join(cwd, 'test_template.json')
|
|
786
|
+
|
|
787
|
+
config = %{
|
|
788
|
+
host logs-${tag}.google.com
|
|
789
|
+
port 777
|
|
790
|
+
scheme https
|
|
791
|
+
path /es/
|
|
792
|
+
user john
|
|
793
|
+
password doe
|
|
794
|
+
template_name logstash
|
|
795
|
+
template_file #{template_file}
|
|
796
|
+
verify_es_version_at_startup false
|
|
797
|
+
default_elasticsearch_version 6
|
|
798
|
+
customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
# connection start
|
|
802
|
+
stub_request(:head, "https://logs-test.google.com:777/es//").
|
|
803
|
+
with(basic_auth: ['john', 'doe']).
|
|
804
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
805
|
+
# check if template exists
|
|
806
|
+
stub_request(:get, "https://logs-test.google.com:777/es//_template/logstash").
|
|
807
|
+
with(basic_auth: ['john', 'doe']).
|
|
808
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
809
|
+
stub_request(:put, "https://logs-test.google.com:777/es//_template/logstash").
|
|
810
|
+
with(basic_auth: ['john', 'doe']).
|
|
811
|
+
to_return(status: 200, body: "", headers: {})
|
|
812
|
+
|
|
813
|
+
driver(config)
|
|
814
|
+
|
|
815
|
+
stub_elastic("https://logs-test.google.com:777/es//_bulk")
|
|
816
|
+
driver.run(default_tag: 'test') do
|
|
817
|
+
driver.feed(sample_record)
|
|
818
|
+
end
|
|
819
|
+
end
|
|
820
|
+
|
|
510
821
|
def test_custom_template_with_rollover_index_create
|
|
511
822
|
cwd = File.dirname(__FILE__)
|
|
512
823
|
template_file = File.join(cwd, 'test_alias_template.json')
|
|
@@ -555,9 +866,255 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
|
555
866
|
|
|
556
867
|
driver(config)
|
|
557
868
|
|
|
869
|
+
stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
870
|
+
driver.run(default_tag: 'test') do
|
|
871
|
+
driver.feed(sample_record)
|
|
872
|
+
end
|
|
873
|
+
|
|
558
874
|
assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
|
|
559
875
|
end
|
|
560
876
|
|
|
877
|
+
class CustomTemplateIndexLifecycleManagementTest < self
|
|
878
|
+
def setup
|
|
879
|
+
begin
|
|
880
|
+
require "elasticsearch/xpack"
|
|
881
|
+
rescue LoadError
|
|
882
|
+
omit "ILM testcase needs elasticsearch-xpack gem."
|
|
883
|
+
end
|
|
884
|
+
end
|
|
885
|
+
|
|
886
|
+
def test_custom_template_with_rollover_index_create_and_default_ilm
|
|
887
|
+
cwd = File.dirname(__FILE__)
|
|
888
|
+
template_file = File.join(cwd, 'test_alias_template.json')
|
|
889
|
+
|
|
890
|
+
config = %{
|
|
891
|
+
host logs.google.com
|
|
892
|
+
port 777
|
|
893
|
+
scheme https
|
|
894
|
+
path /es/
|
|
895
|
+
user john
|
|
896
|
+
password doe
|
|
897
|
+
template_name myapp_alias_template
|
|
898
|
+
template_file #{template_file}
|
|
899
|
+
customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
|
|
900
|
+
rollover_index true
|
|
901
|
+
index_date_pattern now/w{xxxx.ww}
|
|
902
|
+
deflector_alias myapp_deflector
|
|
903
|
+
index_prefix mylogs
|
|
904
|
+
application_name myapp
|
|
905
|
+
ilm_policy_id fluentd-policy
|
|
906
|
+
enable_ilm true
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
# connection start
|
|
910
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
911
|
+
with(basic_auth: ['john', 'doe']).
|
|
912
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
913
|
+
# check if template exists
|
|
914
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
|
|
915
|
+
with(basic_auth: ['john', 'doe']).
|
|
916
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
917
|
+
# creation
|
|
918
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
|
|
919
|
+
with(basic_auth: ['john', 'doe']).
|
|
920
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
921
|
+
# creation of index which can rollover
|
|
922
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
923
|
+
with(basic_auth: ['john', 'doe']).
|
|
924
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
925
|
+
# check if alias exists
|
|
926
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
|
|
927
|
+
with(basic_auth: ['john', 'doe']).
|
|
928
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
929
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
|
|
930
|
+
with(basic_auth: ['john', 'doe']).
|
|
931
|
+
to_return(status: 404, body: "", headers: {})
|
|
932
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
|
|
933
|
+
with(basic_auth: ['john', 'doe'],
|
|
934
|
+
body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"myapp_deflector-*\"}").
|
|
935
|
+
to_return(status: 200, body: "", headers: {})
|
|
936
|
+
# put the alias for the index
|
|
937
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector").
|
|
938
|
+
with(basic_auth: ['john', 'doe'],
|
|
939
|
+
:body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
|
|
940
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
941
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
942
|
+
with(basic_auth: ['john', 'doe']).
|
|
943
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
944
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
945
|
+
with(basic_auth: ['john', 'doe']).
|
|
946
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
947
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
948
|
+
with(basic_auth: ['john', 'doe'],
|
|
949
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
|
950
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
951
|
+
|
|
952
|
+
driver(config)
|
|
953
|
+
|
|
954
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
955
|
+
driver.run(default_tag: 'test') do
|
|
956
|
+
driver.feed(sample_record)
|
|
957
|
+
end
|
|
958
|
+
|
|
959
|
+
assert_requested(elastic_request)
|
|
960
|
+
end
|
|
961
|
+
|
|
962
|
+
def test_custom_template_with_rollover_index_create_and_default_ilm_and_placeholders
|
|
963
|
+
cwd = File.dirname(__FILE__)
|
|
964
|
+
template_file = File.join(cwd, 'test_alias_template.json')
|
|
965
|
+
|
|
966
|
+
config = %{
|
|
967
|
+
host logs.google.com
|
|
968
|
+
port 777
|
|
969
|
+
scheme https
|
|
970
|
+
path /es/
|
|
971
|
+
user john
|
|
972
|
+
password doe
|
|
973
|
+
template_name myapp_alias_template
|
|
974
|
+
template_file #{template_file}
|
|
975
|
+
customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
|
|
976
|
+
rollover_index true
|
|
977
|
+
index_date_pattern now/w{xxxx.ww}
|
|
978
|
+
index_name fluentd-${tag}
|
|
979
|
+
deflector_alias myapp_deflector-${tag}
|
|
980
|
+
index_prefix mylogs
|
|
981
|
+
application_name myapp
|
|
982
|
+
ilm_policy_id fluentd-policy
|
|
983
|
+
enable_ilm true
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
# connection start
|
|
987
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
988
|
+
with(basic_auth: ['john', 'doe']).
|
|
989
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
990
|
+
# check if template exists
|
|
991
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
|
|
992
|
+
with(basic_auth: ['john', 'doe']).
|
|
993
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
994
|
+
# creation
|
|
995
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
|
|
996
|
+
with(basic_auth: ['john', 'doe']).
|
|
997
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
998
|
+
# creation of index which can rollover
|
|
999
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1000
|
+
with(basic_auth: ['john', 'doe']).
|
|
1001
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1002
|
+
# check if alias exists
|
|
1003
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector-custom-test").
|
|
1004
|
+
with(basic_auth: ['john', 'doe']).
|
|
1005
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1006
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector-custom-test").
|
|
1007
|
+
with(basic_auth: ['john', 'doe']).
|
|
1008
|
+
to_return(status: 404, body: "", headers: {})
|
|
1009
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector-custom-test").
|
|
1010
|
+
with(basic_auth: ['john', 'doe'],
|
|
1011
|
+
body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector-custom-test\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"myapp_deflector-custom-test-*\"}").
|
|
1012
|
+
to_return(status: 200, body: "", headers: {})
|
|
1013
|
+
# put the alias for the index
|
|
1014
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector-custom-test").
|
|
1015
|
+
with(basic_auth: ['john', 'doe'],
|
|
1016
|
+
:body => "{\"aliases\":{\"myapp_deflector-custom-test\":{\"is_write_index\":true}}}").
|
|
1017
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1018
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1019
|
+
with(basic_auth: ['john', 'doe']).
|
|
1020
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1021
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1022
|
+
with(basic_auth: ['john', 'doe']).
|
|
1023
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1024
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1025
|
+
with(basic_auth: ['john', 'doe'],
|
|
1026
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1027
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1028
|
+
|
|
1029
|
+
driver(config)
|
|
1030
|
+
|
|
1031
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1032
|
+
driver.run(default_tag: 'custom-test') do
|
|
1033
|
+
driver.feed(sample_record)
|
|
1034
|
+
end
|
|
1035
|
+
assert_equal('fluentd-custom-test', index_cmds.first['index']['_index'])
|
|
1036
|
+
|
|
1037
|
+
assert_equal ["myapp_deflector-custom-test"], driver.instance.alias_indexes
|
|
1038
|
+
|
|
1039
|
+
assert_requested(elastic_request)
|
|
1040
|
+
end
|
|
1041
|
+
|
|
1042
|
+
def test_custom_template_with_rollover_index_create_and_custom_ilm
|
|
1043
|
+
cwd = File.dirname(__FILE__)
|
|
1044
|
+
template_file = File.join(cwd, 'test_alias_template.json')
|
|
1045
|
+
|
|
1046
|
+
config = %{
|
|
1047
|
+
host logs.google.com
|
|
1048
|
+
port 777
|
|
1049
|
+
scheme https
|
|
1050
|
+
path /es/
|
|
1051
|
+
user john
|
|
1052
|
+
password doe
|
|
1053
|
+
template_name myapp_alias_template
|
|
1054
|
+
template_file #{template_file}
|
|
1055
|
+
customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
|
|
1056
|
+
rollover_index true
|
|
1057
|
+
index_date_pattern now/w{xxxx.ww}
|
|
1058
|
+
deflector_alias myapp_deflector
|
|
1059
|
+
index_prefix mylogs
|
|
1060
|
+
application_name myapp
|
|
1061
|
+
ilm_policy_id fluentd-policy
|
|
1062
|
+
enable_ilm true
|
|
1063
|
+
ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
# connection start
|
|
1067
|
+
stub_request(:head, "https://logs.google.com:777/es//").
|
|
1068
|
+
with(basic_auth: ['john', 'doe']).
|
|
1069
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1070
|
+
# check if template exists
|
|
1071
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
|
|
1072
|
+
with(basic_auth: ['john', 'doe']).
|
|
1073
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1074
|
+
# creation
|
|
1075
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
|
|
1076
|
+
with(basic_auth: ['john', 'doe']).
|
|
1077
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1078
|
+
# creation of index which can rollover
|
|
1079
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
|
|
1080
|
+
with(basic_auth: ['john', 'doe']).
|
|
1081
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1082
|
+
# check if alias exists
|
|
1083
|
+
stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
|
|
1084
|
+
with(basic_auth: ['john', 'doe']).
|
|
1085
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1086
|
+
stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
|
|
1087
|
+
with(basic_auth: ['john', 'doe']).
|
|
1088
|
+
to_return(status: 404, body: "", headers: {})
|
|
1089
|
+
stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
|
|
1090
|
+
with(basic_auth: ['john', 'doe']).
|
|
1091
|
+
to_return(status: 200, body: "", headers: {})
|
|
1092
|
+
# put the alias for the index
|
|
1093
|
+
stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector").
|
|
1094
|
+
with(basic_auth: ['john', 'doe'],
|
|
1095
|
+
:body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
|
|
1096
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1097
|
+
stub_request(:get, "https://logs.google.com:777/es//_xpack").
|
|
1098
|
+
with(basic_auth: ['john', 'doe']).
|
|
1099
|
+
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
|
1100
|
+
stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1101
|
+
with(basic_auth: ['john', 'doe']).
|
|
1102
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1103
|
+
stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
|
|
1104
|
+
with(basic_auth: ['john', 'doe'],
|
|
1105
|
+
:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
|
|
1106
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1107
|
+
|
|
1108
|
+
driver(config)
|
|
1109
|
+
|
|
1110
|
+
elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1111
|
+
driver.run(default_tag: 'test') do
|
|
1112
|
+
driver.feed(sample_record)
|
|
1113
|
+
end
|
|
1114
|
+
|
|
1115
|
+
assert_requested(elastic_request)
|
|
1116
|
+
end
|
|
1117
|
+
end
|
|
561
1118
|
|
|
562
1119
|
def test_template_overwrite
|
|
563
1120
|
cwd = File.dirname(__FILE__)
|
|
@@ -590,6 +1147,11 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
|
590
1147
|
|
|
591
1148
|
driver(config)
|
|
592
1149
|
|
|
1150
|
+
stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1151
|
+
driver.run(default_tag: 'test') do
|
|
1152
|
+
driver.feed(sample_record)
|
|
1153
|
+
end
|
|
1154
|
+
|
|
593
1155
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
594
1156
|
end
|
|
595
1157
|
|
|
@@ -625,6 +1187,11 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
|
625
1187
|
|
|
626
1188
|
driver(config)
|
|
627
1189
|
|
|
1190
|
+
stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1191
|
+
driver.run(default_tag: 'test') do
|
|
1192
|
+
driver.feed(sample_record)
|
|
1193
|
+
end
|
|
1194
|
+
|
|
628
1195
|
assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
|
|
629
1196
|
end
|
|
630
1197
|
|
|
@@ -676,6 +1243,11 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
|
676
1243
|
|
|
677
1244
|
driver(config)
|
|
678
1245
|
|
|
1246
|
+
stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1247
|
+
driver.run(default_tag: 'test') do
|
|
1248
|
+
driver.feed(sample_record)
|
|
1249
|
+
end
|
|
1250
|
+
|
|
679
1251
|
assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
|
|
680
1252
|
end
|
|
681
1253
|
|
|
@@ -700,12 +1272,17 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
|
700
1272
|
with(basic_auth: ['john', 'doe']).
|
|
701
1273
|
to_return(:status => 404, :body => "", :headers => {})
|
|
702
1274
|
|
|
1275
|
+
driver(config)
|
|
1276
|
+
|
|
1277
|
+
stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
703
1278
|
assert_raise(RuntimeError) {
|
|
704
|
-
driver(
|
|
1279
|
+
driver.run(default_tag: 'test') do
|
|
1280
|
+
driver.feed(sample_record)
|
|
1281
|
+
end
|
|
705
1282
|
}
|
|
706
1283
|
end
|
|
707
1284
|
|
|
708
|
-
def
|
|
1285
|
+
def test_template_create_for_host_placeholder
|
|
709
1286
|
cwd = File.dirname(__FILE__)
|
|
710
1287
|
template_file = File.join(cwd, 'test_template.json')
|
|
711
1288
|
|
|
@@ -718,10 +1295,30 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
|
718
1295
|
password doe
|
|
719
1296
|
template_name logstash
|
|
720
1297
|
template_file #{template_file}
|
|
1298
|
+
verify_es_version_at_startup false
|
|
1299
|
+
default_elasticsearch_version 6
|
|
721
1300
|
}
|
|
722
1301
|
|
|
723
|
-
|
|
724
|
-
|
|
1302
|
+
# connection start
|
|
1303
|
+
stub_request(:head, "https://logs-test.google.com:777/es//").
|
|
1304
|
+
with(basic_auth: ['john', 'doe']).
|
|
1305
|
+
to_return(:status => 200, :body => "", :headers => {})
|
|
1306
|
+
# check if template exists
|
|
1307
|
+
stub_request(:get, "https://logs-test.google.com:777/es//_template/logstash").
|
|
1308
|
+
with(basic_auth: ['john', 'doe']).
|
|
1309
|
+
to_return(:status => 404, :body => "", :headers => {})
|
|
1310
|
+
stub_request(:put, "https://logs-test.google.com:777/es//_template/logstash").
|
|
1311
|
+
with(basic_auth: ['john', 'doe']).
|
|
1312
|
+
to_return(status: 200, body: "", headers: {})
|
|
1313
|
+
stub_request(:post, "https://logs-test.google.com:777/es//_bulk").
|
|
1314
|
+
with(basic_auth: ['john', 'doe']).
|
|
1315
|
+
to_return(status: 200, body: "", headers: {})
|
|
1316
|
+
|
|
1317
|
+
driver(config)
|
|
1318
|
+
|
|
1319
|
+
stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1320
|
+
driver.run(default_tag: 'test') do
|
|
1321
|
+
driver.feed(sample_record)
|
|
725
1322
|
end
|
|
726
1323
|
end
|
|
727
1324
|
|
|
@@ -749,8 +1346,11 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
|
749
1346
|
raise Faraday::ConnectionFailed, "Test message"
|
|
750
1347
|
end
|
|
751
1348
|
|
|
752
|
-
|
|
753
|
-
|
|
1349
|
+
driver(config)
|
|
1350
|
+
|
|
1351
|
+
stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1352
|
+
driver.run(default_tag: 'test') do
|
|
1353
|
+
driver.feed(sample_record)
|
|
754
1354
|
end
|
|
755
1355
|
|
|
756
1356
|
assert_equal(4, connection_resets)
|
|
@@ -783,6 +1383,11 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
|
783
1383
|
|
|
784
1384
|
driver(config)
|
|
785
1385
|
|
|
1386
|
+
stub_elastic("https://logs.google.com:778/es//_bulk")
|
|
1387
|
+
driver.run(default_tag: 'test') do
|
|
1388
|
+
driver.feed(sample_record)
|
|
1389
|
+
end
|
|
1390
|
+
|
|
786
1391
|
assert_equal(4, connection_resets)
|
|
787
1392
|
end
|
|
788
1393
|
|
|
@@ -918,6 +1523,11 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
|
918
1523
|
|
|
919
1524
|
driver(config)
|
|
920
1525
|
|
|
1526
|
+
stub_elastic("https://logs.google.com:777/es//_bulk")
|
|
1527
|
+
driver.run(default_tag: 'test') do
|
|
1528
|
+
driver.feed(sample_record)
|
|
1529
|
+
end
|
|
1530
|
+
|
|
921
1531
|
assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
|
|
922
1532
|
|
|
923
1533
|
assert_not_requested(:put, "https://logs.google.com:777/es//_template/logstash1")
|