fluent-plugin-elasticsearch 3.5.6 → 3.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,63 @@
1
+ require 'helper'
2
+ require 'elasticsearch'
3
+ require 'fluent/plugin/elasticsearch_index_lifecycle_management'
4
+
5
+ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
6
+ include Fluent::Plugin::ElasticsearchIndexLifecycleManagement
7
+
8
+ def setup
9
+ begin
10
+ require "elasticsearch/xpack"
11
+ rescue LoadError
12
+ omit "ILM testcase needs elasticsearch-xpack gem."
13
+ end
14
+ Fluent::Plugin::ElasticsearchIndexLifecycleManagement.module_eval(<<-CODE)
15
+ def client
16
+ Elasticsearch::Client.new url: 'localhost:9200'
17
+ end
18
+ def log
19
+ log_device = Fluent::Test::DummyLogDevice.new
20
+ dl_opts = {:log_level => ServerEngine::DaemonLogger::INFO}
21
+ logger = ServerEngine::DaemonLogger.new(log_device, dl_opts)
22
+ Fluent::Log.new(logger)
23
+ end
24
+ CODE
25
+ end
26
+
27
+ def test_xpack_info
28
+ stub_request(:get, "http://localhost:9200/_xpack").
29
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
30
+ expected = {"features"=>{"ilm"=>{"available"=>true, "enabled"=>true}}}
31
+ assert_equal(expected, xpack_info)
32
+ end
33
+
34
+ def test_verify_ilm_working
35
+ stub_request(:get, "http://localhost:9200/_xpack").
36
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
37
+ assert_nothing_raised { verify_ilm_working }
38
+ end
39
+
40
+ def test_ilm_policy_doesnt_exists
41
+ stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluentd-policy%22%7D").
42
+ to_return(:status => 404, :body => "", :headers => {})
43
+ assert_false(ilm_policy_exists?(policy_id: "fluentd-policy"))
44
+ end
45
+
46
+ def test_ilm_policy_exists
47
+ stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluent-policy%22%7D").
48
+ to_return(:status => 200, :body => "", :headers => {})
49
+ assert_true(ilm_policy_exists?(policy_id: "fluent-policy"))
50
+ end
51
+
52
+ def test_create_ilm_policy
53
+ stub_request(:get, "http://localhost:9200/_ilm/policy/fluent-policy").
54
+ to_return(:status => 404, :body => "", :headers => {})
55
+ stub_request(:put, "http://localhost:9200/_ilm/policy/fluent-policy").
56
+ with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
57
+ :headers => {'Content-Type'=>'application/json'}).
58
+ to_return(:status => 200, :body => "", :headers => {})
59
+ create_ilm_policy("fluent-policy")
60
+
61
+ assert_requested(:put, "http://localhost:9200/_ilm/policy/fluent-policy", times: 1)
62
+ end
63
+ end
@@ -269,6 +269,20 @@ class ElasticsearchOutput < Test::Unit::TestCase
269
269
  }
270
270
  end
271
271
 
272
+ test 'invalid configuration of index lifecycle management' do
273
+ cwd = File.dirname(__FILE__)
274
+ template_file = File.join(cwd, 'test_template.json')
275
+
276
+ config = %{
277
+ enable_ilm true
278
+ template_name logstash
279
+ template_file #{template_file}
280
+ }
281
+ assert_raise(Fluent::ConfigError) {
282
+ driver(config)
283
+ }
284
+ end
285
+
272
286
  test 'Detected Elasticsearch 7' do
273
287
  config = %{
274
288
  type_name changed
@@ -347,6 +361,27 @@ class ElasticsearchOutput < Test::Unit::TestCase
347
361
  end
348
362
  end
349
363
 
364
+ test 'Detected exclusive features which are host placeholder, template installation, and verify Elasticsearch version at startup' do
365
+ cwd = File.dirname(__FILE__)
366
+ template_file = File.join(cwd, 'test_template.json')
367
+
368
+ assert_raise_message(/host placeholder, template installation, and verify Elasticsearch version at startup are exclusive feature at same time./) do
369
+ config = %{
370
+ host logs-${tag}.google.com
371
+ port 777
372
+ scheme https
373
+ path /es/
374
+ user john
375
+ password doe
376
+ template_name logstash
377
+ template_file #{template_file}
378
+ verify_es_version_at_startup true
379
+ default_elasticsearch_version 6
380
+ }
381
+ driver(config)
382
+ end
383
+ end
384
+
350
385
  sub_test_case 'connection exceptions' do
351
386
  test 'default connection exception' do
352
387
  driver(Fluent::Config::Element.new(
@@ -469,10 +504,243 @@ class ElasticsearchOutput < Test::Unit::TestCase
469
504
  to_return(:status => 200, :body => "", :headers => {})
470
505
 
471
506
  driver(config)
472
-
507
+ stub_elastic("https://logs.google.com:777/es//_bulk")
508
+ driver.run(default_tag: 'test') do
509
+ driver.feed(sample_record)
510
+ end
473
511
  assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
474
512
  end
475
513
 
514
+ class TemplateIndexLifecycleManagementTest < self
515
+ def setup
516
+ begin
517
+ require "elasticsearch/xpack"
518
+ rescue LoadError
519
+ omit "ILM testcase needs elasticsearch-xpack gem."
520
+ end
521
+ end
522
+
523
+
524
+ def test_template_create_with_rollover_index_and_default_ilm
525
+ cwd = File.dirname(__FILE__)
526
+ template_file = File.join(cwd, 'test_template.json')
527
+
528
+ config = %{
529
+ host logs.google.com
530
+ port 777
531
+ scheme https
532
+ path /es/
533
+ user john
534
+ password doe
535
+ template_name logstash
536
+ template_file #{template_file}
537
+ rollover_index true
538
+ index_date_pattern now/w{xxxx.ww}
539
+ deflector_alias myapp_deflector
540
+ enable_ilm true
541
+ }
542
+
543
+ # connection start
544
+ stub_request(:head, "https://logs.google.com:777/es//").
545
+ with(basic_auth: ['john', 'doe']).
546
+ to_return(:status => 200, :body => "", :headers => {})
547
+ # check if template exists
548
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
549
+ with(basic_auth: ['john', 'doe']).
550
+ to_return(:status => 404, :body => "", :headers => {})
551
+ # creation
552
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
553
+ with(basic_auth: ['john', 'doe']).
554
+ to_return(:status => 200, :body => "", :headers => {})
555
+ # check if alias exists
556
+ stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
557
+ with(basic_auth: ['john', 'doe']).
558
+ to_return(:status => 404, :body => "", :headers => {})
559
+ stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
560
+ with(basic_auth: ['john', 'doe']).
561
+ to_return(status: 404, body: "", headers: {})
562
+ stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
563
+ with(basic_auth: ['john', 'doe'],
564
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"myapp_deflector-*\",\"order\":51}").
565
+ to_return(status: 200, body: "", headers: {})
566
+ # put the alias for the index
567
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
568
+ with(basic_auth: ['john', 'doe']).
569
+ to_return(:status => 200, :body => "", :headers => {})
570
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector").
571
+ with(basic_auth: ['john', 'doe'],
572
+ :body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
573
+ to_return(:status => 200, :body => "", :headers => {})
574
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
575
+ with(basic_auth: ['john', 'doe']).
576
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
577
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
578
+ with(basic_auth: ['john', 'doe']).
579
+ to_return(:status => 404, :body => "", :headers => {})
580
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
581
+ with(basic_auth: ['john', 'doe'],
582
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
583
+ to_return(:status => 200, :body => "", :headers => {})
584
+
585
+ driver(config)
586
+
587
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
588
+ driver.run(default_tag: 'test') do
589
+ driver.feed(sample_record)
590
+ end
591
+
592
+ assert_requested(elastic_request)
593
+ end
594
+
595
+ def test_template_create_with_rollover_index_and_custom_ilm
596
+ cwd = File.dirname(__FILE__)
597
+ template_file = File.join(cwd, 'test_template.json')
598
+
599
+ config = %{
600
+ host logs.google.com
601
+ port 777
602
+ scheme https
603
+ path /es/
604
+ user john
605
+ password doe
606
+ template_name logstash
607
+ template_file #{template_file}
608
+ rollover_index true
609
+ index_date_pattern now/w{xxxx.ww}
610
+ deflector_alias myapp_deflector
611
+ ilm_policy_id fluentd-policy
612
+ enable_ilm true
613
+ ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}
614
+ }
615
+
616
+ # connection start
617
+ stub_request(:head, "https://logs.google.com:777/es//").
618
+ with(basic_auth: ['john', 'doe']).
619
+ to_return(:status => 200, :body => "", :headers => {})
620
+ # check if template exists
621
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
622
+ with(basic_auth: ['john', 'doe']).
623
+ to_return(:status => 404, :body => "", :headers => {})
624
+ # creation
625
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
626
+ with(basic_auth: ['john', 'doe']).
627
+ to_return(:status => 200, :body => "", :headers => {})
628
+ # check if alias exists
629
+ stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
630
+ with(basic_auth: ['john', 'doe']).
631
+ to_return(:status => 404, :body => "", :headers => {})
632
+ stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
633
+ with(basic_auth: ['john', 'doe']).
634
+ to_return(status: 404, body: "", headers: {})
635
+ stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
636
+ with(basic_auth: ['john', 'doe'],
637
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"myapp_deflector-*\",\"order\":51}").
638
+ to_return(status: 200, body: "", headers: {})
639
+ # put the alias for the index
640
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
641
+ with(basic_auth: ['john', 'doe']).
642
+ to_return(:status => 200, :body => "", :headers => {})
643
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector").
644
+ with(basic_auth: ['john', 'doe'],
645
+ :body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
646
+ to_return(:status => 200, :body => "", :headers => {})
647
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
648
+ with(basic_auth: ['john', 'doe']).
649
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
650
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
651
+ with(basic_auth: ['john', 'doe']).
652
+ to_return(:status => 404, :body => "", :headers => {})
653
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
654
+ with(basic_auth: ['john', 'doe'],
655
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
656
+ to_return(:status => 200, :body => "", :headers => {})
657
+
658
+ driver(config)
659
+
660
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
661
+ driver.run(default_tag: 'test') do
662
+ driver.feed(sample_record)
663
+ end
664
+
665
+ assert_requested(elastic_request)
666
+ end
667
+
668
+ def test_template_create_with_rollover_index_and_default_ilm_and_placeholders
669
+ cwd = File.dirname(__FILE__)
670
+ template_file = File.join(cwd, 'test_template.json')
671
+
672
+ config = %{
673
+ host logs.google.com
674
+ port 777
675
+ scheme https
676
+ path /es/
677
+ user john
678
+ password doe
679
+ template_name logstash
680
+ template_file #{template_file}
681
+ rollover_index true
682
+ index_date_pattern now/w{xxxx.ww}
683
+ index_name fluentd-${tag}
684
+ deflector_alias myapp_deflector-${tag}
685
+ enable_ilm true
686
+ }
687
+
688
+ # connection start
689
+ stub_request(:head, "https://logs.google.com:777/es//").
690
+ with(basic_auth: ['john', 'doe']).
691
+ to_return(:status => 200, :body => "", :headers => {})
692
+ # check if template exists
693
+ stub_request(:get, "https://logs.google.com:777/es//_template/logstash").
694
+ with(basic_auth: ['john', 'doe']).
695
+ to_return(:status => 404, :body => "", :headers => {})
696
+ # creation
697
+ stub_request(:put, "https://logs.google.com:777/es//_template/logstash").
698
+ with(basic_auth: ['john', 'doe']).
699
+ to_return(:status => 200, :body => "", :headers => {})
700
+ # check if alias exists
701
+ stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector-test").
702
+ with(basic_auth: ['john', 'doe']).
703
+ to_return(:status => 404, :body => "", :headers => {})
704
+ stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector-test").
705
+ with(basic_auth: ['john', 'doe']).
706
+ to_return(status: 404, body: "", headers: {})
707
+ stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector-test").
708
+ with(basic_auth: ['john', 'doe'],
709
+ body: "{\"settings\":{\"number_of_shards\":1,\"index.lifecycle.name\":\"logstash-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector-test\"},\"mappings\":{\"type1\":{\"_source\":{\"enabled\":false},\"properties\":{\"host_name\":{\"type\":\"string\",\"index\":\"not_analyzed\"},\"created_at\":{\"type\":\"date\",\"format\":\"EEE MMM dd HH:mm:ss Z YYYY\"}}}},\"index_patterns\":\"myapp_deflector-test-*\",\"order\":52}").
710
+ to_return(status: 200, body: "", headers: {})
711
+ # put the alias for the index
712
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
713
+ with(basic_auth: ['john', 'doe']).
714
+ to_return(:status => 200, :body => "", :headers => {})
715
+ stub_request(:put, "https://logs.google.com:777/es//%3Clogstash-default-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector-test").
716
+ with(basic_auth: ['john', 'doe'],
717
+ :body => "{\"aliases\":{\"myapp_deflector-test\":{\"is_write_index\":true}}}").
718
+ to_return(:status => 200, :body => "", :headers => {})
719
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
720
+ with(basic_auth: ['john', 'doe']).
721
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
722
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
723
+ with(basic_auth: ['john', 'doe']).
724
+ to_return(:status => 404, :body => "", :headers => {})
725
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/logstash-policy").
726
+ with(basic_auth: ['john', 'doe'],
727
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
728
+ to_return(:status => 200, :body => "", :headers => {})
729
+
730
+ driver(config)
731
+
732
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
733
+ driver.run(default_tag: 'test') do
734
+ driver.feed(sample_record)
735
+ end
736
+ assert_equal('fluentd-test', index_cmds.first['index']['_index'])
737
+
738
+ assert_equal ["myapp_deflector-test"], driver.instance.alias_indexes
739
+
740
+ assert_requested(elastic_request)
741
+ end
742
+ end
743
+
476
744
  def test_custom_template_create
477
745
  cwd = File.dirname(__FILE__)
478
746
  template_file = File.join(cwd, 'test_alias_template.json')
@@ -504,9 +772,52 @@ class ElasticsearchOutput < Test::Unit::TestCase
504
772
 
505
773
  driver(config)
506
774
 
775
+ stub_elastic("https://logs.google.com:777/es//_bulk")
776
+ driver.run(default_tag: 'test') do
777
+ driver.feed(sample_record)
778
+ end
779
+
507
780
  assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
508
781
  end
509
782
 
783
+ def test_custom_template_installation_for_host_placeholder
784
+ cwd = File.dirname(__FILE__)
785
+ template_file = File.join(cwd, 'test_template.json')
786
+
787
+ config = %{
788
+ host logs-${tag}.google.com
789
+ port 777
790
+ scheme https
791
+ path /es/
792
+ user john
793
+ password doe
794
+ template_name logstash
795
+ template_file #{template_file}
796
+ verify_es_version_at_startup false
797
+ default_elasticsearch_version 6
798
+ customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
799
+ }
800
+
801
+ # connection start
802
+ stub_request(:head, "https://logs-test.google.com:777/es//").
803
+ with(basic_auth: ['john', 'doe']).
804
+ to_return(:status => 200, :body => "", :headers => {})
805
+ # check if template exists
806
+ stub_request(:get, "https://logs-test.google.com:777/es//_template/logstash").
807
+ with(basic_auth: ['john', 'doe']).
808
+ to_return(:status => 404, :body => "", :headers => {})
809
+ stub_request(:put, "https://logs-test.google.com:777/es//_template/logstash").
810
+ with(basic_auth: ['john', 'doe']).
811
+ to_return(status: 200, body: "", headers: {})
812
+
813
+ driver(config)
814
+
815
+ stub_elastic("https://logs-test.google.com:777/es//_bulk")
816
+ driver.run(default_tag: 'test') do
817
+ driver.feed(sample_record)
818
+ end
819
+ end
820
+
510
821
  def test_custom_template_with_rollover_index_create
511
822
  cwd = File.dirname(__FILE__)
512
823
  template_file = File.join(cwd, 'test_alias_template.json')
@@ -555,9 +866,255 @@ class ElasticsearchOutput < Test::Unit::TestCase
555
866
 
556
867
  driver(config)
557
868
 
869
+ stub_elastic("https://logs.google.com:777/es//_bulk")
870
+ driver.run(default_tag: 'test') do
871
+ driver.feed(sample_record)
872
+ end
873
+
558
874
  assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
559
875
  end
560
876
 
877
+ class CustomTemplateIndexLifecycleManagementTest < self
878
+ def setup
879
+ begin
880
+ require "elasticsearch/xpack"
881
+ rescue LoadError
882
+ omit "ILM testcase needs elasticsearch-xpack gem."
883
+ end
884
+ end
885
+
886
+ def test_custom_template_with_rollover_index_create_and_default_ilm
887
+ cwd = File.dirname(__FILE__)
888
+ template_file = File.join(cwd, 'test_alias_template.json')
889
+
890
+ config = %{
891
+ host logs.google.com
892
+ port 777
893
+ scheme https
894
+ path /es/
895
+ user john
896
+ password doe
897
+ template_name myapp_alias_template
898
+ template_file #{template_file}
899
+ customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
900
+ rollover_index true
901
+ index_date_pattern now/w{xxxx.ww}
902
+ deflector_alias myapp_deflector
903
+ index_prefix mylogs
904
+ application_name myapp
905
+ ilm_policy_id fluentd-policy
906
+ enable_ilm true
907
+ }
908
+
909
+ # connection start
910
+ stub_request(:head, "https://logs.google.com:777/es//").
911
+ with(basic_auth: ['john', 'doe']).
912
+ to_return(:status => 200, :body => "", :headers => {})
913
+ # check if template exists
914
+ stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
915
+ with(basic_auth: ['john', 'doe']).
916
+ to_return(:status => 404, :body => "", :headers => {})
917
+ # creation
918
+ stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
919
+ with(basic_auth: ['john', 'doe']).
920
+ to_return(:status => 200, :body => "", :headers => {})
921
+ # creation of index which can rollover
922
+ stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
923
+ with(basic_auth: ['john', 'doe']).
924
+ to_return(:status => 200, :body => "", :headers => {})
925
+ # check if alias exists
926
+ stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
927
+ with(basic_auth: ['john', 'doe']).
928
+ to_return(:status => 404, :body => "", :headers => {})
929
+ stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
930
+ with(basic_auth: ['john', 'doe']).
931
+ to_return(status: 404, body: "", headers: {})
932
+ stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
933
+ with(basic_auth: ['john', 'doe'],
934
+ body: "{\"order\":6,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"myapp_deflector-*\"}").
935
+ to_return(status: 200, body: "", headers: {})
936
+ # put the alias for the index
937
+ stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector").
938
+ with(basic_auth: ['john', 'doe'],
939
+ :body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
940
+ to_return(:status => 200, :body => "", :headers => {})
941
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
942
+ with(basic_auth: ['john', 'doe']).
943
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
944
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
945
+ with(basic_auth: ['john', 'doe']).
946
+ to_return(:status => 404, :body => "", :headers => {})
947
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
948
+ with(basic_auth: ['john', 'doe'],
949
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
950
+ to_return(:status => 200, :body => "", :headers => {})
951
+
952
+ driver(config)
953
+
954
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
955
+ driver.run(default_tag: 'test') do
956
+ driver.feed(sample_record)
957
+ end
958
+
959
+ assert_requested(elastic_request)
960
+ end
961
+
962
+ def test_custom_template_with_rollover_index_create_and_default_ilm_and_placeholders
963
+ cwd = File.dirname(__FILE__)
964
+ template_file = File.join(cwd, 'test_alias_template.json')
965
+
966
+ config = %{
967
+ host logs.google.com
968
+ port 777
969
+ scheme https
970
+ path /es/
971
+ user john
972
+ password doe
973
+ template_name myapp_alias_template
974
+ template_file #{template_file}
975
+ customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
976
+ rollover_index true
977
+ index_date_pattern now/w{xxxx.ww}
978
+ index_name fluentd-${tag}
979
+ deflector_alias myapp_deflector-${tag}
980
+ index_prefix mylogs
981
+ application_name myapp
982
+ ilm_policy_id fluentd-policy
983
+ enable_ilm true
984
+ }
985
+
986
+ # connection start
987
+ stub_request(:head, "https://logs.google.com:777/es//").
988
+ with(basic_auth: ['john', 'doe']).
989
+ to_return(:status => 200, :body => "", :headers => {})
990
+ # check if template exists
991
+ stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
992
+ with(basic_auth: ['john', 'doe']).
993
+ to_return(:status => 404, :body => "", :headers => {})
994
+ # creation
995
+ stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
996
+ with(basic_auth: ['john', 'doe']).
997
+ to_return(:status => 200, :body => "", :headers => {})
998
+ # creation of index which can rollover
999
+ stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1000
+ with(basic_auth: ['john', 'doe']).
1001
+ to_return(:status => 200, :body => "", :headers => {})
1002
+ # check if alias exists
1003
+ stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector-custom-test").
1004
+ with(basic_auth: ['john', 'doe']).
1005
+ to_return(:status => 404, :body => "", :headers => {})
1006
+ stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector-custom-test").
1007
+ with(basic_auth: ['john', 'doe']).
1008
+ to_return(status: 404, body: "", headers: {})
1009
+ stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector-custom-test").
1010
+ with(basic_auth: ['john', 'doe'],
1011
+ body: "{\"order\":8,\"settings\":{\"index.lifecycle.name\":\"fluentd-policy\",\"index.lifecycle.rollover_alias\":\"myapp_deflector-custom-test\"},\"mappings\":{},\"aliases\":{\"myapp-logs-alias\":{}},\"index_patterns\":\"myapp_deflector-custom-test-*\"}").
1012
+ to_return(status: 200, body: "", headers: {})
1013
+ # put the alias for the index
1014
+ stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector-custom-test").
1015
+ with(basic_auth: ['john', 'doe'],
1016
+ :body => "{\"aliases\":{\"myapp_deflector-custom-test\":{\"is_write_index\":true}}}").
1017
+ to_return(:status => 200, :body => "", :headers => {})
1018
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1019
+ with(basic_auth: ['john', 'doe']).
1020
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1021
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1022
+ with(basic_auth: ['john', 'doe']).
1023
+ to_return(:status => 404, :body => "", :headers => {})
1024
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1025
+ with(basic_auth: ['john', 'doe'],
1026
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}").
1027
+ to_return(:status => 200, :body => "", :headers => {})
1028
+
1029
+ driver(config)
1030
+
1031
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1032
+ driver.run(default_tag: 'custom-test') do
1033
+ driver.feed(sample_record)
1034
+ end
1035
+ assert_equal('fluentd-custom-test', index_cmds.first['index']['_index'])
1036
+
1037
+ assert_equal ["myapp_deflector-custom-test"], driver.instance.alias_indexes
1038
+
1039
+ assert_requested(elastic_request)
1040
+ end
1041
+
1042
+ def test_custom_template_with_rollover_index_create_and_custom_ilm
1043
+ cwd = File.dirname(__FILE__)
1044
+ template_file = File.join(cwd, 'test_alias_template.json')
1045
+
1046
+ config = %{
1047
+ host logs.google.com
1048
+ port 777
1049
+ scheme https
1050
+ path /es/
1051
+ user john
1052
+ password doe
1053
+ template_name myapp_alias_template
1054
+ template_file #{template_file}
1055
+ customize_template {"--appid--": "myapp-logs","--index_prefix--":"mylogs"}
1056
+ rollover_index true
1057
+ index_date_pattern now/w{xxxx.ww}
1058
+ deflector_alias myapp_deflector
1059
+ index_prefix mylogs
1060
+ application_name myapp
1061
+ ilm_policy_id fluentd-policy
1062
+ enable_ilm true
1063
+ ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_size":"70gb", "max_age":"30d"}}}}}}
1064
+ }
1065
+
1066
+ # connection start
1067
+ stub_request(:head, "https://logs.google.com:777/es//").
1068
+ with(basic_auth: ['john', 'doe']).
1069
+ to_return(:status => 200, :body => "", :headers => {})
1070
+ # check if template exists
1071
+ stub_request(:get, "https://logs.google.com:777/es//_template/myapp_alias_template").
1072
+ with(basic_auth: ['john', 'doe']).
1073
+ to_return(:status => 404, :body => "", :headers => {})
1074
+ # creation
1075
+ stub_request(:put, "https://logs.google.com:777/es//_template/myapp_alias_template").
1076
+ with(basic_auth: ['john', 'doe']).
1077
+ to_return(:status => 200, :body => "", :headers => {})
1078
+ # creation of index which can rollover
1079
+ stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E").
1080
+ with(basic_auth: ['john', 'doe']).
1081
+ to_return(:status => 200, :body => "", :headers => {})
1082
+ # check if alias exists
1083
+ stub_request(:head, "https://logs.google.com:777/es//_alias/myapp_deflector").
1084
+ with(basic_auth: ['john', 'doe']).
1085
+ to_return(:status => 404, :body => "", :headers => {})
1086
+ stub_request(:get, "https://logs.google.com:777/es//_template/myapp_deflector").
1087
+ with(basic_auth: ['john', 'doe']).
1088
+ to_return(status: 404, body: "", headers: {})
1089
+ stub_request(:put, "https://logs.google.com:777/es//_template/myapp_deflector").
1090
+ with(basic_auth: ['john', 'doe']).
1091
+ to_return(status: 200, body: "", headers: {})
1092
+ # put the alias for the index
1093
+ stub_request(:put, "https://logs.google.com:777/es//%3Cmylogs-myapp-%7Bnow%2Fw%7Bxxxx.ww%7D%7D-000001%3E/_alias/myapp_deflector").
1094
+ with(basic_auth: ['john', 'doe'],
1095
+ :body => "{\"aliases\":{\"myapp_deflector\":{\"is_write_index\":true}}}").
1096
+ to_return(:status => 200, :body => "", :headers => {})
1097
+ stub_request(:get, "https://logs.google.com:777/es//_xpack").
1098
+ with(basic_auth: ['john', 'doe']).
1099
+ to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
1100
+ stub_request(:get, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1101
+ with(basic_auth: ['john', 'doe']).
1102
+ to_return(:status => 404, :body => "", :headers => {})
1103
+ stub_request(:put, "https://logs.google.com:777/es//_ilm/policy/fluentd-policy").
1104
+ with(basic_auth: ['john', 'doe'],
1105
+ :body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"70gb\",\"max_age\":\"30d\"}}}}}}").
1106
+ to_return(:status => 200, :body => "", :headers => {})
1107
+
1108
+ driver(config)
1109
+
1110
+ elastic_request = stub_elastic("https://logs.google.com:777/es//_bulk")
1111
+ driver.run(default_tag: 'test') do
1112
+ driver.feed(sample_record)
1113
+ end
1114
+
1115
+ assert_requested(elastic_request)
1116
+ end
1117
+ end
561
1118
 
562
1119
  def test_template_overwrite
563
1120
  cwd = File.dirname(__FILE__)
@@ -590,6 +1147,11 @@ class ElasticsearchOutput < Test::Unit::TestCase
590
1147
 
591
1148
  driver(config)
592
1149
 
1150
+ stub_elastic("https://logs.google.com:777/es//_bulk")
1151
+ driver.run(default_tag: 'test') do
1152
+ driver.feed(sample_record)
1153
+ end
1154
+
593
1155
  assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
594
1156
  end
595
1157
 
@@ -625,6 +1187,11 @@ class ElasticsearchOutput < Test::Unit::TestCase
625
1187
 
626
1188
  driver(config)
627
1189
 
1190
+ stub_elastic("https://logs.google.com:777/es//_bulk")
1191
+ driver.run(default_tag: 'test') do
1192
+ driver.feed(sample_record)
1193
+ end
1194
+
628
1195
  assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
629
1196
  end
630
1197
 
@@ -676,6 +1243,11 @@ class ElasticsearchOutput < Test::Unit::TestCase
676
1243
 
677
1244
  driver(config)
678
1245
 
1246
+ stub_elastic("https://logs.google.com:777/es//_bulk")
1247
+ driver.run(default_tag: 'test') do
1248
+ driver.feed(sample_record)
1249
+ end
1250
+
679
1251
  assert_requested(:put, "https://logs.google.com:777/es//_template/myapp_alias_template", times: 1)
680
1252
  end
681
1253
 
@@ -700,12 +1272,17 @@ class ElasticsearchOutput < Test::Unit::TestCase
700
1272
  with(basic_auth: ['john', 'doe']).
701
1273
  to_return(:status => 404, :body => "", :headers => {})
702
1274
 
1275
+ driver(config)
1276
+
1277
+ stub_elastic("https://logs.google.com:777/es//_bulk")
703
1278
  assert_raise(RuntimeError) {
704
- driver(config)
1279
+ driver.run(default_tag: 'test') do
1280
+ driver.feed(sample_record)
1281
+ end
705
1282
  }
706
1283
  end
707
1284
 
708
- def test_template_installation_exclusive_for_host_placeholder
1285
+ def test_template_create_for_host_placeholder
709
1286
  cwd = File.dirname(__FILE__)
710
1287
  template_file = File.join(cwd, 'test_template.json')
711
1288
 
@@ -718,10 +1295,30 @@ class ElasticsearchOutput < Test::Unit::TestCase
718
1295
  password doe
719
1296
  template_name logstash
720
1297
  template_file #{template_file}
1298
+ verify_es_version_at_startup false
1299
+ default_elasticsearch_version 6
721
1300
  }
722
1301
 
723
- assert_raise(Fluent::ConfigError) do
724
- driver(config)
1302
+ # connection start
1303
+ stub_request(:head, "https://logs-test.google.com:777/es//").
1304
+ with(basic_auth: ['john', 'doe']).
1305
+ to_return(:status => 200, :body => "", :headers => {})
1306
+ # check if template exists
1307
+ stub_request(:get, "https://logs-test.google.com:777/es//_template/logstash").
1308
+ with(basic_auth: ['john', 'doe']).
1309
+ to_return(:status => 404, :body => "", :headers => {})
1310
+ stub_request(:put, "https://logs-test.google.com:777/es//_template/logstash").
1311
+ with(basic_auth: ['john', 'doe']).
1312
+ to_return(status: 200, body: "", headers: {})
1313
+ stub_request(:post, "https://logs-test.google.com:777/es//_bulk").
1314
+ with(basic_auth: ['john', 'doe']).
1315
+ to_return(status: 200, body: "", headers: {})
1316
+
1317
+ driver(config)
1318
+
1319
+ stub_elastic("https://logs.google.com:777/es//_bulk")
1320
+ driver.run(default_tag: 'test') do
1321
+ driver.feed(sample_record)
725
1322
  end
726
1323
  end
727
1324
 
@@ -749,8 +1346,11 @@ class ElasticsearchOutput < Test::Unit::TestCase
749
1346
  raise Faraday::ConnectionFailed, "Test message"
750
1347
  end
751
1348
 
752
- assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
753
- driver(config)
1349
+ driver(config)
1350
+
1351
+ stub_elastic("https://logs.google.com:777/es//_bulk")
1352
+ driver.run(default_tag: 'test') do
1353
+ driver.feed(sample_record)
754
1354
  end
755
1355
 
756
1356
  assert_equal(4, connection_resets)
@@ -783,6 +1383,11 @@ class ElasticsearchOutput < Test::Unit::TestCase
783
1383
 
784
1384
  driver(config)
785
1385
 
1386
+ stub_elastic("https://logs.google.com:778/es//_bulk")
1387
+ driver.run(default_tag: 'test') do
1388
+ driver.feed(sample_record)
1389
+ end
1390
+
786
1391
  assert_equal(4, connection_resets)
787
1392
  end
788
1393
 
@@ -918,6 +1523,11 @@ class ElasticsearchOutput < Test::Unit::TestCase
918
1523
 
919
1524
  driver(config)
920
1525
 
1526
+ stub_elastic("https://logs.google.com:777/es//_bulk")
1527
+ driver.run(default_tag: 'test') do
1528
+ driver.feed(sample_record)
1529
+ end
1530
+
921
1531
  assert_requested(:put, "https://logs.google.com:777/es//_template/logstash", times: 1)
922
1532
 
923
1533
  assert_not_requested(:put, "https://logs.google.com:777/es//_template/logstash1")