fluent-plugin-elb-access-log 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: e6cf91d3c27190b1dc3433a92e5710e803616256
4
- data.tar.gz: 60e98a16ff1535f1a3bf57807255aa8403eaac01
3
+ metadata.gz: d3a20dc8a60a8ce35bae8050d88f6f3cb89fd5cd
4
+ data.tar.gz: b9df7709325ba5f796658df242b7d04727f65f96
5
5
  SHA512:
6
- metadata.gz: 1e1945d726b97db0f0ad9b7038b98958bac3196ec51d77dbf72e04afaf078064c68a7dcb1132df0709f9d1921bc00d7d1e6360a2fad93f6446d4043f903ffa2c
7
- data.tar.gz: '0495808a43015a235ba716d5bd43a36f40111014f25bfa4960da80dfd7cb3da7e1174ee0c67e5d6f6b678a900d76cd5f5ec6325604321acdc90727db3bcf39ec'
6
+ metadata.gz: 41d7b97d9044b10ea91d65d20556024fa1f357e9e6ba1cb3c8e3827f553b5258ee8b96dc3e964acaba24eba0780f2078ed6517868eeaa28a2057a6efd0ecf7f1
7
+ data.tar.gz: 9f3f1b43320c87f24112a6187e65c1bb293975b7f3db0816eb16c987bcdf4d5b6ad594d0950235d4b164760fc22a6b9997a99820cc158185d5e433528453a721
data/README.md CHANGED
@@ -53,6 +53,7 @@ Or install it yourself as:
53
53
  #type_cast true
54
54
  #parse_request true
55
55
  #split_addr_port true
56
+ #file_filter REGEXP
56
57
  </source>
57
58
  ```
58
59
 
@@ -84,6 +84,7 @@ class FluentPluginElbAccessLogInput < Fluent::Input
84
84
  config_param :type_cast, :bool, default: true
85
85
  config_param :parse_request, :bool, default: true
86
86
  config_param :split_addr_port, :bool, default: true
87
+ config_param :file_filter, :string, default: nil
87
88
 
88
89
  def configure(conf)
89
90
  super
@@ -115,6 +116,10 @@ class FluentPluginElbAccessLogInput < Fluent::Input
115
116
  if @filter
116
117
  @filter = Hash[@filter.map {|k, v| [k.to_s, Regexp.new(v.to_s)] }]
117
118
  end
119
+
120
+ if @file_filter
121
+ @file_filter = Regexp.new(@file_filter)
122
+ end
118
123
  end
119
124
 
120
125
  def start
@@ -165,8 +170,12 @@ class FluentPluginElbAccessLogInput < Fluent::Input
165
170
  last_timestamp = timestamp
166
171
 
167
172
  prefixes(timestamp).each do |prefix|
168
- client.list_objects(bucket: @s3_bucket, prefix: prefix).each do |page|
173
+ client.list_objects_v2(bucket: @s3_bucket, prefix: prefix).each do |page|
169
174
  page.contents.each do |obj|
175
+ if @file_filter and obj.key !~ @file_filter
176
+ next
177
+ end
178
+
170
179
  account_id, logfile_const, region, elb_name, logfile_datetime, ip, logfile_suffix = obj.key.split('_', 7)
171
180
  logfile_datetime = Time.parse(logfile_datetime)
172
181
 
@@ -1,3 +1,3 @@
1
1
  module FluentPluginElbAccessLog
2
- VERSION = '0.5.0'
2
+ VERSION = '0.6.0'
3
3
  end
@@ -45,9 +45,9 @@ describe FluentPluginElbAccessLogInput do
45
45
 
46
46
  context 'when access log does not exist' do
47
47
  before do
48
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
49
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) { [] }
50
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
48
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
49
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) { [] }
50
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
51
51
  expect(driver.instance).to_not receive(:save_timestamp).with(today)
52
52
  expect(driver.instance.log).to_not receive(:warn)
53
53
 
@@ -73,15 +73,15 @@ https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
73
73
  end
74
74
 
75
75
  before do
76
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) do
76
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) do
77
77
  [double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])]
78
78
  end
79
79
 
80
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
80
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
81
81
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
82
82
  end
83
83
 
84
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
84
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
85
85
  [double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])]
86
86
  end
87
87
 
@@ -421,6 +421,142 @@ https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
421
421
  end
422
422
  end
423
423
 
424
+ context 'with file_filter' do
425
+ let(:today_access_log) do
426
+ gzip(<<-EOS)
427
+ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
428
+ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
429
+ EOS
430
+ end
431
+
432
+ let(:tomorrow_access_log) do
433
+ gzip(<<-EOS)
434
+ https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
435
+ https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
436
+ EOS
437
+ end
438
+
439
+ before do
440
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) do
441
+ [double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])]
442
+ end
443
+
444
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
445
+ [double('today_objects', contents: [double('today_object', key: today_object_key)])]
446
+ end
447
+
448
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
449
+ [double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])]
450
+ end
451
+
452
+ expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
453
+ double('today_s3_object', body: StringIO.new(today_access_log))
454
+ end
455
+
456
+ expect(client).to_not receive(:get_object).with(bucket: s3_bucket, key: tomorrow_object_key) do
457
+ double('tomorrow_s3_object', body: StringIO.new(tomorrow_access_log))
458
+ end
459
+
460
+ expect(driver.instance).to receive(:save_timestamp).with(today)
461
+ expect(driver.instance.log).to_not receive(:warn)
462
+
463
+ driver_run(driver)
464
+ end
465
+
466
+ let(:expected_emits) do
467
+ [["elb.access_log",
468
+ Time.parse('2015-05-24 19:55:36 UTC').to_i,
469
+ {"type"=>"https",
470
+ "timestamp"=>"2015-05-24T19:55:36.000000Z",
471
+ "elb"=>"hoge",
472
+ "client_port"=>57673,
473
+ "target_port"=>80,
474
+ "request_processing_time"=>5.3e-05,
475
+ "target_processing_time"=>0.000913,
476
+ "response_processing_time"=>3.6e-05,
477
+ "elb_status_code"=>200,
478
+ "target_status_code"=>200,
479
+ "received_bytes"=>0,
480
+ "sent_bytes"=>3,
481
+ "request"=>
482
+ "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
483
+ "user_agent"=>"curl/7.30.0",
484
+ "ssl_cipher"=>"ssl_cipher",
485
+ "ssl_protocol"=>"ssl_protocol",
486
+ "target_group_arn"=>
487
+ "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx",
488
+ "trace_id"=>"Root=xxx",
489
+ "domain_name"=>"-",
490
+ "chosen_cert_arn"=>
491
+ "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx",
492
+ "client"=>"14.14.124.20",
493
+ "target"=>"10.0.199.184",
494
+ "request.method"=>"GET",
495
+ "request.uri"=>
496
+ "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
497
+ "request.http_version"=>"HTTP/1.1",
498
+ "request.uri.scheme"=>"http",
499
+ "request.uri.user"=>nil,
500
+ "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
501
+ "request.uri.port"=>80,
502
+ "request.uri.path"=>"/",
503
+ "request.uri.query"=>nil,
504
+ "request.uri.fragment"=>nil}],
505
+ ["elb.access_log",
506
+ Time.parse('2015-05-24 19:55:36 UTC').to_i,
507
+ {"type"=>"https",
508
+ "timestamp"=>"2015-05-24T19:55:36.000000Z",
509
+ "elb"=>"hoge",
510
+ "client_port"=>57673,
511
+ "target_port"=>80,
512
+ "request_processing_time"=>5.3e-05,
513
+ "target_processing_time"=>0.000913,
514
+ "response_processing_time"=>3.6e-05,
515
+ "elb_status_code"=>200,
516
+ "target_status_code"=>200,
517
+ "received_bytes"=>0,
518
+ "sent_bytes"=>3,
519
+ "request"=>
520
+ "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
521
+ "user_agent"=>"curl/7.30.0",
522
+ "ssl_cipher"=>"ssl_cipher",
523
+ "ssl_protocol"=>"ssl_protocol",
524
+ "target_group_arn"=>
525
+ "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx",
526
+ "trace_id"=>"Root=xxx",
527
+ "domain_name"=>"-",
528
+ "chosen_cert_arn"=>
529
+ "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx",
530
+ "client"=>"14.14.124.20",
531
+ "target"=>"10.0.199.184",
532
+ "request.method"=>"GET",
533
+ "request.uri"=>
534
+ "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
535
+ "request.http_version"=>"HTTP/1.1",
536
+ "request.uri.scheme"=>"http",
537
+ "request.uri.user"=>nil,
538
+ "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
539
+ "request.uri.port"=>80,
540
+ "request.uri.path"=>"/",
541
+ "request.uri.query"=>nil,
542
+ "request.uri.fragment"=>nil}]]
543
+ end
544
+
545
+ let(:fluentd_conf) do
546
+ {
547
+ interval: 0,
548
+ account_id: account_id,
549
+ s3_bucket: s3_bucket,
550
+ region: region,
551
+ start_datetime: (today - 1).to_s,
552
+ elb_type: 'alb',
553
+ file_filter: today.iso8601,
554
+ }
555
+ end
556
+
557
+ it { is_expected.to match_table expected_emits }
558
+ end
559
+
424
560
  context 'when include bad URI' do
425
561
  let(:today_access_log) do
426
562
  gzip(<<-EOS)
@@ -429,10 +565,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
429
565
  end
430
566
 
431
567
  before do
432
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
433
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
568
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
569
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
434
570
 
435
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
571
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
436
572
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
437
573
  end
438
574
 
@@ -505,10 +641,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
505
641
  end
506
642
 
507
643
  before do
508
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
509
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
644
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
645
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
510
646
 
511
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
647
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
512
648
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
513
649
  end
514
650
 
@@ -577,10 +713,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
577
713
  let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 600).iso8601}_52.68.51.1_8hSqR3o4.log.gz" }
578
714
 
579
715
  before do
580
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
581
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
716
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
717
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
582
718
 
583
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
719
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
584
720
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
585
721
  end
586
722
 
@@ -646,10 +782,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
646
782
  end
647
783
 
648
784
  before do
649
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
650
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
785
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
786
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
651
787
 
652
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
788
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
653
789
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
654
790
  end
655
791
 
@@ -739,10 +875,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
739
875
  let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 601).iso8601}_52.68.51.1_8hSqR3o4.log.gz" }
740
876
 
741
877
  before do
742
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
743
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
878
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
879
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
744
880
 
745
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
881
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
746
882
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
747
883
  end
748
884
 
@@ -764,10 +900,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
764
900
  end
765
901
 
766
902
  before do
767
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
768
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
903
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
904
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
769
905
 
770
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
906
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
771
907
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
772
908
  end
773
909
 
@@ -794,10 +930,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
794
930
  let(:history) { driver.instance.instance_variable_get(:@history) }
795
931
 
796
932
  before do
797
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
798
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
933
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
934
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
799
935
 
800
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
936
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
801
937
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
802
938
  end
803
939
 
@@ -837,10 +973,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
837
973
  end
838
974
 
839
975
  before do
840
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
841
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
976
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
977
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
842
978
 
843
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
979
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
844
980
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
845
981
  end
846
982
 
@@ -906,10 +1042,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
906
1042
  end
907
1043
 
908
1044
  before do
909
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
910
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
1045
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
1046
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
911
1047
 
912
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
1048
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
913
1049
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
914
1050
  end
915
1051
 
@@ -933,10 +1069,10 @@ https xxx hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200
933
1069
  end
934
1070
 
935
1071
  before do
936
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
937
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
1072
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
1073
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
938
1074
 
939
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
1075
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
940
1076
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
941
1077
  end
942
1078
 
@@ -962,10 +1098,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
962
1098
  end
963
1099
 
964
1100
  before do
965
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
966
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
1101
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
1102
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
967
1103
 
968
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
1104
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
969
1105
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
970
1106
  end
971
1107
 
@@ -44,9 +44,9 @@ describe FluentPluginElbAccessLogInput do
44
44
 
45
45
  context 'when access log does not exist' do
46
46
  before do
47
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
48
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) { [] }
49
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
47
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
48
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) { [] }
49
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
50
50
  expect(driver.instance).to_not receive(:save_timestamp).with(today)
51
51
  expect(driver.instance).to receive(:save_history)
52
52
  expect(driver.instance.log).to_not receive(:warn)
@@ -73,15 +73,15 @@ describe FluentPluginElbAccessLogInput do
73
73
  end
74
74
 
75
75
  before do
76
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) do
76
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) do
77
77
  [double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])]
78
78
  end
79
79
 
80
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
80
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
81
81
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
82
82
  end
83
83
 
84
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
84
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
85
85
  [double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])]
86
86
  end
87
87
 
@@ -387,6 +387,128 @@ describe FluentPluginElbAccessLogInput do
387
387
  end
388
388
  end
389
389
 
390
+ context 'with file_filter' do
391
+ let(:today_access_log) do
392
+ <<-EOS
393
+ 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol
394
+ 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol
395
+ EOS
396
+ end
397
+
398
+ let(:tomorrow_access_log) do
399
+ <<-EOS
400
+ 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol
401
+ 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol
402
+ EOS
403
+ end
404
+
405
+ before do
406
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) do
407
+ [double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])]
408
+ end
409
+
410
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
411
+ [double('today_objects', contents: [double('today_object', key: today_object_key)])]
412
+ end
413
+
414
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
415
+ [double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])]
416
+ end
417
+
418
+ expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
419
+ double('today_s3_object', body: StringIO.new(today_access_log))
420
+ end
421
+
422
+ expect(client).to_not receive(:get_object).with(bucket: s3_bucket, key: tomorrow_object_key) do
423
+ double('tomorrow_s3_object', body: StringIO.new(tomorrow_access_log))
424
+ end
425
+
426
+ expect(driver.instance).to receive(:save_timestamp).with(today)
427
+ expect(driver.instance).to receive(:save_history)
428
+ expect(driver.instance.log).to_not receive(:warn)
429
+
430
+ driver_run(driver)
431
+ end
432
+
433
+ let(:expected_emits) do
434
+ [["elb.access_log",
435
+ Time.parse('2015-05-24 19:55:36 UTC').to_i,
436
+ {"timestamp"=>"2015-05-24T19:55:36.000000Z",
437
+ "elb"=>"hoge",
438
+ "client"=>"14.14.124.20",
439
+ "client_port"=>57673,
440
+ "backend"=>"10.0.199.184",
441
+ "backend_port"=>80,
442
+ "request_processing_time"=>5.3e-05,
443
+ "backend_processing_time"=>0.000913,
444
+ "response_processing_time"=>3.6e-05,
445
+ "elb_status_code"=>200,
446
+ "backend_status_code"=>200,
447
+ "received_bytes"=>0,
448
+ "sent_bytes"=>3,
449
+ "request"=>
450
+ "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
451
+ "user_agent"=>"curl/7.30.0",
452
+ "ssl_cipher"=>"ssl_cipher",
453
+ "ssl_protocol"=>"ssl_protocol",
454
+ "request.method"=>"GET",
455
+ "request.uri"=>
456
+ "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
457
+ "request.http_version"=>"HTTP/1.1",
458
+ "request.uri.scheme"=>"http",
459
+ "request.uri.user"=>nil,
460
+ "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
461
+ "request.uri.port"=>80,
462
+ "request.uri.path"=>"/",
463
+ "request.uri.query"=>nil,
464
+ "request.uri.fragment"=>nil}],
465
+ ["elb.access_log",
466
+ Time.parse('2015-05-24 19:55:36 UTC').to_i,
467
+ {"timestamp"=>"2015-05-24T19:55:36.000000Z",
468
+ "elb"=>"hoge",
469
+ "client"=>"14.14.124.20",
470
+ "client_port"=>57673,
471
+ "backend"=>"10.0.199.184",
472
+ "backend_port"=>80,
473
+ "request_processing_time"=>5.3e-05,
474
+ "backend_processing_time"=>0.000913,
475
+ "response_processing_time"=>3.6e-05,
476
+ "elb_status_code"=>200,
477
+ "backend_status_code"=>200,
478
+ "received_bytes"=>0,
479
+ "sent_bytes"=>3,
480
+ "request"=>
481
+ "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
482
+ "user_agent"=>"curl/7.30.0",
483
+ "ssl_cipher"=>"ssl_cipher",
484
+ "ssl_protocol"=>"ssl_protocol",
485
+ "request.method"=>"GET",
486
+ "request.uri"=>
487
+ "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
488
+ "request.http_version"=>"HTTP/1.1",
489
+ "request.uri.scheme"=>"http",
490
+ "request.uri.user"=>nil,
491
+ "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
492
+ "request.uri.port"=>80,
493
+ "request.uri.path"=>"/",
494
+ "request.uri.query"=>nil,
495
+ "request.uri.fragment"=>nil}]]
496
+ end
497
+
498
+ let(:fluentd_conf) do
499
+ {
500
+ interval: 0,
501
+ account_id: account_id,
502
+ s3_bucket: s3_bucket,
503
+ region: region,
504
+ start_datetime: (today - 1).to_s,
505
+ file_filter: today.iso8601,
506
+ }
507
+ end
508
+
509
+ it { is_expected.to match_table expected_emits }
510
+ end
511
+
390
512
  context 'when include bad URI' do
391
513
  let(:today_access_log) do
392
514
  <<-EOS
@@ -395,10 +517,10 @@ describe FluentPluginElbAccessLogInput do
395
517
  end
396
518
 
397
519
  before do
398
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
399
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
520
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
521
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
400
522
 
401
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
523
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
402
524
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
403
525
  end
404
526
 
@@ -464,10 +586,10 @@ describe FluentPluginElbAccessLogInput do
464
586
  end
465
587
 
466
588
  before do
467
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
468
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
589
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
590
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
469
591
 
470
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
592
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
471
593
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
472
594
  end
473
595
 
@@ -529,10 +651,10 @@ describe FluentPluginElbAccessLogInput do
529
651
  let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 600).iso8601}_52.68.51.1_8hSqR3o4.log" }
530
652
 
531
653
  before do
532
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
533
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
654
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
655
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
534
656
 
535
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
657
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
536
658
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
537
659
  end
538
660
 
@@ -592,10 +714,10 @@ describe FluentPluginElbAccessLogInput do
592
714
  end
593
715
 
594
716
  before do
595
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
596
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
717
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
718
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
597
719
 
598
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
720
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
599
721
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
600
722
  end
601
723
 
@@ -675,10 +797,10 @@ describe FluentPluginElbAccessLogInput do
675
797
  let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 601).iso8601}_52.68.51.1_8hSqR3o4.log" }
676
798
 
677
799
  before do
678
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
679
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
800
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
801
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
680
802
 
681
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
803
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
682
804
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
683
805
  end
684
806
 
@@ -701,10 +823,10 @@ describe FluentPluginElbAccessLogInput do
701
823
  end
702
824
 
703
825
  before do
704
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
705
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
826
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
827
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
706
828
 
707
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
829
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
708
830
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
709
831
  end
710
832
 
@@ -732,10 +854,10 @@ describe FluentPluginElbAccessLogInput do
732
854
  let(:history) { driver.instance.instance_variable_get(:@history) }
733
855
 
734
856
  before do
735
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
736
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
857
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
858
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
737
859
 
738
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
860
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
739
861
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
740
862
  end
741
863
 
@@ -776,10 +898,10 @@ describe FluentPluginElbAccessLogInput do
776
898
  end
777
899
 
778
900
  before do
779
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
780
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
901
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
902
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
781
903
 
782
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
904
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
783
905
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
784
906
  end
785
907
 
@@ -839,10 +961,10 @@ describe FluentPluginElbAccessLogInput do
839
961
  end
840
962
 
841
963
  before do
842
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
843
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
964
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
965
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
844
966
 
845
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
967
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
846
968
  [double('today_objects', contents: [double('today_object', key: today_object_key)])]
847
969
  end
848
970
 
@@ -30,9 +30,9 @@ describe 'FluentPluginElbAccessLogInput#client' do
30
30
  Timecop.freeze(today)
31
31
  allow_any_instance_of(FluentPluginElbAccessLogInput).to receive(:load_history) { [] }
32
32
  allow_any_instance_of(FluentPluginElbAccessLogInput).to receive(:parse_tsfile) { nil }
33
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
34
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) { [] }
35
- expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
33
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
34
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) { [] }
35
+ expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
36
36
  allow(FileUtils).to receive(:touch)
37
37
  expect(driver.instance).to_not receive(:save_timestamp).with(today)
38
38
  expect(driver.instance).to receive(:save_history)
@@ -49,6 +49,7 @@ describe 'FluentPluginElbAccessLogInput#configure' do
49
49
  expect(driver.instance.type_cast).to be_truthy
50
50
  expect(driver.instance.parse_request).to be_truthy
51
51
  expect(driver.instance.split_addr_port).to be_truthy
52
+ expect(driver.instance.filter).to be_nil
52
53
  end
53
54
  end
54
55
 
@@ -70,6 +71,7 @@ describe 'FluentPluginElbAccessLogInput#configure' do
70
71
  let(:elb_type) { 'alb' }
71
72
  let(:filter) { 'elb_status_code:^2' }
72
73
  let(:filter_operator) { 'or' }
74
+ let(:file_filter) { '.*my-elb.*' }
73
75
 
74
76
  let(:fluentd_conf) do
75
77
  {
@@ -97,6 +99,7 @@ describe 'FluentPluginElbAccessLogInput#configure' do
97
99
  type_cast: 'false',
98
100
  parse_request: 'false',
99
101
  split_addr_port: 'false',
102
+ file_filter: file_filter,
100
103
  }
101
104
  end
102
105
 
@@ -124,6 +127,7 @@ describe 'FluentPluginElbAccessLogInput#configure' do
124
127
  expect(driver.instance.type_cast).to be_falsey
125
128
  expect(driver.instance.parse_request).to be_falsey
126
129
  expect(driver.instance.split_addr_port).to be_falsey
130
+ expect(driver.instance.file_filter).to eq Regexp.new(file_filter)
127
131
  end
128
132
  end
129
133
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elb-access-log
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.0
4
+ version: 0.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Genki Sugawara
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-01-19 00:00:00.000000000 Z
11
+ date: 2018-03-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd