fluent-plugin-elb-access-log 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -0
- data/lib/fluent/plugin/in_elb_access_log.rb +10 -1
- data/lib/fluent_plugin_elb_access_log/version.rb +1 -1
- data/spec/in_elb_access_log_alb_spec.rb +175 -39
- data/spec/in_elb_access_log_clb_spec.rb +155 -33
- data/spec/in_elb_access_log_client_spec.rb +3 -3
- data/spec/in_elb_access_log_config_spec.rb +4 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d3a20dc8a60a8ce35bae8050d88f6f3cb89fd5cd
|
4
|
+
data.tar.gz: b9df7709325ba5f796658df242b7d04727f65f96
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 41d7b97d9044b10ea91d65d20556024fa1f357e9e6ba1cb3c8e3827f553b5258ee8b96dc3e964acaba24eba0780f2078ed6517868eeaa28a2057a6efd0ecf7f1
|
7
|
+
data.tar.gz: 9f3f1b43320c87f24112a6187e65c1bb293975b7f3db0816eb16c987bcdf4d5b6ad594d0950235d4b164760fc22a6b9997a99820cc158185d5e433528453a721
|
data/README.md
CHANGED
@@ -84,6 +84,7 @@ class FluentPluginElbAccessLogInput < Fluent::Input
|
|
84
84
|
config_param :type_cast, :bool, default: true
|
85
85
|
config_param :parse_request, :bool, default: true
|
86
86
|
config_param :split_addr_port, :bool, default: true
|
87
|
+
config_param :file_filter, :string, default: nil
|
87
88
|
|
88
89
|
def configure(conf)
|
89
90
|
super
|
@@ -115,6 +116,10 @@ class FluentPluginElbAccessLogInput < Fluent::Input
|
|
115
116
|
if @filter
|
116
117
|
@filter = Hash[@filter.map {|k, v| [k.to_s, Regexp.new(v.to_s)] }]
|
117
118
|
end
|
119
|
+
|
120
|
+
if @file_filter
|
121
|
+
@file_filter = Regexp.new(@file_filter)
|
122
|
+
end
|
118
123
|
end
|
119
124
|
|
120
125
|
def start
|
@@ -165,8 +170,12 @@ class FluentPluginElbAccessLogInput < Fluent::Input
|
|
165
170
|
last_timestamp = timestamp
|
166
171
|
|
167
172
|
prefixes(timestamp).each do |prefix|
|
168
|
-
client.
|
173
|
+
client.list_objects_v2(bucket: @s3_bucket, prefix: prefix).each do |page|
|
169
174
|
page.contents.each do |obj|
|
175
|
+
if @file_filter and obj.key !~ @file_filter
|
176
|
+
next
|
177
|
+
end
|
178
|
+
|
170
179
|
account_id, logfile_const, region, elb_name, logfile_datetime, ip, logfile_suffix = obj.key.split('_', 7)
|
171
180
|
logfile_datetime = Time.parse(logfile_datetime)
|
172
181
|
|
@@ -45,9 +45,9 @@ describe FluentPluginElbAccessLogInput do
|
|
45
45
|
|
46
46
|
context 'when access log does not exist' do
|
47
47
|
before do
|
48
|
-
expect(client).to receive(:
|
49
|
-
expect(client).to receive(:
|
50
|
-
expect(client).to receive(:
|
48
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
49
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) { [] }
|
50
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
51
51
|
expect(driver.instance).to_not receive(:save_timestamp).with(today)
|
52
52
|
expect(driver.instance.log).to_not receive(:warn)
|
53
53
|
|
@@ -73,15 +73,15 @@ https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
|
|
73
73
|
end
|
74
74
|
|
75
75
|
before do
|
76
|
-
expect(client).to receive(:
|
76
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) do
|
77
77
|
[double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])]
|
78
78
|
end
|
79
79
|
|
80
|
-
expect(client).to receive(:
|
80
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
81
81
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
82
82
|
end
|
83
83
|
|
84
|
-
expect(client).to receive(:
|
84
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
|
85
85
|
[double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])]
|
86
86
|
end
|
87
87
|
|
@@ -421,6 +421,142 @@ https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
|
|
421
421
|
end
|
422
422
|
end
|
423
423
|
|
424
|
+
context 'with file_filter' do
|
425
|
+
let(:today_access_log) do
|
426
|
+
gzip(<<-EOS)
|
427
|
+
https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
|
428
|
+
https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
|
429
|
+
EOS
|
430
|
+
end
|
431
|
+
|
432
|
+
let(:tomorrow_access_log) do
|
433
|
+
gzip(<<-EOS)
|
434
|
+
https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
|
435
|
+
https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
|
436
|
+
EOS
|
437
|
+
end
|
438
|
+
|
439
|
+
before do
|
440
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) do
|
441
|
+
[double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])]
|
442
|
+
end
|
443
|
+
|
444
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
445
|
+
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
446
|
+
end
|
447
|
+
|
448
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
|
449
|
+
[double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])]
|
450
|
+
end
|
451
|
+
|
452
|
+
expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
|
453
|
+
double('today_s3_object', body: StringIO.new(today_access_log))
|
454
|
+
end
|
455
|
+
|
456
|
+
expect(client).to_not receive(:get_object).with(bucket: s3_bucket, key: tomorrow_object_key) do
|
457
|
+
double('tomorrow_s3_object', body: StringIO.new(tomorrow_access_log))
|
458
|
+
end
|
459
|
+
|
460
|
+
expect(driver.instance).to receive(:save_timestamp).with(today)
|
461
|
+
expect(driver.instance.log).to_not receive(:warn)
|
462
|
+
|
463
|
+
driver_run(driver)
|
464
|
+
end
|
465
|
+
|
466
|
+
let(:expected_emits) do
|
467
|
+
[["elb.access_log",
|
468
|
+
Time.parse('2015-05-24 19:55:36 UTC').to_i,
|
469
|
+
{"type"=>"https",
|
470
|
+
"timestamp"=>"2015-05-24T19:55:36.000000Z",
|
471
|
+
"elb"=>"hoge",
|
472
|
+
"client_port"=>57673,
|
473
|
+
"target_port"=>80,
|
474
|
+
"request_processing_time"=>5.3e-05,
|
475
|
+
"target_processing_time"=>0.000913,
|
476
|
+
"response_processing_time"=>3.6e-05,
|
477
|
+
"elb_status_code"=>200,
|
478
|
+
"target_status_code"=>200,
|
479
|
+
"received_bytes"=>0,
|
480
|
+
"sent_bytes"=>3,
|
481
|
+
"request"=>
|
482
|
+
"GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
|
483
|
+
"user_agent"=>"curl/7.30.0",
|
484
|
+
"ssl_cipher"=>"ssl_cipher",
|
485
|
+
"ssl_protocol"=>"ssl_protocol",
|
486
|
+
"target_group_arn"=>
|
487
|
+
"arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx",
|
488
|
+
"trace_id"=>"Root=xxx",
|
489
|
+
"domain_name"=>"-",
|
490
|
+
"chosen_cert_arn"=>
|
491
|
+
"arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx",
|
492
|
+
"client"=>"14.14.124.20",
|
493
|
+
"target"=>"10.0.199.184",
|
494
|
+
"request.method"=>"GET",
|
495
|
+
"request.uri"=>
|
496
|
+
"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
|
497
|
+
"request.http_version"=>"HTTP/1.1",
|
498
|
+
"request.uri.scheme"=>"http",
|
499
|
+
"request.uri.user"=>nil,
|
500
|
+
"request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
|
501
|
+
"request.uri.port"=>80,
|
502
|
+
"request.uri.path"=>"/",
|
503
|
+
"request.uri.query"=>nil,
|
504
|
+
"request.uri.fragment"=>nil}],
|
505
|
+
["elb.access_log",
|
506
|
+
Time.parse('2015-05-24 19:55:36 UTC').to_i,
|
507
|
+
{"type"=>"https",
|
508
|
+
"timestamp"=>"2015-05-24T19:55:36.000000Z",
|
509
|
+
"elb"=>"hoge",
|
510
|
+
"client_port"=>57673,
|
511
|
+
"target_port"=>80,
|
512
|
+
"request_processing_time"=>5.3e-05,
|
513
|
+
"target_processing_time"=>0.000913,
|
514
|
+
"response_processing_time"=>3.6e-05,
|
515
|
+
"elb_status_code"=>200,
|
516
|
+
"target_status_code"=>200,
|
517
|
+
"received_bytes"=>0,
|
518
|
+
"sent_bytes"=>3,
|
519
|
+
"request"=>
|
520
|
+
"GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
|
521
|
+
"user_agent"=>"curl/7.30.0",
|
522
|
+
"ssl_cipher"=>"ssl_cipher",
|
523
|
+
"ssl_protocol"=>"ssl_protocol",
|
524
|
+
"target_group_arn"=>
|
525
|
+
"arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx",
|
526
|
+
"trace_id"=>"Root=xxx",
|
527
|
+
"domain_name"=>"-",
|
528
|
+
"chosen_cert_arn"=>
|
529
|
+
"arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx",
|
530
|
+
"client"=>"14.14.124.20",
|
531
|
+
"target"=>"10.0.199.184",
|
532
|
+
"request.method"=>"GET",
|
533
|
+
"request.uri"=>
|
534
|
+
"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
|
535
|
+
"request.http_version"=>"HTTP/1.1",
|
536
|
+
"request.uri.scheme"=>"http",
|
537
|
+
"request.uri.user"=>nil,
|
538
|
+
"request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
|
539
|
+
"request.uri.port"=>80,
|
540
|
+
"request.uri.path"=>"/",
|
541
|
+
"request.uri.query"=>nil,
|
542
|
+
"request.uri.fragment"=>nil}]]
|
543
|
+
end
|
544
|
+
|
545
|
+
let(:fluentd_conf) do
|
546
|
+
{
|
547
|
+
interval: 0,
|
548
|
+
account_id: account_id,
|
549
|
+
s3_bucket: s3_bucket,
|
550
|
+
region: region,
|
551
|
+
start_datetime: (today - 1).to_s,
|
552
|
+
elb_type: 'alb',
|
553
|
+
file_filter: today.iso8601,
|
554
|
+
}
|
555
|
+
end
|
556
|
+
|
557
|
+
it { is_expected.to match_table expected_emits }
|
558
|
+
end
|
559
|
+
|
424
560
|
context 'when include bad URI' do
|
425
561
|
let(:today_access_log) do
|
426
562
|
gzip(<<-EOS)
|
@@ -429,10 +565,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
|
|
429
565
|
end
|
430
566
|
|
431
567
|
before do
|
432
|
-
expect(client).to receive(:
|
433
|
-
expect(client).to receive(:
|
568
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
569
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
434
570
|
|
435
|
-
expect(client).to receive(:
|
571
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
436
572
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
437
573
|
end
|
438
574
|
|
@@ -505,10 +641,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
|
|
505
641
|
end
|
506
642
|
|
507
643
|
before do
|
508
|
-
expect(client).to receive(:
|
509
|
-
expect(client).to receive(:
|
644
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
645
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
510
646
|
|
511
|
-
expect(client).to receive(:
|
647
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
512
648
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
513
649
|
end
|
514
650
|
|
@@ -577,10 +713,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
|
|
577
713
|
let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 600).iso8601}_52.68.51.1_8hSqR3o4.log.gz" }
|
578
714
|
|
579
715
|
before do
|
580
|
-
expect(client).to receive(:
|
581
|
-
expect(client).to receive(:
|
716
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
717
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
582
718
|
|
583
|
-
expect(client).to receive(:
|
719
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
584
720
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
585
721
|
end
|
586
722
|
|
@@ -646,10 +782,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
|
|
646
782
|
end
|
647
783
|
|
648
784
|
before do
|
649
|
-
expect(client).to receive(:
|
650
|
-
expect(client).to receive(:
|
785
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
786
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
651
787
|
|
652
|
-
expect(client).to receive(:
|
788
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
653
789
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
654
790
|
end
|
655
791
|
|
@@ -739,10 +875,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
|
|
739
875
|
let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 601).iso8601}_52.68.51.1_8hSqR3o4.log.gz" }
|
740
876
|
|
741
877
|
before do
|
742
|
-
expect(client).to receive(:
|
743
|
-
expect(client).to receive(:
|
878
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
879
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
744
880
|
|
745
|
-
expect(client).to receive(:
|
881
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
746
882
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
747
883
|
end
|
748
884
|
|
@@ -764,10 +900,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
|
|
764
900
|
end
|
765
901
|
|
766
902
|
before do
|
767
|
-
expect(client).to receive(:
|
768
|
-
expect(client).to receive(:
|
903
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
904
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
769
905
|
|
770
|
-
expect(client).to receive(:
|
906
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
771
907
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
772
908
|
end
|
773
909
|
|
@@ -794,10 +930,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
|
|
794
930
|
let(:history) { driver.instance.instance_variable_get(:@history) }
|
795
931
|
|
796
932
|
before do
|
797
|
-
expect(client).to receive(:
|
798
|
-
expect(client).to receive(:
|
933
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
934
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
799
935
|
|
800
|
-
expect(client).to receive(:
|
936
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
801
937
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
802
938
|
end
|
803
939
|
|
@@ -837,10 +973,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
|
|
837
973
|
end
|
838
974
|
|
839
975
|
before do
|
840
|
-
expect(client).to receive(:
|
841
|
-
expect(client).to receive(:
|
976
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
977
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
842
978
|
|
843
|
-
expect(client).to receive(:
|
979
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
844
980
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
845
981
|
end
|
846
982
|
|
@@ -906,10 +1042,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
|
|
906
1042
|
end
|
907
1043
|
|
908
1044
|
before do
|
909
|
-
expect(client).to receive(:
|
910
|
-
expect(client).to receive(:
|
1045
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
1046
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
911
1047
|
|
912
|
-
expect(client).to receive(:
|
1048
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
913
1049
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
914
1050
|
end
|
915
1051
|
|
@@ -933,10 +1069,10 @@ https xxx hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200
|
|
933
1069
|
end
|
934
1070
|
|
935
1071
|
before do
|
936
|
-
expect(client).to receive(:
|
937
|
-
expect(client).to receive(:
|
1072
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
1073
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
938
1074
|
|
939
|
-
expect(client).to receive(:
|
1075
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
940
1076
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
941
1077
|
end
|
942
1078
|
|
@@ -962,10 +1098,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
|
|
962
1098
|
end
|
963
1099
|
|
964
1100
|
before do
|
965
|
-
expect(client).to receive(:
|
966
|
-
expect(client).to receive(:
|
1101
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
1102
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
967
1103
|
|
968
|
-
expect(client).to receive(:
|
1104
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
969
1105
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
970
1106
|
end
|
971
1107
|
|
@@ -44,9 +44,9 @@ describe FluentPluginElbAccessLogInput do
|
|
44
44
|
|
45
45
|
context 'when access log does not exist' do
|
46
46
|
before do
|
47
|
-
expect(client).to receive(:
|
48
|
-
expect(client).to receive(:
|
49
|
-
expect(client).to receive(:
|
47
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
48
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) { [] }
|
49
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
50
50
|
expect(driver.instance).to_not receive(:save_timestamp).with(today)
|
51
51
|
expect(driver.instance).to receive(:save_history)
|
52
52
|
expect(driver.instance.log).to_not receive(:warn)
|
@@ -73,15 +73,15 @@ describe FluentPluginElbAccessLogInput do
|
|
73
73
|
end
|
74
74
|
|
75
75
|
before do
|
76
|
-
expect(client).to receive(:
|
76
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) do
|
77
77
|
[double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])]
|
78
78
|
end
|
79
79
|
|
80
|
-
expect(client).to receive(:
|
80
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
81
81
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
82
82
|
end
|
83
83
|
|
84
|
-
expect(client).to receive(:
|
84
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
|
85
85
|
[double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])]
|
86
86
|
end
|
87
87
|
|
@@ -387,6 +387,128 @@ describe FluentPluginElbAccessLogInput do
|
|
387
387
|
end
|
388
388
|
end
|
389
389
|
|
390
|
+
context 'with file_filter' do
|
391
|
+
let(:today_access_log) do
|
392
|
+
<<-EOS
|
393
|
+
2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol
|
394
|
+
2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol
|
395
|
+
EOS
|
396
|
+
end
|
397
|
+
|
398
|
+
let(:tomorrow_access_log) do
|
399
|
+
<<-EOS
|
400
|
+
2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol
|
401
|
+
2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol
|
402
|
+
EOS
|
403
|
+
end
|
404
|
+
|
405
|
+
before do
|
406
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) do
|
407
|
+
[double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])]
|
408
|
+
end
|
409
|
+
|
410
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
411
|
+
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
412
|
+
end
|
413
|
+
|
414
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
|
415
|
+
[double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])]
|
416
|
+
end
|
417
|
+
|
418
|
+
expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
|
419
|
+
double('today_s3_object', body: StringIO.new(today_access_log))
|
420
|
+
end
|
421
|
+
|
422
|
+
expect(client).to_not receive(:get_object).with(bucket: s3_bucket, key: tomorrow_object_key) do
|
423
|
+
double('tomorrow_s3_object', body: StringIO.new(tomorrow_access_log))
|
424
|
+
end
|
425
|
+
|
426
|
+
expect(driver.instance).to receive(:save_timestamp).with(today)
|
427
|
+
expect(driver.instance).to receive(:save_history)
|
428
|
+
expect(driver.instance.log).to_not receive(:warn)
|
429
|
+
|
430
|
+
driver_run(driver)
|
431
|
+
end
|
432
|
+
|
433
|
+
let(:expected_emits) do
|
434
|
+
[["elb.access_log",
|
435
|
+
Time.parse('2015-05-24 19:55:36 UTC').to_i,
|
436
|
+
{"timestamp"=>"2015-05-24T19:55:36.000000Z",
|
437
|
+
"elb"=>"hoge",
|
438
|
+
"client"=>"14.14.124.20",
|
439
|
+
"client_port"=>57673,
|
440
|
+
"backend"=>"10.0.199.184",
|
441
|
+
"backend_port"=>80,
|
442
|
+
"request_processing_time"=>5.3e-05,
|
443
|
+
"backend_processing_time"=>0.000913,
|
444
|
+
"response_processing_time"=>3.6e-05,
|
445
|
+
"elb_status_code"=>200,
|
446
|
+
"backend_status_code"=>200,
|
447
|
+
"received_bytes"=>0,
|
448
|
+
"sent_bytes"=>3,
|
449
|
+
"request"=>
|
450
|
+
"GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
|
451
|
+
"user_agent"=>"curl/7.30.0",
|
452
|
+
"ssl_cipher"=>"ssl_cipher",
|
453
|
+
"ssl_protocol"=>"ssl_protocol",
|
454
|
+
"request.method"=>"GET",
|
455
|
+
"request.uri"=>
|
456
|
+
"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
|
457
|
+
"request.http_version"=>"HTTP/1.1",
|
458
|
+
"request.uri.scheme"=>"http",
|
459
|
+
"request.uri.user"=>nil,
|
460
|
+
"request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
|
461
|
+
"request.uri.port"=>80,
|
462
|
+
"request.uri.path"=>"/",
|
463
|
+
"request.uri.query"=>nil,
|
464
|
+
"request.uri.fragment"=>nil}],
|
465
|
+
["elb.access_log",
|
466
|
+
Time.parse('2015-05-24 19:55:36 UTC').to_i,
|
467
|
+
{"timestamp"=>"2015-05-24T19:55:36.000000Z",
|
468
|
+
"elb"=>"hoge",
|
469
|
+
"client"=>"14.14.124.20",
|
470
|
+
"client_port"=>57673,
|
471
|
+
"backend"=>"10.0.199.184",
|
472
|
+
"backend_port"=>80,
|
473
|
+
"request_processing_time"=>5.3e-05,
|
474
|
+
"backend_processing_time"=>0.000913,
|
475
|
+
"response_processing_time"=>3.6e-05,
|
476
|
+
"elb_status_code"=>200,
|
477
|
+
"backend_status_code"=>200,
|
478
|
+
"received_bytes"=>0,
|
479
|
+
"sent_bytes"=>3,
|
480
|
+
"request"=>
|
481
|
+
"GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
|
482
|
+
"user_agent"=>"curl/7.30.0",
|
483
|
+
"ssl_cipher"=>"ssl_cipher",
|
484
|
+
"ssl_protocol"=>"ssl_protocol",
|
485
|
+
"request.method"=>"GET",
|
486
|
+
"request.uri"=>
|
487
|
+
"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
|
488
|
+
"request.http_version"=>"HTTP/1.1",
|
489
|
+
"request.uri.scheme"=>"http",
|
490
|
+
"request.uri.user"=>nil,
|
491
|
+
"request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
|
492
|
+
"request.uri.port"=>80,
|
493
|
+
"request.uri.path"=>"/",
|
494
|
+
"request.uri.query"=>nil,
|
495
|
+
"request.uri.fragment"=>nil}]]
|
496
|
+
end
|
497
|
+
|
498
|
+
let(:fluentd_conf) do
|
499
|
+
{
|
500
|
+
interval: 0,
|
501
|
+
account_id: account_id,
|
502
|
+
s3_bucket: s3_bucket,
|
503
|
+
region: region,
|
504
|
+
start_datetime: (today - 1).to_s,
|
505
|
+
file_filter: today.iso8601,
|
506
|
+
}
|
507
|
+
end
|
508
|
+
|
509
|
+
it { is_expected.to match_table expected_emits }
|
510
|
+
end
|
511
|
+
|
390
512
|
context 'when include bad URI' do
|
391
513
|
let(:today_access_log) do
|
392
514
|
<<-EOS
|
@@ -395,10 +517,10 @@ describe FluentPluginElbAccessLogInput do
|
|
395
517
|
end
|
396
518
|
|
397
519
|
before do
|
398
|
-
expect(client).to receive(:
|
399
|
-
expect(client).to receive(:
|
520
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
521
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
400
522
|
|
401
|
-
expect(client).to receive(:
|
523
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
402
524
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
403
525
|
end
|
404
526
|
|
@@ -464,10 +586,10 @@ describe FluentPluginElbAccessLogInput do
|
|
464
586
|
end
|
465
587
|
|
466
588
|
before do
|
467
|
-
expect(client).to receive(:
|
468
|
-
expect(client).to receive(:
|
589
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
590
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
469
591
|
|
470
|
-
expect(client).to receive(:
|
592
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
471
593
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
472
594
|
end
|
473
595
|
|
@@ -529,10 +651,10 @@ describe FluentPluginElbAccessLogInput do
|
|
529
651
|
let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 600).iso8601}_52.68.51.1_8hSqR3o4.log" }
|
530
652
|
|
531
653
|
before do
|
532
|
-
expect(client).to receive(:
|
533
|
-
expect(client).to receive(:
|
654
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
655
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
534
656
|
|
535
|
-
expect(client).to receive(:
|
657
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
536
658
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
537
659
|
end
|
538
660
|
|
@@ -592,10 +714,10 @@ describe FluentPluginElbAccessLogInput do
|
|
592
714
|
end
|
593
715
|
|
594
716
|
before do
|
595
|
-
expect(client).to receive(:
|
596
|
-
expect(client).to receive(:
|
717
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
718
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
597
719
|
|
598
|
-
expect(client).to receive(:
|
720
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
599
721
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
600
722
|
end
|
601
723
|
|
@@ -675,10 +797,10 @@ describe FluentPluginElbAccessLogInput do
|
|
675
797
|
let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 601).iso8601}_52.68.51.1_8hSqR3o4.log" }
|
676
798
|
|
677
799
|
before do
|
678
|
-
expect(client).to receive(:
|
679
|
-
expect(client).to receive(:
|
800
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
801
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
680
802
|
|
681
|
-
expect(client).to receive(:
|
803
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
682
804
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
683
805
|
end
|
684
806
|
|
@@ -701,10 +823,10 @@ describe FluentPluginElbAccessLogInput do
|
|
701
823
|
end
|
702
824
|
|
703
825
|
before do
|
704
|
-
expect(client).to receive(:
|
705
|
-
expect(client).to receive(:
|
826
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
827
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
706
828
|
|
707
|
-
expect(client).to receive(:
|
829
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
708
830
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
709
831
|
end
|
710
832
|
|
@@ -732,10 +854,10 @@ describe FluentPluginElbAccessLogInput do
|
|
732
854
|
let(:history) { driver.instance.instance_variable_get(:@history) }
|
733
855
|
|
734
856
|
before do
|
735
|
-
expect(client).to receive(:
|
736
|
-
expect(client).to receive(:
|
857
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
858
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
737
859
|
|
738
|
-
expect(client).to receive(:
|
860
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
739
861
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
740
862
|
end
|
741
863
|
|
@@ -776,10 +898,10 @@ describe FluentPluginElbAccessLogInput do
|
|
776
898
|
end
|
777
899
|
|
778
900
|
before do
|
779
|
-
expect(client).to receive(:
|
780
|
-
expect(client).to receive(:
|
901
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
902
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
781
903
|
|
782
|
-
expect(client).to receive(:
|
904
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
783
905
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
784
906
|
end
|
785
907
|
|
@@ -839,10 +961,10 @@ describe FluentPluginElbAccessLogInput do
|
|
839
961
|
end
|
840
962
|
|
841
963
|
before do
|
842
|
-
expect(client).to receive(:
|
843
|
-
expect(client).to receive(:
|
964
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
965
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
844
966
|
|
845
|
-
expect(client).to receive(:
|
967
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
|
846
968
|
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
847
969
|
end
|
848
970
|
|
@@ -30,9 +30,9 @@ describe 'FluentPluginElbAccessLogInput#client' do
|
|
30
30
|
Timecop.freeze(today)
|
31
31
|
allow_any_instance_of(FluentPluginElbAccessLogInput).to receive(:load_history) { [] }
|
32
32
|
allow_any_instance_of(FluentPluginElbAccessLogInput).to receive(:parse_tsfile) { nil }
|
33
|
-
expect(client).to receive(:
|
34
|
-
expect(client).to receive(:
|
35
|
-
expect(client).to receive(:
|
33
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
34
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) { [] }
|
35
|
+
expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
36
36
|
allow(FileUtils).to receive(:touch)
|
37
37
|
expect(driver.instance).to_not receive(:save_timestamp).with(today)
|
38
38
|
expect(driver.instance).to receive(:save_history)
|
@@ -49,6 +49,7 @@ describe 'FluentPluginElbAccessLogInput#configure' do
|
|
49
49
|
expect(driver.instance.type_cast).to be_truthy
|
50
50
|
expect(driver.instance.parse_request).to be_truthy
|
51
51
|
expect(driver.instance.split_addr_port).to be_truthy
|
52
|
+
expect(driver.instance.filter).to be_nil
|
52
53
|
end
|
53
54
|
end
|
54
55
|
|
@@ -70,6 +71,7 @@ describe 'FluentPluginElbAccessLogInput#configure' do
|
|
70
71
|
let(:elb_type) { 'alb' }
|
71
72
|
let(:filter) { 'elb_status_code:^2' }
|
72
73
|
let(:filter_operator) { 'or' }
|
74
|
+
let(:file_filter) { '.*my-elb.*' }
|
73
75
|
|
74
76
|
let(:fluentd_conf) do
|
75
77
|
{
|
@@ -97,6 +99,7 @@ describe 'FluentPluginElbAccessLogInput#configure' do
|
|
97
99
|
type_cast: 'false',
|
98
100
|
parse_request: 'false',
|
99
101
|
split_addr_port: 'false',
|
102
|
+
file_filter: file_filter,
|
100
103
|
}
|
101
104
|
end
|
102
105
|
|
@@ -124,6 +127,7 @@ describe 'FluentPluginElbAccessLogInput#configure' do
|
|
124
127
|
expect(driver.instance.type_cast).to be_falsey
|
125
128
|
expect(driver.instance.parse_request).to be_falsey
|
126
129
|
expect(driver.instance.split_addr_port).to be_falsey
|
130
|
+
expect(driver.instance.file_filter).to eq Regexp.new(file_filter)
|
127
131
|
end
|
128
132
|
end
|
129
133
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elb-access-log
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Genki Sugawara
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-03-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|