fluent-plugin-elb-access-log 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e6cf91d3c27190b1dc3433a92e5710e803616256
-  data.tar.gz: 60e98a16ff1535f1a3bf57807255aa8403eaac01
+  metadata.gz: d3a20dc8a60a8ce35bae8050d88f6f3cb89fd5cd
+  data.tar.gz: b9df7709325ba5f796658df242b7d04727f65f96
 SHA512:
-  metadata.gz: 1e1945d726b97db0f0ad9b7038b98958bac3196ec51d77dbf72e04afaf078064c68a7dcb1132df0709f9d1921bc00d7d1e6360a2fad93f6446d4043f903ffa2c
-  data.tar.gz: '0495808a43015a235ba716d5bd43a36f40111014f25bfa4960da80dfd7cb3da7e1174ee0c67e5d6f6b678a900d76cd5f5ec6325604321acdc90727db3bcf39ec'
+  metadata.gz: 41d7b97d9044b10ea91d65d20556024fa1f357e9e6ba1cb3c8e3827f553b5258ee8b96dc3e964acaba24eba0780f2078ed6517868eeaa28a2057a6efd0ecf7f1
+  data.tar.gz: 9f3f1b43320c87f24112a6187e65c1bb293975b7f3db0816eb16c987bcdf4d5b6ad594d0950235d4b164760fc22a6b9997a99820cc158185d5e433528453a721

data/README.md

@@ -53,6 +53,7 @@ Or install it yourself as:
   #type_cast true
   #parse_request true
   #split_addr_port true
+  #file_filter REGEXP
 </source>
 ```
 

data/lib/fluent/plugin/in_elb_access_log.rb

@@ -84,6 +84,7 @@ class FluentPluginElbAccessLogInput < Fluent::Input
   config_param :type_cast,         :bool,    default: true
   config_param :parse_request,     :bool,    default: true
   config_param :split_addr_port,   :bool,    default: true
+  config_param :file_filter,       :string,  default: nil
 
   def configure(conf)
     super
@@ -115,6 +116,10 @@ class FluentPluginElbAccessLogInput < Fluent::Input
     if @filter
       @filter = Hash[@filter.map {|k, v| [k.to_s, Regexp.new(v.to_s)] }]
     end
+
+    if @file_filter
+      @file_filter = Regexp.new(@file_filter)
+    end
   end
 
   def start
@@ -165,8 +170,12 @@ class FluentPluginElbAccessLogInput < Fluent::Input
     last_timestamp = timestamp
 
     prefixes(timestamp).each do |prefix|
-      client.list_objects(bucket: @s3_bucket, prefix: prefix).each do |page|
+      client.list_objects_v2(bucket: @s3_bucket, prefix: prefix).each do |page|
         page.contents.each do |obj|
+          if @file_filter and obj.key !~ @file_filter
+            next
+          end
+
           account_id, logfile_const, region, elb_name, logfile_datetime, ip, logfile_suffix = obj.key.split('_', 7)
           logfile_datetime = Time.parse(logfile_datetime)
 

data/lib/fluent_plugin_elb_access_log/version.rb

@@ -1,3 +1,3 @@
 module FluentPluginElbAccessLog
-  VERSION = '0.5.0'
+  VERSION = '0.6.0'
 end

data/spec/in_elb_access_log_alb_spec.rb

@@ -45,9 +45,9 @@ describe FluentPluginElbAccessLogInput do
 
   context 'when access log does not exist' do
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
       expect(driver.instance).to_not receive(:save_timestamp).with(today)
       expect(driver.instance.log).to_not receive(:warn)
 
@@ -73,15 +73,15 @@ https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
     end
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) do
         [double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])]
       end
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
         [double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])]
       end
 
@@ -421,6 +421,142 @@ https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
     end
   end
 
+  context 'with file_filter' do
+    let(:today_access_log) do
+      gzip(<<-EOS)
+https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
+https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
+      EOS
+    end
+
+    let(:tomorrow_access_log) do
+      gzip(<<-EOS)
+https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
+https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
+      EOS
+    end
+
+    before do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) do
+        [double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])]
+      end
+
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
+        [double('today_objects', contents: [double('today_object', key: today_object_key)])]
+      end
+
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
+        [double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])]
+      end
+
+      expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
+        double('today_s3_object', body: StringIO.new(today_access_log))
+      end
+
+      expect(client).to_not receive(:get_object).with(bucket: s3_bucket, key: tomorrow_object_key) do
+        double('tomorrow_s3_object', body: StringIO.new(tomorrow_access_log))
+      end
+
+      expect(driver.instance).to receive(:save_timestamp).with(today)
+      expect(driver.instance.log).to_not receive(:warn)
+
+      driver_run(driver)
+    end
+
+    let(:expected_emits) do
+      [["elb.access_log",
+        Time.parse('2015-05-24 19:55:36 UTC').to_i,
+        {"type"=>"https",
+         "timestamp"=>"2015-05-24T19:55:36.000000Z",
+         "elb"=>"hoge",
+         "client_port"=>57673,
+         "target_port"=>80,
+         "request_processing_time"=>5.3e-05,
+         "target_processing_time"=>0.000913,
+         "response_processing_time"=>3.6e-05,
+         "elb_status_code"=>200,
+         "target_status_code"=>200,
+         "received_bytes"=>0,
+         "sent_bytes"=>3,
+         "request"=>
+          "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+         "user_agent"=>"curl/7.30.0",
+         "ssl_cipher"=>"ssl_cipher",
+         "ssl_protocol"=>"ssl_protocol",
+         "target_group_arn"=>
+          "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx",
+         "trace_id"=>"Root=xxx",
+         "domain_name"=>"-",
+         "chosen_cert_arn"=>
+          "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx",
+         "client"=>"14.14.124.20",
+         "target"=>"10.0.199.184",
+         "request.method"=>"GET",
+         "request.uri"=>
+          "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+         "request.http_version"=>"HTTP/1.1",
+         "request.uri.scheme"=>"http",
+         "request.uri.user"=>nil,
+         "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
+         "request.uri.port"=>80,
+         "request.uri.path"=>"/",
+         "request.uri.query"=>nil,
+         "request.uri.fragment"=>nil}],
+       ["elb.access_log",
+        Time.parse('2015-05-24 19:55:36 UTC').to_i,
+        {"type"=>"https",
+         "timestamp"=>"2015-05-24T19:55:36.000000Z",
+         "elb"=>"hoge",
+         "client_port"=>57673,
+         "target_port"=>80,
+         "request_processing_time"=>5.3e-05,
+         "target_processing_time"=>0.000913,
+         "response_processing_time"=>3.6e-05,
+         "elb_status_code"=>200,
+         "target_status_code"=>200,
+         "received_bytes"=>0,
+         "sent_bytes"=>3,
+         "request"=>
+          "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+         "user_agent"=>"curl/7.30.0",
+         "ssl_cipher"=>"ssl_cipher",
+         "ssl_protocol"=>"ssl_protocol",
+         "target_group_arn"=>
+          "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx",
+         "trace_id"=>"Root=xxx",
+         "domain_name"=>"-",
+         "chosen_cert_arn"=>
+          "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx",
+         "client"=>"14.14.124.20",
+         "target"=>"10.0.199.184",
+         "request.method"=>"GET",
+         "request.uri"=>
+          "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+         "request.http_version"=>"HTTP/1.1",
+         "request.uri.scheme"=>"http",
+         "request.uri.user"=>nil,
+         "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
+         "request.uri.port"=>80,
+         "request.uri.path"=>"/",
+         "request.uri.query"=>nil,
+         "request.uri.fragment"=>nil}]]
+    end
+
+    let(:fluentd_conf) do
+      {
+        interval: 0,
+        account_id: account_id,
+        s3_bucket: s3_bucket,
+        region: region,
+        start_datetime: (today - 1).to_s,
+        elb_type: 'alb',
+        file_filter: today.iso8601,
+      }
+    end
+
+    it { is_expected.to match_table expected_emits }
+  end
+
   context 'when include bad URI' do
     let(:today_access_log) do
       gzip(<<-EOS)
@@ -429,10 +565,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
     end
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -505,10 +641,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
     end
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -577,10 +713,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
     let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 600).iso8601}_52.68.51.1_8hSqR3o4.log.gz" }
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -646,10 +782,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
     end
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -739,10 +875,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
     let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 601).iso8601}_52.68.51.1_8hSqR3o4.log.gz" }
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -764,10 +900,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
     end
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -794,10 +930,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
     let(:history) { driver.instance.instance_variable_get(:@history) }
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -837,10 +973,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
     end
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -906,10 +1042,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
     end
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -933,10 +1069,10 @@ https xxx hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200
     end
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -962,10 +1098,10 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
     end
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 

data/spec/in_elb_access_log_clb_spec.rb

@@ -44,9 +44,9 @@ describe FluentPluginElbAccessLogInput do
 
   context 'when access log does not exist' do
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
       expect(driver.instance).to_not receive(:save_timestamp).with(today)
       expect(driver.instance).to receive(:save_history)
       expect(driver.instance.log).to_not receive(:warn)
@@ -73,15 +73,15 @@ describe FluentPluginElbAccessLogInput do
     end
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) do
         [double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])]
       end
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
         [double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])]
       end
 
@@ -387,6 +387,128 @@ describe FluentPluginElbAccessLogInput do
     end
   end
 
+  context 'with file_filter' do
+    let(:today_access_log) do
+      <<-EOS
+2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol
+2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol
+      EOS
+    end
+
+    let(:tomorrow_access_log) do
+      <<-EOS
+2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol
+2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol
+      EOS
+    end
+
+    before do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) do
+        [double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])]
+      end
+
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
+        [double('today_objects', contents: [double('today_object', key: today_object_key)])]
+      end
+
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
+        [double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])]
+      end
+
+      expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
+        double('today_s3_object', body: StringIO.new(today_access_log))
+      end
+
+      expect(client).to_not receive(:get_object).with(bucket: s3_bucket, key: tomorrow_object_key) do
+        double('tomorrow_s3_object', body: StringIO.new(tomorrow_access_log))
+      end
+
+      expect(driver.instance).to receive(:save_timestamp).with(today)
+      expect(driver.instance).to receive(:save_history)
+      expect(driver.instance.log).to_not receive(:warn)
+
+      driver_run(driver)
+    end
+
+    let(:expected_emits) do
+      [["elb.access_log",
+        Time.parse('2015-05-24 19:55:36 UTC').to_i,
+        {"timestamp"=>"2015-05-24T19:55:36.000000Z",
+         "elb"=>"hoge",
+         "client"=>"14.14.124.20",
+         "client_port"=>57673,
+         "backend"=>"10.0.199.184",
+         "backend_port"=>80,
+         "request_processing_time"=>5.3e-05,
+         "backend_processing_time"=>0.000913,
+         "response_processing_time"=>3.6e-05,
+         "elb_status_code"=>200,
+         "backend_status_code"=>200,
+         "received_bytes"=>0,
+         "sent_bytes"=>3,
+         "request"=>
+          "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+         "user_agent"=>"curl/7.30.0",
+         "ssl_cipher"=>"ssl_cipher",
+         "ssl_protocol"=>"ssl_protocol",
+         "request.method"=>"GET",
+         "request.uri"=>
+          "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+         "request.http_version"=>"HTTP/1.1",
+         "request.uri.scheme"=>"http",
+         "request.uri.user"=>nil,
+         "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
+         "request.uri.port"=>80,
+         "request.uri.path"=>"/",
+         "request.uri.query"=>nil,
+         "request.uri.fragment"=>nil}],
+       ["elb.access_log",
+        Time.parse('2015-05-24 19:55:36 UTC').to_i,
+        {"timestamp"=>"2015-05-24T19:55:36.000000Z",
+         "elb"=>"hoge",
+         "client"=>"14.14.124.20",
+         "client_port"=>57673,
+         "backend"=>"10.0.199.184",
+         "backend_port"=>80,
+         "request_processing_time"=>5.3e-05,
+         "backend_processing_time"=>0.000913,
+         "response_processing_time"=>3.6e-05,
+         "elb_status_code"=>200,
+         "backend_status_code"=>200,
+         "received_bytes"=>0,
+         "sent_bytes"=>3,
+         "request"=>
+          "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+         "user_agent"=>"curl/7.30.0",
+         "ssl_cipher"=>"ssl_cipher",
+         "ssl_protocol"=>"ssl_protocol",
+         "request.method"=>"GET",
+         "request.uri"=>
+          "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+         "request.http_version"=>"HTTP/1.1",
+         "request.uri.scheme"=>"http",
+         "request.uri.user"=>nil,
+         "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
+         "request.uri.port"=>80,
+         "request.uri.path"=>"/",
+         "request.uri.query"=>nil,
+         "request.uri.fragment"=>nil}]]
+    end
+
+    let(:fluentd_conf) do
+      {
+        interval: 0,
+        account_id: account_id,
+        s3_bucket: s3_bucket,
+        region: region,
+        start_datetime: (today - 1).to_s,
+        file_filter: today.iso8601,
+      }
+    end
+
+    it { is_expected.to match_table expected_emits }
+  end
+
   context 'when include bad URI' do
     let(:today_access_log) do
       <<-EOS
@@ -395,10 +517,10 @@ describe FluentPluginElbAccessLogInput do
     end
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -464,10 +586,10 @@ describe FluentPluginElbAccessLogInput do
     end
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -529,10 +651,10 @@ describe FluentPluginElbAccessLogInput do
     let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 600).iso8601}_52.68.51.1_8hSqR3o4.log" }
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -592,10 +714,10 @@ describe FluentPluginElbAccessLogInput do
     end
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -675,10 +797,10 @@ describe FluentPluginElbAccessLogInput do
     let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 601).iso8601}_52.68.51.1_8hSqR3o4.log" }
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -701,10 +823,10 @@ describe FluentPluginElbAccessLogInput do
     end
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -732,10 +854,10 @@ describe FluentPluginElbAccessLogInput do
     let(:history) { driver.instance.instance_variable_get(:@history) }
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -776,10 +898,10 @@ describe FluentPluginElbAccessLogInput do
     end
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
@@ -839,10 +961,10 @@ describe FluentPluginElbAccessLogInput do
     end
 
     before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+      expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) do
         [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 

data/spec/in_elb_access_log_client_spec.rb

@@ -30,9 +30,9 @@ describe 'FluentPluginElbAccessLogInput#client' do
     Timecop.freeze(today)
     allow_any_instance_of(FluentPluginElbAccessLogInput).to receive(:load_history) { [] }
     allow_any_instance_of(FluentPluginElbAccessLogInput).to receive(:parse_tsfile) { nil }
-    expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-    expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) { [] }
-    expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+    expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+    expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: today_prefix) { [] }
+    expect(client).to receive(:list_objects_v2).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
     allow(FileUtils).to receive(:touch)
     expect(driver.instance).to_not receive(:save_timestamp).with(today)
     expect(driver.instance).to receive(:save_history)

data/spec/in_elb_access_log_config_spec.rb

@@ -49,6 +49,7 @@ describe 'FluentPluginElbAccessLogInput#configure' do
       expect(driver.instance.type_cast).to be_truthy
       expect(driver.instance.parse_request).to be_truthy
       expect(driver.instance.split_addr_port).to be_truthy
+      expect(driver.instance.filter).to be_nil
     end
   end
 
@@ -70,6 +71,7 @@ describe 'FluentPluginElbAccessLogInput#configure' do
     let(:elb_type) { 'alb' }
     let(:filter) { 'elb_status_code:^2' }
     let(:filter_operator) { 'or' }
+    let(:file_filter) { '.*my-elb.*' }
 
     let(:fluentd_conf) do
       {
@@ -97,6 +99,7 @@ describe 'FluentPluginElbAccessLogInput#configure' do
         type_cast: 'false',
         parse_request: 'false',
         split_addr_port: 'false',
+        file_filter: file_filter,
       }
     end
 
@@ -124,6 +127,7 @@ describe 'FluentPluginElbAccessLogInput#configure' do
       expect(driver.instance.type_cast).to be_falsey
       expect(driver.instance.parse_request).to be_falsey
       expect(driver.instance.split_addr_port).to be_falsey
+      expect(driver.instance.file_filter).to eq Regexp.new(file_filter)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-access-log
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Genki Sugawara
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-19 00:00:00.000000000 Z
+date: 2018-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd