fluent-plugin-elb-access-log 0.3.7 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c576eb75af304916ce519a79377d8fe14204a8fd
-  data.tar.gz: 659724f105a175477d751d33924a4eb862c9af8e
+  metadata.gz: 1c73cb54a193ee0d843ee5d038d2f7cf4e8c7cdf
+  data.tar.gz: 4da300a290b3d16f5eeffad7b4e5ee3ccbf66264
 SHA512:
-  metadata.gz: 9b4f3229cc5bcded62629cf3e818f013edd7e7d6faab21c1bbb46f1b029bfc6c45bcabb7ebec72e5c0f544417a662ee0b19ebe85a7f3df3bd3b62430ef64942a
-  data.tar.gz: 157d4fa797fc74bb6c78e2ba09c36279fc44b743a342cbeba17e8cdd7161ef2f3e3d4ad28a54ea8f343629f33b8c6a2de1f2f7ab3a030d5f11031818f0165cfb
+  metadata.gz: aed16e4faac1a13e8f9138443d925645f06110cd3168df5726596549896f2242a68e241c38e062c0c898f5c494cfc4d18888070630ec64367851fd0df0a09368
+  data.tar.gz: 2cb130b4b344352c6ebfa514288cf7705ca1c4da5893e7ff1d51594b416f34bb3d53b4bfa6c2994a837922c976a71e5e5ffda5ede54118dd3402228dc62c5a03

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/.simplecov

@@ -0,0 +1,4 @@
+SimpleCov.start do
+  # exclude directories and files
+  add_filter "/spec/"
+end

data/README.md

@@ -1,6 +1,6 @@
 # fluent-plugin-elb-access-log
 
-Fluentd input plugin for [AWS ELB Access Logs](http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/access-log-collection.html).
+Fluentd input plugin for AWS ELB Access Logs.
 
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-elb-access-log.svg)](http://badge.fury.io/rb/fluent-plugin-elb-access-log)
 [![Build Status](https://travis-ci.org/winebarrel/fluent-plugin-elb-access-log.svg?branch=master)](https://travis-ci.org/winebarrel/fluent-plugin-elb-access-log)
@@ -55,6 +55,8 @@ Or install it yourself as:
 
 ### CLB
 
+see http://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html
+
 ```json
 // elb.access_log:
 {
@@ -90,6 +92,8 @@ Or install it yourself as:
 
 ### ALB
 
+see http://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html
+
 ```json
 {
   "type": "https",

data/fluent-plugin-elb-access-log.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'fluentd'
+  spec.add_dependency 'fluentd', '>= 0.14'
   spec.add_dependency 'aws-sdk-s3', '~> 1.8'
   spec.add_dependency 'addressable'
   spec.add_development_dependency 'bundler'

data/lib/fluent/plugin/in_elb_access_log.rb

@@ -1,7 +1,15 @@
+require 'csv'
+require 'fileutils'
+require 'logger'
+require 'time'
+require 'addressable/uri'
+require 'aws-sdk-s3'
+require 'zlib'
+
 require 'fluent/input'
 require 'fluent_plugin_elb_access_log/version'
 
-class Fluent::ElbAccessLogInput < Fluent::Input
+class Fluent::Plugin::ElbAccessLogInput < Fluent::Input
   Fluent::Plugin.register_input('elb_access_log', self)
 
   USER_AGENT_SUFFIX = "fluent-plugin-elb-access-log/#{FluentPluginElbAccessLog::VERSION}"
@@ -52,14 +60,6 @@ class Fluent::ElbAccessLogInput < Fluent::Input
 
   ELB_TYPES = %(clb alb)
 
-  unless method_defined?(:log)
-    define_method('log') { $log }
-  end
-
-  unless method_defined?(:router)
-    define_method('router') { Fluent::Engine }
-  end
-
   config_param :elb_type,          :string,  default: 'clb'
   config_param :aws_key_id,        :string,  default: nil, secret: true
   config_param :aws_sec_key,       :string,  default: nil, secret: true
@@ -80,17 +80,6 @@ class Fluent::ElbAccessLogInput < Fluent::Input
   config_param :sampling_interval, :integer, default: 1
   config_param :debug,             :bool,    default: false
 
-  def initialize
-    super
-    require 'csv'
-    require 'fileutils'
-    require 'logger'
-    require 'time'
-    require 'addressable/uri'
-    require 'aws-sdk-s3'
-    require 'zlib'
-  end
-
   def configure(conf)
     super
 
@@ -212,8 +201,13 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     records = parse_log(access_log)
 
     records.each do |record|
-      time = Time.parse(record['timestamp'])
-      router.emit(@tag, time.to_i, record)
+      begin
+        time = Time.parse(record['timestamp'])
+        router.emit(@tag, time.to_i, record)
+      rescue ArgumentError => e
+        @log.warn("#{e.message}: #{record}")
+        @log.warn('A record that has bad timestamp is not emitted.')
+      end
     end
   end
 
@@ -226,9 +220,6 @@ class Fluent::ElbAccessLogInput < Fluent::Input
         line = parse_clb_line(line)
       when 'alb'
         line = parse_alb_line(line)
-      else
-        # It is a bug if an exception is thrown
-        raise 'must not happen'
       end
 
       parsed_access_log << line if line
@@ -238,32 +229,24 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     access_log_fields = ACCESS_LOG_FIELDS.fetch(@elb_type)
 
     parsed_access_log.each do |row|
-      begin
-        record = Hash[access_log_fields.keys.zip(row)]
+      record = Hash[access_log_fields.keys.zip(row)]
 
-        access_log_fields.each do |name, conv|
-          record[name] = record[name].send(conv) if conv
-        end
+      access_log_fields.each do |name, conv|
+        record[name] = record[name].send(conv) if conv
+      end
 
-        split_address_port!(record, 'client')
+      split_address_port!(record, 'client')
 
-        case @elb_type
-        when 'clb'
-          split_address_port!(record, 'backend')
-        when 'alb'
-          split_address_port!(record, 'target')
-        else
-          # It is a bug if an exception is thrown
-          raise 'must not happen'
-        end
+      case @elb_type
+      when 'clb'
+        split_address_port!(record, 'backend')
+      when 'alb'
+        split_address_port!(record, 'target')
+      end
 
-        parse_request!(record)
+      parse_request!(record)
 
-        records << record
-      rescue ArgumentError => e
-        @log.warn("#{e.message}: #{row}")
-        @log.warn('A record that has bad timestamp is not emitted.')
-      end
+      records << record
     end
 
     records
@@ -289,7 +272,7 @@ class Fluent::ElbAccessLogInput < Fluent::Input
         parsed[13] = ssl_cipher
         parsed[14] = ssl_protocol
       rescue => e2
-        @log.warn("#{e.message}: #{line}")
+        @log.warn("#{e.message}: #{e2.message}: #{line}")
       end
     end
 
@@ -320,7 +303,7 @@ class Fluent::ElbAccessLogInput < Fluent::Input
         parsed[18] = unquote(domain_name)
         parsed[19] = unquote(chosen_cert_arn)
       rescue => e2
-        @log.warn("#{e.message}: #{line}")
+        @log.warn("#{e.message}: #{e2.message}: #{line}")
       end
     end
 
@@ -443,4 +426,4 @@ class Fluent::ElbAccessLogInput < Fluent::Input
       @log.error_backtrace(e.backtrace)
     end
   end # TimerWatcher
-end # Fluent::ElbAccessLogInput
+end # Fluent::Plugin::ElbAccessLogInput

data/lib/fluent_plugin_elb_access_log/version.rb

@@ -1,3 +1,3 @@
 module FluentPluginElbAccessLog
-  VERSION = '0.3.7'
+  VERSION = '0.4.0'
 end

data/spec/in_elb_access_log_alb_spec.rb

@@ -1,9 +1,9 @@
-describe Fluent::ElbAccessLogInput do
+describe Fluent::Plugin::ElbAccessLogInput do
   let(:account_id) { '123456789012' }
   let(:s3_bucket) { 'my-bucket' }
   let(:region) { 'us-west-1' }
   let(:driver) { create_driver(fluentd_conf) }
-  let(:client){ Aws::S3::Client.new(stub_responses: true) }
+  let!(:client){ Aws::S3::Client.new(stub_responses: true) }
 
   let(:fluentd_conf) do
     {
@@ -29,9 +29,9 @@ describe Fluent::ElbAccessLogInput do
 
   before do
     Timecop.freeze(today)
-    allow_any_instance_of(Fluent::ElbAccessLogInput).to receive(:client) { client }
-    allow_any_instance_of(Fluent::ElbAccessLogInput).to receive(:load_history) { [] }
-    allow_any_instance_of(Fluent::ElbAccessLogInput).to receive(:parse_tsfile) { nil }
+    allow(Aws::S3::Client).to receive(:new) { client }
+    allow_any_instance_of(Fluent::Plugin::ElbAccessLogInput).to receive(:load_history) { [] }
+    allow_any_instance_of(Fluent::Plugin::ElbAccessLogInput).to receive(:parse_tsfile) { nil }
     allow(FileUtils).to receive(:touch)
     expect(driver.instance.log).to_not receive(:error)
   end
@@ -40,7 +40,7 @@ describe Fluent::ElbAccessLogInput do
     Timecop.return
   end
 
-  subject { driver.emits }
+  subject { driver.events }
 
   context 'when access log does not exist' do
     before do
@@ -51,7 +51,7 @@ describe Fluent::ElbAccessLogInput do
       expect(driver.instance).to receive(:save_history)
       expect(driver.instance.log).to_not receive(:warn)
 
-      driver.run
+      driver_run(driver)
     end
 
     it { is_expected.to be_empty }
@@ -97,7 +97,7 @@ https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
       expect(driver.instance).to receive(:save_history)
       expect(driver.instance.log).to_not receive(:warn)
 
-      driver.run
+      driver_run(driver)
     end
 
     let(:expected_emits) do
@@ -302,7 +302,7 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
       allow(Addressable::URI).to receive(:parse).and_raise('parse error')
       expect(driver.instance.log).to receive(:warn).with('parse error: http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/')
 
-      driver.run
+      driver_run(driver)
     end
 
     let(:expected_emits) do
@@ -376,7 +376,7 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
       expect(driver.instance).to receive(:save_history)
       expect(driver.instance.log).to_not receive(:warn)
 
-      driver.run
+      driver_run(driver)
     end
 
     let(:expected_emits) do
@@ -449,7 +449,7 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
       expect(driver.instance).to receive(:save_history)
       expect(driver.instance.log).to_not receive(:warn)
 
-      driver.run
+      driver_run(driver)
     end
 
     let(:expected_emits) do
@@ -521,7 +521,7 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
       expect(CSV).to receive(:parse_line).and_raise('parse error')
       expect(driver.instance.log).to_not receive(:warn)
 
-      driver.run
+      driver_run(driver)
     end
 
     let(:expected_emits) do
@@ -610,7 +610,7 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
       expect(driver.instance).to receive(:save_history)
       expect(driver.instance.log).to_not receive(:warn)
 
-      driver.run
+      driver_run(driver)
     end
 
     it { is_expected.to be_empty }
@@ -639,7 +639,7 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
       expect(driver.instance).to receive(:save_history)
       expect(driver.instance.log).to_not receive(:warn)
 
-      driver.run
+      driver_run(driver)
     end
 
     it { is_expected.to be_empty }
@@ -675,7 +675,7 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
 
     context 'when history.length <= 100' do
       before do
-        driver.run
+        driver_run(driver)
       end
 
       it { is_expected.to eq 1 }
@@ -684,7 +684,7 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
     context 'when history.length > 100' do
       before do
         history.concat (1..100).map(&:to_s)
-        driver.run
+        driver_run(driver)
       end
 
       it { is_expected.to eq 100 }
@@ -714,7 +714,7 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
       expect(driver.instance).to receive(:save_history)
       expect(driver.instance.log).to_not receive(:warn)
 
-      driver.run
+      driver_run(driver)
     end
 
     let(:expected_emits) do
@@ -760,4 +760,94 @@ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.0000
 
     it { is_expected.to match_table expected_emits }
   end
+
+  context 'when inflate fails' do
+    let(:today_access_log) do
+      <<-EOS
+https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
+      EOS
+    end
+
+    before do
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+        [double('today_objects', contents: [double('today_object', key: today_object_key)])]
+      end
+
+      expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
+        double('today_s3_object', body: StringIO.new(today_access_log))
+      end
+
+      expect(driver.instance).to receive(:save_history)
+    end
+
+    specify do
+      expect(driver.instance.log).to receive(:warn).with('incorrect header check: "https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.')
+      driver_run(driver)
+    end
+  end
+
+  context 'when bad timestamp' do
+    let(:today_access_log) do
+      Zlib::Deflate.deflate(<<-EOS)
+https xxx hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
+      EOS
+    end
+
+    before do
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+        [double('today_objects', contents: [double('today_object', key: today_object_key)])]
+      end
+
+      expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
+        double('today_s3_object', body: StringIO.new(today_access_log))
+      end
+
+      expect(driver.instance).to receive(:save_timestamp).with(today)
+      expect(driver.instance).to receive(:save_history)
+    end
+
+    specify do
+      expect(driver.instance.log).to receive(:warn).with(/no time information in "xxx":/)
+      expect(driver.instance.log).to receive(:warn).with('A record that has bad timestamp is not emitted.')
+      driver_run(driver)
+    end
+  end
+
+  context 'when unquote fails' do
+    let(:today_access_log) do
+      Zlib::Deflate.deflate(<<-EOS)
+https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
+      EOS
+    end
+
+    before do
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+        [double('today_objects', contents: [double('today_object', key: today_object_key)])]
+      end
+
+      expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
+        double('today_s3_object', body: StringIO.new(today_access_log))
+      end
+
+      expect(driver.instance).to receive(:save_timestamp).with(today)
+      expect(driver.instance).to receive(:save_history)
+
+      expect(CSV).to receive(:parse_line).and_raise('parse error')
+      expect(driver.instance).to receive(:unquote).and_raise('unquote error')
+    end
+
+    specify do
+      expect(driver.instance.log).to receive(:warn).with(/parse error: unquote error:/)
+      driver_run(driver)
+    end
+  end
 end

data/spec/in_elb_access_log_clb_spec.rb

@@ -1,9 +1,9 @@
-describe Fluent::ElbAccessLogInput do
+describe Fluent::Plugin::ElbAccessLogInput do
   let(:account_id) { '123456789012' }
   let(:s3_bucket) { 'my-bucket' }
   let(:region) { 'us-west-1' }
   let(:driver) { create_driver(fluentd_conf) }
-  let(:client){ Aws::S3::Client.new(stub_responses: true) }
+  let!(:client){ Aws::S3::Client.new(stub_responses: true) }
 
   let(:fluentd_conf) do
     {
@@ -28,18 +28,18 @@ describe Fluent::ElbAccessLogInput do
 
   before do
     Timecop.freeze(today)
-    allow_any_instance_of(Fluent::ElbAccessLogInput).to receive(:client) { client }
-    allow_any_instance_of(Fluent::ElbAccessLogInput).to receive(:load_history) { [] }
-    allow_any_instance_of(Fluent::ElbAccessLogInput).to receive(:parse_tsfile) { nil }
+    allow(Aws::S3::Client).to receive(:new) { client }
+    allow_any_instance_of(Fluent::Plugin::ElbAccessLogInput).to receive(:load_history) { [] }
+    allow_any_instance_of(Fluent::Plugin::ElbAccessLogInput).to receive(:parse_tsfile) { nil }
     allow(FileUtils).to receive(:touch)
     expect(driver.instance.log).to_not receive(:error)
- end
+  end
 
   after do
     Timecop.return
   end
 
-  subject { driver.emits }
+  subject { driver.events }
 
   context 'when access log does not exist' do
     before do
@@ -50,7 +50,7 @@ describe Fluent::ElbAccessLogInput do
       expect(driver.instance).to receive(:save_history)
       expect(driver.instance.log).to_not receive(:warn)
 
-      driver.run
+      driver_run(driver)
     end
 
     it { is_expected.to be_empty }
@@ -96,7 +96,7 @@ describe Fluent::ElbAccessLogInput do
       expect(driver.instance).to receive(:save_history)
       expect(driver.instance.log).to_not receive(:warn)
 
-      driver.run
+      driver_run(driver)
     end
 
     let(:expected_emits) do
@@ -272,7 +272,7 @@ describe Fluent::ElbAccessLogInput do
       allow(Addressable::URI).to receive(:parse).and_raise('parse error')
       expect(driver.instance.log).to receive(:warn).with('parse error: http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/')
 
-      driver.run
+      driver_run(driver)
     end
 
     let(:expected_emits) do
@@ -338,7 +338,7 @@ describe Fluent::ElbAccessLogInput do
       expect(driver.instance).to receive(:save_history)
       expect(driver.instance.log).to_not receive(:warn)
 
-      driver.run
+      driver_run(driver)
     end
 
     let(:expected_emits) do
@@ -403,7 +403,7 @@ describe Fluent::ElbAccessLogInput do
       expect(driver.instance).to receive(:save_history)
       expect(driver.instance.log).to_not receive(:warn)
 
-      driver.run
+      driver_run(driver)
     end
 
     let(:expected_emits) do
@@ -468,7 +468,7 @@ describe Fluent::ElbAccessLogInput do
       expect(CSV).to receive(:parse_line).and_raise('parse error')
       expect(driver.instance.log).to_not receive(:warn)
 
-      driver.run
+      driver_run(driver)
     end
 
     let(:expected_emits) do
@@ -546,7 +546,7 @@ describe Fluent::ElbAccessLogInput do
       expect(driver.instance).to receive(:save_history)
       expect(driver.instance.log).to_not receive(:warn)
 
-      driver.run
+      driver_run(driver)
     end
 
     it { is_expected.to be_empty }
@@ -575,7 +575,7 @@ describe Fluent::ElbAccessLogInput do
       expect(driver.instance).to receive(:save_history)
       expect(driver.instance.log).to_not receive(:warn)
 
-      driver.run
+      driver_run(driver)
     end
 
     it { is_expected.to be_empty }
@@ -611,7 +611,7 @@ describe Fluent::ElbAccessLogInput do
 
     context 'when history.length <= 100' do
       before do
-        driver.run
+        driver_run(driver)
       end
 
       it { is_expected.to eq 1 }
@@ -620,7 +620,7 @@ describe Fluent::ElbAccessLogInput do
     context 'when history.length > 100' do
       before do
         history.concat (1..100).map(&:to_s)
-        driver.run
+        driver_run(driver)
       end
 
       it { is_expected.to eq 100 }
@@ -650,7 +650,7 @@ describe Fluent::ElbAccessLogInput do
       expect(driver.instance).to receive(:save_history)
       expect(driver.instance.log).to_not receive(:warn)
 
-      driver.run
+      driver_run(driver)
     end
 
     let(:expected_emits) do
@@ -689,4 +689,36 @@ describe Fluent::ElbAccessLogInput do
 
     it { is_expected.to match_table expected_emits }
   end
+
+  context 'when no user_agent' do
+    let(:today_access_log) do
+      <<-EOS
+2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol
+      EOS
+    end
+
+    before do
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+        [double('today_objects', contents: [double('today_object', key: today_object_key)])]
+      end
+
+      expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
+        double('today_s3_object', body: StringIO.new(today_access_log))
+      end
+
+      expect(driver.instance).to receive(:save_timestamp).with(today)
+      expect(driver.instance).to receive(:save_history)
+
+      expect(CSV).to receive(:parse_line).and_raise('parse error')
+      expect(driver.instance).to receive(:unquote).and_raise('unquote error')
+    end
+
+    specify do
+      expect(driver.instance.log).to receive(:warn).with(/parse error: unquote error:/)
+      driver_run(driver)
+    end
+  end
 end

data/spec/in_elb_access_log_config_spec.rb

@@ -1,4 +1,4 @@
-describe 'Fluent::ElbAccessLogInput#configure' do
+describe 'Fluent::Plugin::ElbAccessLogInput#configure' do
   let(:account_id) { '123456789012' }
   let(:s3_bucket) { 'my-bucket' }
   let(:region) { 'us-west-1' }
@@ -11,8 +11,8 @@ describe 'Fluent::ElbAccessLogInput#configure' do
     Timecop.freeze(today)
     allow(FileUtils).to receive(:touch)
     allow(File).to receive(:read) { nil }
-    allow_any_instance_of(Fluent::ElbAccessLogInput).to receive(:load_history) { [] }
-    allow_any_instance_of(Fluent::ElbAccessLogInput).to receive(:parse_tsfile) { nil }
+    allow_any_instance_of(Fluent::Plugin::ElbAccessLogInput).to receive(:load_history) { [] }
+    allow_any_instance_of(Fluent::Plugin::ElbAccessLogInput).to receive(:parse_tsfile) { nil }
   end
 
   context 'when default' do
@@ -140,7 +140,7 @@ describe 'Fluent::ElbAccessLogInput#configure' do
     end
 
     before do
-      allow_any_instance_of(Fluent::ElbAccessLogInput).to receive(:parse_tsfile) { Time.parse(tsfile_datetime) }
+      allow_any_instance_of(Fluent::Plugin::ElbAccessLogInput).to receive(:parse_tsfile) { Time.parse(tsfile_datetime) }
     end
 
     it do
@@ -162,7 +162,7 @@ describe 'Fluent::ElbAccessLogInput#configure' do
     end
 
     before do
-      allow_any_instance_of(Fluent::ElbAccessLogInput).to receive(:parse_tsfile) { Time.parse(tsfile_datetime) }
+      allow_any_instance_of(Fluent::Plugin::ElbAccessLogInput).to receive(:parse_tsfile) { Time.parse(tsfile_datetime) }
       allow_any_instance_of(Fluent::Test::TestLogger).to receive(:warn).with("start_datetime(#{start_datetime}) is set. but tsfile datetime(#{tsfile_datetime}) is used")
     end
 

data/spec/spec_helper.rb

@@ -1,4 +1,8 @@
+require 'coveralls'
+Coveralls.wear!
+
 require 'fluent/test'
+require 'fluent/test/driver/input'
 require 'fluent/plugin/in_elb_access_log'
 
 require 'aws-sdk-s3'
@@ -6,16 +10,6 @@ require 'time'
 require 'timecop'
 require 'rspec/match_table'
 
-if ENV['TRAVIS']
-  require 'simplecov'
-  require 'coveralls'
-
-  SimpleCov.formatter = Coveralls::SimpleCov::Formatter
-  SimpleCov.start do
-    add_filter "spec/"
-  end
-end
-
 # Disable Test::Unit
 module Test::Unit::RunCount; def run(*); end; end
 
@@ -46,8 +40,14 @@ region #{region}
 #{additional_options}
   EOS
 
-  tag = options[:tag] || 'test.default'
-  Fluent::Test::OutputTestDriver.new(Fluent::ElbAccessLogInput, tag).configure(fluentd_conf)
+  Fluent::Test::Driver::Input.new(Fluent::Plugin::ElbAccessLogInput).configure(fluentd_conf)
+end
+
+def driver_run(driver)
+  driver.run do
+    coolio_loop = driver.instance.instance_variable_get(:@loop)
+    sleep 0.1 until coolio_loop.instance_variable_get(:@running)
+  end
 end
 
 # prevent Test::Unit's AutoRunner from executing during RSpec's rake task

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-access-log
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.4.0
 platform: ruby
 authors:
 - Genki Sugawara
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.14'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
@@ -157,8 +157,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".coveralls.yml"
 - ".gitignore"
 - ".rspec"
+- ".simplecov"
 - ".travis.yml"
 - Gemfile
 - LICENSE.txt