fluent-plugin-elb-access-log 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6df223556ac6d0e817804ee03aa63403bc3bd33f
+  data.tar.gz: e25d57033fe58bde5b24d13bfff67b7ae848e684
+SHA512:
+  metadata.gz: 0b2b2c56a9ae17ec2452c36f32a1297809b0e51cedbc901757e0af089ae13b78e5f767f18ec68b77f330a803822a379276dfcd8089e540ab3faab3333c87aab4
+  data.tar.gz: 1f98ee986f5de353aeceb3e8e1baf196aa1322c17887f5f42df69a96aa9e4b68cb357f47524aa9f44b25078e5b160bfd0b2988b4f07cafeb5846b8198956bfe0

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1,2 @@
+--require spec_helper
+--colour

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+- 2.0.0
+script:
+- bundle install
+- bundle exec rake

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-elb-access-log.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Genki Sugawara
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,75 @@
+# fluent-plugin-elb-access-log
+
+Fluentd input plugin for [AWS ELB Access Logs](http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/access-log-collection.html).
+
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-elb-access-log.png)](http://badge.fury.io/rb/fluent-plugin-elb-access-log)
+[![Build Status](https://travis-ci.org/winebarrel/fluent-plugin-elb-access-log.svg?branch=master)](https://travis-ci.org/winebarrel/fluent-plugin-elb-access-log)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-elb-access-log'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-elb-access-log
+
+## Configuration
+
+```apache
+<source>
+  type elb_access_log
+  #aws_key_id YOUR_ACCESS_KEY_ID
+  #aws_sec_key YOUR_SECRET_ACCESS_KEY
+  #profile PROFILE_NAME
+  #credentials_path path/to/credentials_file
+
+  account_id 123456789012 # required
+  s3_bucket BUCKET_NAME # required
+  s3_region us-west-1 # required
+  #s3_prefix PREFIX
+
+  #tag elb.access_log
+  #tsfile_path /var/tmp/fluent-plugin-elb-access-log.ts
+  #interval 300
+  #start_datetime 2015/05/24 17:00
+  #debug false
+</source>
+```
+
+```javascript
+// elb.access_log:
+{
+  "timestamp":"2015-05-24T08:25:36.229576Z",
+  "elb":"hoge",
+  "client_port":"14.14.124.20:52232",
+  "backend_port":"10.0.199.184:80",
+  "request_processing_time":5.5e-05,
+  "backend_processing_time":0.000893,
+  "response_processing_time":5.7e-05,
+  "elb_status_code":200,
+  "backend_status_code":200,
+  "received_bytes":0,
+  "sent_bytes":3,
+  "request":"GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+  "request.method":"GET",
+  "request.uri":"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+  "request.http_version":"HTTP/1.1",
+  "request.uri.scheme":"http",
+  "request.uri.userinfo":null,
+  "request.uri.host":"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
+  "request.uri.port":80,
+  "request.uri.registry":null,
+  "request.uri.path":"/",
+  "request.uri.opaque":null,
+  "request.uri.query":null,
+  "request.uri.fragment":null
+}
+```

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new('spec')
+
+task :default => :spec

data/fluent-plugin-elb-access-log.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fluent_plugin_elb_access_log/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'fluent-plugin-elb-access-log'
+  spec.version       = FluentPluginElbAccessLog::VERSION
+  spec.authors       = ['Genki Sugawara']
+  spec.email         = ['sugawara@cookpad.com']
+  spec.summary       = %q{Fluentd input plugin for AWS ELB Access Logs.}
+  spec.description   = %q{Fluentd input plugin for AWS ELB Access Logs.}
+  spec.homepage      = 'https://github.com/winebarrel/fluent-plugin-elb-access-log'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'fluentd'
+  spec.add_dependency 'aws-sdk', '~> 2'
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec', '>= 3.0.0'
+  spec.add_development_dependency 'timecop'
+end

data/lib/fluent/plugin/in_elb_access_log.rb

@@ -0,0 +1,213 @@
+require 'fluent_plugin_elb_access_log/version'
+
+class Fluent::ElbAccessLogInput < Fluent::Input
+  Fluent::Plugin.register_input('elb_access_log', self)
+
+  USER_AGENT_SUFFIX = "fluent-plugin-elb-access-log/#{FluentPluginElbAccessLog::VERSION}"
+
+  # http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/access-log-collection.html#access-log-entry-format
+  ACCESS_LOG_FIELDS = {
+    'timestamp'                => nil,
+    'elb'                      => nil,
+    'client_port'              => nil,
+    'backend_port'             => nil,
+    'request_processing_time'  => :to_f,
+    'backend_processing_time'  => :to_f,
+    'response_processing_time' => :to_f,
+    'elb_status_code'          => :to_i,
+    'backend_status_code'      => :to_i,
+    'received_bytes'           => :to_i,
+    'sent_bytes'               => :to_i,
+    'request'                  => nil,
+  }
+
+  unless method_defined?(:log)
+    define_method('log') { $log }
+  end
+
+  unless method_defined?(:router)
+    define_method('router') { Fluent::Engine }
+  end
+
+  config_param :aws_key_id,       :string,  :default => nil
+  config_param :aws_sec_key,      :string,  :default => nil
+  config_param :profile,          :string,  :default => nil
+  config_param :credentials_path, :string,  :default => nil
+  config_param :account_id,       :string
+  config_param :s3_bucket,        :string
+  config_param :s3_region,        :string
+  config_param :s3_prefix,        :string,  :default => nil
+  config_param :tag,              :string,  :default => 'elb.access_log'
+  config_param :tsfile_path,      :string,  :default => '/var/tmp/fluent-plugin-elb-access-log.ts'
+  config_param :interval,         :time,    :default => 300
+  config_param :start_datetime,   :string,  :default => nil
+  config_param :debug,            :bool,    :default => false
+
+  def initialize
+    super
+    require 'csv'
+    require 'fileutils'
+    require 'logger'
+    require 'time'
+    require 'uri'
+    require 'aws-sdk'
+  end
+
+  def configure(conf)
+    super
+
+    FileUtils.touch(@tsfile_path)
+
+    if @start_datetime
+      @start_datetime = Time.parse(@start_datetime).utc
+    else
+      @start_datetime = Time.parse(File.read(@tsfile_path)).utc rescue Time.now.utc
+    end
+  end
+
+  def start
+    super
+
+    # Load client
+    client
+
+    @loop = Coolio::Loop.new
+    timestamp = @start_datetime
+
+    timer = TimerWatcher.new(@interval, true, log) do
+      new_timestamp = fetch(timestamp)
+
+      if timestamp != new_timestamp
+        save_timestamp(new_timestamp)
+        timestamp = new_timestamp
+      end
+    end
+
+    @loop.attach(timer)
+    @thread = Thread.new(&method(:run))
+  end
+
+  def shutdown
+    @loop.stop
+    @thread.join
+  end
+
+  private
+
+  def run
+    @loop.run
+  rescue => e
+    log.error(e.message)
+    log.error_backtrace(e.backtrace)
+  end
+
+  def fetch(timestamp)
+    last_timestamp = timestamp
+
+    prefixes(timestamp).each do |prefix|
+      client.list_objects(:bucket => @s3_bucket, :prefix => prefix).each do |page|
+        page.contents.each do |obj|
+          account_id, logfile_const, region, elb_name, logfile_datetime, ip, logfile_suffix = obj.key.split('_', 7)
+          logfile_datetime = Time.parse(logfile_datetime)
+
+          if logfile_suffix !~ /\.log\z/ or logfile_datetime <= timestamp
+            next
+          end
+
+          access_log = client.get_object(bucket: @s3_bucket, key: obj.key).first.body.string
+          emit_access_log(access_log)
+          last_timestamp = logfile_datetime
+        end
+      end
+    end
+
+    last_timestamp
+  end
+
+  def prefixes(timestamp)
+    base_prefix = "AWSLogs/#{@account_id}/elasticloadbalancing/#{@s3_region}/"
+    base_prefix = "#{@s3_prefix}/#{base_prefix}" if @s3_prefix
+
+    [timestamp - 86400, timestamp, timestamp + 86400].map do |date|
+      base_prefix + date.strftime('%Y/%m/%d/')
+    end
+  end
+
+  def emit_access_log(access_log)
+    access_log = CSV.parse(access_log, :col_sep => ' ')
+
+    access_log.each do |row|
+      record = Hash[ACCESS_LOG_FIELDS.keys.zip(row)]
+
+      ACCESS_LOG_FIELDS.each do |name, conv|
+        record[name] = record[name].send(conv) if conv
+      end
+
+      parse_request!(record)
+
+      time = Time.parse(record['timestamp'])
+      router.emit(@tag, time, record)
+    end
+  end
+
+  def parse_request!(record)
+    request = record['request']
+    method, uri, http_version = request.split(' ', 3)
+
+    record['request.method'] = method
+    record['request.uri'] = uri
+    record['request.http_version'] = http_version
+
+    uri = URI.parse(uri)
+
+    [:scheme ,:userinfo, :host, :port, :registry, :path, :opaque, :query, :fragment].each do |key|
+      record["request.uri.#{key}"] = uri.send(key)
+    end
+  end
+
+  def save_timestamp(timestamp)
+    open(@tsfile_path, 'w') do |tsfile|
+      tsfile << timestamp.to_s
+    end
+  end
+
+  def client
+    return @client if @client
+
+    options = {:user_agent_suffix => USER_AGENT_SUFFIX}
+    options[:region] = @region if @region
+
+    if @aws_key_id and @aws_sec_key
+      options[:access_key_id] = @aws_key_id
+      options[:secret_access_key] = @aws_sec_key
+    elsif @profile
+      credentials_opts = {:profile_name => @profile}
+      credentials_opts[:path] = @credentials_path if @credentials_path
+      credentials = Aws::SharedCredentials.new(credentials_opts)
+      options[:credentials] = credentials
+    end
+
+    if @debug
+      options[:logger] = Logger.new(log.out)
+      options[:log_level] = :debug
+      #options[:http_wire_trace] = true
+    end
+
+    @client = Aws::S3::Client.new(options)
+  end
+
+  class TimerWatcher < Coolio::TimerWatcher
+    def initialize(interval, repeat, log, &callback)
+      @callback = callback
+      @log = log
+      super(interval, repeat)
+    end
+
+    def on_timer
+      @callback.call
+    rescue => e
+      @log.error(e.message)
+      @log.error_backtrace(e.backtrace)
+    end
+  end # TimerWatcher
+end # Fluent::ElbAccessLogInput

data/lib/fluent_plugin_elb_access_log/version.rb

@@ -0,0 +1,3 @@
+module FluentPluginElbAccessLog
+  VERSION = '0.1.0'
+end

data/spec/in_elb_access_log_spec.rb

@@ -0,0 +1,279 @@
+describe Fluent::ElbAccessLogInput do
+  let(:account_id) { '123456789012' }
+  let(:s3_bucket) { 'my-bucket' }
+  let(:s3_region) { 'us-west-1' }
+  let(:driver) { create_driver(fluentd_conf) }
+  let(:client){ Aws::S3::Client.new(stub_responses: true) }
+
+  let(:fluentd_conf) do
+    {
+      account_id: account_id,
+      s3_bucket: s3_bucket,
+      s3_region: s3_region,
+      start_datetime: (today - 1).to_s,
+    }
+  end
+
+  let(:today) { Time.parse('2015/05/24 18:30 UTC') }
+  let(:yesterday) { today - 86400 }
+  let(:tomorrow) { today + 86400 }
+
+  let(:today_prefix) { "AWSLogs/#{account_id}/elasticloadbalancing/#{s3_region}/#{today.strftime('%Y/%m/%d')}/" }
+  let(:yesterday_prefix) { "AWSLogs/#{account_id}/elasticloadbalancing/#{s3_region}/#{yesterday.strftime('%Y/%m/%d')}/" }
+  let(:tomorrow_prefix) { "AWSLogs/#{account_id}/elasticloadbalancing/#{s3_region}/#{tomorrow.strftime('%Y/%m/%d')}/" }
+
+  let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{today.iso8601}_52.68.51.1_8hSqR3o4.log" }
+  let(:yesterday_object_key) { "#{yesterday_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{yesterday.iso8601}_52.68.51.1_8hSqR3o4.log" }
+  let(:tomorrow_object_key) { "#{tomorrow_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{tomorrow.iso8601}_52.68.51.1_8hSqR3o4.log" }
+
+  before do
+    Timecop.freeze(today)
+    allow_any_instance_of(Fluent::ElbAccessLogInput).to receive(:client) { client }
+  end
+
+  after do
+    Timecop.return
+  end
+
+  subject { x = driver.emits; x }
+
+  context 'when access log does not exist' do
+    before do
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) { [] }
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+      expect(driver.instance).to_not receive(:save_timestamp).with(today)
+
+      driver.run
+    end
+
+    it { is_expected.to be_empty }
+  end
+
+  context 'when access log exists' do
+    let(:today_access_log) do
+      <<-EOS
+2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
+2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
+      EOS
+    end
+
+    let(:tomorrow_access_log) do
+      <<-EOS
+2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
+2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
+      EOS
+    end
+
+    before do
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) do
+        [double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])]
+      end
+
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+        [double('today_objects', contents: [double('today_object', key: today_object_key)])]
+      end
+
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
+        [double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])]
+      end
+
+      expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
+        [double('today_s3_object', body: StringIO.new(today_access_log))]
+      end
+
+      expect(client).to receive(:get_object).with(bucket: s3_bucket, key: tomorrow_object_key) do
+        [double('tomorrow_s3_object', body: StringIO.new(tomorrow_access_log))]
+      end
+
+      expect(driver.instance).to receive(:save_timestamp).with(tomorrow)
+
+      driver.run
+    end
+
+    let(:expected_emits) do
+      [["elb.access_log",
+        Time.parse('2015-05-24 19:55:36 UTC'),
+        {"timestamp"=>"2015-05-24T19:55:36.000000Z",
+         "elb"=>"hoge",
+         "client_port"=>"14.14.124.20:57673",
+         "backend_port"=>"10.0.199.184:80",
+         "request_processing_time"=>5.3e-05,
+         "backend_processing_time"=>0.000913,
+         "response_processing_time"=>3.6e-05,
+         "elb_status_code"=>200,
+         "backend_status_code"=>200,
+         "received_bytes"=>0,
+         "sent_bytes"=>3,
+         "request"=>
+          "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+         "request.method"=>"GET",
+         "request.uri"=>
+          "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+         "request.http_version"=>"HTTP/1.1",
+         "request.uri.scheme"=>"http",
+         "request.uri.userinfo"=>nil,
+         "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
+         "request.uri.port"=>80,
+         "request.uri.registry"=>nil,
+         "request.uri.path"=>"/",
+         "request.uri.opaque"=>nil,
+         "request.uri.query"=>nil,
+         "request.uri.fragment"=>nil}],
+       ["elb.access_log",
+        Time.parse('2015-05-24 19:55:36 UTC'),
+        {"timestamp"=>"2015-05-24T19:55:36.000000Z",
+         "elb"=>"hoge",
+         "client_port"=>"14.14.124.20:57673",
+         "backend_port"=>"10.0.199.184:80",
+         "request_processing_time"=>5.3e-05,
+         "backend_processing_time"=>0.000913,
+         "response_processing_time"=>3.6e-05,
+         "elb_status_code"=>200,
+         "backend_status_code"=>200,
+         "received_bytes"=>0,
+         "sent_bytes"=>3,
+         "request"=>
+          "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+         "request.method"=>"GET",
+         "request.uri"=>
+          "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+         "request.http_version"=>"HTTP/1.1",
+         "request.uri.scheme"=>"http",
+         "request.uri.userinfo"=>nil,
+         "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
+         "request.uri.port"=>80,
+         "request.uri.registry"=>nil,
+         "request.uri.path"=>"/",
+         "request.uri.opaque"=>nil,
+         "request.uri.query"=>nil,
+         "request.uri.fragment"=>nil}],
+       ["elb.access_log",
+        Time.parse('2015-05-25 19:55:36 UTC'),
+        {"timestamp"=>"2015-05-25T19:55:36.000000Z",
+         "elb"=>"hoge",
+         "client_port"=>"14.14.124.20:57673",
+         "backend_port"=>"10.0.199.184:80",
+         "request_processing_time"=>5.3e-05,
+         "backend_processing_time"=>0.000913,
+         "response_processing_time"=>3.6e-05,
+         "elb_status_code"=>200,
+         "backend_status_code"=>200,
+         "received_bytes"=>0,
+         "sent_bytes"=>3,
+         "request"=>
+          "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+         "request.method"=>"GET",
+         "request.uri"=>
+          "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+         "request.http_version"=>"HTTP/1.1",
+         "request.uri.scheme"=>"http",
+         "request.uri.userinfo"=>nil,
+         "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
+         "request.uri.port"=>80,
+         "request.uri.registry"=>nil,
+         "request.uri.path"=>"/",
+         "request.uri.opaque"=>nil,
+         "request.uri.query"=>nil,
+         "request.uri.fragment"=>nil}],
+       ["elb.access_log",
+        Time.parse('2015-05-25 19:55:36 UTC'),
+        {"timestamp"=>"2015-05-25T19:55:36.000000Z",
+         "elb"=>"hoge",
+         "client_port"=>"14.14.124.20:57673",
+         "backend_port"=>"10.0.199.184:80",
+         "request_processing_time"=>5.3e-05,
+         "backend_processing_time"=>0.000913,
+         "response_processing_time"=>3.6e-05,
+         "elb_status_code"=>200,
+         "backend_status_code"=>200,
+         "received_bytes"=>0,
+         "sent_bytes"=>3,
+         "request"=>
+          "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+         "request.method"=>"GET",
+         "request.uri"=>
+          "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+         "request.http_version"=>"HTTP/1.1",
+         "request.uri.scheme"=>"http",
+         "request.uri.userinfo"=>nil,
+         "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
+         "request.uri.port"=>80,
+         "request.uri.registry"=>nil,
+         "request.uri.path"=>"/",
+         "request.uri.opaque"=>nil,
+         "request.uri.query"=>nil,
+         "request.uri.fragment"=>nil}]]
+    end
+
+    it { is_expected.to eq expected_emits }
+  end
+
+  context 'when access log exists (with tag option)' do
+    let(:today_access_log) do
+      <<-EOS
+2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
+      EOS
+    end
+
+  let(:fluentd_conf) do
+    {
+      account_id: account_id,
+      s3_bucket: s3_bucket,
+      s3_region: s3_region,
+      start_datetime: (today - 1).to_s,
+      tag: 'any.tag'
+    }
+  end
+
+    before do
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+        [double('today_objects', contents: [double('today_object', key: today_object_key)])]
+      end
+
+      expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
+        [double('today_s3_object', body: StringIO.new(today_access_log))]
+      end
+
+      expect(driver.instance).to receive(:save_timestamp).with(today)
+
+      driver.run
+    end
+
+    let(:expected_emits) do
+      [["any.tag",
+        Time.parse('2015-05-24 19:55:36 UTC'),
+        {"timestamp"=>"2015-05-24T19:55:36.000000Z",
+         "elb"=>"hoge",
+         "client_port"=>"14.14.124.20:57673",
+         "backend_port"=>"10.0.199.184:80",
+         "request_processing_time"=>5.3e-05,
+         "backend_processing_time"=>0.000913,
+         "response_processing_time"=>3.6e-05,
+         "elb_status_code"=>200,
+         "backend_status_code"=>200,
+         "received_bytes"=>0,
+         "sent_bytes"=>3,
+         "request"=>
+          "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+         "request.method"=>"GET",
+         "request.uri"=>
+          "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+         "request.http_version"=>"HTTP/1.1",
+         "request.uri.scheme"=>"http",
+         "request.uri.userinfo"=>nil,
+         "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
+         "request.uri.port"=>80,
+         "request.uri.registry"=>nil,
+         "request.uri.path"=>"/",
+         "request.uri.opaque"=>nil,
+         "request.uri.query"=>nil,
+         "request.uri.fragment"=>nil}]]
+    end
+
+    it { is_expected.to eq expected_emits }
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,40 @@
+require 'fluent/test'
+require 'fluent/plugin/in_elb_access_log'
+
+require 'aws-sdk'
+require 'time'
+require 'timecop'
+
+# Disable Test::Unit
+module Test::Unit::RunCount; def run(*); end; end
+
+RSpec.configure do |config|
+  config.before(:all) do
+    Fluent::Test.setup
+  end
+end
+
+def create_driver(options = {})
+  options = {
+    interval: 0,
+  }.merge(options)
+
+  account_id = options.fetch(:account_id) || '123456789012'
+  s3_bucket = options.fetch(:s3_bucket) || 'my-bucket'
+  s3_region = options.fetch(:s3_region) || 'us-west-1'
+
+  additional_options = options.select {|k, v| v }.map {|key, value|
+    "#{key} #{value}"
+  }.join("\n")
+
+  fluentd_conf = <<-EOS
+type elb_access_log
+account_id #{account_id}
+s3_bucket #{s3_bucket}
+s3_region #{s3_region}
+#{additional_options}
+  EOS
+
+  tag = options[:tag] || 'test.default'
+  Fluent::Test::OutputTestDriver.new(Fluent::ElbAccessLogInput, tag).configure(fluentd_conf)
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-elb-access-log
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Genki Sugawara
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-05-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd input plugin for AWS ELB Access Logs.
+email:
+- sugawara@cookpad.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-elb-access-log.gemspec
+- lib/fluent/plugin/in_elb_access_log.rb
+- lib/fluent_plugin_elb_access_log/version.rb
+- spec/in_elb_access_log_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/winebarrel/fluent-plugin-elb-access-log
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Fluentd input plugin for AWS ELB Access Logs.
+test_files:
+- spec/in_elb_access_log_spec.rb
+- spec/spec_helper.rb