fluent-plugin-elb-access-log 0.3.7.pre.beta1 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f4c4c5dcff53f3ac390c49257c2dccc74e827a06
-  data.tar.gz: e62f5e6f828379ef785829e4ec5fef39b2bb451c
+  metadata.gz: c576eb75af304916ce519a79377d8fe14204a8fd
+  data.tar.gz: 659724f105a175477d751d33924a4eb862c9af8e
 SHA512:
-  metadata.gz: c197cd9d6a3c4b25a463b433dfad6ee3b814d90969290a6f1cfb4ee4af784294e18cc596287c966102e43448b97273bc539aeb770e500e44c4d5f2d31d56ff65
-  data.tar.gz: 0c69eeca3fffbd7f58629f9fdbb58d1a4596556c8d8e919e7fbe39d0d2bc2671531f7a4cf39ede26260bbc56cc48f09555f553579c5e3f2c8e29aea66d71239b
+  metadata.gz: 9b4f3229cc5bcded62629cf3e818f013edd7e7d6faab21c1bbb46f1b029bfc6c45bcabb7ebec72e5c0f544417a662ee0b19ebe85a7f3df3bd3b62430ef64942a
+  data.tar.gz: 157d4fa797fc74bb6c78e2ba09c36279fc44b743a342cbeba17e8cdd7161ef2f3e3d4ad28a54ea8f343629f33b8c6a2de1f2f7ab3a030d5f11031818f0165cfb

data/README.md

@@ -4,6 +4,7 @@ Fluentd input plugin for [AWS ELB Access Logs](http://docs.aws.amazon.com/Elasti
 
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-elb-access-log.svg)](http://badge.fury.io/rb/fluent-plugin-elb-access-log)
 [![Build Status](https://travis-ci.org/winebarrel/fluent-plugin-elb-access-log.svg?branch=master)](https://travis-ci.org/winebarrel/fluent-plugin-elb-access-log)
+[![Coverage Status](https://coveralls.io/repos/github/winebarrel/fluent-plugin-elb-access-log/badge.svg?branch=master)](https://coveralls.io/github/winebarrel/fluent-plugin-elb-access-log?branch=master)
 
 ## Installation
 
@@ -46,10 +47,15 @@ Or install it yourself as:
   #history_length 100
   #sampling_interval 1
   #debug false
+  #elb_type clb # or alb
 </source>
 ```
 
-```javascript
+## Outout
+
+### CLB
+
+```json
 // elb.access_log:
 {
   "timestamp":"2015-05-24T08:25:36.229576Z",
@@ -82,9 +88,41 @@ Or install it yourself as:
 }
 ```
 
-# Difference with [fluent-plugin-elb-log](https://github.com/shinsaka/fluent-plugin-elb-log)
+### ALB
 
-* Use AWS SDK for Ruby V2.
-* It is possible to change the record tag.
-* List objects with prefix.
-* Perse request line URI.
+```json
+{
+  "type": "https",
+  "timestamp": "2015-05-24T19:55:36.000000Z",
+  "elb": "hoge",
+  "client_port": 57673,
+  "target_port": 80,
+  "request_processing_time": 5.3e-05,
+  "target_processing_time": 0.000913,
+  "response_processing_time": 3.6e-05,
+  "elb_status_code": 200,
+  "target_status_code": 200,
+  "received_bytes": 0,
+  "sent_bytes": 3,
+  "request": "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+  "user_agent": "curl/7.30.0",
+  "ssl_cipher": "ssl_cipher",
+  "ssl_protocol": "ssl_protocol",
+  "target_group_arn": "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx",
+  "trace_id": "Root=xxx",
+  "domain_name": "-",
+  "chosen_cert_arn": "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx",
+  "client": "14.14.124.20",
+  "target": "10.0.199.184",
+  "request.method": "GET",
+  "request.uri": "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+  "request.http_version": "HTTP/1.1",
+  "request.uri.scheme": "http",
+  "request.uri.user": null,
+  "request.uri.host": "hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
+  "request.uri.port": 80,
+  "request.uri.path": "/",
+  "request.uri.query": null,
+  "request.uri.fragment": null
+}
+```

data/fluent-plugin-elb-access-log.gemspec

@@ -19,11 +19,13 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'fluentd'
-  spec.add_dependency 'aws-sdk', '~> 2.1.2'
+  spec.add_dependency 'aws-sdk-s3', '~> 1.8'
   spec.add_dependency 'addressable'
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec', '>= 3.0.0'
   spec.add_development_dependency 'timecop'
   spec.add_development_dependency 'test-unit', '>= 3.1.0'
+  spec.add_development_dependency 'rspec-match_table', '>= 0.1.1'
+  spec.add_development_dependency 'coveralls'
 end

data/lib/fluent/plugin/in_elb_access_log.rb

@@ -6,25 +6,52 @@ class Fluent::ElbAccessLogInput < Fluent::Input
 
   USER_AGENT_SUFFIX = "fluent-plugin-elb-access-log/#{FluentPluginElbAccessLog::VERSION}"
 
-  # http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/access-log-collection.html#access-log-entry-format
   ACCESS_LOG_FIELDS = {
-    'timestamp'                => nil,
-    'elb'                      => nil,
-    'client_port'              => nil,
-    'backend_port'             => nil,
-    'request_processing_time'  => :to_f,
-    'backend_processing_time'  => :to_f,
-    'response_processing_time' => :to_f,
-    'elb_status_code'          => :to_i,
-    'backend_status_code'      => :to_i,
-    'received_bytes'           => :to_i,
-    'sent_bytes'               => :to_i,
-    'request'                  => nil,
-    'user_agent'               => nil,
-    'ssl_cipher'               => nil,
-    'ssl_protocol'             => nil,
+    # http://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html
+    'clb' => {
+      'timestamp'                => nil,
+      'elb'                      => nil,
+      'client_port'              => nil,
+      'backend_port'             => nil,
+      'request_processing_time'  => :to_f,
+      'backend_processing_time'  => :to_f,
+      'response_processing_time' => :to_f,
+      'elb_status_code'          => :to_i,
+      'backend_status_code'      => :to_i,
+      'received_bytes'           => :to_i,
+      'sent_bytes'               => :to_i,
+      'request'                  => nil,
+      'user_agent'               => nil,
+      'ssl_cipher'               => nil,
+      'ssl_protocol'             => nil,
+    },
+    # http://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html
+    'alb' => {
+      'type'                     => nil,
+      'timestamp'                => nil,
+      'elb'                      => nil,
+      'client_port'              => nil,
+      'target_port'              => nil,
+      'request_processing_time'  => :to_f,
+      'target_processing_time'   => :to_f,
+      'response_processing_time' => :to_f,
+      'elb_status_code'          => :to_i,
+      'target_status_code'       => :to_i,
+      'received_bytes'           => :to_i,
+      'sent_bytes'               => :to_i,
+      'request'                  => nil,
+      'user_agent'               => nil,
+      'ssl_cipher'               => nil,
+      'ssl_protocol'             => nil,
+      'target_group_arn'         => nil,
+      'trace_id'                 => nil,
+      'domain_name'              => nil,
+      'chosen_cert_arn'          => nil,
+    },
   }
 
+  ELB_TYPES = %(clb alb)
+
   unless method_defined?(:log)
     define_method('log') { $log }
   end
@@ -33,24 +60,25 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     define_method('router') { Fluent::Engine }
   end
 
-  config_param :aws_key_id,        :string,  :default => nil, :secret => true
-  config_param :aws_sec_key,       :string,  :default => nil, :secret => true
-  config_param :profile,           :string,  :default => nil
-  config_param :credentials_path,  :string,  :default => nil
-  config_param :http_proxy,        :string,  :default => nil
+  config_param :elb_type,          :string,  default: 'clb'
+  config_param :aws_key_id,        :string,  default: nil, secret: true
+  config_param :aws_sec_key,       :string,  default: nil, secret: true
+  config_param :profile,           :string,  default: nil
+  config_param :credentials_path,  :string,  default: nil
+  config_param :http_proxy,        :string,  default: nil
   config_param :account_id,        :string
   config_param :region,            :string
   config_param :s3_bucket,         :string
-  config_param :s3_prefix,         :string,  :default => nil
-  config_param :tag,               :string,  :default => 'elb.access_log'
-  config_param :tsfile_path,       :string,  :default => '/var/tmp/fluent-plugin-elb-access-log.ts'
-  config_param :histfile_path,     :string,  :default => '/var/tmp/fluent-plugin-elb-access-log.history'
-  config_param :interval,          :time,    :default => 300
-  config_param :start_datetime,    :string,  :default => nil
-  config_param :buffer_sec,        :time,    :default => 600
-  config_param :history_length,    :integer, :default => 100
-  config_param :sampling_interval, :integer, :default => 1
-  config_param :debug,             :bool,    :default => false
+  config_param :s3_prefix,         :string,  default: nil
+  config_param :tag,               :string,  default: 'elb.access_log'
+  config_param :tsfile_path,       :string,  default: '/var/tmp/fluent-plugin-elb-access-log.ts'
+  config_param :histfile_path,     :string,  default: '/var/tmp/fluent-plugin-elb-access-log.history'
+  config_param :interval,          :time,    default: 300
+  config_param :start_datetime,    :string,  default: nil
+  config_param :buffer_sec,        :time,    default: 600
+  config_param :history_length,    :integer, default: 100
+  config_param :sampling_interval, :integer, default: 1
+  config_param :debug,             :bool,    default: false
 
   def initialize
     super
@@ -59,12 +87,17 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     require 'logger'
     require 'time'
     require 'addressable/uri'
-    require 'aws-sdk'
+    require 'aws-sdk-s3'
+    require 'zlib'
   end
 
   def configure(conf)
     super
 
+    unless ELB_TYPES.include?(@elb_type)
+      raise raise Fluent::ConfigError, "Invalid ELB type: #{@elb_type}"
+    end
+
     FileUtils.touch(@tsfile_path)
     FileUtils.touch(@histfile_path)
     tsfile_start_datetime = parse_tsfile
@@ -113,6 +146,7 @@ class Fluent::ElbAccessLogInput < Fluent::Input
   def shutdown
     @loop.stop
     @thread.join
+    super
   end
 
   private
@@ -128,17 +162,28 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     last_timestamp = timestamp
 
     prefixes(timestamp).each do |prefix|
-      client.list_objects(:bucket => @s3_bucket, :prefix => prefix).each do |page|
+      client.list_objects(bucket: @s3_bucket, prefix: prefix).each do |page|
         page.contents.each do |obj|
           account_id, logfile_const, region, elb_name, logfile_datetime, ip, logfile_suffix = obj.key.split('_', 7)
           logfile_datetime = Time.parse(logfile_datetime)
 
-          if logfile_suffix !~ /\.log\z/ or logfile_datetime <= (timestamp - @buffer_sec)
+          if logfile_suffix !~ /\.log(\.gz)?\z/ or logfile_datetime <= (timestamp - @buffer_sec)
             next
           end
 
           unless @history.include?(obj.key)
             access_log = client.get_object(bucket: @s3_bucket, key: obj.key).body.string
+
+            if obj.key.end_with?('.gz')
+              begin
+                inflated = Zlib::Inflate.inflate(access_log)
+                access_log = inflated
+              rescue Zlib::Error => e
+                @log.warn("#{e.message}: #{access_log.inspect.slice(0, 64)}")
+                next
+              end
+            end
+
             emit_access_log(access_log)
             last_timestamp = logfile_datetime
             @history.push(obj.key)
@@ -164,40 +209,71 @@ class Fluent::ElbAccessLogInput < Fluent::Input
       access_log = sampling(access_log)
     end
 
+    records = parse_log(access_log)
+
+    records.each do |record|
+      time = Time.parse(record['timestamp'])
+      router.emit(@tag, time.to_i, record)
+    end
+  end
+
+  def parse_log(access_log)
     parsed_access_log = []
 
     access_log.split("\n").each do |line|
-      line = parse_line(line)
+      case @elb_type
+      when 'clb'
+        line = parse_clb_line(line)
+      when 'alb'
+        line = parse_alb_line(line)
+      else
+        # It is a bug if an exception is thrown
+        raise 'must not happen'
+      end
+
       parsed_access_log << line if line
     end
 
+    records = []
+    access_log_fields = ACCESS_LOG_FIELDS.fetch(@elb_type)
+
     parsed_access_log.each do |row|
-      record = Hash[ACCESS_LOG_FIELDS.keys.zip(row)]
+      begin
+        record = Hash[access_log_fields.keys.zip(row)]
 
-      ACCESS_LOG_FIELDS.each do |name, conv|
-        record[name] = record[name].send(conv) if conv
-      end
+        access_log_fields.each do |name, conv|
+          record[name] = record[name].send(conv) if conv
+        end
 
-      split_address_port!(record, 'client')
-      split_address_port!(record, 'backend')
+        split_address_port!(record, 'client')
 
-      parse_request!(record)
+        case @elb_type
+        when 'clb'
+          split_address_port!(record, 'backend')
+        when 'alb'
+          split_address_port!(record, 'target')
+        else
+          # It is a bug if an exception is thrown
+          raise 'must not happen'
+        end
 
-      begin
-        time = Time.strptime(record['timestamp'].to_s, '%FT%T.%L %z')
-        router.emit(@tag, time.to_i, record)
+        parse_request!(record)
+
+        records << record
       rescue ArgumentError => e
         @log.warn("#{e.message}: #{row}")
         @log.warn('A record that has bad timestamp is not emitted.')
       end
     end
+
+    records
   end
 
-  def parse_line(line)
+  def parse_clb_line(line)
     parsed = nil
 
     begin
-      parsed = CSV.parse_line(line, :col_sep => ' ')
+      parsed = CSV.parse_line(line, col_sep: ' ')
     rescue => e
       begin
         parsed = line.split(' ', 12)
@@ -207,9 +283,9 @@ class Fluent::ElbAccessLogInput < Fluent::Input
         parsed[11].sub!(/\A"/, '')
         parsed[11].sub!(/"(.*)\z/, '')
 
-        user_agent, ssl_cipher, ssl_protocol = $1.strip.split(' ', 3)
-        user_agent.sub!(/\A"/, '').sub!(/"\z/, '') if user_agent
-        parsed[12] = user_agent
+        user_agent, ssl_cipher, ssl_protocol = rsplit($1.strip, ' ', 3)
+
+        parsed[12] = unquote(user_agent)
         parsed[13] = ssl_cipher
         parsed[14] = ssl_protocol
       rescue => e2
@@ -220,6 +296,37 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     parsed
   end
 
+  def parse_alb_line(line)
+    parsed = nil
+
+    begin
+      parsed = CSV.parse_line(line, col_sep: ' ')
+    rescue => e
+      begin
+        parsed = line.split(' ', 13)
+
+        # request
+        parsed[12] ||= ''
+        parsed[12].sub!(/\A"/, '')
+        parsed[12].sub!(/"(.*)\z/, '')
+
+        user_agent, ssl_cipher, ssl_protocol, target_group_arn, trace_id, domain_name, chosen_cert_arn = rsplit($1.strip, ' ', 7)
+
+        parsed[13] = unquote(user_agent)
+        parsed[14] = ssl_cipher
+        parsed[15] = ssl_protocol
+        parsed[16] = target_group_arn
+        parsed[17] = unquote(trace_id)
+        parsed[18] = unquote(domain_name)
+        parsed[19] = unquote(chosen_cert_arn)
+      rescue => e2
+        @log.warn("#{e.message}: #{line}")
+      end
+    end
+
+    parsed
+  end
+
   def sampling(access_log)
     access_log.split("\n").each_with_index.select {|row, i| (i % @sampling_interval).zero? }.map {|row, i| row }.join("\n")
   end
@@ -279,7 +386,7 @@ class Fluent::ElbAccessLogInput < Fluent::Input
   def client
     return @client if @client
 
-    options = {:user_agent_suffix => USER_AGENT_SUFFIX}
+    options = {user_agent_suffix: USER_AGENT_SUFFIX}
     options[:region] = @region if @region
     options[:http_proxy] = @http_proxy if @http_proxy
 
@@ -287,7 +394,7 @@ class Fluent::ElbAccessLogInput < Fluent::Input
       options[:access_key_id] = @aws_key_id
       options[:secret_access_key] = @aws_sec_key
     elsif @profile
-      credentials_opts = {:profile_name => @profile}
+      credentials_opts = {profile_name: @profile}
       credentials_opts[:path] = @credentials_path if @credentials_path
       credentials = Aws::SharedCredentials.new(credentials_opts)
       options[:credentials] = credentials
@@ -302,6 +409,26 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     @client = Aws::S3::Client.new(options)
   end
 
+  def rsplit(str, sep, n)
+    str = str.dup
+    substrs = []
+
+    (n - 1).times do
+      pos = str.rindex(sep)
+      next unless pos
+      substr = str.slice!(pos..-1).slice(sep.length..-1)
+      substrs << substr
+    end
+
+    substrs << str
+    substrs.reverse
+  end
+
+  def unquote(str)
+    return nil if (str || '').empty?
+    str.sub(/\A"/, '').sub(/"\z/, '')
+  end
+
   class TimerWatcher < Coolio::TimerWatcher
     def initialize(interval, repeat, log, &callback)
       @callback = callback

data/lib/fluent_plugin_elb_access_log/version.rb

@@ -1,3 +1,3 @@
 module FluentPluginElbAccessLog
-  VERSION = '0.3.7-beta1'
+  VERSION = '0.3.7'
 end