fluent-plugin-elb-log 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9e2cdd52d7bb6925077930b4af585a251b7adefc
-  data.tar.gz: 64429389036f8c97f4a502dad80cfebce71aaa83
+  metadata.gz: 6d34dc14c5627860aa7787a9c0e0dbac19edaf9d
+  data.tar.gz: 57c8a63726d0ad90f47653b6f3d20286d1527129
 SHA512:
-  metadata.gz: 05735444de0f2eceaaa76a417cd146971bc90ac614626dbaab1d4881b85651c06ab24aabad039fa9c90047f96e53e9dc406ba12c47743522d8f6947e34dce660
-  data.tar.gz: 8592521b71b5bb4df3311ea87f833627e129e7582ccb5cb8360a5bab67140e4efc284a35d273b767ec088db432aebea1505ba2b76efa2e1871027e989181e2a9
+  metadata.gz: 6854587f02206b4cdcd14863516d4df01b79eda6af2765d4250c3139e21ea1bb20896b7dba86cd29287aafa5b00e7983d02456ee50bfd29da6519d6c57622012
+  data.tar.gz: cfb29cc94c144bf8a877a3cd287510be473c31f23fa2f6879b0421ccba0e8a992559057e27f5f2c0edc12bd6877544872c878b570511bc1eb05b58d0e173b612

data/README.md

@@ -5,8 +5,6 @@
 
 ## Requirements
 
-## Requirements
-
 | fluent-plugin-elb-log | fluentd    | ruby   |
 |-----------------------|------------|--------|
 | >= 0.3.0              | >= v0.14.0 | >= 2.1 |
@@ -36,6 +34,12 @@
      - user_agent
      - option1, option2, option3
 
+## Support Application Load Balancer (ver 0.4.0 or later)
+- Support Access Logs for Application Load Balancer
+ - https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html
+- Existing ELB is called Classic Load Balancer
+ - http://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html
+
 ## When SSL certification error
 log:
 ```
@@ -50,7 +54,7 @@ SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
 
 ```config
 <source>
-  type elb_log
+  @type elb_log
 
   # following attibutes are required
   region            <region name>
@@ -70,7 +74,7 @@ SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
 ### Example setting
 ```config
 <source>
-  type elb_log
+  @type elb_log
   region            us-east-1
   s3_bucketname     my-elblog-bucket
   s3_prefix         prodcution/web
@@ -83,7 +87,7 @@ SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
 </source>
 
 <match **>
-  type stdout
+  @type stdout
 </match>
 ```
 
@@ -119,6 +123,9 @@ SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
     "user_agent":"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)",
     "ssl_cipher":"DHE-RSA-AES128-SHA",
     "ssl_protocol":"TLSv1.2",
+    "type":"http",
+    "target_group_arn":"arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4e4ee9f2d",
+    "trace_id":"\"Root=1-xxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx\""
     "option3":null
 }
 ```

data/fluent-plugin-elb-log.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-elb-log"
-  spec.version       = "0.3.0"
+  spec.version       = "0.4.0"
   spec.authors       = ["shinsaka"]
   spec.email         = ["shinx1265@gmail.com"]
   spec.summary       = "Amazon ELB log input plugin"

data/lib/fluent/plugin/in_elb_log.rb

@@ -1,4 +1,6 @@
 require 'time'
+require 'zlib'
+require 'fileutils'
 require 'aws-sdk'
 require 'fluent/input'
 
@@ -7,8 +9,8 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
 
   helpers :timer
 
-  LOGFILE_REGEXP = /^((?<prefix>.+?)\/|)AWSLogs\/(?<account_id>[0-9]{12})\/elasticloadbalancing\/(?<region>.+?)\/(?<logfile_date>[0-9]{4}\/[0-9]{2}\/[0-9]{2})\/[0-9]{12}_elasticloadbalancing_.+?_(?<logfile_elb_name>[^_]+)_(?<elb_timestamp>[0-9]{8}T[0-9]{4}Z)_(?<elb_ip_address>.+?)_(?<logfile_hash>.+)\.log$/
-  ACCESSLOG_REGEXP = /^(?<time>\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}\.\d{6}Z) (?<elb>.+?) (?<client>[^ ]+)\:(?<client_port>.+?) (?<backend>.+?)(\:(?<backend_port>.+?))? (?<request_processing_time>.+?) (?<backend_processing_time>.+?) (?<response_processing_time>.+?) (?<elb_status_code>.+?) (?<backend_status_code>.+?) (?<received_bytes>.+?) (?<sent_bytes>.+?) \"(?<request_method>.+?) (?<request_uri>.+?) (?<request_protocol>.+?)\"( \"(?<user_agent>.*?)\" (?<ssl_cipher>.+?) (?<ssl_protocol>.+)(| (?<option3>.*)))?/
+  LOGFILE_REGEXP = /^((?<prefix>.+?)\/|)AWSLogs\/(?<account_id>[0-9]{12})\/elasticloadbalancing\/(?<region>.+?)\/(?<logfile_date>[0-9]{4}\/[0-9]{2}\/[0-9]{2})\/[0-9]{12}_elasticloadbalancing_.+?_(?<logfile_elb_name>[^_]+)_(?<elb_timestamp>[0-9]{8}T[0-9]{4}Z)_(?<elb_ip_address>.+?)_(?<logfile_hash>.+)\.log(.gz)?$/
+  ACCESSLOG_REGEXP = /^((?<type>[a-z0-9]+) )?(?<time>\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}\.\d{6}Z) (?<elb>.+?) (?<client>[^ ]+)\:(?<client_port>.+?) (?<backend>.+?)(\:(?<backend_port>.+?))? (?<request_processing_time>.+?) (?<backend_processing_time>.+?) (?<response_processing_time>.+?) (?<elb_status_code>.+?) (?<backend_status_code>.+?) (?<received_bytes>.+?) (?<sent_bytes>.+?) \"(?<request_method>.+?) (?<request_uri>.+?) (?<request_protocol>.+?)\"( \"(?<user_agent>.*?)\" (?<ssl_cipher>.+?) (?<ssl_protocol>[^\s]+)( (?<target_group_arn>arn:.+) (?<trace_id>[^\s]+))?(| (?<option3>.*)))?/
 
   config_param :access_key_id, :string, default: nil, secret: true
   config_param :secret_access_key, :string, default: nil, secret: true
@@ -146,38 +148,48 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
   end
 
   def get_object_keys(timestamp)
-    # get values from object file name
     begin
       object_keys = []
+      get_object_keys_from_s3.each do |object_key|
+        matches = LOGFILE_REGEXP.match(object_key)
+        next unless matches
+
+        # snip old items
+        elb_timestamp_unixtime = Time.parse(matches[:elb_timestamp]).to_i
+        next if elb_timestamp_unixtime <= timestamp
+
+        log.debug object_key
+        object_keys << {
+          key: object_key,
+          prefix: matches[:prefix],
+          account_id: matches[:account_id],
+          region: matches[:region],
+          logfile_date: matches[:logfile_date],
+          logfile_elb_name: matches[:logfile_elb_name],
+          elb_timestamp: matches[:elb_timestamp],
+          elb_ip_address: matches[:elb_ip_address],
+          logfile_hash: matches[:logfile_hash],
+          elb_timestamp_unixtime: elb_timestamp_unixtime,
+        }
+      end
+      return object_keys
+    rescue => e
+      log.warn "error occurred: #{e.message}"
+    end
+  end
 
+  def get_object_keys_from_s3
+    begin
       objects = s3_client.list_objects(
         bucket: @s3_bucketname,
         max_keys: 100,
         prefix: @s3_prefix,
       )
-  
+
+      object_keys = []
       objects.each do |object|
         object.contents.each do |content|
-          matches = LOGFILE_REGEXP.match(content.key)
-          next unless matches
-
-          # snip old items
-          elb_timestamp_unixtime = Time.parse(matches[:elb_timestamp]).to_i
-          next if elb_timestamp_unixtime <= timestamp
-
-          log.debug content.key
-          object_keys << {
-            key: content.key,
-            prefix: matches[:prefix],
-            account_id: matches[:account_id],
-            region: matches[:region],
-            logfile_date: matches[:logfile_date],
-            logfile_elb_name: matches[:logfile_elb_name],
-            elb_timestamp: matches[:elb_timestamp],
-            elb_ip_address: matches[:elb_ip_address],
-            logfile_hash: matches[:logfile_hash],
-            elb_timestamp_unixtime: elb_timestamp_unixtime,
-          }
+          object_keys << content.key
         end
       end
       return object_keys
@@ -200,17 +212,24 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
     begin
       log.debug "getting object from s3 name is #{object_name}"
 
-      # read an object from S3 to a file and write buffer file
-      File.open(@buf_file, File::WRONLY|File::CREAT|File::TRUNC) do |file|
-        s3_client.get_object(
-          bucket: @s3_bucketname,
-          key: object_name
-        ) do |chunk|
-          file.write(chunk)
+      Tempfile.create('fluent-elblog') do |tfile|
+        s3_client.get_object(bucket: @s3_bucketname, key: object_name) do |chunk|
+          tfile.write(chunk)
+        end
+        tfile.close
+
+        if File.extname(object_name) != '.gz'
+          FileUtils.cp(tfile.path, @buf_file)
+        else
+          File.open(@buf_file, File::WRONLY|File::CREAT|File::TRUNC) do |bfile|
+            Zlib::GzipReader.open(tfile.path) do |gz|
+              bfile.write gz.read
+            end
+          end
         end
       end
     rescue => e
-      log.warn "error occurred: #{e.message}"
+      log.warn "error occurred: #{e.message}, #{e.backtrace}"
     end
   end
 
@@ -224,7 +243,7 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
             log.info "nomatch log found: #{line} in #{record_common['key']}"
             next
           end
-  
+
           record = {
             "time" => line_match[:time].gsub(/Z/, "+0000"),
             "elb" => line_match[:elb],
@@ -245,6 +264,9 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
             "user_agent" => line_match[:user_agent],
             "ssl_cipher" => line_match[:ssl_cipher],
             "ssl_protocol" => line_match[:ssl_protocol],
+            "type" => line_match[:type],
+            "target_group_arn" => line_match[:target_group_arn],
+            "trace_id" => line_match[:trace_id],
             "option3" => line_match[:option3],
           }
 

data/test/plugin/in_elb_log.rb

@@ -113,4 +113,89 @@ class Elb_LogInputTest < Test::Unit::TestCase
     assert_equal('secret_access_key is required', exception.message)
   end
 
+  def test_logfilename_classic_lb_parse
+    logfile_classic = 'classic/AWSLogs/123456789012/elasticloadbalancing/ap-northeast-1/2017/05/03/123456789012_elasticloadbalancing_ap-northeast-1_elbname_20170503T1250Z_10.0.0.1_43nzjpdj.log'
+
+    m = Fluent::Plugin::Elb_LogInput::LOGFILE_REGEXP.match(logfile_classic)
+    assert_equal('classic', m[:prefix])
+    assert_equal('123456789012', m[:account_id])
+    assert_equal('ap-northeast-1', m[:region])
+    assert_equal('2017/05/03', m[:logfile_date])
+    assert_equal('elbname', m[:logfile_elb_name])
+    assert_equal('20170503T1250Z', m[:elb_timestamp])
+    assert_equal('10.0.0.1', m[:elb_ip_address])
+    assert_equal('43nzjpdj', m[:logfile_hash])
+  end
+
+  def test_logfilename_appication_lb_parse
+    logfile_applb = 'applb/AWSLogs/123456789012/elasticloadbalancing/ap-northeast-1/2017/05/03/123456789012_elasticloadbalancing_ap-northeast-1_app.elbname.59bfa19e900030c2_20170503T1310Z_10.0.0.1_2tko12gv.log.gz'
+
+    m = Fluent::Plugin::Elb_LogInput::LOGFILE_REGEXP.match(logfile_applb)
+    assert_equal('applb', m[:prefix])
+    assert_equal('123456789012', m[:account_id])
+    assert_equal('ap-northeast-1', m[:region])
+    assert_equal('2017/05/03', m[:logfile_date])
+    assert_equal('app.elbname.59bfa19e900030c2', m[:logfile_elb_name])
+    assert_equal('20170503T1310Z', m[:elb_timestamp])
+    assert_equal('10.0.0.1', m[:elb_ip_address])
+    assert_equal('2tko12gv', m[:logfile_hash])
+  end
+
+  def test_log_classic_lb_parse
+    log = '2017-05-05T12:53:50.128456Z elbname 10.11.12.13:37852 192.168.30.186:443 0.00004 0.085372 0.000039 301 301 0 0 "GET https://elbname-123456789.ap-northeast-1.elb.amazonaws.com:443/ HTTP/1.1" "curl/7.51.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2'
+
+    m = Fluent::Plugin::Elb_LogInput::ACCESSLOG_REGEXP.match(log)
+    assert_equal('2017-05-05T12:53:50.128456Z', m[:time])
+    assert_equal('elbname', m[:elb])
+    assert_equal('10.11.12.13', m[:client])
+    assert_equal('37852', m[:client_port])
+    assert_equal('192.168.30.186', m[:backend])
+    assert_equal('443', m[:backend_port])
+    assert_equal('0.00004', m[:request_processing_time])
+    assert_equal('0.085372', m[:backend_processing_time])
+    assert_equal('0.000039', m[:response_processing_time])
+    assert_equal('301', m[:elb_status_code])
+    assert_equal('301', m[:backend_status_code])
+    assert_equal('0', m[:received_bytes])
+    assert_equal('0', m[:sent_bytes])
+    assert_equal('GET', m[:request_method])
+    assert_equal('https://elbname-123456789.ap-northeast-1.elb.amazonaws.com:443/', m[:request_uri])
+    assert_equal('HTTP/1.1', m[:request_protocol])
+    assert_equal('curl/7.51.0', m[:user_agent])
+    assert_equal('ECDHE-RSA-AES128-GCM-SHA256', m[:ssl_cipher])
+    assert_equal('TLSv1.2', m[:ssl_protocol])
+    assert_equal(nil, m[:type])
+    assert_equal(nil, m[:target_group_arn])
+    assert_equal(nil, m[:trace_id])
+    assert_equal(nil, m[:option3])
+  end
+
+  def test_log_application_lb_parse
+    log = 'https 2017-05-05T13:07:53.468529Z app/elbname/59bfa19e900030c2 10.20.30.40:52730 192.168.30.186:443 0.006 0.000 0.086 301 301 117 507 "GET https://elbname-1121128512.ap-northeast-1.elb.amazonaws.com:443/ HTTP/1.1" "curl/7.51.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4e4ee9f2d "Root=1-590c7929-4eb4cb393d46a01d22db8473"'
+
+    m = Fluent::Plugin::Elb_LogInput::ACCESSLOG_REGEXP.match(log)
+    assert_equal('2017-05-05T13:07:53.468529Z', m[:time])
+    assert_equal('app/elbname/59bfa19e900030c2', m[:elb])
+    assert_equal('10.20.30.40', m[:client])
+    assert_equal('52730', m[:client_port])
+    assert_equal('192.168.30.186', m[:backend])
+    assert_equal('443', m[:backend_port])
+    assert_equal('0.006', m[:request_processing_time])
+    assert_equal('0.000', m[:backend_processing_time])
+    assert_equal('0.086', m[:response_processing_time])
+    assert_equal('301', m[:elb_status_code])
+    assert_equal('301', m[:backend_status_code])
+    assert_equal('117', m[:received_bytes])
+    assert_equal('507', m[:sent_bytes])
+    assert_equal('GET', m[:request_method])
+    assert_equal('https://elbname-1121128512.ap-northeast-1.elb.amazonaws.com:443/', m[:request_uri])
+    assert_equal('HTTP/1.1', m[:request_protocol])
+    assert_equal('curl/7.51.0', m[:user_agent])
+    assert_equal('ECDHE-RSA-AES128-GCM-SHA256', m[:ssl_cipher])
+    assert_equal('TLSv1.2', m[:ssl_protocol])
+    assert_equal('https', m[:type])
+    assert_equal('arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4e4ee9f2d', m[:target_group_arn])
+    assert_equal('"Root=1-590c7929-4eb4cb393d46a01d22db8473"', m[:trace_id])
+    assert_equal(nil, m[:option3])
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-log
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - shinsaka
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-29 00:00:00.000000000 Z
+date: 2017-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -138,7 +138,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Amazon ELB log input plugin