fluent-plugin-elb-log 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8eab2080580611847117b9bc50bd3d5c46313d98b558f77d86faa1928eb1ea9f
-  data.tar.gz: d9e85e0d495c32e2310463adf50d4e660e03d7d6fd8e90eed3c2ed72ff7abb3e
+  metadata.gz: ef83fe5f1ca41f2f28d7d1d67fae59876fc915200009564db4115e11a97788ed
+  data.tar.gz: 8f94bf0325d3ab08838e90b721d676153f56c061be2d311a9f5f5ca1cd86848c
 SHA512:
-  metadata.gz: acff79c5a45017124e20ce1c4a9c19b957ae918f9ae7101a66370e0bf867402fc2302e9e79d0333c40ec684758d1439240961733a74ae00ca462c9b00e26e212
-  data.tar.gz: 401653c2a8a3a2d222da52faa01f9f993cc1f167a0ff62677ffd2301d130bea6e51b281faa7414ee1f4b5946fb26623d1dd862c077a5904582645ab3f9c36841
+  metadata.gz: 15ec8f47aa7811c44aa98ea8ad57c97280e3d4899018c555a930f363c3cd5248ff2f33fceabb6071d222d5e1b5799caa18a20bdec1ce24d486ec85cca27407b0
+  data.tar.gz: 436c4b85afbbaa887653091b263f224ce5bbfbf8e973a8bfa99b852cc7e0b01d3bea53032bba1b78be627aae1463df5f968873cc399051c88bf6fe79daca7543

data/README.md

@@ -86,7 +86,7 @@ SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
 ```
 
 ### json output example
-```
+```json
 {
     "account_id":"123456789012",
     "region":"ap-northeast-1",
@@ -118,9 +118,17 @@ SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt (If you using amazon linux)
     "ssl_cipher":"DHE-RSA-AES128-SHA",
     "ssl_protocol":"TLSv1.2",
     "type":"http",
-    "target_group_arn":"arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4e4ee9f2d",
-    "trace_id":"\"Root=1-xxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx\""
-    "option3":null
+    "target_group_arn": "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4e4ee9f2d",
+    "trace_id": "\"Root=1-xxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx\"",
+    "domain_name": "-",
+    "chosen_cert_arn": "-",
+    "matched_rule_priority": "0",
+    "request_creation_time": "2099-10-26T06:10:03.050000Z",
+    "actions_executed": "forward",
+    "redirect_url": "-",
+    "error_reason": "-",
+    "option1": "\"192.168.0.1:443\"",
+    "option2": "\"301\"",
+    "option3": null
 }
-```
-
+```

data/fluent-plugin-elb-log.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-elb-log"
-  spec.version       = "1.1.0"
+  spec.version       = "1.2.0"
   spec.authors       = ["shinsaka"]
   spec.email         = ["shinx1265@gmail.com"]
   spec.summary       = "Amazon ELB log input plugin"
@@ -17,11 +17,11 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "fluentd", "~> 1"
+  spec.add_dependency "fluentd", ">= 0.14.0", "< 2"
   spec.add_dependency "aws-sdk-s3", "~> 1"
   spec.add_dependency "aws-sdk-ec2", "~> 1"
 
-  spec.add_development_dependency "bundler", "~> 2"
+  spec.add_development_dependency "bundler", ">=1.17"
   spec.add_development_dependency "rake", "~> 12"
   spec.add_development_dependency "test-unit", "~> 3.2"
   spec.add_development_dependency "webmock", "~>3"

data/fluent.conf.sample

@@ -1,5 +1,5 @@
 <source>
-  type elb_log
+  @type elb_log
   region            us-east-1
   s3_bucketname     my-elblog-bucket
   s3_prefix         prodcution/web
@@ -12,6 +12,5 @@
 </source>
 
 <match elb.access>
-  type stdout
+  @type stdout
 </match>
-

data/lib/fluent/plugin/in_elb_log.rb

@@ -12,8 +12,7 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
   helpers :timer
 
   LOGFILE_REGEXP = /^((?<prefix>.+?)\/|)AWSLogs\/(?<account_id>[0-9]{12})\/elasticloadbalancing\/(?<region>.+?)\/(?<logfile_date>[0-9]{4}\/[0-9]{2}\/[0-9]{2})\/[0-9]{12}_elasticloadbalancing_.+?_(?<logfile_elb_name>[^_]+)_(?<elb_timestamp>[0-9]{8}T[0-9]{4}Z)_(?<elb_ip_address>.+?)_(?<logfile_hash>.+)\.log(.gz)?$/
-  ACCESSLOG_REGEXP = /^((?<type>[a-z0-9]+) )?(?<time>\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}\.\d{6}Z) (?<elb>.+?) (?<client>[^ ]+)\:(?<client_port>.+?) (?<backend>.+?)(\:(?<backend_port>.+?))? (?<request_processing_time>.+?) (?<backend_processing_time>.+?) (?<response_processing_time>.+?) (?<elb_status_code>.+?) (?<backend_status_code>.+?) (?<received_bytes>.+?) (?<sent_bytes>.+?) \"(?<request_method>.+?) (?<request_uri>.+?) (?<request_protocol>.+?)\"( \"(?<user_agent>.*?)\" (?<ssl_cipher>.+?) (?<ssl_protocol>[^\s]+)( (?<target_group_arn>arn:\S+) (?<trace_id>[^\s]+))?(| (?<option3>.*)))?/
-
+  ACCESSLOG_REGEXP = /^((?<type>[a-z0-9]+) )?(?<time>\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}\.\d{6}Z) (?<elb>.+?) (?<client>[^ ]+)\:(?<client_port>.+?) (?<backend>.+?)(\:(?<backend_port>.+?))? (?<request_processing_time>.+?) (?<backend_processing_time>.+?) (?<response_processing_time>.+?) (?<elb_status_code>.+?) (?<backend_status_code>.+?) (?<received_bytes>.+?) (?<sent_bytes>.+?) \"(?<request_method>.+?) (?<request_uri>.+?) (?<request_protocol>.+?)\"(\s+\"(?<user_agent>.+?)\" (?<ssl_cipher>.+?) (?<ssl_protocol>[^\s]+)( (?<target_group_arn>arn:\S+) (?<trace_id>[^\s]+))?( \"(?<domain_name>.+?)\" \"(?<chosen_cert_arn>.+?)\" (?<matched_rule_priority>.+?) (?<request_creation_time>.+?) \"(?<actions_executed>.+?)\" \"(?<redirect_url>.+?)\" \"(?<error_reason>[^\s]+)\"( |$))?((?<option1>[^\s]+)( |$))?((?<option2>[^\s]+)( |$))?( (?<option3>.*))?)?/
   config_param :access_key_id, :string, default: nil, secret: true
   config_param :secret_access_key, :string, default: nil, secret: true
   config_param :region, :string
@@ -319,6 +318,15 @@ class Fluent::Plugin::Elb_LogInput < Fluent::Plugin::Input
       "type" => item[:type],
       "target_group_arn" => item[:target_group_arn],
       "trace_id" => item[:trace_id],
+      "domain_name" => item[:domain_name],
+      "chosen_cert_arn" => item[:chosen_cert_arn],
+      "matched_rule_priority" => item[:matched_rule_priority],
+      "request_creation_time" => item[:request_creation_time],
+      "actions_executed" => item[:actions_executed],
+      "redirect_url" => item[:redirect_url],
+      "error_reason" => item[:error_reason],
+      "option1" => item[:option1],
+      "option2" => item[:option2],
       "option3" => item[:option3]
     }
   end

data/test/plugin/in_elb_log.rb

@@ -251,6 +251,45 @@ class Elb_LogInputTest < Test::Unit::TestCase
     assert_equal('-', m[:ssl_protocol])
     assert_equal('arn:aws:elasticloadbalancing:us-east-1:123456789123:targetgroup/example-service/1234abcd1234abcd', m[:target_group_arn])
     assert_equal('"Root=1-xxxxxxxx-yyyyyyyyyyyyyyyyyyyzzzzz"', m[:trace_id])
+    assert_equal('"-" "-" 3', m[:option3])
+  end
+
+  def test_alb_all_field
+    log = 'http 2019-10-26T06:10:03.157333Z app/my-alb/520e61ffffffffff 60.11.22.33:51306 192.168.30.111:443 0.010 0.097 0.001 301 301 414 507 "GET http://www.example.com:80/ HTTP/1.1" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36" ssl1 ssl2 arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4ffffffff "Root=1-123abcde-d03aafc8497211546b64c54c" "domainname" "certarn" 50000 2019-10-26T06:10:03.050000Z "forward" "redirect://url-something.com/" "error_reason" "192.168.30.186:443" "301"'
+    m = Fluent::Plugin::Elb_LogInput::ACCESSLOG_REGEXP.match(log)
+    assert_equal('http', m[:type])
+    assert_equal('2019-10-26T06:10:03.157333Z', m[:time])
+
+    assert_equal('app/my-alb/520e61ffffffffff', m[:elb])
+    assert_equal('60.11.22.33', m[:client])
+    assert_equal('51306', m[:client_port])
+    assert_equal('192.168.30.111', m[:backend])
+    assert_equal('443', m[:backend_port])
+
+    assert_equal('0.010', m[:request_processing_time])
+    assert_equal('0.097', m[:backend_processing_time])
+    assert_equal('0.001', m[:response_processing_time])
+    assert_equal('301', m[:elb_status_code])
+    assert_equal('301', m[:backend_status_code])
+    assert_equal('414', m[:received_bytes])
+    assert_equal('507', m[:sent_bytes])
+    assert_equal('GET', m[:request_method])
+    assert_equal('http://www.example.com:80/', m[:request_uri])
+    assert_equal('HTTP/1.1', m[:request_protocol])
+    assert_equal('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36', m[:user_agent])
+    assert_equal('ssl1', m[:ssl_cipher])
+    assert_equal('ssl2', m[:ssl_protocol])
+    assert_equal('arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/lbgrp1/605122a4ffffffff', m[:target_group_arn])
+    assert_equal('"Root=1-123abcde-d03aafc8497211546b64c54c"', m[:trace_id])
+    assert_equal('domainname', m[:domain_name])
+    assert_equal('certarn', m[:chosen_cert_arn])
+    assert_equal('50000', m[:matched_rule_priority])
+    assert_equal('2019-10-26T06:10:03.050000Z', m[:request_creation_time])
+    assert_equal('forward', m[:actions_executed])
+    assert_equal('redirect://url-something.com/', m[:redirect_url])
+    assert_equal('error_reason', m[:error_reason])
+    assert_equal('"192.168.30.186:443"', m[:option1])
+    assert_equal('"301"', m[:option2])
     assert_equal(nil, m[:option3])
   end
 end

metadata

@@ -1,29 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-log
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - shinsaka
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-26 00:00:00.000000000 Z
+date: 2019-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1'
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1'
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
@@ -56,16 +62,16 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1.17'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1.17'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -159,7 +165,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.0.pre2
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: Amazon ELB log input plugin