fluent-plugin-elb-access-log 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 08f77efb6afe4e6a4ff1e3c2d9fb956eb4cbb5a5
-  data.tar.gz: 5f4249955cf58fc6c77c7281a1911014e7572940
+  metadata.gz: 7b6fd483aeb5428ce74df156ed7efe4990cee287
+  data.tar.gz: f34fee8557781f1756558a9945b83d77b4f104e9
 SHA512:
-  metadata.gz: c5216d607dce35496ecc0a1ee43472d38932b70b8dacf4b2d1e0d7284e6eaac3297948edb87f4660fc75570e1650b84d0a49eeabecf37c94743fe3dfcda4faaf
-  data.tar.gz: 7936a97c6014943af80f42ecb952672012957d5d8a4de93327e0d2ebaa4d98821cb01b1ed82d0d12edc5f66996e699bd89f58bd9fc3c28054ed3dbf4eb99180b
+  metadata.gz: 1ac2032b68b88c5985910767a0bec383cc6bab13eb58c983fcac3b0e23d8f4529a15d464fc08ac9834f4c32a29b10cf991e789652f3c641974ebd3ec9f1ba248
+  data.tar.gz: 5606a780fb2c93df73c350baa05ac9f12cd7adb42720f1031c50cc325ef20e76844d1c74edb2ca318cb2a6bc6ae2fec90816f72494568d8877cd93168a0c2513

data/README.md

@@ -70,12 +70,10 @@ Or install it yourself as:
   "request.uri":"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
   "request.http_version":"HTTP/1.1",
   "request.uri.scheme":"http",
-  "request.uri.userinfo":null,
+  "request.uri.user":null,
   "request.uri.host":"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
   "request.uri.port":80,
-  "request.uri.registry":null,
   "request.uri.path":"/",
-  "request.uri.opaque":null,
   "request.uri.query":null,
   "request.uri.fragment":null
 }

data/fluent-plugin-elb-access-log.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'fluentd'
   spec.add_dependency 'aws-sdk', '~> 2'
+  spec.add_dependency 'addressable'
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec', '>= 3.0.0'

data/lib/fluent/plugin/in_elb_access_log.rb

@@ -54,7 +54,7 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     require 'fileutils'
     require 'logger'
     require 'time'
-    require 'uri'
+    require 'addressable/uri'
     require 'aws-sdk'
   end
 
@@ -193,9 +193,9 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     record['request.http_version'] = http_version
 
     begin
-      uri = URI.parse(uri)
+      uri = Addressable::URI.parse(uri)
 
-      [:scheme ,:userinfo, :host, :port, :registry, :path, :opaque, :query, :fragment].each do |key|
+      [:scheme ,:user, :host, :port, :path, :query, :fragment].each do |key|
         record["request.uri.#{key}"] = uri.send(key)
       end
     rescue => e

data/lib/fluent_plugin_elb_access_log/version.rb

@@ -1,3 +1,3 @@
 module FluentPluginElbAccessLog
-  VERSION = '0.2.0'
+  VERSION = '0.2.1'
 end

data/spec/in_elb_access_log_spec.rb

@@ -118,12 +118,10 @@ describe Fluent::ElbAccessLogInput do
           "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
          "request.http_version"=>"HTTP/1.1",
          "request.uri.scheme"=>"http",
-         "request.uri.userinfo"=>nil,
+         "request.uri.user"=>nil,
          "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
          "request.uri.port"=>80,
-         "request.uri.registry"=>nil,
          "request.uri.path"=>"/",
-         "request.uri.opaque"=>nil,
          "request.uri.query"=>nil,
          "request.uri.fragment"=>nil}],
        ["elb.access_log",
@@ -148,12 +146,10 @@ describe Fluent::ElbAccessLogInput do
           "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
          "request.http_version"=>"HTTP/1.1",
          "request.uri.scheme"=>"http",
-         "request.uri.userinfo"=>nil,
+         "request.uri.user"=>nil,
          "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
          "request.uri.port"=>80,
-         "request.uri.registry"=>nil,
          "request.uri.path"=>"/",
-         "request.uri.opaque"=>nil,
          "request.uri.query"=>nil,
          "request.uri.fragment"=>nil}],
        ["elb.access_log",
@@ -178,12 +174,10 @@ describe Fluent::ElbAccessLogInput do
           "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
          "request.http_version"=>"HTTP/1.1",
          "request.uri.scheme"=>"http",
-         "request.uri.userinfo"=>nil,
+         "request.uri.user"=>nil,
          "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
          "request.uri.port"=>80,
-         "request.uri.registry"=>nil,
          "request.uri.path"=>"/",
-         "request.uri.opaque"=>nil,
          "request.uri.query"=>nil,
          "request.uri.fragment"=>nil}],
        ["elb.access_log",
@@ -208,12 +202,10 @@ describe Fluent::ElbAccessLogInput do
           "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
          "request.http_version"=>"HTTP/1.1",
          "request.uri.scheme"=>"http",
-         "request.uri.userinfo"=>nil,
+         "request.uri.user"=>nil,
          "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
          "request.uri.port"=>80,
-         "request.uri.registry"=>nil,
          "request.uri.path"=>"/",
-         "request.uri.opaque"=>nil,
          "request.uri.query"=>nil,
          "request.uri.fragment"=>nil}]]
     end
@@ -237,131 +229,60 @@ describe Fluent::ElbAccessLogInput do
         is_expected.to eq expected_emits
       end
     end
+  end
+
+  context 'when include bad URI' do
+    let(:today_access_log) do
+      <<-EOS
+2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
+      EOS
+    end
+
+    before do
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
 
-    context 'when include bad URI' do
-      let(:today_access_log) do
-        <<-EOS
-  2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
-  2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://'hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
-        EOS
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+        [double('today_objects', contents: [double('today_object', key: today_object_key)])]
       end
 
-      let(:expected_emits) do
-        [["elb.access_log",
-          Time.parse('2015-05-24 19:55:36 UTC').to_i,
-          {"timestamp"=>"2015-05-24T19:55:36.000000Z",
-           "elb"=>"hoge",
-           "client"=>"14.14.124.20",
-           "client_port"=>57673,
-           "backend"=>"10.0.199.184",
-           "backend_port"=>80,
-           "request_processing_time"=>5.3e-05,
-           "backend_processing_time"=>0.000913,
-           "response_processing_time"=>3.6e-05,
-           "elb_status_code"=>200,
-           "backend_status_code"=>200,
-           "received_bytes"=>0,
-           "sent_bytes"=>3,
-           "request"=>
-            "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
-           "request.method"=>"GET",
-           "request.uri"=>
-            "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
-           "request.http_version"=>"HTTP/1.1",
-           "request.uri.scheme"=>"http",
-           "request.uri.userinfo"=>nil,
-           "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
-           "request.uri.port"=>80,
-           "request.uri.registry"=>nil,
-           "request.uri.path"=>"/",
-           "request.uri.opaque"=>nil,
-           "request.uri.query"=>nil,
-           "request.uri.fragment"=>nil}],
-         ["elb.access_log",
-          Time.parse('2015-05-24 19:55:36 UTC').to_i,
-          {"timestamp"=>"2015-05-24T19:55:36.000000Z",
-           "elb"=>"hoge",
-           "client"=>"14.14.124.20",
-           "client_port"=>57673,
-           "backend"=>"10.0.199.184",
-           "backend_port"=>80,
-           "request_processing_time"=>5.3e-05,
-           "backend_processing_time"=>0.000913,
-           "response_processing_time"=>3.6e-05,
-           "elb_status_code"=>200,
-           "backend_status_code"=>200,
-           "received_bytes"=>0,
-           "sent_bytes"=>3,
-           "request"=>
-            "GET http://'hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
-           "request.method"=>"GET",
-           "request.uri"=>
-            "http://'hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
-           "request.http_version"=>"HTTP/1.1"}],
-         ["elb.access_log",
-          Time.parse('2015-05-25 19:55:36 UTC').to_i,
-          {"timestamp"=>"2015-05-25T19:55:36.000000Z",
-           "elb"=>"hoge",
-           "client"=>"14.14.124.20",
-           "client_port"=>57673,
-           "backend"=>"10.0.199.184",
-           "backend_port"=>80,
-           "request_processing_time"=>5.3e-05,
-           "backend_processing_time"=>0.000913,
-           "response_processing_time"=>3.6e-05,
-           "elb_status_code"=>200,
-           "backend_status_code"=>200,
-           "received_bytes"=>0,
-           "sent_bytes"=>3,
-           "request"=>
-            "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
-           "request.method"=>"GET",
-           "request.uri"=>
-            "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
-           "request.http_version"=>"HTTP/1.1",
-           "request.uri.scheme"=>"http",
-           "request.uri.userinfo"=>nil,
-           "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
-           "request.uri.port"=>80,
-           "request.uri.registry"=>nil,
-           "request.uri.path"=>"/",
-           "request.uri.opaque"=>nil,
-           "request.uri.query"=>nil,
-           "request.uri.fragment"=>nil}],
-         ["elb.access_log",
-          Time.parse('2015-05-25 19:55:36 UTC').to_i,
-          {"timestamp"=>"2015-05-25T19:55:36.000000Z",
-           "elb"=>"hoge",
-           "client"=>"14.14.124.20",
-           "client_port"=>57673,
-           "backend"=>"10.0.199.184",
-           "backend_port"=>80,
-           "request_processing_time"=>5.3e-05,
-           "backend_processing_time"=>0.000913,
-           "response_processing_time"=>3.6e-05,
-           "elb_status_code"=>200,
-           "backend_status_code"=>200,
-           "received_bytes"=>0,
-           "sent_bytes"=>3,
-           "request"=>
-            "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
-           "request.method"=>"GET",
-           "request.uri"=>
-            "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
-           "request.http_version"=>"HTTP/1.1",
-           "request.uri.scheme"=>"http",
-           "request.uri.userinfo"=>nil,
-           "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
-           "request.uri.port"=>80,
-           "request.uri.registry"=>nil,
-           "request.uri.path"=>"/",
-           "request.uri.opaque"=>nil,
-           "request.uri.query"=>nil,
-           "request.uri.fragment"=>nil}]]
+      expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
+        [double('today_s3_object', body: StringIO.new(today_access_log))]
       end
 
-      it {  is_expected.to eq expected_emits }
+      expect(driver.instance).to receive(:save_timestamp).with(today)
+      expect(driver.instance).to receive(:save_history)
+
+      allow(Addressable::URI).to receive(:parse).and_raise('parse error')
+
+      driver.run
     end
+
+    let(:expected_emits) do
+      [["elb.access_log",
+        Time.parse('2015-05-24 19:55:36 UTC').to_i,
+        {"timestamp"=>"2015-05-24T19:55:36.000000Z",
+         "elb"=>"hoge",
+         "client"=>"14.14.124.20",
+         "client_port"=>57673,
+         "backend"=>"10.0.199.184",
+         "backend_port"=>80,
+         "request_processing_time"=>5.3e-05,
+         "backend_processing_time"=>0.000913,
+         "response_processing_time"=>3.6e-05,
+         "elb_status_code"=>200,
+         "backend_status_code"=>200,
+         "received_bytes"=>0,
+         "sent_bytes"=>3,
+         "request"=>
+          "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+         "request.method"=>"GET",
+         "request.uri"=>
+          "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+         "request.http_version"=>"HTTP/1.1"}]]
+    end
+
+    it { is_expected.to eq expected_emits }
   end
 
   context 'when access log exists (with tag option)' do
@@ -422,12 +343,10 @@ describe Fluent::ElbAccessLogInput do
           "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
          "request.http_version"=>"HTTP/1.1",
          "request.uri.scheme"=>"http",
-         "request.uri.userinfo"=>nil,
+         "request.uri.user"=>nil,
          "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
          "request.uri.port"=>80,
-         "request.uri.registry"=>nil,
          "request.uri.path"=>"/",
-         "request.uri.opaque"=>nil,
          "request.uri.query"=>nil,
          "request.uri.fragment"=>nil}]]
     end
@@ -485,12 +404,10 @@ describe Fluent::ElbAccessLogInput do
           "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
          "request.http_version"=>"HTTP/1.1",
          "request.uri.scheme"=>"http",
-         "request.uri.userinfo"=>nil,
+         "request.uri.user"=>nil,
          "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
          "request.uri.port"=>80,
-         "request.uri.registry"=>nil,
          "request.uri.path"=>"/",
-         "request.uri.opaque"=>nil,
          "request.uri.query"=>nil,
          "request.uri.fragment"=>nil}]]
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-access-log
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Genki Sugawara
@@ -38,6 +38,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement