fluent-plugin-elb-access-log 0.1.5 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 797ad8f847603d045b5eca038c0f5afed8730332
-  data.tar.gz: cf272aa08b78a320f4b5845d2be210ad7df91272
+  metadata.gz: 23af18097699523eb79eacdfff39c155e14a074e
+  data.tar.gz: 9132f72caaa0e09e2bbbad4fefef5bb6654642ee
 SHA512:
-  metadata.gz: 059252da8f5468d7e579f9edf8beab4cd14b20f55a99a8bce0e9aede7cbf30e5253ba66ac67b2872b145b678bc9762bb0cfdc42641ff1c4c7ee3ebb8545f7ca3
-  data.tar.gz: 73fa7e9b2e49b46bc6a45db1a932521fd137a42d6fe4c0c5e3fc88634ee265fcae99b247cddce55ec1a21bcf410e69ef35e9d8e1f1a2cbe5a87d70099efd370a
+  metadata.gz: f3273e599277bb2ee5367e8934a4c8cb32d6337145c5eee60114558d2d111f4fde60149b3ef2ed811144ab35cdf5fb6e20a2cdbe2f8a8e3b6b8ccd5d4ad49c6d
+  data.tar.gz: 4c44fdc553b5afddb444cdead9a2c6bf00304059a4eea0f9abe2d6550588679536122afe2ffe36e5e52c9e0640841a3d287815c43d2096d01fec33d9f82fc842

data/lib/fluent/plugin/in_elb_access_log.rb

@@ -88,7 +88,7 @@ class Fluent::ElbAccessLogInput < Fluent::Input
       end
 
       if @history.length > @history_length
-        @history.shift(history.length - @history_length)
+        @history.shift(@history.length - @history_length)
       end
     end
 
@@ -179,10 +179,14 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     record['request.uri'] = uri
     record['request.http_version'] = http_version
 
-    uri = URI.parse(uri)
+    begin
+      uri = URI.parse(uri)
 
-    [:scheme ,:userinfo, :host, :port, :registry, :path, :opaque, :query, :fragment].each do |key|
-      record["request.uri.#{key}"] = uri.send(key)
+      [:scheme ,:userinfo, :host, :port, :registry, :path, :opaque, :query, :fragment].each do |key|
+        record["request.uri.#{key}"] = uri.send(key)
+      end
+    rescue => e
+      @log.warn(e.message)
     end
   end
 

data/lib/fluent_plugin_elb_access_log/version.rb

@@ -1,3 +1,3 @@
 module FluentPluginElbAccessLog
-  VERSION = '0.1.5'
+  VERSION = '0.1.7'
 end

data/spec/in_elb_access_log_spec.rb

@@ -216,6 +216,131 @@ describe Fluent::ElbAccessLogInput do
     end
 
     it { is_expected.to eq expected_emits }
+
+    context 'when include bad URI' do
+      let(:today_access_log) do
+        <<-EOS
+  2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
+  2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://'hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
+        EOS
+      end
+
+      let(:expected_emits) do
+        [["elb.access_log",
+          Time.parse('2015-05-24 19:55:36 UTC').to_i,
+          {"timestamp"=>"2015-05-24T19:55:36.000000Z",
+           "elb"=>"hoge",
+           "client"=>"14.14.124.20",
+           "client_port"=>57673,
+           "backend"=>"10.0.199.184",
+           "backend_port"=>80,
+           "request_processing_time"=>5.3e-05,
+           "backend_processing_time"=>0.000913,
+           "response_processing_time"=>3.6e-05,
+           "elb_status_code"=>200,
+           "backend_status_code"=>200,
+           "received_bytes"=>0,
+           "sent_bytes"=>3,
+           "request"=>
+            "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+           "request.method"=>"GET",
+           "request.uri"=>
+            "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+           "request.http_version"=>"HTTP/1.1",
+           "request.uri.scheme"=>"http",
+           "request.uri.userinfo"=>nil,
+           "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
+           "request.uri.port"=>80,
+           "request.uri.registry"=>nil,
+           "request.uri.path"=>"/",
+           "request.uri.opaque"=>nil,
+           "request.uri.query"=>nil,
+           "request.uri.fragment"=>nil}],
+         ["elb.access_log",
+          Time.parse('2015-05-24 19:55:36 UTC').to_i,
+          {"timestamp"=>"2015-05-24T19:55:36.000000Z",
+           "elb"=>"hoge",
+           "client"=>"14.14.124.20",
+           "client_port"=>57673,
+           "backend"=>"10.0.199.184",
+           "backend_port"=>80,
+           "request_processing_time"=>5.3e-05,
+           "backend_processing_time"=>0.000913,
+           "response_processing_time"=>3.6e-05,
+           "elb_status_code"=>200,
+           "backend_status_code"=>200,
+           "received_bytes"=>0,
+           "sent_bytes"=>3,
+           "request"=>
+            "GET http://'hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+           "request.method"=>"GET",
+           "request.uri"=>
+            "http://'hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+           "request.http_version"=>"HTTP/1.1"}],
+         ["elb.access_log",
+          Time.parse('2015-05-25 19:55:36 UTC').to_i,
+          {"timestamp"=>"2015-05-25T19:55:36.000000Z",
+           "elb"=>"hoge",
+           "client"=>"14.14.124.20",
+           "client_port"=>57673,
+           "backend"=>"10.0.199.184",
+           "backend_port"=>80,
+           "request_processing_time"=>5.3e-05,
+           "backend_processing_time"=>0.000913,
+           "response_processing_time"=>3.6e-05,
+           "elb_status_code"=>200,
+           "backend_status_code"=>200,
+           "received_bytes"=>0,
+           "sent_bytes"=>3,
+           "request"=>
+            "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+           "request.method"=>"GET",
+           "request.uri"=>
+            "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+           "request.http_version"=>"HTTP/1.1",
+           "request.uri.scheme"=>"http",
+           "request.uri.userinfo"=>nil,
+           "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
+           "request.uri.port"=>80,
+           "request.uri.registry"=>nil,
+           "request.uri.path"=>"/",
+           "request.uri.opaque"=>nil,
+           "request.uri.query"=>nil,
+           "request.uri.fragment"=>nil}],
+         ["elb.access_log",
+          Time.parse('2015-05-25 19:55:36 UTC').to_i,
+          {"timestamp"=>"2015-05-25T19:55:36.000000Z",
+           "elb"=>"hoge",
+           "client"=>"14.14.124.20",
+           "client_port"=>57673,
+           "backend"=>"10.0.199.184",
+           "backend_port"=>80,
+           "request_processing_time"=>5.3e-05,
+           "backend_processing_time"=>0.000913,
+           "response_processing_time"=>3.6e-05,
+           "elb_status_code"=>200,
+           "backend_status_code"=>200,
+           "received_bytes"=>0,
+           "sent_bytes"=>3,
+           "request"=>
+            "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+           "request.method"=>"GET",
+           "request.uri"=>
+            "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+           "request.http_version"=>"HTTP/1.1",
+           "request.uri.scheme"=>"http",
+           "request.uri.userinfo"=>nil,
+           "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
+           "request.uri.port"=>80,
+           "request.uri.registry"=>nil,
+           "request.uri.path"=>"/",
+           "request.uri.opaque"=>nil,
+           "request.uri.query"=>nil,
+           "request.uri.fragment"=>nil}]]
+      end
+
+      it {  is_expected.to eq expected_emits }
+    end
   end
 
   context 'when access log exists (with tag option)' do
@@ -402,4 +527,48 @@ describe Fluent::ElbAccessLogInput do
 
     it { is_expected.to be_empty }
   end
+
+  describe 'history#length' do
+    let(:today_access_log) do
+      <<-EOS
+2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
+      EOS
+    end
+
+    let(:history) { driver.instance.instance_variable_get(:@history) }
+
+    before do
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+        [double('today_objects', contents: [double('today_object', key: today_object_key)])]
+      end
+
+      expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
+        [double('today_s3_object', body: StringIO.new(today_access_log))]
+      end
+
+      expect(driver.instance).to receive(:save_timestamp).with(today)
+    end
+
+    subject { history.length }
+
+    context 'when history.length <= 100' do
+      before do
+        driver.run
+      end
+
+      it { is_expected.to eq 1 }
+    end
+
+    context 'when history.length > 100' do
+      before do
+        history.concat (1..100).map(&:to_s)
+        driver.run
+      end
+
+      it { is_expected.to eq 100 }
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-access-log
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.7
 platform: ruby
 authors:
 - Genki Sugawara