fluent-plugin-elb-access-log 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d199a2443e0155561cf75236566e4a2e6cb1e528
-  data.tar.gz: 97fd5bb09c4f6b5d653c15fcef661fd886e14b81
+  metadata.gz: 20da40ca2f682d7cd5e2da5680a59a3d1d19ffcb
+  data.tar.gz: 493f3437ffc8614d3a7d4b144a5dbd34c2400beb
 SHA512:
-  metadata.gz: 74953f1b6b2d0be08589596a255f6774bd9df447ebb9f52ea0c1902e78fb40769ec0cfc9a190abb0cb0a894d08aeccb5fcaa61170cc8ed8b768992878fead595
-  data.tar.gz: 7e5477cf26c6e07217b78b02b7ed198c9b0a072e9c4dcb058cebcd53191f7c8dbf8f722c3a9db149ba47953dbd7ddea1d88ac006f3643bbb71fe8c42685e3d1f
+  metadata.gz: dc79f4a01261b1e125f9e9b699328723673edeb075d69aef266db31384d94254e3d49ba4adda7aa3c2eb96697300abd952aa18397b70ba7cce3f49e8ef267c06
+  data.tar.gz: 09760b23d4c257d7f6ef6f2604c50369ed3c68eab9a7fb0e4f0e8c9852ec076fc38fcec0b8f06c45c250978a5d1f88e5b2f3e7b4bb0f81be34a7ee84e9903b17

data/lib/fluent/plugin/in_elb_access_log.rb

@@ -158,11 +158,8 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     parsed_access_log = []
 
     access_log.split("\n").each do |line|
-      begin
-        parsed_access_log << CSV.parse_line(line, :col_sep => ' ')
-      rescue => e
-        @log.warn("#{e.message}: #{line}")
-      end
+      line = parse_line(line)
+      parsed_access_log << line if line
     end
 
     parsed_access_log.each do |row|
@@ -182,6 +179,27 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     end
   end
 
+  def parse_line(line)
+    parsed = nil
+
+    begin
+      parsed = CSV.parse_line(line, :col_sep => ' ')
+    rescue => e
+      begin
+        parsed = line.split(' ', 12)
+
+        # request
+        parsed[11] ||= ''
+        parsed[11].gsub!(/\A"/, '')
+        parsed[11].gsub!(/".*\z/, '')
+      rescue => e2
+        @log.warn("#{e.message}: #{line}")
+      end
+    end
+
+    parsed
+  end
+
   def sampling(access_log)
     access_log.split("\n").each_with_index.select {|row, i| (i % @sampling_interval).zero? }.map {|row, i| row }.join("\n")
   end

data/lib/fluent_plugin_elb_access_log/version.rb

@@ -1,3 +1,3 @@
 module FluentPluginElbAccessLog
-  VERSION = '0.2.3'
+  VERSION = '0.2.4'
 end

data/spec/in_elb_access_log_spec.rb

@@ -415,6 +415,67 @@ describe Fluent::ElbAccessLogInput do
     it { is_expected.to eq expected_emits }
   end
 
+  context 'when parse error' do
+    let(:today_access_log) do
+      <<-EOS
+2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
+      EOS
+    end
+
+    before do
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+        [double('today_objects', contents: [double('today_object', key: today_object_key)])]
+      end
+
+      expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
+        [double('today_s3_object', body: StringIO.new(today_access_log))]
+      end
+
+      expect(driver.instance).to receive(:save_timestamp).with(today)
+      expect(driver.instance).to receive(:save_history)
+
+      expect(CSV).to receive(:parse_line).and_raise('parse error')
+
+      driver.run
+    end
+
+    let(:expected_emits) do
+      [["elb.access_log",
+        Time.parse('2015-05-24 19:55:36 UTC').to_i,
+        {"timestamp"=>"2015-05-24T19:55:36.000000Z",
+         "elb"=>"hoge",
+         "client"=>"14.14.124.20",
+         "client_port"=>57673,
+         "backend"=>"10.0.199.184",
+         "backend_port"=>80,
+         "request_processing_time"=>5.3e-05,
+         "backend_processing_time"=>0.000913,
+         "response_processing_time"=>3.6e-05,
+         "elb_status_code"=>200,
+         "backend_status_code"=>200,
+         "received_bytes"=>0,
+         "sent_bytes"=>3,
+         "request"=>
+          "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+         "request.method"=>"GET",
+         "request.uri"=>
+          "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+         "request.http_version"=>"HTTP/1.1",
+         "request.uri.scheme"=>"http",
+         "request.uri.user"=>nil,
+         "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
+         "request.uri.port"=>80,
+         "request.uri.path"=>"/",
+         "request.uri.query"=>nil,
+         "request.uri.fragment"=>nil}]]
+    end
+
+    it { is_expected.to eq expected_emits }
+  end
+
   context 'when access old log exists (timeout)' do
     let(:today_access_log) do
       <<-EOS

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-access-log
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Genki Sugawara
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-26 00:00:00.000000000 Z
+date: 2015-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd