fluent-plugin-elb-access-log 0.3.4 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e6e81bc4652c523c92b8c7ddcfc955f8ca97ebb5
-  data.tar.gz: b99c326e7572840df8b104f06419a5a671fee9d8
+  metadata.gz: f85d937e44c08597e68f473469740f4d3f4cd1e4
+  data.tar.gz: 9aca5b3731371a67cbcfffd9af0e72bbbff48c4d
 SHA512:
-  metadata.gz: d797ed6e97c7a8e1b09b0e30eba86f948a8bdd164055a168c16627048281baf1fa0f5ad2c1f67f2cd26adc120f592499adf758386684534781718fe913b46d7e
-  data.tar.gz: 6bb0190f7fa03b5763ec544b75df50f1404f6f99f6964981ea37d0aa289e0b314fa68adc8d8801b89ac7d8d7cab68224439ae7c633639e2c8a680101c319c798
+  metadata.gz: 5e82edceb9328fab1dbf16dd938f4a960371f18fde1811b4694b276f9930f7ca7d7fde71a27533a46ab010c0b328a379c223d07717020f206848bbfd7ac8cfc8
+  data.tar.gz: 6f09b24ee4f1ccea68332197db41c380c0b0e1f4066559bfbbbc2279d23f4f7714efb155141129a143131aa02ce46f8f7d31dff6780fff4bbab38ea3bda2c433

data/lib/fluent/plugin/in_elb_access_log.rb

@@ -166,7 +166,7 @@ class Fluent::ElbAccessLogInput < Fluent::Input
 
     parsed_access_log = []
 
-    access_log.split("\n").each do |line|
+    normalize_line_feeds(access_log).split("\n").each do |line|
       line = parse_line(line)
       parsed_access_log << line if line
     end
@@ -215,6 +215,32 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     parsed
   end
 
+  # This method is required because fields of user-agent are sometimes separated
+  # to several lines like flollowing example,
+  # 2017-06-07T22:47:14.827494Z baby 162.243.126.163:37036 10.6.49.1:80 0.000042 0.004133 0.00002 301 301 0 232 "GET http://example.com:80/ HTTP/1.0" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133
+  #          Safari/537.36
+  #" - -
+  def normalize_line_feeds(str)
+    in_quotation = false
+    normalized_str = ''
+    previous_ch = nil
+
+    str.each_char do |current_ch|
+      if in_quotation && current_ch == "\n"
+        normalized_str << ' '
+      else
+        normalized_str << current_ch
+      end
+
+      if current_ch == '"' && previous_ch != '"'
+        in_quotation = !in_quotation
+      end
+      previous_ch = current_ch
+    end
+
+    normalized_str
+  end
+
   def sampling(access_log)
     access_log.split("\n").each_with_index.select {|row, i| (i % @sampling_interval).zero? }.map {|row, i| row }.join("\n")
   end

data/lib/fluent_plugin_elb_access_log/version.rb

@@ -1,3 +1,3 @@
 module FluentPluginElbAccessLog
-  VERSION = '0.3.4'
+  VERSION = '0.3.5'
 end

data/spec/in_elb_access_log_spec.rb

@@ -678,4 +678,68 @@ describe Fluent::ElbAccessLogInput do
 
     it { is_expected.to eq expected_emits }
   end
+
+  context 'when a record has a line feed in an user-agent quotation' do
+    let(:today_access_log) do
+      <<-'EOS'
+2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "Dummy user agent
+   ""7.30.0""
+  - " ssl_cipher ssl_protocol
+      EOS
+    end
+
+    before do
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
+
+      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
+        [double('today_objects', contents: [double('today_object', key: today_object_key)])]
+      end
+
+      expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
+        double('today_s3_object', body: StringIO.new(today_access_log))
+      end
+
+      expect(driver.instance).to receive(:save_timestamp).with(today)
+      expect(driver.instance).to receive(:save_history)
+
+      driver.run
+    end
+
+    let(:expected_emits) do
+      [["elb.access_log",
+        Time.parse('2015-05-24 19:55:36 UTC').to_i,
+        {"timestamp"=>"2015-05-24T19:55:36.000000Z",
+         "elb"=>"hoge",
+         "client"=>"14.14.124.20",
+         "client_port"=>57673,
+         "backend"=>"10.0.199.184",
+         "backend_port"=>80,
+         "request_processing_time"=>5.3e-05,
+         "backend_processing_time"=>0.000913,
+         "response_processing_time"=>3.6e-05,
+         "elb_status_code"=>200,
+         "backend_status_code"=>200,
+         "received_bytes"=>0,
+         "sent_bytes"=>3,
+         "request"=>
+          "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
+         "user_agent"=>'Dummy user agent "7.30.0" - ',
+         "ssl_cipher"=>"ssl_cipher",
+         "ssl_protocol"=>"ssl_protocol",
+         "request.method"=>"GET",
+         "request.uri"=>
+          "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
+         "request.http_version"=>"HTTP/1.1",
+         "request.uri.scheme"=>"http",
+         "request.uri.user"=>nil,
+         "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
+         "request.uri.port"=>80,
+         "request.uri.path"=>"/",
+         "request.uri.query"=>nil,
+         "request.uri.fragment"=>nil}]]
+    end
+
+    it { is_expected.to eq expected_emits }
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-access-log
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.5
 platform: ruby
 authors:
 - Genki Sugawara
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-18 00:00:00.000000000 Z
+date: 2017-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd