fluent-plugin-elb-access-log 0.3.5 → 0.3.6.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f85d937e44c08597e68f473469740f4d3f4cd1e4
-  data.tar.gz: 9aca5b3731371a67cbcfffd9af0e72bbbff48c4d
+  metadata.gz: b6eaa3c13f77a8734ed7221b6d9a27dfcbc2d8c3
+  data.tar.gz: 6e4f32e43b6ff3a9299a8172fb5645466c5b8322
 SHA512:
-  metadata.gz: 5e82edceb9328fab1dbf16dd938f4a960371f18fde1811b4694b276f9930f7ca7d7fde71a27533a46ab010c0b328a379c223d07717020f206848bbfd7ac8cfc8
-  data.tar.gz: 6f09b24ee4f1ccea68332197db41c380c0b0e1f4066559bfbbbc2279d23f4f7714efb155141129a143131aa02ce46f8f7d31dff6780fff4bbab38ea3bda2c433
+  metadata.gz: 5bf817d46c52c8d70016c14add800602af362b71ea2de5c3420b66641766bdb5138d08090abf4d5d0b8f4a1036f049fcaec739dfa8a578b84400bb85a5df0ce6
+  data.tar.gz: 371117dac3d925ec4ed8d214311cfc8c3b24daac639a46a71bec779faa1d2b78ecd8662acb7f6de5b37eb2bfde179e3012d48291e402220b5d703448f99e9681

data/lib/fluent/plugin/in_elb_access_log.rb

@@ -166,7 +166,7 @@ class Fluent::ElbAccessLogInput < Fluent::Input
 
     parsed_access_log = []
 
-    normalize_line_feeds(access_log).split("\n").each do |line|
+    access_log.split("\n").each do |line|
       line = parse_line(line)
       parsed_access_log << line if line
     end
@@ -183,8 +183,13 @@ class Fluent::ElbAccessLogInput < Fluent::Input
 
       parse_request!(record)
 
-      time = Time.parse(record['timestamp']) rescue Time.now
-      router.emit(@tag, time.to_i, record)
+      begin
+        time = Time.parse(record['timestamp'])
+        router.emit(@tag, time.to_i, record)
+      rescue ArgumentError => e
+        @log.warn("#{e.message}: #{line}")
+        @log.warn('A record that has bad timestamp is not emitted.')
+      end
     end
   end
 
@@ -215,32 +220,6 @@ class Fluent::ElbAccessLogInput < Fluent::Input
     parsed
   end
 
-  # This method is required because fields of user-agent are sometimes separated
-  # to several lines like flollowing example,
-  # 2017-06-07T22:47:14.827494Z baby 162.243.126.163:37036 10.6.49.1:80 0.000042 0.004133 0.00002 301 301 0 232 "GET http://example.com:80/ HTTP/1.0" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133
-  #          Safari/537.36
-  #" - -
-  def normalize_line_feeds(str)
-    in_quotation = false
-    normalized_str = ''
-    previous_ch = nil
-
-    str.each_char do |current_ch|
-      if in_quotation && current_ch == "\n"
-        normalized_str << ' '
-      else
-        normalized_str << current_ch
-      end
-
-      if current_ch == '"' && previous_ch != '"'
-        in_quotation = !in_quotation
-      end
-      previous_ch = current_ch
-    end
-
-    normalized_str
-  end
-
   def sampling(access_log)
     access_log.split("\n").each_with_index.select {|row, i| (i % @sampling_interval).zero? }.map {|row, i| row }.join("\n")
   end

data/lib/fluent_plugin_elb_access_log/version.rb

@@ -1,3 +1,3 @@
 module FluentPluginElbAccessLog
-  VERSION = '0.3.5'
+  VERSION = '0.3.6.beta1'
 end

data/spec/in_elb_access_log_spec.rb

@@ -678,68 +678,4 @@ describe Fluent::ElbAccessLogInput do
 
     it { is_expected.to eq expected_emits }
   end
-
-  context 'when a record has a line feed in an user-agent quotation' do
-    let(:today_access_log) do
-      <<-'EOS'
-2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "Dummy user agent
-   ""7.30.0""
-  - " ssl_cipher ssl_protocol
-      EOS
-    end
-
-    before do
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
-
-      expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
-        [double('today_objects', contents: [double('today_object', key: today_object_key)])]
-      end
-
-      expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
-        double('today_s3_object', body: StringIO.new(today_access_log))
-      end
-
-      expect(driver.instance).to receive(:save_timestamp).with(today)
-      expect(driver.instance).to receive(:save_history)
-
-      driver.run
-    end
-
-    let(:expected_emits) do
-      [["elb.access_log",
-        Time.parse('2015-05-24 19:55:36 UTC').to_i,
-        {"timestamp"=>"2015-05-24T19:55:36.000000Z",
-         "elb"=>"hoge",
-         "client"=>"14.14.124.20",
-         "client_port"=>57673,
-         "backend"=>"10.0.199.184",
-         "backend_port"=>80,
-         "request_processing_time"=>5.3e-05,
-         "backend_processing_time"=>0.000913,
-         "response_processing_time"=>3.6e-05,
-         "elb_status_code"=>200,
-         "backend_status_code"=>200,
-         "received_bytes"=>0,
-         "sent_bytes"=>3,
-         "request"=>
-          "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
-         "user_agent"=>'Dummy user agent "7.30.0" - ',
-         "ssl_cipher"=>"ssl_cipher",
-         "ssl_protocol"=>"ssl_protocol",
-         "request.method"=>"GET",
-         "request.uri"=>
-          "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
-         "request.http_version"=>"HTTP/1.1",
-         "request.uri.scheme"=>"http",
-         "request.uri.user"=>nil,
-         "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
-         "request.uri.port"=>80,
-         "request.uri.path"=>"/",
-         "request.uri.query"=>nil,
-         "request.uri.fragment"=>nil}]]
-    end
-
-    it { is_expected.to eq expected_emits }
-  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elb-access-log
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.6.beta1
 platform: ruby
 authors:
 - Genki Sugawara
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-12 00:00:00.000000000 Z
+date: 2017-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -157,9 +157,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.11