fluent-plugin-elb-access-log 0.3.4 → 0.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: e6e81bc4652c523c92b8c7ddcfc955f8ca97ebb5
4
- data.tar.gz: b99c326e7572840df8b104f06419a5a671fee9d8
3
+ metadata.gz: f85d937e44c08597e68f473469740f4d3f4cd1e4
4
+ data.tar.gz: 9aca5b3731371a67cbcfffd9af0e72bbbff48c4d
5
5
  SHA512:
6
- metadata.gz: d797ed6e97c7a8e1b09b0e30eba86f948a8bdd164055a168c16627048281baf1fa0f5ad2c1f67f2cd26adc120f592499adf758386684534781718fe913b46d7e
7
- data.tar.gz: 6bb0190f7fa03b5763ec544b75df50f1404f6f99f6964981ea37d0aa289e0b314fa68adc8d8801b89ac7d8d7cab68224439ae7c633639e2c8a680101c319c798
6
+ metadata.gz: 5e82edceb9328fab1dbf16dd938f4a960371f18fde1811b4694b276f9930f7ca7d7fde71a27533a46ab010c0b328a379c223d07717020f206848bbfd7ac8cfc8
7
+ data.tar.gz: 6f09b24ee4f1ccea68332197db41c380c0b0e1f4066559bfbbbc2279d23f4f7714efb155141129a143131aa02ce46f8f7d31dff6780fff4bbab38ea3bda2c433
@@ -166,7 +166,7 @@ class Fluent::ElbAccessLogInput < Fluent::Input
166
166
 
167
167
  parsed_access_log = []
168
168
 
169
- access_log.split("\n").each do |line|
169
+ normalize_line_feeds(access_log).split("\n").each do |line|
170
170
  line = parse_line(line)
171
171
  parsed_access_log << line if line
172
172
  end
@@ -215,6 +215,32 @@ class Fluent::ElbAccessLogInput < Fluent::Input
215
215
  parsed
216
216
  end
217
217
 
218
+ # This method is required because fields of user-agent are sometimes separated
219
+ # to several lines like flollowing example,
220
+ # 2017-06-07T22:47:14.827494Z baby 162.243.126.163:37036 10.6.49.1:80 0.000042 0.004133 0.00002 301 301 0 232 "GET http://example.com:80/ HTTP/1.0" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133
221
+ # Safari/537.36
222
+ #" - -
223
+ def normalize_line_feeds(str)
224
+ in_quotation = false
225
+ normalized_str = ''
226
+ previous_ch = nil
227
+
228
+ str.each_char do |current_ch|
229
+ if in_quotation && current_ch == "\n"
230
+ normalized_str << ' '
231
+ else
232
+ normalized_str << current_ch
233
+ end
234
+
235
+ if current_ch == '"' && previous_ch != '"'
236
+ in_quotation = !in_quotation
237
+ end
238
+ previous_ch = current_ch
239
+ end
240
+
241
+ normalized_str
242
+ end
243
+
218
244
  def sampling(access_log)
219
245
  access_log.split("\n").each_with_index.select {|row, i| (i % @sampling_interval).zero? }.map {|row, i| row }.join("\n")
220
246
  end
@@ -1,3 +1,3 @@
1
1
  module FluentPluginElbAccessLog
2
- VERSION = '0.3.4'
2
+ VERSION = '0.3.5'
3
3
  end
@@ -678,4 +678,68 @@ describe Fluent::ElbAccessLogInput do
678
678
 
679
679
  it { is_expected.to eq expected_emits }
680
680
  end
681
+
682
+ context 'when a record has a line feed in an user-agent quotation' do
683
+ let(:today_access_log) do
684
+ <<-'EOS'
685
+ 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "Dummy user agent
686
+ ""7.30.0""
687
+ - " ssl_cipher ssl_protocol
688
+ EOS
689
+ end
690
+
691
+ before do
692
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
693
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
694
+
695
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
696
+ [double('today_objects', contents: [double('today_object', key: today_object_key)])]
697
+ end
698
+
699
+ expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
700
+ double('today_s3_object', body: StringIO.new(today_access_log))
701
+ end
702
+
703
+ expect(driver.instance).to receive(:save_timestamp).with(today)
704
+ expect(driver.instance).to receive(:save_history)
705
+
706
+ driver.run
707
+ end
708
+
709
+ let(:expected_emits) do
710
+ [["elb.access_log",
711
+ Time.parse('2015-05-24 19:55:36 UTC').to_i,
712
+ {"timestamp"=>"2015-05-24T19:55:36.000000Z",
713
+ "elb"=>"hoge",
714
+ "client"=>"14.14.124.20",
715
+ "client_port"=>57673,
716
+ "backend"=>"10.0.199.184",
717
+ "backend_port"=>80,
718
+ "request_processing_time"=>5.3e-05,
719
+ "backend_processing_time"=>0.000913,
720
+ "response_processing_time"=>3.6e-05,
721
+ "elb_status_code"=>200,
722
+ "backend_status_code"=>200,
723
+ "received_bytes"=>0,
724
+ "sent_bytes"=>3,
725
+ "request"=>
726
+ "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
727
+ "user_agent"=>'Dummy user agent "7.30.0" - ',
728
+ "ssl_cipher"=>"ssl_cipher",
729
+ "ssl_protocol"=>"ssl_protocol",
730
+ "request.method"=>"GET",
731
+ "request.uri"=>
732
+ "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
733
+ "request.http_version"=>"HTTP/1.1",
734
+ "request.uri.scheme"=>"http",
735
+ "request.uri.user"=>nil,
736
+ "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
737
+ "request.uri.port"=>80,
738
+ "request.uri.path"=>"/",
739
+ "request.uri.query"=>nil,
740
+ "request.uri.fragment"=>nil}]]
741
+ end
742
+
743
+ it { is_expected.to eq expected_emits }
744
+ end
681
745
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elb-access-log
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.4
4
+ version: 0.3.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Genki Sugawara
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-05-18 00:00:00.000000000 Z
11
+ date: 2017-06-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd