fluent-plugin-elb-access-log 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 08f77efb6afe4e6a4ff1e3c2d9fb956eb4cbb5a5
4
- data.tar.gz: 5f4249955cf58fc6c77c7281a1911014e7572940
3
+ metadata.gz: 7b6fd483aeb5428ce74df156ed7efe4990cee287
4
+ data.tar.gz: f34fee8557781f1756558a9945b83d77b4f104e9
5
5
  SHA512:
6
- metadata.gz: c5216d607dce35496ecc0a1ee43472d38932b70b8dacf4b2d1e0d7284e6eaac3297948edb87f4660fc75570e1650b84d0a49eeabecf37c94743fe3dfcda4faaf
7
- data.tar.gz: 7936a97c6014943af80f42ecb952672012957d5d8a4de93327e0d2ebaa4d98821cb01b1ed82d0d12edc5f66996e699bd89f58bd9fc3c28054ed3dbf4eb99180b
6
+ metadata.gz: 1ac2032b68b88c5985910767a0bec383cc6bab13eb58c983fcac3b0e23d8f4529a15d464fc08ac9834f4c32a29b10cf991e789652f3c641974ebd3ec9f1ba248
7
+ data.tar.gz: 5606a780fb2c93df73c350baa05ac9f12cd7adb42720f1031c50cc325ef20e76844d1c74edb2ca318cb2a6bc6ae2fec90816f72494568d8877cd93168a0c2513
data/README.md CHANGED
@@ -70,12 +70,10 @@ Or install it yourself as:
70
70
  "request.uri":"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
71
71
  "request.http_version":"HTTP/1.1",
72
72
  "request.uri.scheme":"http",
73
- "request.uri.userinfo":null,
73
+ "request.uri.user":null,
74
74
  "request.uri.host":"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
75
75
  "request.uri.port":80,
76
- "request.uri.registry":null,
77
76
  "request.uri.path":"/",
78
- "request.uri.opaque":null,
79
77
  "request.uri.query":null,
80
78
  "request.uri.fragment":null
81
79
  }
@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
20
20
 
21
21
  spec.add_dependency 'fluentd'
22
22
  spec.add_dependency 'aws-sdk', '~> 2'
23
+ spec.add_dependency 'addressable'
23
24
  spec.add_development_dependency 'bundler'
24
25
  spec.add_development_dependency 'rake'
25
26
  spec.add_development_dependency 'rspec', '>= 3.0.0'
@@ -54,7 +54,7 @@ class Fluent::ElbAccessLogInput < Fluent::Input
54
54
  require 'fileutils'
55
55
  require 'logger'
56
56
  require 'time'
57
- require 'uri'
57
+ require 'addressable/uri'
58
58
  require 'aws-sdk'
59
59
  end
60
60
 
@@ -193,9 +193,9 @@ class Fluent::ElbAccessLogInput < Fluent::Input
193
193
  record['request.http_version'] = http_version
194
194
 
195
195
  begin
196
- uri = URI.parse(uri)
196
+ uri = Addressable::URI.parse(uri)
197
197
 
198
- [:scheme ,:userinfo, :host, :port, :registry, :path, :opaque, :query, :fragment].each do |key|
198
+ [:scheme ,:user, :host, :port, :path, :query, :fragment].each do |key|
199
199
  record["request.uri.#{key}"] = uri.send(key)
200
200
  end
201
201
  rescue => e
@@ -1,3 +1,3 @@
1
1
  module FluentPluginElbAccessLog
2
- VERSION = '0.2.0'
2
+ VERSION = '0.2.1'
3
3
  end
@@ -118,12 +118,10 @@ describe Fluent::ElbAccessLogInput do
118
118
  "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
119
119
  "request.http_version"=>"HTTP/1.1",
120
120
  "request.uri.scheme"=>"http",
121
- "request.uri.userinfo"=>nil,
121
+ "request.uri.user"=>nil,
122
122
  "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
123
123
  "request.uri.port"=>80,
124
- "request.uri.registry"=>nil,
125
124
  "request.uri.path"=>"/",
126
- "request.uri.opaque"=>nil,
127
125
  "request.uri.query"=>nil,
128
126
  "request.uri.fragment"=>nil}],
129
127
  ["elb.access_log",
@@ -148,12 +146,10 @@ describe Fluent::ElbAccessLogInput do
148
146
  "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
149
147
  "request.http_version"=>"HTTP/1.1",
150
148
  "request.uri.scheme"=>"http",
151
- "request.uri.userinfo"=>nil,
149
+ "request.uri.user"=>nil,
152
150
  "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
153
151
  "request.uri.port"=>80,
154
- "request.uri.registry"=>nil,
155
152
  "request.uri.path"=>"/",
156
- "request.uri.opaque"=>nil,
157
153
  "request.uri.query"=>nil,
158
154
  "request.uri.fragment"=>nil}],
159
155
  ["elb.access_log",
@@ -178,12 +174,10 @@ describe Fluent::ElbAccessLogInput do
178
174
  "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
179
175
  "request.http_version"=>"HTTP/1.1",
180
176
  "request.uri.scheme"=>"http",
181
- "request.uri.userinfo"=>nil,
177
+ "request.uri.user"=>nil,
182
178
  "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
183
179
  "request.uri.port"=>80,
184
- "request.uri.registry"=>nil,
185
180
  "request.uri.path"=>"/",
186
- "request.uri.opaque"=>nil,
187
181
  "request.uri.query"=>nil,
188
182
  "request.uri.fragment"=>nil}],
189
183
  ["elb.access_log",
@@ -208,12 +202,10 @@ describe Fluent::ElbAccessLogInput do
208
202
  "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
209
203
  "request.http_version"=>"HTTP/1.1",
210
204
  "request.uri.scheme"=>"http",
211
- "request.uri.userinfo"=>nil,
205
+ "request.uri.user"=>nil,
212
206
  "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
213
207
  "request.uri.port"=>80,
214
- "request.uri.registry"=>nil,
215
208
  "request.uri.path"=>"/",
216
- "request.uri.opaque"=>nil,
217
209
  "request.uri.query"=>nil,
218
210
  "request.uri.fragment"=>nil}]]
219
211
  end
@@ -237,131 +229,60 @@ describe Fluent::ElbAccessLogInput do
237
229
  is_expected.to eq expected_emits
238
230
  end
239
231
  end
232
+ end
233
+
234
+ context 'when include bad URI' do
235
+ let(:today_access_log) do
236
+ <<-EOS
237
+ 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
238
+ EOS
239
+ end
240
+
241
+ before do
242
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
243
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
240
244
 
241
- context 'when include bad URI' do
242
- let(:today_access_log) do
243
- <<-EOS
244
- 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
245
- 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://'hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
246
- EOS
245
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
246
+ [double('today_objects', contents: [double('today_object', key: today_object_key)])]
247
247
  end
248
248
 
249
- let(:expected_emits) do
250
- [["elb.access_log",
251
- Time.parse('2015-05-24 19:55:36 UTC').to_i,
252
- {"timestamp"=>"2015-05-24T19:55:36.000000Z",
253
- "elb"=>"hoge",
254
- "client"=>"14.14.124.20",
255
- "client_port"=>57673,
256
- "backend"=>"10.0.199.184",
257
- "backend_port"=>80,
258
- "request_processing_time"=>5.3e-05,
259
- "backend_processing_time"=>0.000913,
260
- "response_processing_time"=>3.6e-05,
261
- "elb_status_code"=>200,
262
- "backend_status_code"=>200,
263
- "received_bytes"=>0,
264
- "sent_bytes"=>3,
265
- "request"=>
266
- "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
267
- "request.method"=>"GET",
268
- "request.uri"=>
269
- "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
270
- "request.http_version"=>"HTTP/1.1",
271
- "request.uri.scheme"=>"http",
272
- "request.uri.userinfo"=>nil,
273
- "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
274
- "request.uri.port"=>80,
275
- "request.uri.registry"=>nil,
276
- "request.uri.path"=>"/",
277
- "request.uri.opaque"=>nil,
278
- "request.uri.query"=>nil,
279
- "request.uri.fragment"=>nil}],
280
- ["elb.access_log",
281
- Time.parse('2015-05-24 19:55:36 UTC').to_i,
282
- {"timestamp"=>"2015-05-24T19:55:36.000000Z",
283
- "elb"=>"hoge",
284
- "client"=>"14.14.124.20",
285
- "client_port"=>57673,
286
- "backend"=>"10.0.199.184",
287
- "backend_port"=>80,
288
- "request_processing_time"=>5.3e-05,
289
- "backend_processing_time"=>0.000913,
290
- "response_processing_time"=>3.6e-05,
291
- "elb_status_code"=>200,
292
- "backend_status_code"=>200,
293
- "received_bytes"=>0,
294
- "sent_bytes"=>3,
295
- "request"=>
296
- "GET http://'hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
297
- "request.method"=>"GET",
298
- "request.uri"=>
299
- "http://'hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
300
- "request.http_version"=>"HTTP/1.1"}],
301
- ["elb.access_log",
302
- Time.parse('2015-05-25 19:55:36 UTC').to_i,
303
- {"timestamp"=>"2015-05-25T19:55:36.000000Z",
304
- "elb"=>"hoge",
305
- "client"=>"14.14.124.20",
306
- "client_port"=>57673,
307
- "backend"=>"10.0.199.184",
308
- "backend_port"=>80,
309
- "request_processing_time"=>5.3e-05,
310
- "backend_processing_time"=>0.000913,
311
- "response_processing_time"=>3.6e-05,
312
- "elb_status_code"=>200,
313
- "backend_status_code"=>200,
314
- "received_bytes"=>0,
315
- "sent_bytes"=>3,
316
- "request"=>
317
- "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
318
- "request.method"=>"GET",
319
- "request.uri"=>
320
- "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
321
- "request.http_version"=>"HTTP/1.1",
322
- "request.uri.scheme"=>"http",
323
- "request.uri.userinfo"=>nil,
324
- "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
325
- "request.uri.port"=>80,
326
- "request.uri.registry"=>nil,
327
- "request.uri.path"=>"/",
328
- "request.uri.opaque"=>nil,
329
- "request.uri.query"=>nil,
330
- "request.uri.fragment"=>nil}],
331
- ["elb.access_log",
332
- Time.parse('2015-05-25 19:55:36 UTC').to_i,
333
- {"timestamp"=>"2015-05-25T19:55:36.000000Z",
334
- "elb"=>"hoge",
335
- "client"=>"14.14.124.20",
336
- "client_port"=>57673,
337
- "backend"=>"10.0.199.184",
338
- "backend_port"=>80,
339
- "request_processing_time"=>5.3e-05,
340
- "backend_processing_time"=>0.000913,
341
- "response_processing_time"=>3.6e-05,
342
- "elb_status_code"=>200,
343
- "backend_status_code"=>200,
344
- "received_bytes"=>0,
345
- "sent_bytes"=>3,
346
- "request"=>
347
- "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
348
- "request.method"=>"GET",
349
- "request.uri"=>
350
- "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
351
- "request.http_version"=>"HTTP/1.1",
352
- "request.uri.scheme"=>"http",
353
- "request.uri.userinfo"=>nil,
354
- "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
355
- "request.uri.port"=>80,
356
- "request.uri.registry"=>nil,
357
- "request.uri.path"=>"/",
358
- "request.uri.opaque"=>nil,
359
- "request.uri.query"=>nil,
360
- "request.uri.fragment"=>nil}]]
249
+ expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
250
+ [double('today_s3_object', body: StringIO.new(today_access_log))]
361
251
  end
362
252
 
363
- it { is_expected.to eq expected_emits }
253
+ expect(driver.instance).to receive(:save_timestamp).with(today)
254
+ expect(driver.instance).to receive(:save_history)
255
+
256
+ allow(Addressable::URI).to receive(:parse).and_raise('parse error')
257
+
258
+ driver.run
364
259
  end
260
+
261
+ let(:expected_emits) do
262
+ [["elb.access_log",
263
+ Time.parse('2015-05-24 19:55:36 UTC').to_i,
264
+ {"timestamp"=>"2015-05-24T19:55:36.000000Z",
265
+ "elb"=>"hoge",
266
+ "client"=>"14.14.124.20",
267
+ "client_port"=>57673,
268
+ "backend"=>"10.0.199.184",
269
+ "backend_port"=>80,
270
+ "request_processing_time"=>5.3e-05,
271
+ "backend_processing_time"=>0.000913,
272
+ "response_processing_time"=>3.6e-05,
273
+ "elb_status_code"=>200,
274
+ "backend_status_code"=>200,
275
+ "received_bytes"=>0,
276
+ "sent_bytes"=>3,
277
+ "request"=>
278
+ "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
279
+ "request.method"=>"GET",
280
+ "request.uri"=>
281
+ "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
282
+ "request.http_version"=>"HTTP/1.1"}]]
283
+ end
284
+
285
+ it { is_expected.to eq expected_emits }
365
286
  end
366
287
 
367
288
  context 'when access log exists (with tag option)' do
@@ -422,12 +343,10 @@ describe Fluent::ElbAccessLogInput do
422
343
  "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
423
344
  "request.http_version"=>"HTTP/1.1",
424
345
  "request.uri.scheme"=>"http",
425
- "request.uri.userinfo"=>nil,
346
+ "request.uri.user"=>nil,
426
347
  "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
427
348
  "request.uri.port"=>80,
428
- "request.uri.registry"=>nil,
429
349
  "request.uri.path"=>"/",
430
- "request.uri.opaque"=>nil,
431
350
  "request.uri.query"=>nil,
432
351
  "request.uri.fragment"=>nil}]]
433
352
  end
@@ -485,12 +404,10 @@ describe Fluent::ElbAccessLogInput do
485
404
  "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
486
405
  "request.http_version"=>"HTTP/1.1",
487
406
  "request.uri.scheme"=>"http",
488
- "request.uri.userinfo"=>nil,
407
+ "request.uri.user"=>nil,
489
408
  "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
490
409
  "request.uri.port"=>80,
491
- "request.uri.registry"=>nil,
492
410
  "request.uri.path"=>"/",
493
- "request.uri.opaque"=>nil,
494
411
  "request.uri.query"=>nil,
495
412
  "request.uri.fragment"=>nil}]]
496
413
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elb-access-log
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Genki Sugawara
@@ -38,6 +38,20 @@ dependencies:
38
38
  - - ~>
39
39
  - !ruby/object:Gem::Version
40
40
  version: '2'
41
+ - !ruby/object:Gem::Dependency
42
+ name: addressable
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - '>='
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - '>='
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
41
55
  - !ruby/object:Gem::Dependency
42
56
  name: bundler
43
57
  requirement: !ruby/object:Gem::Requirement