fluent-plugin-elb-access-log 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -3
- data/fluent-plugin-elb-access-log.gemspec +1 -0
- data/lib/fluent/plugin/in_elb_access_log.rb +3 -3
- data/lib/fluent_plugin_elb_access_log/version.rb +1 -1
- data/spec/in_elb_access_log_spec.rb +54 -137
- metadata +15 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7b6fd483aeb5428ce74df156ed7efe4990cee287
|
4
|
+
data.tar.gz: f34fee8557781f1756558a9945b83d77b4f104e9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1ac2032b68b88c5985910767a0bec383cc6bab13eb58c983fcac3b0e23d8f4529a15d464fc08ac9834f4c32a29b10cf991e789652f3c641974ebd3ec9f1ba248
|
7
|
+
data.tar.gz: 5606a780fb2c93df73c350baa05ac9f12cd7adb42720f1031c50cc325ef20e76844d1c74edb2ca318cb2a6bc6ae2fec90816f72494568d8877cd93168a0c2513
|
data/README.md
CHANGED
@@ -70,12 +70,10 @@ Or install it yourself as:
|
|
70
70
|
"request.uri":"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
|
71
71
|
"request.http_version":"HTTP/1.1",
|
72
72
|
"request.uri.scheme":"http",
|
73
|
-
"request.uri.
|
73
|
+
"request.uri.user":null,
|
74
74
|
"request.uri.host":"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
|
75
75
|
"request.uri.port":80,
|
76
|
-
"request.uri.registry":null,
|
77
76
|
"request.uri.path":"/",
|
78
|
-
"request.uri.opaque":null,
|
79
77
|
"request.uri.query":null,
|
80
78
|
"request.uri.fragment":null
|
81
79
|
}
|
@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
|
|
20
20
|
|
21
21
|
spec.add_dependency 'fluentd'
|
22
22
|
spec.add_dependency 'aws-sdk', '~> 2'
|
23
|
+
spec.add_dependency 'addressable'
|
23
24
|
spec.add_development_dependency 'bundler'
|
24
25
|
spec.add_development_dependency 'rake'
|
25
26
|
spec.add_development_dependency 'rspec', '>= 3.0.0'
|
@@ -54,7 +54,7 @@ class Fluent::ElbAccessLogInput < Fluent::Input
|
|
54
54
|
require 'fileutils'
|
55
55
|
require 'logger'
|
56
56
|
require 'time'
|
57
|
-
require 'uri'
|
57
|
+
require 'addressable/uri'
|
58
58
|
require 'aws-sdk'
|
59
59
|
end
|
60
60
|
|
@@ -193,9 +193,9 @@ class Fluent::ElbAccessLogInput < Fluent::Input
|
|
193
193
|
record['request.http_version'] = http_version
|
194
194
|
|
195
195
|
begin
|
196
|
-
uri = URI.parse(uri)
|
196
|
+
uri = Addressable::URI.parse(uri)
|
197
197
|
|
198
|
-
[:scheme ,:
|
198
|
+
[:scheme ,:user, :host, :port, :path, :query, :fragment].each do |key|
|
199
199
|
record["request.uri.#{key}"] = uri.send(key)
|
200
200
|
end
|
201
201
|
rescue => e
|
@@ -118,12 +118,10 @@ describe Fluent::ElbAccessLogInput do
|
|
118
118
|
"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
|
119
119
|
"request.http_version"=>"HTTP/1.1",
|
120
120
|
"request.uri.scheme"=>"http",
|
121
|
-
"request.uri.
|
121
|
+
"request.uri.user"=>nil,
|
122
122
|
"request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
|
123
123
|
"request.uri.port"=>80,
|
124
|
-
"request.uri.registry"=>nil,
|
125
124
|
"request.uri.path"=>"/",
|
126
|
-
"request.uri.opaque"=>nil,
|
127
125
|
"request.uri.query"=>nil,
|
128
126
|
"request.uri.fragment"=>nil}],
|
129
127
|
["elb.access_log",
|
@@ -148,12 +146,10 @@ describe Fluent::ElbAccessLogInput do
|
|
148
146
|
"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
|
149
147
|
"request.http_version"=>"HTTP/1.1",
|
150
148
|
"request.uri.scheme"=>"http",
|
151
|
-
"request.uri.
|
149
|
+
"request.uri.user"=>nil,
|
152
150
|
"request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
|
153
151
|
"request.uri.port"=>80,
|
154
|
-
"request.uri.registry"=>nil,
|
155
152
|
"request.uri.path"=>"/",
|
156
|
-
"request.uri.opaque"=>nil,
|
157
153
|
"request.uri.query"=>nil,
|
158
154
|
"request.uri.fragment"=>nil}],
|
159
155
|
["elb.access_log",
|
@@ -178,12 +174,10 @@ describe Fluent::ElbAccessLogInput do
|
|
178
174
|
"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
|
179
175
|
"request.http_version"=>"HTTP/1.1",
|
180
176
|
"request.uri.scheme"=>"http",
|
181
|
-
"request.uri.
|
177
|
+
"request.uri.user"=>nil,
|
182
178
|
"request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
|
183
179
|
"request.uri.port"=>80,
|
184
|
-
"request.uri.registry"=>nil,
|
185
180
|
"request.uri.path"=>"/",
|
186
|
-
"request.uri.opaque"=>nil,
|
187
181
|
"request.uri.query"=>nil,
|
188
182
|
"request.uri.fragment"=>nil}],
|
189
183
|
["elb.access_log",
|
@@ -208,12 +202,10 @@ describe Fluent::ElbAccessLogInput do
|
|
208
202
|
"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
|
209
203
|
"request.http_version"=>"HTTP/1.1",
|
210
204
|
"request.uri.scheme"=>"http",
|
211
|
-
"request.uri.
|
205
|
+
"request.uri.user"=>nil,
|
212
206
|
"request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
|
213
207
|
"request.uri.port"=>80,
|
214
|
-
"request.uri.registry"=>nil,
|
215
208
|
"request.uri.path"=>"/",
|
216
|
-
"request.uri.opaque"=>nil,
|
217
209
|
"request.uri.query"=>nil,
|
218
210
|
"request.uri.fragment"=>nil}]]
|
219
211
|
end
|
@@ -237,131 +229,60 @@ describe Fluent::ElbAccessLogInput do
|
|
237
229
|
is_expected.to eq expected_emits
|
238
230
|
end
|
239
231
|
end
|
232
|
+
end
|
233
|
+
|
234
|
+
context 'when include bad URI' do
|
235
|
+
let(:today_access_log) do
|
236
|
+
<<-EOS
|
237
|
+
2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
|
238
|
+
EOS
|
239
|
+
end
|
240
|
+
|
241
|
+
before do
|
242
|
+
expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
|
243
|
+
expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
|
240
244
|
|
241
|
-
|
242
|
-
|
243
|
-
<<-EOS
|
244
|
-
2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
|
245
|
-
2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://'hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" - -
|
246
|
-
EOS
|
245
|
+
expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
|
246
|
+
[double('today_objects', contents: [double('today_object', key: today_object_key)])]
|
247
247
|
end
|
248
248
|
|
249
|
-
|
250
|
-
[
|
251
|
-
Time.parse('2015-05-24 19:55:36 UTC').to_i,
|
252
|
-
{"timestamp"=>"2015-05-24T19:55:36.000000Z",
|
253
|
-
"elb"=>"hoge",
|
254
|
-
"client"=>"14.14.124.20",
|
255
|
-
"client_port"=>57673,
|
256
|
-
"backend"=>"10.0.199.184",
|
257
|
-
"backend_port"=>80,
|
258
|
-
"request_processing_time"=>5.3e-05,
|
259
|
-
"backend_processing_time"=>0.000913,
|
260
|
-
"response_processing_time"=>3.6e-05,
|
261
|
-
"elb_status_code"=>200,
|
262
|
-
"backend_status_code"=>200,
|
263
|
-
"received_bytes"=>0,
|
264
|
-
"sent_bytes"=>3,
|
265
|
-
"request"=>
|
266
|
-
"GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
|
267
|
-
"request.method"=>"GET",
|
268
|
-
"request.uri"=>
|
269
|
-
"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
|
270
|
-
"request.http_version"=>"HTTP/1.1",
|
271
|
-
"request.uri.scheme"=>"http",
|
272
|
-
"request.uri.userinfo"=>nil,
|
273
|
-
"request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
|
274
|
-
"request.uri.port"=>80,
|
275
|
-
"request.uri.registry"=>nil,
|
276
|
-
"request.uri.path"=>"/",
|
277
|
-
"request.uri.opaque"=>nil,
|
278
|
-
"request.uri.query"=>nil,
|
279
|
-
"request.uri.fragment"=>nil}],
|
280
|
-
["elb.access_log",
|
281
|
-
Time.parse('2015-05-24 19:55:36 UTC').to_i,
|
282
|
-
{"timestamp"=>"2015-05-24T19:55:36.000000Z",
|
283
|
-
"elb"=>"hoge",
|
284
|
-
"client"=>"14.14.124.20",
|
285
|
-
"client_port"=>57673,
|
286
|
-
"backend"=>"10.0.199.184",
|
287
|
-
"backend_port"=>80,
|
288
|
-
"request_processing_time"=>5.3e-05,
|
289
|
-
"backend_processing_time"=>0.000913,
|
290
|
-
"response_processing_time"=>3.6e-05,
|
291
|
-
"elb_status_code"=>200,
|
292
|
-
"backend_status_code"=>200,
|
293
|
-
"received_bytes"=>0,
|
294
|
-
"sent_bytes"=>3,
|
295
|
-
"request"=>
|
296
|
-
"GET http://'hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
|
297
|
-
"request.method"=>"GET",
|
298
|
-
"request.uri"=>
|
299
|
-
"http://'hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
|
300
|
-
"request.http_version"=>"HTTP/1.1"}],
|
301
|
-
["elb.access_log",
|
302
|
-
Time.parse('2015-05-25 19:55:36 UTC').to_i,
|
303
|
-
{"timestamp"=>"2015-05-25T19:55:36.000000Z",
|
304
|
-
"elb"=>"hoge",
|
305
|
-
"client"=>"14.14.124.20",
|
306
|
-
"client_port"=>57673,
|
307
|
-
"backend"=>"10.0.199.184",
|
308
|
-
"backend_port"=>80,
|
309
|
-
"request_processing_time"=>5.3e-05,
|
310
|
-
"backend_processing_time"=>0.000913,
|
311
|
-
"response_processing_time"=>3.6e-05,
|
312
|
-
"elb_status_code"=>200,
|
313
|
-
"backend_status_code"=>200,
|
314
|
-
"received_bytes"=>0,
|
315
|
-
"sent_bytes"=>3,
|
316
|
-
"request"=>
|
317
|
-
"GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
|
318
|
-
"request.method"=>"GET",
|
319
|
-
"request.uri"=>
|
320
|
-
"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
|
321
|
-
"request.http_version"=>"HTTP/1.1",
|
322
|
-
"request.uri.scheme"=>"http",
|
323
|
-
"request.uri.userinfo"=>nil,
|
324
|
-
"request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
|
325
|
-
"request.uri.port"=>80,
|
326
|
-
"request.uri.registry"=>nil,
|
327
|
-
"request.uri.path"=>"/",
|
328
|
-
"request.uri.opaque"=>nil,
|
329
|
-
"request.uri.query"=>nil,
|
330
|
-
"request.uri.fragment"=>nil}],
|
331
|
-
["elb.access_log",
|
332
|
-
Time.parse('2015-05-25 19:55:36 UTC').to_i,
|
333
|
-
{"timestamp"=>"2015-05-25T19:55:36.000000Z",
|
334
|
-
"elb"=>"hoge",
|
335
|
-
"client"=>"14.14.124.20",
|
336
|
-
"client_port"=>57673,
|
337
|
-
"backend"=>"10.0.199.184",
|
338
|
-
"backend_port"=>80,
|
339
|
-
"request_processing_time"=>5.3e-05,
|
340
|
-
"backend_processing_time"=>0.000913,
|
341
|
-
"response_processing_time"=>3.6e-05,
|
342
|
-
"elb_status_code"=>200,
|
343
|
-
"backend_status_code"=>200,
|
344
|
-
"received_bytes"=>0,
|
345
|
-
"sent_bytes"=>3,
|
346
|
-
"request"=>
|
347
|
-
"GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
|
348
|
-
"request.method"=>"GET",
|
349
|
-
"request.uri"=>
|
350
|
-
"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
|
351
|
-
"request.http_version"=>"HTTP/1.1",
|
352
|
-
"request.uri.scheme"=>"http",
|
353
|
-
"request.uri.userinfo"=>nil,
|
354
|
-
"request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
|
355
|
-
"request.uri.port"=>80,
|
356
|
-
"request.uri.registry"=>nil,
|
357
|
-
"request.uri.path"=>"/",
|
358
|
-
"request.uri.opaque"=>nil,
|
359
|
-
"request.uri.query"=>nil,
|
360
|
-
"request.uri.fragment"=>nil}]]
|
249
|
+
expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
|
250
|
+
[double('today_s3_object', body: StringIO.new(today_access_log))]
|
361
251
|
end
|
362
252
|
|
363
|
-
|
253
|
+
expect(driver.instance).to receive(:save_timestamp).with(today)
|
254
|
+
expect(driver.instance).to receive(:save_history)
|
255
|
+
|
256
|
+
allow(Addressable::URI).to receive(:parse).and_raise('parse error')
|
257
|
+
|
258
|
+
driver.run
|
364
259
|
end
|
260
|
+
|
261
|
+
let(:expected_emits) do
|
262
|
+
[["elb.access_log",
|
263
|
+
Time.parse('2015-05-24 19:55:36 UTC').to_i,
|
264
|
+
{"timestamp"=>"2015-05-24T19:55:36.000000Z",
|
265
|
+
"elb"=>"hoge",
|
266
|
+
"client"=>"14.14.124.20",
|
267
|
+
"client_port"=>57673,
|
268
|
+
"backend"=>"10.0.199.184",
|
269
|
+
"backend_port"=>80,
|
270
|
+
"request_processing_time"=>5.3e-05,
|
271
|
+
"backend_processing_time"=>0.000913,
|
272
|
+
"response_processing_time"=>3.6e-05,
|
273
|
+
"elb_status_code"=>200,
|
274
|
+
"backend_status_code"=>200,
|
275
|
+
"received_bytes"=>0,
|
276
|
+
"sent_bytes"=>3,
|
277
|
+
"request"=>
|
278
|
+
"GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
|
279
|
+
"request.method"=>"GET",
|
280
|
+
"request.uri"=>
|
281
|
+
"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
|
282
|
+
"request.http_version"=>"HTTP/1.1"}]]
|
283
|
+
end
|
284
|
+
|
285
|
+
it { is_expected.to eq expected_emits }
|
365
286
|
end
|
366
287
|
|
367
288
|
context 'when access log exists (with tag option)' do
|
@@ -422,12 +343,10 @@ describe Fluent::ElbAccessLogInput do
|
|
422
343
|
"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
|
423
344
|
"request.http_version"=>"HTTP/1.1",
|
424
345
|
"request.uri.scheme"=>"http",
|
425
|
-
"request.uri.
|
346
|
+
"request.uri.user"=>nil,
|
426
347
|
"request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
|
427
348
|
"request.uri.port"=>80,
|
428
|
-
"request.uri.registry"=>nil,
|
429
349
|
"request.uri.path"=>"/",
|
430
|
-
"request.uri.opaque"=>nil,
|
431
350
|
"request.uri.query"=>nil,
|
432
351
|
"request.uri.fragment"=>nil}]]
|
433
352
|
end
|
@@ -485,12 +404,10 @@ describe Fluent::ElbAccessLogInput do
|
|
485
404
|
"http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
|
486
405
|
"request.http_version"=>"HTTP/1.1",
|
487
406
|
"request.uri.scheme"=>"http",
|
488
|
-
"request.uri.
|
407
|
+
"request.uri.user"=>nil,
|
489
408
|
"request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
|
490
409
|
"request.uri.port"=>80,
|
491
|
-
"request.uri.registry"=>nil,
|
492
410
|
"request.uri.path"=>"/",
|
493
|
-
"request.uri.opaque"=>nil,
|
494
411
|
"request.uri.query"=>nil,
|
495
412
|
"request.uri.fragment"=>nil}]]
|
496
413
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elb-access-log
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Genki Sugawara
|
@@ -38,6 +38,20 @@ dependencies:
|
|
38
38
|
- - ~>
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '2'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: addressable
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
41
55
|
- !ruby/object:Gem::Dependency
|
42
56
|
name: bundler
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|