fluent-plugin-elb-access-log 0.3.7.pre.beta1 → 0.3.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,763 @@
1
+ describe Fluent::ElbAccessLogInput do
2
+ let(:account_id) { '123456789012' }
3
+ let(:s3_bucket) { 'my-bucket' }
4
+ let(:region) { 'us-west-1' }
5
+ let(:driver) { create_driver(fluentd_conf) }
6
+ let(:client){ Aws::S3::Client.new(stub_responses: true) }
7
+
8
+ let(:fluentd_conf) do
9
+ {
10
+ account_id: account_id,
11
+ s3_bucket: s3_bucket,
12
+ region: region,
13
+ start_datetime: (today - 1).to_s,
14
+ elb_type: 'alb',
15
+ }
16
+ end
17
+
18
+ let(:today) { Time.parse('2015/05/24 18:30 UTC') }
19
+ let(:yesterday) { today - 86400 }
20
+ let(:tomorrow) { today + 86400 }
21
+
22
+ let(:today_prefix) { "AWSLogs/#{account_id}/elasticloadbalancing/#{region}/#{today.strftime('%Y/%m/%d')}/" }
23
+ let(:yesterday_prefix) { "AWSLogs/#{account_id}/elasticloadbalancing/#{region}/#{yesterday.strftime('%Y/%m/%d')}/" }
24
+ let(:tomorrow_prefix) { "AWSLogs/#{account_id}/elasticloadbalancing/#{region}/#{tomorrow.strftime('%Y/%m/%d')}/" }
25
+
26
+ let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{today.iso8601}_52.68.51.1_8hSqR3o4.log.gz" }
27
+ let(:yesterday_object_key) { "#{yesterday_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{yesterday.iso8601}_52.68.51.1_8hSqR3o4.log.gz" }
28
+ let(:tomorrow_object_key) { "#{tomorrow_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{tomorrow.iso8601}_52.68.51.1_8hSqR3o4.log.gz" }
29
+
30
+ before do
31
+ Timecop.freeze(today)
32
+ allow_any_instance_of(Fluent::ElbAccessLogInput).to receive(:client) { client }
33
+ allow_any_instance_of(Fluent::ElbAccessLogInput).to receive(:load_history) { [] }
34
+ allow_any_instance_of(Fluent::ElbAccessLogInput).to receive(:parse_tsfile) { nil }
35
+ allow(FileUtils).to receive(:touch)
36
+ expect(driver.instance.log).to_not receive(:error)
37
+ end
38
+
39
+ after do
40
+ Timecop.return
41
+ end
42
+
43
+ subject { driver.emits }
44
+
45
+ context 'when access log does not exist' do
46
+ before do
47
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
48
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) { [] }
49
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
50
+ expect(driver.instance).to_not receive(:save_timestamp).with(today)
51
+ expect(driver.instance).to receive(:save_history)
52
+ expect(driver.instance.log).to_not receive(:warn)
53
+
54
+ driver.run
55
+ end
56
+
57
+ it { is_expected.to be_empty }
58
+ end
59
+
60
+ context 'when access log exists' do
61
+ let(:today_access_log) do
62
+ Zlib::Deflate.deflate(<<-EOS)
63
+ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
64
+ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
65
+ EOS
66
+ end
67
+
68
+ let(:tomorrow_access_log) do
69
+ Zlib::Deflate.deflate(<<-EOS)
70
+ https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
71
+ https 2015-05-25T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
72
+ EOS
73
+ end
74
+
75
+ before do
76
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) do
77
+ [double('yesterday_objects', contents: [double('yesterday_object', key: yesterday_object_key)])]
78
+ end
79
+
80
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
81
+ [double('today_objects', contents: [double('today_object', key: today_object_key)])]
82
+ end
83
+
84
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) do
85
+ [double('tomorrow_objects', contents: [double('tomorrow_object', key: tomorrow_object_key)])]
86
+ end
87
+
88
+ expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
89
+ double('today_s3_object', body: StringIO.new(today_access_log))
90
+ end
91
+
92
+ expect(client).to receive(:get_object).with(bucket: s3_bucket, key: tomorrow_object_key) do
93
+ double('tomorrow_s3_object', body: StringIO.new(tomorrow_access_log))
94
+ end
95
+
96
+ expect(driver.instance).to receive(:save_timestamp).with(tomorrow)
97
+ expect(driver.instance).to receive(:save_history)
98
+ expect(driver.instance.log).to_not receive(:warn)
99
+
100
+ driver.run
101
+ end
102
+
103
+ let(:expected_emits) do
104
+ [["elb.access_log",
105
+ Time.parse('2015-05-24 19:55:36 UTC').to_i,
106
+ {"type"=>"https",
107
+ "timestamp"=>"2015-05-24T19:55:36.000000Z",
108
+ "elb"=>"hoge",
109
+ "client_port"=>57673,
110
+ "target_port"=>80,
111
+ "request_processing_time"=>5.3e-05,
112
+ "target_processing_time"=>0.000913,
113
+ "response_processing_time"=>3.6e-05,
114
+ "elb_status_code"=>200,
115
+ "target_status_code"=>200,
116
+ "received_bytes"=>0,
117
+ "sent_bytes"=>3,
118
+ "request"=>
119
+ "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
120
+ "user_agent"=>"curl/7.30.0",
121
+ "ssl_cipher"=>"ssl_cipher",
122
+ "ssl_protocol"=>"ssl_protocol",
123
+ "target_group_arn"=>
124
+ "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx",
125
+ "trace_id"=>"Root=xxx",
126
+ "domain_name"=>"-",
127
+ "chosen_cert_arn"=>
128
+ "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx",
129
+ "client"=>"14.14.124.20",
130
+ "target"=>"10.0.199.184",
131
+ "request.method"=>"GET",
132
+ "request.uri"=>
133
+ "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
134
+ "request.http_version"=>"HTTP/1.1",
135
+ "request.uri.scheme"=>"http",
136
+ "request.uri.user"=>nil,
137
+ "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
138
+ "request.uri.port"=>80,
139
+ "request.uri.path"=>"/",
140
+ "request.uri.query"=>nil,
141
+ "request.uri.fragment"=>nil}],
142
+ ["elb.access_log",
143
+ Time.parse('2015-05-24 19:55:36 UTC').to_i,
144
+ {"type"=>"https",
145
+ "timestamp"=>"2015-05-24T19:55:36.000000Z",
146
+ "elb"=>"hoge",
147
+ "client_port"=>57673,
148
+ "target_port"=>80,
149
+ "request_processing_time"=>5.3e-05,
150
+ "target_processing_time"=>0.000913,
151
+ "response_processing_time"=>3.6e-05,
152
+ "elb_status_code"=>200,
153
+ "target_status_code"=>200,
154
+ "received_bytes"=>0,
155
+ "sent_bytes"=>3,
156
+ "request"=>
157
+ "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
158
+ "user_agent"=>"curl/7.30.0",
159
+ "ssl_cipher"=>"ssl_cipher",
160
+ "ssl_protocol"=>"ssl_protocol",
161
+ "target_group_arn"=>
162
+ "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx",
163
+ "trace_id"=>"Root=xxx",
164
+ "domain_name"=>"-",
165
+ "chosen_cert_arn"=>
166
+ "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx",
167
+ "client"=>"14.14.124.20",
168
+ "target"=>"10.0.199.184",
169
+ "request.method"=>"GET",
170
+ "request.uri"=>
171
+ "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
172
+ "request.http_version"=>"HTTP/1.1",
173
+ "request.uri.scheme"=>"http",
174
+ "request.uri.user"=>nil,
175
+ "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
176
+ "request.uri.port"=>80,
177
+ "request.uri.path"=>"/",
178
+ "request.uri.query"=>nil,
179
+ "request.uri.fragment"=>nil}],
180
+ ["elb.access_log",
181
+ Time.parse('2015-05-25 19:55:36 UTC').to_i,
182
+ {"type"=>"https",
183
+ "timestamp"=>"2015-05-25T19:55:36.000000Z",
184
+ "elb"=>"hoge",
185
+ "client_port"=>57673,
186
+ "target_port"=>80,
187
+ "request_processing_time"=>5.3e-05,
188
+ "target_processing_time"=>0.000913,
189
+ "response_processing_time"=>3.6e-05,
190
+ "elb_status_code"=>200,
191
+ "target_status_code"=>200,
192
+ "received_bytes"=>0,
193
+ "sent_bytes"=>3,
194
+ "request"=>
195
+ "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
196
+ "user_agent"=>"curl/7.30.0",
197
+ "ssl_cipher"=>"ssl_cipher",
198
+ "ssl_protocol"=>"ssl_protocol",
199
+ "target_group_arn"=>
200
+ "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx",
201
+ "trace_id"=>"Root=xxx",
202
+ "domain_name"=>"-",
203
+ "chosen_cert_arn"=>
204
+ "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx",
205
+ "client"=>"14.14.124.20",
206
+ "target"=>"10.0.199.184",
207
+ "request.method"=>"GET",
208
+ "request.uri"=>
209
+ "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
210
+ "request.http_version"=>"HTTP/1.1",
211
+ "request.uri.scheme"=>"http",
212
+ "request.uri.user"=>nil,
213
+ "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
214
+ "request.uri.port"=>80,
215
+ "request.uri.path"=>"/",
216
+ "request.uri.query"=>nil,
217
+ "request.uri.fragment"=>nil}],
218
+ ["elb.access_log",
219
+ Time.parse('2015-05-25 19:55:36 UTC').to_i,
220
+ {"type"=>"https",
221
+ "timestamp"=>"2015-05-25T19:55:36.000000Z",
222
+ "elb"=>"hoge",
223
+ "client_port"=>57673,
224
+ "target_port"=>80,
225
+ "request_processing_time"=>5.3e-05,
226
+ "target_processing_time"=>0.000913,
227
+ "response_processing_time"=>3.6e-05,
228
+ "elb_status_code"=>200,
229
+ "target_status_code"=>200,
230
+ "received_bytes"=>0,
231
+ "sent_bytes"=>3,
232
+ "request"=>
233
+ "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
234
+ "user_agent"=>"curl/7.30.0",
235
+ "ssl_cipher"=>"ssl_cipher",
236
+ "ssl_protocol"=>"ssl_protocol",
237
+ "target_group_arn"=>
238
+ "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx",
239
+ "trace_id"=>"Root=xxx",
240
+ "domain_name"=>"-",
241
+ "chosen_cert_arn"=>
242
+ "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx",
243
+ "client"=>"14.14.124.20",
244
+ "target"=>"10.0.199.184",
245
+ "request.method"=>"GET",
246
+ "request.uri"=>
247
+ "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
248
+ "request.http_version"=>"HTTP/1.1",
249
+ "request.uri.scheme"=>"http",
250
+ "request.uri.user"=>nil,
251
+ "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
252
+ "request.uri.port"=>80,
253
+ "request.uri.path"=>"/",
254
+ "request.uri.query"=>nil,
255
+ "request.uri.fragment"=>nil}]]
256
+ end
257
+
258
+ it { is_expected.to match_table expected_emits }
259
+
260
+ context 'when sampling' do
261
+ let(:fluentd_conf) do
262
+ {
263
+ account_id: account_id,
264
+ s3_bucket: s3_bucket,
265
+ region: region,
266
+ start_datetime: (today - 1).to_s,
267
+ sampling_interval: 2,
268
+ elb_type: 'alb',
269
+ }
270
+ end
271
+
272
+ it do
273
+ expected_emits.delete_at(3)
274
+ expected_emits.delete_at(1)
275
+ is_expected.to match_table expected_emits
276
+ end
277
+ end
278
+ end
279
+
280
+ context 'when include bad URI' do
281
+ let(:today_access_log) do
282
+ Zlib::Deflate.deflate(<<-EOS)
283
+ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
284
+ EOS
285
+ end
286
+
287
+ before do
288
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
289
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
290
+
291
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
292
+ [double('today_objects', contents: [double('today_object', key: today_object_key)])]
293
+ end
294
+
295
+ expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
296
+ double('today_s3_object', body: StringIO.new(today_access_log))
297
+ end
298
+
299
+ expect(driver.instance).to receive(:save_timestamp).with(today)
300
+ expect(driver.instance).to receive(:save_history)
301
+
302
+ allow(Addressable::URI).to receive(:parse).and_raise('parse error')
303
+ expect(driver.instance.log).to receive(:warn).with('parse error: http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/')
304
+
305
+ driver.run
306
+ end
307
+
308
+ let(:expected_emits) do
309
+ [["elb.access_log",
310
+ Time.parse('2015-05-24 19:55:36 UTC').to_i,
311
+ {"chosen_cert_arn"=>
312
+ "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx",
313
+ "client"=>"14.14.124.20",
314
+ "client_port"=>57673,
315
+ "domain_name"=>"-",
316
+ "elb"=>"hoge",
317
+ "elb_status_code"=>200,
318
+ "received_bytes"=>0,
319
+ "request"=>
320
+ "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
321
+ "request.http_version"=>"HTTP/1.1",
322
+ "request.method"=>"GET",
323
+ "request.uri"=>
324
+ "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
325
+ "request_processing_time"=>5.3e-05,
326
+ "response_processing_time"=>3.6e-05,
327
+ "sent_bytes"=>3,
328
+ "ssl_cipher"=>"ssl_cipher",
329
+ "ssl_protocol"=>"ssl_protocol",
330
+ "target"=>"10.0.199.184",
331
+ "target_group_arn"=>
332
+ "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx",
333
+ "target_port"=>80,
334
+ "target_processing_time"=>0.000913,
335
+ "target_status_code"=>200,
336
+ "timestamp"=>"2015-05-24T19:55:36.000000Z",
337
+ "trace_id"=>"Root=xxx",
338
+ "type"=>"https",
339
+ "user_agent"=>"curl/7.30.0"}]]
340
+ end
341
+
342
+ it { is_expected.to match_table expected_emits }
343
+ end
344
+
345
+ context 'when access log exists (with tag option)' do
346
+ let(:today_access_log) do
347
+ Zlib::Deflate.deflate(<<-EOS)
348
+ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
349
+ EOS
350
+ end
351
+
352
+ let(:fluentd_conf) do
353
+ {
354
+ account_id: account_id,
355
+ s3_bucket: s3_bucket,
356
+ region: region,
357
+ start_datetime: (today - 1).to_s,
358
+ tag: 'any.tag',
359
+ elb_type: 'alb',
360
+ }
361
+ end
362
+
363
+ before do
364
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
365
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
366
+
367
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
368
+ [double('today_objects', contents: [double('today_object', key: today_object_key)])]
369
+ end
370
+
371
+ expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
372
+ double('today_s3_object', body: StringIO.new(today_access_log))
373
+ end
374
+
375
+ expect(driver.instance).to receive(:save_timestamp).with(today)
376
+ expect(driver.instance).to receive(:save_history)
377
+ expect(driver.instance.log).to_not receive(:warn)
378
+
379
+ driver.run
380
+ end
381
+
382
+ let(:expected_emits) do
383
+ [["any.tag",
384
+ Time.parse('2015-05-24 19:55:36 UTC').to_i,
385
+ {"chosen_cert_arn"=>
386
+ "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx",
387
+ "client"=>"14.14.124.20",
388
+ "client_port"=>57673,
389
+ "domain_name"=>"-",
390
+ "elb"=>"hoge",
391
+ "elb_status_code"=>200,
392
+ "received_bytes"=>0,
393
+ "request"=>
394
+ "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
395
+ "request.http_version"=>"HTTP/1.1",
396
+ "request.method"=>"GET",
397
+ "request.uri"=>
398
+ "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
399
+ "request.uri.fragment"=>nil,
400
+ "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
401
+ "request.uri.path"=>"/",
402
+ "request.uri.port"=>80,
403
+ "request.uri.query"=>nil,
404
+ "request.uri.scheme"=>"http",
405
+ "request.uri.user"=>nil,
406
+ "request_processing_time"=>5.3e-05,
407
+ "response_processing_time"=>3.6e-05,
408
+ "sent_bytes"=>3,
409
+ "ssl_cipher"=>"ssl_cipher",
410
+ "ssl_protocol"=>"ssl_protocol",
411
+ "target"=>"10.0.199.184",
412
+ "target_group_arn"=>
413
+ "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx",
414
+ "target_port"=>80,
415
+ "target_processing_time"=>0.000913,
416
+ "target_status_code"=>200,
417
+ "timestamp"=>"2015-05-24T19:55:36.000000Z",
418
+ "trace_id"=>"Root=xxx",
419
+ "type"=>"https",
420
+ "user_agent"=>"curl/7.30.0"}]]
421
+ end
422
+
423
+ it { is_expected.to match_table expected_emits }
424
+ end
425
+
426
+
427
+ context 'when access old log exists' do
428
+ let(:today_access_log) do
429
+ Zlib::Deflate.deflate(<<-EOS)
430
+ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
431
+ EOS
432
+ end
433
+
434
+ let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 600).iso8601}_52.68.51.1_8hSqR3o4.log.gz" }
435
+
436
+ before do
437
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
438
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
439
+
440
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
441
+ [double('today_objects', contents: [double('today_object', key: today_object_key)])]
442
+ end
443
+
444
+ expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
445
+ double('today_s3_object', body: StringIO.new(today_access_log))
446
+ end
447
+
448
+ expect(driver.instance).to_not receive(:save_timestamp)
449
+ expect(driver.instance).to receive(:save_history)
450
+ expect(driver.instance.log).to_not receive(:warn)
451
+
452
+ driver.run
453
+ end
454
+
455
+ let(:expected_emits) do
456
+ [["elb.access_log",
457
+ Time.parse('2015-05-24 19:55:36 UTC').to_i,
458
+ {"chosen_cert_arn"=>
459
+ "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx",
460
+ "client"=>"14.14.124.20",
461
+ "client_port"=>57673,
462
+ "domain_name"=>"-",
463
+ "elb"=>"hoge",
464
+ "elb_status_code"=>200,
465
+ "received_bytes"=>0,
466
+ "request"=>
467
+ "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
468
+ "request.http_version"=>"HTTP/1.1",
469
+ "request.method"=>"GET",
470
+ "request.uri"=>
471
+ "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
472
+ "request.uri.fragment"=>nil,
473
+ "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
474
+ "request.uri.path"=>"/",
475
+ "request.uri.port"=>80,
476
+ "request.uri.query"=>nil,
477
+ "request.uri.scheme"=>"http",
478
+ "request.uri.user"=>nil,
479
+ "request_processing_time"=>5.3e-05,
480
+ "response_processing_time"=>3.6e-05,
481
+ "sent_bytes"=>3,
482
+ "ssl_cipher"=>"ssl_cipher",
483
+ "ssl_protocol"=>"ssl_protocol",
484
+ "target"=>"10.0.199.184",
485
+ "target_group_arn"=>
486
+ "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx",
487
+ "target_port"=>80,
488
+ "target_processing_time"=>0.000913,
489
+ "target_status_code"=>200,
490
+ "timestamp"=>"2015-05-24T19:55:36.000000Z",
491
+ "trace_id"=>"Root=xxx",
492
+ "type"=>"https",
493
+ "user_agent"=>"curl/7.30.0"}]]
494
+ end
495
+
496
+ it { is_expected.to match_table expected_emits }
497
+ end
498
+
499
+ context 'when parse error' do
500
+ let(:today_access_log) do
501
+ Zlib::Deflate.deflate(<<-EOS)
502
+ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
503
+ EOS
504
+ end
505
+
506
+ before do
507
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
508
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
509
+
510
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
511
+ [double('today_objects', contents: [double('today_object', key: today_object_key)])]
512
+ end
513
+
514
+ expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
515
+ double('today_s3_object', body: StringIO.new(today_access_log))
516
+ end
517
+
518
+ expect(driver.instance).to receive(:save_timestamp).with(today)
519
+ expect(driver.instance).to receive(:save_history)
520
+
521
+ expect(CSV).to receive(:parse_line).and_raise('parse error')
522
+ expect(driver.instance.log).to_not receive(:warn)
523
+
524
+ driver.run
525
+ end
526
+
527
+ let(:expected_emits) do
528
+ [["elb.access_log",
529
+ Time.parse('2015-05-24 19:55:36 UTC').to_i,
530
+ {"chosen_cert_arn"=>
531
+ "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx",
532
+ "client"=>"14.14.124.20",
533
+ "client_port"=>57673,
534
+ "domain_name"=>"-",
535
+ "elb"=>"hoge",
536
+ "elb_status_code"=>200,
537
+ "received_bytes"=>0,
538
+ "request"=>
539
+ "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
540
+ "request.http_version"=>"HTTP/1.1",
541
+ "request.method"=>"GET",
542
+ "request.uri"=>
543
+ "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
544
+ "request.uri.fragment"=>nil,
545
+ "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
546
+ "request.uri.path"=>"/",
547
+ "request.uri.port"=>80,
548
+ "request.uri.query"=>nil,
549
+ "request.uri.scheme"=>"http",
550
+ "request.uri.user"=>nil,
551
+ "request_processing_time"=>5.3e-05,
552
+ "response_processing_time"=>3.6e-05,
553
+ "sent_bytes"=>3,
554
+ "ssl_cipher"=>"ssl_cipher",
555
+ "ssl_protocol"=>"ssl_protocol",
556
+ "target"=>"10.0.199.184",
557
+ "target_group_arn"=>
558
+ "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx",
559
+ "target_port"=>80,
560
+ "target_processing_time"=>0.000913,
561
+ "target_status_code"=>200,
562
+ "timestamp"=>"2015-05-24T19:55:36.000000Z",
563
+ "trace_id"=>"Root=xxx",
564
+ "type"=>"https",
565
+ "user_agent"=>"curl/7.30.0"}]]
566
+ end
567
+
568
+ it { is_expected.to match_table expected_emits }
569
+
570
+ context 'when no user_agent' do
571
+ let(:today_access_log) do
572
+ Zlib::Deflate.deflate(<<-EOS)
573
+ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1"
574
+ EOS
575
+ end
576
+
577
+ before do
578
+ expected_emits[0][2]['user_agent'] = nil
579
+ expected_emits[0][2]['ssl_cipher'] = nil
580
+ expected_emits[0][2]['ssl_protocol'] = nil
581
+ expected_emits[0][2]['target_group_arn'] = nil
582
+ expected_emits[0][2]['trace_id'] = nil
583
+ expected_emits[0][2]['domain_name'] = nil
584
+ expected_emits[0][2]['chosen_cert_arn'] = nil
585
+ end
586
+
587
+ it { is_expected.to match_table expected_emits }
588
+ end
589
+ end
590
+
591
+ context 'when access old log exists (timeout)' do
592
+ let(:today_access_log) do
593
+ Zlib::Deflate.deflate(<<-EOS)
594
+ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
595
+ EOS
596
+ end
597
+
598
+ let(:today_object_key) { "#{today_prefix}#{account_id}_elasticloadbalancing_ap-northeast-1_hoge_#{(today - 601).iso8601}_52.68.51.1_8hSqR3o4.log.gz" }
599
+
600
+ before do
601
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
602
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
603
+
604
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
605
+ [double('today_objects', contents: [double('today_object', key: today_object_key)])]
606
+ end
607
+
608
+ expect(client).to_not receive(:get_object)
609
+ expect(driver.instance).to_not receive(:save_timestamp)
610
+ expect(driver.instance).to receive(:save_history)
611
+ expect(driver.instance.log).to_not receive(:warn)
612
+
613
+ driver.run
614
+ end
615
+
616
+ it { is_expected.to be_empty }
617
+ end
618
+
619
+ context 'when emitted log exists' do
620
+ let(:today_access_log) do
621
+ Zlib::Deflate.deflate(<<-EOS)
622
+ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
623
+ EOS
624
+ end
625
+
626
+ before do
627
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
628
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
629
+
630
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
631
+ [double('today_objects', contents: [double('today_object', key: today_object_key)])]
632
+ end
633
+
634
+ expect(client).to_not receive(:get_object)
635
+
636
+ history = driver.instance.instance_variable_get(:@history)
637
+ history << today_object_key
638
+ expect(driver.instance).to_not receive(:save_timestamp)
639
+ expect(driver.instance).to receive(:save_history)
640
+ expect(driver.instance.log).to_not receive(:warn)
641
+
642
+ driver.run
643
+ end
644
+
645
+ it { is_expected.to be_empty }
646
+ end
647
+
648
+ describe 'history#length' do
649
+ let(:today_access_log) do
650
+ Zlib::Deflate.deflate(<<-EOS)
651
+ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.30.0" ssl_cipher ssl_protocol arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
652
+ EOS
653
+ end
654
+
655
+ let(:history) { driver.instance.instance_variable_get(:@history) }
656
+
657
+ before do
658
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
659
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
660
+
661
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
662
+ [double('today_objects', contents: [double('today_object', key: today_object_key)])]
663
+ end
664
+
665
+ expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
666
+ double('today_s3_object', body: StringIO.new(today_access_log))
667
+ end
668
+
669
+ expect(driver.instance).to receive(:save_timestamp).with(today)
670
+ expect(driver.instance).to receive(:save_history)
671
+ expect(driver.instance.log).to_not receive(:warn)
672
+ end
673
+
674
+ subject { history.length }
675
+
676
+ context 'when history.length <= 100' do
677
+ before do
678
+ driver.run
679
+ end
680
+
681
+ it { is_expected.to eq 1 }
682
+ end
683
+
684
+ context 'when history.length > 100' do
685
+ before do
686
+ history.concat (1..100).map(&:to_s)
687
+ driver.run
688
+ end
689
+
690
+ it { is_expected.to eq 100 }
691
+ end
692
+ end
693
+
694
+ context 'when no user_agent' do
695
+ let(:today_access_log) do
696
+ Zlib::Deflate.deflate(<<-EOS)
697
+ https 2015-05-24T19:55:36.000000Z hoge 14.14.124.20:57673 10.0.199.184:80 0.000053 0.000913 0.000036 200 200 0 3 "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1" arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx "Root=xxx" "-" "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx"
698
+ EOS
699
+ end
700
+
701
+ before do
702
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: yesterday_prefix) { [] }
703
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: tomorrow_prefix) { [] }
704
+
705
+ expect(client).to receive(:list_objects).with(bucket: s3_bucket, prefix: today_prefix) do
706
+ [double('today_objects', contents: [double('today_object', key: today_object_key)])]
707
+ end
708
+
709
+ expect(client).to receive(:get_object).with(bucket: s3_bucket, key: today_object_key) do
710
+ double('today_s3_object', body: StringIO.new(today_access_log))
711
+ end
712
+
713
+ expect(driver.instance).to receive(:save_timestamp).with(today)
714
+ expect(driver.instance).to receive(:save_history)
715
+ expect(driver.instance.log).to_not receive(:warn)
716
+
717
+ driver.run
718
+ end
719
+
720
+ let(:expected_emits) do
721
+ [["elb.access_log",
722
+ Time.parse('2015-05-24 19:55:36 UTC').to_i,
723
+ {"chosen_cert_arn"=>nil,
724
+ "client"=>"14.14.124.20",
725
+ "client_port"=>57673,
726
+ "domain_name"=>nil,
727
+ "elb"=>"hoge",
728
+ "elb_status_code"=>200,
729
+ "received_bytes"=>0,
730
+ "request"=>
731
+ "GET http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/ HTTP/1.1",
732
+ "request.http_version"=>"HTTP/1.1",
733
+ "request.method"=>"GET",
734
+ "request.uri"=>
735
+ "http://hoge-1876938939.ap-northeast-1.elb.amazonaws.com:80/",
736
+ "request.uri.fragment"=>nil,
737
+ "request.uri.host"=>"hoge-1876938939.ap-northeast-1.elb.amazonaws.com",
738
+ "request.uri.path"=>"/",
739
+ "request.uri.port"=>80,
740
+ "request.uri.query"=>nil,
741
+ "request.uri.scheme"=>"http",
742
+ "request.uri.user"=>nil,
743
+ "request_processing_time"=>5.3e-05,
744
+ "response_processing_time"=>3.6e-05,
745
+ "sent_bytes"=>3,
746
+ "ssl_cipher"=>"Root=xxx",
747
+ "ssl_protocol"=>"-",
748
+ "target"=>"10.0.199.184",
749
+ "target_group_arn"=>
750
+ "arn:aws:acm:ap-northeast-1:123456789012:certificate/xxx",
751
+ "target_port"=>80,
752
+ "target_processing_time"=>0.000913,
753
+ "target_status_code"=>200,
754
+ "timestamp"=>"2015-05-24T19:55:36.000000Z",
755
+ "trace_id"=>nil,
756
+ "type"=>"https",
757
+ "user_agent"=>
758
+ "arn:aws:elasticloadbalancing:ap-northeast-1:123456789012:targetgroup/app/xxx"}]]
759
+ end
760
+
761
+ it { is_expected.to match_table expected_emits }
762
+ end
763
+ end