fluent-plugin-s3-file-inclusion 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,713 @@
1
+ require 'fluent/test'
2
+ require 'fluent/test/helpers'
3
+ require 'fluent/test/log'
4
+ require 'fluent/test/driver/output'
5
+ require 'aws-sdk-s3'
6
+ require 'fluent/plugin/out_s3'
7
+
8
+ require 'test/unit/rr'
9
+ require 'zlib'
10
+ require 'fileutils'
11
+ require 'timecop'
12
+ require 'ostruct'
13
+
14
+ include Fluent::Test::Helpers
15
+
16
+ class S3OutputTest < Test::Unit::TestCase
17
+ def setup
18
+ # Fluent::Test.setup
19
+ end
20
+
21
+ def teardown
22
+ Dir.glob('test/tmp/*').each {|file| FileUtils.rm_f(file) }
23
+ end
24
+
25
+ CONFIG = %[
26
+ aws_key_id test_key_id
27
+ aws_sec_key test_sec_key
28
+ s3_bucket test_bucket
29
+ path log
30
+ utc
31
+ buffer_type memory
32
+ time_slice_format %Y%m%d-%H
33
+ ]
34
+
35
+ def create_driver(conf = CONFIG)
36
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::S3Output) do
37
+ def format(tag, time, record)
38
+ super
39
+ end
40
+
41
+ def write(chunk)
42
+ chunk.read
43
+ end
44
+
45
+ private
46
+
47
+ def ensure_bucket
48
+ end
49
+
50
+ def check_apikeys
51
+ end
52
+ end.configure(conf)
53
+ end
54
+
55
+ def test_configure
56
+ d = create_driver
57
+ assert_equal 'test_key_id', d.instance.aws_key_id
58
+ assert_equal 'test_sec_key', d.instance.aws_sec_key
59
+ assert_equal 'test_bucket', d.instance.s3_bucket
60
+ assert_equal 'log', d.instance.path
61
+ assert_equal 'gz', d.instance.instance_variable_get(:@compressor).ext
62
+ assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@compressor).content_type
63
+ assert_equal false, d.instance.force_path_style
64
+ assert_equal nil, d.instance.compute_checksums
65
+ assert_equal nil, d.instance.signature_version
66
+ assert_equal true, d.instance.check_bucket
67
+ assert_equal true, d.instance.check_object
68
+ end
69
+
70
+ def test_s3_endpoint_with_valid_endpoint
71
+ d = create_driver(CONFIG + 's3_endpoint riak-cs.example.com')
72
+ assert_equal 'riak-cs.example.com', d.instance.s3_endpoint
73
+ end
74
+
75
+ data('US West (Oregon)' => 's3-us-west-2.amazonaws.com',
76
+ 'EU (Frankfurt)' => 's3.eu-central-1.amazonaws.com',
77
+ 'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com')
78
+ def test_s3_endpoint_with_invalid_endpoint(endpoint)
79
+ assert_raise(Fluent::ConfigError, "s3_endpoint parameter is not supported, use s3_region instead. This parameter is for S3 compatible services") {
80
+ create_driver(CONFIG + "s3_endpoint #{endpoint}")
81
+ }
82
+ end
83
+
84
+ def test_configure_with_mime_type_json
85
+ conf = CONFIG.clone
86
+ conf << "\nstore_as json\n"
87
+ d = create_driver(conf)
88
+ assert_equal 'json', d.instance.instance_variable_get(:@compressor).ext
89
+ assert_equal 'application/json', d.instance.instance_variable_get(:@compressor).content_type
90
+ end
91
+
92
+ def test_configure_with_mime_type_text
93
+ conf = CONFIG.clone
94
+ conf << "\nstore_as text\n"
95
+ d = create_driver(conf)
96
+ assert_equal 'txt', d.instance.instance_variable_get(:@compressor).ext
97
+ assert_equal 'text/plain', d.instance.instance_variable_get(:@compressor).content_type
98
+ end
99
+
100
+ def test_configure_with_mime_type_lzo
101
+ conf = CONFIG.clone
102
+ conf << "\nstore_as lzo\n"
103
+ d = create_driver(conf)
104
+ assert_equal 'lzo', d.instance.instance_variable_get(:@compressor).ext
105
+ assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@compressor).content_type
106
+ rescue => e
107
+ # TODO: replace code with disable lzop command
108
+ assert(e.is_a?(Fluent::ConfigError))
109
+ end
110
+
111
+ def test_configure_with_path_style
112
+ conf = CONFIG.clone
113
+ conf << "\nforce_path_style true\n"
114
+ d = create_driver(conf)
115
+ assert d.instance.force_path_style
116
+ end
117
+
118
+ def test_configure_with_compute_checksums
119
+ conf = CONFIG.clone
120
+ conf << "\ncompute_checksums false\n"
121
+ d = create_driver(conf)
122
+ assert_equal false, d.instance.compute_checksums
123
+ end
124
+
125
+ def test_configure_with_hex_random_length
126
+ conf = CONFIG.clone
127
+ assert_raise Fluent::ConfigError do
128
+ create_driver(conf + "\nhex_random_length 17\n")
129
+ end
130
+ assert_nothing_raised do
131
+ create_driver(conf + "\nhex_random_length 16\n")
132
+ end
133
+ end
134
+
135
+ def test_configure_with_no_check_on_s3
136
+ conf = CONFIG.clone
137
+ conf << "\ncheck_bucket false\ncheck_object false\n"
138
+ d = create_driver(conf)
139
+ assert_equal false, d.instance.check_bucket
140
+ assert_equal false, d.instance.check_object
141
+ end
142
+
143
+ def test_configure_with_grant
144
+ conf = CONFIG.clone
145
+ conf << "\grant_full_control id='0123456789'\ngrant_read id='1234567890'\ngrant_read_acp id='2345678901'\ngrant_write_acp id='3456789012'\n"
146
+ d = create_driver(conf)
147
+ assert_equal "id='0123456789'", d.instance.grant_full_control
148
+ assert_equal "id='1234567890'", d.instance.grant_read
149
+ assert_equal "id='2345678901'", d.instance.grant_read_acp
150
+ assert_equal "id='3456789012'", d.instance.grant_write_acp
151
+ end
152
+
153
+ def test_format
154
+ d = create_driver
155
+
156
+ time = event_time("2011-01-02 13:14:15 UTC")
157
+ d.run(default_tag: "test") do
158
+ d.feed(time, { "a" => 1 })
159
+ d.feed(time, { "a" => 2 })
160
+ end
161
+ expected = [
162
+ %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n],
163
+ %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
164
+ ]
165
+ assert_equal(expected, d.formatted)
166
+ end
167
+
168
+ def test_format_included_tag_and_time
169
+ config = [CONFIG, 'include_tag_key true', 'include_time_key true'].join("\n")
170
+ d = create_driver(config)
171
+
172
+ time = event_time("2011-01-02 13:14:15 UTC")
173
+ d.run(default_tag: "test") do
174
+ d.feed(time, { "a" => 1 })
175
+ d.feed(time, { "a" => 2 })
176
+ end
177
+ expected = [
178
+ %[2011-01-02T13:14:15Z\ttest\t{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n],
179
+ %[2011-01-02T13:14:15Z\ttest\t{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
180
+ ]
181
+ assert_equal(expected, d.formatted)
182
+ end
183
+
184
+ def test_format_with_format_ltsv
185
+ config = [CONFIG, 'format ltsv'].join("\n")
186
+ d = create_driver(config)
187
+
188
+ time = event_time("2011-01-02 13:14:15 UTC")
189
+ d.run(default_tag: "test") do
190
+ d.feed(time, {"a"=>1, "b"=>1})
191
+ d.feed(time, {"a"=>2, "b"=>2})
192
+ end
193
+ expected = [
194
+ %[a:1\tb:1\n],
195
+ %[a:2\tb:2\n]
196
+ ]
197
+ assert_equal(expected, d.formatted)
198
+ end
199
+
200
+ def test_format_with_format_json
201
+ config = [CONFIG, 'format json'].join("\n")
202
+ d = create_driver(config)
203
+
204
+ time = event_time("2011-01-02 13:14:15 UTC")
205
+ d.run(default_tag: "test") do
206
+ d.feed(time, {"a"=>1})
207
+ d.feed(time, {"a"=>2})
208
+ end
209
+ expected = [
210
+ %[{"a":1}\n],
211
+ %[{"a":2}\n]
212
+ ]
213
+ assert_equal(expected, d.formatted)
214
+ end
215
+
216
+ def test_format_with_format_json_included_tag
217
+ config = [CONFIG, 'format json', 'include_tag_key true'].join("\n")
218
+ d = create_driver(config)
219
+
220
+ time = event_time("2011-01-02 13:14:15 UTC")
221
+ d.run(default_tag: "test") do
222
+ d.feed(time, {"a"=>1})
223
+ d.feed(time, {"a"=>2})
224
+ end
225
+ expected = [
226
+ %[{"a":1,"tag":"test"}\n],
227
+ %[{"a":2,"tag":"test"}\n]
228
+ ]
229
+ assert_equal(expected, d.formatted)
230
+ end
231
+
232
+ def test_format_with_format_json_included_time
233
+ config = [CONFIG, 'format json', 'include_time_key true'].join("\n")
234
+ d = create_driver(config)
235
+
236
+ time = event_time("2011-01-02 13:14:15 UTC")
237
+ d.run(default_tag: "test") do
238
+ d.feed(time, {"a"=>1})
239
+ d.feed(time, {"a"=>2})
240
+ end
241
+ expected = [
242
+ %[{"a":1,"time":"2011-01-02T13:14:15Z"}\n],
243
+ %[{"a":2,"time":"2011-01-02T13:14:15Z"}\n]
244
+ ]
245
+ assert_equal(expected, d.formatted)
246
+ end
247
+
248
+ def test_format_with_format_json_included_tag_and_time
249
+ config = [CONFIG, 'format json', 'include_tag_key true', 'include_time_key true'].join("\n")
250
+ d = create_driver(config)
251
+
252
+ time = event_time("2011-01-02 13:14:15 UTC")
253
+ d.run(default_tag: "test") do
254
+ d.feed(time, {"a"=>1})
255
+ d.feed(time, {"a"=>2})
256
+ end
257
+ expected = [
258
+ %[{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n],
259
+ %[{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
260
+ ]
261
+ assert_equal(expected, d.formatted)
262
+ end
263
+
264
+ CONFIG_TIME_SLICE = <<EOC
265
+ aws_key_id test_key_id
266
+ aws_sec_key test_sec_key
267
+ s3_bucket test_bucket
268
+ s3_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
269
+ time_slice_format %Y%m%d-%H
270
+ path log
271
+ utc
272
+ buffer_type memory
273
+ @log_level debug
274
+ check_bucket true
275
+ check_object true
276
+ EOC
277
+
278
+ def create_time_sliced_driver(conf = CONFIG_TIME_SLICE)
279
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::S3Output) do
280
+ def format(tag, time, record)
281
+ super
282
+ end
283
+
284
+ def write(chunk)
285
+ super
286
+ end
287
+
288
+ private
289
+
290
+ def check_apikeys
291
+ end
292
+ end.configure(conf)
293
+ end
294
+
295
+ def test_write_with_hardened_s3_policy
296
+ # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
297
+ setup_mocks_hardened_policy
298
+ s3_local_file_path = "/tmp/s3-test.txt"
299
+ # @s3_object_key_format will be hard_coded with timestamp only,
300
+ # as in this case, it will not check for object existence, not even bucker existence
301
+ # check_bukcet and check_object both of this config parameter should be false
302
+ # @s3_object_key_format = "%{path}/%{time_slice}_%{hms_slice}.%{file_extension}"
303
+ setup_s3_object_mocks_hardened_policy()
304
+
305
+ # We must use TimeSlicedOutputTestDriver instead of BufferedOutputTestDriver,
306
+ # to make assertions on chunks' keys
307
+ config = CONFIG_TIME_SLICE.gsub(/check_object true/, "check_object false\n")
308
+ config = config.gsub(/check_bucket true/, "check_bucket false\n")
309
+ d = create_time_sliced_driver(config)
310
+
311
+ time = event_time("2011-01-02 13:14:15 UTC")
312
+ d.run(default_tag: "test") do
313
+ d.feed(time, {"a"=>1})
314
+ d.feed(time, {"a"=>2})
315
+ end
316
+
317
+ Zlib::GzipReader.open(s3_local_file_path) do |gz|
318
+ data = gz.read
319
+ assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
320
+ %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
321
+ data
322
+ end
323
+ FileUtils.rm_f(s3_local_file_path)
324
+ end
325
+
326
+ def test_write_with_custom_s3_object_key_format
327
+ # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
328
+ setup_mocks(true)
329
+ s3_local_file_path = "/tmp/s3-test.txt"
330
+ setup_s3_object_mocks(s3_local_file_path: s3_local_file_path)
331
+
332
+ d = create_time_sliced_driver
333
+
334
+ time = event_time("2011-01-02 13:14:15 UTC")
335
+ d.run(default_tag: "test") do
336
+ d.feed(time, {"a"=>1})
337
+ d.feed(time, {"a"=>2})
338
+ end
339
+
340
+ Zlib::GzipReader.open(s3_local_file_path) do |gz|
341
+ data = gz.read
342
+ assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
343
+ %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
344
+ data
345
+ end
346
+ FileUtils.rm_f(s3_local_file_path)
347
+ end
348
+
349
+ def test_write_with_custom_s3_object_key_format_containing_uuid_flush_placeholder
350
+
351
+ # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
352
+ setup_mocks(true)
353
+
354
+ uuid = "5755e23f-9b54-42d8-8818-2ea38c6f279e"
355
+ stub(::SecureRandom).uuid{ uuid }
356
+
357
+ s3_local_file_path = "/tmp/s3-test.txt"
358
+ s3path = "log/events/ts=20110102-13/events_0-#{uuid}.gz"
359
+ setup_s3_object_mocks(s3_local_file_path: s3_local_file_path, s3path: s3path)
360
+
361
+ config = CONFIG_TIME_SLICE.gsub(/%{hostname}/,"%{uuid_flush}")
362
+ d = create_time_sliced_driver(config)
363
+
364
+ time = event_time("2011-01-02 13:14:15 UTC")
365
+ d.run(default_tag: "test") do
366
+ d.feed(time, {"a"=>1})
367
+ d.feed(time, {"a"=>2})
368
+ end
369
+
370
+ Zlib::GzipReader.open(s3_local_file_path) do |gz|
371
+ data = gz.read
372
+ assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
373
+ %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
374
+ data
375
+ end
376
+ FileUtils.rm_f(s3_local_file_path)
377
+ Dir.glob('tmp/*').each {|file| FileUtils.rm_f(file) }
378
+ end
379
+
380
+ # ToDo: need to test hex_random does not change on retry, but it is difficult with
381
+ # the current fluentd test helper because it does not provide a way to run with the same chunks
382
+ def test_write_with_custom_s3_object_key_format_containing_hex_random_placeholder
383
+ unique_hex = "5226c3c4fb3d49b15226c3c4fb3d49b1"
384
+ hex_random = unique_hex.reverse[0...5]
385
+
386
+ config = CONFIG_TIME_SLICE.gsub(/%{hostname}/,"%{hex_random}") << "\nhex_random_length #{hex_random.length}"
387
+ config = config.gsub(/buffer_type memory/, "buffer_type file\nbuffer_path test/tmp/buf")
388
+
389
+ # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
390
+ setup_mocks(true)
391
+
392
+ s3path = "log/events/ts=20110102-13/events_0-#{hex_random}.gz"
393
+ s3_local_file_path = "/tmp/s3-test.txt"
394
+ setup_s3_object_mocks(s3_local_file_path: s3_local_file_path, s3path: s3path)
395
+
396
+ d = create_time_sliced_driver(config)
397
+ stub(Fluent::UniqueId).hex(anything) { unique_hex }
398
+
399
+ time = event_time("2011-01-02 13:14:15 UTC")
400
+ d.run(default_tag: "test") do
401
+ d.feed(time, {"a"=>1})
402
+ d.feed(time, {"a"=>2})
403
+ end
404
+
405
+ Zlib::GzipReader.open(s3_local_file_path) do |gz|
406
+ data = gz.read
407
+ assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
408
+ %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
409
+ data
410
+ end
411
+ FileUtils.rm_f(s3_local_file_path)
412
+ end
413
+
414
+ class MockResponse
415
+ attr_reader :data
416
+
417
+ def initialize(data)
418
+ @data = data
419
+ end
420
+ end
421
+
422
+ def setup_mocks(exists_return = false)
423
+ @s3_client = stub(Aws::S3::Client.new(stub_responses: true))
424
+ stub(@s3_client).config { OpenStruct.new({region: "us-east-1"}) }
425
+ # aws-sdk-s3 calls Client#put_object inside Object#put
426
+ mock(@s3_client).put_object(anything).at_least(0) { MockResponse.new({}) }
427
+ mock(Aws::S3::Client).new(anything).at_least(0) { @s3_client }
428
+ @s3_resource = mock(Aws::S3::Resource.new(client: @s3_client))
429
+ mock(Aws::S3::Resource).new(client: @s3_client) { @s3_resource }
430
+ @s3_bucket = mock(Aws::S3::Bucket.new(name: "test",
431
+ client: @s3_client))
432
+ @s3_bucket.exists? { exists_return }
433
+ @s3_object = mock(Aws::S3::Object.new(bucket_name: "test_bucket",
434
+ key: "test",
435
+ client: @s3_client))
436
+ @s3_object.exists?.at_least(0) { false }
437
+ @s3_bucket.object(anything).at_least(0) { @s3_object }
438
+ @s3_resource.bucket(anything) { @s3_bucket }
439
+ end
440
+
441
+ def setup_s3_object_mocks(params = {})
442
+ s3path = params[:s3path] || "log/events/ts=20110102-13/events_0-#{Socket.gethostname}.gz"
443
+ s3_local_file_path = params[:s3_local_file_path] || "/tmp/s3-test.txt"
444
+
445
+ # Assert content of event logs which are being sent to S3
446
+ s3obj = stub(Aws::S3::Object.new(bucket_name: "test_bucket",
447
+ key: "test",
448
+ client: @s3_client))
449
+ s3obj.exists? { false }
450
+
451
+ tempfile = File.new(s3_local_file_path, "w")
452
+ stub(Tempfile).new("s3-") { tempfile }
453
+ s3obj.put(body: tempfile,
454
+ content_type: "application/x-gzip",
455
+ storage_class: "STANDARD")
456
+
457
+ @s3_bucket.object(s3path) { s3obj }
458
+ end
459
+
460
+ def setup_mocks_hardened_policy()
461
+ @s3_client = stub(Aws::S3::Client.new(:stub_responses => true))
462
+ stub(@s3_client).config { OpenStruct.new({region: "us-east-1"}) }
463
+ mock(@s3_client).put_object(anything).at_least(0) { MockResponse.new({}) }
464
+ mock(Aws::S3::Client).new(anything).at_least(0) { @s3_client }
465
+ @s3_resource = mock(Aws::S3::Resource.new(:client => @s3_client))
466
+ mock(Aws::S3::Resource).new(:client => @s3_client) { @s3_resource }
467
+ @s3_bucket = mock(Aws::S3::Bucket.new(:name => "test",
468
+ :client => @s3_client))
469
+ @s3_object = mock(Aws::S3::Object.new(:bucket_name => "test_bucket",
470
+ :key => "test",
471
+ :client => @s3_client))
472
+ @s3_bucket.object(anything).at_least(0) { @s3_object }
473
+ @s3_resource.bucket(anything) { @s3_bucket }
474
+ end
475
+
476
+ def setup_s3_object_mocks_hardened_policy(params = {})
477
+ s3_local_file_path = params[:s3_local_file_path] || "/tmp/s3-test.txt"
478
+
479
+ # Assert content of event logs which are being sent to S3
480
+ s3obj = stub(Aws::S3::Object.new(:bucket_name => "test_bucket",
481
+ :key => "test",
482
+ :client => @s3_client))
483
+
484
+ tempfile = File.new(s3_local_file_path, "w")
485
+ stub(Tempfile).new("s3-") { tempfile }
486
+ s3obj.put(:body => tempfile,
487
+ :content_type => "application/x-gzip",
488
+ :storage_class => "STANDARD")
489
+ end
490
+
491
+ def test_auto_create_bucket_false_with_non_existence_bucket
492
+ setup_mocks
493
+
494
+ config = CONFIG_TIME_SLICE + 'auto_create_bucket false'
495
+ d = create_time_sliced_driver(config)
496
+ assert_raise(RuntimeError, "The specified bucket does not exist: bucket = test_bucket") {
497
+ d.run {}
498
+ }
499
+ end
500
+
501
+ def test_auto_create_bucket_true_with_non_existence_bucket
502
+ setup_mocks
503
+ @s3_resource.create_bucket(bucket: "test_bucket")
504
+
505
+ config = CONFIG_TIME_SLICE + 'auto_create_bucket true'
506
+ d = create_time_sliced_driver(config)
507
+ assert_nothing_raised { d.run {} }
508
+ end
509
+
510
+ def test_credentials
511
+ d = create_time_sliced_driver
512
+ assert_nothing_raised { d.run {} }
513
+ client = d.instance.instance_variable_get(:@s3).client
514
+ credentials = client.config.credentials
515
+ assert_instance_of(Aws::Credentials, credentials)
516
+ end
517
+
518
+ def test_assume_role_credentials
519
+ expected_credentials = Aws::Credentials.new("test_key", "test_secret")
520
+ mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
521
+ role_session_name: "test_session",
522
+ client: anything){
523
+ expected_credentials
524
+ }
525
+ config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
526
+ config += %[
527
+ <assume_role_credentials>
528
+ role_arn test_arn
529
+ role_session_name test_session
530
+ </assume_role_credentials>
531
+ ]
532
+ d = create_time_sliced_driver(config)
533
+ assert_nothing_raised { d.run {} }
534
+ client = d.instance.instance_variable_get(:@s3).client
535
+ credentials = client.config.credentials
536
+ assert_equal(expected_credentials, credentials)
537
+ end
538
+
539
+ def test_assume_role_credentials_with_region
540
+ expected_credentials = Aws::Credentials.new("test_key", "test_secret")
541
+ sts_client = Aws::STS::Client.new(region: 'ap-northeast-1')
542
+ mock(Aws::STS::Client).new(region: 'ap-northeast-1'){ sts_client }
543
+ mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
544
+ role_session_name: "test_session",
545
+ client: sts_client){
546
+ expected_credentials
547
+ }
548
+ config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
549
+ config += %[
550
+ s3_region ap-northeast-1
551
+ <assume_role_credentials>
552
+ role_arn test_arn
553
+ role_session_name test_session
554
+ </assume_role_credentials>
555
+ ]
556
+ d = create_time_sliced_driver(config)
557
+ assert_nothing_raised { d.run {} }
558
+ client = d.instance.instance_variable_get(:@s3).client
559
+ credentials = client.config.credentials
560
+ assert_equal(expected_credentials, credentials)
561
+ end
562
+
563
+ def test_web_identity_credentials
564
+ expected_credentials = Aws::Credentials.new("test_key", "test_secret")
565
+ mock(Aws::AssumeRoleWebIdentityCredentials).new(
566
+ role_arn: "test_arn",
567
+ role_session_name: "test_session",
568
+ web_identity_token_file: "test_file",
569
+ client: anything
570
+ ){
571
+ expected_credentials
572
+ }
573
+
574
+ config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
575
+ config += %[
576
+ <web_identity_credentials>
577
+ role_arn test_arn
578
+ role_session_name test_session
579
+ web_identity_token_file test_file
580
+ </web_identity_credentials>
581
+ ]
582
+ d = create_time_sliced_driver(config)
583
+ assert_nothing_raised { d.run {} }
584
+ client = d.instance.instance_variable_get(:@s3).client
585
+ credentials = client.config.credentials
586
+ assert_equal(expected_credentials, credentials)
587
+ end
588
+
589
+ def test_web_identity_credentials_with_sts_region
590
+ expected_credentials = Aws::Credentials.new("test_key", "test_secret")
591
+ sts_client = Aws::STS::Client.new(region: 'us-east-1')
592
+ mock(Aws::STS::Client).new(region: 'us-east-1'){ sts_client }
593
+ mock(Aws::AssumeRoleWebIdentityCredentials).new(
594
+ role_arn: "test_arn",
595
+ role_session_name: "test_session",
596
+ web_identity_token_file: "test_file",
597
+ client: sts_client
598
+ ){
599
+ expected_credentials
600
+ }
601
+
602
+ config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
603
+ config += %[
604
+ s3_region us-west-2
605
+ <web_identity_credentials>
606
+ role_arn test_arn
607
+ role_session_name test_session
608
+ web_identity_token_file test_file
609
+ sts_region us-east-1
610
+ </web_identity_credentials>
611
+ ]
612
+ d = create_time_sliced_driver(config)
613
+ assert_nothing_raised { d.run {} }
614
+ client = d.instance.instance_variable_get(:@s3).client
615
+ credentials = client.config.credentials
616
+ assert_equal(expected_credentials, credentials)
617
+ end
618
+
619
+ def test_instance_profile_credentials
620
+ expected_credentials = Aws::Credentials.new("test_key", "test_secret")
621
+ mock(Aws::InstanceProfileCredentials).new({}).returns(expected_credentials)
622
+ config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
623
+ config += %[
624
+ <instance_profile_credentials>
625
+ </instance_profile_credentials>
626
+ ]
627
+ d = create_time_sliced_driver(config)
628
+ assert_nothing_raised { d.run {} }
629
+ client = d.instance.instance_variable_get(:@s3).client
630
+ credentials = client.config.credentials
631
+ assert_equal(expected_credentials, credentials)
632
+ end
633
+
634
+ def test_ecs_credentials
635
+ ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] = "/credential_provider_version/credentials?id=task_UUID"
636
+
637
+ expected_credentials = Aws::Credentials.new("test_key", "test_secret")
638
+ mock(Aws::ECSCredentials).new({}).returns(expected_credentials)
639
+ config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
640
+ config += %[
641
+ <instance_profile_credentials>
642
+ </instance_profile_credentials>
643
+ ]
644
+ d = create_time_sliced_driver(config)
645
+ assert_nothing_raised { d.run {} }
646
+ client = d.instance.instance_variable_get(:@s3).client
647
+ credentials = client.config.credentials
648
+ assert_equal(expected_credentials, credentials)
649
+
650
+ ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] = nil
651
+ end
652
+
653
+ def test_instance_profile_credentials_aws_iam_retries
654
+ expected_credentials = Aws::Credentials.new("test_key", "test_secret")
655
+ mock(Aws::InstanceProfileCredentials).new({ retries: 10 }).returns(expected_credentials)
656
+ config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
657
+ config += %[
658
+ aws_iam_retries 10
659
+ ]
660
+ d = create_time_sliced_driver(config)
661
+ assert_nothing_raised { d.run {} }
662
+ client = d.instance.instance_variable_get(:@s3).client
663
+ credentials = client.config.credentials
664
+ assert_equal(expected_credentials, credentials)
665
+ end
666
+
667
+ def test_shared_credentials
668
+ expected_credentials = Aws::Credentials.new("test_key", "test_secret")
669
+ mock(Aws::SharedCredentials).new({}).returns(expected_credentials)
670
+ config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
671
+ config += %[
672
+ <shared_credentials>
673
+ </shared_credentials>
674
+ ]
675
+ d = create_time_sliced_driver(config)
676
+ assert_nothing_raised { d.run {} }
677
+ client = d.instance.instance_variable_get(:@s3).client
678
+ credentials = client.config.credentials
679
+ assert_equal(expected_credentials, credentials)
680
+ end
681
+
682
+ def test_signature_version
683
+ config = [CONFIG, 'signature_version s3'].join("\n")
684
+ d = create_driver(config)
685
+
686
+ signature_version = d.instance.instance_variable_get(:@signature_version)
687
+ assert_equal("s3", signature_version)
688
+ end
689
+
690
+ def test_warn_for_delay
691
+ setup_mocks(true)
692
+ s3_local_file_path = "/tmp/s3-test.txt"
693
+ setup_s3_object_mocks(s3_local_file_path: s3_local_file_path)
694
+
695
+ config = CONFIG_TIME_SLICE + 'warn_for_delay 1d'
696
+ d = create_time_sliced_driver(config)
697
+
698
+ delayed_time = event_time("2011-01-02 13:14:15 UTC")
699
+ now = delayed_time.to_i + 86000 + 1
700
+ d.instance.log.out.flush_logs = false
701
+ Timecop.freeze(Time.at(now)) do
702
+ d.run(default_tag: "test") do
703
+ d.feed(delayed_time, {"a"=>1})
704
+ d.feed(delayed_time, {"a"=>2})
705
+ end
706
+ end
707
+ logs = d.instance.log.out.logs
708
+ assert_true logs.any? {|log| log.include?('out_s3: delayed events were put') }
709
+ d.instance.log.out.flush_logs = true
710
+ d.instance.log.out.reset
711
+ FileUtils.rm_f(s3_local_file_path)
712
+ end
713
+ end