fluent-plugin-s3 1.3.1 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +4 -6
- data/ChangeLog +22 -0
- data/Gemfile +0 -2
- data/README.md +8 -3
- data/VERSION +1 -1
- data/lib/fluent/plugin/in_s3.rb +3 -3
- data/lib/fluent/plugin/out_s3.rb +42 -19
- data/test/test_out_s3.rb +163 -9
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 48c05ddd12cfd327c2e92032892887d9ec87b9276d8a6511ade65d0cb0a027e0
|
4
|
+
data.tar.gz: 4a2dc83ce85ced8e92f3f96bc1661b1678ff84fa5512c62584c60b93616f32db
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b607bcbd43ff07b0a1240647a17683a68918e448b6190dca09e56c38db92d0908189d4a5729654fe6a027b243b996040445c84b131b0327023dc226f0b9f1dc6
|
7
|
+
data.tar.gz: 8959c36b3045111088161e3634f4165b2d4b9420f16ac7bc62ccdbba4066cbc6fbf586c22f8b42ab639e21df6ab18823bfdd659fc553039ca67b7633011a1439
|
data/.travis.yml
CHANGED
@@ -1,11 +1,10 @@
|
|
1
1
|
language: ruby
|
2
2
|
|
3
3
|
rvm:
|
4
|
-
- 2.
|
5
|
-
- 2.
|
6
|
-
- 2.
|
7
|
-
- 2.
|
8
|
-
- 2.5.0
|
4
|
+
- 2.4
|
5
|
+
- 2.5
|
6
|
+
- 2.6
|
7
|
+
- 2.7
|
9
8
|
- ruby-head
|
10
9
|
|
11
10
|
gemfile:
|
@@ -14,7 +13,6 @@ gemfile:
|
|
14
13
|
branches:
|
15
14
|
only:
|
16
15
|
- master
|
17
|
-
- v0.12
|
18
16
|
|
19
17
|
before_install: gem update bundler
|
20
18
|
script: bundle exec rake test
|
data/ChangeLog
CHANGED
@@ -1,3 +1,25 @@
|
|
1
|
+
Release 1.5.0 - 2020/12/18
|
2
|
+
|
3
|
+
* out_s3: Fix IAM credentials handling. Prefer assume_role_credentials and region parameter bug.
|
4
|
+
|
5
|
+
Release 1.4.0 - 2020/08/02
|
6
|
+
|
7
|
+
* Remove uuidtools dependency
|
8
|
+
* in_s3: Add error info to polloing retry log
|
9
|
+
|
10
|
+
Release 1.3.4 - 2020/07/07
|
11
|
+
|
12
|
+
* Add sts_http_proxy and sts_endpoint_url to assume_role_credentials
|
13
|
+
|
14
|
+
Release 1.3.3 - 2020/06/25
|
15
|
+
|
16
|
+
* Allow fips/gov included endpoint
|
17
|
+
* Support sts_region parameter
|
18
|
+
|
19
|
+
Release 1.3.2 - 2020/05/18
|
20
|
+
|
21
|
+
* out_s3: Show warning message for object conflict case.
|
22
|
+
|
1
23
|
Release 1.3.1 - 2020/04/15
|
2
24
|
|
3
25
|
* out_s3: Support S3 Dual-Stack Endpoints in output plugin via enable_dual_stack parameter
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -31,8 +31,13 @@ We must setup SQS queue and S3 event notification before use this plugin.
|
|
31
31
|
|
32
32
|
Simply use RubyGems:
|
33
33
|
|
34
|
-
|
35
|
-
$ gem install fluent-plugin-s3
|
34
|
+
# install latest version
|
35
|
+
$ gem install fluent-plugin-s3 --no-document # for fluentd v1.0 or later
|
36
|
+
# If you need to install specifiv version, use -v option
|
37
|
+
$ gem install fluent-plugin-s3 -v 1.3.0 --no-document
|
38
|
+
# For v0.12. This is for old v0.12 users. Don't use v0.12 for new deployment
|
39
|
+
$ gem install fluent-plugin-s3 -v "~> 0.8" --no-document # for fluentd v0.12
|
40
|
+
|
36
41
|
|
37
42
|
## Configuration: credentials
|
38
43
|
|
@@ -337,7 +342,7 @@ E.g., "logs/" in the example configuration above.
|
|
337
342
|
time-slice in text that are formatted with **time_slice_format**.
|
338
343
|
* %{index} is the sequential number starts from 0, increments when multiple files are uploaded to S3 in the same time slice.
|
339
344
|
* %{file_extension} depends on **store_as** parameter.
|
340
|
-
* %{uuid_flush} a uuid that is replaced everytime the buffer will be flushed.
|
345
|
+
* %{uuid_flush} a uuid that is replaced everytime the buffer will be flushed.
|
341
346
|
* %{hostname} is replaced with `Socket.gethostname` result.
|
342
347
|
* %{hex_random} a random hex string that is replaced for each buffer chunk, not
|
343
348
|
assured to be unique. This is used to follow a way of performance tuning, `Add
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.5.0
|
data/lib/fluent/plugin/in_s3.rb
CHANGED
@@ -118,11 +118,11 @@ module Fluent::Plugin
|
|
118
118
|
def configure(conf)
|
119
119
|
super
|
120
120
|
|
121
|
-
if @s3_endpoint && @s3_endpoint.end_with?('amazonaws.com')
|
121
|
+
if @s3_endpoint && (@s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) })
|
122
122
|
raise Fluent::ConfigError, "s3_endpoint parameter is not supported for S3, use s3_region instead. This parameter is for S3 compatible services"
|
123
123
|
end
|
124
124
|
|
125
|
-
if @sqs.endpoint && @sqs.endpoint.end_with?('amazonaws.com')
|
125
|
+
if @sqs.endpoint && (@sqs.endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @sqs.endpoint.include?(e) })
|
126
126
|
raise Fluent::ConfigError, "sqs/endpoint parameter is not supported for SQS, use s3_region instead. This parameter is for SQS compatible services"
|
127
127
|
end
|
128
128
|
|
@@ -192,7 +192,7 @@ module Fluent::Plugin
|
|
192
192
|
end
|
193
193
|
end
|
194
194
|
rescue => e
|
195
|
-
log.warn("SQS Polling Failed. Retry in #{@sqs.retry_error_interval} seconds")
|
195
|
+
log.warn("SQS Polling Failed. Retry in #{@sqs.retry_error_interval} seconds", error: e)
|
196
196
|
sleep(@sqs.retry_error_interval)
|
197
197
|
retry
|
198
198
|
end
|
data/lib/fluent/plugin/out_s3.rb
CHANGED
@@ -5,6 +5,7 @@ require 'aws-sdk-s3'
|
|
5
5
|
require 'zlib'
|
6
6
|
require 'time'
|
7
7
|
require 'tempfile'
|
8
|
+
require 'securerandom'
|
8
9
|
|
9
10
|
module Fluent::Plugin
|
10
11
|
class S3Output < Output
|
@@ -39,6 +40,12 @@ module Fluent::Plugin
|
|
39
40
|
config_param :duration_seconds, :integer, default: nil
|
40
41
|
desc "A unique identifier that is used by third parties when assuming roles in their customers' accounts."
|
41
42
|
config_param :external_id, :string, default: nil, secret: true
|
43
|
+
desc "The region of the STS endpoint to use."
|
44
|
+
config_param :sts_region, :string, default: nil
|
45
|
+
desc "A http proxy url for requests to aws sts service"
|
46
|
+
config_param :sts_http_proxy, :string, default: nil, secret: true
|
47
|
+
desc "A url for a regional sts api endpoint, the default is global"
|
48
|
+
config_param :sts_endpoint_url, :string, default: nil
|
42
49
|
end
|
43
50
|
# See the following link for additional params that could be added:
|
44
51
|
# https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html#assume_role_with_web_identity-instance_method
|
@@ -53,6 +60,8 @@ module Fluent::Plugin
|
|
53
60
|
config_param :policy, :string, default: nil
|
54
61
|
desc "The duration, in seconds, of the role session (900-43200)"
|
55
62
|
config_param :duration_seconds, :integer, default: nil
|
63
|
+
desc "The region of the STS endpoint to use."
|
64
|
+
config_param :sts_region, :string, default: nil
|
56
65
|
end
|
57
66
|
config_section :instance_profile_credentials, multi: false do
|
58
67
|
desc "Number of times to retry when retrieving credentials"
|
@@ -171,7 +180,7 @@ module Fluent::Plugin
|
|
171
180
|
|
172
181
|
Aws.use_bundled_cert! if @use_bundled_cert
|
173
182
|
|
174
|
-
if @s3_endpoint && @s3_endpoint.end_with?('amazonaws.com')
|
183
|
+
if @s3_endpoint && (@s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) })
|
175
184
|
raise Fluent::ConfigError, "s3_endpoint parameter is not supported for S3, use s3_region instead. This parameter is for S3 compatible services"
|
176
185
|
end
|
177
186
|
|
@@ -209,6 +218,8 @@ module Fluent::Plugin
|
|
209
218
|
end
|
210
219
|
end
|
211
220
|
|
221
|
+
check_s3_path_safety(conf)
|
222
|
+
|
212
223
|
# For backward compatibility
|
213
224
|
# TODO: Remove time_slice_format when end of support compat_parameters
|
214
225
|
@configured_time_slice_format = conf['time_slice_format']
|
@@ -374,7 +385,7 @@ module Fluent::Plugin
|
|
374
385
|
end
|
375
386
|
|
376
387
|
def uuid_random
|
377
|
-
|
388
|
+
SecureRandom.uuid
|
378
389
|
end
|
379
390
|
|
380
391
|
# This is stolen from Fluentd
|
@@ -431,17 +442,6 @@ module Fluent::Plugin
|
|
431
442
|
}
|
432
443
|
|
433
444
|
if @s3_object_key_format.include?('%{uuid_flush}')
|
434
|
-
# test uuidtools works or not
|
435
|
-
begin
|
436
|
-
require 'uuidtools'
|
437
|
-
rescue LoadError
|
438
|
-
raise Fluent::ConfigError, "uuidtools gem not found. Install uuidtools gem first"
|
439
|
-
end
|
440
|
-
begin
|
441
|
-
uuid_random
|
442
|
-
rescue => e
|
443
|
-
raise Fluent::ConfigError, "Generating uuid doesn't work. Can't use %{uuid_flush} on this environment. #{e}"
|
444
|
-
end
|
445
445
|
@uuid_flush_enabled = true
|
446
446
|
end
|
447
447
|
|
@@ -451,6 +451,16 @@ module Fluent::Plugin
|
|
451
451
|
}
|
452
452
|
end
|
453
453
|
|
454
|
+
def check_s3_path_safety(conf)
|
455
|
+
unless conf.has_key?('s3_object_key_format')
|
456
|
+
log.warn "The default value of s3_object_key_format will use ${chunk_id} instead of %{index} to avoid object conflict in v2"
|
457
|
+
end
|
458
|
+
|
459
|
+
if (@buffer_config.flush_thread_count > 1) && ['${chunk_id}', '%{uuid_flush}'].none? { |key| @s3_object_key_format.include?(key) }
|
460
|
+
log.warn "No ${chunk_id} or %{uuid_flush} in s3_object_key_format with multiple flush threads. Recommend to set ${chunk_id} or %{uuid_flush} to avoid data lost by object conflict"
|
461
|
+
end
|
462
|
+
end
|
463
|
+
|
454
464
|
def check_apikeys
|
455
465
|
@bucket.objects(prefix: @path, :max_keys => 1).first
|
456
466
|
rescue Aws::S3::Errors::NoSuchBucket
|
@@ -463,20 +473,31 @@ module Fluent::Plugin
|
|
463
473
|
options = {}
|
464
474
|
credentials_options = {}
|
465
475
|
case
|
466
|
-
when @aws_key_id && @aws_sec_key
|
467
|
-
options[:access_key_id] = @aws_key_id
|
468
|
-
options[:secret_access_key] = @aws_sec_key
|
469
476
|
when @assume_role_credentials
|
470
477
|
c = @assume_role_credentials
|
478
|
+
iam_user_credentials = @aws_key_id && @aws_sec_key ? Aws::Credentials.new(@aws_key_id, @aws_sec_key) : nil
|
479
|
+
region = c.sts_region || @s3_region
|
471
480
|
credentials_options[:role_arn] = c.role_arn
|
472
481
|
credentials_options[:role_session_name] = c.role_session_name
|
473
482
|
credentials_options[:policy] = c.policy if c.policy
|
474
483
|
credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
|
475
484
|
credentials_options[:external_id] = c.external_id if c.external_id
|
476
|
-
if
|
477
|
-
|
485
|
+
credentials_options[:sts_endpoint_url] = c.sts_endpoint_url if c.sts_endpoint_url
|
486
|
+
credentials_options[:sts_http_proxy] = c.sts_http_proxy if c.sts_http_proxy
|
487
|
+
if c.sts_http_proxy && c.sts_endpoint_url
|
488
|
+
credentials_options[:client] = Aws::STS::Client.new(region: region, http_proxy: c.sts_http_proxy, endpoint: c.sts_endpoint_url, credentials: iam_user_credentials)
|
489
|
+
elsif c.sts_http_proxy
|
490
|
+
credentials_options[:client] = Aws::STS::Client.new(region: region, http_proxy: c.sts_http_proxy, credentials: iam_user_credentials)
|
491
|
+
elsif c.sts_endpoint_url
|
492
|
+
credentials_options[:client] = Aws::STS::Client.new(region: region, endpoint: c.sts_endpoint_url, credentials: iam_user_credentials)
|
493
|
+
else
|
494
|
+
credentials_options[:client] = Aws::STS::Client.new(region: region, credentials: iam_user_credentials)
|
478
495
|
end
|
496
|
+
|
479
497
|
options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
|
498
|
+
when @aws_key_id && @aws_sec_key
|
499
|
+
options[:access_key_id] = @aws_key_id
|
500
|
+
options[:secret_access_key] = @aws_sec_key
|
480
501
|
when @web_identity_credentials
|
481
502
|
c = @web_identity_credentials
|
482
503
|
credentials_options[:role_arn] = c.role_arn
|
@@ -484,7 +505,9 @@ module Fluent::Plugin
|
|
484
505
|
credentials_options[:web_identity_token_file] = c.web_identity_token_file
|
485
506
|
credentials_options[:policy] = c.policy if c.policy
|
486
507
|
credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
|
487
|
-
if
|
508
|
+
if c.sts_region
|
509
|
+
credentials_options[:client] = Aws::STS::Client.new(:region => c.sts_region)
|
510
|
+
elsif @s3_region
|
488
511
|
credentials_options[:client] = Aws::STS::Client.new(:region => @s3_region)
|
489
512
|
end
|
490
513
|
options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
|
data/test/test_out_s3.rb
CHANGED
@@ -9,7 +9,6 @@ require 'test/unit/rr'
|
|
9
9
|
require 'zlib'
|
10
10
|
require 'fileutils'
|
11
11
|
require 'timecop'
|
12
|
-
require 'uuidtools'
|
13
12
|
require 'ostruct'
|
14
13
|
|
15
14
|
include Fluent::Test::Helpers
|
@@ -349,17 +348,11 @@ EOC
|
|
349
348
|
|
350
349
|
def test_write_with_custom_s3_object_key_format_containing_uuid_flush_placeholder
|
351
350
|
|
352
|
-
begin
|
353
|
-
require 'uuidtools'
|
354
|
-
rescue LoadError
|
355
|
-
pend("uuidtools not found. skip this test")
|
356
|
-
end
|
357
|
-
|
358
351
|
# Partial mock the S3Bucket, not to make an actual connection to Amazon S3
|
359
352
|
setup_mocks(true)
|
360
353
|
|
361
354
|
uuid = "5755e23f-9b54-42d8-8818-2ea38c6f279e"
|
362
|
-
stub(::
|
355
|
+
stub(::SecureRandom).uuid{ uuid }
|
363
356
|
|
364
357
|
s3_local_file_path = "/tmp/s3-test.txt"
|
365
358
|
s3path = "log/events/ts=20110102-13/events_0-#{uuid}.gz"
|
@@ -546,7 +539,7 @@ EOC
|
|
546
539
|
def test_assume_role_credentials_with_region
|
547
540
|
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
548
541
|
sts_client = Aws::STS::Client.new(region: 'ap-northeast-1')
|
549
|
-
mock(Aws::STS::Client).new(region: 'ap-northeast-1'){ sts_client }
|
542
|
+
mock(Aws::STS::Client).new(region: 'ap-northeast-1', credentials: nil){ sts_client }
|
550
543
|
mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
|
551
544
|
role_session_name: "test_session",
|
552
545
|
client: sts_client){
|
@@ -567,6 +560,137 @@ EOC
|
|
567
560
|
assert_equal(expected_credentials, credentials)
|
568
561
|
end
|
569
562
|
|
563
|
+
def test_assume_role_with_iam_credentials
|
564
|
+
expected_credentials = Aws::Credentials.new("test_key_id", "test_sec_key")
|
565
|
+
sts_client = Aws::STS::Client.new(region: 'ap-northeast-1', credentials: expected_credentials)
|
566
|
+
mock(Aws::Credentials).new("test_key_id", "test_sec_key") { expected_credentials }
|
567
|
+
mock(Aws::STS::Client).new(region: 'ap-northeast-1', credentials: expected_credentials){ sts_client }
|
568
|
+
mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
|
569
|
+
role_session_name: "test_session",
|
570
|
+
client: sts_client){
|
571
|
+
expected_credentials
|
572
|
+
}
|
573
|
+
config = CONFIG_TIME_SLICE
|
574
|
+
config += %[
|
575
|
+
s3_region ap-northeast-1
|
576
|
+
|
577
|
+
<assume_role_credentials>
|
578
|
+
role_arn test_arn
|
579
|
+
role_session_name test_session
|
580
|
+
</assume_role_credentials>
|
581
|
+
]
|
582
|
+
d = create_time_sliced_driver(config)
|
583
|
+
assert_nothing_raised { d.run {} }
|
584
|
+
client = d.instance.instance_variable_get(:@s3).client
|
585
|
+
credentials = client.config.credentials
|
586
|
+
assert_equal(expected_credentials, credentials)
|
587
|
+
end
|
588
|
+
|
589
|
+
def test_assume_role_credentials_with_region_and_sts_http_proxy
|
590
|
+
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
591
|
+
expected_region = "ap-northeast-1"
|
592
|
+
expected_sts_http_proxy = 'http://example.com'
|
593
|
+
sts_client = Aws::STS::Client.new(region: expected_region, http_proxy: expected_sts_http_proxy)
|
594
|
+
mock(Aws::STS::Client).new(region:expected_region, http_proxy: expected_sts_http_proxy, credentials: nil){ sts_client }
|
595
|
+
mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
|
596
|
+
role_session_name: "test_session",
|
597
|
+
client: sts_client,
|
598
|
+
sts_http_proxy: expected_sts_http_proxy){
|
599
|
+
expected_credentials
|
600
|
+
}
|
601
|
+
config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
|
602
|
+
config += %[
|
603
|
+
s3_region #{expected_region}
|
604
|
+
<assume_role_credentials>
|
605
|
+
role_arn test_arn
|
606
|
+
role_session_name test_session
|
607
|
+
sts_http_proxy #{expected_sts_http_proxy}
|
608
|
+
</assume_role_credentials>
|
609
|
+
]
|
610
|
+
d = create_time_sliced_driver(config)
|
611
|
+
assert_nothing_raised { d.run {} }
|
612
|
+
client = d.instance.instance_variable_get(:@s3).client
|
613
|
+
credentials = client.config.credentials
|
614
|
+
assert_equal(expected_credentials, credentials)
|
615
|
+
end
|
616
|
+
|
617
|
+
def test_assume_role_credentials_with_sts_http_proxy
|
618
|
+
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
619
|
+
expected_sts_http_proxy = 'http://example.com'
|
620
|
+
sts_client = Aws::STS::Client.new(region: "us-east-1", http_proxy: expected_sts_http_proxy)
|
621
|
+
mock(Aws::STS::Client).new(region: "us-east-1", http_proxy: expected_sts_http_proxy, credentials: nil){ sts_client }
|
622
|
+
mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
|
623
|
+
role_session_name: "test_session",
|
624
|
+
client: sts_client,
|
625
|
+
sts_http_proxy: expected_sts_http_proxy){
|
626
|
+
expected_credentials
|
627
|
+
}
|
628
|
+
config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
|
629
|
+
config += %[
|
630
|
+
<assume_role_credentials>
|
631
|
+
role_arn test_arn
|
632
|
+
role_session_name test_session
|
633
|
+
sts_http_proxy #{expected_sts_http_proxy}
|
634
|
+
</assume_role_credentials>
|
635
|
+
]
|
636
|
+
d = create_time_sliced_driver(config)
|
637
|
+
assert_nothing_raised { d.run {} }
|
638
|
+
client = d.instance.instance_variable_get(:@s3).client
|
639
|
+
credentials = client.config.credentials
|
640
|
+
assert_equal(expected_credentials, credentials)
|
641
|
+
end
|
642
|
+
|
643
|
+
def test_assume_role_credentials_with_sts_endpoint_url
|
644
|
+
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
645
|
+
expected_sts_endpoint_url = 'http://example.com'
|
646
|
+
sts_client = Aws::STS::Client.new(region: "us-east-1", endpoint: expected_sts_endpoint_url)
|
647
|
+
mock(Aws::STS::Client).new(region: "us-east-1", endpoint: expected_sts_endpoint_url, credentials: nil){ sts_client }
|
648
|
+
mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
|
649
|
+
role_session_name: "test_session",
|
650
|
+
client: sts_client,
|
651
|
+
sts_endpoint_url: expected_sts_endpoint_url){
|
652
|
+
expected_credentials
|
653
|
+
}
|
654
|
+
config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
|
655
|
+
config += %[
|
656
|
+
<assume_role_credentials>
|
657
|
+
role_arn test_arn
|
658
|
+
role_session_name test_session
|
659
|
+
sts_endpoint_url #{expected_sts_endpoint_url}
|
660
|
+
</assume_role_credentials>
|
661
|
+
]
|
662
|
+
d = create_time_sliced_driver(config)
|
663
|
+
assert_nothing_raised { d.run {} }
|
664
|
+
client = d.instance.instance_variable_get(:@s3).client
|
665
|
+
credentials = client.config.credentials
|
666
|
+
assert_equal(expected_credentials, credentials)
|
667
|
+
end
|
668
|
+
|
669
|
+
def test_assume_role_credentials_with_sts_region
|
670
|
+
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
671
|
+
expected_sts_region = 'ap-south-1'
|
672
|
+
sts_client = Aws::STS::Client.new(region: expected_sts_region)
|
673
|
+
mock(Aws::STS::Client).new(region: expected_sts_region, credentials: nil){ sts_client }
|
674
|
+
mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
|
675
|
+
role_session_name: "test_session",
|
676
|
+
client: sts_client){
|
677
|
+
expected_credentials
|
678
|
+
}
|
679
|
+
config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
|
680
|
+
config += %[
|
681
|
+
<assume_role_credentials>
|
682
|
+
role_arn test_arn
|
683
|
+
role_session_name test_session
|
684
|
+
sts_region #{expected_sts_region}
|
685
|
+
</assume_role_credentials>
|
686
|
+
]
|
687
|
+
d = create_time_sliced_driver(config)
|
688
|
+
assert_nothing_raised { d.run {} }
|
689
|
+
client = d.instance.instance_variable_get(:@s3).client
|
690
|
+
credentials = client.config.credentials
|
691
|
+
assert_equal(expected_credentials, credentials)
|
692
|
+
end
|
693
|
+
|
570
694
|
def test_web_identity_credentials
|
571
695
|
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
572
696
|
mock(Aws::AssumeRoleWebIdentityCredentials).new(
|
@@ -593,6 +717,36 @@ EOC
|
|
593
717
|
assert_equal(expected_credentials, credentials)
|
594
718
|
end
|
595
719
|
|
720
|
+
def test_web_identity_credentials_with_sts_region
|
721
|
+
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
722
|
+
sts_client = Aws::STS::Client.new(region: 'us-east-1')
|
723
|
+
mock(Aws::STS::Client).new(region: 'us-east-1'){ sts_client }
|
724
|
+
mock(Aws::AssumeRoleWebIdentityCredentials).new(
|
725
|
+
role_arn: "test_arn",
|
726
|
+
role_session_name: "test_session",
|
727
|
+
web_identity_token_file: "test_file",
|
728
|
+
client: sts_client
|
729
|
+
){
|
730
|
+
expected_credentials
|
731
|
+
}
|
732
|
+
|
733
|
+
config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
|
734
|
+
config += %[
|
735
|
+
s3_region us-west-2
|
736
|
+
<web_identity_credentials>
|
737
|
+
role_arn test_arn
|
738
|
+
role_session_name test_session
|
739
|
+
web_identity_token_file test_file
|
740
|
+
sts_region us-east-1
|
741
|
+
</web_identity_credentials>
|
742
|
+
]
|
743
|
+
d = create_time_sliced_driver(config)
|
744
|
+
assert_nothing_raised { d.run {} }
|
745
|
+
client = d.instance.instance_variable_get(:@s3).client
|
746
|
+
credentials = client.config.credentials
|
747
|
+
assert_equal(expected_credentials, credentials)
|
748
|
+
end
|
749
|
+
|
596
750
|
def test_instance_profile_credentials
|
597
751
|
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
598
752
|
mock(Aws::InstanceProfileCredentials).new({}).returns(expected_credentials)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2020-
|
12
|
+
date: 2020-12-18 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|