fluent-plugin-s3 1.3.0 → 1.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +4 -6
- data/ChangeLog +22 -0
- data/Gemfile +0 -2
- data/README.md +18 -10
- data/VERSION +1 -1
- data/lib/fluent/plugin/in_s3.rb +3 -3
- data/lib/fluent/plugin/out_s3.rb +44 -15
- data/test/test_out_s3.rb +31 -8
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 592d53153cbf95a1134849af1fa27baf14f86f74a8af0e7737ec94a1317ca1a6
|
4
|
+
data.tar.gz: d2c0f9b0d40ae43f885c6840133621d0f9685340f1e8e74302934a19488b7d59
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2a7ac8b2606ca7ab6506ed7bb467cbe63b720ec6a333f894772919dba348ceed70c4044a276d1df80e94c6cda583f6ddc7dfa7f8d3f36f5b0290f43303878eb1
|
7
|
+
data.tar.gz: '082ebb2c021a5dc38554ede86fe9179fdc666fdd1a427dee9c4a0a13f707c9f59eac3688574675549c8406172d3ad870dd243c15c3e7e19e93d2d099e31e0e72'
|
data/.travis.yml
CHANGED
@@ -1,11 +1,10 @@
|
|
1
1
|
language: ruby
|
2
2
|
|
3
3
|
rvm:
|
4
|
-
- 2.
|
5
|
-
- 2.
|
6
|
-
- 2.
|
7
|
-
- 2.
|
8
|
-
- 2.5.0
|
4
|
+
- 2.4
|
5
|
+
- 2.5
|
6
|
+
- 2.6
|
7
|
+
- 2.7
|
9
8
|
- ruby-head
|
10
9
|
|
11
10
|
gemfile:
|
@@ -14,7 +13,6 @@ gemfile:
|
|
14
13
|
branches:
|
15
14
|
only:
|
16
15
|
- master
|
17
|
-
- v0.12
|
18
16
|
|
19
17
|
before_install: gem update bundler
|
20
18
|
script: bundle exec rake test
|
data/ChangeLog
CHANGED
@@ -1,3 +1,25 @@
|
|
1
|
+
Release 1.4.0 - 2020/08/02
|
2
|
+
|
3
|
+
* Remove uuidtools dependency
|
4
|
+
* in_s3: Add error info to polloing retry log
|
5
|
+
|
6
|
+
Release 1.3.4 - 2020/07/07
|
7
|
+
|
8
|
+
* Add sts_http_proxy and sts_endpoint_url to assume_role_credentials
|
9
|
+
|
10
|
+
Release 1.3.3 - 2020/06/25
|
11
|
+
|
12
|
+
* Allow fips/gov included endpoint
|
13
|
+
* Support sts_region parameter
|
14
|
+
|
15
|
+
Release 1.3.2 - 2020/05/18
|
16
|
+
|
17
|
+
* out_s3: Show warning message for object conflict case.
|
18
|
+
|
19
|
+
Release 1.3.1 - 2020/04/15
|
20
|
+
|
21
|
+
* out_s3: Support S3 Dual-Stack Endpoints in output plugin via enable_dual_stack parameter
|
22
|
+
|
1
23
|
Release 1.3.0 - 2020/02/10
|
2
24
|
|
3
25
|
* in_s3/out_s3: Support AssumeRoleWebIdentityCredentials via `web_identity_credentials` section for EKS.
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -31,8 +31,13 @@ We must setup SQS queue and S3 event notification before use this plugin.
|
|
31
31
|
|
32
32
|
Simply use RubyGems:
|
33
33
|
|
34
|
-
|
35
|
-
$ gem install fluent-plugin-s3
|
34
|
+
# install latest version
|
35
|
+
$ gem install fluent-plugin-s3 --no-document # for fluentd v1.0 or later
|
36
|
+
# If you need to install specifiv version, use -v option
|
37
|
+
$ gem install fluent-plugin-s3 -v 1.3.0 --no-document
|
38
|
+
# For v0.12. This is for old v0.12 users. Don't use v0.12 for new deployment
|
39
|
+
$ gem install fluent-plugin-s3 -v "~> 0.8" --no-document # for fluentd v0.12
|
40
|
+
|
36
41
|
|
37
42
|
## Configuration: credentials
|
38
43
|
|
@@ -301,6 +306,10 @@ See also AWS article: [Working with Regions](https://aws.amazon.com/blogs/develo
|
|
301
306
|
|
302
307
|
Enable [S3 Transfer Acceleration](https://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html) for uploads. **IMPORTANT**: For this to work, you must first enable this feature on your destination S3 bucket.
|
303
308
|
|
309
|
+
**enable_dual_stack**
|
310
|
+
|
311
|
+
Enable [Amazon S3 Dual-Stack Endpoints](https://docs.aws.amazon.com/AmazonS3/latest/dev/dual-stack-endpoints.html) for uploads. Will make it possible to use either IPv4 or IPv6 when connecting to S3.
|
312
|
+
|
304
313
|
**use_bundled_cert**
|
305
314
|
|
306
315
|
For cases where the default SSL certificate is unavailable (e.g. Windows), you can set this option to true in order to use the AWS SDK bundled certificate. Default is false.
|
@@ -333,7 +342,7 @@ E.g., "logs/" in the example configuration above.
|
|
333
342
|
time-slice in text that are formatted with **time_slice_format**.
|
334
343
|
* %{index} is the sequential number starts from 0, increments when multiple files are uploaded to S3 in the same time slice.
|
335
344
|
* %{file_extension} depends on **store_as** parameter.
|
336
|
-
* %{uuid_flush} a uuid that is replaced everytime the buffer will be flushed.
|
345
|
+
* %{uuid_flush} a uuid that is replaced everytime the buffer will be flushed.
|
337
346
|
* %{hostname} is replaced with `Socket.gethostname` result.
|
338
347
|
* %{hex_random} a random hex string that is replaced for each buffer chunk, not
|
339
348
|
assured to be unique. This is used to follow a way of performance tuning, `Add
|
@@ -422,16 +431,16 @@ Change one line format in the S3 object. Supported formats are "out_file",
|
|
422
431
|
|
423
432
|
|
424
433
|
At this format, "time" and "tag" are omitted. But you can set these
|
425
|
-
information to the record by setting
|
426
|
-
"include_time_key" / "time_key" option. If you set following configuration in
|
434
|
+
information to the record by setting `<inject>` option. If you set following configuration in
|
427
435
|
S3 output:
|
428
436
|
|
429
437
|
# v1
|
430
438
|
<format>
|
431
439
|
@type json
|
432
|
-
include_time_key true
|
433
|
-
time_key log_time # default is time
|
434
440
|
</format>
|
441
|
+
<inject>
|
442
|
+
time_key log_time
|
443
|
+
</inject>
|
435
444
|
# v0.12
|
436
445
|
@format json
|
437
446
|
include_time_key true
|
@@ -441,15 +450,14 @@ then the record has log_time field.
|
|
441
450
|
|
442
451
|
{"log_time":"time string",...}
|
443
452
|
|
453
|
+
See also [official Inject Section article](https://docs.fluentd.org/configuration/inject-section).
|
454
|
+
|
444
455
|
* ltsv
|
445
456
|
|
446
457
|
key1:value1\tkey2:value2
|
447
458
|
key1:value1\tkey2:value2
|
448
459
|
...
|
449
460
|
|
450
|
-
|
451
|
-
"ltsv" format also accepts "include_xxx" related options. See "json" section.
|
452
|
-
|
453
461
|
* single_value
|
454
462
|
|
455
463
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.4.0
|
data/lib/fluent/plugin/in_s3.rb
CHANGED
@@ -118,11 +118,11 @@ module Fluent::Plugin
|
|
118
118
|
def configure(conf)
|
119
119
|
super
|
120
120
|
|
121
|
-
if @s3_endpoint && @s3_endpoint.end_with?('amazonaws.com')
|
121
|
+
if @s3_endpoint && (@s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) })
|
122
122
|
raise Fluent::ConfigError, "s3_endpoint parameter is not supported for S3, use s3_region instead. This parameter is for S3 compatible services"
|
123
123
|
end
|
124
124
|
|
125
|
-
if @sqs.endpoint && @sqs.endpoint.end_with?('amazonaws.com')
|
125
|
+
if @sqs.endpoint && (@sqs.endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @sqs.endpoint.include?(e) })
|
126
126
|
raise Fluent::ConfigError, "sqs/endpoint parameter is not supported for SQS, use s3_region instead. This parameter is for SQS compatible services"
|
127
127
|
end
|
128
128
|
|
@@ -192,7 +192,7 @@ module Fluent::Plugin
|
|
192
192
|
end
|
193
193
|
end
|
194
194
|
rescue => e
|
195
|
-
log.warn("SQS Polling Failed. Retry in #{@sqs.retry_error_interval} seconds")
|
195
|
+
log.warn("SQS Polling Failed. Retry in #{@sqs.retry_error_interval} seconds", error: e)
|
196
196
|
sleep(@sqs.retry_error_interval)
|
197
197
|
retry
|
198
198
|
end
|
data/lib/fluent/plugin/out_s3.rb
CHANGED
@@ -5,6 +5,7 @@ require 'aws-sdk-s3'
|
|
5
5
|
require 'zlib'
|
6
6
|
require 'time'
|
7
7
|
require 'tempfile'
|
8
|
+
require 'securerandom'
|
8
9
|
|
9
10
|
module Fluent::Plugin
|
10
11
|
class S3Output < Output
|
@@ -39,6 +40,12 @@ module Fluent::Plugin
|
|
39
40
|
config_param :duration_seconds, :integer, default: nil
|
40
41
|
desc "A unique identifier that is used by third parties when assuming roles in their customers' accounts."
|
41
42
|
config_param :external_id, :string, default: nil, secret: true
|
43
|
+
desc "The region of the STS endpoint to use."
|
44
|
+
config_param :sts_region, :string, default: nil
|
45
|
+
desc "A http proxy url for requests to aws sts service"
|
46
|
+
config_param :sts_http_proxy, :string, default: nil, secret: true
|
47
|
+
desc "A url for a regional sts api endpoint, the default is global"
|
48
|
+
config_param :sts_endpoint_url, :string, default: nil
|
42
49
|
end
|
43
50
|
# See the following link for additional params that could be added:
|
44
51
|
# https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html#assume_role_with_web_identity-instance_method
|
@@ -53,6 +60,8 @@ module Fluent::Plugin
|
|
53
60
|
config_param :policy, :string, default: nil
|
54
61
|
desc "The duration, in seconds, of the role session (900-43200)"
|
55
62
|
config_param :duration_seconds, :integer, default: nil
|
63
|
+
desc "The region of the STS endpoint to use."
|
64
|
+
config_param :sts_region, :string, default: nil
|
56
65
|
end
|
57
66
|
config_section :instance_profile_credentials, multi: false do
|
58
67
|
desc "Number of times to retry when retrieving credentials"
|
@@ -84,6 +93,8 @@ module Fluent::Plugin
|
|
84
93
|
config_param :s3_endpoint, :string, default: nil
|
85
94
|
desc "If true, S3 Transfer Acceleration will be enabled for uploads. IMPORTANT: You must first enable this feature on your destination S3 bucket"
|
86
95
|
config_param :enable_transfer_acceleration, :bool, default: false
|
96
|
+
desc "If true, use Amazon S3 Dual-Stack Endpoints. Will make it possible to use either IPv4 or IPv6 when connecting to S3."
|
97
|
+
config_param :enable_dual_stack, :bool, default: false
|
87
98
|
desc "If false, the certificate of endpoint will not be verified"
|
88
99
|
config_param :ssl_verify_peer, :bool, :default => true
|
89
100
|
desc "The format of S3 object keys"
|
@@ -169,7 +180,7 @@ module Fluent::Plugin
|
|
169
180
|
|
170
181
|
Aws.use_bundled_cert! if @use_bundled_cert
|
171
182
|
|
172
|
-
if @s3_endpoint && @s3_endpoint.end_with?('amazonaws.com')
|
183
|
+
if @s3_endpoint && (@s3_endpoint.end_with?('amazonaws.com') && !['fips', 'gov'].any? { |e| @s3_endpoint.include?(e) })
|
173
184
|
raise Fluent::ConfigError, "s3_endpoint parameter is not supported for S3, use s3_region instead. This parameter is for S3 compatible services"
|
174
185
|
end
|
175
186
|
|
@@ -207,6 +218,8 @@ module Fluent::Plugin
|
|
207
218
|
end
|
208
219
|
end
|
209
220
|
|
221
|
+
check_s3_path_safety(conf)
|
222
|
+
|
210
223
|
# For backward compatibility
|
211
224
|
# TODO: Remove time_slice_format when end of support compat_parameters
|
212
225
|
@configured_time_slice_format = conf['time_slice_format']
|
@@ -223,6 +236,7 @@ module Fluent::Plugin
|
|
223
236
|
options[:region] = @s3_region if @s3_region
|
224
237
|
options[:endpoint] = @s3_endpoint if @s3_endpoint
|
225
238
|
options[:use_accelerate_endpoint] = @enable_transfer_acceleration
|
239
|
+
options[:use_dualstack_endpoint] = @enable_dual_stack
|
226
240
|
options[:http_proxy] = @proxy_uri if @proxy_uri
|
227
241
|
options[:force_path_style] = @force_path_style
|
228
242
|
options[:compute_checksums] = @compute_checksums unless @compute_checksums.nil?
|
@@ -371,7 +385,7 @@ module Fluent::Plugin
|
|
371
385
|
end
|
372
386
|
|
373
387
|
def uuid_random
|
374
|
-
|
388
|
+
SecureRandom.uuid
|
375
389
|
end
|
376
390
|
|
377
391
|
# This is stolen from Fluentd
|
@@ -428,17 +442,6 @@ module Fluent::Plugin
|
|
428
442
|
}
|
429
443
|
|
430
444
|
if @s3_object_key_format.include?('%{uuid_flush}')
|
431
|
-
# test uuidtools works or not
|
432
|
-
begin
|
433
|
-
require 'uuidtools'
|
434
|
-
rescue LoadError
|
435
|
-
raise Fluent::ConfigError, "uuidtools gem not found. Install uuidtools gem first"
|
436
|
-
end
|
437
|
-
begin
|
438
|
-
uuid_random
|
439
|
-
rescue => e
|
440
|
-
raise Fluent::ConfigError, "Generating uuid doesn't work. Can't use %{uuid_flush} on this environment. #{e}"
|
441
|
-
end
|
442
445
|
@uuid_flush_enabled = true
|
443
446
|
end
|
444
447
|
|
@@ -448,6 +451,16 @@ module Fluent::Plugin
|
|
448
451
|
}
|
449
452
|
end
|
450
453
|
|
454
|
+
def check_s3_path_safety(conf)
|
455
|
+
unless conf.has_key?('s3_object_key_format')
|
456
|
+
log.warn "The default value of s3_object_key_format will use ${chunk_id} instead of %{index} to avoid object conflict in v2"
|
457
|
+
end
|
458
|
+
|
459
|
+
if (@buffer_config.flush_thread_count > 1) && ['${chunk_id}', '%{uuid_flush}'].none? { |key| @s3_object_key_format.include?(key) }
|
460
|
+
log.warn "No ${chunk_id} or %{uuid_flush} in s3_object_key_format with multiple flush threads. Recommend to set ${chunk_id} or %{uuid_flush} to avoid data lost by object conflict"
|
461
|
+
end
|
462
|
+
end
|
463
|
+
|
451
464
|
def check_apikeys
|
452
465
|
@bucket.objects(prefix: @path, :max_keys => 1).first
|
453
466
|
rescue Aws::S3::Errors::NoSuchBucket
|
@@ -470,7 +483,21 @@ module Fluent::Plugin
|
|
470
483
|
credentials_options[:policy] = c.policy if c.policy
|
471
484
|
credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
|
472
485
|
credentials_options[:external_id] = c.external_id if c.external_id
|
473
|
-
if
|
486
|
+
credentials_options[:sts_endpoint_url] = c.sts_endpoint_url if c.sts_endpoint_url
|
487
|
+
credentials_options[:sts_http_proxy] = c.sts_http_proxy if c.sts_http_proxy
|
488
|
+
if c.sts_http_proxy && c.sts_endpoint_url
|
489
|
+
credentials_options[:client] = Aws::STS::Client.new(http_proxy: c.sts_http_proxy, endpoint: c.sts_endpoint_url)
|
490
|
+
elsif @region && c.sts_http_proxy
|
491
|
+
credentials_options[:client] = Aws::STS::Client.new(region: @region, http_proxy: c.sts_http_proxy)
|
492
|
+
elsif @region && c.sts_endpoint_url
|
493
|
+
credentials_options[:client] = Aws::STS::Client.new(region: @region, endpoint: c.sts_endpoint_url)
|
494
|
+
elsif c.sts_http_proxy
|
495
|
+
credentials_options[:client] = Aws::STS::Client.new(http_proxy: c.sts_http_proxy)
|
496
|
+
elsif c.sts_endpoint_url
|
497
|
+
credentials_options[:client] = Aws::STS::Client.new(endpoint: c.sts_endpoint_url)
|
498
|
+
elsif c.sts_region
|
499
|
+
credentials_options[:client] = Aws::STS::Client.new(region: c.sts_region)
|
500
|
+
elsif @s3_region
|
474
501
|
credentials_options[:client] = Aws::STS::Client.new(region: @s3_region)
|
475
502
|
end
|
476
503
|
options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
|
@@ -481,7 +508,9 @@ module Fluent::Plugin
|
|
481
508
|
credentials_options[:web_identity_token_file] = c.web_identity_token_file
|
482
509
|
credentials_options[:policy] = c.policy if c.policy
|
483
510
|
credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
|
484
|
-
if
|
511
|
+
if c.sts_region
|
512
|
+
credentials_options[:client] = Aws::STS::Client.new(:region => c.sts_region)
|
513
|
+
elsif @s3_region
|
485
514
|
credentials_options[:client] = Aws::STS::Client.new(:region => @s3_region)
|
486
515
|
end
|
487
516
|
options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
|
data/test/test_out_s3.rb
CHANGED
@@ -9,7 +9,6 @@ require 'test/unit/rr'
|
|
9
9
|
require 'zlib'
|
10
10
|
require 'fileutils'
|
11
11
|
require 'timecop'
|
12
|
-
require 'uuidtools'
|
13
12
|
require 'ostruct'
|
14
13
|
|
15
14
|
include Fluent::Test::Helpers
|
@@ -349,17 +348,11 @@ EOC
|
|
349
348
|
|
350
349
|
def test_write_with_custom_s3_object_key_format_containing_uuid_flush_placeholder
|
351
350
|
|
352
|
-
begin
|
353
|
-
require 'uuidtools'
|
354
|
-
rescue LoadError
|
355
|
-
pend("uuidtools not found. skip this test")
|
356
|
-
end
|
357
|
-
|
358
351
|
# Partial mock the S3Bucket, not to make an actual connection to Amazon S3
|
359
352
|
setup_mocks(true)
|
360
353
|
|
361
354
|
uuid = "5755e23f-9b54-42d8-8818-2ea38c6f279e"
|
362
|
-
stub(::
|
355
|
+
stub(::SecureRandom).uuid{ uuid }
|
363
356
|
|
364
357
|
s3_local_file_path = "/tmp/s3-test.txt"
|
365
358
|
s3path = "log/events/ts=20110102-13/events_0-#{uuid}.gz"
|
@@ -593,6 +586,36 @@ EOC
|
|
593
586
|
assert_equal(expected_credentials, credentials)
|
594
587
|
end
|
595
588
|
|
589
|
+
def test_web_identity_credentials_with_sts_region
|
590
|
+
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
591
|
+
sts_client = Aws::STS::Client.new(region: 'us-east-1')
|
592
|
+
mock(Aws::STS::Client).new(region: 'us-east-1'){ sts_client }
|
593
|
+
mock(Aws::AssumeRoleWebIdentityCredentials).new(
|
594
|
+
role_arn: "test_arn",
|
595
|
+
role_session_name: "test_session",
|
596
|
+
web_identity_token_file: "test_file",
|
597
|
+
client: sts_client
|
598
|
+
){
|
599
|
+
expected_credentials
|
600
|
+
}
|
601
|
+
|
602
|
+
config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
|
603
|
+
config += %[
|
604
|
+
s3_region us-west-2
|
605
|
+
<web_identity_credentials>
|
606
|
+
role_arn test_arn
|
607
|
+
role_session_name test_session
|
608
|
+
web_identity_token_file test_file
|
609
|
+
sts_region us-east-1
|
610
|
+
</web_identity_credentials>
|
611
|
+
]
|
612
|
+
d = create_time_sliced_driver(config)
|
613
|
+
assert_nothing_raised { d.run {} }
|
614
|
+
client = d.instance.instance_variable_get(:@s3).client
|
615
|
+
credentials = client.config.credentials
|
616
|
+
assert_equal(expected_credentials, credentials)
|
617
|
+
end
|
618
|
+
|
596
619
|
def test_instance_profile_credentials
|
597
620
|
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
598
621
|
mock(Aws::InstanceProfileCredentials).new({}).returns(expected_credentials)
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2020-
|
12
|
+
date: 2020-08-03 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|