fluent-plugin-s3 0.8.8 → 1.0.0.rc1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +5 -15
- data/ChangeLog +0 -42
- data/README.md +0 -37
- data/VERSION +1 -1
- data/fluent-plugin-s3.gemspec +1 -2
- data/lib/fluent/plugin/in_s3.rb +49 -74
- data/lib/fluent/plugin/out_s3.rb +128 -162
- data/lib/fluent/plugin/s3_compressor_gzip_command.rb +3 -3
- data/lib/fluent/plugin/s3_compressor_lzma2.rb +3 -3
- data/lib/fluent/plugin/s3_compressor_lzo.rb +3 -3
- data/lib/fluent/plugin/s3_extractor_gzip_command.rb +2 -2
- data/lib/fluent/plugin/s3_extractor_lzma2.rb +2 -2
- data/lib/fluent/plugin/s3_extractor_lzo.rb +2 -3
- data/test/test_in_s3.rb +30 -62
- data/test/test_out_s3.rb +164 -192
- metadata +7 -22
- data/lib/fluent/log-ext.rb +0 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7ec3e12f232c01e8d49337f8400f81eb721ecb72
|
4
|
+
data.tar.gz: 214c840df68e54ff4c96c5278b0bad06156e028e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: db47889fef5cb149f2f5c590887ad83989af1c7bee798b784dcc0b33ff48a6bd0c1e0c9b4a2b83d301f924882cb9b11cd3b382f1785e673fc09b71701d1ba2f2
|
7
|
+
data.tar.gz: bacf47f05d717afdcf2298cf31ee154392c8081809c4f337563e706bd8ba3248cf0e2cbccfab80be7811dd01d1df8bd21f29718af1c07860ceeec14274a6010c
|
data/.travis.yml
CHANGED
@@ -1,22 +1,19 @@
|
|
1
1
|
language: ruby
|
2
2
|
|
3
3
|
rvm:
|
4
|
-
- 1.
|
5
|
-
- 2.
|
6
|
-
- 2.1
|
7
|
-
- 2.2.3
|
4
|
+
- 2.1.10
|
5
|
+
- 2.2.5
|
8
6
|
- 2.3.1
|
9
|
-
- 2.4.0
|
7
|
+
- 2.4.0-rc1
|
10
8
|
- ruby-head
|
9
|
+
- rbx
|
11
10
|
|
12
11
|
gemfile:
|
13
12
|
- Gemfile
|
14
|
-
- Gemfile.v0.12
|
15
13
|
|
16
14
|
branches:
|
17
15
|
only:
|
18
16
|
- master
|
19
|
-
- v0.12
|
20
17
|
|
21
18
|
before_install: gem update bundler
|
22
19
|
script: bundle exec rake test
|
@@ -25,12 +22,5 @@ sudo: false
|
|
25
22
|
|
26
23
|
matrix:
|
27
24
|
allow_failures:
|
28
|
-
- rvm: 1.9.3
|
29
25
|
- rvm: ruby-head
|
30
|
-
|
31
|
-
- rvm: 1.9.3
|
32
|
-
gemfile: Gemfile
|
33
|
-
- rvm: 2.0
|
34
|
-
gemfile: Gemfile
|
35
|
-
- rvm: 2.4.0
|
36
|
-
gemfile: Gemfile.v0.12
|
26
|
+
- rvm: rbx
|
data/ChangeLog
CHANGED
@@ -1,45 +1,3 @@
|
|
1
|
-
Release 0.8.8 - 2018/09/06
|
2
|
-
|
3
|
-
* out_s3: Improve check_apikeys performance
|
4
|
-
|
5
|
-
|
6
|
-
Release 0.8.7 - 2017/10/27
|
7
|
-
|
8
|
-
* out_s3: support for granting a permission to a canonical user id
|
9
|
-
|
10
|
-
|
11
|
-
Release 0.8.6 - 2017/10/11
|
12
|
-
|
13
|
-
* out_s3: specified s3_object_key_format is applied when check_object is false
|
14
|
-
|
15
|
-
|
16
|
-
Release 0.8.5 - 2017/08/04
|
17
|
-
|
18
|
-
* out_s3: Fix typo error in debug log message
|
19
|
-
|
20
|
-
|
21
|
-
Release 0.8.4 - 2017/06/26
|
22
|
-
|
23
|
-
* in_s3: Fix object handling for URL encoded key
|
24
|
-
* Improve error message for startup API error
|
25
|
-
|
26
|
-
|
27
|
-
Release 0.8.3 - 2017/05/09
|
28
|
-
|
29
|
-
* Add tmp_dir parameter to change the directory for tempfile
|
30
|
-
|
31
|
-
|
32
|
-
Release 0.8.2 - 2017/02/29
|
33
|
-
|
34
|
-
* Log debug message when `debug` or `trace` is set in `@log_level`
|
35
|
-
|
36
|
-
|
37
|
-
Release 0.8.1 - 2017/01/29
|
38
|
-
|
39
|
-
* out_s3: Add ssl_verify_peer option to disable cert check
|
40
|
-
* out_s3: Fix %{hostname} regression in path
|
41
|
-
|
42
|
-
|
43
1
|
Release 0.8.0 - 2016/12/20
|
44
2
|
|
45
3
|
* out_s3: Add check_object / check_bucket parameters for only put permission
|
data/README.md
CHANGED
@@ -77,10 +77,6 @@ recommend using `s3_region` instead of `s3_endpoint`.
|
|
77
77
|
endpoint for S3 compatible services. For example, Riak CS based storage or
|
78
78
|
something. This option doesn't work on S3, use `s3_region` instead.
|
79
79
|
|
80
|
-
**ssl_verify_peer**
|
81
|
-
|
82
|
-
Verify SSL certificate of the endpoint. Default is true. Set false when you want to ignore the endpoint SSL certificate.
|
83
|
-
|
84
80
|
**s3_object_key_format**
|
85
81
|
|
86
82
|
The format of S3 object keys. You can use several built-in variables:
|
@@ -299,39 +295,6 @@ To use cross-account access, you will need to create a bucket policy granting
|
|
299
295
|
the specific access required. Refer to the [AWS
|
300
296
|
documentation](http://docs.aws.amazon.com/AmazonS3/latest/dev/example-walkthroughs-managing-access-example3.html) for examples.
|
301
297
|
|
302
|
-
**grant_full_control**
|
303
|
-
|
304
|
-
Allows grantee READ, READ_ACP, and WRITE_ACP permissions on the object.
|
305
|
-
This is useful for cross-account access using IAM roles.
|
306
|
-
|
307
|
-
Valid values are `id="Grantee-CanonicalUserID"`. Please specify the grantee's canonical user ID.
|
308
|
-
|
309
|
-
e.g. `id="79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be"`
|
310
|
-
|
311
|
-
Note that a canonical user ID is different from an AWS account ID.
|
312
|
-
Please refer to [AWS documentation](https://docs.aws.amazon.com/general/latest/gr/acct-identifiers.html) for more details.
|
313
|
-
|
314
|
-
**grant_read**
|
315
|
-
|
316
|
-
Allows grantee to read the object data and its metadata.
|
317
|
-
Valid values are `id="Grantee-CanonicalUserID"`. Please specify the grantee's canonical user ID.
|
318
|
-
|
319
|
-
e.g. `id="79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be"`
|
320
|
-
|
321
|
-
**grant_read_acp**
|
322
|
-
|
323
|
-
Allows grantee to read the object ACL.
|
324
|
-
Valid values are `id="Grantee-CanonicalUserID"`. Please specify the grantee's canonical user ID.
|
325
|
-
|
326
|
-
e.g. `id="79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be"`
|
327
|
-
|
328
|
-
**grant_write_acp**
|
329
|
-
|
330
|
-
Allows grantee to write the ACL for the applicable object.
|
331
|
-
Valid values are `id="Grantee-CanonicalUserID"`. Please specify the grantee's canonical user ID.
|
332
|
-
|
333
|
-
e.g. `id="79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be"`
|
334
|
-
|
335
298
|
**hex_random_length**
|
336
299
|
|
337
300
|
The length of `%{hex_random}` placeholder. Default is 4 as written in
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
1.0.0.rc1
|
data/fluent-plugin-s3.gemspec
CHANGED
@@ -17,10 +17,9 @@ Gem::Specification.new do |gem|
|
|
17
17
|
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
18
|
gem.require_paths = ['lib']
|
19
19
|
|
20
|
-
gem.add_dependency "fluentd", [">= 0.
|
20
|
+
gem.add_dependency "fluentd", [">= 0.14.2", "< 2"]
|
21
21
|
gem.add_dependency "aws-sdk", [">= 2.3.22", "< 3"]
|
22
22
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
23
|
-
gem.add_development_dependency "rr", "= 1.1.2"
|
24
23
|
gem.add_development_dependency "test-unit", ">= 3.0.8"
|
25
24
|
gem.add_development_dependency "test-unit-rr", ">= 1.0.3"
|
26
25
|
gem.add_development_dependency "timecop"
|
data/lib/fluent/plugin/in_s3.rb
CHANGED
@@ -1,119 +1,107 @@
|
|
1
1
|
require 'fluent/input'
|
2
|
-
require 'fluent/log-ext'
|
3
2
|
|
4
|
-
|
5
|
-
|
3
|
+
require 'aws-sdk-resources'
|
4
|
+
require 'zlib'
|
5
|
+
require 'time'
|
6
|
+
require 'tempfile'
|
7
|
+
|
8
|
+
module Fluent::Plugin
|
9
|
+
class S3Input < Fluent::Plugin::Input
|
6
10
|
Fluent::Plugin.register_input('s3', self)
|
7
11
|
|
12
|
+
helpers :compat_parameters, :parser, :thread
|
13
|
+
|
8
14
|
def initialize
|
9
15
|
super
|
10
|
-
require 'aws-sdk-resources'
|
11
|
-
require 'zlib'
|
12
|
-
require 'time'
|
13
|
-
require 'tempfile'
|
14
|
-
require 'cgi/util'
|
15
|
-
|
16
16
|
@extractor = nil
|
17
17
|
end
|
18
18
|
|
19
|
-
|
20
|
-
class << self
|
21
|
-
unless method_defined?(:desc)
|
22
|
-
def desc(description)
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
unless Fluent::Config::ConfigureProxy.method_defined?(:desc)
|
27
|
-
Fluent::Config::ConfigureProxy.class_eval do
|
28
|
-
def desc(description)
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
19
|
+
DEFAULT_PARSE_TYPE = "none"
|
32
20
|
|
33
21
|
desc "AWS access key id"
|
34
|
-
config_param :aws_key_id, :string, :
|
22
|
+
config_param :aws_key_id, :string, default: nil, secret: true
|
35
23
|
desc "AWS secret key."
|
36
|
-
config_param :aws_sec_key, :string, :
|
37
|
-
config_section :assume_role_credentials, :
|
24
|
+
config_param :aws_sec_key, :string, default: nil, secret: true
|
25
|
+
config_section :assume_role_credentials, multi: false do
|
38
26
|
desc "The Amazon Resource Name (ARN) of the role to assume"
|
39
27
|
config_param :role_arn, :string
|
40
28
|
desc "An identifier for the assumed role session"
|
41
29
|
config_param :role_session_name, :string
|
42
30
|
desc "An IAM policy in JSON format"
|
43
|
-
config_param :policy, :string, :
|
31
|
+
config_param :policy, :string, default: nil
|
44
32
|
desc "The duration, in seconds, of the role session (900-3600)"
|
45
|
-
config_param :duration_seconds, :integer, :
|
33
|
+
config_param :duration_seconds, :integer, default: nil
|
46
34
|
desc "A unique identifier that is used by third parties when assuming roles in their customers' accounts."
|
47
|
-
config_param :external_id, :string, :
|
35
|
+
config_param :external_id, :string, default: nil
|
48
36
|
end
|
49
|
-
config_section :instance_profile_credentials, :
|
37
|
+
config_section :instance_profile_credentials, multi: false do
|
50
38
|
desc "Number of times to retry when retrieving credentials"
|
51
|
-
config_param :retries, :integer, :
|
39
|
+
config_param :retries, :integer, default: nil
|
52
40
|
desc "IP address (default:169.254.169.254)"
|
53
|
-
config_param :ip_address, :string, :
|
41
|
+
config_param :ip_address, :string, default: nil
|
54
42
|
desc "Port number (default:80)"
|
55
|
-
config_param :port, :integer, :
|
43
|
+
config_param :port, :integer, default: nil
|
56
44
|
desc "Number of seconds to wait for the connection to open"
|
57
|
-
config_param :http_open_timeout, :float, :
|
45
|
+
config_param :http_open_timeout, :float, default: nil
|
58
46
|
desc "Number of seconds to wait for one block to be read"
|
59
|
-
config_param :http_read_timeout, :float, :
|
47
|
+
config_param :http_read_timeout, :float, default: nil
|
60
48
|
# config_param :delay, :integer or :proc, :default => nil
|
61
49
|
# config_param :http_degub_output, :io, :default => nil
|
62
50
|
end
|
63
|
-
config_section :shared_credentials, :
|
51
|
+
config_section :shared_credentials, multi: false do
|
64
52
|
desc "Path to the shared file. (default: $HOME/.aws/credentials)"
|
65
|
-
config_param :path, :string, :
|
53
|
+
config_param :path, :string, default: nil
|
66
54
|
desc "Profile name. Default to 'default' or ENV['AWS_PROFILE']"
|
67
|
-
config_param :profile_name, :string, :
|
55
|
+
config_param :profile_name, :string, default: nil
|
68
56
|
end
|
69
57
|
desc "S3 bucket name"
|
70
58
|
config_param :s3_bucket, :string
|
71
59
|
desc "S3 region name"
|
72
|
-
config_param :s3_region, :string, :
|
60
|
+
config_param :s3_region, :string, default: ENV["AWS_REGION"] || "us-east-1"
|
73
61
|
desc "Archive format on S3"
|
74
|
-
config_param :store_as, :string, :
|
62
|
+
config_param :store_as, :string, default: "gzip"
|
75
63
|
desc "Check AWS key on start"
|
76
|
-
config_param :check_apikey_on_start, :bool, :
|
64
|
+
config_param :check_apikey_on_start, :bool, default: true
|
77
65
|
desc "URI of proxy environment"
|
78
|
-
config_param :proxy_uri, :string, :
|
79
|
-
desc "Change one line format in the S3 object (none,json,ltsv,single_value)"
|
80
|
-
config_param :format, :string, :default => 'none'
|
66
|
+
config_param :proxy_uri, :string, default: nil
|
81
67
|
|
82
|
-
config_section :sqs, :
|
68
|
+
config_section :sqs, required: true, multi: false do
|
83
69
|
desc "SQS queue name"
|
84
|
-
config_param :queue_name, :string, :
|
70
|
+
config_param :queue_name, :string, default: nil
|
85
71
|
desc "Skip message deletion"
|
86
|
-
config_param :skip_delete, :bool, :
|
72
|
+
config_param :skip_delete, :bool, default: false
|
87
73
|
desc "The long polling interval."
|
88
|
-
config_param :wait_time_seconds, :integer, :
|
74
|
+
config_param :wait_time_seconds, :integer, default: 20
|
89
75
|
end
|
90
76
|
|
91
77
|
desc "Tag string"
|
92
|
-
config_param :tag, :string, :
|
78
|
+
config_param :tag, :string, default: "input.s3"
|
79
|
+
|
80
|
+
config_section :parse do
|
81
|
+
config_set_default :@type, DEFAULT_PARSE_TYPE
|
82
|
+
end
|
93
83
|
|
94
84
|
attr_reader :bucket
|
95
85
|
|
96
86
|
def configure(conf)
|
97
87
|
super
|
98
88
|
|
89
|
+
parser_config = conf.elements("parse").first
|
99
90
|
unless @sqs.queue_name
|
100
|
-
raise ConfigError, "sqs/queue_name is required"
|
91
|
+
raise Fluent::ConfigError, "sqs/queue_name is required"
|
101
92
|
end
|
102
93
|
|
103
94
|
@extractor = EXTRACTOR_REGISTRY.lookup(@store_as).new(log: log)
|
104
95
|
@extractor.configure(conf)
|
105
96
|
|
106
|
-
@parser =
|
107
|
-
@parser.configure(conf)
|
97
|
+
@parser = parser_create(conf: parser_config, default_type: DEFAULT_PARSE_TYPE)
|
108
98
|
end
|
109
99
|
|
110
100
|
def start
|
111
101
|
super
|
112
102
|
|
113
103
|
s3_client = create_s3_client
|
114
|
-
|
115
|
-
log.debug("Succeeded to create S3 client")
|
116
|
-
@s3 = Aws::S3::Resource.new(:client => s3_client)
|
104
|
+
@s3 = Aws::S3::Resource.new(client: s3_client)
|
117
105
|
@bucket = @s3.bucket(@s3_bucket)
|
118
106
|
|
119
107
|
raise "#{@bucket.name} is not found." unless @bucket.exists?
|
@@ -121,20 +109,17 @@ module Fluent
|
|
121
109
|
check_apikeys if @check_apikey_on_start
|
122
110
|
|
123
111
|
sqs_client = create_sqs_client
|
124
|
-
log.debug("Succeeded to create SQS client")
|
125
112
|
response = sqs_client.get_queue_url(queue_name: @sqs.queue_name)
|
126
113
|
sqs_queue_url = response.queue_url
|
127
|
-
log.debug("Succeeded to get SQS queue URL")
|
128
114
|
|
129
115
|
@poller = Aws::SQS::QueuePoller.new(sqs_queue_url, client: sqs_client)
|
130
116
|
|
131
117
|
@running = true
|
132
|
-
|
118
|
+
thread_create(:in_s3, &method(:run))
|
133
119
|
end
|
134
120
|
|
135
121
|
def shutdown
|
136
122
|
@running = false
|
137
|
-
@thread.join
|
138
123
|
super
|
139
124
|
end
|
140
125
|
|
@@ -150,7 +135,6 @@ module Fluent
|
|
150
135
|
@poller.poll(options) do |message|
|
151
136
|
begin
|
152
137
|
body = Yajl.load(message.body)
|
153
|
-
log.debug(body)
|
154
138
|
next unless body["Records"] # skip test queue
|
155
139
|
|
156
140
|
process(body)
|
@@ -201,10 +185,6 @@ module Fluent
|
|
201
185
|
options = setup_credentials
|
202
186
|
options[:region] = @s3_region if @s3_region
|
203
187
|
options[:proxy_uri] = @proxy_uri if @proxy_uri
|
204
|
-
log.on_trace do
|
205
|
-
options[:http_wire_trace] = true
|
206
|
-
options[:logger] = log
|
207
|
-
end
|
208
188
|
|
209
189
|
Aws::S3::Client.new(options)
|
210
190
|
end
|
@@ -212,28 +192,23 @@ module Fluent
|
|
212
192
|
def create_sqs_client
|
213
193
|
options = setup_credentials
|
214
194
|
options[:region] = @s3_region if @s3_region
|
215
|
-
log.on_trace do
|
216
|
-
options[:http_wire_trace] = true
|
217
|
-
options[:logger] = log
|
218
|
-
end
|
219
195
|
|
220
196
|
Aws::SQS::Client.new(options)
|
221
197
|
end
|
222
198
|
|
223
199
|
def check_apikeys
|
224
200
|
@bucket.objects.first
|
225
|
-
log.debug("Succeeded to verify API keys")
|
226
201
|
rescue => e
|
227
|
-
raise "can't call S3 API. Please check your
|
202
|
+
raise "can't call S3 API. Please check your aws_key_id / aws_sec_key or s3_region configuration. error = #{e.inspect}"
|
228
203
|
end
|
229
204
|
|
230
205
|
def process(body)
|
231
206
|
s3 = body["Records"].first["s3"]
|
232
|
-
key =
|
207
|
+
key = s3["object"]["key"]
|
233
208
|
|
234
209
|
io = @bucket.object(key).get.body
|
235
210
|
content = @extractor.extract(io)
|
236
|
-
es = MultiEventStream.new
|
211
|
+
es = Fluent::MultiEventStream.new
|
237
212
|
content.each_line do |line|
|
238
213
|
@parser.parse(line) do |time, record|
|
239
214
|
es.add(time, record)
|
@@ -243,7 +218,7 @@ module Fluent
|
|
243
218
|
end
|
244
219
|
|
245
220
|
class Extractor
|
246
|
-
include Configurable
|
221
|
+
include Fluent::Configurable
|
247
222
|
|
248
223
|
attr_reader :log
|
249
224
|
|
@@ -274,7 +249,7 @@ module Fluent
|
|
274
249
|
begin
|
275
250
|
Open3.capture3("#{command} -V")
|
276
251
|
rescue Errno::ENOENT
|
277
|
-
raise ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
|
252
|
+
raise Fluent::ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
|
278
253
|
end
|
279
254
|
end
|
280
255
|
end
|
@@ -319,7 +294,7 @@ module Fluent
|
|
319
294
|
end
|
320
295
|
end
|
321
296
|
|
322
|
-
EXTRACTOR_REGISTRY = Registry.new(:s3_extractor_type, 'fluent/plugin/s3_extractor_')
|
297
|
+
EXTRACTOR_REGISTRY = Fluent::Registry.new(:s3_extractor_type, 'fluent/plugin/s3_extractor_')
|
323
298
|
{
|
324
299
|
'gzip' => GzipExtractor,
|
325
300
|
'text' => TextExtractor,
|
data/lib/fluent/plugin/out_s3.rb
CHANGED
@@ -1,189 +1,153 @@
|
|
1
1
|
require 'fluent/output'
|
2
|
-
require '
|
2
|
+
require 'aws-sdk-resources'
|
3
|
+
require 'zlib'
|
4
|
+
require 'time'
|
5
|
+
require 'tempfile'
|
3
6
|
|
4
|
-
module Fluent
|
5
|
-
class S3Output < Fluent::
|
7
|
+
module Fluent::Plugin
|
8
|
+
class S3Output < Fluent::Plugin::Output
|
6
9
|
Fluent::Plugin.register_output('s3', self)
|
7
10
|
|
11
|
+
helpers :compat_parameters, :formatter, :inject
|
12
|
+
|
8
13
|
def initialize
|
9
14
|
super
|
10
|
-
require 'aws-sdk-resources'
|
11
|
-
require 'zlib'
|
12
|
-
require 'time'
|
13
|
-
require 'tempfile'
|
14
|
-
|
15
15
|
@compressor = nil
|
16
16
|
@uuid_flush_enabled = false
|
17
17
|
end
|
18
18
|
|
19
|
-
# For fluentd v0.12.16 or earlier
|
20
|
-
class << self
|
21
|
-
unless method_defined?(:desc)
|
22
|
-
def desc(description)
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
unless Fluent::Config::ConfigureProxy.method_defined?(:desc)
|
27
|
-
Fluent::Config::ConfigureProxy.class_eval do
|
28
|
-
def desc(description)
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
19
|
desc "Path prefix of the files on S3"
|
34
|
-
config_param :path, :string, :
|
20
|
+
config_param :path, :string, default: ""
|
35
21
|
desc "The Server-side encryption algorithm used when storing this object in S3 (AES256, aws:kms)"
|
36
|
-
config_param :use_server_side_encryption, :string, :
|
22
|
+
config_param :use_server_side_encryption, :string, default: nil
|
37
23
|
desc "AWS access key id"
|
38
|
-
config_param :aws_key_id, :string, :
|
24
|
+
config_param :aws_key_id, :string, default: nil, secret: true
|
39
25
|
desc "AWS secret key."
|
40
|
-
config_param :aws_sec_key, :string, :
|
41
|
-
config_section :assume_role_credentials, :
|
26
|
+
config_param :aws_sec_key, :string, default: nil, secret: true
|
27
|
+
config_section :assume_role_credentials, multi: false do
|
42
28
|
desc "The Amazon Resource Name (ARN) of the role to assume"
|
43
|
-
config_param :role_arn, :string, :
|
29
|
+
config_param :role_arn, :string, secret: true
|
44
30
|
desc "An identifier for the assumed role session"
|
45
31
|
config_param :role_session_name, :string
|
46
32
|
desc "An IAM policy in JSON format"
|
47
|
-
config_param :policy, :string, :
|
33
|
+
config_param :policy, :string, default: nil
|
48
34
|
desc "The duration, in seconds, of the role session (900-3600)"
|
49
|
-
config_param :duration_seconds, :integer, :
|
35
|
+
config_param :duration_seconds, :integer, default: nil
|
50
36
|
desc "A unique identifier that is used by third parties when assuming roles in their customers' accounts."
|
51
|
-
config_param :external_id, :string, :
|
37
|
+
config_param :external_id, :string, default: nil, secret: true
|
52
38
|
end
|
53
|
-
config_section :instance_profile_credentials, :
|
39
|
+
config_section :instance_profile_credentials, multi: false do
|
54
40
|
desc "Number of times to retry when retrieving credentials"
|
55
|
-
config_param :retries, :integer, :
|
41
|
+
config_param :retries, :integer, default: nil
|
56
42
|
desc "IP address (default:169.254.169.254)"
|
57
|
-
config_param :ip_address, :string, :
|
43
|
+
config_param :ip_address, :string, default: nil
|
58
44
|
desc "Port number (default:80)"
|
59
|
-
config_param :port, :integer, :
|
45
|
+
config_param :port, :integer, default: nil
|
60
46
|
desc "Number of seconds to wait for the connection to open"
|
61
|
-
config_param :http_open_timeout, :float, :
|
47
|
+
config_param :http_open_timeout, :float, default: nil
|
62
48
|
desc "Number of seconds to wait for one block to be read"
|
63
|
-
config_param :http_read_timeout, :float, :
|
49
|
+
config_param :http_read_timeout, :float, default: nil
|
64
50
|
# config_param :delay, :integer or :proc, :default => nil
|
65
51
|
# config_param :http_degub_output, :io, :default => nil
|
66
52
|
end
|
67
|
-
config_section :shared_credentials, :
|
53
|
+
config_section :shared_credentials, multi: false do
|
68
54
|
desc "Path to the shared file. (default: $HOME/.aws/credentials)"
|
69
|
-
config_param :path, :string, :
|
55
|
+
config_param :path, :string, default: nil
|
70
56
|
desc "Profile name. Default to 'default' or ENV['AWS_PROFILE']"
|
71
|
-
config_param :profile_name, :string, :
|
57
|
+
config_param :profile_name, :string, default: nil
|
72
58
|
end
|
73
59
|
desc "The number of attempts to load instance profile credentials from the EC2 metadata service using IAM role"
|
74
|
-
config_param :aws_iam_retries, :integer, :
|
60
|
+
config_param :aws_iam_retries, :integer, default: nil, deprecated: "Use 'instance_profile_credentials' instead"
|
75
61
|
desc "S3 bucket name"
|
76
62
|
config_param :s3_bucket, :string
|
77
63
|
desc "S3 region name"
|
78
|
-
config_param :s3_region, :string, :
|
64
|
+
config_param :s3_region, :string, default: ENV["AWS_REGION"] || "us-east-1"
|
79
65
|
desc "Use 's3_region' instead"
|
80
|
-
config_param :s3_endpoint, :string, :
|
81
|
-
desc "If false, the certificate of endpoint will not be verified"
|
82
|
-
config_param :ssl_verify_peer, :bool, :default => true
|
66
|
+
config_param :s3_endpoint, :string, default: nil
|
83
67
|
desc "The format of S3 object keys"
|
84
|
-
config_param :s3_object_key_format, :string, :
|
68
|
+
config_param :s3_object_key_format, :string, default: "%{path}%{time_slice}_%{index}.%{file_extension}"
|
85
69
|
desc "If true, the bucket name is always left in the request URI and never moved to the host as a sub-domain"
|
86
|
-
config_param :force_path_style, :bool, :
|
70
|
+
config_param :force_path_style, :bool, default: false
|
87
71
|
desc "Archive format on S3"
|
88
|
-
config_param :store_as, :string, :
|
72
|
+
config_param :store_as, :string, default: "gzip"
|
89
73
|
desc "Create S3 bucket if it does not exists"
|
90
|
-
config_param :auto_create_bucket, :bool, :
|
74
|
+
config_param :auto_create_bucket, :bool, default: true
|
91
75
|
desc "Check AWS key on start"
|
92
|
-
config_param :check_apikey_on_start, :bool, :
|
76
|
+
config_param :check_apikey_on_start, :bool, default: true
|
93
77
|
desc "URI of proxy environment"
|
94
|
-
config_param :proxy_uri, :string, :
|
78
|
+
config_param :proxy_uri, :string, default: nil
|
95
79
|
desc "Use S3 reduced redundancy storage for 33% cheaper pricing. Deprecated. Use storage_class instead"
|
96
|
-
config_param :reduced_redundancy, :bool, :
|
80
|
+
config_param :reduced_redundancy, :bool, default: false, deprecated: "Use storage_class parameter instead."
|
97
81
|
desc "The type of storage to use for the object(STANDARD,REDUCED_REDUNDANCY,STANDARD_IA)"
|
98
|
-
config_param :storage_class, :string, :
|
99
|
-
desc "Change one line format in the S3 object (out_file,json,ltsv,single_value)"
|
100
|
-
config_param :format, :string, :default => 'out_file'
|
82
|
+
config_param :storage_class, :string, default: "STANDARD"
|
101
83
|
desc "Permission for the object in S3"
|
102
|
-
config_param :acl, :string, :
|
103
|
-
desc "Allows grantee READ, READ_ACP, and WRITE_ACP permissions on the object"
|
104
|
-
config_param :grant_full_control, :string, :default => nil
|
105
|
-
desc "Allows grantee to read the object data and its metadata"
|
106
|
-
config_param :grant_read, :string, :default => nil
|
107
|
-
desc "Allows grantee to read the object ACL"
|
108
|
-
config_param :grant_read_acp, :string, :default => nil
|
109
|
-
desc "Allows grantee to write the ACL for the applicable object"
|
110
|
-
config_param :grant_write_acp, :string, :default => nil
|
84
|
+
config_param :acl, :string, default: nil
|
111
85
|
desc "The length of `%{hex_random}` placeholder(4-16)"
|
112
|
-
config_param :hex_random_length, :integer, :
|
86
|
+
config_param :hex_random_length, :integer, default: 4
|
113
87
|
desc "Overwrite already existing path"
|
114
|
-
config_param :overwrite, :bool, :
|
88
|
+
config_param :overwrite, :bool, default: false
|
115
89
|
desc "Check bucket if exists or not"
|
116
|
-
config_param :check_bucket, :bool, :
|
90
|
+
config_param :check_bucket, :bool, default: true
|
117
91
|
desc "Check object before creation"
|
118
|
-
config_param :check_object, :bool, :
|
92
|
+
config_param :check_object, :bool, default: true
|
119
93
|
desc "Specifies the AWS KMS key ID to use for object encryption"
|
120
|
-
config_param :ssekms_key_id, :string, :
|
94
|
+
config_param :ssekms_key_id, :string, default: nil, secret: true
|
121
95
|
desc "Specifies the algorithm to use to when encrypting the object"
|
122
|
-
config_param :sse_customer_algorithm, :string, :
|
96
|
+
config_param :sse_customer_algorithm, :string, default: nil
|
123
97
|
desc "Specifies the customer-provided encryption key for Amazon S3 to use in encrypting data"
|
124
|
-
config_param :sse_customer_key, :string, :
|
98
|
+
config_param :sse_customer_key, :string, default: nil, secret: true
|
125
99
|
desc "Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321"
|
126
|
-
config_param :sse_customer_key_md5, :string, :
|
100
|
+
config_param :sse_customer_key_md5, :string, default: nil, secret: true
|
127
101
|
desc "AWS SDK uses MD5 for API request/response by default"
|
128
|
-
config_param :compute_checksums, :bool, :
|
102
|
+
config_param :compute_checksums, :bool, default: nil # use nil to follow SDK default configuration
|
129
103
|
desc "Signature version for API Request (s3,v4)"
|
130
|
-
config_param :signature_version, :string, :
|
104
|
+
config_param :signature_version, :string, default: nil # use nil to follow SDK default configuration
|
131
105
|
desc "Given a threshold to treat events as delay, output warning logs if delayed events were put into s3"
|
132
|
-
config_param :warn_for_delay, :time, :
|
133
|
-
|
134
|
-
|
106
|
+
config_param :warn_for_delay, :time, default: nil
|
107
|
+
|
108
|
+
DEFAULT_FORMAT_TYPE = "out_file"
|
109
|
+
|
110
|
+
config_section :format do
|
111
|
+
config_set_default :@type, DEFAULT_FORMAT_TYPE
|
112
|
+
end
|
135
113
|
|
136
114
|
attr_reader :bucket
|
137
115
|
|
138
116
|
MAX_HEX_RANDOM_LENGTH = 16
|
139
117
|
|
140
118
|
def configure(conf)
|
119
|
+
compat_parameters_convert(conf, :buffer, :formatter, :inject)
|
120
|
+
|
141
121
|
super
|
142
122
|
|
143
123
|
if @s3_endpoint && @s3_endpoint.end_with?('amazonaws.com')
|
144
|
-
raise ConfigError, "s3_endpoint parameter is not supported for S3, use s3_region instead. This parameter is for S3 compatible services"
|
124
|
+
raise Fluent::ConfigError, "s3_endpoint parameter is not supported for S3, use s3_region instead. This parameter is for S3 compatible services"
|
145
125
|
end
|
146
126
|
|
147
127
|
begin
|
148
|
-
|
128
|
+
buffer_type = @buffer_config[:@type]
|
129
|
+
@compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(buffer_type: buffer_type, log: log)
|
149
130
|
rescue
|
150
|
-
|
131
|
+
log.warn "#{@store_as} not found. Use 'text' instead"
|
151
132
|
@compressor = TextCompressor.new
|
152
133
|
end
|
153
134
|
@compressor.configure(conf)
|
154
135
|
|
155
|
-
@formatter =
|
156
|
-
@formatter.configure(conf)
|
157
|
-
|
158
|
-
if @localtime
|
159
|
-
@path_slicer = Proc.new {|path|
|
160
|
-
Time.now.strftime(path)
|
161
|
-
}
|
162
|
-
else
|
163
|
-
@path_slicer = Proc.new {|path|
|
164
|
-
Time.now.utc.strftime(path)
|
165
|
-
}
|
166
|
-
end
|
136
|
+
@formatter = formatter_create
|
167
137
|
|
168
138
|
if @hex_random_length > MAX_HEX_RANDOM_LENGTH
|
169
|
-
raise ConfigError, "hex_random_length parameter must be less than or equal to #{MAX_HEX_RANDOM_LENGTH}"
|
139
|
+
raise Fluent::ConfigError, "hex_random_length parameter must be less than or equal to #{MAX_HEX_RANDOM_LENGTH}"
|
170
140
|
end
|
171
141
|
|
172
142
|
if @reduced_redundancy
|
173
|
-
|
143
|
+
log.warn "reduced_redundancy parameter is deprecated. Use storage_class parameter instead"
|
174
144
|
@storage_class = "REDUCED_REDUNDANCY"
|
175
145
|
end
|
176
146
|
|
177
|
-
@
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
log.warn "Set 'check_object false' and s3_object_key_format is specified. Check s3_object_key_format is unique in each write. If not, existing file will be overwritten."
|
182
|
-
else
|
183
|
-
log.warn "Set 'check_object false' and s3_object_key_format is not specified. Use '%{path}/%{date_slice}_%{hms_slice}.%{file_extension}' for s3_object_key_format"
|
184
|
-
@s3_object_key_format = "%{path}/%{date_slice}_%{hms_slice}.%{file_extension}"
|
185
|
-
end
|
186
|
-
end
|
147
|
+
@s3_object_key_format = process_s3_object_key_format
|
148
|
+
# For backward compatibility
|
149
|
+
# TODO: Remove time_slice_format when end of support compat_parameters
|
150
|
+
@configured_time_slice_format = conf['time_slice_format']
|
187
151
|
@values_for_s3_object_chunk = {}
|
188
152
|
end
|
189
153
|
|
@@ -195,48 +159,48 @@ module Fluent
|
|
195
159
|
options[:force_path_style] = @force_path_style
|
196
160
|
options[:compute_checksums] = @compute_checksums unless @compute_checksums.nil?
|
197
161
|
options[:signature_version] = @signature_version unless @signature_version.nil?
|
198
|
-
options[:ssl_verify_peer] = @ssl_verify_peer
|
199
|
-
log.on_trace do
|
200
|
-
options[:http_wire_trace] = true
|
201
|
-
options[:logger] = log
|
202
|
-
end
|
203
162
|
|
204
163
|
s3_client = Aws::S3::Client.new(options)
|
205
|
-
@s3 = Aws::S3::Resource.new(:
|
164
|
+
@s3 = Aws::S3::Resource.new(client: s3_client)
|
206
165
|
@bucket = @s3.bucket(@s3_bucket)
|
207
166
|
|
208
167
|
check_apikeys if @check_apikey_on_start
|
209
168
|
ensure_bucket if @check_bucket
|
210
169
|
|
170
|
+
if !@check_object
|
171
|
+
@s3_object_key_format = "%{path}/%{date_slice}_%{hms_slice}.%{file_extension}"
|
172
|
+
end
|
173
|
+
|
211
174
|
super
|
212
175
|
end
|
213
176
|
|
214
177
|
def format(tag, time, record)
|
215
|
-
|
178
|
+
r = inject_values_to_record(tag, time, record)
|
179
|
+
@formatter.format(tag, time, r)
|
216
180
|
end
|
217
181
|
|
218
182
|
def write(chunk)
|
219
183
|
i = 0
|
220
184
|
previous_path = nil
|
185
|
+
time_slice_format = @configured_time_slice_format || timekey_to_timeformat(@buffer_config['timekey'])
|
186
|
+
time_slice = Time.at(chunk.metadata.timekey).utc.strftime(time_slice_format)
|
221
187
|
|
222
188
|
if @check_object
|
223
189
|
begin
|
224
|
-
path =
|
190
|
+
path = extract_placeholders(@path, chunk.metadata)
|
225
191
|
|
226
192
|
@values_for_s3_object_chunk[chunk.unique_id] ||= {
|
227
|
-
"hex_random" => hex_random(chunk),
|
193
|
+
"%{hex_random}" => hex_random(chunk),
|
228
194
|
}
|
229
195
|
values_for_s3_object_key = {
|
230
|
-
"path" => path,
|
231
|
-
"time_slice" =>
|
232
|
-
"file_extension" => @compressor.ext,
|
233
|
-
"index" => i,
|
196
|
+
"%{path}" => path,
|
197
|
+
"%{time_slice}" => time_slice,
|
198
|
+
"%{file_extension}" => @compressor.ext,
|
199
|
+
"%{index}" => i,
|
234
200
|
}.merge!(@values_for_s3_object_chunk[chunk.unique_id])
|
235
|
-
values_for_s3_object_key[
|
201
|
+
values_for_s3_object_key["%{uuid_flush}".freeze] = uuid_random if @uuid_flush_enabled
|
236
202
|
|
237
|
-
s3path = @s3_object_key_format.gsub(%r(%{[^}]+}))
|
238
|
-
values_for_s3_object_key[expr[2...expr.size-1]]
|
239
|
-
}
|
203
|
+
s3path = @s3_object_key_format.gsub(%r(%{[^}]+}), values_for_s3_object_key)
|
240
204
|
if (i > 0) && (s3path == previous_path)
|
241
205
|
if @overwrite
|
242
206
|
log.warn "#{s3path} already exists, but will overwrite"
|
@@ -256,29 +220,32 @@ module Fluent
|
|
256
220
|
hms_slicer = Time.now.utc.strftime("%H%M%S")
|
257
221
|
end
|
258
222
|
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
"
|
263
|
-
"file_extension" => @compressor.ext,
|
264
|
-
"hms_slice" => hms_slicer,
|
265
|
-
}
|
266
|
-
s3path = @s3_object_key_format.gsub(%r(%{[^}]+})) { |expr|
|
267
|
-
values_for_s3_object_key[expr[2...expr.size-1]]
|
223
|
+
path = extract_placeholders(@path, chunk.metadata)
|
224
|
+
|
225
|
+
@values_for_s3_object_chunk[chunk.unique_id] ||= {
|
226
|
+
"%{hex_random}" => hex_random(chunk),
|
268
227
|
}
|
228
|
+
values_for_s3_object_key = {
|
229
|
+
"%{path}" => path,
|
230
|
+
"%{time_slice}" => time_slice,
|
231
|
+
"%{file_extension}" => @compressor.ext,
|
232
|
+
}.merge!(@values_for_s3_object_chunk[chunk.unique_id])
|
233
|
+
values_for_s3_object_key["%{uuid_flush}".freeze] = uuid_random if @uuid_flush_enabled
|
234
|
+
|
235
|
+
s3path = @s3_object_key_format.gsub(%r(%{[^}]+}), values_for_s3_object_key)
|
269
236
|
end
|
270
237
|
|
271
|
-
tmp = Tempfile.new("s3-"
|
238
|
+
tmp = Tempfile.new("s3-")
|
272
239
|
tmp.binmode
|
273
240
|
begin
|
274
241
|
@compressor.compress(chunk, tmp)
|
275
242
|
tmp.rewind
|
276
|
-
log.debug { "out_s3: write chunk: {key:#{chunk.key},
|
243
|
+
log.debug { "out_s3: write chunk: {key:#{chunk.key},tsuffix:#{tsuffix(chunk)}} to s3://#{@s3_bucket}/#{s3path}" }
|
277
244
|
|
278
245
|
put_options = {
|
279
|
-
:
|
280
|
-
:
|
281
|
-
:
|
246
|
+
body: tmp,
|
247
|
+
content_type: @compressor.content_type,
|
248
|
+
storage_class: @storage_class,
|
282
249
|
}
|
283
250
|
put_options[:server_side_encryption] = @use_server_side_encryption if @use_server_side_encryption
|
284
251
|
put_options[:ssekms_key_id] = @ssekms_key_id if @ssekms_key_id
|
@@ -286,16 +253,12 @@ module Fluent
|
|
286
253
|
put_options[:sse_customer_key] = @sse_customer_key if @sse_customer_key
|
287
254
|
put_options[:sse_customer_key_md5] = @sse_customer_key_md5 if @sse_customer_key_md5
|
288
255
|
put_options[:acl] = @acl if @acl
|
289
|
-
put_options[:grant_full_control] = @grant_full_control if @grant_full_control
|
290
|
-
put_options[:grant_read] = @grant_read if @grant_read
|
291
|
-
put_options[:grant_read_acp] = @grant_read_acp if @grant_read_acp
|
292
|
-
put_options[:grant_write_acp] = @grant_write_acp if @grant_write_acp
|
293
256
|
@bucket.object(s3path).put(put_options)
|
294
257
|
|
295
258
|
@values_for_s3_object_chunk.delete(chunk.unique_id)
|
296
259
|
|
297
260
|
if @warn_for_delay
|
298
|
-
if Time.
|
261
|
+
if Time.at(chunk.metadata.timekey) < Time.now - @warn_for_delay
|
299
262
|
log.warn { "out_s3: delayed events were put to s3://#{@s3_bucket}/#{s3path}" }
|
300
263
|
end
|
301
264
|
end
|
@@ -306,14 +269,8 @@ module Fluent
|
|
306
269
|
|
307
270
|
private
|
308
271
|
|
309
|
-
# v0.14 has a useful Fluent::UniqueId.hex(unique_id) method, though
|
310
|
-
def unique_hex(chunk)
|
311
|
-
unique_id = chunk.unique_id
|
312
|
-
unique_id.unpack('C*').map {|x| x.to_s(16) }.join('')
|
313
|
-
end
|
314
|
-
|
315
272
|
def hex_random(chunk)
|
316
|
-
unique_hex =
|
273
|
+
unique_hex = Fluent::UniqueId.hex(chunk.unique_id)
|
317
274
|
unique_hex.reverse! # unique_hex is like (time_sec, time_usec, rand) => reversing gives more randomness
|
318
275
|
unique_hex[0...@hex_random_length]
|
319
276
|
end
|
@@ -322,25 +279,36 @@ module Fluent
|
|
322
279
|
::UUIDTools::UUID.random_create.to_s
|
323
280
|
end
|
324
281
|
|
282
|
+
# This is stolen from Fluentd
|
283
|
+
def timekey_to_timeformat(timekey)
|
284
|
+
case timekey
|
285
|
+
when nil then ''
|
286
|
+
when 0...60 then '%Y%m%d%H%M%S' # 60 exclusive
|
287
|
+
when 60...3600 then '%Y%m%d%H%M'
|
288
|
+
when 3600...86400 then '%Y%m%d%H'
|
289
|
+
else '%Y%m%d'
|
290
|
+
end
|
291
|
+
end
|
292
|
+
|
325
293
|
def ensure_bucket
|
326
294
|
if !@bucket.exists?
|
327
295
|
if @auto_create_bucket
|
328
296
|
log.info "Creating bucket #{@s3_bucket} on #{@s3_endpoint}"
|
329
|
-
@s3.create_bucket(:
|
297
|
+
@s3.create_bucket(bucket: @s3_bucket)
|
330
298
|
else
|
331
299
|
raise "The specified bucket does not exist: bucket = #{@s3_bucket}"
|
332
300
|
end
|
333
301
|
end
|
334
302
|
end
|
335
303
|
|
336
|
-
def
|
304
|
+
def process_s3_object_key_format
|
337
305
|
%W(%{uuid} %{uuid:random} %{uuid:hostname} %{uuid:timestamp}).each { |ph|
|
338
|
-
if
|
339
|
-
raise ConfigError, %!#{ph} placeholder is removed!
|
306
|
+
if @s3_object_key_format.include?(ph)
|
307
|
+
raise ConfigError, %!#{ph} placeholder in s3_object_key_format is removed!
|
340
308
|
end
|
341
309
|
}
|
342
310
|
|
343
|
-
if
|
311
|
+
if @s3_object_key_format.include?('%{uuid_flush}')
|
344
312
|
# test uuidtools works or not
|
345
313
|
begin
|
346
314
|
require 'uuidtools'
|
@@ -355,18 +323,18 @@ module Fluent
|
|
355
323
|
@uuid_flush_enabled = true
|
356
324
|
end
|
357
325
|
|
358
|
-
|
326
|
+
@s3_object_key_format.gsub('%{hostname}') { |expr|
|
359
327
|
log.warn "%{hostname} will be removed in the future. Use \"\#{Socket.gethostname}\" instead"
|
360
328
|
Socket.gethostname
|
361
329
|
}
|
362
330
|
end
|
363
331
|
|
364
332
|
def check_apikeys
|
365
|
-
@bucket.objects(:
|
333
|
+
@bucket.objects(prefix: @path).first
|
366
334
|
rescue Aws::S3::Errors::NoSuchBucket
|
367
335
|
# ignore NoSuchBucket Error because ensure_bucket checks it.
|
368
336
|
rescue => e
|
369
|
-
raise "can't call S3 API. Please check your
|
337
|
+
raise "can't call S3 API. Please check your aws_key_id / aws_sec_key or s3_region configuration. error = #{e.inspect}"
|
370
338
|
end
|
371
339
|
|
372
340
|
def setup_credentials
|
@@ -384,7 +352,7 @@ module Fluent
|
|
384
352
|
credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
|
385
353
|
credentials_options[:external_id] = c.external_id if c.external_id
|
386
354
|
if @s3_region
|
387
|
-
credentials_options[:client] = Aws::STS::Client.new(:
|
355
|
+
credentials_options[:client] = Aws::STS::Client.new(region: @s3_region)
|
388
356
|
end
|
389
357
|
options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
|
390
358
|
when @instance_profile_credentials
|
@@ -405,7 +373,7 @@ module Fluent
|
|
405
373
|
credentials_options[:profile_name] = c.profile_name if c.profile_name
|
406
374
|
options[:credentials] = Aws::SharedCredentials.new(credentials_options)
|
407
375
|
when @aws_iam_retries
|
408
|
-
|
376
|
+
log.warn("'aws_iam_retries' parameter is deprecated. Use 'instance_profile_credentials' instead")
|
409
377
|
credentials_options[:retries] = @aws_iam_retries
|
410
378
|
if ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
|
411
379
|
options[:credentials] = Aws::ECSCredentials.new(credentials_options)
|
@@ -420,9 +388,7 @@ module Fluent
|
|
420
388
|
end
|
421
389
|
|
422
390
|
class Compressor
|
423
|
-
include Configurable
|
424
|
-
|
425
|
-
config_param :tmp_dir, :string, :default => nil
|
391
|
+
include Fluent::Configurable
|
426
392
|
|
427
393
|
def initialize(opts = {})
|
428
394
|
super()
|
@@ -454,7 +420,7 @@ module Fluent
|
|
454
420
|
begin
|
455
421
|
Open3.capture3("#{command} -V")
|
456
422
|
rescue Errno::ENOENT
|
457
|
-
raise ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
|
423
|
+
raise Fluent::ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
|
458
424
|
end
|
459
425
|
end
|
460
426
|
end
|
@@ -501,7 +467,7 @@ module Fluent
|
|
501
467
|
end
|
502
468
|
end
|
503
469
|
|
504
|
-
COMPRESSOR_REGISTRY = Registry.new(:s3_compressor_type, 'fluent/plugin/s3_compressor_')
|
470
|
+
COMPRESSOR_REGISTRY = Fluent::Registry.new(:s3_compressor_type, 'fluent/plugin/s3_compressor_')
|
505
471
|
{
|
506
472
|
'gzip' => GzipCompressor,
|
507
473
|
'json' => JsonCompressor,
|