fluent-plugin-s3 0.6.0.pre1 → 0.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ChangeLog +7 -0
- data/README.rdoc +89 -2
- data/VERSION +1 -2
- data/fluent-plugin-s3.gemspec +1 -1
- data/lib/fluent/plugin/out_s3.rb +71 -42
- data/test/test_out_s3.rb +50 -0
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9a324977d9062b01736950c76f666d42161e93d6
|
4
|
+
data.tar.gz: 6d70b12fabf887f6ac991798ac4e5a0200c5afe5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 23a7cb16fca1b7dbe7f7c785d3096d8670c1bc7af033685e57ebfe7b8bcecc225bd3ad55abb72e71514d983e7319bea13629df0f1f51638fffffd60ac23c2e27
|
7
|
+
data.tar.gz: 30b2f703798b58b3b866b1f3fe1bdcf64081c4c73eede9d570a631d4c297a7be662267c59bbd46a17c286047a00481898493e8bf1cefed2acf2c15c3fc55b00f
|
data/ChangeLog
CHANGED
data/README.rdoc
CHANGED
@@ -21,7 +21,7 @@ Simply use RubyGems:
|
|
21
21
|
type s3
|
22
22
|
|
23
23
|
aws_key_id YOUR_AWS_KEY_ID
|
24
|
-
aws_sec_key
|
24
|
+
aws_sec_key YOUR_AWS_SECRET_KEY
|
25
25
|
s3_bucket YOUR_S3_BUCKET_NAME
|
26
26
|
s3_region ap-northeast-1
|
27
27
|
s3_object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
|
@@ -52,6 +52,7 @@ Simply use RubyGems:
|
|
52
52
|
- %{index}
|
53
53
|
- %{file_extension}
|
54
54
|
- %{uuid_flush}
|
55
|
+
- %{hex_random}
|
55
56
|
|
56
57
|
to decide keys dynamically.
|
57
58
|
|
@@ -59,7 +60,8 @@ to decide keys dynamically.
|
|
59
60
|
%{time_slice} is the time-slice in text that are formatted with *time_slice_format*.
|
60
61
|
%{index} is the sequential number starts from 0, increments when multiple files are uploaded to S3 in the same time slice.
|
61
62
|
%{file_extention} is always "gz" for now.
|
62
|
-
%{uuid_flush} a uuid that is replaced
|
63
|
+
%{uuid_flush} a uuid that is replaced for each buffer chunk to be flushed
|
64
|
+
%{hex_random} a random hex string that is replaced for each buffer chunk, not assured to be unique. This is used to follow a way of peformance tuning, `Add a Hex Hash Prefix to Key Name`, written in [Request Rate and Performance Considerations - Amazon Simple Storage Service](https://docs.aws.amazon.com/AmazonS3/latest/dev/request-rate-perf-considerations.html). You can configure the length of string with a `hex_random_length` parameter (Default: 4).
|
63
65
|
|
64
66
|
The default format is "%{path}%{time_slice}_%{index}.%{file_extension}".
|
65
67
|
|
@@ -87,6 +89,8 @@ The {fluent-mixin-config-placeholders}[https://github.com/tagomoris/fluent-mixin
|
|
87
89
|
|
88
90
|
s3_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
|
89
91
|
|
92
|
+
[force_path_style] :force_path_style (Boolean) — default: false — When set to true, the bucket name is always left in the request URI and never moved to the host as a sub-domain. See Plugins::S3BucketDns for more details.
|
93
|
+
|
90
94
|
[store_as] archive format on S3. You can use serveral format:
|
91
95
|
|
92
96
|
- gzip (default)
|
@@ -173,6 +177,89 @@ You can change key name by "message_key" option.
|
|
173
177
|
To use cross-account access, you will need to create a bucket policy granting
|
174
178
|
the specific access required. Refer to the {AWS documentation}[http://docs.aws.amazon.com/AmazonS3/latest/dev/example-walkthroughs-managing-access-example3.html] for examples.
|
175
179
|
|
180
|
+
[hex_random_length] The length of `%{hex_random}` placeholder. Default is 4 as written in [Request Rate and Performance Considerations - Amazon Simple Storage Service](https://docs.aws.amazon.com/AmazonS3/latest/dev/request-rate-perf-considerations.html).
|
181
|
+
|
182
|
+
[overwrite] Overwrite already existing path. Default is false, which raises an error if a s3 object of the same path already exists, or increment the `%{index}` placeholder until finding an absent path.
|
183
|
+
|
184
|
+
=== assume_role_credentials
|
185
|
+
|
186
|
+
Typically, you use AssumeRole for cross-account access or federation.
|
187
|
+
|
188
|
+
<match *>
|
189
|
+
type s3
|
190
|
+
|
191
|
+
<assume_role_credentials>
|
192
|
+
role_arn ROLE_ARN
|
193
|
+
role_session_name ROLE_SESSION_NAME
|
194
|
+
</assume_role_credentials>
|
195
|
+
</match>
|
196
|
+
|
197
|
+
See also:
|
198
|
+
|
199
|
+
- {Using IAM Roles - AWS Identity and Access Management}[http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html]
|
200
|
+
- {Aws::STS::Client}[http://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html]
|
201
|
+
- {Aws::AssumeRoleCredentials}[http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html]
|
202
|
+
|
203
|
+
[role_arn (required)] The Amazon Resource Name (ARN) of the role to assume.
|
204
|
+
|
205
|
+
[role_session_name (required)] An identifier for the assumed role session.
|
206
|
+
|
207
|
+
[policy] An IAM policy in JSON format.
|
208
|
+
|
209
|
+
[duration_seconds] The duration, in seconds, of the role session. The value can range from 900 seconds (15 minutes) to 3600 seconds (1 hour). By default, the value is set to 3600 seconds.
|
210
|
+
|
211
|
+
[external_id] A unique identifier that is used by third parties when assuming roles in their customers' accounts.
|
212
|
+
|
213
|
+
=== instance_profile_credentials
|
214
|
+
|
215
|
+
Retrieve temporary security credentials via HTTP request. This is useful on EC2 instance.
|
216
|
+
|
217
|
+
<match *>
|
218
|
+
type s3
|
219
|
+
|
220
|
+
<instance_profile_credentials>
|
221
|
+
ip_address IP_ADDRESS
|
222
|
+
port PORT
|
223
|
+
</instance_profile_credentials>
|
224
|
+
</match>
|
225
|
+
|
226
|
+
See also:
|
227
|
+
|
228
|
+
- {Aws::InstanceProfileCredentials}[http://docs.aws.amazon.com/sdkforruby/api/Aws/InstanceProfileCredentials.html]
|
229
|
+
- {Temporary Security Credentials - AWS Identity and Access Management}[http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html]
|
230
|
+
- {Instance Metadata and User Data - Amazon Elastic Compute Cloud}[http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html]
|
231
|
+
|
232
|
+
[retries] Number of times to retry when retrieving credentials. Default is 5.
|
233
|
+
|
234
|
+
[ip_address] Default is 169.254.169.254.
|
235
|
+
|
236
|
+
[port] Default is 80.
|
237
|
+
|
238
|
+
[http_open_timeout] Default is 5.
|
239
|
+
|
240
|
+
[http_read_timeout] Default is 5.
|
241
|
+
|
242
|
+
=== shared_credentials
|
243
|
+
|
244
|
+
This loads AWS access credentials from local ini file. This is useful for local developing.
|
245
|
+
|
246
|
+
<match *>
|
247
|
+
type s3
|
248
|
+
|
249
|
+
<shared_credentials>
|
250
|
+
path PATH
|
251
|
+
profile_name PROFILE_NAME
|
252
|
+
</shared_credentials>
|
253
|
+
</match>
|
254
|
+
|
255
|
+
See also:
|
256
|
+
|
257
|
+
- {Aws::SharedCredentials}[http://docs.aws.amazon.com/sdkforruby/api/Aws/SharedCredentials.html]
|
258
|
+
|
259
|
+
[path] Path to the shared file. Defaults to "#{Dir.home}/.aws/credentials".
|
260
|
+
|
261
|
+
[profile_name] Defaults to 'default' or `ENV['AWS_PROFILE']`.
|
262
|
+
|
176
263
|
== IAM Policy
|
177
264
|
|
178
265
|
The following is an example for a minimal IAM policy needed to write to an s3 bucket (matches my-s3bucket/logs, my-s3bucket-test, etc.).
|
data/VERSION
CHANGED
@@ -1,2 +1 @@
|
|
1
|
-
0.6.0
|
2
|
-
|
1
|
+
0.6.0
|
data/fluent-plugin-s3.gemspec
CHANGED
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
|
|
17
17
|
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
18
|
gem.require_paths = ['lib']
|
19
19
|
|
20
|
-
gem.add_dependency "fluentd", [">= 0.10.
|
20
|
+
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
21
21
|
gem.add_dependency "aws-sdk", "~> 2"
|
22
22
|
gem.add_dependency "yajl-ruby", "~> 1.0"
|
23
23
|
gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
|
data/lib/fluent/plugin/out_s3.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
module Fluent
|
2
2
|
require 'fluent/mixin/config_placeholders'
|
3
|
+
require 'securerandom'
|
3
4
|
|
4
5
|
class S3Output < Fluent::TimeSlicedOutput
|
5
6
|
Fluent::Plugin.register_output('s3', self)
|
@@ -43,6 +44,7 @@ module Fluent
|
|
43
44
|
config_param :s3_region, :string, :default => ENV["AWS_REGION"] || "us-east-1"
|
44
45
|
config_param :s3_endpoint, :string, :default => nil
|
45
46
|
config_param :s3_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
|
47
|
+
config_param :force_path_style, :bool, :default => false
|
46
48
|
config_param :store_as, :string, :default => "gzip"
|
47
49
|
config_param :auto_create_bucket, :bool, :default => true
|
48
50
|
config_param :check_apikey_on_start, :bool, :default => true
|
@@ -51,6 +53,8 @@ module Fluent
|
|
51
53
|
config_param :storage_class, :string, :default => "STANDARD"
|
52
54
|
config_param :format, :string, :default => 'out_file'
|
53
55
|
config_param :acl, :string, :default => :private
|
56
|
+
config_param :hex_random_length, :integer, :default => 4
|
57
|
+
config_param :overwrite, :bool, :default => false
|
54
58
|
|
55
59
|
attr_reader :bucket
|
56
60
|
|
@@ -69,15 +73,14 @@ module Fluent
|
|
69
73
|
|
70
74
|
begin
|
71
75
|
@compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
|
72
|
-
rescue
|
76
|
+
rescue
|
73
77
|
$log.warn "#{@store_as} not found. Use 'text' instead"
|
74
78
|
@compressor = TextCompressor.new
|
75
79
|
end
|
76
80
|
@compressor.configure(conf)
|
77
81
|
|
78
|
-
|
79
|
-
conf
|
80
|
-
@formatter = TextFormatter.create(conf)
|
82
|
+
@formatter = Plugin.new_formatter(@format)
|
83
|
+
@formatter.configure(conf)
|
81
84
|
|
82
85
|
if @localtime
|
83
86
|
@path_slicer = Proc.new {|path|
|
@@ -90,49 +93,17 @@ module Fluent
|
|
90
93
|
end
|
91
94
|
|
92
95
|
@storage_class = "REDUCED_REDUNDANCY" if @reduced_redundancy
|
96
|
+
@values_for_s3_object_chunk = {}
|
93
97
|
end
|
94
98
|
|
95
99
|
def start
|
96
100
|
super
|
97
|
-
options =
|
98
|
-
credentials_options = {}
|
99
|
-
case
|
100
|
-
when @aws_key_id && @aws_sec_key
|
101
|
-
options[:access_key_id] = @aws_key_id
|
102
|
-
options[:secret_access_key] = @aws_sec_key
|
103
|
-
when @assume_role_credentials
|
104
|
-
c = @assume_role_credentials
|
105
|
-
credentials_options[:role_arn] = c.role_arn
|
106
|
-
credentials_options[:role_session_name] = c.role_session_name
|
107
|
-
credentials_options[:policy] = c.policy if c.policy
|
108
|
-
credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
|
109
|
-
credentials_options[:external_id] = c.external_id if c.external_id
|
110
|
-
options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
|
111
|
-
when @instance_profile_credentials
|
112
|
-
c = @instance_profile_credentials
|
113
|
-
credentials_options[:retries] = c.retries if c.retries
|
114
|
-
credentials_options[:ip_address] = c.ip_address if c.ip_address
|
115
|
-
credentials_options[:port] = c.port if c.port
|
116
|
-
credentials_options[:http_open_timeout] = c.http_open_timeout if c.http_open_timeout
|
117
|
-
credentials_options[:http_read_timeout] = c.http_read_timeout if c.http_read_timeout
|
118
|
-
options[:credentials] = Aws::InstanceProfileCredentials.new(credentials_options)
|
119
|
-
when @shared_credentials
|
120
|
-
c = @shared_credentials
|
121
|
-
credentials_options[:path] = c.path if c.path
|
122
|
-
credentials_options[:profile_name] = c.profile_name if c.profile_name
|
123
|
-
options[:credentials] = Aws::SharedCredentials.new(credentials_options)
|
124
|
-
when @aws_iam_retries
|
125
|
-
$log.warn("'aws_iam_retries' parameter is deprecated. Use 'instance_profile_credentials' instead")
|
126
|
-
credentials_options[:retries] = @aws_iam_retries
|
127
|
-
options[:credentials] = Aws::InstanceProfileCredentials.new(credentials_options)
|
128
|
-
else
|
129
|
-
# Use default credentials
|
130
|
-
# See http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html
|
131
|
-
end
|
101
|
+
options = setup_credentials
|
132
102
|
options[:region] = @s3_region if @s3_region
|
133
103
|
options[:endpoint] = @s3_endpoint if @s3_endpoint
|
134
104
|
options[:http_proxy] = @proxy_uri if @proxy_uri
|
135
105
|
options[:s3_server_side_encryption] = @use_server_side_encryption.to_sym if @use_server_side_encryption
|
106
|
+
options[:force_path_style] = @force_path_style
|
136
107
|
|
137
108
|
s3_client = Aws::S3::Client.new(options)
|
138
109
|
@s3 = Aws::S3::Resource.new(:client => s3_client)
|
@@ -140,6 +111,9 @@ module Fluent
|
|
140
111
|
|
141
112
|
check_apikeys if @check_apikey_on_start
|
142
113
|
ensure_bucket
|
114
|
+
|
115
|
+
# Securerandom.hex(2) returns 4 length hex
|
116
|
+
@hex_random_n = (@hex_random_length + 1) / 2
|
143
117
|
end
|
144
118
|
|
145
119
|
def format(tag, time, record)
|
@@ -152,18 +126,28 @@ module Fluent
|
|
152
126
|
|
153
127
|
begin
|
154
128
|
path = @path_slicer.call(@path)
|
129
|
+
|
130
|
+
@values_for_s3_object_chunk[chunk.key] ||= {
|
131
|
+
"hex_random" => hex_random,
|
132
|
+
"uuid_flush" => uuid_random,
|
133
|
+
}
|
155
134
|
values_for_s3_object_key = {
|
156
135
|
"path" => path,
|
157
136
|
"time_slice" => chunk.key,
|
158
137
|
"file_extension" => @compressor.ext,
|
159
138
|
"index" => i,
|
160
|
-
|
161
|
-
|
139
|
+
}.merge!(@values_for_s3_object_chunk[chunk.key])
|
140
|
+
|
162
141
|
s3path = @s3_object_key_format.gsub(%r(%{[^}]+})) { |expr|
|
163
142
|
values_for_s3_object_key[expr[2...expr.size-1]]
|
164
143
|
}
|
165
144
|
if (i > 0) && (s3path == previous_path)
|
166
|
-
|
145
|
+
if @overwrite
|
146
|
+
log.warn "#{s3path} already exists, but will overwrite"
|
147
|
+
break
|
148
|
+
else
|
149
|
+
raise "duplicated path is generated. use %{index} in s3_object_key_format: path = #{s3path}"
|
150
|
+
end
|
167
151
|
end
|
168
152
|
|
169
153
|
i += 1
|
@@ -174,16 +158,22 @@ module Fluent
|
|
174
158
|
begin
|
175
159
|
@compressor.compress(chunk, tmp)
|
176
160
|
tmp.rewind
|
161
|
+
log.debug { "out_s3: trying to write {object_id:#{chunk.object_id},time_slice:#{chunk.key}} to s3://#{@s3_bucket}/#{s3path}" }
|
177
162
|
@bucket.object(s3path).put(:body => tmp,
|
178
163
|
:content_type => @compressor.content_type,
|
179
164
|
:storage_class => @storage_class)
|
180
165
|
ensure
|
166
|
+
@values_for_s3_object_chunk.delete(chunk.key)
|
181
167
|
tmp.close(true) rescue nil
|
182
168
|
end
|
183
169
|
end
|
184
170
|
|
185
171
|
private
|
186
172
|
|
173
|
+
def hex_random
|
174
|
+
SecureRandom.hex(@hex_random_n)[0...@hex_random_length]
|
175
|
+
end
|
176
|
+
|
187
177
|
def ensure_bucket
|
188
178
|
if !@bucket.exists?
|
189
179
|
if @auto_create_bucket
|
@@ -203,6 +193,45 @@ module Fluent
|
|
203
193
|
raise "can't call S3 API. Please check your aws_key_id / aws_sec_key or s3_region configuration. error = #{e.inspect}"
|
204
194
|
end
|
205
195
|
|
196
|
+
def setup_credentials
|
197
|
+
options = {}
|
198
|
+
credentials_options = {}
|
199
|
+
case
|
200
|
+
when @aws_key_id && @aws_sec_key
|
201
|
+
options[:access_key_id] = @aws_key_id
|
202
|
+
options[:secret_access_key] = @aws_sec_key
|
203
|
+
when @assume_role_credentials
|
204
|
+
c = @assume_role_credentials
|
205
|
+
credentials_options[:role_arn] = c.role_arn
|
206
|
+
credentials_options[:role_session_name] = c.role_session_name
|
207
|
+
credentials_options[:policy] = c.policy if c.policy
|
208
|
+
credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
|
209
|
+
credentials_options[:external_id] = c.external_id if c.external_id
|
210
|
+
options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
|
211
|
+
when @instance_profile_credentials
|
212
|
+
c = @instance_profile_credentials
|
213
|
+
credentials_options[:retries] = c.retries if c.retries
|
214
|
+
credentials_options[:ip_address] = c.ip_address if c.ip_address
|
215
|
+
credentials_options[:port] = c.port if c.port
|
216
|
+
credentials_options[:http_open_timeout] = c.http_open_timeout if c.http_open_timeout
|
217
|
+
credentials_options[:http_read_timeout] = c.http_read_timeout if c.http_read_timeout
|
218
|
+
options[:credentials] = Aws::InstanceProfileCredentials.new(credentials_options)
|
219
|
+
when @shared_credentials
|
220
|
+
c = @shared_credentials
|
221
|
+
credentials_options[:path] = c.path if c.path
|
222
|
+
credentials_options[:profile_name] = c.profile_name if c.profile_name
|
223
|
+
options[:credentials] = Aws::SharedCredentials.new(credentials_options)
|
224
|
+
when @aws_iam_retries
|
225
|
+
$log.warn("'aws_iam_retries' parameter is deprecated. Use 'instance_profile_credentials' instead")
|
226
|
+
credentials_options[:retries] = @aws_iam_retries
|
227
|
+
options[:credentials] = Aws::InstanceProfileCredentials.new(credentials_options)
|
228
|
+
else
|
229
|
+
# Use default credentials
|
230
|
+
# See http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html
|
231
|
+
end
|
232
|
+
options
|
233
|
+
end
|
234
|
+
|
206
235
|
class Compressor
|
207
236
|
include Configurable
|
208
237
|
|
data/test/test_out_s3.rb
CHANGED
@@ -44,6 +44,7 @@ class S3OutputTest < Test::Unit::TestCase
|
|
44
44
|
assert_equal 'log', d.instance.path
|
45
45
|
assert_equal 'gz', d.instance.instance_variable_get(:@compressor).ext
|
46
46
|
assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@compressor).content_type
|
47
|
+
assert_equal false, d.instance.force_path_style
|
47
48
|
end
|
48
49
|
|
49
50
|
def test_s3_endpoint_with_valid_endpoint
|
@@ -87,6 +88,13 @@ class S3OutputTest < Test::Unit::TestCase
|
|
87
88
|
assert(e.is_a?(Fluent::ConfigError))
|
88
89
|
end
|
89
90
|
|
91
|
+
def test_configure_with_path_style
|
92
|
+
conf = CONFIG.clone
|
93
|
+
conf << "\nforce_path_style true\n"
|
94
|
+
d = create_driver(conf)
|
95
|
+
assert d.instance.force_path_style
|
96
|
+
end
|
97
|
+
|
90
98
|
def test_path_slicing
|
91
99
|
config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
|
92
100
|
d = create_driver(config)
|
@@ -317,6 +325,48 @@ class S3OutputTest < Test::Unit::TestCase
|
|
317
325
|
FileUtils.rm_f(s3_test_file_path)
|
318
326
|
end
|
319
327
|
|
328
|
+
# ToDo: need to test hex_random does not change on retry, but it is difficult with
|
329
|
+
# the current fluentd test helper because it does not provide a way to run with the same chunks
|
330
|
+
def test_write_with_custom_s3_object_key_format_containing_hex_random_placeholder
|
331
|
+
# Partial mock the S3Bucket, not to make an actual connection to Amazon S3
|
332
|
+
setup_mocks(true)
|
333
|
+
|
334
|
+
hex = "012345"
|
335
|
+
mock(SecureRandom).hex(3) { hex }
|
336
|
+
|
337
|
+
# Assert content of event logs which are being sent to S3
|
338
|
+
s3obj = stub(Aws::S3::Object.new(:bucket_name => "test_bucket",
|
339
|
+
:key => "test",
|
340
|
+
:client => @s3_client))
|
341
|
+
s3obj.exists? { false }
|
342
|
+
s3_test_file_path = "/tmp/s3-test.txt"
|
343
|
+
tempfile = File.new(s3_test_file_path, "w")
|
344
|
+
mock(Tempfile).new("s3-") { tempfile }
|
345
|
+
s3obj.put(:body => tempfile,
|
346
|
+
:content_type => "application/x-gzip",
|
347
|
+
:storage_class => "STANDARD")
|
348
|
+
@s3_bucket.object("log/events/ts=20110102-13/events_0-#{hex[0...5]}.gz") { s3obj }
|
349
|
+
|
350
|
+
# We must use TimeSlicedOutputTestDriver instead of BufferedOutputTestDriver,
|
351
|
+
# to make assertions on chunks' keys
|
352
|
+
config = CONFIG_TIME_SLICE.gsub(/%{hostname}/,"%{hex_random}") << "\nhex_random_length 5"
|
353
|
+
d = create_time_sliced_driver(config)
|
354
|
+
|
355
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
356
|
+
d.emit({"a"=>1}, time)
|
357
|
+
d.emit({"a"=>2}, time)
|
358
|
+
|
359
|
+
# Finally, the instance of S3Output is initialized and then invoked
|
360
|
+
d.run
|
361
|
+
Zlib::GzipReader.open(s3_test_file_path) do |gz|
|
362
|
+
data = gz.read
|
363
|
+
assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
|
364
|
+
%[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
|
365
|
+
data
|
366
|
+
end
|
367
|
+
FileUtils.rm_f(s3_test_file_path)
|
368
|
+
end
|
369
|
+
|
320
370
|
def setup_mocks(exists_return = false)
|
321
371
|
@s3_client = stub(Aws::S3::Client.new(:stub_responses => true))
|
322
372
|
mock(Aws::S3::Client).new(anything).at_least(0) { @s3_client }
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.0
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2015-09
|
12
|
+
date: 2015-10-09 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -17,7 +17,7 @@ dependencies:
|
|
17
17
|
requirements:
|
18
18
|
- - ">="
|
19
19
|
- !ruby/object:Gem::Version
|
20
|
-
version: 0.10.
|
20
|
+
version: 0.10.58
|
21
21
|
- - "<"
|
22
22
|
- !ruby/object:Gem::Version
|
23
23
|
version: '2'
|
@@ -27,7 +27,7 @@ dependencies:
|
|
27
27
|
requirements:
|
28
28
|
- - ">="
|
29
29
|
- !ruby/object:Gem::Version
|
30
|
-
version: 0.10.
|
30
|
+
version: 0.10.58
|
31
31
|
- - "<"
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '2'
|
@@ -150,9 +150,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
150
150
|
version: '0'
|
151
151
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
152
152
|
requirements:
|
153
|
-
- - "
|
153
|
+
- - ">="
|
154
154
|
- !ruby/object:Gem::Version
|
155
|
-
version:
|
155
|
+
version: '0'
|
156
156
|
requirements: []
|
157
157
|
rubyforge_project:
|
158
158
|
rubygems_version: 2.2.3
|