fluent-plugin-s3-file-inclusion 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +13 -0
- data/.travis.yml +24 -0
- data/AUTHORS +2 -0
- data/ChangeLog +375 -0
- data/Gemfile +3 -0
- data/Gemfile.v0.12 +6 -0
- data/README.md +845 -0
- data/Rakefile +14 -0
- data/VERSION +1 -0
- data/appveyor.yml +25 -0
- data/fluent-plugin-s3.gemspec +26 -0
- data/lib/fluent/log-ext.rb +12 -0
- data/lib/fluent/plugin/in_s3.rb +419 -0
- data/lib/fluent/plugin/out_s3.rb +642 -0
- data/lib/fluent/plugin/s3_compressor_gzip_command.rb +52 -0
- data/lib/fluent/plugin/s3_compressor_lzma2.rb +35 -0
- data/lib/fluent/plugin/s3_compressor_lzo.rb +35 -0
- data/lib/fluent/plugin/s3_extractor_gzip_command.rb +46 -0
- data/lib/fluent/plugin/s3_extractor_lzma2.rb +40 -0
- data/lib/fluent/plugin/s3_extractor_lzo.rb +40 -0
- data/test/test_in_s3.rb +513 -0
- data/test/test_out_s3.rb +713 -0
- metadata +169 -0
data/test/test_out_s3.rb
ADDED
@@ -0,0 +1,713 @@
|
|
1
|
+
require 'fluent/test'
|
2
|
+
require 'fluent/test/helpers'
|
3
|
+
require 'fluent/test/log'
|
4
|
+
require 'fluent/test/driver/output'
|
5
|
+
require 'aws-sdk-s3'
|
6
|
+
require 'fluent/plugin/out_s3'
|
7
|
+
|
8
|
+
require 'test/unit/rr'
|
9
|
+
require 'zlib'
|
10
|
+
require 'fileutils'
|
11
|
+
require 'timecop'
|
12
|
+
require 'ostruct'
|
13
|
+
|
14
|
+
include Fluent::Test::Helpers
|
15
|
+
|
16
|
+
class S3OutputTest < Test::Unit::TestCase
|
17
|
+
def setup
|
18
|
+
# Fluent::Test.setup
|
19
|
+
end
|
20
|
+
|
21
|
+
def teardown
|
22
|
+
Dir.glob('test/tmp/*').each {|file| FileUtils.rm_f(file) }
|
23
|
+
end
|
24
|
+
|
25
|
+
CONFIG = %[
|
26
|
+
aws_key_id test_key_id
|
27
|
+
aws_sec_key test_sec_key
|
28
|
+
s3_bucket test_bucket
|
29
|
+
path log
|
30
|
+
utc
|
31
|
+
buffer_type memory
|
32
|
+
time_slice_format %Y%m%d-%H
|
33
|
+
]
|
34
|
+
|
35
|
+
def create_driver(conf = CONFIG)
|
36
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::S3Output) do
|
37
|
+
def format(tag, time, record)
|
38
|
+
super
|
39
|
+
end
|
40
|
+
|
41
|
+
def write(chunk)
|
42
|
+
chunk.read
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
def ensure_bucket
|
48
|
+
end
|
49
|
+
|
50
|
+
def check_apikeys
|
51
|
+
end
|
52
|
+
end.configure(conf)
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_configure
|
56
|
+
d = create_driver
|
57
|
+
assert_equal 'test_key_id', d.instance.aws_key_id
|
58
|
+
assert_equal 'test_sec_key', d.instance.aws_sec_key
|
59
|
+
assert_equal 'test_bucket', d.instance.s3_bucket
|
60
|
+
assert_equal 'log', d.instance.path
|
61
|
+
assert_equal 'gz', d.instance.instance_variable_get(:@compressor).ext
|
62
|
+
assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@compressor).content_type
|
63
|
+
assert_equal false, d.instance.force_path_style
|
64
|
+
assert_equal nil, d.instance.compute_checksums
|
65
|
+
assert_equal nil, d.instance.signature_version
|
66
|
+
assert_equal true, d.instance.check_bucket
|
67
|
+
assert_equal true, d.instance.check_object
|
68
|
+
end
|
69
|
+
|
70
|
+
def test_s3_endpoint_with_valid_endpoint
|
71
|
+
d = create_driver(CONFIG + 's3_endpoint riak-cs.example.com')
|
72
|
+
assert_equal 'riak-cs.example.com', d.instance.s3_endpoint
|
73
|
+
end
|
74
|
+
|
75
|
+
data('US West (Oregon)' => 's3-us-west-2.amazonaws.com',
|
76
|
+
'EU (Frankfurt)' => 's3.eu-central-1.amazonaws.com',
|
77
|
+
'Asia Pacific (Tokyo)' => 's3-ap-northeast-1.amazonaws.com')
|
78
|
+
def test_s3_endpoint_with_invalid_endpoint(endpoint)
|
79
|
+
assert_raise(Fluent::ConfigError, "s3_endpoint parameter is not supported, use s3_region instead. This parameter is for S3 compatible services") {
|
80
|
+
create_driver(CONFIG + "s3_endpoint #{endpoint}")
|
81
|
+
}
|
82
|
+
end
|
83
|
+
|
84
|
+
def test_configure_with_mime_type_json
|
85
|
+
conf = CONFIG.clone
|
86
|
+
conf << "\nstore_as json\n"
|
87
|
+
d = create_driver(conf)
|
88
|
+
assert_equal 'json', d.instance.instance_variable_get(:@compressor).ext
|
89
|
+
assert_equal 'application/json', d.instance.instance_variable_get(:@compressor).content_type
|
90
|
+
end
|
91
|
+
|
92
|
+
def test_configure_with_mime_type_text
|
93
|
+
conf = CONFIG.clone
|
94
|
+
conf << "\nstore_as text\n"
|
95
|
+
d = create_driver(conf)
|
96
|
+
assert_equal 'txt', d.instance.instance_variable_get(:@compressor).ext
|
97
|
+
assert_equal 'text/plain', d.instance.instance_variable_get(:@compressor).content_type
|
98
|
+
end
|
99
|
+
|
100
|
+
def test_configure_with_mime_type_lzo
|
101
|
+
conf = CONFIG.clone
|
102
|
+
conf << "\nstore_as lzo\n"
|
103
|
+
d = create_driver(conf)
|
104
|
+
assert_equal 'lzo', d.instance.instance_variable_get(:@compressor).ext
|
105
|
+
assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@compressor).content_type
|
106
|
+
rescue => e
|
107
|
+
# TODO: replace code with disable lzop command
|
108
|
+
assert(e.is_a?(Fluent::ConfigError))
|
109
|
+
end
|
110
|
+
|
111
|
+
def test_configure_with_path_style
|
112
|
+
conf = CONFIG.clone
|
113
|
+
conf << "\nforce_path_style true\n"
|
114
|
+
d = create_driver(conf)
|
115
|
+
assert d.instance.force_path_style
|
116
|
+
end
|
117
|
+
|
118
|
+
def test_configure_with_compute_checksums
|
119
|
+
conf = CONFIG.clone
|
120
|
+
conf << "\ncompute_checksums false\n"
|
121
|
+
d = create_driver(conf)
|
122
|
+
assert_equal false, d.instance.compute_checksums
|
123
|
+
end
|
124
|
+
|
125
|
+
def test_configure_with_hex_random_length
|
126
|
+
conf = CONFIG.clone
|
127
|
+
assert_raise Fluent::ConfigError do
|
128
|
+
create_driver(conf + "\nhex_random_length 17\n")
|
129
|
+
end
|
130
|
+
assert_nothing_raised do
|
131
|
+
create_driver(conf + "\nhex_random_length 16\n")
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
def test_configure_with_no_check_on_s3
|
136
|
+
conf = CONFIG.clone
|
137
|
+
conf << "\ncheck_bucket false\ncheck_object false\n"
|
138
|
+
d = create_driver(conf)
|
139
|
+
assert_equal false, d.instance.check_bucket
|
140
|
+
assert_equal false, d.instance.check_object
|
141
|
+
end
|
142
|
+
|
143
|
+
def test_configure_with_grant
|
144
|
+
conf = CONFIG.clone
|
145
|
+
conf << "\grant_full_control id='0123456789'\ngrant_read id='1234567890'\ngrant_read_acp id='2345678901'\ngrant_write_acp id='3456789012'\n"
|
146
|
+
d = create_driver(conf)
|
147
|
+
assert_equal "id='0123456789'", d.instance.grant_full_control
|
148
|
+
assert_equal "id='1234567890'", d.instance.grant_read
|
149
|
+
assert_equal "id='2345678901'", d.instance.grant_read_acp
|
150
|
+
assert_equal "id='3456789012'", d.instance.grant_write_acp
|
151
|
+
end
|
152
|
+
|
153
|
+
def test_format
|
154
|
+
d = create_driver
|
155
|
+
|
156
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
157
|
+
d.run(default_tag: "test") do
|
158
|
+
d.feed(time, { "a" => 1 })
|
159
|
+
d.feed(time, { "a" => 2 })
|
160
|
+
end
|
161
|
+
expected = [
|
162
|
+
%[2011-01-02T13:14:15Z\ttest\t{"a":1}\n],
|
163
|
+
%[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
|
164
|
+
]
|
165
|
+
assert_equal(expected, d.formatted)
|
166
|
+
end
|
167
|
+
|
168
|
+
def test_format_included_tag_and_time
|
169
|
+
config = [CONFIG, 'include_tag_key true', 'include_time_key true'].join("\n")
|
170
|
+
d = create_driver(config)
|
171
|
+
|
172
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
173
|
+
d.run(default_tag: "test") do
|
174
|
+
d.feed(time, { "a" => 1 })
|
175
|
+
d.feed(time, { "a" => 2 })
|
176
|
+
end
|
177
|
+
expected = [
|
178
|
+
%[2011-01-02T13:14:15Z\ttest\t{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n],
|
179
|
+
%[2011-01-02T13:14:15Z\ttest\t{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
|
180
|
+
]
|
181
|
+
assert_equal(expected, d.formatted)
|
182
|
+
end
|
183
|
+
|
184
|
+
def test_format_with_format_ltsv
|
185
|
+
config = [CONFIG, 'format ltsv'].join("\n")
|
186
|
+
d = create_driver(config)
|
187
|
+
|
188
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
189
|
+
d.run(default_tag: "test") do
|
190
|
+
d.feed(time, {"a"=>1, "b"=>1})
|
191
|
+
d.feed(time, {"a"=>2, "b"=>2})
|
192
|
+
end
|
193
|
+
expected = [
|
194
|
+
%[a:1\tb:1\n],
|
195
|
+
%[a:2\tb:2\n]
|
196
|
+
]
|
197
|
+
assert_equal(expected, d.formatted)
|
198
|
+
end
|
199
|
+
|
200
|
+
def test_format_with_format_json
|
201
|
+
config = [CONFIG, 'format json'].join("\n")
|
202
|
+
d = create_driver(config)
|
203
|
+
|
204
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
205
|
+
d.run(default_tag: "test") do
|
206
|
+
d.feed(time, {"a"=>1})
|
207
|
+
d.feed(time, {"a"=>2})
|
208
|
+
end
|
209
|
+
expected = [
|
210
|
+
%[{"a":1}\n],
|
211
|
+
%[{"a":2}\n]
|
212
|
+
]
|
213
|
+
assert_equal(expected, d.formatted)
|
214
|
+
end
|
215
|
+
|
216
|
+
def test_format_with_format_json_included_tag
|
217
|
+
config = [CONFIG, 'format json', 'include_tag_key true'].join("\n")
|
218
|
+
d = create_driver(config)
|
219
|
+
|
220
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
221
|
+
d.run(default_tag: "test") do
|
222
|
+
d.feed(time, {"a"=>1})
|
223
|
+
d.feed(time, {"a"=>2})
|
224
|
+
end
|
225
|
+
expected = [
|
226
|
+
%[{"a":1,"tag":"test"}\n],
|
227
|
+
%[{"a":2,"tag":"test"}\n]
|
228
|
+
]
|
229
|
+
assert_equal(expected, d.formatted)
|
230
|
+
end
|
231
|
+
|
232
|
+
def test_format_with_format_json_included_time
|
233
|
+
config = [CONFIG, 'format json', 'include_time_key true'].join("\n")
|
234
|
+
d = create_driver(config)
|
235
|
+
|
236
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
237
|
+
d.run(default_tag: "test") do
|
238
|
+
d.feed(time, {"a"=>1})
|
239
|
+
d.feed(time, {"a"=>2})
|
240
|
+
end
|
241
|
+
expected = [
|
242
|
+
%[{"a":1,"time":"2011-01-02T13:14:15Z"}\n],
|
243
|
+
%[{"a":2,"time":"2011-01-02T13:14:15Z"}\n]
|
244
|
+
]
|
245
|
+
assert_equal(expected, d.formatted)
|
246
|
+
end
|
247
|
+
|
248
|
+
def test_format_with_format_json_included_tag_and_time
|
249
|
+
config = [CONFIG, 'format json', 'include_tag_key true', 'include_time_key true'].join("\n")
|
250
|
+
d = create_driver(config)
|
251
|
+
|
252
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
253
|
+
d.run(default_tag: "test") do
|
254
|
+
d.feed(time, {"a"=>1})
|
255
|
+
d.feed(time, {"a"=>2})
|
256
|
+
end
|
257
|
+
expected = [
|
258
|
+
%[{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n],
|
259
|
+
%[{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
|
260
|
+
]
|
261
|
+
assert_equal(expected, d.formatted)
|
262
|
+
end
|
263
|
+
|
264
|
+
CONFIG_TIME_SLICE = <<EOC
|
265
|
+
aws_key_id test_key_id
|
266
|
+
aws_sec_key test_sec_key
|
267
|
+
s3_bucket test_bucket
|
268
|
+
s3_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
|
269
|
+
time_slice_format %Y%m%d-%H
|
270
|
+
path log
|
271
|
+
utc
|
272
|
+
buffer_type memory
|
273
|
+
@log_level debug
|
274
|
+
check_bucket true
|
275
|
+
check_object true
|
276
|
+
EOC
|
277
|
+
|
278
|
+
def create_time_sliced_driver(conf = CONFIG_TIME_SLICE)
|
279
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::S3Output) do
|
280
|
+
def format(tag, time, record)
|
281
|
+
super
|
282
|
+
end
|
283
|
+
|
284
|
+
def write(chunk)
|
285
|
+
super
|
286
|
+
end
|
287
|
+
|
288
|
+
private
|
289
|
+
|
290
|
+
def check_apikeys
|
291
|
+
end
|
292
|
+
end.configure(conf)
|
293
|
+
end
|
294
|
+
|
295
|
+
def test_write_with_hardened_s3_policy
|
296
|
+
# Partial mock the S3Bucket, not to make an actual connection to Amazon S3
|
297
|
+
setup_mocks_hardened_policy
|
298
|
+
s3_local_file_path = "/tmp/s3-test.txt"
|
299
|
+
# @s3_object_key_format will be hard_coded with timestamp only,
|
300
|
+
# as in this case, it will not check for object existence, not even bucker existence
|
301
|
+
# check_bukcet and check_object both of this config parameter should be false
|
302
|
+
# @s3_object_key_format = "%{path}/%{time_slice}_%{hms_slice}.%{file_extension}"
|
303
|
+
setup_s3_object_mocks_hardened_policy()
|
304
|
+
|
305
|
+
# We must use TimeSlicedOutputTestDriver instead of BufferedOutputTestDriver,
|
306
|
+
# to make assertions on chunks' keys
|
307
|
+
config = CONFIG_TIME_SLICE.gsub(/check_object true/, "check_object false\n")
|
308
|
+
config = config.gsub(/check_bucket true/, "check_bucket false\n")
|
309
|
+
d = create_time_sliced_driver(config)
|
310
|
+
|
311
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
312
|
+
d.run(default_tag: "test") do
|
313
|
+
d.feed(time, {"a"=>1})
|
314
|
+
d.feed(time, {"a"=>2})
|
315
|
+
end
|
316
|
+
|
317
|
+
Zlib::GzipReader.open(s3_local_file_path) do |gz|
|
318
|
+
data = gz.read
|
319
|
+
assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
|
320
|
+
%[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
|
321
|
+
data
|
322
|
+
end
|
323
|
+
FileUtils.rm_f(s3_local_file_path)
|
324
|
+
end
|
325
|
+
|
326
|
+
def test_write_with_custom_s3_object_key_format
|
327
|
+
# Partial mock the S3Bucket, not to make an actual connection to Amazon S3
|
328
|
+
setup_mocks(true)
|
329
|
+
s3_local_file_path = "/tmp/s3-test.txt"
|
330
|
+
setup_s3_object_mocks(s3_local_file_path: s3_local_file_path)
|
331
|
+
|
332
|
+
d = create_time_sliced_driver
|
333
|
+
|
334
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
335
|
+
d.run(default_tag: "test") do
|
336
|
+
d.feed(time, {"a"=>1})
|
337
|
+
d.feed(time, {"a"=>2})
|
338
|
+
end
|
339
|
+
|
340
|
+
Zlib::GzipReader.open(s3_local_file_path) do |gz|
|
341
|
+
data = gz.read
|
342
|
+
assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
|
343
|
+
%[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
|
344
|
+
data
|
345
|
+
end
|
346
|
+
FileUtils.rm_f(s3_local_file_path)
|
347
|
+
end
|
348
|
+
|
349
|
+
def test_write_with_custom_s3_object_key_format_containing_uuid_flush_placeholder
|
350
|
+
|
351
|
+
# Partial mock the S3Bucket, not to make an actual connection to Amazon S3
|
352
|
+
setup_mocks(true)
|
353
|
+
|
354
|
+
uuid = "5755e23f-9b54-42d8-8818-2ea38c6f279e"
|
355
|
+
stub(::SecureRandom).uuid{ uuid }
|
356
|
+
|
357
|
+
s3_local_file_path = "/tmp/s3-test.txt"
|
358
|
+
s3path = "log/events/ts=20110102-13/events_0-#{uuid}.gz"
|
359
|
+
setup_s3_object_mocks(s3_local_file_path: s3_local_file_path, s3path: s3path)
|
360
|
+
|
361
|
+
config = CONFIG_TIME_SLICE.gsub(/%{hostname}/,"%{uuid_flush}")
|
362
|
+
d = create_time_sliced_driver(config)
|
363
|
+
|
364
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
365
|
+
d.run(default_tag: "test") do
|
366
|
+
d.feed(time, {"a"=>1})
|
367
|
+
d.feed(time, {"a"=>2})
|
368
|
+
end
|
369
|
+
|
370
|
+
Zlib::GzipReader.open(s3_local_file_path) do |gz|
|
371
|
+
data = gz.read
|
372
|
+
assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
|
373
|
+
%[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
|
374
|
+
data
|
375
|
+
end
|
376
|
+
FileUtils.rm_f(s3_local_file_path)
|
377
|
+
Dir.glob('tmp/*').each {|file| FileUtils.rm_f(file) }
|
378
|
+
end
|
379
|
+
|
380
|
+
# ToDo: need to test hex_random does not change on retry, but it is difficult with
|
381
|
+
# the current fluentd test helper because it does not provide a way to run with the same chunks
|
382
|
+
def test_write_with_custom_s3_object_key_format_containing_hex_random_placeholder
|
383
|
+
unique_hex = "5226c3c4fb3d49b15226c3c4fb3d49b1"
|
384
|
+
hex_random = unique_hex.reverse[0...5]
|
385
|
+
|
386
|
+
config = CONFIG_TIME_SLICE.gsub(/%{hostname}/,"%{hex_random}") << "\nhex_random_length #{hex_random.length}"
|
387
|
+
config = config.gsub(/buffer_type memory/, "buffer_type file\nbuffer_path test/tmp/buf")
|
388
|
+
|
389
|
+
# Partial mock the S3Bucket, not to make an actual connection to Amazon S3
|
390
|
+
setup_mocks(true)
|
391
|
+
|
392
|
+
s3path = "log/events/ts=20110102-13/events_0-#{hex_random}.gz"
|
393
|
+
s3_local_file_path = "/tmp/s3-test.txt"
|
394
|
+
setup_s3_object_mocks(s3_local_file_path: s3_local_file_path, s3path: s3path)
|
395
|
+
|
396
|
+
d = create_time_sliced_driver(config)
|
397
|
+
stub(Fluent::UniqueId).hex(anything) { unique_hex }
|
398
|
+
|
399
|
+
time = event_time("2011-01-02 13:14:15 UTC")
|
400
|
+
d.run(default_tag: "test") do
|
401
|
+
d.feed(time, {"a"=>1})
|
402
|
+
d.feed(time, {"a"=>2})
|
403
|
+
end
|
404
|
+
|
405
|
+
Zlib::GzipReader.open(s3_local_file_path) do |gz|
|
406
|
+
data = gz.read
|
407
|
+
assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
|
408
|
+
%[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
|
409
|
+
data
|
410
|
+
end
|
411
|
+
FileUtils.rm_f(s3_local_file_path)
|
412
|
+
end
|
413
|
+
|
414
|
+
class MockResponse
|
415
|
+
attr_reader :data
|
416
|
+
|
417
|
+
def initialize(data)
|
418
|
+
@data = data
|
419
|
+
end
|
420
|
+
end
|
421
|
+
|
422
|
+
def setup_mocks(exists_return = false)
|
423
|
+
@s3_client = stub(Aws::S3::Client.new(stub_responses: true))
|
424
|
+
stub(@s3_client).config { OpenStruct.new({region: "us-east-1"}) }
|
425
|
+
# aws-sdk-s3 calls Client#put_object inside Object#put
|
426
|
+
mock(@s3_client).put_object(anything).at_least(0) { MockResponse.new({}) }
|
427
|
+
mock(Aws::S3::Client).new(anything).at_least(0) { @s3_client }
|
428
|
+
@s3_resource = mock(Aws::S3::Resource.new(client: @s3_client))
|
429
|
+
mock(Aws::S3::Resource).new(client: @s3_client) { @s3_resource }
|
430
|
+
@s3_bucket = mock(Aws::S3::Bucket.new(name: "test",
|
431
|
+
client: @s3_client))
|
432
|
+
@s3_bucket.exists? { exists_return }
|
433
|
+
@s3_object = mock(Aws::S3::Object.new(bucket_name: "test_bucket",
|
434
|
+
key: "test",
|
435
|
+
client: @s3_client))
|
436
|
+
@s3_object.exists?.at_least(0) { false }
|
437
|
+
@s3_bucket.object(anything).at_least(0) { @s3_object }
|
438
|
+
@s3_resource.bucket(anything) { @s3_bucket }
|
439
|
+
end
|
440
|
+
|
441
|
+
def setup_s3_object_mocks(params = {})
|
442
|
+
s3path = params[:s3path] || "log/events/ts=20110102-13/events_0-#{Socket.gethostname}.gz"
|
443
|
+
s3_local_file_path = params[:s3_local_file_path] || "/tmp/s3-test.txt"
|
444
|
+
|
445
|
+
# Assert content of event logs which are being sent to S3
|
446
|
+
s3obj = stub(Aws::S3::Object.new(bucket_name: "test_bucket",
|
447
|
+
key: "test",
|
448
|
+
client: @s3_client))
|
449
|
+
s3obj.exists? { false }
|
450
|
+
|
451
|
+
tempfile = File.new(s3_local_file_path, "w")
|
452
|
+
stub(Tempfile).new("s3-") { tempfile }
|
453
|
+
s3obj.put(body: tempfile,
|
454
|
+
content_type: "application/x-gzip",
|
455
|
+
storage_class: "STANDARD")
|
456
|
+
|
457
|
+
@s3_bucket.object(s3path) { s3obj }
|
458
|
+
end
|
459
|
+
|
460
|
+
def setup_mocks_hardened_policy()
|
461
|
+
@s3_client = stub(Aws::S3::Client.new(:stub_responses => true))
|
462
|
+
stub(@s3_client).config { OpenStruct.new({region: "us-east-1"}) }
|
463
|
+
mock(@s3_client).put_object(anything).at_least(0) { MockResponse.new({}) }
|
464
|
+
mock(Aws::S3::Client).new(anything).at_least(0) { @s3_client }
|
465
|
+
@s3_resource = mock(Aws::S3::Resource.new(:client => @s3_client))
|
466
|
+
mock(Aws::S3::Resource).new(:client => @s3_client) { @s3_resource }
|
467
|
+
@s3_bucket = mock(Aws::S3::Bucket.new(:name => "test",
|
468
|
+
:client => @s3_client))
|
469
|
+
@s3_object = mock(Aws::S3::Object.new(:bucket_name => "test_bucket",
|
470
|
+
:key => "test",
|
471
|
+
:client => @s3_client))
|
472
|
+
@s3_bucket.object(anything).at_least(0) { @s3_object }
|
473
|
+
@s3_resource.bucket(anything) { @s3_bucket }
|
474
|
+
end
|
475
|
+
|
476
|
+
def setup_s3_object_mocks_hardened_policy(params = {})
|
477
|
+
s3_local_file_path = params[:s3_local_file_path] || "/tmp/s3-test.txt"
|
478
|
+
|
479
|
+
# Assert content of event logs which are being sent to S3
|
480
|
+
s3obj = stub(Aws::S3::Object.new(:bucket_name => "test_bucket",
|
481
|
+
:key => "test",
|
482
|
+
:client => @s3_client))
|
483
|
+
|
484
|
+
tempfile = File.new(s3_local_file_path, "w")
|
485
|
+
stub(Tempfile).new("s3-") { tempfile }
|
486
|
+
s3obj.put(:body => tempfile,
|
487
|
+
:content_type => "application/x-gzip",
|
488
|
+
:storage_class => "STANDARD")
|
489
|
+
end
|
490
|
+
|
491
|
+
def test_auto_create_bucket_false_with_non_existence_bucket
|
492
|
+
setup_mocks
|
493
|
+
|
494
|
+
config = CONFIG_TIME_SLICE + 'auto_create_bucket false'
|
495
|
+
d = create_time_sliced_driver(config)
|
496
|
+
assert_raise(RuntimeError, "The specified bucket does not exist: bucket = test_bucket") {
|
497
|
+
d.run {}
|
498
|
+
}
|
499
|
+
end
|
500
|
+
|
501
|
+
def test_auto_create_bucket_true_with_non_existence_bucket
|
502
|
+
setup_mocks
|
503
|
+
@s3_resource.create_bucket(bucket: "test_bucket")
|
504
|
+
|
505
|
+
config = CONFIG_TIME_SLICE + 'auto_create_bucket true'
|
506
|
+
d = create_time_sliced_driver(config)
|
507
|
+
assert_nothing_raised { d.run {} }
|
508
|
+
end
|
509
|
+
|
510
|
+
def test_credentials
|
511
|
+
d = create_time_sliced_driver
|
512
|
+
assert_nothing_raised { d.run {} }
|
513
|
+
client = d.instance.instance_variable_get(:@s3).client
|
514
|
+
credentials = client.config.credentials
|
515
|
+
assert_instance_of(Aws::Credentials, credentials)
|
516
|
+
end
|
517
|
+
|
518
|
+
def test_assume_role_credentials
|
519
|
+
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
520
|
+
mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
|
521
|
+
role_session_name: "test_session",
|
522
|
+
client: anything){
|
523
|
+
expected_credentials
|
524
|
+
}
|
525
|
+
config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
|
526
|
+
config += %[
|
527
|
+
<assume_role_credentials>
|
528
|
+
role_arn test_arn
|
529
|
+
role_session_name test_session
|
530
|
+
</assume_role_credentials>
|
531
|
+
]
|
532
|
+
d = create_time_sliced_driver(config)
|
533
|
+
assert_nothing_raised { d.run {} }
|
534
|
+
client = d.instance.instance_variable_get(:@s3).client
|
535
|
+
credentials = client.config.credentials
|
536
|
+
assert_equal(expected_credentials, credentials)
|
537
|
+
end
|
538
|
+
|
539
|
+
def test_assume_role_credentials_with_region
|
540
|
+
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
541
|
+
sts_client = Aws::STS::Client.new(region: 'ap-northeast-1')
|
542
|
+
mock(Aws::STS::Client).new(region: 'ap-northeast-1'){ sts_client }
|
543
|
+
mock(Aws::AssumeRoleCredentials).new(role_arn: "test_arn",
|
544
|
+
role_session_name: "test_session",
|
545
|
+
client: sts_client){
|
546
|
+
expected_credentials
|
547
|
+
}
|
548
|
+
config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
|
549
|
+
config += %[
|
550
|
+
s3_region ap-northeast-1
|
551
|
+
<assume_role_credentials>
|
552
|
+
role_arn test_arn
|
553
|
+
role_session_name test_session
|
554
|
+
</assume_role_credentials>
|
555
|
+
]
|
556
|
+
d = create_time_sliced_driver(config)
|
557
|
+
assert_nothing_raised { d.run {} }
|
558
|
+
client = d.instance.instance_variable_get(:@s3).client
|
559
|
+
credentials = client.config.credentials
|
560
|
+
assert_equal(expected_credentials, credentials)
|
561
|
+
end
|
562
|
+
|
563
|
+
def test_web_identity_credentials
|
564
|
+
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
565
|
+
mock(Aws::AssumeRoleWebIdentityCredentials).new(
|
566
|
+
role_arn: "test_arn",
|
567
|
+
role_session_name: "test_session",
|
568
|
+
web_identity_token_file: "test_file",
|
569
|
+
client: anything
|
570
|
+
){
|
571
|
+
expected_credentials
|
572
|
+
}
|
573
|
+
|
574
|
+
config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
|
575
|
+
config += %[
|
576
|
+
<web_identity_credentials>
|
577
|
+
role_arn test_arn
|
578
|
+
role_session_name test_session
|
579
|
+
web_identity_token_file test_file
|
580
|
+
</web_identity_credentials>
|
581
|
+
]
|
582
|
+
d = create_time_sliced_driver(config)
|
583
|
+
assert_nothing_raised { d.run {} }
|
584
|
+
client = d.instance.instance_variable_get(:@s3).client
|
585
|
+
credentials = client.config.credentials
|
586
|
+
assert_equal(expected_credentials, credentials)
|
587
|
+
end
|
588
|
+
|
589
|
+
def test_web_identity_credentials_with_sts_region
|
590
|
+
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
591
|
+
sts_client = Aws::STS::Client.new(region: 'us-east-1')
|
592
|
+
mock(Aws::STS::Client).new(region: 'us-east-1'){ sts_client }
|
593
|
+
mock(Aws::AssumeRoleWebIdentityCredentials).new(
|
594
|
+
role_arn: "test_arn",
|
595
|
+
role_session_name: "test_session",
|
596
|
+
web_identity_token_file: "test_file",
|
597
|
+
client: sts_client
|
598
|
+
){
|
599
|
+
expected_credentials
|
600
|
+
}
|
601
|
+
|
602
|
+
config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
|
603
|
+
config += %[
|
604
|
+
s3_region us-west-2
|
605
|
+
<web_identity_credentials>
|
606
|
+
role_arn test_arn
|
607
|
+
role_session_name test_session
|
608
|
+
web_identity_token_file test_file
|
609
|
+
sts_region us-east-1
|
610
|
+
</web_identity_credentials>
|
611
|
+
]
|
612
|
+
d = create_time_sliced_driver(config)
|
613
|
+
assert_nothing_raised { d.run {} }
|
614
|
+
client = d.instance.instance_variable_get(:@s3).client
|
615
|
+
credentials = client.config.credentials
|
616
|
+
assert_equal(expected_credentials, credentials)
|
617
|
+
end
|
618
|
+
|
619
|
+
def test_instance_profile_credentials
|
620
|
+
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
621
|
+
mock(Aws::InstanceProfileCredentials).new({}).returns(expected_credentials)
|
622
|
+
config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
|
623
|
+
config += %[
|
624
|
+
<instance_profile_credentials>
|
625
|
+
</instance_profile_credentials>
|
626
|
+
]
|
627
|
+
d = create_time_sliced_driver(config)
|
628
|
+
assert_nothing_raised { d.run {} }
|
629
|
+
client = d.instance.instance_variable_get(:@s3).client
|
630
|
+
credentials = client.config.credentials
|
631
|
+
assert_equal(expected_credentials, credentials)
|
632
|
+
end
|
633
|
+
|
634
|
+
def test_ecs_credentials
|
635
|
+
ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] = "/credential_provider_version/credentials?id=task_UUID"
|
636
|
+
|
637
|
+
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
638
|
+
mock(Aws::ECSCredentials).new({}).returns(expected_credentials)
|
639
|
+
config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
|
640
|
+
config += %[
|
641
|
+
<instance_profile_credentials>
|
642
|
+
</instance_profile_credentials>
|
643
|
+
]
|
644
|
+
d = create_time_sliced_driver(config)
|
645
|
+
assert_nothing_raised { d.run {} }
|
646
|
+
client = d.instance.instance_variable_get(:@s3).client
|
647
|
+
credentials = client.config.credentials
|
648
|
+
assert_equal(expected_credentials, credentials)
|
649
|
+
|
650
|
+
ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] = nil
|
651
|
+
end
|
652
|
+
|
653
|
+
def test_instance_profile_credentials_aws_iam_retries
|
654
|
+
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
655
|
+
mock(Aws::InstanceProfileCredentials).new({ retries: 10 }).returns(expected_credentials)
|
656
|
+
config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
|
657
|
+
config += %[
|
658
|
+
aws_iam_retries 10
|
659
|
+
]
|
660
|
+
d = create_time_sliced_driver(config)
|
661
|
+
assert_nothing_raised { d.run {} }
|
662
|
+
client = d.instance.instance_variable_get(:@s3).client
|
663
|
+
credentials = client.config.credentials
|
664
|
+
assert_equal(expected_credentials, credentials)
|
665
|
+
end
|
666
|
+
|
667
|
+
def test_shared_credentials
|
668
|
+
expected_credentials = Aws::Credentials.new("test_key", "test_secret")
|
669
|
+
mock(Aws::SharedCredentials).new({}).returns(expected_credentials)
|
670
|
+
config = CONFIG_TIME_SLICE.split("\n").reject{|x| x =~ /.+aws_.+/}.join("\n")
|
671
|
+
config += %[
|
672
|
+
<shared_credentials>
|
673
|
+
</shared_credentials>
|
674
|
+
]
|
675
|
+
d = create_time_sliced_driver(config)
|
676
|
+
assert_nothing_raised { d.run {} }
|
677
|
+
client = d.instance.instance_variable_get(:@s3).client
|
678
|
+
credentials = client.config.credentials
|
679
|
+
assert_equal(expected_credentials, credentials)
|
680
|
+
end
|
681
|
+
|
682
|
+
def test_signature_version
|
683
|
+
config = [CONFIG, 'signature_version s3'].join("\n")
|
684
|
+
d = create_driver(config)
|
685
|
+
|
686
|
+
signature_version = d.instance.instance_variable_get(:@signature_version)
|
687
|
+
assert_equal("s3", signature_version)
|
688
|
+
end
|
689
|
+
|
690
|
+
def test_warn_for_delay
|
691
|
+
setup_mocks(true)
|
692
|
+
s3_local_file_path = "/tmp/s3-test.txt"
|
693
|
+
setup_s3_object_mocks(s3_local_file_path: s3_local_file_path)
|
694
|
+
|
695
|
+
config = CONFIG_TIME_SLICE + 'warn_for_delay 1d'
|
696
|
+
d = create_time_sliced_driver(config)
|
697
|
+
|
698
|
+
delayed_time = event_time("2011-01-02 13:14:15 UTC")
|
699
|
+
now = delayed_time.to_i + 86000 + 1
|
700
|
+
d.instance.log.out.flush_logs = false
|
701
|
+
Timecop.freeze(Time.at(now)) do
|
702
|
+
d.run(default_tag: "test") do
|
703
|
+
d.feed(delayed_time, {"a"=>1})
|
704
|
+
d.feed(delayed_time, {"a"=>2})
|
705
|
+
end
|
706
|
+
end
|
707
|
+
logs = d.instance.log.out.logs
|
708
|
+
assert_true logs.any? {|log| log.include?('out_s3: delayed events were put') }
|
709
|
+
d.instance.log.out.flush_logs = true
|
710
|
+
d.instance.log.out.reset
|
711
|
+
FileUtils.rm_f(s3_local_file_path)
|
712
|
+
end
|
713
|
+
end
|