fluent-plugin-s3 1.1.2 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ChangeLog +4 -0
- data/VERSION +1 -1
- data/lib/fluent/plugin/in_s3.rb +15 -2
- data/test/test_in_s3.rb +101 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 16e7bfdfa2c8a9ac640bc5aab2f4f340ff764874
|
4
|
+
data.tar.gz: a4711544ff30e4312474b85a730aa00fb8cd3c1b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d084d89f1fc8ba64405c417e2f71d5a2316d47de454be606d28b54ad9a52d567e09e401a5f1bf72268c638d37e5b17812e4c3d85be6221081c6921fe2160adc1
|
7
|
+
data.tar.gz: fdd479ee6b6bbae6a0d4bd2e25817d70b6f82fe2d8ab87ca835b4b803cd86f7b3432266bdd4ed099edcda8501d11d790818ed4336740f79126cc87d621c4031a
|
data/ChangeLog
CHANGED
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.1.
|
1
|
+
1.1.3
|
data/lib/fluent/plugin/in_s3.rb
CHANGED
@@ -284,10 +284,23 @@ module Fluent::Plugin
|
|
284
284
|
'application/x-gzip'.freeze
|
285
285
|
end
|
286
286
|
|
287
|
+
# https://bugs.ruby-lang.org/issues/9790
|
288
|
+
# https://bugs.ruby-lang.org/issues/11180
|
289
|
+
# https://github.com/exAspArk/multiple_files_gzip_reader
|
287
290
|
def extract(io)
|
288
|
-
|
289
|
-
|
291
|
+
parts = []
|
292
|
+
loop do
|
293
|
+
unused = nil
|
294
|
+
Zlib::GzipReader.wrap(io) do |gz|
|
295
|
+
parts << gz.read
|
296
|
+
unused = gz.unused
|
297
|
+
gz.finish
|
298
|
+
end
|
299
|
+
io.pos -= unused ? unused.length : 0
|
300
|
+
break if io.eof?
|
290
301
|
end
|
302
|
+
io.close
|
303
|
+
parts.join
|
291
304
|
end
|
292
305
|
end
|
293
306
|
|
data/test/test_in_s3.rb
CHANGED
@@ -89,6 +89,7 @@ class S3InputTest < Test::Unit::TestCase
|
|
89
89
|
|
90
90
|
data("json" => ["json", "json", "application/json"],
|
91
91
|
"text" => ["text", "txt", "text/plain"],
|
92
|
+
"gzip" => ["gzip", "gz", "application/x-gzip"],
|
92
93
|
"gzip_command" => ["gzip_command", "gz", "application/x-gzip"],
|
93
94
|
"lzo" => ["lzo", "lzo", "application/x-lzop"],
|
94
95
|
"lzma2" => ["lzma2", "xz", "application/x-xz"])
|
@@ -262,4 +263,104 @@ class S3InputTest < Test::Unit::TestCase
|
|
262
263
|
]
|
263
264
|
assert_equal(expected_records, events.map {|_tag, _time, record| record })
|
264
265
|
end
|
266
|
+
|
267
|
+
def test_gzip_single_stream
|
268
|
+
setup_mocks
|
269
|
+
d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as gzip\nformat none\n")
|
270
|
+
|
271
|
+
s3_object = stub(Object.new)
|
272
|
+
s3_response = stub(Object.new)
|
273
|
+
s3_response.body {
|
274
|
+
io = StringIO.new
|
275
|
+
Zlib::GzipWriter.wrap(io) do |gz|
|
276
|
+
gz.write "aaa\nbbb\n"
|
277
|
+
gz.finish
|
278
|
+
end
|
279
|
+
io.rewind
|
280
|
+
io
|
281
|
+
}
|
282
|
+
s3_object.get { s3_response }
|
283
|
+
@s3_bucket.object(anything).at_least(1) { s3_object }
|
284
|
+
|
285
|
+
body = {
|
286
|
+
"Records" => [
|
287
|
+
{
|
288
|
+
"s3" => {
|
289
|
+
"object" => {
|
290
|
+
"key" => "test_key"
|
291
|
+
}
|
292
|
+
}
|
293
|
+
}
|
294
|
+
]
|
295
|
+
}
|
296
|
+
message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
|
297
|
+
@sqs_poller.get_messages(anything, anything) do |config, stats|
|
298
|
+
config.before_request.call(stats) if config.before_request
|
299
|
+
stats.request_count += 1
|
300
|
+
if stats.request_count >= 1
|
301
|
+
d.instance.instance_variable_set(:@running, false)
|
302
|
+
end
|
303
|
+
[message]
|
304
|
+
end
|
305
|
+
d.run(expect_emits: 1)
|
306
|
+
events = d.events
|
307
|
+
expected_records = [
|
308
|
+
{ "message" => "aaa\n" },
|
309
|
+
{ "message" => "bbb\n" }
|
310
|
+
]
|
311
|
+
assert_equal(expected_records, events.map {|_tag, _time, record| record })
|
312
|
+
end
|
313
|
+
|
314
|
+
def test_gzip_multiple_steams
|
315
|
+
setup_mocks
|
316
|
+
d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as gzip\nformat none\n")
|
317
|
+
|
318
|
+
s3_object = stub(Object.new)
|
319
|
+
s3_response = stub(Object.new)
|
320
|
+
s3_response.body {
|
321
|
+
io = StringIO.new
|
322
|
+
Zlib::GzipWriter.wrap(io) do |gz|
|
323
|
+
gz.write "aaa\nbbb\n"
|
324
|
+
gz.finish
|
325
|
+
end
|
326
|
+
Zlib::GzipWriter.wrap(io) do |gz|
|
327
|
+
gz.write "ccc\nddd\n"
|
328
|
+
gz.finish
|
329
|
+
end
|
330
|
+
io.rewind
|
331
|
+
io
|
332
|
+
}
|
333
|
+
s3_object.get { s3_response }
|
334
|
+
@s3_bucket.object(anything).at_least(1) { s3_object }
|
335
|
+
|
336
|
+
body = {
|
337
|
+
"Records" => [
|
338
|
+
{
|
339
|
+
"s3" => {
|
340
|
+
"object" => {
|
341
|
+
"key" => "test_key"
|
342
|
+
}
|
343
|
+
}
|
344
|
+
}
|
345
|
+
]
|
346
|
+
}
|
347
|
+
message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
|
348
|
+
@sqs_poller.get_messages(anything, anything) do |config, stats|
|
349
|
+
config.before_request.call(stats) if config.before_request
|
350
|
+
stats.request_count += 1
|
351
|
+
if stats.request_count >= 1
|
352
|
+
d.instance.instance_variable_set(:@running, false)
|
353
|
+
end
|
354
|
+
[message]
|
355
|
+
end
|
356
|
+
d.run(expect_emits: 1)
|
357
|
+
events = d.events
|
358
|
+
expected_records = [
|
359
|
+
{ "message" => "aaa\n" },
|
360
|
+
{ "message" => "bbb\n" },
|
361
|
+
{ "message" => "ccc\n" },
|
362
|
+
{ "message" => "ddd\n" }
|
363
|
+
]
|
364
|
+
assert_equal(expected_records, events.map {|_tag, _time, record| record })
|
365
|
+
end
|
265
366
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-04-
|
12
|
+
date: 2018-04-15 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -163,7 +163,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
163
163
|
version: '0'
|
164
164
|
requirements: []
|
165
165
|
rubyforge_project:
|
166
|
-
rubygems_version: 2.6.
|
166
|
+
rubygems_version: 2.6.14.1
|
167
167
|
signing_key:
|
168
168
|
specification_version: 4
|
169
169
|
summary: Amazon S3 output plugin for Fluentd event collector
|