fluent-plugin-s3 0.5.9 → 0.5.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ChangeLog +5 -0
- data/README.rdoc +2 -0
- data/VERSION +1 -1
- data/fluent-plugin-s3.gemspec +1 -0
- data/lib/fluent/plugin/out_s3.rb +2 -1
- data/test/test_out_s3.rb +48 -0
- metadata +5 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f674b76e6ab0fcde41c2a1f580b36f2b0e346f01
|
4
|
+
data.tar.gz: 32de8cacec06dd1f7a0ab241e05eafca9ecaf755
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8417dad2be0be87130030d41fdfb59ba553d5f574c71f0eb3f685145dd7b6ff1011ff9dd89d35b65b4dcb422be0e54f297c312d51cbdef2e70728c4ce5dbe42f
|
7
|
+
data.tar.gz: e0f4aaace4ebf98f33cabaa782ae39b8999d0fee860d9b7a0eec990f0661faddffdf8510ea29609ab786587f6044f799abb9392f649fd423d7570aed42456085
|
data/ChangeLog
CHANGED
data/README.rdoc
CHANGED
@@ -51,6 +51,7 @@ Simply use RubyGems:
|
|
51
51
|
- %{time_slice}
|
52
52
|
- %{index}
|
53
53
|
- %{file_extension}
|
54
|
+
- %{uuid_flush}
|
54
55
|
|
55
56
|
to decide keys dynamically.
|
56
57
|
|
@@ -58,6 +59,7 @@ to decide keys dynamically.
|
|
58
59
|
%{time_slice} is the time-slice in text that are formatted with *time_slice_format*.
|
59
60
|
%{index} is the sequential number starts from 0, increments when multiple files are uploaded to S3 in the same time slice.
|
60
61
|
%{file_extention} is always "gz" for now.
|
62
|
+
%{uuid_flush} a uuid that is replaced everytime the buffer will be flushed
|
61
63
|
|
62
64
|
The default format is "%{path}%{time_slice}_%{index}.%{file_extension}".
|
63
65
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.5.
|
1
|
+
0.5.10
|
data/fluent-plugin-s3.gemspec
CHANGED
@@ -4,6 +4,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
4
4
|
Gem::Specification.new do |gem|
|
5
5
|
gem.name = "fluent-plugin-s3"
|
6
6
|
gem.description = "Amazon S3 output plugin for Fluentd event collector"
|
7
|
+
gem.license = "Apache-2.0"
|
7
8
|
gem.homepage = "https://github.com/fluent/fluent-plugin-s3"
|
8
9
|
gem.summary = gem.description
|
9
10
|
gem.version = File.read("VERSION").strip
|
data/lib/fluent/plugin/out_s3.rb
CHANGED
@@ -107,7 +107,8 @@ module Fluent
|
|
107
107
|
"path" => path,
|
108
108
|
"time_slice" => chunk.key,
|
109
109
|
"file_extension" => @compressor.ext,
|
110
|
-
"index" => i
|
110
|
+
"index" => i,
|
111
|
+
"uuid_flush" => uuid_random
|
111
112
|
}
|
112
113
|
s3path = @s3_object_key_format.gsub(%r(%{[^}]+})) { |expr|
|
113
114
|
values_for_s3_object_key[expr[2...expr.size-1]]
|
data/test/test_out_s3.rb
CHANGED
@@ -288,6 +288,54 @@ class S3OutputTest < Test::Unit::TestCase
|
|
288
288
|
d.run
|
289
289
|
end
|
290
290
|
|
291
|
+
def test_write_with_custom_s3_object_key_format_containing_uuid_flush_placeholder
|
292
|
+
# Assert content of event logs which are being sent to S3
|
293
|
+
s3obj = flexmock(AWS::S3::S3Object)
|
294
|
+
s3obj.should_receive(:exists?).with_any_args.and_return { false }
|
295
|
+
s3obj.should_receive(:write).with(
|
296
|
+
on { |pathname|
|
297
|
+
data = nil
|
298
|
+
# Event logs are compressed in GZip
|
299
|
+
pathname.open { |f|
|
300
|
+
gz = Zlib::GzipReader.new(f)
|
301
|
+
data = gz.read
|
302
|
+
gz.close
|
303
|
+
}
|
304
|
+
assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
|
305
|
+
%[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
|
306
|
+
data
|
307
|
+
|
308
|
+
pathname.to_s.match(%r|s3-|)
|
309
|
+
},
|
310
|
+
{:content_type => "application/x-gzip", :reduced_redundancy => false})
|
311
|
+
|
312
|
+
# Assert the key of S3Object, which event logs are stored in
|
313
|
+
s3obj_col = flexmock(AWS::S3::ObjectCollection)
|
314
|
+
s3obj_col.should_receive(:[]).with(
|
315
|
+
on { |key|
|
316
|
+
key =~ /.*.[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}.*/
|
317
|
+
}).
|
318
|
+
and_return {
|
319
|
+
s3obj
|
320
|
+
}
|
321
|
+
|
322
|
+
# Partial mock the S3Bucket, not to make an actual connection to Amazon S3
|
323
|
+
s3bucket, _ = setup_mocks(true)
|
324
|
+
s3bucket.should_receive(:objects).with_any_args.and_return { s3obj_col }
|
325
|
+
|
326
|
+
# We must use TimeSlicedOutputTestDriver instead of BufferedOutputTestDriver,
|
327
|
+
# to make assertions on chunks' keys
|
328
|
+
config = CONFIG_TIME_SLICE.clone.gsub(/%{hostname}/,"%{uuid_flush}")
|
329
|
+
d = create_time_sliced_driver(config)
|
330
|
+
|
331
|
+
time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
332
|
+
d.emit({"a"=>1}, time)
|
333
|
+
d.emit({"a"=>2}, time)
|
334
|
+
|
335
|
+
# Finally, the instance of S3Output is initialized and then invoked
|
336
|
+
d.run
|
337
|
+
end
|
338
|
+
|
291
339
|
def setup_mocks(exists_return = false)
|
292
340
|
s3bucket = flexmock(AWS::S3::Bucket)
|
293
341
|
s3bucket.should_receive(:exists?).with_any_args.and_return { exists_return }
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2015-
|
12
|
+
date: 2015-07-27 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -136,7 +136,8 @@ files:
|
|
136
136
|
- lib/fluent/plugin/s3_compressor_lzo.rb
|
137
137
|
- test/test_out_s3.rb
|
138
138
|
homepage: https://github.com/fluent/fluent-plugin-s3
|
139
|
-
licenses:
|
139
|
+
licenses:
|
140
|
+
- Apache-2.0
|
140
141
|
metadata: {}
|
141
142
|
post_install_message:
|
142
143
|
rdoc_options: []
|
@@ -154,7 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
154
155
|
version: '0'
|
155
156
|
requirements: []
|
156
157
|
rubyforge_project:
|
157
|
-
rubygems_version: 2.2.
|
158
|
+
rubygems_version: 2.2.3
|
158
159
|
signing_key:
|
159
160
|
specification_version: 4
|
160
161
|
summary: Amazon S3 output plugin for Fluentd event collector
|