fluent-plugin-s3 0.5.9 → 0.5.10

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: feddb8710e1325aa7cc3a277082a67d6bc67c5f9
4
- data.tar.gz: 95e0def196fbe430df629400990695f783716872
3
+ metadata.gz: f674b76e6ab0fcde41c2a1f580b36f2b0e346f01
4
+ data.tar.gz: 32de8cacec06dd1f7a0ab241e05eafca9ecaf755
5
5
  SHA512:
6
- metadata.gz: 744be5bcaf22b43c565ac5a61271dec1abc1532dcf81d6fb5cb2352fabcdad36367ee9c83c56dd30fac6494f72abd5098eb119d2cf6f0c02a51975ef8f11a7be
7
- data.tar.gz: f8f607b792454e660e40412dd1df7352f55478813696bb935190e104f6716d2a8b8131c10a6f3b4f6a094c78521833c66486b7d3e72526ac7afc498821c32ed5
6
+ metadata.gz: 8417dad2be0be87130030d41fdfb59ba553d5f574c71f0eb3f685145dd7b6ff1011ff9dd89d35b65b4dcb422be0e54f297c312d51cbdef2e70728c4ce5dbe42f
7
+ data.tar.gz: e0f4aaace4ebf98f33cabaa782ae39b8999d0fee860d9b7a0eec990f0661faddffdf8510ea29609ab786587f6044f799abb9392f649fd423d7570aed42456085
data/ChangeLog CHANGED
@@ -1,3 +1,8 @@
1
+ Release 0.5.10 - 2015/07/27
2
+
3
+ * Add "uuid_flush" placeholder to s3_object_key_format for creating unique object
4
+
5
+
1
6
  Release 0.5.9 - 2015/06/10
2
7
 
3
8
  * Add secret option to AWS key related parameters
@@ -51,6 +51,7 @@ Simply use RubyGems:
51
51
  - %{time_slice}
52
52
  - %{index}
53
53
  - %{file_extension}
54
+ - %{uuid_flush}
54
55
 
55
56
  to decide keys dynamically.
56
57
 
@@ -58,6 +59,7 @@ to decide keys dynamically.
58
59
  %{time_slice} is the time-slice in text that are formatted with *time_slice_format*.
59
60
  %{index} is the sequential number starts from 0, increments when multiple files are uploaded to S3 in the same time slice.
60
61
  %{file_extention} is always "gz" for now.
62
+ %{uuid_flush} a uuid that is replaced everytime the buffer will be flushed
61
63
 
62
64
  The default format is "%{path}%{time_slice}_%{index}.%{file_extension}".
63
65
 
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.5.9
1
+ 0.5.10
@@ -4,6 +4,7 @@ $:.push File.expand_path('../lib', __FILE__)
4
4
  Gem::Specification.new do |gem|
5
5
  gem.name = "fluent-plugin-s3"
6
6
  gem.description = "Amazon S3 output plugin for Fluentd event collector"
7
+ gem.license = "Apache-2.0"
7
8
  gem.homepage = "https://github.com/fluent/fluent-plugin-s3"
8
9
  gem.summary = gem.description
9
10
  gem.version = File.read("VERSION").strip
@@ -107,7 +107,8 @@ module Fluent
107
107
  "path" => path,
108
108
  "time_slice" => chunk.key,
109
109
  "file_extension" => @compressor.ext,
110
- "index" => i
110
+ "index" => i,
111
+ "uuid_flush" => uuid_random
111
112
  }
112
113
  s3path = @s3_object_key_format.gsub(%r(%{[^}]+})) { |expr|
113
114
  values_for_s3_object_key[expr[2...expr.size-1]]
@@ -288,6 +288,54 @@ class S3OutputTest < Test::Unit::TestCase
288
288
  d.run
289
289
  end
290
290
 
291
+ def test_write_with_custom_s3_object_key_format_containing_uuid_flush_placeholder
292
+ # Assert content of event logs which are being sent to S3
293
+ s3obj = flexmock(AWS::S3::S3Object)
294
+ s3obj.should_receive(:exists?).with_any_args.and_return { false }
295
+ s3obj.should_receive(:write).with(
296
+ on { |pathname|
297
+ data = nil
298
+ # Event logs are compressed in GZip
299
+ pathname.open { |f|
300
+ gz = Zlib::GzipReader.new(f)
301
+ data = gz.read
302
+ gz.close
303
+ }
304
+ assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
305
+ %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
306
+ data
307
+
308
+ pathname.to_s.match(%r|s3-|)
309
+ },
310
+ {:content_type => "application/x-gzip", :reduced_redundancy => false})
311
+
312
+ # Assert the key of S3Object, which event logs are stored in
313
+ s3obj_col = flexmock(AWS::S3::ObjectCollection)
314
+ s3obj_col.should_receive(:[]).with(
315
+ on { |key|
316
+ key =~ /.*.[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}.*/
317
+ }).
318
+ and_return {
319
+ s3obj
320
+ }
321
+
322
+ # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
323
+ s3bucket, _ = setup_mocks(true)
324
+ s3bucket.should_receive(:objects).with_any_args.and_return { s3obj_col }
325
+
326
+ # We must use TimeSlicedOutputTestDriver instead of BufferedOutputTestDriver,
327
+ # to make assertions on chunks' keys
328
+ config = CONFIG_TIME_SLICE.clone.gsub(/%{hostname}/,"%{uuid_flush}")
329
+ d = create_time_sliced_driver(config)
330
+
331
+ time = Time.parse("2011-01-02 13:14:15 UTC").to_i
332
+ d.emit({"a"=>1}, time)
333
+ d.emit({"a"=>2}, time)
334
+
335
+ # Finally, the instance of S3Output is initialized and then invoked
336
+ d.run
337
+ end
338
+
291
339
  def setup_mocks(exists_return = false)
292
340
  s3bucket = flexmock(AWS::S3::Bucket)
293
341
  s3bucket.should_receive(:exists?).with_any_args.and_return { exists_return }
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-s3
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.9
4
+ version: 0.5.10
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2015-06-10 00:00:00.000000000 Z
12
+ date: 2015-07-27 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
@@ -136,7 +136,8 @@ files:
136
136
  - lib/fluent/plugin/s3_compressor_lzo.rb
137
137
  - test/test_out_s3.rb
138
138
  homepage: https://github.com/fluent/fluent-plugin-s3
139
- licenses: []
139
+ licenses:
140
+ - Apache-2.0
140
141
  metadata: {}
141
142
  post_install_message:
142
143
  rdoc_options: []
@@ -154,7 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
154
155
  version: '0'
155
156
  requirements: []
156
157
  rubyforge_project:
157
- rubygems_version: 2.2.2
158
+ rubygems_version: 2.2.3
158
159
  signing_key:
159
160
  specification_version: 4
160
161
  summary: Amazon S3 output plugin for Fluentd event collector