fluent-plugin-s3 0.5.9 → 0.5.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: feddb8710e1325aa7cc3a277082a67d6bc67c5f9
-  data.tar.gz: 95e0def196fbe430df629400990695f783716872
+  metadata.gz: f674b76e6ab0fcde41c2a1f580b36f2b0e346f01
+  data.tar.gz: 32de8cacec06dd1f7a0ab241e05eafca9ecaf755
 SHA512:
-  metadata.gz: 744be5bcaf22b43c565ac5a61271dec1abc1532dcf81d6fb5cb2352fabcdad36367ee9c83c56dd30fac6494f72abd5098eb119d2cf6f0c02a51975ef8f11a7be
-  data.tar.gz: f8f607b792454e660e40412dd1df7352f55478813696bb935190e104f6716d2a8b8131c10a6f3b4f6a094c78521833c66486b7d3e72526ac7afc498821c32ed5
+  metadata.gz: 8417dad2be0be87130030d41fdfb59ba553d5f574c71f0eb3f685145dd7b6ff1011ff9dd89d35b65b4dcb422be0e54f297c312d51cbdef2e70728c4ce5dbe42f
+  data.tar.gz: e0f4aaace4ebf98f33cabaa782ae39b8999d0fee860d9b7a0eec990f0661faddffdf8510ea29609ab786587f6044f799abb9392f649fd423d7570aed42456085

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 0.5.10 - 2015/07/27
+
+  * Add "uuid_flush" placeholder to s3_object_key_format for creating unique object
+
+
 Release 0.5.9 - 2015/06/10
 
   * Add secret option to AWS key related parameters

data/README.rdoc

@@ -51,6 +51,7 @@ Simply use RubyGems:
 - %{time_slice}
 - %{index}
 - %{file_extension}
+- %{uuid_flush}
 
 to decide keys dynamically.
 
@@ -58,6 +59,7 @@ to decide keys dynamically.
 %{time_slice} is the time-slice in text that are formatted with *time_slice_format*.
 %{index} is the sequential number starts from 0, increments when multiple files are uploaded to S3 in the same time slice.
 %{file_extention} is always "gz" for now.
+%{uuid_flush} a uuid that is replaced everytime the buffer will be flushed
 
 The default format is "%{path}%{time_slice}_%{index}.%{file_extension}".
 

data/VERSION

@@ -1 +1 @@
-0.5.9
+0.5.10

data/fluent-plugin-s3.gemspec

@@ -4,6 +4,7 @@ $:.push File.expand_path('../lib', __FILE__)
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-s3"
   gem.description = "Amazon S3 output plugin for Fluentd event collector"
+  gem.license     = "Apache-2.0"
   gem.homepage    = "https://github.com/fluent/fluent-plugin-s3"
   gem.summary     = gem.description
   gem.version     = File.read("VERSION").strip

data/lib/fluent/plugin/out_s3.rb

@@ -107,7 +107,8 @@ module Fluent
           "path" => path,
           "time_slice" => chunk.key,
           "file_extension" => @compressor.ext,
-          "index" => i
+          "index" => i,
+          "uuid_flush" => uuid_random
         }
         s3path = @s3_object_key_format.gsub(%r(%{[^}]+})) { |expr|
           values_for_s3_object_key[expr[2...expr.size-1]]

data/test/test_out_s3.rb

@@ -288,6 +288,54 @@ class S3OutputTest < Test::Unit::TestCase
     d.run
   end
 
+  def test_write_with_custom_s3_object_key_format_containing_uuid_flush_placeholder
+    # Assert content of event logs which are being sent to S3
+    s3obj = flexmock(AWS::S3::S3Object)
+    s3obj.should_receive(:exists?).with_any_args.and_return { false }
+    s3obj.should_receive(:write).with(
+      on { |pathname|
+        data = nil
+        # Event logs are compressed in GZip
+        pathname.open { |f|
+          gz = Zlib::GzipReader.new(f)
+          data = gz.read
+          gz.close
+        }
+        assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
+                     %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
+                     data
+
+        pathname.to_s.match(%r|s3-|)
+      },
+      {:content_type => "application/x-gzip", :reduced_redundancy => false})
+
+    # Assert the key of S3Object, which event logs are stored in
+    s3obj_col = flexmock(AWS::S3::ObjectCollection)
+    s3obj_col.should_receive(:[]).with(
+      on { |key|
+        key =~ /.*.[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}.*/
+      }).
+      and_return {
+        s3obj
+      }
+
+    # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
+    s3bucket, _ = setup_mocks(true)
+    s3bucket.should_receive(:objects).with_any_args.and_return { s3obj_col }
+
+    # We must use TimeSlicedOutputTestDriver instead of BufferedOutputTestDriver,
+    # to make assertions on chunks' keys
+    config = CONFIG_TIME_SLICE.clone.gsub(/%{hostname}/,"%{uuid_flush}")
+    d = create_time_sliced_driver(config)
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    # Finally, the instance of S3Output is initialized and then invoked
+    d.run
+  end
+
   def setup_mocks(exists_return = false)
     s3bucket = flexmock(AWS::S3::Bucket)
     s3bucket.should_receive(:exists?).with_any_args.and_return { exists_return }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 0.5.9
+  version: 0.5.10
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-10 00:00:00.000000000 Z
+date: 2015-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -136,7 +136,8 @@ files:
 - lib/fluent/plugin/s3_compressor_lzo.rb
 - test/test_out_s3.rb
 homepage: https://github.com/fluent/fluent-plugin-s3
-licenses: []
+licenses:
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -154,7 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.2.3
 signing_key: 
 specification_version: 4
 summary: Amazon S3 output plugin for Fluentd event collector