fluent-plugin-s3 0.6.6 → 0.6.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 563ab768bef3f13148f841becf15b1d7e8e88ed2
-  data.tar.gz: b3b54160a015d14b801914e522fef06ad5d7bb1a
+  metadata.gz: cc5bd6dc0616525d3beb069168b7f59bc985b188
+  data.tar.gz: 5021dbf9b12861ee087ad2b5a1800d9430e79874
 SHA512:
-  metadata.gz: 27b44ab95d94a7cf93cd1337aea917326bf596693b7af529a75cfb343c7a65e97311c9345d21ee8bf3983c69b84a2773f26081c4cc6f928fb6f25ba9a621b285
-  data.tar.gz: 205930d50c3e1d1d42383e149b870aa4e8b8232780288026fcdeb214624626927b3e68601a1f361aaa1dd3632e3a81119d9e964615e890bbf19ebe94733ecd06
+  metadata.gz: f0c104943d41ecaacefcf35565d3bf3de8ff8c0301d3edb02994f8425cfcc81f43833c1eed478dfc8fd2965017068da32d43a590644e82bbd1cd98277f3888bd
+  data.tar.gz: 4580b5c87c33db7fc7ba65c250c3c2e1330c10be6e75bde2144175df41276cfe929031379df785dba68b5d8196890e6ba51b8fbd5081e8e769ea96b24718c6ae

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 0.6.7 - 2016/03/31
+
+  * Add signature_version parameter
+  * Add warn_for_delay parameter
+
+
 Release 0.6.6 - 2016/03/16
 
   * Fix ACL handling in PUT operation

data/README.md

@@ -293,6 +293,16 @@ AWS SDK uses MD5 for API request/response by default. On FIPS enabled environmen
 OpenSSL returns an error because MD5 is disabled. If you want to use
 this plugin on FIPS enabled environment, set `compute_checksums false`.
 
+**signature_version**
+
+Signature version for API request. `s3` means signature version 2 and
+`v4` means signature version 4. Default is `nil` (Following SDK's default).
+It would be useful when you use S3 compatible storage that accepts only signature version 2.
+
+**warn_for_delay**
+
+Given a threshold to treat events as delay, output warning logs if delayed events were put into s3.
+
 ### assume_role_credentials
 
 Typically, you use AssumeRole for cross-account access or federation.

data/VERSION

@@ -1 +1 @@
-0.6.6
+0.6.7

data/fluent-plugin-s3.gemspec

@@ -24,4 +24,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
   gem.add_development_dependency "test-unit-rr", ">= 1.0.3"
+  gem.add_development_dependency "timecop"
 end

data/lib/fluent/plugin/out_s3.rb

@@ -105,6 +105,10 @@ module Fluent
     config_param :ssekms_key_id, :string, :default => nil, :secret => true
     desc "AWS SDK uses MD5 for API request/response by default"
     config_param :compute_checksums, :bool, :default => nil # use nil to follow SDK default configuration
+    desc "Signature version for API Request (s3,v4)"
+    config_param :signature_version, :string, :default => nil # use nil to follow SDK default configuration
+    desc "Given a threshold to treat events as delay, output warning logs if delayed events were put into s3"
+    config_param :warn_for_delay, :time, :default => nil
 
     attr_reader :bucket
 
@@ -159,6 +163,7 @@ module Fluent
       options[:http_proxy] = @proxy_uri if @proxy_uri
       options[:force_path_style] = @force_path_style
       options[:compute_checksums] = @compute_checksums unless @compute_checksums.nil?
+      options[:signature_version] = @signature_version unless @signature_version.nil?
 
       s3_client = Aws::S3::Client.new(options)
       @s3 = Aws::S3::Resource.new(:client => s3_client)
@@ -227,6 +232,12 @@ module Fluent
         @bucket.object(s3path).put(put_options)
 
         @values_for_s3_object_chunk.delete(chunk.unique_id)
+
+        if @warn_for_delay
+          if Time.strptime(chunk.key, @time_slice_format) < Time.now - @warn_for_delay
+            log.warn { "out_s3: delayed events were put to s3://#{@s3_bucket}/#{s3path}" }
+          end
+        end
       ensure
         tmp.close(true) rescue nil
       end

data/test/test_out_s3.rb

@@ -4,6 +4,7 @@ require 'fluent/plugin/out_s3'
 require 'test/unit/rr'
 require 'zlib'
 require 'fileutils'
+require 'timecop'
 
 class S3OutputTest < Test::Unit::TestCase
   def setup
@@ -50,6 +51,7 @@ class S3OutputTest < Test::Unit::TestCase
     assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@compressor).content_type
     assert_equal false, d.instance.force_path_style
     assert_equal nil, d.instance.compute_checksums
+    assert_equal nil, d.instance.signature_version
   end
 
   def test_s3_endpoint_with_valid_endpoint
@@ -274,19 +276,8 @@ class S3OutputTest < Test::Unit::TestCase
   def test_write_with_custom_s3_object_key_format
     # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
     setup_mocks(true)
-
-    # Assert content of event logs which are being sent to S3
-    s3obj = stub(Aws::S3::Object.new(:bucket_name => "test_bucket",
-                                     :key => "test",
-                                     :client => @s3_client))
-    s3obj.exists? { false }
-    s3_test_file_path = "/tmp/s3-test.txt"
-    tempfile = File.new(s3_test_file_path, "w")
-    mock(Tempfile).new("s3-") { tempfile }
-    s3obj.put(:body => tempfile,
-              :content_type => "application/x-gzip",
-              :storage_class => "STANDARD")
-    @s3_bucket.object("log/events/ts=20110102-13/events_0-testing.node.local.gz") { s3obj }
+    s3_local_file_path = "/tmp/s3-test.txt"
+    setup_s3_object_mocks(s3_local_file_path: s3_local_file_path)
 
     # We must use TimeSlicedOutputTestDriver instead of BufferedOutputTestDriver,
     # to make assertions on chunks' keys
@@ -298,13 +289,13 @@ class S3OutputTest < Test::Unit::TestCase
 
     # Finally, the instance of S3Output is initialized and then invoked
     d.run
-    Zlib::GzipReader.open(s3_test_file_path) do |gz|
+    Zlib::GzipReader.open(s3_local_file_path) do |gz|
       data = gz.read
       assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
                    %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
                    data
     end
-    FileUtils.rm_f(s3_test_file_path)
+    FileUtils.rm_f(s3_local_file_path)
   end
 
   def test_write_with_custom_s3_object_key_format_containing_uuid_flush_placeholder
@@ -314,18 +305,9 @@ class S3OutputTest < Test::Unit::TestCase
     uuid = "5755e23f-9b54-42d8-8818-2ea38c6f279e"
     stub(UUIDTools::UUID).random_create{ uuid }
 
-    # Assert content of event logs which are being sent to S3
-    s3obj = stub(Aws::S3::Object.new(:bucket_name => "test_bucket",
-                                     :key => "test",
-                                     :client => @s3_client))
-    s3obj.exists? { false }
-    s3_test_file_path = "/tmp/s3-test.txt"
-    tempfile = File.new(s3_test_file_path, "w")
-    mock(Tempfile).new("s3-") { tempfile }
-    s3obj.put(:body => tempfile,
-              :content_type => "application/x-gzip",
-              :storage_class => "STANDARD")
-    @s3_bucket.object("log/events/ts=20110102-13/events_0-#{uuid}.gz") { s3obj }
+    s3_local_file_path = "/tmp/s3-test.txt"
+    s3path = "log/events/ts=20110102-13/events_0-#{uuid}.gz"
+    setup_s3_object_mocks(s3_local_file_path: s3_local_file_path, s3path: s3path)
 
     # We must use TimeSlicedOutputTestDriver instead of BufferedOutputTestDriver,
     # to make assertions on chunks' keys
@@ -338,13 +320,13 @@ class S3OutputTest < Test::Unit::TestCase
 
     # Finally, the instance of S3Output is initialized and then invoked
     d.run
-    Zlib::GzipReader.open(s3_test_file_path) do |gz|
+    Zlib::GzipReader.open(s3_local_file_path) do |gz|
       data = gz.read
       assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
                    %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
                    data
     end
-    FileUtils.rm_f(s3_test_file_path)
+    FileUtils.rm_f(s3_local_file_path)
     Dir.glob('tmp/*').each {|file| FileUtils.rm_f(file) }
   end
 
@@ -375,18 +357,9 @@ class S3OutputTest < Test::Unit::TestCase
     # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
     setup_mocks(true)
 
-    # Assert content of event logs which are being sent to S3
-    s3obj = stub(Aws::S3::Object.new(:bucket_name => "test_bucket",
-                                     :key => "test",
-                                     :client => @s3_client))
-    s3obj.exists? { false }
-    s3_test_file_path = "/tmp/s3-test.txt"
-    tempfile = File.new(s3_test_file_path, "w")
-    mock(Tempfile).new("s3-") { tempfile }
-    s3obj.put(:body => tempfile,
-              :content_type => "application/x-gzip",
-              :storage_class => "STANDARD")
-    @s3_bucket.object("log/events/ts=20110102-13/events_0-#{hex}.gz") { s3obj }
+    s3path = "log/events/ts=20110102-13/events_0-#{hex}.gz"
+    s3_local_file_path = "/tmp/s3-test.txt"
+    setup_s3_object_mocks(s3_local_file_path: s3_local_file_path, s3path: s3path)
 
     d = create_time_sliced_driver(config)
 
@@ -396,13 +369,13 @@ class S3OutputTest < Test::Unit::TestCase
 
     # Finally, the instance of S3Output is initialized and then invoked
     d.run
-    Zlib::GzipReader.open(s3_test_file_path) do |gz|
+    Zlib::GzipReader.open(s3_local_file_path) do |gz|
       data = gz.read
       assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
                    %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
                    data
     end
-    FileUtils.rm_f(s3_test_file_path)
+    FileUtils.rm_f(s3_local_file_path)
   end
 
   def setup_mocks(exists_return = false)
@@ -420,6 +393,25 @@ class S3OutputTest < Test::Unit::TestCase
     @s3_resource.bucket(anything) { @s3_bucket }
   end
 
+  def setup_s3_object_mocks(params = {})
+    s3path = params[:s3path] || "log/events/ts=20110102-13/events_0-testing.node.local.gz"
+    s3_local_file_path = params[:s3_local_file_path] || "/tmp/s3-test.txt"
+
+    # Assert content of event logs which are being sent to S3
+    s3obj = stub(Aws::S3::Object.new(:bucket_name => "test_bucket",
+                                     :key => "test",
+                                     :client => @s3_client))
+    s3obj.exists? { false }
+
+    tempfile = File.new(s3_local_file_path, "w")
+    stub(Tempfile).new("s3-") { tempfile }
+    s3obj.put(:body => tempfile,
+              :content_type => "application/x-gzip",
+              :storage_class => "STANDARD")
+
+    @s3_bucket.object(s3path) { s3obj }
+  end
+
   def test_auto_create_bucket_false_with_non_existence_bucket
     setup_mocks
 
@@ -534,4 +526,36 @@ class S3OutputTest < Test::Unit::TestCase
     credentials = client.config.credentials
     assert_equal(expected_credentials, credentials)
   end
+
+  def test_signature_version
+    config = [CONFIG, 'signature_version s3'].join("\n")
+    d = create_driver(config)
+
+    signature_version = d.instance.instance_variable_get(:@signature_version)
+    assert_equal("s3", signature_version)
+  end
+
+  def test_warn_for_delay
+    setup_mocks(true)
+    s3_local_file_path = "/tmp/s3-test.txt"
+    setup_s3_object_mocks(s3_local_file_path: s3_local_file_path)
+
+    config = CONFIG_TIME_SLICE + 'warn_for_delay 1d'
+    d = create_time_sliced_driver(config)
+
+    delayed_time = Time.parse("2011-01-02 13:14:15 UTC")
+    now = delayed_time + 86000 + 1
+    Timecop.freeze(now)
+
+    d.emit({"a"=>1}, delayed_time.to_i)
+    d.emit({"a"=>2}, delayed_time.to_i)
+
+    d.run
+
+    logs = d.instance.log.logs
+    assert_true logs.any? {|log| log.include?('[warn]: out_s3: delayed events were put') }
+
+    Timecop.return
+    FileUtils.rm_f(s3_local_file_path)
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 0.6.6
+  version: 0.6.7
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-17 00:00:00.000000000 Z
+date: 2016-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -115,6 +115,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.3
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Amazon S3 output plugin for Fluentd event collector
 email: frsyuki@gmail.com
 executables: []