fluent-plugin-s3 0.6.8 → 0.6.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8b5cf42881912f6452810b283b3bd9be7a94160f
-  data.tar.gz: a7f3fb4b37a91b18b72d8f7bdef7a80c783ba64b
+  metadata.gz: 4d3125cb720171313d0ac3b19ee194e3996454d4
+  data.tar.gz: ca82560974fb26d500e418888b6ca03cdc020f2c
 SHA512:
-  metadata.gz: 21e938107f0fa66ddedf3634013c9e0472ac2a5855f26dfecd3bd01bebce120957e7e656693a4bf993a28afc9272514ac61b5b8678d7f313bec080161c47c442
-  data.tar.gz: 2c334c3fe1c8f34baae53b5668a02971279c8a574e063a04c6f9e4ce925601060bd957356e700abc9d49f0ed3aa3f891ad633e815f62d1eb85b60741268a0aee
+  metadata.gz: 981012e05d6804b223b3e683e3da31754f034c818a4bdcc3820be7643c48a21ecd08a76e9df945adf35b77ed4e70e4b1da33bd4155765ee26cabefbadda6fb36
+  data.tar.gz: 751fbb6b67929ea0bb0934e345a83f0afb3f8cea60cffc64f4f6947f623122643c50e6b5bb1e9c1266a01fa708b3150e74777fd71bdd4bc809cfbcac59f2a23b

data/.travis.yml

@@ -5,12 +5,13 @@ rvm:
   - 2.0
   - 2.1
   - 2.2.3
-  - 2.3.0
+  - 2.3.1
   - ruby-head
   - rbx
 
 gemfile:
  - Gemfile
+ - Gemfile.v0.12
 
 branches:
   only:
@@ -25,3 +26,8 @@ matrix:
   allow_failures:
     - rvm: ruby-head
     - rvm: rbx
+  exclude:
+    - rvm: 1.9.3
+      gemfile: Gemfile
+    - rvm: 2.0
+      gemfile: Gemfile

data/ChangeLog

@@ -1,3 +1,10 @@
+Release 0.6.9 - 2016/07/28
+
+  * Support v0.14
+  * Fix aws_iam_retries warning
+  * Fix race condition at start with buffers
+
+
 Release 0.6.8 - 2016/04/19
 
   * Add sse customer key options

data/Gemfile

@@ -1,3 +1,5 @@
 source "http://rubygems.org"
 
+gem 'json', '= 1.8.3'
+
 gemspec

data/Gemfile.v0.12

@@ -0,0 +1,6 @@
+source "http://rubygems.org"
+
+gem 'json', '= 1.8.3'
+gem 'fluentd', '~> 0.12.0'
+
+gemspec

data/README.md

@@ -126,8 +126,7 @@ You get:
     "log/events/ts=20130112-00/events_0.gz"
 
 The
-[fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-c
-onfig-placeholders) mixin is also incorporated, so additional variables such
+[fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) mixin is also incorporated, so additional variables such
 as %{hostname}, %{uuid}, etc. can be used in the s3_object_key_format. This
 could prove useful in preventing filename conflicts when writing from multiple
 servers.
@@ -158,7 +157,7 @@ See `Use your compression algorithm` section for adding another format.
 **format**
 
 Change one line format in the S3 object. Supported formats are "out_file",
-"json", "ltsv" and "single_value".
+"json", "ltsv" and "single_value". See also [official Formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
 
 * out_file (default).
 
@@ -260,8 +259,7 @@ using IAM roles. Valid values are:
 *   private (default)
 *   public-read
 *   public-read-write (not recommended - see [Canned
-    ACL](http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#cann
-    ed-acl))
+    ACL](http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl))
 *   authenticated-read
 *   bucket-owner-read
 *   bucket-owner-full-control
@@ -337,12 +335,9 @@ Typically, you use AssumeRole for cross-account access or federation.
 See also:
 
 *   [Using IAM Roles - AWS Identity and Access
-    Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.h
-    tml)
-*   [Aws::STS::Client](http://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Clien
-    t.html)
-*   [Aws::AssumeRoleCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws
-    /AssumeRoleCredentials.html)
+    Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
+*   [Aws::STS::Client](http://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
+*   [Aws::AssumeRoleCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html)
 
 **role_arn (required)**
 
@@ -383,14 +378,11 @@ EC2 instance.
 
 See also:
 
-*   [Aws::InstanceProfileCredentials](http://docs.aws.amazon.com/sdkforruby/ap
-    i/Aws/InstanceProfileCredentials.html)
+*   [Aws::InstanceProfileCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/InstanceProfileCredentials.html)
 *   [Temporary Security Credentials - AWS Identity and Access
-    Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials
-    _temp.html)
+    Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html)
 *   [Instance Metadata and User Data - Amazon Elastic Compute
-    Cloud](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-met
-    adata.html)
+    Cloud](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
 
 **retries**
 
@@ -428,8 +420,7 @@ local developing.
 
 See also:
 
-*   [Aws::SharedCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/Sha
-    redCredentials.html)
+*   [Aws::SharedCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/SharedCredentials.html)
 
 **path**
 
@@ -456,8 +447,7 @@ Note that the bucket must already exist and **auto_create_bucket** has no
 effect in this case.
 
 Refer to the [AWS
-documentation](http://docs.aws.amazon.com/IAM/latest/UserGuide/ExampleIAMPolic
-ies.html) for example policies.
+documentation](http://docs.aws.amazon.com/IAM/latest/UserGuide/ExampleIAMPolicies.html) for example policies.
 
 Using [IAM
 roles](http://docs.aws.amazon.com/IAM/latest/UserGuide/WorkingWithRoles.html)

data/VERSION

@@ -1 +1 @@
-0.6.8
+0.6.9

data/fluent-plugin-s3.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "yajl-ruby", "~> 1.0"
   gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
   gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "rr", "= 1.1.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
   gem.add_development_dependency "test-unit-rr", ">= 1.0.3"
   gem.add_development_dependency "timecop"

data/lib/fluent/plugin/out_s3.rb

@@ -1,6 +1,5 @@
 module Fluent
   require 'fluent/mixin/config_placeholders'
-  require 'securerandom'
 
   class S3Output < Fluent::TimeSlicedOutput
     Fluent::Plugin.register_output('s3', self)
@@ -70,7 +69,7 @@ module Fluent
       config_param :profile_name, :string, :default => nil
     end
     desc "The number of attempts to load instance profile credentials from the EC2 metadata service using IAM role"
-    config_param :aws_iam_retries, :integer, :default => 5
+    config_param :aws_iam_retries, :integer, :default => nil
     desc "S3 bucket name"
     config_param :s3_bucket, :string
     desc "S3 region name"
@@ -165,7 +164,6 @@ module Fluent
     end
 
     def start
-      super
       options = setup_credentials
       options[:region] = @s3_region if @s3_region
       options[:endpoint] = @s3_endpoint if @s3_endpoint
@@ -181,8 +179,7 @@ module Fluent
       check_apikeys if @check_apikey_on_start
       ensure_bucket
 
-      # Securerandom.hex(2) returns 4 length hex
-      @hex_random_n = (@hex_random_length + 1) / 2
+      super
     end
 
     def format(tag, time, record)
@@ -257,25 +254,16 @@ module Fluent
 
     private
 
-    # tsuffix is the one which file buffer filename has
-    def tsuffix(chunk)
-      if chunk.is_a?(Fluent::FileBufferChunk)
-        unique_id = chunk.unique_id
-        tsuffix = unique_id[0...(unique_id.size/2)].unpack('C*').map {|x| x.to_s(16) }.join('') # size: 16
-      else
-        nil
-      end
+    # v0.14 has a useful Fluent::UniqueId.hex(unique_id) method, though
+    def unique_hex(chunk)
+      unique_id = chunk.unique_id
+      unique_id.unpack('C*').map {|x| x.to_s(16) }.join('')
     end
 
     def hex_random(chunk)
-      if chunk.is_a?(Fluent::FileBufferChunk)
-        # let me use tsuffix because its value is kept on retrying even after rebooting
-        tsuffix = tsuffix(chunk)
-        tsuffix.reverse! # tsuffix is like (time_sec, time_usec, rand) => reversing gives more randomness
-        tsuffix[0...@hex_random_length]
-      else
-        SecureRandom.hex(@hex_random_n)[0...@hex_random_length]
-      end
+      unique_hex = unique_hex(chunk)
+      unique_hex.reverse! # unique_hex is like (time_sec, time_usec, rand) => reversing gives more randomness
+      unique_hex[0...@hex_random_length]
     end
 
     def ensure_bucket

data/test/test_out_s3.rb

@@ -26,7 +26,7 @@ class S3OutputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::S3Output) do
+    Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::S3Output) do
       def write(chunk)
         chunk.read
       end
@@ -247,7 +247,7 @@ class S3OutputTest < Test::Unit::TestCase
 
     assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
                  %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
-                 data
+                 data.first
   end
 
   CONFIG_TIME_SLICE = %[
@@ -330,38 +330,24 @@ class S3OutputTest < Test::Unit::TestCase
     Dir.glob('tmp/*').each {|file| FileUtils.rm_f(file) }
   end
 
-  def test_write_with_custom_s3_object_key_format_containing_hex_random_placeholder_memory_buffer
-    hex = "012345"
-    mock(SecureRandom).hex(3) { hex }
-
-    config = CONFIG_TIME_SLICE.gsub(/%{hostname}/,"%{hex_random}") << "\nhex_random_length 5"
-    write_with_custom_s3_object_key_format_containing_hex_random_placeholder(config, hex[0...5])
-  end
-
-  def test_write_with_custom_s3_object_key_format_containing_hex_random_placeholder_file_buffer
-    tsuffix = "5226c3c4fb3d49b1"
-    any_instance_of(Fluent::FileBufferChunk) do |klass|
-      unique_id = "R&\xC3\xC4\xFB=I\xB1R&\xC3\xC4\xFB=I\xB1" # corresponding unique_id with tsuffxi
-      stub(klass).unique_id { unique_id }
-    end
-    hex = tsuffix.reverse
+  # ToDo: need to test hex_random does not change on retry, but it is difficult with
+  # the current fluentd test helper because it does not provide a way to run with the same chunks
+  def test_write_with_custom_s3_object_key_format_containing_hex_random_placeholder
+    unique_hex = "5226c3c4fb3d49b15226c3c4fb3d49b1"
+    hex_random = unique_hex.reverse[0...5]
 
-    config = CONFIG_TIME_SLICE.gsub(/%{hostname}/,"%{hex_random}") << "\nhex_random_length 16"
+    config = CONFIG_TIME_SLICE.gsub(/%{hostname}/,"%{hex_random}") << "\nhex_random_length #{hex_random.length}"
     config = config.gsub(/buffer_type memory/, "buffer_type file\nbuffer_path test/tmp/buf")
-    write_with_custom_s3_object_key_format_containing_hex_random_placeholder(config, hex)
-  end
 
-  # ToDo: need to test hex_random does not change on retry, but it is difficult with
-  # the current fluentd test helper because it does not provide a way to run with the same chunks
-  def write_with_custom_s3_object_key_format_containing_hex_random_placeholder(config, hex)
     # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
     setup_mocks(true)
 
-    s3path = "log/events/ts=20110102-13/events_0-#{hex}.gz"
+    s3path = "log/events/ts=20110102-13/events_0-#{hex_random}.gz"
     s3_local_file_path = "/tmp/s3-test.txt"
     setup_s3_object_mocks(s3_local_file_path: s3_local_file_path, s3path: s3path)
 
     d = create_time_sliced_driver(config)
+    stub(d.instance).unique_hex { unique_hex }
 
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
     d.emit({"a"=>1}, time)
@@ -553,7 +539,7 @@ class S3OutputTest < Test::Unit::TestCase
     d.run
 
     logs = d.instance.log.logs
-    assert_true logs.any? {|log| log.include?('[warn]: out_s3: delayed events were put') }
+    assert_true logs.any? {|log| log.include?('out_s3: delayed events were put') }
 
     Timecop.return
     FileUtils.rm_f(s3_local_file_path)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 0.6.8
+  version: 0.6.9
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-19 00:00:00.000000000 Z
+date: 2016-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -87,6 +87,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.1.2
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -140,6 +154,7 @@ files:
 - AUTHORS
 - ChangeLog
 - Gemfile
+- Gemfile.v0.12
 - README.md
 - Rakefile
 - VERSION
@@ -170,7 +185,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Amazon S3 output plugin for Fluentd event collector