fluent-plugin-s3 0.2.6 → 0.3.0

data/ChangeLog

@@ -1,4 +1,19 @@
-Release 0.2.6 - 2012/01/15
+Release 0.3.0 - 2013/02/19
+
+  * Enable dynamic and configurable S3 object kyes
+    https://github.com/fluent/fluent-plugin-s3/pull/12
+  * Fix a lot of temporary files were left on /tmp when the plugin failed to write to S3
+    https://github.com/fluent/fluent-plugin-s3/pull/15
+  * Enable fluent-mixin-config-placeholders to support hostname, uuid and other parameters in configuration
+    https://github.com/fluent/fluent-plugin-s3/pull/19
+  * Update 'aws-sdk' version requirement to '~> 1.8.2'
+    https://github.com/fluent/fluent-plugin-s3/pull/21
+  * Create new S3 bucket if not exists
+    https://github.com/fluent/fluent-plugin-s3/pull/22
+  * Check the permission and bucket existence at start method, not write method.
+
+
+Release 0.2.6 - 2013/01/15
 
   * Add use_ssl option
 

data/README.rdoc

@@ -22,6 +22,7 @@ Simply use RubyGems:
       aws_sec_key YOUR_AWS_SECRET/KEY
       s3_bucket YOUR_S3_BUCKET_NAME
       s3_endpoint s3-ap-northeast-1.amazonaws.com
+      s3_object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
       path logs/
       buffer_path /var/log/fluent/s3
 
@@ -38,6 +39,48 @@ Simply use RubyGems:
 
 [s3_endpoint] s3 endpoint name. Example, Tokyo region is "s3-ap-northeast-1.amazonaws.com".
 
+[s3_object_key_format] The format of S3 object keys. You can use several built-in variables:
+
+- %{path}
+- %{time_slice}
+- %{index}
+- %{file_extension}
+
+to decide keys dynamically.
+
+%{path} is exactly the value of *path* configured in the configuration file. E.g., "logs/" in the example configuration above.
+%{time_slice} is the time-slice in text that are formatted with *time_slice_format*.
+%{index} is the sequential number starts from 0, increments when multiple files are uploaded to S3 in the same time slice.
+%{file_extention} is always "gz" for now.
+
+The default format is "%{path}%{time_slice}_%{index}.%{file_extension}".
+
+For instance, using the example configuration above, actual object keys on S3 will be something like:
+
+    "logs/20130111-22_0.gz"
+    "logs/20130111-23_0.gz"
+    "logs/20130111-23_1.gz"
+    "logs/20130112-00_0.gz"
+
+With the configuration:
+
+    s3_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}.%{file_extension}
+    path log
+    time_slice_format %Y%m%d-%H
+
+You get:
+
+    "log/events/ts=20130111-22/events_0.gz"
+    "log/events/ts=20130111-23/events_0.gz"
+    "log/events/ts=20130111-23/events_1.gz"
+    "log/events/ts=20130112-00/events_0.gz"
+
+The {fluent-mixin-config-placeholders}[https://github.com/tagomoris/fluent-mixin-config-placeholders] mixin is also incorporated, so additional variables such as %{hostname}, %{uuid}, etc. can be used in the s3_object_key_format. This could prove useful in preventing filename conflicts when writing from multiple servers.
+
+    s3_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
+
+[auto_create_bucket] Create S3 bucket if it does not exists. Default is true.
+
 [path] path prefix of the files on S3. Default is "" (no prefix).
 
 [buffer_path (required)] path prefix of the files to buffer logs.
@@ -48,8 +91,6 @@ Simply use RubyGems:
 
 [utc] Use UTC instead of local time.
 
-The actual path on S3 will be: "{path}{time_slice_format}_{sequential_number}.gz"
-
 
 == Copyright
 

data/VERSION

@@ -1 +1 @@
-0.2.6
+0.3.0

data/fluent-plugin-s3.gemspec

@@ -17,7 +17,9 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_dependency "fluentd", "~> 0.10.0"
-  gem.add_dependency "aws-sdk", "~> 1.7"
+  gem.add_dependency "aws-sdk", "~> 1.8.2"
   gem.add_dependency "yajl-ruby", "~> 1.0"
+  gem.add_dependency "fluent-mixin-config-placeholders", "~> 0.2.0"
   gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "flexmock", ">= 1.2.0"
 end

data/lib/fluent/plugin/out_s3.rb

@@ -1,5 +1,6 @@
 module Fluent
 
+require 'fluent/mixin/config_placeholders'
 
 class S3Output < Fluent::TimeSlicedOutput
   Fluent::Plugin.register_output('s3', self)
@@ -27,6 +28,16 @@ class S3Output < Fluent::TimeSlicedOutput
   config_param :aws_sec_key, :string, :default => nil
   config_param :s3_bucket, :string
   config_param :s3_endpoint, :string, :default => nil
+  config_param :s3_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
+  config_param :auto_create_bucket, :bool, :default => true
+
+  attr_reader :bucket
+
+  include Fluent::Mixin::ConfigPlaceholders
+
+  def placeholders
+    [:percent]
+  end
 
   def configure(conf)
     super
@@ -63,6 +74,9 @@ class S3Output < Fluent::TimeSlicedOutput
 
     @s3 = AWS::S3.new(options)
     @bucket = @s3.buckets[@s3_bucket]
+
+    ensure_bucket
+    check_apikeys
   end
 
   def format(tag, time, record)
@@ -88,7 +102,15 @@ class S3Output < Fluent::TimeSlicedOutput
   def write(chunk)
     i = 0
     begin
-      s3path = "#{@path}#{chunk.key}_#{i}.gz"
+      values_for_s3_object_key = {
+        "path" => @path,
+        "time_slice" => chunk.key,
+        "file_extension" => "gz",
+        "index" => i
+      }
+      s3path = @s3_object_key_format.gsub(%r(%{[^}]+})) { |expr|
+        values_for_s3_object_key[expr[2...expr.size-1]]
+      }
       i += 1
     end while @bucket.objects[s3path].exists?
 
@@ -99,11 +121,30 @@ class S3Output < Fluent::TimeSlicedOutput
       w.close
       @bucket.objects[s3path].write(Pathname.new(tmp.path), :content_type => 'application/x-gzip')
     ensure
+      tmp.close(true) rescue nil
       w.close rescue nil
     end
   end
-end
 
+  private
+
+  def ensure_bucket
+    if !@bucket.exists?
+      if @auto_create_bucket
+        $log.info "Creating bucket #{@s3_bucket} on #{@s3_endpoint}"
+        @s3.buckets.create(@s3_bucket)
+      else
+        raise "The specified bucket does not exist: bucket = #{@s3_bucket}"
+      end
+    end
+  end
 
+  def check_apikeys
+    @bucket.empty?
+  rescue
+    raise "aws_key_id or aws_sec_key is invalid. Please check your configuration"
+  end
 end
 
+
+end

data/test/out_s3.rb

@@ -1,8 +1,12 @@
 require 'fluent/test'
 require 'fluent/plugin/out_s3'
 
+require 'flexmock/test_unit'
+require 'zlib'
+
 class S3OutputTest < Test::Unit::TestCase
   def setup
+    require 'aws-sdk'
     Fluent::Test.setup
   end
 
@@ -20,6 +24,11 @@ class S3OutputTest < Test::Unit::TestCase
       def write(chunk)
         chunk.read
       end
+
+      private
+
+      def check_apikeys
+      end
     end.configure(conf)
   end
 
@@ -115,7 +124,7 @@ class S3OutputTest < Test::Unit::TestCase
     d.run
   end
 
-  def test_write
+  def test_chunk_to_write
     d = create_driver
 
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
@@ -130,5 +139,109 @@ class S3OutputTest < Test::Unit::TestCase
                  data
   end
 
-end
+  CONFIG2 = %[
+    hostname testing.node.local
+    aws_key_id test_key_id
+    aws_sec_key test_sec_key
+    s3_bucket test_bucket
+    s3_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
+    time_slice_format %Y%m%d-%H
+    path log
+    utc
+    buffer_type memory
+    auto_create_bucket false
+  ]
+
+  def create_time_sliced_driver(additional_conf = '')
+    d = Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::S3Output) do
+      private
+
+      def check_apikeys
+      end
+    end.configure([CONFIG2, additional_conf].join("\n"))
+    d
+  end
+
+  def test_write_with_custom_s3_object_key_format
+    # Assert content of event logs which are being sent to S3
+    s3obj = flexmock(AWS::S3::S3Object)
+    s3obj.should_receive(:exists?).with_any_args.
+      and_return { false }
+    s3obj.should_receive(:write).with(
+      on { |pathname|
+        data = nil
+        # Event logs are compressed in GZip
+        pathname.open { |f|
+          gz = Zlib::GzipReader.new(f)
+          data = gz.read
+          gz.close
+        }
+        assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
+                         %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
+                     data
+
+        pathname.to_s.match(%r|s3-|)
+      },
+      {:content_type=>"application/x-gzip"})
+
+    # Assert the key of S3Object, which event logs are stored in
+    s3obj_col = flexmock(AWS::S3::ObjectCollection)
+    s3obj_col.should_receive(:[]).with(
+      on { |key|
+        key == "log/events/ts=20110102-13/events_0-testing.node.local.gz"
+      }).
+      and_return {
+        s3obj
+      }
+
+    # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
+    flexmock(AWS::S3::Bucket).new_instances do |bucket|
+      bucket.should_receive(:objects).with_any_args.
+        and_return {
+          s3obj_col
+        }
+    end
+
+    # We must use TimeSlicedOutputTestDriver instead of BufferedOutputTestDriver,
+    # to make assertions on chunks' keys
+    d = create_time_sliced_driver
 
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    # Finally, the instance of S3Output is initialized and then invoked
+    d.run
+  end
+
+  def setup_mocks
+    s3bucket = flexmock(AWS::S3::Bucket)
+    s3bucket.should_receive(:exists?).with_any_args.and_return { false }
+    s3bucket_col = flexmock(AWS::S3::BucketCollection)
+    s3bucket_col.should_receive(:[]).with_any_args.and_return { s3bucket }
+    flexmock(AWS::S3).new_instances do |bucket|
+      bucket.should_receive(:buckets).with_any_args.and_return { s3bucket_col }
+    end
+
+    return s3bucket, s3bucket_col
+  end
+
+  def test_auto_create_bucket_false_with_non_existence_bucket
+    s3bucket, s3bucket_col = setup_mocks
+
+    d = create_time_sliced_driver('auto_create_bucket false')
+    assert_raise(RuntimeError, "The specified bucket does not exist: bucket = test_bucket") { 
+      d.run
+    }
+  end
+
+  def test_auto_create_bucket_true_with_non_existence_bucket
+    s3bucket, s3bucket_col = setup_mocks
+    s3bucket_col.should_receive(:create).with_any_args.and_return { true }
+
+    d = create_time_sliced_driver('auto_create_bucket true')
+    assert_nothing_raised { 
+      d.run
+    }
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-14 00:00:00.000000000 Z
+date: 2013-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -34,7 +34,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: 1.8.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -42,7 +42,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: 1.8.2
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -59,6 +59,22 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-config-placeholders
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -75,6 +91,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: flexmock
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.0
 description: Amazon S3 output plugin for Fluent event collector
 email: frsyuki@gmail.com
 executables: []
@@ -104,7 +136,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -957364254525035982
+      hash: -741948344279215557
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -113,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -957364254525035982
+      hash: -741948344279215557
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23