fluent-plugin-s3 0.3.0 → 0.3.1

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 0.3.1 - 2013/03/28
+
+  * Support json and text mime-types
+    https://github.com/fluent/fluent-plugin-s3/pull/20
+
+
 Release 0.3.0 - 2013/02/19
 
   * Enable dynamic and configurable S3 object kyes

data/README.rdoc

@@ -31,9 +31,9 @@ Simply use RubyGems:
       utc
     </match>
 
-[aws_key_id (required)] AWS access key id.
+[aws_key_id] AWS access key id. This parameter is required when your agent is not running on EC2 instance with an IAM Instance Profile.
 
-[aws_sec_key (required)] AWS secret key.
+[aws_sec_key] AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Instance Profile.
 
 [s3_bucket (required)] S3 bucket name.
 

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.3.1

data/lib/fluent/plugin/out_s3.rb

@@ -29,6 +29,7 @@ class S3Output < Fluent::TimeSlicedOutput
   config_param :s3_bucket, :string
   config_param :s3_endpoint, :string, :default => nil
   config_param :s3_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
+  config_param :store_as, :string, :default => "gzip"
   config_param :auto_create_bucket, :bool, :default => true
 
   attr_reader :bucket
@@ -59,6 +60,12 @@ class S3Output < Fluent::TimeSlicedOutput
       end
     end
 
+    @ext, @mime_type = case @store_as
+      when 'gzip' then ['gz', 'application/x-gzip']
+      when 'json' then ['json', 'application/json']
+      else ['txt', 'text/plain']
+    end
+
     @timef = TimeFormatter.new(@time_format, @localtime)
   end
 
@@ -101,11 +108,12 @@ class S3Output < Fluent::TimeSlicedOutput
 
   def write(chunk)
     i = 0
+
     begin
       values_for_s3_object_key = {
         "path" => @path,
         "time_slice" => chunk.key,
-        "file_extension" => "gz",
+        "file_extension" => @ext,
         "index" => i
       }
       s3path = @s3_object_key_format.gsub(%r(%{[^}]+})) { |expr|
@@ -115,11 +123,16 @@ class S3Output < Fluent::TimeSlicedOutput
     end while @bucket.objects[s3path].exists?
 
     tmp = Tempfile.new("s3-")
-    w = Zlib::GzipWriter.new(tmp)
     begin
-      chunk.write_to(w)
-      w.close
-      @bucket.objects[s3path].write(Pathname.new(tmp.path), :content_type => 'application/x-gzip')
+      if @store_as == "gzip"
+        w = Zlib::GzipWriter.new(tmp)
+        chunk.write_to(w)
+        w.close
+      else
+        chunk.write_to(tmp)
+        tmp.close
+      end
+      @bucket.objects[s3path].write(Pathname.new(tmp.path), :content_type => @mime_type)
     ensure
       tmp.close(true) rescue nil
       w.close rescue nil

data/test/out_s3.rb

@@ -39,6 +39,24 @@ class S3OutputTest < Test::Unit::TestCase
     assert_equal 'test_bucket', d.instance.s3_bucket
     assert_equal 'log', d.instance.path
     assert d.instance.instance_variable_get(:@use_ssl)
+    assert_equal 'gz', d.instance.instance_variable_get(:@ext)
+    assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@mime_type)
+  end
+
+  def test_configure_with_mime_type_json
+    conf = CONFIG.clone
+    conf << "\nstore_as json\n"
+    d = create_driver(conf)
+    assert_equal 'json', d.instance.instance_variable_get(:@ext)
+    assert_equal 'application/json', d.instance.instance_variable_get(:@mime_type)
+  end
+
+  def test_configure_with_mime_type_text
+    conf = CONFIG.clone
+    conf << "\nstore_as text\n"
+    d = create_driver(conf)
+    assert_equal 'txt', d.instance.instance_variable_get(:@ext)
+    assert_equal 'text/plain', d.instance.instance_variable_get(:@mime_type)
   end
 
   def test_format

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-19 00:00:00.000000000 Z
+date: 2013-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -136,7 +136,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -741948344279215557
+      hash: -4273077554576087642
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -145,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -741948344279215557
+      hash: -4273077554576087642
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23