fluent-plugin-s3 0.3.1 → 0.3.3

data/ChangeLog

@@ -1,3 +1,18 @@
+Release 0.3.3 - 2013/06/18
+
+  * Fix require bug on case-sensitive environment
+
+
+Release 0.3.2 - 2013/06/18
+
+  * Support lzo mime-type
+    https://github.com/fluent/fluent-plugin-s3/pull/29
+  * Add proxy_uri option
+    https://github.com/fluent/fluent-plugin-s3/issues/25
+  * Add check_apikey_on_start option
+    https://github.com/fluent/fluent-plugin-s3/pull/28
+
+
 Release 0.3.1 - 2013/03/28
 
   * Support json and text mime-types

data/README.rdoc

@@ -37,7 +37,7 @@ Simply use RubyGems:
 
 [s3_bucket (required)] S3 bucket name.
 
-[s3_endpoint] s3 endpoint name. Example, Tokyo region is "s3-ap-northeast-1.amazonaws.com".
+[s3_endpoint] s3 endpoint name. For example, US West (Oregon) Region is "s3-us-west-2.amazonaws.com". The full list of endpoints are available here. > http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
 
 [s3_object_key_format] The format of S3 object keys. You can use several built-in variables:
 
@@ -79,8 +79,19 @@ The {fluent-mixin-config-placeholders}[https://github.com/tagomoris/fluent-mixin
 
     s3_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
 
+[store_as] archive format on S3. You can use serveral format:
+
+- gzip (default)
+- json
+- text
+- lzo (Need lzop command)
+
 [auto_create_bucket] Create S3 bucket if it does not exists. Default is true.
 
+[check_apikey_on_start] Check AWS key on start. Default is true.
+
+[proxy_uri] uri of proxy environment.
+
 [path] path prefix of the files on S3. Default is "" (no prefix).
 
 [buffer_path (required)] path prefix of the files to buffer logs.

data/VERSION

@@ -1 +1 @@
-0.3.1
+0.3.3

data/lib/fluent/plugin/out_s3.rb

@@ -11,6 +11,7 @@ class S3Output < Fluent::TimeSlicedOutput
     require 'zlib'
     require 'time'
     require 'tempfile'
+    require 'open3'
 
     @use_ssl = true
   end
@@ -31,6 +32,8 @@ class S3Output < Fluent::TimeSlicedOutput
   config_param :s3_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
   config_param :store_as, :string, :default => "gzip"
   config_param :auto_create_bucket, :bool, :default => true
+  config_param :check_apikey_on_start, :bool, :default => true
+  config_param :proxy_uri, :string, :default => nil
 
   attr_reader :bucket
 
@@ -62,6 +65,13 @@ class S3Output < Fluent::TimeSlicedOutput
 
     @ext, @mime_type = case @store_as
       when 'gzip' then ['gz', 'application/x-gzip']
+      when 'lzo' then
+        begin
+          Open3.capture3('lzop -V')
+        rescue Errno::ENOENT
+          raise ConfigError, "'lzop' utility must be in PATH for LZO compression"
+        end
+        ['lzo', 'application/x-lzop']
       when 'json' then ['json', 'application/json']
       else ['txt', 'text/plain']
     end
@@ -77,13 +87,14 @@ class S3Output < Fluent::TimeSlicedOutput
       options[:secret_access_key] = @aws_sec_key
     end
     options[:s3_endpoint] = @s3_endpoint if @s3_endpoint
+    options[:proxy_uri] = @proxy_uri if @proxy_uri
     options[:use_ssl] = @use_ssl
 
     @s3 = AWS::S3.new(options)
     @bucket = @s3.buckets[@s3_bucket]
 
     ensure_bucket
-    check_apikeys
+    check_apikeys if @check_apikey_on_start
   end
 
   def format(tag, time, record)
@@ -128,6 +139,13 @@ class S3Output < Fluent::TimeSlicedOutput
         w = Zlib::GzipWriter.new(tmp)
         chunk.write_to(w)
         w.close
+      elsif @store_as == "lzo"
+        w = Tempfile.new("chunk-tmp")
+        chunk.write_to(w)
+        w.close
+        tmp.close
+        # We don't check the return code because we can't recover lzop failure.
+        system "lzop -qf1 -o #{tmp.path} #{w.path}"
       else
         chunk.write_to(tmp)
         tmp.close
@@ -136,6 +154,7 @@ class S3Output < Fluent::TimeSlicedOutput
     ensure
       tmp.close(true) rescue nil
       w.close rescue nil
+      w.unlink rescue nil
     end
   end
 

data/test/out_s3.rb

@@ -59,6 +59,17 @@ class S3OutputTest < Test::Unit::TestCase
     assert_equal 'text/plain', d.instance.instance_variable_get(:@mime_type)
   end
 
+  def test_configure_with_mime_type_lzo
+    conf = CONFIG.clone
+    conf << "\nstore_as lzo\n"
+    d = create_driver(conf)
+    assert_equal 'lzo', d.instance.instance_variable_get(:@ext)
+    assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@mime_type)
+  rescue => e
+    # TODO: replace code with disable lzop command
+    assert(e.is_a?(Fluent::ConfigError))
+  end
+
   def test_format
     d = create_driver
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-28 00:00:00.000000000 Z
+date: 2013-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -136,7 +136,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4273077554576087642
+      hash: -3746525337841491520
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -145,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4273077554576087642
+      hash: -3746525337841491520
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23