fluent-plugin-s3 0.5.2 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 94693d17ac1d8a9064b7ae64bac1496a74acf7de
-  data.tar.gz: 300d4d8cb66c403a8237afcda3def9de6761c74a
+  metadata.gz: b4371b4a30965b7724f988a1c101f84af199dc09
+  data.tar.gz: b01e6df47133d2fc117d020e670b6ecfafef1b74
 SHA512:
-  metadata.gz: de718a336390d3121a194afc54f7e5597fb6c0f8d250bb5096af21e3d8165b3dc5f7a6dd467cc25fcdab6107939cfb131f464f654d0297466ff4d2ee3a6dca0e
-  data.tar.gz: 0379d7d50711f3f2ad4beec0774361f05f47da8b0aa0b7c364e69d0cd8475491d0764bd0f872cb7331c5edd2503926ff7d9f747959439c5544f36a42245e68c7
+  metadata.gz: 8abdf2a7ce8ea84e235355a9528eb09a39e01f0ddfbca63ad8d474b22f4003d7530ef9c94ab00342e3169a6afded6a4328dfa51b06a36370810eac72673481c8
+  data.tar.gz: 7f5ae08030d544e093ce716c35970bf2cb7cdbfebcf97947c9a214b95d1b458ace7fc449e5e52f44e7e86938d0f0baf4164aeb731d6b4f7b4153aeffbf497f62

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 0.5.3 - 2015/02/06
+
+  * Add error information in API check
+  * Add GzipWriter fallback to gzip_command
+
+
 Release 0.5.2 - 2015/02/05
 
   * Add experimental gzip_command compressor

data/VERSION

@@ -1 +1 @@
-0.5.2
+0.5.3

data/lib/fluent/plugin/out_s3.rb

@@ -44,13 +44,12 @@ module Fluent
       end
 
       begin
-        @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new
+        @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
       rescue => e
         $log.warn "#{@store_as} not found. Use 'text' instead"
         @compressor = TextCompressor.new
       end
       @compressor.configure(conf)
-      @compressor.buffer_type = @buffer_type
 
       # TODO: use Plugin.new_formatter instead of TextFormatter.create
       conf['format'] = @format
@@ -139,19 +138,25 @@ module Fluent
       @bucket.empty?
     rescue AWS::S3::Errors::NoSuchBucket
       # ignore NoSuchBucket Error because ensure_bucket checks it.
-    rescue
-      raise "can't call S3 API. Please check your aws_key_id / aws_sec_key or s3_region configuration"
+    rescue => e
+      raise "can't call S3 API. Please check your aws_key_id / aws_sec_key or s3_region configuration. error = #{e.inspect}"
     end
 
     class Compressor
       include Configurable
 
+      def initialize(opts = {})
+        super()
+        @buffer_type = opts[:buffer_type]
+        @log = opts[:log]
+      end
+
+      attr_reader :buffer_type, :log
+
       def configure(conf)
         super
       end
 
-      attr_accessor :buffer_type
-
       def ext
       end
 
@@ -190,7 +195,6 @@ module Fluent
         w.close
       ensure
         w.close rescue nil
-        w.unlink rescue nil
       end
     end
 

data/lib/fluent/plugin/s3_compressor_gzip_command.rb

@@ -26,15 +26,24 @@ module Fluent
                  w = Tempfile.new("chunk-gzip-tmp")
                  chunk.write_to(w)
                  w.close
-                 tmp.close
                  w.path
                end
-        # We don't check the return code because we can't recover gzip failure.
-        system "gzip #{@command_parameter} -c #{path} > #{tmp.path}"
+
+        res = system "gzip #{@command_parameter} -c #{path} > #{tmp.path}"
+        unless res
+          log.warn "failed to execute gzip command. Fallback to GzipWriter. status = #{$?}"
+          begin
+            tmp.truncate(0)
+            gw = Zlib::GzipWriter.new(tmp)
+            chunk.write_to(gw)
+            gw.close
+          ensure
+            gw.close rescue nil
+          end
+        end
       ensure
         unless chunk_is_file
-          w.close rescue nil
-          w.unlink rescue nil
+          w.close(true) rescue nil
         end
       end
     end

data/lib/fluent/plugin/s3_compressor_lzma2.rb

@@ -22,7 +22,7 @@ module Fluent
         w = Tempfile.new("chunk-xz-tmp")
         chunk.write_to(w)
         w.close
-        tmp.close
+
         # We don't check the return code because we can't recover lzop failure.
         system "xz #{@command_parameter} -c #{w.path} > #{tmp.path}"
       ensure

data/lib/fluent/plugin/s3_compressor_lzo.rb

@@ -22,7 +22,7 @@ module Fluent
         w = Tempfile.new("chunk-tmp")
         chunk.write_to(w)
         w.close
-        tmp.close
+
         # We don't check the return code because we can't recover lzop failure.
         system "lzop #{@command_parameter} -o #{tmp.path} #{w.path}"
       ensure

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi