s3backup 0.6.5 → 0.6.6

data/History.txt

@@ -10,3 +10,10 @@
 * mod bug if directory name has non english and shell special char
 * retry 5 times if data post fails.
   
+=== 0.6.6 2010-01-19
+* add config max_retry_count
+* if ocurred "Connection reset by peer",retry  max_retry_count times. default 6 waiting (30*try_count) second next post.
+* show http://scie.nti.st/2008/3/14/amazon-s3-and-connection-reset-by-peer
+* don't use raise,change  exit and error log
+* config BUFESIZE to buffer_size
+* mod bug if directory name has '\' 

data/lib/s3backup/backup.rb

@@ -11,14 +11,14 @@ module S3backup
         @s3_obj = S3Wrapper.new(config,true)
       rescue => err
         S3log.error(err.backtrace.join("\n")+"\n"+err.message)
-        exit -1
+        exit(-1)
       end
       @manager = Manager.new(@s3_obj,config)
     end
     def check_config(config)
       unless config["directories"]
         S3log.error("directories doesn't exist in config file.")
-        exit -1
+        exit(-1)
       end
       unless config["directories"].class == Array 
         dir = config["directories"] 
@@ -28,11 +28,11 @@ module S3backup
       config["directories"].each do |dir|
         unless File.directory? dir
           S3log.error("#{dir} isn't exist.")
-          exit -1
+          exit(-1)
         end
         if File.expand_path(dir) != dir
           S3log.error("#{dir.length} must be absolute path.")
-          exit -1
+          exit(-1)
         end
       end
     end
@@ -44,7 +44,7 @@ module S3backup
       end
       rescue => err
         S3log.error(err.backtrace.join("\n")+"\n"+err.message)
-        exit -1
+        exit(-1)
       end
     end
   end

data/lib/s3backup/cli.rb

@@ -35,18 +35,18 @@ module S3backup
           }
           opt.on("-h","--help","print this message and quit") {
             puts opt.help
-            exit 0
+            exit(0)
           }
           opt.parse!(arguments)
         end
       rescue OptionParser::ParseError => err
         S3log.error(err.message)
-        exit 1
+        exit(-1)
       end
       S3log.set_debug(options[:verbose])
       if !File.file?(options[:config_file])
         S3log.error("config #{options[:config_file]} is not exist.")
-        exit 1
+        exit(-1)
       end
       if options[:log]
         S3log.set_logfile(File.open(options[:log],"a"))
@@ -55,7 +55,7 @@ module S3backup
         require 's3backup/restore'
         if !File.directory?(options[:output_dir])
           S3log.error("output directory #{options[:output_dir]} is not exist.")
-          exit 1
+          exit(-1)
         end
         rt = Restore.new(options[:output_dir],YAML.load_file(options[:config_file]))
         rt.start

data/lib/s3backup/manager.rb

@@ -6,9 +6,9 @@ require 's3backup/tree_info'
 require 's3backup/crypt'
 module S3backup
   class Manager
-    DEFAULT_BUF_READ_SIZE=1024*1024*64
+    DEFAULT_BUF_READ_SIZE=1024*1024*32
     def shell_name(str)
-      str.gsub!(/[!"$&'()*,:;<=>?\[\]^`{|}\s]/, '\\\\\&')
+      str.gsub!(/[!"$&'()*,:;<=>?\[\]\\^`{|}\s]/, '\\\\\&')
       a=[]
       str.each_byte{|i| 
         if i < 0x80 
@@ -26,15 +26,22 @@ module S3backup
     def set_config(config)
       if config["password"] and config["password"] != ""
         unless config["salt"] 
-          raise "salt doesn't exist in config file.\n"
+          S3log.error("salt doesn't exist in config file.\n")
+          exit(-1)
         end
         unless config["salt"] =~ /[0-9A-Fa-f]{16}/
-          raise "salt format shoud be HexString and length should be 16.\n"
+          S3log.error("salt format shoud be HexString and length should be 16.\n")
+          exit(-1)
         end
-        if config["BUF_SIEZE"]
-          size=config["BUF_SIEZE"]
-          if size > 1000*1000*1000*5
-            raise "BUF_SIZE must be less than 5G"
+        if config["buffer_size"]
+          if config["buffer_size"].class == String
+            @buf_size = config["buffer_size"].to_i
+          else
+            @buf_size = config["buffer_size"]
+          end
+          if @buf_size > 1000*1000*1000*5
+            S3log.error("buffer_size must be less than 5G\n")
+            exit(-1)
           end
         else
           @buf_size = DEFAULT_BUF_READ_SIZE
@@ -57,7 +64,8 @@ module S3backup
       S3log.debug(cmd)
       system(cmd) 
       unless $?.success?
-        raise "feiled #{cmd} execute. #{$?.inspect}"
+        S3log.error("feiled #{cmd} execute. #{$?.inspect}")
+        exit(-1)
       end
     end
     def from_tgz(path,dir)
@@ -65,7 +73,8 @@ module S3backup
       S3log.debug(cmd)
       system(cmd) 
       unless $?.success?
-        raise "feiled #{cmd} execute. #{$?.inspect}"
+        S3log.error("feiled #{cmd} execute. #{$?.inspect}")
+        exit(-1)
       end
     end
     def get_chain(key)

data/lib/s3backup/restore.rb

@@ -11,7 +11,7 @@ module S3backup
         @s3_obj = S3Wrapper.new(config,false)
       rescue => err
         S3log.error(err.backtrace.join("\n")+"\n"+err.message)
-        exit -1
+        exit(-1)
       end
       @manager = Manager.new(@s3_obj,config)
     end
@@ -33,7 +33,7 @@ module S3backup
         end
       rescue => err
         S3log.error(err.backtrace.join("\n")+"\n"+err.message)
-        exit -1
+        exit(-1)
       end
     end
   end

data/lib/s3backup/s3wrapper.rb

@@ -3,7 +3,7 @@ require 'aws/s3'
 module S3backup
   class S3Wrapper
     attr_reader :bucket
-    MAX_TRY_COUNT = 5
+    DEFAULT_MAX_RETRY_COUNT = 6
     def initialize(config,create_flg)
       @bucket= nil
       @s3objects =nil
@@ -35,7 +35,8 @@ module S3backup
       #Bucket作成
       ret = AWS::S3::Bucket.create(@bucket_name) 
       unless ret
-        raise "AWS::S3::Bucket create error"
+        S3log.error("AWS::S3::Bucket create error possibly already exist #{@bucket_name} by anyone else?)")
+        exit(-1)
       end
     end
     def rename(orig,dest)
@@ -57,18 +58,22 @@ module S3backup
       key_name = CGI.escape(key)
       S3log.info("S3Object.store(#{key_name})")
       count = 0;
-      while count < MAX_TRY_COUNT do
+      while true do
         begin
           AWS::S3::S3Object.store(key_name,val,@bucket_name)
           break;
         rescue => ex
           count+=1;
+          S3log.debug("post #{count} times failed. #{key_name}\n")
+          if count >= @max_retry_count
+            raise ex
+            S3log.error("post #{count} times failed #{key_name} #{ex.class}:#{ex.message}\n")
+            exit(-1)
+          end
+          sleep(count*30)
           AWS::S3::Base.establish_connection!(@args)
         end
       end
-      if count >= MAX_TRY_COUNT
-        raise "post error occurd path #{key_name} \n"
-      end
     end
     def set_config(config)
       err_msg = "" 
@@ -83,6 +88,15 @@ module S3backup
       unless config["bucket"]
         err_msg += "bucket doesn't exist in config file.\n"
       end
+      if config["max_retry_count"]
+        if config["max_retry_count"].class == String
+          @max_retry_count = config["max_retry_count"].to_i
+        else
+          @max_retry_count = config["max_retry_count"]
+        end
+      else
+        @max_retry_count = DEFAULT_MAX_RETRY_COUNT
+      end
       @proxy = {}
       @proxy[:host] = config["proxy_host"] if config["proxy_host"]
       @proxy[:port] = config["proxy_port"] if config["proxy_port"]
@@ -90,7 +104,8 @@ module S3backup
       @proxy[:password] = config["proxy_password"] if config["proxy_password"]
       @bucket_name = config["bucket"]
       if err_msg != ""
-        raise err_msg
+        S3log.error(err_msg)
+        exit(-1)
       end
     end
     def delete(key)

data/lib/s3backup.rb

@@ -2,5 +2,5 @@ $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
 module S3backup
-  VERSION = '0.6.5'
+  VERSION = '0.6.6'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: s3backup
 version: !ruby/object:Gem::Version 
-  version: 0.6.5
+  version: 0.6.6
 platform: ruby
 authors: 
 - Takeshi Morita
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-18 00:00:00 +09:00
+date: 2010-01-19 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency