s3backup 0.6.10 → 0.7.01

data/History.txt

@@ -32,3 +32,8 @@
 
 === 0.6.10 2010-01-25
 * mod bug if directory name has '#' 
+
+=== 0.7.01 2010-01-27
+* mod bug if passward is not set. very sorry.
+* change resume function  default off. because use huge memory.
+* default max_retry_count 20 to 30 because resume function change off.

data/README.rdoc

@@ -28,6 +28,7 @@ To use remotebackup,you should prepare backup configuretion file by yaml such be
  proxy_user: login name for proxy server if you use proxy.
  proxy_password: login password for proxy server if you use proxy.
  log_level: 'output log level. value is debug or info or warn or error(optional default info)'
+ resume: 'if set true,when Ctrl-c or another program stop, next run program resume before backup. default false. caution!! this option use huge memory'
 *If directories isn't specified when restore, it restores all directories in bucket.
 
 == COMMAND:

data/backup.yml

@@ -13,3 +13,4 @@ proxy_port: 'port of proxy server(optional)'
 proxy_user: 'user name of proxy server(optional)'
 proxy_password: 'password of proxy server(optional)'
 log_level: 'output log level. value is debug or info or warn or error(optional default info)'
+resume: 'if set true,when Ctrl-c or another program stop, next run program resume before backup. default false. caution!! this option use huge memory'

data/lib/s3backup/manager.rb

@@ -21,6 +21,7 @@ module S3backup
     end
     def initialize(target,config)
       @target = target
+      @resume = false
       set_config(config)
     end
     def set_config(config)
@@ -33,20 +34,36 @@ module S3backup
           S3log.error("salt format shoud be HexString and length should be 16.\n")
           exit(-1)
         end
-        if config["buffer_size"]
-          if config["buffer_size"].class == String
-            @buf_size = config["buffer_size"].to_i
-          else
-            @buf_size = config["buffer_size"]
-          end
-          if @buf_size > 1000*1000*1000*5
-            S3log.error("buffer_size must be less than 5G\n")
-            exit(-1)
-          end
+        @aes = Crypt.new(config["password"],config["salt"])
+      end
+      if config["buffer_size"]
+        if config["buffer_size"].class == String
+          @buf_size = config["buffer_size"].to_i
         else
-          @buf_size = DEFAULT_BUF_READ_SIZE
+          @buf_size = config["buffer_size"]
         end
-        @aes = Crypt.new(config["password"],config["salt"])
+        if @buf_size > 1000*1000*1000*5
+          S3log.error("buffer_size must be less than 5G\n")
+          exit(-1)
+        end
+      else
+        @buf_size = DEFAULT_BUF_READ_SIZE
+      end
+      if config["buffer_size"]
+        if config["buffer_size"].class == String
+          @buf_size = config["buffer_size"].to_i
+        else
+          @buf_size = config["buffer_size"]
+        end
+        if @buf_size > 1000*1000*1000*5
+          S3log.error("buffer_size must be less than 5G\n")
+          exit(-1)
+        end
+      else
+        @buf_size = DEFAULT_BUF_READ_SIZE
+      end
+      if config["resume"] == true
+        @resume = true
       end
     end
     #指定されたディレクトリをtar gzip形式で圧縮する
@@ -175,17 +192,25 @@ module S3backup
       #前回と今回のファイル・ツリーを比較
       diff_info = tree_info.diff(old_tree)
       S3log.debug("diff_info=#{diff_info.inspect}")
-
+      dir_map = nil
+      if @resume
+        new_dir_map = tree_info.make_dir_map
+        old_dir_map = old_tree.make_dir_map
+      else
+        #メモリ節約のため開放
+        old_tree = nil 
+      end
       update_dir = diff_info[:directory][:add] + diff_info[:directory][:modify]
       #更新されたディレクトリをアップロード
       update_dir.each do |udir|
         GC.start
         store_directory(udir)
-        udir_info = tree_info.get_dir_info(udir)
-        #前回のファイル・ツリー情報のうち、今回アップデートしたディレクトリ情報ファイル情報を更新
-        old_tree.update_dir(udir,udir_info)
-        #更新したファイル・ツリー情報をアップロード(途中で失敗しても、resumeできるようにするため。)
-        @target.post(target_tree_name,old_tree.dump_yaml)
+        if @resume
+          #前回のファイル・ツリー情報のうち、今回アップデートしたディレクトリ情報ファイル情報を更新
+          old_dir_map = old_tree.update_dir(udir,old_dir_map,new_dir_map[udir])
+          #更新したファイル・ツリー情報をアップロード(途中で失敗しても、resumeできるようにするため。)
+          @target.post(target_tree_name,old_tree.dump_yaml)
+        end
       end
       diff_info[:directory][:remove].each do |rm_dir|
         delete_direcory(rm_dir)
@@ -226,6 +251,7 @@ module S3backup
       return dirs.compact
     end
     def expand_tree(dir,tree_info,output_dir)
+      now = Time.new
       tree = tree_info.hierarchie(dir)
       top = tree[0].keys[0]
       top_dir = File.dirname(top)
@@ -241,7 +267,7 @@ module S3backup
           dir_len = k.length
           relative_path = k.slice(top_dir_len,dir_len - top_dir_len)
           dir = output_dir + relative_path
-          File.utime(v[:atime],v[:mtime],dir)
+          File.utime(now,v[:mtime],dir)
         end
       }
     end

data/lib/s3backup/s3wrapper.rb

@@ -3,7 +3,7 @@ require 'aws/s3'
 module S3backup
   class S3Wrapper
     attr_reader :bucket
-    DEFAULT_MAX_RETRY_COUNT = 20
+    DEFAULT_MAX_RETRY_COUNT = 30
     def initialize(config,create_flg)
       @bucket= nil
       @s3objects =nil

data/lib/s3backup/tree_info.rb

@@ -3,25 +3,17 @@ module S3backup
   class TreeInfo
     attr_reader :fileMap
     def initialize(target)
-      @dir_map={}
-      @orig = nil
       if target.nil?
         @fileMap = {:file => Hash.new,:symlink => Hash.new,:directory => Hash.new}
       elsif File.directory?(target)
-        @orig = {:type=>"directory",:target=>target}
         @fileMap = {:file => Hash.new,:symlink => Hash.new,:directory => Hash.new}
         stat = File.stat(target)
-        @fileMap[:directory][target] = {:mtime => stat.mtime, :atime => stat.atime}
-        @dir_map[target] = {:mtime => stat.mtime, :atime => stat.atime,:file=>{},:symlink=>{}}
+        @fileMap[:directory][target] = {:mtime => stat.mtime}
         makeFileMap(target)
       elsif File.file?(target)
-        @orig = {:type=>"file",:target=>target}
         load_yaml(File.read(target))
-        make_dir_map
       else
-        @orig = {:type=>"data",:target=>""}
         load_yaml(target)
-        make_dir_map
       end
     end
     def makeFileMap(dir)
@@ -32,69 +24,62 @@ module S3backup
         name = dir + "/" + e
         if File.directory?(name)
           stat = File.stat(name)
-          @dir_map[name] = {:mtime => stat.mtime, :atime => stat.atime,:file=>{},:symlink=>{}}
-          @fileMap[:directory][name] = {:mtime => stat.mtime, :atime => stat.atime}
+          @fileMap[:directory][name] = {:mtime => stat.mtime}
           makeFileMap(name)
         elsif File.symlink?(name)
-          @dir_map[dir][:symlink][name] = {:source => File.readlink(name)}
           @fileMap[:symlink][name] = {:source => File.readlink(name)}
         else
           stat = File.stat(name)
-          @dir_map[dir][:file][name] ={:size => stat.size,:date => stat.mtime}
           @fileMap[:file][name] = {:size => stat.size,:date => stat.mtime}
         end
       end
     end
     def make_dir_map
+      dir_map = {};
       @fileMap[:directory].each do |k,v|
-        @dir_map[k] = {:mtime => v[:mtime], :atime => v[:atime],:file=>{},:symlink=>{}}
+        dir_map[k] = {:mtime => v[:mtime],:file=>{},:symlink=>{}}
       end
       @fileMap[:file].each do |k,v|
-        target = @dir_map[File.dirname(k)]
+        target = dir_map[File.dirname(k)]
         #不整合だけど適当に作る
         unless target
-          S3log.warn("Tree Data isn't correct.#{@orig.inspect}")
-          target = {:mtime => DateTime.now.to_s,:atime => DateTime.now.to_s,:file=>{},:symlink=>{}}
-          @dir_map[File.dirname(k)] = target
+          S3log.warn("Tree Data isn't correct.")
+          target = {:mtime => DateTime.now.to_s,:file=>{},:symlink=>{}}
+          dir_map[File.dirname(k)] = target
         end
         target[:file][k] = {:size => v[:size], :date => v[:date]}
       end
       @fileMap[:symlink].each do |k,v|
-        target = @dir_map[File.dirname(k)]
+        target = dir_map[File.dirname(k)]
         #不整合だけど適当に作る
         unless target
-          S3log.warn("Tree Data isn't correct.#{@orig.inspect}")
-          target = {:mtime => DateTime.now.to_s,:atime => DateTime.now.to_s,:file=>{},:symlink=>{}}
-          @dir_map[File.dirname(k)] = target
+          S3log.warn("Tree Data isn't correct.")
+          target = {:mtime => DateTime.now.to_s,:file=>{},:symlink=>{}}
+          dir_map[File.dirname(k)] = target
         end
         target[:symlink][k] = {:source => v[:source]}
       end
+      return dir_map;
     end
-    def get_dir_info(name)
-      return @dir_map[name]
-    end
-    def update_dir_map(name,dir_info)
-      @dir_map[name][:file] = dir_info[:file]
-      @dir_map[name][:symlink] = dir_info[:symlink]
-      @dir_map[name][:mtime] = dir_info[:mtime]
-      @dir_map[name][:atime] = dir_info[:atime]
-    end
-    def update_dir(name,dir_info)
-      @dir_map[name] = {:file => {},:symlink=>{}} unless @dir_map[name]
-      @dir_map[name][:file].each do |k,v|
+    def update_dir(name,dir_map,dir_info)
+      dir_map[name] = {:file => {},:symlink=>{}} unless dir_map[name]
+      dir_map[name][:file].each do |k,v|
         @fileMap[:file].delete(k)
       end
-      @dir_map[name][:symlink].each do |k,v|
+      dir_map[name][:symlink].each do |k,v|
         @fileMap[:symlink].delete(k)
       end
-      @fileMap[:directory][name] = {:mtime => dir_info[:mtime],:atime =>dir_info[:atime]}
+      @fileMap[:directory][name] = {:mtime => dir_info[:mtime]}
       dir_info[:file].each do |k,v|
         @fileMap[:file][k] = v
       end
       dir_info[:symlink].each do |k,v|
         @fileMap[:symlink][k] = v
       end
-      update_dir_map(name,dir_info)
+      dir_map[name][:file] = dir_info[:file]
+      dir_map[name][:symlink] = dir_info[:symlink]
+      dir_map[name][:mtime] = dir_info[:mtime]
+      return dir_map
     end
     def load_yaml(data)
       @fileMap = YAML.load(data)

data/lib/s3backup.rb

@@ -2,5 +2,5 @@ $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
 module S3backup
-  VERSION = '0.6.10'
+  VERSION = '0.7.01'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: s3backup
 version: !ruby/object:Gem::Version 
-  version: 0.6.10
+  version: 0.7.01
 platform: ruby
 authors: 
 - Takeshi Morita
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-25 00:00:00 +09:00
+date: 2010-01-27 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency