s3backup 0.7.3 → 0.8.1

data/History.txt

@@ -41,3 +41,9 @@
 === 0.7.2 2010-01-29
 * mod bug if directory name is started '-' 
 * dereese use memory
+
+=== 0.8.1 2010-02-11
+* mod bug if directory name is same , that can't restore(because it was not backuped). 
+* change memory to sqlite3.
+* add config  temporary. temporary is directory for temporary file 
+* resume is mandatory.

data/README.rdoc

@@ -28,7 +28,8 @@ To use remotebackup,you should prepare backup configuretion file by yaml such be
  proxy_user: login name for proxy server if you use proxy.
  proxy_password: login password for proxy server if you use proxy.
  log_level: 'output log level. value is debug or info or warn or error(optional default info)'
- resume: 'if set true,when Ctrl-c or another program stop, next run program resume before backup. default false. caution!! this option use huge memory'
+ temporary: 'temporary directory path. default(/tmp)
+
 *If directories isn't specified when restore, it restores all directories in bucket.
 
 == COMMAND:

data/Rakefile

@@ -14,7 +14,7 @@ $hoe = Hoe.spec 's3backup' do
   self.developer 'Takeshi Morita', 'morita@ibrains.co.jp'
   self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
   self.rubyforge_name       = self.name # TODO this is default value
-  self.extra_deps         = [['aws-s3','>= 0.6.2']]
+  self.extra_deps         = [['aws-s3','>= 0.6.2'],['sqlite3',">= 0.0.7"]]
 
 end
 

data/backup.yml

@@ -13,4 +13,4 @@ proxy_port: 'port of proxy server(optional)'
 proxy_user: 'user name of proxy server(optional)'
 proxy_password: 'password of proxy server(optional)'
 log_level: 'output log level. value is debug or info or warn or error(optional default info)'
-resume: 'if set true,when Ctrl-c or another program stop, next run program resume before backup. default false. caution!! this option use huge memory'
+temporary: 'temporary directory path. default(/tmp)

data/lib/s3backup/manager.rb

@@ -1,5 +1,5 @@
 require 'cgi'
-require 'tempfile'
+require 'time'
 require 'fileutils'
 require 's3backup/s3log'
 require 's3backup/tree_info'
@@ -22,6 +22,7 @@ module S3backup
     def initialize(target,config)
       @target = target
       @resume = false
+      @temporary = "/tmp"
       set_config(config)
     end
     def set_config(config)
@@ -49,23 +50,36 @@ module S3backup
       else
         @buf_size = DEFAULT_BUF_READ_SIZE
       end
-      if config["buffer_size"]
-        if config["buffer_size"].class == String
-          @buf_size = config["buffer_size"].to_i
-        else
-          @buf_size = config["buffer_size"]
-        end
-        if @buf_size > 1000*1000*1000*5
-          S3log.error("buffer_size must be less than 5G\n")
-          exit(-1)
-        end
-      else
-        @buf_size = DEFAULT_BUF_READ_SIZE
+      if config["temporary"]
+        @temporary = config["temporary"]
       end
       if config["resume"] == true
         @resume = true
       end
     end
+    def to_gz(file,remain=false)
+      if remain
+        cmd = "(cd #{shell_name(File.dirname(file))};gzip -c  #{shell_name(file)} >  #{shell_name(file)}.gz " + 
+                                              "2>/dev/null)"
+      else
+        cmd = "(cd #{shell_name(File.dirname(file))};gzip #{shell_name(file)} > /dev/null 2>&1)"
+      end
+      S3log.debug(cmd)
+      system(cmd) 
+      unless $?.success?
+        S3log.error("feiled #{cmd} execute. #{$?.inspect}")
+        exit(-1)
+      end
+    end
+    def from_gz(file)
+      cmd = "(cd #{shell_name(File.dirname(file))};gunzip #{shell_name(file)} > /dev/null 2>&1)"
+      S3log.debug(cmd)
+      system(cmd) 
+      unless $?.success?
+        S3log.error("feiled #{cmd} execute. #{$?.inspect}")
+        exit(-1)
+      end
+    end
     #指定されたディレクトリをtar gzip形式で圧縮する
     def to_tgz(path,dir)
       #サブディレクトリを圧縮の対象外にする。
@@ -75,9 +89,12 @@ module S3backup
         sub_dir.push(file) if File.directory?(dir+"/"+file)
       end
       exclude = ""
-      exclude = exclude + " --exclude=" + sub_dir.map{|d| shell_name(d)}.join("  --exclude=") if sub_dir.length != 0
-      cmd = "(cd #{shell_name(File.dirname(dir))};tar -czvf #{shell_name(path)} #{exclude} -- #{shell_name(File.basename(dir))}" + 
-                                 " > /dev/null 2>&1)"
+      if sub_dir.length != 0
+        exclude = " --exclude=#{shell_name(File.basename(dir))}/" + sub_dir.map{|d| shell_name(d)}.join(
+                          "  --exclude=#{shell_name(File.basename(dir))}/") 
+      end
+      cmd = "(cd #{shell_name(File.dirname(dir))};tar -czvf #{shell_name(path)} #{exclude} -- " + 
+        "#{shell_name(File.basename(dir))} > /dev/null 2>&1)"
       S3log.debug(cmd)
       system(cmd) 
       unless $?.success?
@@ -94,38 +111,33 @@ module S3backup
         exit(-1)
       end
     end
-    def get_chain(key)
+    def get_chain(key,path)
       data = nil
-      data_set = nil
       i=1
       if @aes
         key = @aes.encrypt(key)
       end
-      while 1
-        key_name = i.to_s()+"_"+key
-        data = @target.get(key_name)
-        if data == nil
-          break
-        end
-        if i==1
-          data_set = ''
-        end
-        if @aes
-          data = @aes.decrypt(data)
+      File.open(path,"w") do |f|
+        while 1
+          key_name = i.to_s()+"_"+key
+          data = @target.get(key_name)
+          if data == nil
+            break
+          end
+          if @aes
+            data = @aes.decrypt(data)
+          end
+          f.write(data)
+          i+=1
         end
-        data_set += data
-        i+=1
       end
-      return data_set
     end
     def get_directory(dir,out_dir)
-      data = get_chain(dir)
-      tmp = Tempfile.open("s3backup")
-      tmp.write(data)
-      tmp.close
+      file_name = @temporary + "/"+ CGI.escape(dir)
+      get_chain(dir,file_name + ".tgz")
       #tgzのファイルをcur_dirに展開
-      from_tgz(tmp.path,out_dir)
-      tmp.close(true)
+      from_tgz(file_name + ".tgz",out_dir)
+      #File.unlink(file_name)
     end
     def get_directories(dirs,prefix,output_dir)
       prefix_len = prefix.length
@@ -138,10 +150,9 @@ module S3backup
       end
     end
     def store_directory(dir)
-      tmp = Tempfile.open("s3backup")
-      tmp.close
+      tmp_file = @temporary + "/bk_" + CGI.escape(dir)
       #tgzのファイルをtmp.pathに作成
-      to_tgz(tmp.path,dir)
+      to_tgz(tmp_file,dir)
       #S3にディレクトリの絶対パスをキーにして、圧縮したデータをストア
       i=1
       key = nil
@@ -156,20 +167,21 @@ module S3backup
         @target.delete(cnt.to_s() + "_" + key)
         cnt+=1
       end
-      f = File.open(tmp.path,"r")
-      begin
-        while 1
-          key_name = i.to_s()+"_"+key
-          data = f.readpartial(@buf_size)
-          if @aes
-            data = @aes.encrypt(data)
+      File.open(tmp_file,"r") do |f|
+        begin
+          while 1
+            key_name = i.to_s()+"_"+key
+            data = f.readpartial(@buf_size)
+            if @aes
+              data = @aes.encrypt(data)
+            end
+            @target.post(key_name,data)
+            i+=1
           end
-          @target.post(key_name,data)
-          i+=1
+        rescue EOFError
         end
-      rescue EOFError
       end
-      tmp.close(true)
+      File.unlink(tmp_file)
     end
     def delete_direcory(dir)
       if @aes
@@ -182,41 +194,56 @@ module S3backup
     end
     def differential_copy(dir)
       #現在のファイル・ツリーを比較
-      tree_info = TreeInfo.new(dir)
-
-      target_tree_name = "tree_"+dir+".yml"
-      tree_data = nil
+      tree_info = TreeInfo.new({:format=>:directory,:directory=>dir,:db=>@temporary + "/new_" + 
+                               Time.now.to_i.to_s + "_" + Process.pid.to_s + ".db"})
+      target_db_name = dir+".gz"
       #前回のファイル・ツリーを取得
-      old_tree = TreeInfo.new(@target.get(target_tree_name))
-
-      #前回と今回のファイル・ツリーを比較
-      diff_info = tree_info.diff(old_tree)
-      S3log.debug("diff_info=#{diff_info.inspect}")
-      dir_map = nil
-      if @resume
-        new_dir_map = tree_info.make_dir_map
-        old_dir_map = old_tree.make_dir_map
+      data = @target.get(target_db_name)
+      old_tree = nil
+      if data
+        db_name = @temporary + "/old_" + Time.now.to_i.to_s + "_" + Process.pid.to_s + ".db"
+        File.open(db_name + ".gz","w") do |f|
+          f.write(data)
+        end
+        from_gz(db_name + ".gz")
+        old_tree = TreeInfo.new({:format=>:database,:db=>db_name})
       else
-        #メモリ節約のため開放
-        old_tree = nil 
+        target_tree_name = "tree_"+dir+".yml"
+        #以前のフォーマットだった場合は変換
+        data = @target.get(target_tree_name)
+        if data
+          old_tree = TreeInfo.new({:format=>:yaml,:data=>data,:db=>@temporary + "/old_" +
+                                  Time.now.to_i.to_s + "_" + Process.pid.to_s + ".db"})
+        else
+          old_tree = TreeInfo.new({:db=>@temporary + "/old_" +
+                                  Time.now.to_i.to_s + "_" + Process.pid.to_s + ".db"})
+        end
       end
-      update_dir = diff_info[:directory][:add] + diff_info[:directory][:modify]
-      #更新されたディレクトリをアップロード
-      update_dir.each do |udir|
-        GC.start
-        store_directory(udir)
-        if @resume
-          #前回のファイル・ツリー情報のうち、今回アップデートしたディレクトリ情報ファイル情報を更新
-          old_dir_map = old_tree.update_dir(udir,old_dir_map,new_dir_map[udir])
+      data = nil;
+      GC.start
+      cnt=0
+      #前回と今回のファイル・ツリーを比較
+      tree_info.modify(old_tree) do |dir_info|
+        cnt+=1
+        S3log.debug("diff_info=#{dir_info[:name]}")
+        #更新されたディレクトリをアップロード
+        store_directory(dir_info[:name])
+        #前回のファイル・ツリー情報のうち、今回アップデートしたディレクトリ情報ファイル情報を更新
+        old_dir_map = old_tree.update_dir(dir_info)
+        if cnt != 0 and cnt % 10 == 0
           #更新したファイル・ツリー情報をアップロード(途中で失敗しても、resumeできるようにするため。)
-          @target.post(target_tree_name,old_tree.dump_yaml)
+          to_gz(old_tree.db_name,true)
+          @target.post(target_db_name,File.read(old_tree.db_name + ".gz"))
         end
       end
-      diff_info[:directory][:remove].each do |rm_dir|
-        delete_direcory(rm_dir)
+      tree_info.remove(old_tree) do |dir_info|
+        delete_direcory(dir_info[:name])
       end
       #今回のファイル・ツリーをAWS S3に登録
-      @target.post(target_tree_name,tree_info.dump_yaml)
+      to_gz(tree_info.db_name)
+      @target.post(target_db_name,File.read(tree_info.db_name + ".gz"))
+      tree_info.close(true)
+      old_tree.close(true)
     end
     def get_target_tree(dir)
       base_dir = dir
@@ -228,8 +255,8 @@ module S3backup
         if base == before_base
           break
         end
-        tree_file_name = "tree_"+base+".yml"
-        tree_data = @target.get(tree_file_name)
+        tree_db_name = base+".gz"
+        tree_data = @target.get(tree_db_name)
         if tree_data
           break
         end
@@ -239,16 +266,12 @@ module S3backup
       unless tree_data
         return nil
       end
-      return TreeInfo.new(tree_data)
-    end
-    def get_target_bases
-      files = @target.find(/^tree_.*\.yml/)
-      dirs = files.map do |d| 
-        m=/tree_(.*)\.yml/.match(d)
-        next nil unless m
-        m[1]
+      db_name = @temporary + "/" + Time.now.to_i.to_s + "_" + Process.pid.to_s + ".db"
+      File.open(db_name + ".gz","w") do |f|
+        f.write(tree_data)
       end
-      return dirs.compact
+      from_gz(db_name + ".gz")
+      return TreeInfo.new({:format=>:database,:db=>db_name})
     end
     def expand_tree(dir,tree_info,output_dir)
       now = Time.new
@@ -267,7 +290,7 @@ module S3backup
           dir_len = k.length
           relative_path = k.slice(top_dir_len,dir_len - top_dir_len)
           dir = output_dir + relative_path
-          File.utime(now,v[:mtime],dir)
+          File.utime(now,Time.parse(v[:mtime]),dir)
         end
       }
     end
@@ -278,7 +301,7 @@ module S3backup
         return
       end
       expand_tree(dir,tree,output_dir)
-      S3log.debug("expand_tree=#{tree.inspect}")
+      tree.close(true)
     end
   end
 end

data/lib/s3backup/restore.rb

@@ -34,8 +34,8 @@ module S3backup
       end
     end
     def start
+      S3log.error("directories is not defined") unless @directories
       begin
-        @directories = @manager.get_target_bases unless @directories
         @directories.each do |dir|
           @manager.restore(dir,@output_dir)
         end

data/lib/s3backup/s3wrapper.rb

@@ -68,6 +68,7 @@ module S3backup
       while true do
         begin
           if AWS::S3::S3Object.exists? key_name,@bucket_name
+            S3log.info("AWS::S3::S3Object.value(#{key_name})\n")
             data =  AWS::S3::S3Object.value(key_name,@bucket_name)
           end
           break;

data/lib/s3backup/tree_info.rb

@@ -1,22 +1,112 @@
 require 'yaml'
+require 'sqlite3'
 module S3backup
   class TreeInfo
-    attr_reader :fileMap
-    def initialize(target)
-      if target.nil?
-        @fileMap = {:file => Hash.new,:symlink => Hash.new,:directory => Hash.new}
-      elsif File.directory?(target)
-        @fileMap = {:file => Hash.new,:symlink => Hash.new,:directory => Hash.new}
-        stat = File.stat(target)
-        @fileMap[:directory][target] = {:mtime => stat.mtime}
-        makeFileMap(target)
-      elsif File.file?(target)
-        load_yaml(File.read(target))
-      else
-        load_yaml(target)
+    attr_reader :db_name,:db
+    def make_table
+      sql = "create table directory ( id INTEGER PRIMARY KEY ,name varchar(2048), mtime integer, parent_directory_id integer)"
+      @db.execute(sql)
+      sql = "create table file ( name varchar(2048), size  integer, mtime integer,directory_id integer)"
+      @db.execute(sql)
+      sql = "create table symlink ( name varchar(2048), source varchar(2048),directory_id integer)"
+      @db.execute(sql)
+      sql = "CREATE INDEX idx_directory_name ON directory(name)"
+      @db.execute(sql)
+      sql = "CREATE INDEX idx_directory_parent_directory_id ON directory(parent_directory_id)"
+      @db.execute(sql)
+      sql = "CREATE INDEX idx_file_directory_id ON file(directory_id)"
+      @db.execute(sql)
+      sql = "CREATE INDEX idx_file_name  ON file(name)"
+      @db.execute(sql)
+      sql = "CREATE INDEX idx_symlink_name  ON symlink(name)"
+      @db.execute(sql)
+      sql = "CREATE INDEX idx_symlink_directory_id  ON symlink(directory_id)"
+      @db.execute(sql)
+    end
+    def check_dirs(p_id,p_name)
+      @db.execute('select id,name from directory where parent_directory_id = ?',p_id) do |row|
+        id = row[0]
+        name = row[1]
+        if File.basename(name) == File.basename(p_name)
+          sql = "insert into file(name,size,mtime,directory_id) values (:name, :size, :mtime,:directory_id)"
+          @db.execute(sql,:name=>"zetteiarienainamae#{id}",:size=>0,:mtime =>0,:directory_id=>p_id)
+        end
+        check_dirs(id,name)
+      end
+    end
+    def convert_yaml_to_sqlite3(file_map)
+      file_map[:directory].keys().sort{|a,b| a<=>b}.each do |key|
+        file_at = file_map[:directory][key]
+        sql = "insert into directory(name,mtime) values (:name, :mtime)"
+        @db.execute(sql,:name=>key,:mtime => file_at[:mtime] )
+      end
+      @db.execute('select id,name from directory' ) do |row|
+        dir_id = row[0].to_i
+        parent = File.dirname(row[1])
+        @db.execute('select id from directory where name =?',parent ) do |row|
+          @db.execute("update directory set parent_directory_id = #{row[0]} where id = #{dir_id}")
+        end
+      end
+      #for bug (same name directory was not backuped before)
+      @db.execute('select id,name from directory order by id limit 1') do |row|
+        p_id = row[0]
+        name = row[1]
+        check_dirs(p_id,name)
+      end
+      file_map[:file].each do |key,val|
+        file_at = file_map[:file][key]
+        dir_name = File.dirname(key)
+        dir_id = nil
+        @db.execute('select id from directory where name=?',dir_name ) do |row|
+          #rowは結果の配列
+          dir_id = row[0].to_i
+        end
+        unless dir_id
+          STDERR.print "directory name isn't exist ignore #{dir_name}"
+          next
+        end
+        sql = "insert into file(name,size,mtime,directory_id) values (:name, :size, :mtime,:directory_id)"
+        @db.execute(sql,:name=>key,:size=>file_at[:size],:mtime => file_at[:date], :directory_id=>dir_id)
+      end
+      file_map[:symlink].each do |key,val|
+        file_at = file_map[:symlink][key]
+        dir_name = File.dirname(key)
+        sql="select id from directory where name = :name"
+        dir_id = nil
+        @db.execute('select id from directory where name=?',dir_name ) do |row|
+          #rowは結果の配列
+          dir_id = row[0].to_i
+        end
+        unless dir_id
+          STDERR.print "directory name isn't exist ignore #{dir_name}"
+          next
+        end
+        sql = "insert into symlink(name,source,directory_id) values (:name, :source,:directory_id)"
+        @db.execute(sql,:name=>key,:source=>file_at[:source],:directory_id=>dir_id)
+      end
+    end
+    def initialize(opt)
+      @db_name = opt[:db]
+      @db = SQLite3::Database.new(opt[:db])
+      if opt[:format].nil?
+        make_table
+      elsif opt[:format] == :directory
+        make_table
+        stat = File.stat(opt[:directory])
+        sql = "insert into directory(name,mtime) values (:name, :mtime)"
+        @db.execute(sql,:name=>opt[:directory],:mtime =>stat.mtime)
+        dir_id = nil
+        @db.execute('select id from directory where name=?',opt[:directory]) do |row|
+          #rowは結果の配列
+          dir_id = row[0].to_i
+        end
+        makeFileMap(opt[:directory],dir_id)
+      elsif opt[:format] == :yaml
+        make_table
+        convert_yaml_to_sqlite3(YAML.load(opt[:data]))
       end
     end
-    def makeFileMap(dir)
+    def makeFileMap(dir,id)
       Dir.entries(dir).each do |e|
         if e == "." or e == ".."
           next
@@ -24,140 +114,134 @@ module S3backup
         name = dir + "/" + e
         if File.directory?(name)
           stat = File.stat(name)
-          @fileMap[:directory][name] = {:mtime => stat.mtime}
-          makeFileMap(name)
+          sql = "insert into directory(name,mtime,parent_directory_id) values (:name, :mtime,:parent_directory_id)"
+          @db.execute(sql,:name=>name,:mtime =>stat.mtime,:parent_directory_id=>id)
+          dir_id = nil
+          @db.execute('select id from directory where name=?',name) do |row|
+            #rowは結果の配列
+            dir_id = row[0].to_i
+          end
+          makeFileMap(name,dir_id)
         elsif File.symlink?(name)
-          @fileMap[:symlink][name] = {:source => File.readlink(name)}
+          sql = "insert into symlink(name,source,directory_id) values (:name, :source,:directory_id)"
+          @db.execute(sql,:name=>name,:source=>File.readlink(name),:directory_id=>id)
         else
           stat = File.stat(name)
-          @fileMap[:file][name] = {:size => stat.size,:date => stat.mtime}
-        end
-      end
-    end
-    def make_dir_map
-      dir_map = {};
-      @fileMap[:directory].each do |k,v|
-        dir_map[k] = {:mtime => v[:mtime],:file=>{},:symlink=>{}}
-      end
-      @fileMap[:file].each do |k,v|
-        target = dir_map[File.dirname(k)]
-        #不整合だけど適当に作る
-        unless target
-          S3log.warn("Tree Data isn't correct.")
-          target = {:mtime => DateTime.now.to_s,:file=>{},:symlink=>{}}
-          dir_map[File.dirname(k)] = target
-        end
-        target[:file][k] = {:size => v[:size], :date => v[:date]}
-      end
-      @fileMap[:symlink].each do |k,v|
-        target = dir_map[File.dirname(k)]
-        #不整合だけど適当に作る
-        unless target
-          S3log.warn("Tree Data isn't correct.")
-          target = {:mtime => DateTime.now.to_s,:file=>{},:symlink=>{}}
-          dir_map[File.dirname(k)] = target
+          sql = "insert into file(name,size,mtime,directory_id) values (:name, :size, :mtime,:directory_id)"
+          @db.execute(sql,:name=>name,:size=>stat.size,:mtime => stat.mtime, :directory_id=>id)
         end
-        target[:symlink][k] = {:source => v[:source]}
       end
-      return dir_map;
     end
-    def update_dir(name,dir_map,dir_info)
-      dir_map[name] = {:file => {},:symlink=>{}} unless dir_map[name]
-      dir_map[name][:file].each do |k,v|
-        @fileMap[:file].delete(k)
-      end
-      dir_map[name][:symlink].each do |k,v|
-        @fileMap[:symlink].delete(k)
+    def update_dir(dir_info)
+      result = @db.execute("select id from directory where name = ?",dir_info[:name])
+      p_id = nil
+      id = nil
+      @db.execute('select id from directory where name =?',File.dirname(dir_info[:name])) do |row|
+        p_id = row[0]
+      end
+      if result.length != 0
+        id = result[0][0]
+        @db.execute("delete from file where directory_id = #{id}")
+        @db.execute("delete from symlink where directory_id = #{id}")
+
+        @db.execute("update directory  set mtime = ?,parent_directory_id = ?" + 
+                    " where id = ?",dir_info[:mtime],p_id,id)
+      else
+        @db.execute("insert into directory(name,mtime,parent_directory_id) values(?,?,?)",
+          dir_info[:name],dir_info[:mtime],p_id)
+        result = @db.execute("select id from directory where name = ?",dir_info[:name])
+        id = result[0][0]
       end
-      @fileMap[:directory][name] = {:mtime => dir_info[:mtime]}
-      dir_info[:file].each do |k,v|
-        @fileMap[:file][k] = v
+      dir_info[:files].each do |f|
+        @db.execute("insert into file(name,mtime,size,directory_id) values(?,?,?,?)",
+                    f[:name],f[:mtime],f[:size],id)
       end
-      dir_info[:symlink].each do |k,v|
-        @fileMap[:symlink][k] = v
+      dir_info[:links].each do |f|
+        @db.execute("insert into symlink(name,source,directory_id) values(?,?,?)",f[:name],f[:source],id)
       end
-      dir_map[name][:file] = dir_info[:file]
-      dir_map[name][:symlink] = dir_info[:symlink]
-      dir_map[name][:mtime] = dir_info[:mtime]
-      return dir_map
-    end
-    def load_yaml(data)
-      @fileMap = YAML.load(data)
     end
-    def dump_yaml()
-      YAML.dump(@fileMap)
-    end
-    def hierarchie(dir)
-      count = dir.count("/")
-      tree = []
-      @fileMap[:directory].each do |k,v|
-        if k.index(dir) != 0
-          next
-        end
-        level = k.count("/") - count
+    def get_level_directory(tree,p_id,level)
+      @db.execute('select id,name,mtime from directory where parent_directory_id = ?',p_id) do |row|
+        id = row[0]
+        name = row[1]
+        mtime = row[2]
         tree[level] = {} unless tree[level]
-        tree[level][k] = v
+        tree[level][name] = {:mtime=>mtime}
+        get_level_directory(tree,id,level+1)
       end
+    end
+    def hierarchie(dir)
+      tree=[]
+      result = @db.execute('select id,name,mtime from directory where name = ?',dir)
+      if result.length == 0
+        S3log.error("#{dir} is not stored.")
+        exit(-1)
+      end
+      id = result[0][0]
+      name = result[0][1]
+      mtime = result[0][2]
+      tree[0] = {} 
+      tree[0][name]={:mtime=>mtime}
+      get_level_directory(tree,id,1)
       return tree
     end
-    def diff(target)
-      modify_dir_map = {}
-      modify_files = []
-      modify_links = []
-
-      remove_dirs = target.fileMap[:directory].keys -  @fileMap[:directory].keys
-      add_dirs = @fileMap[:directory].keys - target.fileMap[:directory].keys 
-
-      new_info = @fileMap[:file]
-      old_info = target.fileMap[:file]
-
-      remove_files = old_info.keys - new_info.keys
-      remove_files.each do |f|
-        dir = File.dirname(f)
-        modify_dir_map[dir] = true
-      end
-      add_files = new_info.keys - old_info.keys
-      add_files.each do |f|
-        dir = File.dirname(f)
-        modify_dir_map[dir] = true
-      end
-
-      new_info.each do |k,v|
-        next unless old_info[k]
-        if old_info[k][:date] != v[:date] or old_info[k][:size] != v[:size]
-          modify_files.push(k)
-          dir = File.dirname(k)
-          modify_dir_map[dir] = true
+    def modify(target)
+      now_id = 0
+      while 1
+        result = @db.execute("select id,name,mtime from directory where id > ? limit 1",now_id)
+        break if result.length == 0
+        now_id = result[0][0].to_i
+        name = result[0][1]
+        mtime = result[0][2]
+        files = []
+        links = []
+        t_result = target.db.execute("select id,name from directory where name = ?",name)
+        if t_result.length != 0
+          t_id = t_result[0][0]
+          t_files = target.db.execute("select name,size,mtime from file where directory_id = ? order by name",t_id)
+          files = @db.execute("select name,size,mtime from file where directory_id = ? order by name",now_id)
+          if t_files == files
+            t_links = target.db.execute("select name,source from symlink where directory_id = ? order by name",t_id)
+            links = @db.execute("select name,source from symlink where directory_id = ? order by name",now_id)
+            if t_links == links
+              next
+            end
+          end
         end
+        file_infos = []
+        files.each do |f|
+          file_infos.push({:name=>f[0],:size=>f[1],:mtime=>f[2]})
+        end
+        sym_infos = []
+        links.each do |l|
+          sym_infos.push({:name=>f[0],:source=>f[1]})
+        end
+        yield({:name => name,:mtime=>mtime,:files => file_infos ,:links => sym_infos})
       end
-
-      new_info = @fileMap[:symlink]
-      old_info = target.fileMap[:symlink]
-
-      remove_links = old_info.keys - new_info.keys
-      remove_links.each do |f|
-        dir = File.dirname(f)
-        modify_dir_map[dir] = true
-      end
-
-      add_links = new_info.keys - old_info.keys
-      add_links.each do |f|
-        dir = File.dirname(f)
-        modify_dir_map[dir] = true
+    end
+    def remove(target)
+      now_id = 0
+      while 1
+        t_result = target.db.execute("select id,name from directory where id > ? limit 1",now_id)
+        break if t_result.length == 0
+        now_id = t_result[0][0].to_i
+        name = t_result[0][1]
+        result = @db.execute("select id,name from directory where name = ?",name)
+        if result.length == 0
+          yield({:name => name})
+        end
       end
-
-      new_info.each do |k,v|
-        next unless old_info[k]
-        if old_info[k][:source] != v[:source]
-          modify_links.push(k)
-          dir = File.dirname(k)
-          modify_dir_map[dir] = true
+    end
+    def close(delete=false)
+      @db.close
+      if delete
+        if File.exist?(@db_name)
+          File.unlink(@db_name)
+        end
+        if File.exist?(@db_name + ".gz")
+          File.unlink(@db_name+".gz")
         end
       end
-      return {
-        :directory => {:add => add_dirs,:modify => modify_dir_map.keys - add_dirs - remove_dirs,:remove => remove_dirs},
-        :file => {:add => add_files,:modify => modify_files,:remove => remove_files},
-        :symlink => {:add => add_links,:modify => modify_links,:remove => remove_links}}
     end
   end
 end

data/lib/s3backup.rb

@@ -2,5 +2,5 @@ $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
 module S3backup
-  VERSION = '0.7.3'
+  VERSION = '0.8.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: s3backup
 version: !ruby/object:Gem::Version 
-  version: 0.7.3
+  version: 0.8.1
 platform: ruby
 authors: 
 - Takeshi Morita
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-31 00:00:00 +09:00
+date: 2010-02-11 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -22,6 +22,16 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 0.6.2
     version: 
+- !ruby/object:Gem::Dependency 
+  name: sqlite3
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.0.7
+    version: 
 - !ruby/object:Gem::Dependency 
   name: rubyforge
   type: :development