qdumpfs 0.7.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3d4d28c42fc54a4ba9ba806aa57c056774889663cfa011d1cb8ff6d259a3368b
-  data.tar.gz: fcfa95f0abdbaf1a8b571cf088fa83811e1984f4bd7bcd1283e68bdad3ce894b
+  metadata.gz: d0d2013cd5b52dcc99390e498ce0f749b3bf4d74d5229f734606b7b8a47b221f
+  data.tar.gz: e6f9d32055e67cc2b8beed5354befbf52f02ff06448cfa827673fc46a771bda1
 SHA512:
-  metadata.gz: d1af10ef6521b9f775a538dc666e06f4cc4222131413144c8eacd59d795242890db306411e34076b041ac356b0a2e6a0c74c50cf221e01550caf0b18260bd738
-  data.tar.gz: 68351bac00ffe057beb25433f5d158c74814a2a20efa43cd15a8a608cc800f5655b6f41be2b1292e8ebc1b7b63e43f01add315daaa3503d047f4139e7dbf4848
+  metadata.gz: eaed232b7e0803f065d6dde3373745bc7459b7c006ad5f20b2ddeb1431d5adc473c35df9045a913c0fcfd524a92f5d0319fed33e5fa2f645607a5bb0488d1d73
+  data.tar.gz: e434c372a596e239aa97564b4820e5d3c0fb31802f33c9f4b37d031009811819268893462f48c9c3e25728f2302cc56fcd9c5c990ba869dc209edeb1171d0cd8

data/.gitignore

@@ -16,3 +16,4 @@ test*.cmd
 /data/to2
 
 
+/sync_d.cmd

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    qdumpfs (0.7.0)
+    qdumpfs (1.1.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    minitest (5.14.2)
-    rake (13.0.1)
+    minitest (5.14.4)
+    rake (13.0.3)
 
 PLATFORMS
   ruby

data/README.md

@@ -1,13 +1,14 @@
 # Qdumpfs
 
-qdumpfs is a modified version of pdumpfs.
+pdumpfsの個人的改良版です。
 
-## Installation
+Gem化して最近のバージョンのバージョンのRubyに対応。コマンドの拡張などを行っています。
 
-gem install qdumpfs
+## インストール
 
-## Usage
+gem install qdumpfs
 
+## 使用方法
 
 ```
 Usage: qdumpfs [options] <source> <dest>
@@ -19,48 +20,53 @@ Options
     -e, --exclude=PATTERN            exclude files/directories matching PATTERN
     -s, --exclude-by-size=SIZE       exclude files larger than SIZE
     -w, --exclude-by-glob=GLOB       exclude files matching GLOB
-    -c, --command=COMMAND            backup|sync|list|expire|verify|test
+    -c, --command=COMMAND            backup|sync|list|expire|verify|delete
     -l, --limit=HOURS                limit hours
     -k, --keep=KEEPARG               ex: --keep 100Y12M12W30D (100years, 12months, 12weeks, 30days, default)
 ```
 
-## Example
+## 実行例
+
+
+バックアップを実行する場合。
 
-Backup your home directory, run the following command.
 ```
 qdumpfs /home/foo /backup
 ```
 
-You can specify command option(default is "backup").
+"--command backup"オプションを明示することもできます。
+
 ```
 qdumpfs --command=backup /home/foo /backup
 ```
 
-Sync two backup directories.
+"--command sync"でバックアップフォルダを同期できます。
 ```
 qdumpfs --command=sync /backup1 /backup2
 ```
 
-Sync two backup directories(limit 1 hours, keep 100Y12M12W30D).
+バックアップフォルダの同期には膨大な時間が必要な場合があるため、実行時間を制限できます。以下は例えば1時間に制限する場合です。
 ```
 qdumpfs --command=sync --limit=1 /backup1 /backup2
 ```
 
-Sync two backup directories(limit 1 hours, keep specified backups only).
+バックアップフォルダの同期で、1時間でかつ"100Y12M12W30D"を保存する場合のオプションです。
 ```
-qdumpfs --command=sync --limit=1 --keep=5Y6M7W10D --keep/backup1 /backup2
+qdumpfs --command=sync --limit=1 --keep=5Y6M7W10D backup1 /backup2
 ```
 
-Expire backup directory.
+"--command expire"で、"--keep="パターンに該当しないバックアップを削除できます。
 ```
-qdumpfs --command=expire --limit=1 --keep=5Y6M7W10D --keep/backup1 /backup2
+qdumpfs --command=expire --limit=1 --keep=5Y6M7W10D backup1 /backup2
 ```
 
+"--command delete"で、バックアップに存在する指定したパスを削除できます(間違えてバックアップした内容を削除したい場合などに使用)。
+```
+qdumpfs --command=delete --delete-dir=backup1 --limit=1  r:/backup2
+```
 
 ## License
 
 qdumpfs is a free software with ABSOLUTELY NO WARRANTY under the terms of the GNU General Public License version 2.
 
 
-
-

data/lib/qdumpfs.rb

@@ -45,9 +45,13 @@ module Qdumpfs
       opt.on('--delete-to=YYYYMMDD', 'delete backup to YYYY/MM/DD') {|v|
         opts[:delete_to] = Date.parse(v)
       }
+      opt.on('--delete-dir=DIRS', 'relative path from the snapshot root of the backup. ex: --delete-dir=foo,home/bar') {|v|
+        opts[:delete_dirs] = v.split(/,/)
+      }      
       opt.on('--backup-at=YYYYMMDD', 'backup at YYYY/MM/DD') {|v|
         opts[:backup_at] = Date.parse(v)
       }
+      opt.on('-d', '--debug', 'debug mode') {|v| opts[:d] = v }
       opt.parse!(argv)
       option = Option.new(opts, argv)
       if opts[:v]
@@ -61,8 +65,10 @@ module Qdumpfs
         command = Command.new(option)
         command.run
       rescue => e
-#        p e.message
-#        p e.backtrace
+        if option.debug
+          p e.message
+          p e.backtrace
+        end
         puts opt.help
         exit        
       end
@@ -85,8 +91,6 @@ module Qdumpfs
         verify
       elsif @opt.cmd == 'delete'
         delete('delete')
-#      elsif @opt.cmd == 'test'
-#        test
       else
         raise RuntimeError, "unknown command: #{@opt.cmd}"
       end
@@ -96,9 +100,15 @@ module Qdumpfs
     def log_result(src, today, elapsed)
       time  = Time.now.strftime("%Y-%m-%dT%H:%M:%S")
       bytes = convert_bytes(@written_bytes)
-      msg = sprintf("%s: %s -> %s (in %.2f sec, %s written)\n",
-      time, src, today, elapsed, bytes)
+      msg = sprintf("%s: %s -> %s (in %.2f sec, %s written)\n", time, src, today, elapsed, bytes)
       log(msg)
+      log("error files:\n")
+      i = 1
+      @error_files.each do |filename, reason|
+        msg = "#{i}. #{filename}\t#{reason}\n"
+        log(msg)
+        i += 1
+      end
     end
 
     def log(msg, console = true)
@@ -109,6 +119,10 @@ module Qdumpfs
       @opt.report(type, file_name)
     end
 
+    def report_error(file_name, reason)
+      @opt.report_error(file_name, reason)
+    end
+
     def update_file(src, latest, today)
       type = detect_type(src, latest)
       report(type, src)
@@ -133,11 +147,17 @@ module Qdumpfs
     end
     
     def filecount(dir)
-      pscmd = 'Get-ChildItem -Recurse -File | Measure-Object | %{$_.Count}'
-      cmd = "powershell -Command \"#{pscmd}\""
-      result = nil
-      Dir.chdir(dir) do
-        result = `#{cmd}`
+      result = '0'
+      if windows? 
+        pscmd = 'Get-ChildItem -Recurse -File | Measure-Object | %{$_.Count}'
+        cmd = "powershell -Command \"#{pscmd}\""
+        result = nil
+        Dir.chdir(dir) do
+          result = `#{cmd}`
+          result.chomp!
+        end
+      else
+        result = `find #{dir} | wc -l`
         result.chomp!
       end
       result.to_i
@@ -149,29 +169,6 @@ module Qdumpfs
       return src_count, dst_count
     end
 
-#    def get_snapshots(target_dir)
-#      # 指定したディレクトリに含まれるバックアップフォルダ(日付つき)を全て取得
-#      dd   = "[0-9][0-9]"
-#      dddd = dd + dd
-#      # FIXME: Y10K problem.
-#      dirs = []
-#      glob_path = File.join(target_dir, dddd, dd, dd)
-#      Dir.glob(glob_path).sort.find {|dir|
-#        day, month, year = File.split_all(dir).reverse.map {|x| x.to_i }
-#        path = diro
-#        if File.directory?(path) and Date.valid_date?(year, month, day) and
-#          dirs << path
-#        end
-#      }
-#      dirs
-#    end
-
-#    def get_snapshot_date(snapshot)
-#      # バックアップディレクトリのパス(日付つき)から日付を取得して返す
-#      day, month, year = File.split_all(snapshot).reverse.map {|x| x.to_i }
-#      Time.new(year, month, day)
-#    end
-    
     def update_snapshot(src, latest, today)
       # バックアップの差分コピーを実行
       # src: コピー元ディレクトリ ex) i:/from/home
@@ -192,8 +189,9 @@ module Qdumpfs
           # ファイルのアップデート
           update_file(s, l, t)
           dirs[t] = File.stat(s) if File.ftype(s) == "directory"
-        rescue Errno::ENOENT, Errno::EACCES => e
-          wprintf("%s: %s", src, e.message)
+        rescue => e
+          report_error(s, e.message)
+          @error_files << [s, e.message]          
           next
         end
       end
@@ -227,8 +225,9 @@ module Qdumpfs
         end
           chown_if_root(type, s, t)
           dirs[t] = File.stat(s) if File.ftype(s) == "directory"
-        rescue Errno::ENOENT, Errno::EACCES => e
-          wprintf("%s: %s", s, e.message)
+        rescue => e
+          report_error(s, e.message)
+          @error_files << [s, e.message]
           next
         end
       end
@@ -240,7 +239,7 @@ module Qdumpfs
       
       #コピー元のスナップショット
       src_snapshots = BackupDir.scan_backup_dirs(src)
-      @opt.detect_keep_dirs(src_snapshots)
+      @opt.detect_expire_dirs(src_snapshots)
       
       # コピー先の最新スナップショット
       dst_snapshots = BackupDir.scan_backup_dirs(dst)
@@ -308,6 +307,7 @@ module Qdumpfs
       log("##### backup start #####")
       
       @written_bytes = 0
+      @error_files = []
       start_time = Time.now
       if @opt.backup_at
         start_time = to_time(@opt.backup_at)
@@ -364,6 +364,7 @@ module Qdumpfs
       
       start_time = Time.now
       @written_bytes = 0
+      @error_files = []
       src = @opt.src
       dst = @opt.dst
       
@@ -410,6 +411,10 @@ module Qdumpfs
       
       end_time = Time.now
       diff = time_diff(start_time, end_time)
+
+      elapsed = Time.now - start_time
+      log_result(src, dst, elapsed)
+        
       log("##### sync end #{fmt(end_time)} diff=#{diff} last_sync_complete=#{last_sync_complete} #####")
     end
       
@@ -457,9 +462,7 @@ module Qdumpfs
     
     def delete(cmd)
       @opt.validate_directories(1)
-      
       start_time = Time.now
-p @opt.limit_sec      
       limit_time = start_time + (@opt.limit_sec)
       log("##### #{cmd} delete-from=#{@opt.delete_from} delete-to=#{@opt.delete_to} start #{fmt(start_time)} => limit_time=#{fmt(limit_time)} #####")
       @opt.dirs.each do |target_dir|
@@ -482,6 +485,29 @@ p @opt.limit_sec
       log("##### #{cmd} end #####")
     end
 
+    def rm_dir(path)
+      can_delete = true
+      if @opt.dry_run
+        can_delete = false
+      elsif !FileTest.directory?(path)
+        can_delete = false
+      end
+      msg = can_delete ? "...ok..." : "...ng..."
+      print "Deleting #{path} #{msg}"
+
+      return unless can_delete
+
+      #それ以外は日付バックアップディレクトリ全体を削除
+      if windows?
+        # Windowsの場合
+        win_backup_path = to_win_path(path)
+        system("rmdir /S /Q #{win_backup_path}")
+      else
+        # Linux/macOSの場合
+        system("rm -rf #{path}")
+      end
+    end
+
     def delete_target_dir(cmd, target_dir)
       target_dir = to_unix_path(target_dir)
       puts "<<<<< Target dir: #{target_dir} >>>>>"
@@ -494,40 +520,20 @@ p @opt.limit_sec
       else
         raise RuntimeError, "unknown command: #{cmd}"
       end
-      
-#      p @opt.keep_year
-#      p @opt.keep_month
-#      p @opt.keep_day
-      
+    
       snapshots.each do |snapshot|
         next if snapshot.keep
         t_start = Time.now
-        print "Deleting #{snapshot.path} ..."
-       
-        unless @opt.dry_run
-          #http://superuser.com/questions/19762/mass-deleting-files-in-windows/289399#289399
-          ##### here
-          #bundle exec ruby exe/qdumpfs --dry-run --keep=100Y36M30W30D --command expire f:/pc1/pdumpfs/users f:/pc1/pdumpfs/opt f:/pc1/pdumpfs/d
-          
-          if windows?
-            # Windowsの場合
-            win_backup_path = to_win_path(snapshot.path)
-            
-#            byenow = "byenow"
-#            if which(byenow)
-#              print " bynow"
-#              system("byenow -y --delete-ntapi --one-liner #{snapshot.path}")
-#            else
-#              print " pass1"
-#              system("del /F /S /Q #{win_backup_path} > nul")
-#              print " pass2"
-#              system("rmdir /S /Q #{win_backup_path}")
-            #            end
-            system("rmdir /S /Q #{win_backup_path}")
-          else
-            # Linux/macOSの場合
-            system("rm -rf #{snapshot.path}")
+
+        if @opt.delete_dirs.size > 0
+          #削除するディレクトリが指定されている場合日付バックアップディレクトリの下そのディレクトリを削除
+          @opt.delete_dirs.each do |delete_dir|
+            delete_path = File.join(snapshot.path, delete_dir)
+            rm_dir(delete_path)
           end
+        else
+          #それ以外はバックアップディレクトリ全体が対象
+          rm_dir(snapshot.path)
         end
         
         t_end = Time.now
@@ -551,60 +557,6 @@ p @opt.limit_sec
       end      
     end
 
-    # def delete
-    #   @opt.validate_directories(1)
-      
-    #   start_time = Time.now
-    #   limit_time = start_time + (@opt.limit_sec)      
-    #   log("##### delete start #{fmt(start_time)} => limit_time=#{fmt(limit_time)} #####")
-    #   @opt.dirs.each do |target_dir|
-    #     target_start = Time.now
-    #     delete_target_dir(target_dir)
-    #     target_end = Time.now
-        
-    #     # 次回expireにかかる時間を最終expire時間の半分と予想
-    #     next_expire = (target_end - target_start) / 2
-        
-    #     cur_time = Time.now
-    #     in_imit = (cur_time + next_expire) < limit_time
-        
-    #     log("## cur_time=#{fmt(cur_time)} + next_expire=#{next_expire} <  limit_time=#{fmt(limit_time)} in_limit=#{in_limit} ## ")
-    #     unless in_limit
-    #       break
-    #     end
-    #   end        
-    #   log("##### delete end #####")
-    # end
-
-    # def delete_target_dir(target_dir)
-    #   target_dir = to_unix_path(target_dir)
-    #   puts "<<<<< Target dir: #{target_dir} >>>>>"
-      
-    #   snapshots = BackupDir.scan_backup_dirs(target_dir)
-    #   @opt.detect_delete_dirs(snapshots)
-
-    #   snapshots.each do |snapshot|
-    #     next if snapshot.keep
-    #     t_start = Time.now
-    #     print "Deleting #{snapshot.path} ..."
-       
-    #     unless @opt.dry_run
-    #       if windows?
-    #         # Windowsの場合
-    #         win_backup_path = to_win_path(snapshot.path)
-    #         system("rmdir /S /Q #{win_backup_path}")
-    #       else
-    #         # Linux/macOSの場合
-    #         system("rm -rf #{snapshot.path}")
-    #       end
-    #     end
-    #   end
-
-    #   t_end = Time.now
-    #   diff = (t_end - t_start).to_i
-    #   diff_hours = diff / 3600
-    #   puts " done[#{diff} seconds = #{diff_hours} hours]." 
-    # end
     
   end
 

data/lib/qdumpfs/option.rb

@@ -141,14 +141,17 @@ module Qdumpfs
       @keep_day = $1.to_i if keep =~ /(\d+)D/
       @delete_from = @opts[:delete_from]
       @delete_to = @opts[:delete_to]
+      @delete_dirs = @opts[:delete_dirs] || []
       @backup_at = @opts[:backup_at]
       @today = Date.today
+      @debug = @opts[:d]
     end
     attr_reader :dirs, :src, :dst, :cmd
     attr_reader :keep_year, :keep_month, :keep_week, :keep_day
     attr_reader :logdir, :logpath, :verifypath
     attr_reader :logger, :matcher, :reporter, :interval_proc
-    attr_reader :delete_from, :delete_to, :backup_at
+    attr_reader :delete_from, :delete_to, :delete_dirs, :backup_at
+    attr_reader :debug
     
     def report(type, filename)
       if @opts[:v]
@@ -173,6 +176,11 @@ module Qdumpfs
       end
       log(msg)
     end
+
+    def report_error(filename, reason)
+      msg = sprintf("err_file\t%s\t%s\n", filename, reason)
+      log(msg)
+    end
     
     def log(msg, console = true)
       return if (msg.nil? || msg == '')

data/lib/qdumpfs/util.rb

@@ -73,14 +73,14 @@ module QdumpfsFind
         yield file.dup.taint
         begin
           s = File.lstat(file)
-        rescue Errno::ENOENT, Errno::EACCES, Errno::ENOTDIR, Errno::ELOOP, Errno::ENAMETOOLONG => e
+        rescue => e
           logger.print("File.lstat path=#{file} error=#{e.message}")
           next
         end
         if s.directory? then
           begin
             fs = Dir.entries(file, :encoding=>'UTF-8')
-          rescue Errno::ENOENT, Errno::EACCES, Errno::ENOTDIR, Errno::ELOOP, Errno::ENAMETOOLONG => e
+          rescue => e
             logger.print("Dir.entries path=#{file} error=#{e.message}")
             next
           end
@@ -107,23 +107,22 @@ module QdumpfsUtils
 
   # We don't use File.copy for calling @interval_proc.
   def copy_file(src, dest)
-    File.open(src, 'rb') {|r|
-      File.open(dest, 'wb') {|w|
-        block_size = (r.stat.blksize or 8192)
-        begin
+    begin    
+      File.open(src, 'rb') {|r|
+        File.open(dest, 'wb') {|w|
+          block_size = (r.stat.blksize or 8192)
           i = 0
           while true
             block = r.sysread(block_size)
             w.syswrite(block)
             i += 1
             @written_bytes += block.size
-#            @interval_proc.call if i % 10 == 0
           end
-        rescue EOFError => e
-          #            puts e.message, e.backtrace
-        end
+        }
       }
-    }
+    rescue EOFError => e
+      # puts e.message, e.backtrace
+    end    
     unless FileTest.file?(dest)
       raise "copy_file fails #{dest}"
     end

data/lib/qdumpfs/version.rb

@@ -1,3 +1,4 @@
-module Qdumpfs
-  VERSION = "0.7.0"
-end
+module Qdumpfs
+  VERSION = "1.1.0"
+end  
+  

data/run_list.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+curdir=$(dirname $0)
+bundle exec ruby $curdir/exe/qdumpfs --command=list /Volumes/EXT/
+

data/run_qdumpfs.sh

@@ -1,4 +1,4 @@
 #!/bin/sh
 
 curdir=$(dirname $0)
-bundle exec ruby $curdir/exe/qdumpfs $*
+bundle exec ruby $curdir/exe/qdumpfs "$@"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: qdumpfs
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 1.1.0
 platform: ruby
 authors:
 - src
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-01-19 00:00:00.000000000 Z
+date: 2021-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -86,6 +86,7 @@ files:
 - qdumpfs.gemspec
 - run2_qdumpfs.cmd
 - run_list.cmd
+- run_list.sh
 - run_qdumpfs.cmd
 - run_qdumpfs.sh
 - run_sync.cmd