qdumpfs 0.4.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ceaa58593a1a92b6d0c9f2d76bd7ed5a75822c0452a2551781946ec73d8a6ad5
-  data.tar.gz: caf81857a389d2a81094f167e8160195ab8c5fd107fbbef4387d7303a4e12a3b
+  metadata.gz: bc8d8dec60fe04598e9dd5bea8528f5299a7cbb7ee0f0432c3b6d46488917edf
+  data.tar.gz: '029dffccca2865524f208d28f2ebe2f63b2de465e0a5f08de93c597156fd48a9'
 SHA512:
-  metadata.gz: 17439939c7fd6e82f8ad6a560d9faaf5d268c275e121c1939ddc574f0c99e3aba807fe301b282b226b19f2b676b3802bd8016c130ed2d95765380406fca91f1f
-  data.tar.gz: 3f5f378da6ca8e0a4f3a9732227a099909bb15748066d64da7b10c642b66bb1a877d4262b29aa98cf5b44ae9b78bc6b5d0d806fe19fe99e9cb670b490509f7f0
+  metadata.gz: 9c393b76dddd9978adf5dfc77cfb25f3c14bfb9b1aca0767228b8b4c70fdf11206f4a35c9e0f5098b1c7ba1e3b113dc8f51de3ad14b48d39fe1dff8dd21309f1
+  data.tar.gz: 054605d2b4f91e2c0fdbd0518614b81a9f04f0ba378f0d92c5aba28c59456ae88c40dfb156579295c10130c248fc314197bd03398f18435cd1e0596696f444f2

data/.gitignore

@@ -11,4 +11,8 @@ list*
 log.txt
 *test.cmd
 test*.cmd
+*.log
+/data/to
+/data/to2
+
 

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    qdumpfs (0.4.0)
+    qdumpfs (0.8.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    minitest (5.14.1)
+    minitest (5.14.2)
     rake (13.0.1)
 
 PLATFORMS

data/README.md

@@ -1,31 +1,72 @@
 # Qdumpfs
 
-qdumpfs is a modified version of pdumpfs.
+pdumpfsの個人的改良版です。
 
-## Installation
+Gem化して最近のバージョンのバージョンのRubyに対応。コマンドの拡張などを行っています。
+
+## インストール
 
 gem install qdumpfs
 
-```ruby
-gem 'qdumpfs'
+## 使用方法
+
+```
+Usage: qdumpfs [options] <source> <dest>
+    -h, --help                       show this message
+Options
+    -v, --verbose                    verbose message
+    -r, --report                     report message
+    -n, --dry-run                    don't actually run any commands
+    -e, --exclude=PATTERN            exclude files/directories matching PATTERN
+    -s, --exclude-by-size=SIZE       exclude files larger than SIZE
+    -w, --exclude-by-glob=GLOB       exclude files matching GLOB
+    -c, --command=COMMAND            backup|sync|list|expire|verify|delete
+    -l, --limit=HOURS                limit hours
+    -k, --keep=KEEPARG               ex: --keep 100Y12M12W30D (100years, 12months, 12weeks, 30days, default)
 ```
 
-And then execute:
+## 実行例
 
-    $ bundle
 
-Or install it yourself as:
+バックアップを実行する場合。
 
-    $ gem install qdumpfs
+```
+qdumpfs /home/foo /backup
+```
 
-## Usage
+"--command backup"オプションを明示することもできます。
 
-qdumpfs srcdir dstdir
+```
+qdumpfs --command=backup /home/foo /backup
+```
 
-## License
+"--command sync"でバックアップフォルダを同期できます。
+```
+qdumpfs --command=sync /backup1 /backup2
+```
 
-qdumpfs is a free software with ABSOLUTELY NO WARRANTY under the terms of the GNU General Public License version 2.
+バックアップフォルダの同期には膨大な時間が必要な場合があるため、実行時間を制限できます。以下は例えば1時間に制限する場合です。
+```
+qdumpfs --command=sync --limit=1 /backup1 /backup2
+```
+
+バックアップフォルダの同期で、1時間でかつ"100Y12M12W30D"を保存する場合のオプションです。
+```
+qdumpfs --command=sync --limit=1 --keep=5Y6M7W10D backup1 /backup2
+```
+
+"--command expire"で、"--keep="パターンに該当しないバックアップを削除できます。
+```
+qdumpfs --command=expire --limit=1 --keep=5Y6M7W10D backup1 /backup2
+```
+
+"--command delete"で、バックアップに存在する指定したパスを削除できます(間違えてバックアップした内容を削除したい場合などに使用)。
+```
+qdumpfs --command=delete --delete-dir=backup1 --limit=1  r:/backup2
+```
 
+## License
 
+qdumpfs is a free software with ABSOLUTELY NO WARRANTY under the terms of the GNU General Public License version 2.
 
 

data/data/from/0.txt

@@ -0,0 +1 @@
+0

data/data/from/a/1.txt

@@ -0,0 +1 @@
+1

data/exe/sample

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+
+require 'fileutils'
+require "qdumpfs"
+
+from = "./data/from"
+to = "./data/to"
+to2 = "./data/to2"
+
+def create_dir(dir)
+  if FileTest.directory?(dir)
+    FileUtils.rm_rf(dir)
+  end
+  FileUtils.mkpath(dir)
+end
+
+create_dir(to)
+create_dir(to2)
+
+
+arg = ["--backup-at=20210119", from, to]
+Qdumpfs::Command.run(arg)
+arg = ["--backup-at=20210121", from, to]
+Qdumpfs::Command.run(arg)
+arg = ["--backup-at=20210122", from, to]
+Qdumpfs::Command.run(arg)
+arg = ["--backup-at=20210123", from, to]
+Qdumpfs::Command.run(arg)
+arg = ["--backup-at=20210124", from, to]
+Qdumpfs::Command.run(arg)
+
+
+arg = ["--backup-at=20210122", from, to2]
+Qdumpfs::Command.run(arg)
+arg = ["--backup-at=20210123", from, to2]
+
+arg = ["--command=delete", "--delete-from=20210122", "--delete-to=20210123", "--limit=1", to, to2]
+Qdumpfs::Command.run(arg)
+
+

data/lib/qdumpfs.rb

@@ -19,8 +19,8 @@ module Qdumpfs
       opts = {}
       opt = OptionParser.new(argv)
       opt.version = VERSION
-      opt.banner = "Usage: #{opt.program_name} [-h|--help]"
-      opt.separator('')
+      opt.banner = "Usage: #{opt.program_name} [options] <source> <dest>"
+      opt.separator('Options')
       opt.on_head('-h', '--help', 'show this message') do |v|
         puts opt.help
         exit
@@ -34,24 +34,40 @@ module Qdumpfs
       }
       opt.on('-s SIZE', '--exclude-by-size=SIZE', 'exclude files larger than SIZE') {|v| opts[:es] = v }
       opt.on('-w GLOB', '--exclude-by-glob=GLOB', 'exclude files matching GLOB') {|v| opts[:ep] = v }
-      commands = ['backup', 'sync', 'list', 'expire', 'verify', 'test']
+      commands = ['backup', 'sync', 'list', 'expire', 'verify', 'delete']
       opt.on('-c COMMAND', '--command=COMMAND', commands, commands.join('|')) {|v| opts[:c] = v}
       opt.on('-l HOURS', '--limit=HOURS', 'limit hours') {|v| opts[:limit] = v}
       opt.on('-k KEEPARG', '--keep=KEEPARG', 'ex: --keep 100Y12M12W30D (100years, 12months, 12weeks, 30days, default)') {|v| opts[:keep] = v}
-
-
+      opt.on('--logdir=LOGDIR', 'logdir') {|v| opts[:logdir] = v}
+      opt.on('--delete-from=YYYYMMDD', 'delete backup from YYYY/MM/DD') {|v|
+        opts[:delete_from] = Date.parse(v)
+      }
+      opt.on('--delete-to=YYYYMMDD', 'delete backup to YYYY/MM/DD') {|v|
+        opts[:delete_to] = Date.parse(v)
+      }
+      opt.on('--delete-dir=DIRS', 'relative path from the snapshot root of the backup. ex: --delete-dir=foo,home/bar') {|v|
+        opts[:delete_dirs] = v.split(/,/)
+      }      
+      opt.on('--backup-at=YYYYMMDD', 'backup at YYYY/MM/DD') {|v|
+        opts[:backup_at] = Date.parse(v)
+      }
       opt.parse!(argv)
-      option = Option.new(opts, ARGV)
-      
+      option = Option.new(opts, argv)
+      if opts[:v]
+        puts "<<<<< qdumpfs options >>>>> "
+        puts "logdir: #{option.logdir}"
+        puts "logpath: #{option.logpath}"
+        puts "verifypath: #{option.verifypath}"
+        puts 
+      end
       begin
         command = Command.new(option)
         command.run
-      rescue ArgumentError => e
-        puts e.message, e.backtrace
-        puts opt.help
-        exit
       rescue => e
-        puts e.message, e.backtrace
+#        p e.message
+#        p e.backtrace
+        puts opt.help
+        exit        
       end
     end
     
@@ -67,13 +83,13 @@ module Qdumpfs
       elsif @opt.cmd == 'list'
         list
       elsif @opt.cmd == 'expire'
-        expire
+        delete('expire')
       elsif @opt.cmd == 'verify'
         verify
-      elsif @opt.cmd == 'test'
-        test
+      elsif @opt.cmd == 'delete'
+        delete('delete')
       else
-        raise RuntimeError, "unknown command: #{cmd}"
+        raise RuntimeError, "unknown command: #{@opt.cmd}"
       end
     end
     
@@ -134,29 +150,6 @@ module Qdumpfs
       return src_count, dst_count
     end
 
-    def get_snapshots(target_dir)
-      # 指定したディレクトリに含まれるバックアップフォルダ(日付つき)を全て取得
-      dd   = "[0-9][0-9]"
-      dddd = dd + dd
-      # FIXME: Y10K problem.
-      dirs = []
-      glob_path = File.join(target_dir, dddd, dd, dd)
-      Dir.glob(glob_path).sort.find {|dir|
-        day, month, year = File.split_all(dir).reverse.map {|x| x.to_i }
-        path = dir
-        if File.directory?(path) and Date.valid_date?(year, month, day) and
-          dirs << path
-        end
-      }
-      dirs
-    end
-
-    def get_snapshot_date(snapshot)
-      # バックアップディレクトリのパス(日付つき)から日付を取得して返す
-      day, month, year = File.split_all(snapshot).reverse.map {|x| x.to_i }
-      Time.new(year, month, day)
-    end
-    
     def update_snapshot(src, latest, today)
       # バックアップの差分コピーを実行
       # src: コピー元ディレクトリ ex) i:/from/home
@@ -294,6 +287,9 @@ module Qdumpfs
       
       @written_bytes = 0
       start_time = Time.now
+      if @opt.backup_at
+        start_time = to_time(@opt.backup_at)
+      end
       src = @opt.src
       dst = @opt.dst
       
@@ -394,15 +390,7 @@ module Qdumpfs
       diff = time_diff(start_time, end_time)
       log("##### sync end #{fmt(end_time)} diff=#{diff} last_sync_complete=#{last_sync_complete} #####")
     end
-    
-    def open_verify_file
-      filename = File.join(@log_dir, 'verify.txt')
-      if FileTest.file?(filename)
-        File.unlink(filename)
-      end
-      File.open(filename, 'a')
-    end
-    
+      
     def verify
       file = @opt.open_verifyfile
     
@@ -445,74 +433,80 @@ module Qdumpfs
       file.close 
     end
     
-    def expire
+    def delete(cmd)
       @opt.validate_directories(1)
-      
       start_time = Time.now
       limit_time = start_time + (@opt.limit_sec)
-      log("##### expire start #{fmt(start_time)} => limit_time=#{fmt(limit_time)} #####")
-      
+      log("##### #{cmd} delete-from=#{@opt.delete_from} delete-to=#{@opt.delete_to} start #{fmt(start_time)} => limit_time=#{fmt(limit_time)} #####")
       @opt.dirs.each do |target_dir|
         
         target_start = Time.now        
-        expire_target_dir(target_dir)
+        delete_target_dir(cmd, target_dir)
         target_end = Time.now
         
         # 次回expireにかかる時間を最終expire時間の半分と予想
         next_expire = (target_end - target_start) / 2
         
         cur_time = Time.now
-        in_imit = (cur_time + next_expire) < limit_time
+        in_limit = (cur_time + next_expire) < limit_time
         
         log("## cur_time=#{fmt(cur_time)} + next_expire=#{next_expire} <  limit_time=#{fmt(limit_time)} in_limit=#{in_limit} ## ")
         unless in_limit
           break
         end
       end
-      
-      log("##### expire end #####")
+      log("##### #{cmd} end #####")
     end
-    
-    def expire_target_dir(target_dir)
+
+    def rm_dir(path)
+      can_delete = true
+      if @opt.dry_run
+        can_delete = false
+      elsif !FileTest.directory?(path)
+        can_delete = false
+      end
+      msg = can_delete ? "...ok..." : "...ng..."
+      print "Deleting #{path} #{msg}"
+
+      return unless can_delete
+
+      #それ以外は日付バックアップディレクトリ全体を削除
+      if windows?
+        # Windowsの場合
+        win_backup_path = to_win_path(path)
+        system("rmdir /S /Q #{win_backup_path}")
+      else
+        # Linux/macOSの場合
+        system("rm -rf #{path}")
+      end
+    end
+
+    def delete_target_dir(cmd, target_dir)
       target_dir = to_unix_path(target_dir)
       puts "<<<<< Target dir: #{target_dir} >>>>>"
       
       snapshots = BackupDir.scan_backup_dirs(target_dir)
-      @opt.detect_keep_dirs(snapshots)
-      
-#      p @opt.keep_year
-#      p @opt.keep_month
-#      p @opt.keep_day
-      
+      if cmd == 'expire'
+        @opt.detect_expire_dirs(snapshots)
+      elsif cmd == 'delete'
+        @opt.detect_delete_dirs(snapshots, @opt.delete_from, @opt.delete_to) 
+      else
+        raise RuntimeError, "unknown command: #{cmd}"
+      end
+    
       snapshots.each do |snapshot|
         next if snapshot.keep
         t_start = Time.now
-        print "Deleting #{snapshot.path} ..."
-       
-        unless @opt.dry_run
-          #http://superuser.com/questions/19762/mass-deleting-files-in-windows/289399#289399
-          ##### here
-          #bundle exec ruby exe/qdumpfs --dry-run --keep=100Y36M30W30D --command expire f:/pc1/pdumpfs/users f:/pc1/pdumpfs/opt f:/pc1/pdumpfs/d
-          
-          if windows?
-            # Windowsの場合
-            win_backup_path = to_win_path(snapshot.path)
-            
-#            byenow = "byenow"
-#            if which(byenow)
-#              print " bynow"
-#              system("byenow -y --delete-ntapi --one-liner #{snapshot.path}")
-#            else
-#              print " pass1"
-#              system("del /F /S /Q #{win_backup_path} > nul")
-#              print " pass2"
-#              system("rmdir /S /Q #{win_backup_path}")
-            #            end
-            system("rmdir /S /Q #{win_backup_path}")
-          else
-            # Linux/macOSの場合
-            system("rm -rf #{snapshot.path}")
+
+        if @opt.delete_dirs.size > 0
+          #削除するディレクトリが指定されている場合日付バックアップディレクトリの下そのディレクトリを削除
+          @opt.delete_dirs.each do |delete_dir|
+            delete_path = File.join(snapshot.path, delete_dir)
+            rm_dir(delete_path)
           end
+        else
+          #それ以外はバックアップディレクトリ全体が対象
+          rm_dir(snapshot.path)
         end
         
         t_end = Time.now
@@ -533,8 +527,10 @@ module Qdumpfs
       puts "Keep dirs:"
       snapshots.each do |snapshot|
         puts snapshot.path if snapshot.keep
-      end
-      
+      end      
     end
+
+    
   end
+
 end

data/lib/qdumpfs/option.rb

@@ -109,9 +109,9 @@ module Qdumpfs
       @cmd = @opts[:c] || 'backup'
       
       #      @logger = NullLogger.new
-      logfile = @opts[:logfile] || 'log.txt'
+      logfile = 'qdumpfs.log'
       #ログディレクトリの作成      
-      @logdir = File.expand_path('.')
+      @logdir = @opts[:logdir] || Dir.pwd
       Dir.mkdir(@logdir) unless FileTest.directory?(@logdir)
       @logpath = File.join(@logdir, logfile)
       @logger = SimpleLogger.new(@logpath)
@@ -139,12 +139,17 @@ module Qdumpfs
       @keep_month = $1.to_i if keep =~ /(\d+)M/
       @keep_week = $1.to_i if keep =~ /(\d+)W/
       @keep_day = $1.to_i if keep =~ /(\d+)D/
+      @delete_from = @opts[:delete_from]
+      @delete_to = @opts[:delete_to]
+      @delete_dirs = @opts[:delete_dirs]
+      @backup_at = @opts[:backup_at]
       @today = Date.today
     end
     attr_reader :dirs, :src, :dst, :cmd
     attr_reader :keep_year, :keep_month, :keep_week, :keep_day
-    attr_reader :logdir, :logfile, :verifyfile
+    attr_reader :logdir, :logpath, :verifypath
     attr_reader :logger, :matcher, :reporter, :interval_proc
+    attr_reader :delete_from, :delete_to, :delete_dirs, :backup_at
     
     def report(type, filename)
       if @opts[:v]
@@ -162,7 +167,9 @@ module Qdumpfs
       else
         # 何も指定されていない場合
         if type == 'new_file'
-          msg = format_report(type, filename)
+          stat = File.stat(filename)
+          size = stat.size
+          msg = format_report(type, filename, size)
         end
       end
       log(msg)
@@ -204,12 +211,32 @@ module Qdumpfs
       end
     end
     
-    def detect_keep_dirs(backup_dirs)
+    def detect_expire_dirs(backup_dirs)
       detect_year_keep_dirs(backup_dirs)
       detect_month_keep_dirs(backup_dirs)
       detect_week_keep_dirs(backup_dirs)
       detect_day_keep_dirs(backup_dirs)
     end
+
+    def detect_delete_dirs(backup_dirs, delete_from, delete_to)
+      
+      backup_dirs.each do |backup_dir|
+        backup_dir.keep = true
+        if delete_from && delete_to
+          if backup_dir.date >= delete_from && backup_dir.date <= delete_to
+            backup_dir.keep = false
+          end
+        elsif delete_from
+          if backup_dir.date >= delete_from
+            backup_dir.keep = false
+          end
+        elsif delete_to
+          if backup_dir.date <= delete_to
+            backup_dir.keep = false
+          end
+        end
+      end
+    end
     
     def open_verifyfile
       if FileTest.file?(@verifypath)
@@ -227,8 +254,8 @@ module Qdumpfs
     end
         
     private
-    def format_report(type, filename)
-      sprintf("%-12s %s\n", type, filename)
+    def format_report(type, filename, size)
+      sprintf("%s\t%s\t%d\n", type, filename, size)
     end
     
     def format_report_with_size(type, filename, size, format_size)

data/lib/qdumpfs/util.rb

@@ -1,3 +1,4 @@
+# coding: utf-8
 def wprintf(format, *args)
   STDERR.printf("pdumpfs: " + format + "\n", *args)
 end
@@ -278,6 +279,10 @@ module QdumpfsUtils
   
   def to_unix_path(path)
     path.gsub(/\\/, '/')
-  end  
+  end
+
+  def to_time(date)
+    Time.local(date.year, date.month, date.day)
+  end
 end
 

data/lib/qdumpfs/version.rb

@@ -1,3 +1,4 @@
-module Qdumpfs
-  VERSION = "0.4.0"
-end
+module Qdumpfs
+  VERSION = "0.8.0"
+end  
+  

data/sample.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+curdir=$(dirname $0)
+bundle exec ruby $curdir/exe/sample
+

data/test_qdumpfs.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+curdir=$(dirname $0)
+bundle exec ruby $curdir/exe/qdumpfs $* ~/_qdumpfs/src ~/_qdumpfs/dst
+
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: qdumpfs
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.8.0
 platform: ruby
 authors:
 - src
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-16 00:00:00.000000000 Z
+date: 2021-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -57,6 +57,7 @@ email:
 - src@srcw.net
 executables:
 - qdumpfs
+- sample
 extensions: []
 extra_rdoc_files: []
 files:
@@ -73,7 +74,10 @@ files:
 - bin/setup
 - build.cmd
 - build.sh
+- data/from/0.txt
+- data/from/a/1.txt
 - exe/qdumpfs
+- exe/sample
 - lib/qdumpfs.rb
 - lib/qdumpfs/option.rb
 - lib/qdumpfs/util.rb
@@ -85,12 +89,14 @@ files:
 - run_qdumpfs.cmd
 - run_qdumpfs.sh
 - run_sync.cmd
+- sample.sh
 - test_pdumpfs/data/bar
 - test_pdumpfs/data/baz/quux
 - test_pdumpfs/data/file
 - test_pdumpfs/data/foo
 - test_pdumpfs/data/secret/secret
 - test_pdumpfs/pdumpfs-test
+- test_qdumpfs.sh
 homepage: https://github.com/src256/qdumpfs
 licenses:
 - MIT