qdumpfs 0.5.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5a18d42979f9ddc74bca2e274dc76b38ab50377398497b244399e9997a693da
-  data.tar.gz: 017acf5a04a798a3eefab1626b2262a8a8fcfe5a10350b4dd8b2ac7003810c03
+  metadata.gz: d4f23500798845b355c0218ddb9fe5245d18bc0fb206a948d272f379adba6f3e
+  data.tar.gz: 1acc58713bd896da322648ac909bbbbd8359831c530e5f160d022d8091fa4adc
 SHA512:
-  metadata.gz: 1dc38b92a65d33b0ebcf52e9db1ba9a6c0940ac7dc1e2270e86ee8f329fbceb466a490dad1f763f0caeaa72e97bfc3156873e6fd5402606e9eb5a2858c82c29f
-  data.tar.gz: 948fd22961accdd955cdb6c66b73c11548be2e6a8bc99fc41fbec23b8ba3cd82ae70df1c21d4653e4ee9cc28b5ec34b53a50b1caf227bd2240d0bef2176edab7
+  metadata.gz: 4e1d6c3155d669e9a17a199b8ea70749615fe116c496c166bc477036c66715814337d1cbb8db562236d5bed755ba3f21bc45600f5d5e2f070f5eb847f9275057
+  data.tar.gz: 2a4deb71d95507c1ddba720899528874df09f0245b9a668a90aca5fd2d4d06472632ee2b5b1b7c78c1a6d5e894c4c77f21ffa09b1734350f2aff697568aede4c

data/.gitignore

@@ -12,4 +12,7 @@ log.txt
 *test.cmd
 test*.cmd
 *.log
+/data/to
+/data/to2
+
 

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    qdumpfs (0.5.1)
+    qdumpfs (1.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    minitest (5.14.2)
-    rake (13.0.1)
+    minitest (5.14.4)
+    rake (13.0.3)
 
 PLATFORMS
   ruby

data/README.md

@@ -1,13 +1,14 @@
 # Qdumpfs
 
-qdumpfs is a modified version of pdumpfs.
+pdumpfsの個人的改良版です。
 
-## Installation
+Gem化して最近のバージョンのバージョンのRubyに対応。コマンドの拡張などを行っています。
 
-gem install qdumpfs
+## インストール
 
-## Usage
+gem install qdumpfs
 
+## 使用方法
 
 ```
 Usage: qdumpfs [options] <source> <dest>
@@ -19,48 +20,53 @@ Options
     -e, --exclude=PATTERN            exclude files/directories matching PATTERN
     -s, --exclude-by-size=SIZE       exclude files larger than SIZE
     -w, --exclude-by-glob=GLOB       exclude files matching GLOB
-    -c, --command=COMMAND            backup|sync|list|expire|verify|test
+    -c, --command=COMMAND            backup|sync|list|expire|verify|delete
     -l, --limit=HOURS                limit hours
     -k, --keep=KEEPARG               ex: --keep 100Y12M12W30D (100years, 12months, 12weeks, 30days, default)
 ```
 
-## Example
+## 実行例
+
+
+バックアップを実行する場合。
 
-Backup your home directory, run the following command.
 ```
 qdumpfs /home/foo /backup
 ```
 
-You can specify command option(default is "backup").
+"--command backup"オプションを明示することもできます。
+
 ```
 qdumpfs --command=backup /home/foo /backup
 ```
 
-Sync two backup directories.
+"--command sync"でバックアップフォルダを同期できます。
 ```
 qdumpfs --command=sync /backup1 /backup2
 ```
 
-Sync two backup directories(limit 1 hours, keep 100Y12M12W30D).
+バックアップフォルダの同期には膨大な時間が必要な場合があるため、実行時間を制限できます。以下は例えば1時間に制限する場合です。
 ```
 qdumpfs --command=sync --limit=1 /backup1 /backup2
 ```
 
-Sync two backup directories(limit 1 hours, keep specified backups only).
+バックアップフォルダの同期で、1時間でかつ"100Y12M12W30D"を保存する場合のオプションです。
 ```
-qdumpfs --command=sync --limit=1 --keep=5Y6M7W10D --keep/backup1 /backup2
+qdumpfs --command=sync --limit=1 --keep=5Y6M7W10D backup1 /backup2
 ```
 
-Expire backup directory.
+"--command expire"で、"--keep="パターンに該当しないバックアップを削除できます。
 ```
-qdumpfs --command=expire --limit=1 --keep=5Y6M7W10D --keep/backup1 /backup2
+qdumpfs --command=expire --limit=1 --keep=5Y6M7W10D backup1 /backup2
 ```
 
+"--command delete"で、バックアップに存在する指定したパスを削除できます(間違えてバックアップした内容を削除したい場合などに使用)。
+```
+qdumpfs --command=delete --delete-dir=backup1 --limit=1  r:/backup2
+```
 
 ## License
 
 qdumpfs is a free software with ABSOLUTELY NO WARRANTY under the terms of the GNU General Public License version 2.
 
 
-
-

data/data/from/0.txt

@@ -0,0 +1 @@
+0

data/data/from/a/1.txt

@@ -0,0 +1 @@
+1

data/exe/sample

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+
+require 'fileutils'
+require "qdumpfs"
+
+from = "./data/from"
+to = "./data/to"
+to2 = "./data/to2"
+
+def create_dir(dir)
+  if FileTest.directory?(dir)
+    FileUtils.rm_rf(dir)
+  end
+  FileUtils.mkpath(dir)
+end
+
+create_dir(to)
+create_dir(to2)
+
+
+arg = ["--backup-at=20210119", from, to]
+Qdumpfs::Command.run(arg)
+arg = ["--backup-at=20210121", from, to]
+Qdumpfs::Command.run(arg)
+arg = ["--backup-at=20210122", from, to]
+Qdumpfs::Command.run(arg)
+arg = ["--backup-at=20210123", from, to]
+Qdumpfs::Command.run(arg)
+arg = ["--backup-at=20210124", from, to]
+Qdumpfs::Command.run(arg)
+
+
+arg = ["--backup-at=20210122", from, to2]
+Qdumpfs::Command.run(arg)
+arg = ["--backup-at=20210123", from, to2]
+
+arg = ["--command=delete", "--delete-from=20210122", "--delete-to=20210123", "--limit=1", to, to2]
+Qdumpfs::Command.run(arg)
+
+

data/lib/qdumpfs.rb

@@ -34,13 +34,26 @@ module Qdumpfs
       }
       opt.on('-s SIZE', '--exclude-by-size=SIZE', 'exclude files larger than SIZE') {|v| opts[:es] = v }
       opt.on('-w GLOB', '--exclude-by-glob=GLOB', 'exclude files matching GLOB') {|v| opts[:ep] = v }
-      commands = ['backup', 'sync', 'list', 'expire', 'verify']
+      commands = ['backup', 'sync', 'list', 'expire', 'verify', 'delete']
       opt.on('-c COMMAND', '--command=COMMAND', commands, commands.join('|')) {|v| opts[:c] = v}
       opt.on('-l HOURS', '--limit=HOURS', 'limit hours') {|v| opts[:limit] = v}
       opt.on('-k KEEPARG', '--keep=KEEPARG', 'ex: --keep 100Y12M12W30D (100years, 12months, 12weeks, 30days, default)') {|v| opts[:keep] = v}
-      opt.on('--logdir LOGDIR', 'logdir') {|v| opts[:logdir] = v}
+      opt.on('--logdir=LOGDIR', 'logdir') {|v| opts[:logdir] = v}
+      opt.on('--delete-from=YYYYMMDD', 'delete backup from YYYY/MM/DD') {|v|
+        opts[:delete_from] = Date.parse(v)
+      }
+      opt.on('--delete-to=YYYYMMDD', 'delete backup to YYYY/MM/DD') {|v|
+        opts[:delete_to] = Date.parse(v)
+      }
+      opt.on('--delete-dir=DIRS', 'relative path from the snapshot root of the backup. ex: --delete-dir=foo,home/bar') {|v|
+        opts[:delete_dirs] = v.split(/,/)
+      }      
+      opt.on('--backup-at=YYYYMMDD', 'backup at YYYY/MM/DD') {|v|
+        opts[:backup_at] = Date.parse(v)
+      }
+      opt.on('-d', '--debug', 'debug mode') {|v| opts[:d] = v }
       opt.parse!(argv)
-      option = Option.new(opts, ARGV)
+      option = Option.new(opts, argv)
       if opts[:v]
         puts "<<<<< qdumpfs options >>>>> "
         puts "logdir: #{option.logdir}"
@@ -52,6 +65,10 @@ module Qdumpfs
         command = Command.new(option)
         command.run
       rescue => e
+        if option.debug
+          p e.message
+          p e.backtrace
+        end
         puts opt.help
         exit        
       end
@@ -69,13 +86,13 @@ module Qdumpfs
       elsif @opt.cmd == 'list'
         list
       elsif @opt.cmd == 'expire'
-        expire
+        delete('expire')
       elsif @opt.cmd == 'verify'
         verify
-#      elsif @opt.cmd == 'test'
-#        test
-#      else
-        raise RuntimeError, "unknown command: #{cmd}"
+      elsif @opt.cmd == 'delete'
+        delete('delete')
+      else
+        raise RuntimeError, "unknown command: #{@opt.cmd}"
       end
     end
     
@@ -120,11 +137,17 @@ module Qdumpfs
     end
     
     def filecount(dir)
-      pscmd = 'Get-ChildItem -Recurse -File | Measure-Object | %{$_.Count}'
-      cmd = "powershell -Command \"#{pscmd}\""
-      result = nil
-      Dir.chdir(dir) do
-        result = `#{cmd}`
+      result = '0'
+      if windows? 
+        pscmd = 'Get-ChildItem -Recurse -File | Measure-Object | %{$_.Count}'
+        cmd = "powershell -Command \"#{pscmd}\""
+        result = nil
+        Dir.chdir(dir) do
+          result = `#{cmd}`
+          result.chomp!
+        end
+      else
+        result = `find #{dir} | wc -l`
         result.chomp!
       end
       result.to_i
@@ -136,29 +159,6 @@ module Qdumpfs
       return src_count, dst_count
     end
 
-    def get_snapshots(target_dir)
-      # 指定したディレクトリに含まれるバックアップフォルダ(日付つき)を全て取得
-      dd   = "[0-9][0-9]"
-      dddd = dd + dd
-      # FIXME: Y10K problem.
-      dirs = []
-      glob_path = File.join(target_dir, dddd, dd, dd)
-      Dir.glob(glob_path).sort.find {|dir|
-        day, month, year = File.split_all(dir).reverse.map {|x| x.to_i }
-        path = dir
-        if File.directory?(path) and Date.valid_date?(year, month, day) and
-          dirs << path
-        end
-      }
-      dirs
-    end
-
-    def get_snapshot_date(snapshot)
-      # バックアップディレクトリのパス(日付つき)から日付を取得して返す
-      day, month, year = File.split_all(snapshot).reverse.map {|x| x.to_i }
-      Time.new(year, month, day)
-    end
-    
     def update_snapshot(src, latest, today)
       # バックアップの差分コピーを実行
       # src: コピー元ディレクトリ ex) i:/from/home
@@ -179,7 +179,7 @@ module Qdumpfs
           # ファイルのアップデート
           update_file(s, l, t)
           dirs[t] = File.stat(s) if File.ftype(s) == "directory"
-        rescue Errno::ENOENT, Errno::EACCES => e
+        rescue => e
           wprintf("%s: %s", src, e.message)
           next
         end
@@ -214,7 +214,7 @@ module Qdumpfs
         end
           chown_if_root(type, s, t)
           dirs[t] = File.stat(s) if File.ftype(s) == "directory"
-        rescue Errno::ENOENT, Errno::EACCES => e
+        rescue => e
           wprintf("%s: %s", s, e.message)
           next
         end
@@ -227,7 +227,7 @@ module Qdumpfs
       
       #コピー元のスナップショット
       src_snapshots = BackupDir.scan_backup_dirs(src)
-      @opt.detect_keep_dirs(src_snapshots)
+      @opt.detect_expire_dirs(src_snapshots)
       
       # コピー先の最新スナップショット
       dst_snapshots = BackupDir.scan_backup_dirs(dst)
@@ -296,6 +296,9 @@ module Qdumpfs
       
       @written_bytes = 0
       start_time = Time.now
+      if @opt.backup_at
+        start_time = to_time(@opt.backup_at)
+      end
       src = @opt.src
       dst = @opt.dst
       
@@ -439,74 +442,80 @@ module Qdumpfs
       file.close 
     end
     
-    def expire
+    def delete(cmd)
       @opt.validate_directories(1)
-      
       start_time = Time.now
       limit_time = start_time + (@opt.limit_sec)
-      log("##### expire start #{fmt(start_time)} => limit_time=#{fmt(limit_time)} #####")
-      
+      log("##### #{cmd} delete-from=#{@opt.delete_from} delete-to=#{@opt.delete_to} start #{fmt(start_time)} => limit_time=#{fmt(limit_time)} #####")
       @opt.dirs.each do |target_dir|
         
         target_start = Time.now        
-        expire_target_dir(target_dir)
+        delete_target_dir(cmd, target_dir)
         target_end = Time.now
         
         # 次回expireにかかる時間を最終expire時間の半分と予想
         next_expire = (target_end - target_start) / 2
         
         cur_time = Time.now
-        in_imit = (cur_time + next_expire) < limit_time
+        in_limit = (cur_time + next_expire) < limit_time
         
         log("## cur_time=#{fmt(cur_time)} + next_expire=#{next_expire} <  limit_time=#{fmt(limit_time)} in_limit=#{in_limit} ## ")
         unless in_limit
           break
         end
       end
-      
-      log("##### expire end #####")
+      log("##### #{cmd} end #####")
     end
-    
-    def expire_target_dir(target_dir)
+
+    def rm_dir(path)
+      can_delete = true
+      if @opt.dry_run
+        can_delete = false
+      elsif !FileTest.directory?(path)
+        can_delete = false
+      end
+      msg = can_delete ? "...ok..." : "...ng..."
+      print "Deleting #{path} #{msg}"
+
+      return unless can_delete
+
+      #それ以外は日付バックアップディレクトリ全体を削除
+      if windows?
+        # Windowsの場合
+        win_backup_path = to_win_path(path)
+        system("rmdir /S /Q #{win_backup_path}")
+      else
+        # Linux/macOSの場合
+        system("rm -rf #{path}")
+      end
+    end
+
+    def delete_target_dir(cmd, target_dir)
       target_dir = to_unix_path(target_dir)
       puts "<<<<< Target dir: #{target_dir} >>>>>"
       
       snapshots = BackupDir.scan_backup_dirs(target_dir)
-      @opt.detect_keep_dirs(snapshots)
-      
-#      p @opt.keep_year
-#      p @opt.keep_month
-#      p @opt.keep_day
-      
+      if cmd == 'expire'
+        @opt.detect_expire_dirs(snapshots)
+      elsif cmd == 'delete'
+        @opt.detect_delete_dirs(snapshots, @opt.delete_from, @opt.delete_to) 
+      else
+        raise RuntimeError, "unknown command: #{cmd}"
+      end
+    
       snapshots.each do |snapshot|
         next if snapshot.keep
         t_start = Time.now
-        print "Deleting #{snapshot.path} ..."
-       
-        unless @opt.dry_run
-          #http://superuser.com/questions/19762/mass-deleting-files-in-windows/289399#289399
-          ##### here
-          #bundle exec ruby exe/qdumpfs --dry-run --keep=100Y36M30W30D --command expire f:/pc1/pdumpfs/users f:/pc1/pdumpfs/opt f:/pc1/pdumpfs/d
-          
-          if windows?
-            # Windowsの場合
-            win_backup_path = to_win_path(snapshot.path)
-            
-#            byenow = "byenow"
-#            if which(byenow)
-#              print " bynow"
-#              system("byenow -y --delete-ntapi --one-liner #{snapshot.path}")
-#            else
-#              print " pass1"
-#              system("del /F /S /Q #{win_backup_path} > nul")
-#              print " pass2"
-#              system("rmdir /S /Q #{win_backup_path}")
-            #            end
-            system("rmdir /S /Q #{win_backup_path}")
-          else
-            # Linux/macOSの場合
-            system("rm -rf #{snapshot.path}")
+
+        if @opt.delete_dirs.size > 0
+          #削除するディレクトリが指定されている場合日付バックアップディレクトリの下そのディレクトリを削除
+          @opt.delete_dirs.each do |delete_dir|
+            delete_path = File.join(snapshot.path, delete_dir)
+            rm_dir(delete_path)
           end
+        else
+          #それ以外はバックアップディレクトリ全体が対象
+          rm_dir(snapshot.path)
         end
         
         t_end = Time.now
@@ -527,8 +536,10 @@ module Qdumpfs
       puts "Keep dirs:"
       snapshots.each do |snapshot|
         puts snapshot.path if snapshot.keep
-      end
-      
+      end      
     end
+
+    
   end
+
 end

data/lib/qdumpfs/option.rb

@@ -139,12 +139,19 @@ module Qdumpfs
       @keep_month = $1.to_i if keep =~ /(\d+)M/
       @keep_week = $1.to_i if keep =~ /(\d+)W/
       @keep_day = $1.to_i if keep =~ /(\d+)D/
+      @delete_from = @opts[:delete_from]
+      @delete_to = @opts[:delete_to]
+      @delete_dirs = @opts[:delete_dirs] || []
+      @backup_at = @opts[:backup_at]
       @today = Date.today
+      @debug = @opts[:d]
     end
     attr_reader :dirs, :src, :dst, :cmd
     attr_reader :keep_year, :keep_month, :keep_week, :keep_day
     attr_reader :logdir, :logpath, :verifypath
     attr_reader :logger, :matcher, :reporter, :interval_proc
+    attr_reader :delete_from, :delete_to, :delete_dirs, :backup_at
+    attr_reader :debug
     
     def report(type, filename)
       if @opts[:v]
@@ -206,12 +213,32 @@ module Qdumpfs
       end
     end
     
-    def detect_keep_dirs(backup_dirs)
+    def detect_expire_dirs(backup_dirs)
       detect_year_keep_dirs(backup_dirs)
       detect_month_keep_dirs(backup_dirs)
       detect_week_keep_dirs(backup_dirs)
       detect_day_keep_dirs(backup_dirs)
     end
+
+    def detect_delete_dirs(backup_dirs, delete_from, delete_to)
+      
+      backup_dirs.each do |backup_dir|
+        backup_dir.keep = true
+        if delete_from && delete_to
+          if backup_dir.date >= delete_from && backup_dir.date <= delete_to
+            backup_dir.keep = false
+          end
+        elsif delete_from
+          if backup_dir.date >= delete_from
+            backup_dir.keep = false
+          end
+        elsif delete_to
+          if backup_dir.date <= delete_to
+            backup_dir.keep = false
+          end
+        end
+      end
+    end
     
     def open_verifyfile
       if FileTest.file?(@verifypath)

data/lib/qdumpfs/util.rb

@@ -1,3 +1,4 @@
+# coding: utf-8
 def wprintf(format, *args)
   STDERR.printf("pdumpfs: " + format + "\n", *args)
 end
@@ -72,14 +73,14 @@ module QdumpfsFind
         yield file.dup.taint
         begin
           s = File.lstat(file)
-        rescue Errno::ENOENT, Errno::EACCES, Errno::ENOTDIR, Errno::ELOOP, Errno::ENAMETOOLONG => e
+        rescue => e
           logger.print("File.lstat path=#{file} error=#{e.message}")
           next
         end
         if s.directory? then
           begin
             fs = Dir.entries(file, :encoding=>'UTF-8')
-          rescue Errno::ENOENT, Errno::EACCES, Errno::ENOTDIR, Errno::ELOOP, Errno::ENAMETOOLONG => e
+          rescue => e
             logger.print("Dir.entries path=#{file} error=#{e.message}")
             next
           end
@@ -106,23 +107,22 @@ module QdumpfsUtils
 
   # We don't use File.copy for calling @interval_proc.
   def copy_file(src, dest)
-    File.open(src, 'rb') {|r|
-      File.open(dest, 'wb') {|w|
-        block_size = (r.stat.blksize or 8192)
-        begin
+    begin    
+      File.open(src, 'rb') {|r|
+        File.open(dest, 'wb') {|w|
+          block_size = (r.stat.blksize or 8192)
           i = 0
           while true
             block = r.sysread(block_size)
             w.syswrite(block)
             i += 1
             @written_bytes += block.size
-#            @interval_proc.call if i % 10 == 0
           end
-        rescue EOFError => e
-          #            puts e.message, e.backtrace
-        end
+        }
       }
-    }
+    rescue EOFError => e
+      # puts e.message, e.backtrace
+    end    
     unless FileTest.file?(dest)
       raise "copy_file fails #{dest}"
     end
@@ -278,6 +278,10 @@ module QdumpfsUtils
   
   def to_unix_path(path)
     path.gsub(/\\/, '/')
-  end  
+  end
+
+  def to_time(date)
+    Time.local(date.year, date.month, date.day)
+  end
 end
 

data/lib/qdumpfs/version.rb

@@ -1,3 +1,4 @@
-module Qdumpfs
-  VERSION = "0.5.1"
-end
+module Qdumpfs
+  VERSION = "1.0.0"
+end  
+  

data/run_list.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+curdir=$(dirname $0)
+bundle exec ruby $curdir/exe/qdumpfs --command=list /Volumes/EXT/
+

data/run_qdumpfs.sh

@@ -1,4 +1,4 @@
 #!/bin/sh
 
 curdir=$(dirname $0)
-bundle exec ruby $curdir/exe/qdumpfs $*
+bundle exec ruby $curdir/exe/qdumpfs "$@"

data/sample.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+curdir=$(dirname $0)
+bundle exec ruby $curdir/exe/sample
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: qdumpfs
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 1.0.0
 platform: ruby
 authors:
 - src
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-17 00:00:00.000000000 Z
+date: 2021-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -57,6 +57,7 @@ email:
 - src@srcw.net
 executables:
 - qdumpfs
+- sample
 extensions: []
 extra_rdoc_files: []
 files:
@@ -73,7 +74,10 @@ files:
 - bin/setup
 - build.cmd
 - build.sh
+- data/from/0.txt
+- data/from/a/1.txt
 - exe/qdumpfs
+- exe/sample
 - lib/qdumpfs.rb
 - lib/qdumpfs/option.rb
 - lib/qdumpfs/util.rb
@@ -82,9 +86,11 @@ files:
 - qdumpfs.gemspec
 - run2_qdumpfs.cmd
 - run_list.cmd
+- run_list.sh
 - run_qdumpfs.cmd
 - run_qdumpfs.sh
 - run_sync.cmd
+- sample.sh
 - test_pdumpfs/data/bar
 - test_pdumpfs/data/baz/quux
 - test_pdumpfs/data/file