qdumpfs 0.4.0

data/lib/qdumpfs/option.rb

@@ -0,0 +1,302 @@
+# coding: utf-8
+module Qdumpfs
+  # 日毎のバックアップフォルダに対応
+  class BackupDir
+    def self.scan_backup_dirs(target_dir)
+      backup_dirs = []
+      Dir.glob("#{target_dir}/[0-9][0-9][0-9][0-9]/[0-1][0-9]/[0-3][0-9]").sort.each do |path|
+        if  File.directory?(path) && path =~ /(\d\d\d\d)\/(\d\d)\/(\d\d)/
+          #        puts "Backup dir: #{path}"
+          backup_dir = BackupDir.new
+          backup_dir.path = path
+          backup_dir.date = Date.new($1.to_i, $2.to_i, $3.to_i)
+          backup_dirs << backup_dir
+        end
+      end
+      backup_dirs
+    end
+    
+    def self.find(backup_dirs, from_date, to_date)
+      backup_dirs.select{|backup_dir| backup_dir.date >= from_date && backup_dir.date <= to_date}
+    end
+    
+    def initialize
+      @keep = false
+    end
+    attr_accessor :path, :date, :keep
+  end
+  
+  
+  class NullLogger
+    def close
+    end    
+    def print(msg)
+    end
+  end
+  
+  
+  class SimpleLogger
+    def initialize(filename)
+      @file = File.open(filename, "a")
+    end
+    
+    def close
+      @file.close
+    end
+    
+    def print(msg)
+      @file.puts msg
+    end
+  end
+  
+  
+  class Error < StandardError
+  end
+  
+  
+  class NullMatcher
+    def initialize(options = {})
+    end
+    def exclude?(path)
+      false
+    end
+  end
+  
+  
+  class FileMatcher
+    def initialize(options = {})
+      @patterns = options[:patterns] || []
+      @globs    = options[:globs] || []
+      @size     = calc_size(options[:size])
+    end
+    
+    def calc_size(size)
+      table   = { "K" => 1, "M" => 2, "G" => 3, "T" => 4, "P" => 5 }
+      pattern = table.keys.join('')
+      case size
+      when nil
+        -1
+      when /^(\d+)([#{pattern}]?)$/i
+        num  = Regexp.last_match[1].to_i
+        unit = Regexp.last_match[2]
+        num * 1024 ** (table[unit] or 0)
+      else
+        raise "Invalid size: #{size}"
+      end
+    end
+    
+    def exclude?(path)
+      stat = File.lstat(path)
+      if @size >= 0 and stat.file? and stat.size >= @size
+        return true
+      elsif @patterns.find {|pattern| pattern.match(path) }
+        return true
+      elsif stat.file? and
+        @globs.find {|glob| File.fnmatch(glob, File.basename(path)) }
+        return true
+      end
+      return false
+    end
+  end
+
+ 
+  class Option
+    def initialize(opts, dirs)
+      @opts = opts
+      @dirs = dirs
+      @src = dirs[0] if dirs.size > 0
+      @dst = dirs[1] if dirs.size > 1
+      @cmd = @opts[:c] || 'backup'
+      
+      #      @logger = NullLogger.new
+      logfile = @opts[:logfile] || 'log.txt'
+      #ログディレクトリの作成      
+      @logdir = File.expand_path('.')
+      Dir.mkdir(@logdir) unless FileTest.directory?(@logdir)
+      @logpath = File.join(@logdir, logfile)
+      @logger = SimpleLogger.new(@logpath)
+      
+      verifyfile = 'verify.txt'
+      @verifypath = File.join(@logdir, verifyfile)
+
+      @matcher = NullMatcher.new
+      
+      size = @opts[:es]
+      globs = @opts[:eg]
+      patterns = @opts[:ep]
+      if size || globs || patterns
+        @matcher = FileMatcher.new(:size => size, :globs => globs, :patterns => patterns)
+      end      
+      
+      # 同期用のオプションは日常使いのpdumpfs-cleanのオプションより期間短めに設定      
+      @limit = @opts[:limit]
+      @keep_year = 100
+      @keep_month = 12
+      @keep_week = 12
+      @keep_day = 30
+      keep = @opts[:keep]
+      @keep_year = $1.to_i if keep =~ /(\d+)Y/
+      @keep_month = $1.to_i if keep =~ /(\d+)M/
+      @keep_week = $1.to_i if keep =~ /(\d+)W/
+      @keep_day = $1.to_i if keep =~ /(\d+)D/
+      @today = Date.today
+    end
+    attr_reader :dirs, :src, :dst, :cmd
+    attr_reader :keep_year, :keep_month, :keep_week, :keep_day
+    attr_reader :logdir, :logfile, :verifyfile
+    attr_reader :logger, :matcher, :reporter, :interval_proc
+    
+    def report(type, filename)
+      if @opts[:v]
+        stat = File.stat(filename)
+        size = stat.size
+        format_size = convert_bytes(size)
+        msg = format_report_with_size(type, filename, size, format_size)        
+      elsif @opts[:r]
+        if type =~ /^new_file/
+          stat = File.stat(filename)
+          size = stat.size
+          format_size = convert_bytes(size)
+          msg = format_report_with_size(type, filename, size, format_size)
+        end
+      else
+        # 何も指定されていない場合
+        if type == 'new_file'
+          msg = format_report(type, filename)
+        end
+      end
+      log(msg)
+    end
+    
+    def log(msg, console = true)
+      return if (msg.nil? || msg == '')
+      puts msg if console
+      @logger.print(msg)
+    end
+    
+    def dry_run
+      @opts[:n]
+    end
+    
+    def limit_sec
+      @limit.to_i * 3600
+    end
+    
+    def validate_directory(dir)
+      if dir.nil? || !File.directory?(dir)
+        raise ArgumentError, "No such directory: #{dir}"
+      end
+    end
+      
+    def validate_directories(min_count)
+      @dirs.each do |dir|
+        validate_directory(dir)
+      end
+      if @dirs.size == 2 && windows?
+        # ディレクトリが2つだけ指定されている場合、コピー先はntfsである必要がある
+        unless ntfs?(dst)
+          fstype = get_filesystem_type(dst)
+          raise sprintf("only NTFS is supported but %s is %s.", dst, fstype)
+        end      
+      end
+      if @dirs.size < min_count
+        raise "#{min_count} directories required."
+      end
+    end
+    
+    def detect_keep_dirs(backup_dirs)
+      detect_year_keep_dirs(backup_dirs)
+      detect_month_keep_dirs(backup_dirs)
+      detect_week_keep_dirs(backup_dirs)
+      detect_day_keep_dirs(backup_dirs)
+    end
+    
+    def open_verifyfile
+      if FileTest.file?(@verifypath)
+        File.unlink(@verifypath)
+      end
+      File.open(@verifypath, 'a')
+    end
+    
+    def open_listfile
+      filename = File.join(@logdir, "list_" + @src.gsub(/[:\/]/, '_') + '.txt')
+      if FileTest.file?(filename)
+        File.unlink(filename)
+      end
+      File.open(filename, 'a')
+    end
+        
+    private
+    def format_report(type, filename)
+      sprintf("%-12s %s\n", type, filename)
+    end
+    
+    def format_report_with_size(type, filename, size, format_size)
+      sprintf("%s\t%s\t%d\t%s\n", type, filename, size, format_size)
+    end
+    
+    def format_report_with_size_as_csv(type, filename, size, format_size)
+      sprintf("%s,%s,%d,%s\n", type, filename.encode('cp932', undef: :replace), size, format_size)
+    end
+    
+    def format_report_as_csv(type, filename)
+      sprintf("%s,%s\n", type, filename.encode('cp932', undef: :replace))
+    end
+    
+    def convert_bytes(bytes)
+      if bytes < 1024
+        sprintf("%dB", bytes)
+      elsif bytes < 1024 * 1000 # 1000kb
+        sprintf("%.1fKB", bytes.to_f / 1024)
+      elsif bytes < 1024 * 1024 * 1000  # 1000mb
+        sprintf("%.1fMB", bytes.to_f / 1024 / 1024)
+      else
+        sprintf("%.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+      end
+    end
+        
+    def keep_dirs(backup_dirs, num)
+      num.downto(0) do |i|
+        from_date, to_date = yield(i)
+        dirs = BackupDir.find(backup_dirs, from_date, to_date)
+        dirs[0].keep = true if dirs.size > 0
+      end
+    end
+  
+    def detect_year_keep_dirs(backup_dirs)
+      keep_dirs(backup_dirs, @keep_year) do |i|
+      from_date = Date.new(@today.year - i, 1, 1)
+        to_date = Date.new(@today.year - i, 12, 31)
+        [from_date, to_date]
+      end
+    end
+    
+    def detect_month_keep_dirs(backup_dirs)
+      keep_dirs(backup_dirs, @keep_month) do |i|
+        base_date = @today <<  i
+        from_date = Date.new(base_date.year, base_date.month, 1)
+        to_date = from_date >> 1
+        [from_date, to_date]
+      end
+    end
+  
+    def detect_week_keep_dirs(backup_dirs)
+      keep_dirs(backup_dirs, @keep_week) do |i|
+        base_date = @today  - 7 * i
+        from_date = base_date - base_date.cwday # 1
+        to_date = from_date + 6
+        [from_date, to_date]
+      end
+    end
+  
+    def detect_day_keep_dirs(backup_dirs)
+      keep_dirs(backup_dirs, @keep_day) do |i|
+        base_date = @today - i
+        from_date = base_date
+        to_date = base_date
+        [from_date, to_date]
+      end
+    end
+
+  end
+end

data/lib/qdumpfs/util.rb

@@ -0,0 +1,283 @@
+def wprintf(format, *args)
+  STDERR.printf("pdumpfs: " + format + "\n", *args)
+end
+
+#https://stackoverflow.com/questions/2108727/which-in-ruby-checking-if-program-exists-in-path-from-ruby
+# Cross-platform way of finding an executable in the $PATH.
+#
+#   which('ruby') #=> /usr/bin/ruby
+def which(cmd)
+  exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
+  ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
+    exts.each do |ext|
+      exe = File.join(path, "#{cmd}#{ext}")
+      return exe if File.executable?(exe) && !File.directory?(exe)
+    end
+  end
+  nil
+end
+
+class File
+  def self.real_file?(path)
+    FileTest.file?(path) and not FileTest.symlink?(path)
+  end
+
+  def self.anything_exist?(path)
+    FileTest.exist?(path) or FileTest.symlink?(path)
+  end
+
+  def self.real_directory?(path)
+    FileTest.directory?(path) and not FileTest.symlink?(path)
+  end
+
+  def self.force_symlink(src, dest)
+    begin
+      File.unlink(dest) if File.anything_exist?(dest)
+      File.symlink(src, dest)
+    rescue 
+    end
+  end
+
+  def self.force_link(src, dest)
+    File.unlink(dest) if File.anything_exist?(dest)
+    File.link(src, dest)
+  end
+
+  def self.readable_file?(path)
+    FileTest.file?(path) and FileTest.readable?(path)
+  end
+
+  def self.split_all(path)
+    parts = []
+    while true
+      dirname, basename = File.split(path)
+      break if path == dirname
+      parts.unshift(basename) unless basename == "."
+      path = dirname
+    end
+    return parts
+  end
+end
+
+
+module QdumpfsFind
+  def find(logger, *paths)
+    block_given? or return enum_for(__method__, *paths)
+    paths.collect!{|d|
+      raise Errno::ENOENT unless File.exist?(d);
+      d.dup
+    }
+    while file = paths.shift
+      catch(:prune) do
+        yield file.dup.taint
+        begin
+          s = File.lstat(file)
+        rescue Errno::ENOENT, Errno::EACCES, Errno::ENOTDIR, Errno::ELOOP, Errno::ENAMETOOLONG => e
+          logger.print("File.lstat path=#{file} error=#{e.message}")
+          next
+        end
+        if s.directory? then
+          begin
+            fs = Dir.entries(file, :encoding=>'UTF-8')
+          rescue Errno::ENOENT, Errno::EACCES, Errno::ENOTDIR, Errno::ELOOP, Errno::ENAMETOOLONG => e
+            logger.print("Dir.entries path=#{file} error=#{e.message}")
+            next
+          end
+          fs.sort!
+          fs.reverse_each {|f|
+            next if f == "." or f == ".."
+            f = File.join(file, f)
+            paths.unshift f.untaint
+          }
+        end
+      end
+    end
+  end
+  
+  def prune
+    throw :prune
+  end
+  
+  module_function :find, :prune
+end
+
+
+module QdumpfsUtils
+
+  # We don't use File.copy for calling @interval_proc.
+  def copy_file(src, dest)
+    File.open(src, 'rb') {|r|
+      File.open(dest, 'wb') {|w|
+        block_size = (r.stat.blksize or 8192)
+        begin
+          i = 0
+          while true
+            block = r.sysread(block_size)
+            w.syswrite(block)
+            i += 1
+            @written_bytes += block.size
+#            @interval_proc.call if i % 10 == 0
+          end
+        rescue EOFError => e
+          #            puts e.message, e.backtrace
+        end
+      }
+    }
+    unless FileTest.file?(dest)
+      raise "copy_file fails #{dest}"
+    end
+  end 
+  
+  # incomplete substitute for cp -p
+  def copy(src, dest)
+    stat = File.stat(src)
+    copy_file(src, dest)
+    File.chmod(0200, dest) if windows?
+    File.utime(stat.atime, stat.mtime, dest)
+    File.chmod(stat.mode, dest) # not necessary. just to make sure
+  end
+  
+  def convert_bytes(bytes)
+    if bytes < 1024
+      sprintf("%dB", bytes)
+    elsif bytes < 1024 * 1000 # 1000kb
+      sprintf("%.1fKB", bytes.to_f / 1024)
+    elsif bytes < 1024 * 1024 * 1000  # 1000mb
+      sprintf("%.1fMB", bytes.to_f / 1024 / 1024)
+    else
+      sprintf("%.1fGB", bytes.to_f / 1024 / 1024 / 1024)
+    end
+  end
+  
+  def same_file?(f1, f2)
+    #    File.real_file?(f1) and File.real_file?(f2) and
+    #    File.size(f1) == File.size(f2) and File.mtime(f1) == File.mtime(f2)
+    real_file = File.real_file?(f1) and File.real_file?(f2)
+    same_size = File.size(f1) == File.size(f2)
+    
+    #    mtime1 = File.mtime(f1).strftime('%F %T.%N')
+    #    mtime2 = File.mtime(f2).strftime('%F %T.%N')
+    same_mtime = File.mtime(f1).to_i == File.mtime(f2).to_i
+    #    p "#{real_file} #{same_size} #{same_mtime}(#{mtime1}<=>#{mtime2})"
+    real_file and same_size and same_mtime
+  end
+  
+  def detect_type(src, latest = nil)
+    type = "unsupported"
+    if File.real_directory?(src)
+      type = "directory"
+    else
+      if latest and File.real_file?(latest)
+        case File.ftype(src)
+        when "file"
+          same_file = same_file?(src, latest)
+          #          p "same_file? #{src} #{latest} result=#{same_file}"
+          if same_file
+            type = "unchanged"
+          else
+            type = "updated"
+          end
+        when "link"
+          # the latest backup file is a real file but the
+          # current source file is changed to symlink.
+          type = "symlink"
+        end
+      else
+        case File.ftype(src)
+        when "file"
+          type = "new_file"
+        when "link"
+          type = "symlink"
+          end
+      end
+    end
+    return type
+  end
+  
+  def fmt(time)
+    time.strftime('%Y/%m/%d %H:%M:%S')
+  end
+  
+  def chown_if_root(type, src, today)
+    return unless Process.uid == 0 and type != "unsupported"
+    if type == "symlink"
+      if File.respond_to?(:lchown)
+        stat = File.lstat(src)
+        File.lchown(stat.uid, stat.gid, today)
+      end
+    else
+      stat = File.stat(src)
+      File.chown(stat.uid, stat.gid, today)
+    end
+  end
+  
+  def restore_dir_attributes(dirs)
+    dirs.each {|dir, stat|
+      File.utime(stat.atime, stat.mtime, dir)
+      File.chmod(stat.mode, dir)
+    }
+  end    
+  
+  def make_relative_path(path, base)
+    pattern = sprintf("^%s%s?", Regexp.quote(base), File::SEPARATOR)
+    path.sub(Regexp.new(pattern), "")
+  end
+  
+  def fputs(file, msg)
+    puts msg
+    file.puts msg
+  end
+  
+  def time_diff(start_time, end_time)
+    seconds_diff = (start_time - end_time).to_i.abs
+    
+    hours = seconds_diff / 3600
+    seconds_diff -= hours * 3600
+    
+    minutes = seconds_diff / 60
+    seconds_diff -= minutes * 60
+    
+    seconds = seconds_diff
+    
+    '%02d:%02d:%02d' % [hours, minutes, seconds]
+  end
+  
+  def create_latest_symlink(dest, today)
+    # 最新のバックアップに"latest"というシンボリックリンクをはる(Windowsだと動かない)
+    latest_day = File.dirname(make_relative_path(today, dest))
+      latest_symlink = File.join(dest, "latest")
+    #      puts "force_symlink #{latest_day} #{latest_symlink}"
+    File.force_symlink(latest_day, latest_symlink)
+  end
+  
+  def same_directory?(src, dest)
+    src  = File.expand_path(src)
+    dest = File.expand_path(dest)
+    return src == dest
+  end
+  
+  def sub_directory?(src, dest)
+    src  = File.expand_path(src)
+    dest = File.expand_path(dest)
+    src  += File::SEPARATOR unless /#{File::SEPARATOR}$/.match(src)
+    return /^#{Regexp.quote(src)}/.match(dest)
+  end
+  
+  def datedir(date)
+    s = File::SEPARATOR
+    sprintf "%d%s%02d%s%02d", date.year, s, date.month, s, date.day
+  end
+  
+  def past_date?(year, month, day, t)
+    ([year, month, day] <=> [t.year, t.month, t.day]) < 0
+  end
+  
+  def to_win_path(path)
+    path.gsub(/\//, '\\')
+  end
+  
+  def to_unix_path(path)
+    path.gsub(/\\/, '/')
+  end  
+end
+