mclone 0.1.1 → 0.2.0

data/bin/mclone

@@ -13,7 +13,9 @@ Clamp do
 
   using Refinements
 
-  option ['-f', '--force'], :flag, 'Insist on potentially dangerous activities', default: false
+  self.default_subcommand = 'info'
+
+  option ['-f', '--force'], :flag, 'Insist on potentially dangerous actions', default: false
   option ['-n', '--dry-run'], :flag, 'Simulation mode with no on-disk modifications', default: false
   option ['-v', '--verbose'], :flag, 'Verbose operation', default: false
 
@@ -46,9 +48,7 @@ Clamp do
       $stdout.puts
       $stdout.puts '## Volumes'
       $stdout.puts
-      s.volumes.each do |volume|
-        $stdout.puts "* [#{volume.id}] :: (#{volume.root})"
-      end
+      s.volumes.each { |volume| $stdout.puts "* [#{volume.id}] :: (#{volume.root})" }
       stales = []
       intacts = []
       intact_tasks = s.intact_tasks
@@ -56,7 +56,8 @@ Clamp do
         ts = (t = intact_tasks[task]).nil? ? "<#{task.id}>" : "[#{task.id}]"
         svs = s.volumes.volume(task.source_id).nil? ? "<#{task.source_id}>" : "[#{task.source_id}]"
         dvs = s.volumes.volume(task.destination_id).nil? ? "<#{task.destination_id}>" : "[#{task.destination_id}]"
-        xs = ["* #{ts} :: #{task.mode} #{svs}(#{task.source_root}) -> #{dvs}(#{task.destination_root})"]
+        crypter_mode = task.crypter_mode.nil? ? nil : "#{task.crypter_mode}+"
+        xs = ["* #{ts} :: #{crypter_mode}#{task.mode} #{svs}(#{task.source_root}) -> #{dvs}(#{task.destination_root})"]
         xs << "include #{task.include}" unless task.include.nil? || task.include.empty?
         xs << "exclude #{task.exclude}" unless task.exclude.nil? || task.exclude.empty?
         (t.nil? ? stales : intacts) << xs.join(' :: ')
@@ -94,21 +95,35 @@ Clamp do
 
   end
 
-  subcommand ['task'], 'Task operations' do
+  subcommand 'task', 'Task operations' do
 
     def self.set_task_opts
       modes = Task::MODES.collect(&:to_s).join(' | ')
       option ['-m', '--mode'], 'MODE', "Operation mode (#{modes})", default: Task::MODES.first.to_s
-      option ['-i', '--include'], 'PATTERN', 'Include paths pattern', default: '**'
+      option ['-i', '--include'], 'PATTERN', 'Include paths pattern'
       option ['-x', '--exclude'], 'PATTERN', 'Exclude paths pattern'
     end
 
     subcommand ['new', 'create'], 'Create new SOURCE -> DESTINATION task' do
       set_task_opts
+      option ['-d', '--decrypt'], :flag, 'Decrypt source'
+      option ['-e', '--encrypt'], :flag, 'Encrypt destination'
+      option ['-p', '--password'], 'PASSWORD', 'Plain text password'
+      option ['-t', '--token'], 'TOKEN', 'Rclone crypt token (obscured password)'
       parameter 'SOURCE', 'Source path'
       parameter 'DESTINATION', 'Destination path'
       def execute
-        session.create_task!(source, destination, mode: resolve_mode(mode), include: include, exclude: exclude).commit!
+        crypter_mode = nil
+        signal_usage_error 'choose either encryption or decryption mode, not both' if decrypt? && encrypt?
+        signal_usage_error 'specify either plain text password or Rclone crypt token, not both' if !password.nil? && !token.nil?
+        crypter_mode = :encrypt if encrypt?
+        crypter_mode = :decrypt if decrypt?
+        session.create_task!(
+          resolve_mode(mode),
+          source,
+          destination,
+  include: include, exclude: exclude, crypter_mode: crypter_mode, crypter_password: password, crypter_token: token
+        ).commit!
       end
     end
 

data/lib/mclone.rb

@@ -3,7 +3,9 @@
 
 require 'date'
 require 'json'
+require 'open3'
 require 'fileutils'
+require 'shellwords'
 require 'securerandom'
 
 
@@ -11,13 +13,11 @@ require 'securerandom'
 module Mclone
 
 
-  VERSION = '0.1.1'
+  VERSION = '0.2.0'
 
 
   #
-  class Error < StandardError
-
-  end
+  class Error < StandardError; end
 
   #
   module Refinements
@@ -41,6 +41,11 @@ module Mclone
       end
     end
 
+    refine ::String do
+      def escape
+        Mclone.windows? && %r![^\w\-\=\\\/:]!.match?(self) ? %("#{self}") : shellescape
+      end
+    end
   end
 
 
@@ -67,12 +72,31 @@ module Mclone
       @objects.empty?
     end
 
+    #
+    def size
+      @objects.size
+    end
+
     def initialize
       @ids = {} # { id => object }
       @objects = {} # { object => object }
       @modified = false
     end
 
+    protected attr_reader :objects
+
+    #
+    def hash
+      @objects.hash
+    end
+
+    #
+    def eql?(other)
+      equal?(other) || objects == other.objects
+    end
+
+    alias == eql?
+
     # Return ID of the object considered equal to the specified obj or nil
     def id(obj)
       @objects[obj]&.id
@@ -156,6 +180,9 @@ module Mclone
     #
     attr_reader :include, :exclude
 
+    #
+    attr_reader :crypter_mode
+
     def hash
       @hash ||= source_id.hash ^ destination_id.hash ^ source_root.hash ^ destination_root.hash
     end
@@ -172,85 +199,139 @@ module Mclone
     alias == eql?
 
     #
-    def initialize(source_id, source_root, destination_id, destination_root)
+    def initialize(session, mode, source_id, source_root, destination_id, destination_root, include: nil, exclude: nil, crypter_mode: nil, crypter_token: nil, crypter_password: nil)
       @touch = false # Indicates that the time stamp should be updated whenever state of self is altered
+      @session = session
       @id = SecureRandom.hex(4)
       @source_id = source_id
       @destination_id = destination_id
       @source_root = source_root
       @destination_root = destination_root
-      self.mode = :update
-      self.include = '**'
-      self.exclude = ''
+      self.mode = mode
+      self.include = include
+      self.exclude = exclude
+      set_crypter_mode crypter_mode
+      unless crypter_mode.nil?
+        raise(Task::Error, %(either Rclone crypt token or plain text password is expected, not both)) if !crypter_token.nil? && !crypter_password.nil?
+        @assigned_token = register_crypter_token crypter_token
+        @assigned_password = crypter_password
+      end
     ensure
       @touch = true
       touch!
     end
 
+    CRYPTER_MODES = %i[encrypt decrypt].freeze
+
+    @@crypter_tokens = {}
+
+    private def set_crypter_mode(mode)
+      @crypter_mode = mode.nil? ? nil : (CRYPTER_MODES.include?(mode = mode.intern) ? mode : raise(Task::Error, %(unknown crypt mode "#{mode}")))
+    end
+
+    private def register_crypter_token(token)
+      unless token.nil?
+        @@crypter_tokens[id] = @@crypter_tokens[id].nil? ? token : raise(Task::Error, %(attempt to re-register token for task "#{id}"))
+      end
+      token
+    end
+
+    # Lazily determine the crypt token from either assigned values or the token repository
+    def crypter_token
+      # Locally assigned token takes precedence over the repository's
+      unless @assigned_token.nil?
+        @@crypter_tokens[id] = @assigned_token unless @@crypter_tokens[id].nil? # Assign repository entry with this local token if not yet assigned
+        @assigned_token
+      else
+        if @@crypter_tokens[id].nil?
+          # If token is neither locally assigned nor in repository, try to construct it from the user-supplied password
+          # If a user-supplied password is omitted, create a new randomly generated password
+          args = [Mclone.rclone, 'obscure', @assigned_password.nil? ? SecureRandom.alphanumeric(16) : @assigned_password]
+          $stdout << args.collect(&:escape).join(' ') << "\n" if @session.verbose?
+          stdout, status = Open3.capture2(*args)
+          raise(Task::Error, %(Rclone execution failure)) unless status.success?
+          @@crypter_tokens[id] = stdout.strip
+        else
+          @@crypter_tokens[id]
+        end
+      end
+    end
+
     #
-    MODES = [:update, :synchronize, :copy, :move].freeze
+    MODES = %i[update synchronize copy move].freeze
 
     #
     def mode=(mode)
-      unless mode.nil?
-        @mode = MODES.include?(mode = mode.intern) ? mode : raise(Task::Error, %(unknown mode "#{mode}"))
-        touch!
-      end
+      @mode = MODES.include?(mode = mode.intern) ? mode : raise(Task::Error, %(unknown mode "#{mode}"))
+      touch!
+      mode
     end
 
     #
     def include=(mask)
-      unless mask.nil?
-        @include = mask # TODO extensive verification
-        touch!
-      end
+      @include = mask.nil? || mask == '-' ? nil : mask # TODO: verify mask
+      touch!
+      mask
     end
 
     #
     def exclude=(mask)
-      unless mask.nil?
-        @exclude = mask # TODO extensive verification
-        touch!
-      end
+      @exclude = mask.nil? || mask == '-' ? nil : mask # TODO: verify mask
+      touch!
+      mask
     end
 
     #
-    def self.restore(hash)
+    def self.restore(session, hash)
       obj = allocate
-      obj.send(:from_h, hash)
+      obj.send(:from_h, session, hash)
       obj
     end
 
     #
-    private def from_h(hash)
-      initialize(hash.extract(:source, :volume), hash.extract(:source, :root), hash.extract(:destination, :volume), hash.extract(:destination, :root))
+    private def from_h(session, hash)
+      @session = session
       @touch = false
       @id = hash.extract(:task)
+      @mtime = DateTime.parse(hash.extract(:mtime)) rescue DateTime.now # Deleting mtime entry from json can be used to modify data out of mclone
+      @source_id = hash.extract(:source, :volume)
+      @destination_id = hash.extract(:destination, :volume)
+      @source_root = hash.dig(:source, :root)
+      @destination_root = hash.dig(:destination, :root)
       self.mode = hash.extract(:mode)
-      # Deleting the mtime key from json
-      @mtime = DateTime.parse(hash.extract(:mtime)) rescue @mtime
-      self.include = hash.extract(:include) rescue nil
-      self.exclude = hash.extract(:exclude) rescue nil
+      self.include = hash.dig(:include)
+      self.exclude = hash.dig(:exclude)
+      set_crypter_mode hash.dig(:crypter, :mode)
+      @assigned_token = register_crypter_token(hash.dig(:crypter, :token)) unless crypter_mode.nil?
     ensure
       @touch = true
     end
 
     #
-    def to_h
-      {
-        mode: mode,
-        include: include,
-        exclude: exclude,
+    def to_h(volume)
+      hash = {
         task: id,
-        source: {volume: source_id, root: source_root},
-        destination: {volume: destination_id, root: destination_root},
-        mtime: mtime
+        mode: mode,
+        mtime: mtime,
+        source: { volume: source_id },
+        destination: { volume: destination_id }
       }
+      hash[:source][:root] = source_root unless source_root.nil? || source_root.empty?
+      hash[:destination][:root] = destination_root unless destination_root.nil? || destination_root.empty?
+      hash[:include] = include unless include.nil?
+      hash[:exclude] = exclude unless exclude.nil?
+      unless crypter_mode.nil?
+        crypter = hash[:crypter] = { mode: crypter_mode }
+        # Make sure the token won't get into the encrypted volume's task
+        crypter[:token] = crypter_token if (crypter_mode == :encrypt && source_id == volume.id) || (crypter_mode == :decrypt && destination_id == volume.id)
+      end
+      hash
     end
 
     #
     def touch!
       @mtime = DateTime.now if @touch
+      self
     end
   end
 
@@ -270,12 +351,9 @@ module Mclone
     #
     def resolve(id)
       case (ids = super).size
-      when 0
-        raise(Task::Error, %(no task matching "#{id}" pattern found))
-      when 1
-        ids.first
-      else
-        raise(Task::Error, %(ambiguous "#{id}" pattern: two or more tasks match))
+      when 0 then raise(Task::Error, %(no task matching "#{id}" pattern found))
+      when 1 then ids.first
+      else raise(Task::Error, %(ambiguous "#{id}" pattern: two or more tasks match))
       end
     end
 
@@ -302,8 +380,6 @@ module Mclone
     #
     attr_reader :file
 
-    #
-    attr_reader :tasks
 
     #
     def root
@@ -311,31 +387,30 @@ module Mclone
     end
 
     #
-    def initialize(file)
-      @file = file
+    def initialize(session, file)
+      @loaded_tasks = ObjectSet.new
       @id = SecureRandom.hex(4)
-      @tasks = VolumeTaskSet.new(self)
+      @session = session
+      @file = file
     end
 
     #
-    def self.restore(file)
+    def self.restore(session, file)
       obj = allocate
-      obj.send(:from_file, file)
+      obj.send(:from_file, session, file)
       obj
     end
 
     #
-    private def from_file(file)
-      initialize(file)
+    private def from_file(session, file)
       hash = JSON.parse(IO.read(file), symbolize_names: true)
+      @loaded_tasks = ObjectSet.new
+      @id = hash.extract(:volume)
+      @session = session
+      @file = file
       raise(Volume::Error, %(unsupported Mclone volume format version "#{version}")) unless hash.extract(:mclone) == VERSION
-      @id = hash.extract(:volume).to_s
-      hash.extract(:tasks).each { |t| @tasks << Task.restore(t) }
-    end
-
-    #
-    def dirty?
-      tasks.modified?
+      hash.dig(:tasks)&.each { |h| session.tasks << (@loaded_tasks << Task.restore(@session, h)) }
+      self
     end
 
     #
@@ -348,23 +423,43 @@ module Mclone
       equal?(other) || id == other.id
     end
 
+    #
+    def modified?
+      # Comparison against the original loaded tasks set allows to account for task removals
+      (ts = tasks).modified? || ts != @loaded_tasks
+    end
+
     #
     def commit!(force = false)
-      if force || dirty?
-        open(file, 'w') do |stream|
-          stream << JSON.pretty_generate(to_h)
-          tasks.commit!
+      if force || @session.force? || modified?
+        # As a safeguard against malformed volume files generation, first write to a new file
+        # and rename it to a real volume file only in case of normal turn of events
+        _file = "#{file}~"
+        begin
+          open(_file, 'w') do |stream|
+            stream << JSON.pretty_generate(to_h)
+            tasks.commit!
+          end
+          FileUtils.mv(_file, file, force: true)
+        ensure
+          FileUtils.rm_f(_file)
         end
       end
+      self
+    end
+
+    #
+    def tasks
+      TaskSet.new(self).merge!(@session.tasks)
     end
 
     #
     def to_h
-      {mclone: VERSION, volume: id, tasks: tasks.collect(&:to_h)}
+      { mclone: VERSION, volume: id, tasks: tasks.collect { |task| task.to_h(self) } }
     end
 
     # Volume-bound set of tasks belonging to the specific volume
-    class VolumeTaskSet < Mclone::TaskSet
+    class TaskSet < Mclone::TaskSet
 
       def initialize(volume)
         @volume = volume
@@ -388,12 +483,9 @@ module Mclone
     #
     def resolve(id)
       case (ids = super).size
-      when 0
-        raise(Volume::Error, %(no volume matching "#{id}" pattern found))
-      when 1
-        ids.first
-      else
-        raise(Volume::Error, %(ambiguous "#{id}" pattern: two or more volumes match))
+      when 0 then raise(Volume::Error, %(no volume matching "#{id}" pattern found))
+      when 1 then ids.first
+      else raise(Volume::Error, %(ambiguous "#{id}" pattern: two or more volumes match))
       end
     end
 
@@ -429,31 +521,33 @@ module Mclone
     #
     attr_writer :simulate, :verbose, :force
 
+    #
+    attr_reader :tasks
+
     #
     def initialize
       @volumes = VolumeSet.new
+      @tasks = SessionTaskSet.new(self)
     end
 
     #
     def format_volume!(dir)
       mclone = File.join(dir, Volume::FILE)
       raise(Session::Error, %(refuse to overwrite existing Mclone volume file "#{mclone}")) if File.exist?(mclone) && !force?
-      volume = Volume.new(mclone)
-      @volumes << volume
-      volume.commit!(true) unless simulate? # Force creating a new (empty) volume
+      volumes << (volume = Volume.new(self, mclone))
+      volume.commit!(true) unless simulate? # Force creation of a new (empty) volume
       self
     end
 
     #
     def restore_volume!(dir)
-      volume = Volume.restore(File.join(dir, Volume::FILE))
-      @volumes << volume
+      volumes << Volume.restore(self, File.join(dir, Volume::FILE))
       self
     end
 
     #
     def restore_volumes!
-      (Mclone.environment_mounts + Mclone.system_mounts).each { |dir| restore_volume!(dir) rescue Errno::ENOENT }
+      (Mclone.environment_mounts + Mclone.system_mounts + [ENV['HOME']]).each { |dir| restore_volume!(dir) rescue Errno::ENOENT }
       self
     end
 
@@ -467,84 +561,83 @@ module Mclone
     end
 
     #
-    def create_task!(source, destination, mode: :update, include: '**', exclude: '')
-      task = Task.new(*locate(source), *locate(destination))
-      task.mode = mode
-      task.include = include
-      task.exclude = exclude
-      volumes.each do |volume|
-        t = volume.tasks[task]
-        raise(Session::Error, %(refuse to overwrite existing task "#{t.id}")) unless t.nil? || force?
-        volume.tasks << task # It is a volume's responsibility to collect appropriate tasks, see Volume::TaskSet#<<
-      end
+    def create_task!(mode, source, destination, **kws)
+      task = Task.new(self, mode, *locate(source), *locate(destination), **kws)
+      _task = tasks[task]
+      raise(Session::Error, %(refuse to overwrite existing task "#{_task.id}")) unless _task.nil? || force?
+      tasks << task
       self
     end
 
     #
-    def modify_task!(id, mode:, include:, exclude:)
+    def modify_task!(id, mode: nil, include: nil, exclude: nil)
       ts = tasks
       task = ts.task(ts.resolve(id)).clone
-      task.mode = mode
-      task.include = include
-      task.exclude = exclude
-      volumes.each { |volume| volume.tasks << task }
+      task.mode = mode unless mode.nil?
+      task.include = include unless include.nil?
+      task.exclude = exclude unless exclude.nil?
+      tasks << task
       self
     end
 
     #
     def delete_task!(id)
-      ts = tasks
-      task = ts.task(ts.resolve(id))
-      volumes.each { |volume| volume.tasks >> task }
+      tasks >> tasks.task(tasks.resolve(id))
       self
     end
 
     #
     def process_tasks!(*ids)
-      ts = intact_tasks
-      ids = ts.collect(&:id) if ids.empty?
-      rclone = 'rclone' if (rclone = ENV['RCLONE']).nil?
-      ids.collect { |id| ts.task(ts.resolve(id)) }.each do |task|
-        args = [rclone]
+      failed = false
+      intacts = intact_tasks
+      ids = intacts.collect(&:id) if ids.empty?
+      ids.collect { |id| intacts.task(intacts.resolve(id)) }.each do |task|
+        source_path = File.join(volumes.volume(task.source_id).root, task.source_root.nil? || task.source_root.empty? ? '' : task.source_root)
+        destination_path = File.join(volumes.volume(task.destination_id).root, task.destination_root.nil? || task.destination_root.empty? ? '' : task.destination_root)
+        args = [Mclone.rclone]
         opts = [
+          '--config', Mclone.windows? ? 'NUL' : '/dev/null',
           simulate? ? '--dry-run' : nil,
-          verbose? ? '--verbose' : nil
+          verbose? ? '--verbose' : nil,
+          verbose? ? '--progress' : nil
         ].compact
+        opts.append('--crypt-password', task.crypter_token) unless task.crypter_mode.nil?
+        case task.crypter_mode
+        when :encrypt then opts.append('--crypt-remote', destination_path)
+        when :decrypt then opts.append('--crypt-remote', source_path)
+        end
         case task.mode
-        when :update
-          args << 'copy'
-          opts << '--update'
-        when :synchronize
-          args << 'sync'
-        when :copy
-          args << 'copy'
-        when :move
-          args << 'move'
+        when :update then args.push('copy', '--update')
+        when :synchronize then args << 'sync'
+        when :copy then args << 'copy'
+        when :move then args << 'move'
         end
         opts.append('--filter', "- /#{Volume::FILE}")
         opts.append('--filter', "- #{task.exclude}") unless task.exclude.nil? || task.exclude.empty?
         opts.append('--filter', "+ #{task.include}") unless task.include.nil? || task.include.empty?
         args.concat(opts)
-        args.append(File.join(volumes.volume(task.source_id).root, task.source_root), File.join(volumes.volume(task.destination_id).root, task.destination_root))
+        case task.crypter_mode
+        when nil then args.append(source_path, destination_path)
+        when :encrypt then args.append(source_path, ':crypt:')
+        when :decrypt then args.append(':crypt:', destination_path)
+        end
+        $stdout << args.collect(&:escape).join(' ') << "\n" if verbose?
         case system(*args)
-        when nil then raise(Session::Error, %(failed to execute "#{args.first}"))
-        when false then exit($?)
+        when nil
+          $stderr << %(failed to execute "#{args.first}") << "\n" if verbose?
+          failed = true
+        when false
+          $stderr << %(Rclone exited with status #{$?.to_i}) << "\n" if verbose?
+          failed = true
         end
       end
-    end
-
-    # Collect all tasks from all loaded volumes
-    def tasks
-      tasks = SessionTaskSet.new(self)
-      volumes.each { |volume| tasks.merge!(volume.tasks) }
-      tasks
+      raise(Session::Error, "Rclone execution failure(s)") if failed
+      self
     end
 
     # Collect all tasks from all loaded volumes which are ready to be executed
     def intact_tasks
-      tasks = IntactTaskSet.new(self)
-      volumes.each { |volume| tasks.merge!(volume.tasks) }
-      tasks
+      IntactTaskSet.new(self).merge!(tasks)
     end
 
     #
@@ -587,6 +680,11 @@ module Mclone
 
   end
 
+  #
+  def self.rclone
+    @@rclone ||= (rclone = ENV['RCLONE']).nil? ? 'rclone' : rclone
+  end
+
   # Return true if run in the real Windows environment (e.g. not in real *NIX or various emulation layers such as MSYS, Cygwin etc.)
   def self.windows?
     @@windows ||= /^(mingw)/.match?(RbConfig::CONFIG['target_os']) # RubyInstaller's MRI, other MinGW-build MRI
@@ -600,17 +698,16 @@ module Mclone
   PATH_LIST_SEPARATOR = windows? ? ';' : ':'
 
   # Return list of live user-provided mounts (mount points on *NIX and disk drives on Windows) which may contain Mclone volumes
-  # Look for the $mclone_PATH environment variable
+  # Look for the $MCLONE_PATH environment variable
   def self.environment_mounts
     ENV['MCLONE_PATH'].split(PATH_LIST_SEPARATOR).collect { |path| File.directory?(path) ? path : nil }.compact rescue []
   end
-
   # Return list of live system-managed mounts (mount points on *NIX and disk drives on Windows) which may contain Mclone volumes
   case RbConfig::CONFIG['target_os']
   when 'linux'
     # Linux OS
     def self.system_mounts
-      # Query on /proc for currently mounted file systems
+      # Query /proc for currently mounted file systems
       IO.readlines('/proc/self/mountstats').collect do |line|
         mount = line.split[4]
         UNIX_SYSTEM_MOUNTS.match?(mount) || !File.directory?(mount) ? nil : mount