snapsync 0.3.8 → 0.4.0

data/lib/snapsync/cli.rb

@@ -1,9 +1,11 @@
 require 'thor'
 require 'snapsync'
+require 'set'
 
 module Snapsync
     class CLI < Thor
         class_option :debug, type: :boolean, default: false
+        class_option :ssh_debug, type: :boolean, default: false
 
         no_commands do
             def config_from_name(name)
@@ -21,74 +23,78 @@ module Snapsync
                 if options[:debug]
                     Snapsync.logger.level = 'DEBUG'
                 end
+
+                Snapsync.SSH_DEBUG = options[:ssh_debug]
             end
 
             # Resolves a path (or nil) into a list of snapsync targets and
             # yields them
             #
-            # @param [String,nil] dir the path the user gave, or nil if all
+            # @param [AgnosticPath,nil] dir the path the user gave, or nil if all
             #   available auto-sync paths should be processed. If the directory is
             #   a target, it is yield as-is. It can also be the root of a sync-all
             #   target (with proper snapsync target as subdirectories whose name
             #   matches the snapper configurations)
             #
-            # @yieldparam [LocalTarget] target
+            # @yieldparam [SnapperConfig] config
+            # @yieldparam [SyncTarget] target
             def each_target(dir = nil)
                 return enum_for(__method__) if !block_given?
                 if dir
-                    dir = Pathname.new(dir)
+                    dir = Snapsync::path(dir)
                     begin
-                        return yield(nil, LocalTarget.new(dir, create_if_needed: false))
-                    rescue LocalTarget::InvalidTargetPath
+                        return yield(nil, SyncTarget.new(dir, create_if_needed: false))
+                    rescue SyncTarget::InvalidTargetPath
                     end
 
                     SyncAll.new(dir).each_target do |config, target|
                         yield(config, target)
                     end
                 else
-                    autosync = AutoSync.load_default
+                    autosync = AutoSync.new
                     autosync.each_available_target do |config, target|
                         yield(config, target)
                     end
                 end
             end
 
+            # @return [String, Snapsync::Path, Pathname] uuid, mountpoint, relative
             def partition_of(dir)
-                partitions = PartitionsMonitor.new
-                PartitionsMonitor.new.partition_of(dir)
+                PartitionsMonitor.new(dir).partition_of(dir)
             end
         end
 
-        desc 'sync CONFIG DIR', 'synchronizes the snapper configuration CONFIG with the snapsync target DIR'
+        desc 'sync <CONFIG> <DIR>', 'synchronizes the snapper configuration CONFIG with the snapsync target DIR'
         option :autoclean, type: :boolean, default: nil,
             desc: 'whether the target should be cleaned of obsolete snapshots',
             long_desc: "The default is to use the value specified in the target's configuration file. This command line option allows to override the default"
         def sync(config_name, dir)
             handle_class_options
+            dir = Snapsync::path(dir)
 
             config = config_from_name(config_name)
-            target = LocalTarget.new(Pathname.new(dir))
+            target = SyncTarget.new(dir)
             Sync.new(config, target, autoclean: options[:autoclean]).run
         end
 
-        desc 'sync-all DIR', 'synchronizes all snapper configurations into corresponding subdirectories of DIR'
+        desc 'sync-all <DIR>', 'synchronizes all snapper configurations into corresponding subdirectories of DIR'
         option :autoclean, type: :boolean, default: nil,
             desc: 'whether the target should be cleaned of obsolete snapshots',
             long_desc: "The default is to use the value specified in the target's configuration file. This command line option allows to override the default"
         def sync_all(dir)
             handle_class_options
 
-            dir = Pathname.new(dir)
+            dir = Snapsync::path(dir)
             op = SyncAll.new(dir, config_dir: SnapperConfig.default_config_dir, autoclean: options[:autoclean])
             op.run
         end
 
-        desc 'cleanup CONFIG DIR', 'cleans up the snapsync target DIR based on the policy set by the policy command'
+        desc 'cleanup [--debug] [--dry-run] <CONFIG_DIR>', 'cleans up the snapsync target DIR based on the policy set by the policy command'
         option :dry_run, type: :boolean, default: false
         def cleanup(dir)
             handle_class_options
+            target = SyncTarget.new(Snapsync::path(dir))
 
-            target = LocalTarget.new(Pathname.new(dir))
             if target.cleanup
                 target.cleanup.cleanup(target, dry_run: options[:dry_run])
             else
@@ -107,16 +113,18 @@ module Snapsync
                         [args.shift, args]
                     end
 
-                LocalTarget.parse_policy(*policy)
+                SyncTarget.parse_policy(*policy)
                 return *policy
             end
         end
 
-        desc 'init [NAME] DIR [POLICY]', 'creates a synchronization target, optionally adding it to the auto-sync targets and specifying the synchronization and cleanup policies'
+        desc 'init [NAME] <DIR> [POLICY]', 'creates a synchronization target, optionally adding it to the auto-sync targets and specifying the synchronization and cleanup policies'
         long_desc <<-EOD
 NAME must be provided if DIR is to be added to the auto-sync targets (which
 is the default).
 
+DIR can be a remote filesystem path in scp-like format ( [user[:password]@]host:/path/to/drive/snapsync )
+
 By default, the default policy is used. To change this, provide additional
 arguments as would be expected by the policy subcommand. Run snapsync help
 policy for more information
@@ -145,7 +153,9 @@ policy for more information
                 end
                 dir, *policy = *args
             end
-            dir = Pathname.new(dir)
+
+            dir = Snapsync::path(dir)
+            remote = dir.instance_of? RemotePathname
 
             # Parse the policy option early to avoid breaking later
             begin
@@ -180,11 +190,11 @@ policy for more information
 
             dirs.each do |path|
                 begin
-                    LocalTarget.new(path, create_if_needed: false)
+                    SyncTarget.new(path, create_if_needed: false)
                     Snapsync.info "#{path} was already initialized"
-                rescue ArgumentError, LocalTarget::NoUUIDError
+                rescue ArgumentError, SyncTarget::NoUUIDError
                     path.mkpath
-                    target = LocalTarget.new(path)
+                    target = SyncTarget.new(path)
                     target.change_policy(*policy)
                     target.write_config
                     Snapsync.info "initialized #{path} as a snapsync target"
@@ -198,21 +208,18 @@ policy for more information
             end
         end
 
-        desc 'auto-add NAME DIR', "add DIR to the set of targets for auto-sync"
+        desc 'auto-add [NAME] <DIR>', "add DIR to the set of targets for auto-sync"
         option :automount, type: :boolean, default: true,
             desc: 'whether the supporting partition should be auto-mounted by snapsync when needed or not (the default is yes)'
         option :config_file, default: '/etc/snapsync.conf',
             desc: 'the configuration file that should be updated'
         def auto_add(name, dir)
-            uuid, relative = partition_of(Pathname.new(dir))
+            uuid, mountpoint, relative = partition_of(Snapsync::path(dir))
             conf_path = Pathname.new(options[:config_file])
 
-            autosync = AutoSync.new
-            if conf_path.exist?
-                autosync.load_config(conf_path)
-            end
+            autosync = AutoSync.new snapsync_config_file: conf_path
             exists = autosync.each_autosync_target.find do |t|
-                t.partition_uuid == uuid && t.path.cleanpath == relative.cleanpath
+                t.partition_uuid == uuid && t.mountpoint.cleanpath == mountpoint.cleanpath && t.relative.cleanpath == relative.cleanpath
             end
             if exists
                 if !exists.name
@@ -231,7 +238,7 @@ policy for more information
                 end
                 exists.name ||= name
             else
-                autosync.add AutoSync::AutoSyncTarget.new(uuid, relative, options[:automount], name)
+                autosync.add AutoSync::AutoSyncTarget.new(uuid, mountpoint, relative, options[:automount], name)
             end
             autosync.write_config(conf_path)
         end
@@ -239,29 +246,32 @@ policy for more information
         desc 'auto-remove NAME', "remove a target from auto-sync by name"
         def auto_remove(name)
             conf_path = Pathname.new('/etc/snapsync.conf')
-            autosync = AutoSync.new
-            autosync.load_config(conf_path)
+            autosync = AutoSync.new snapsync_config_file: conf_path
             autosync.remove(name: name)
             autosync.write_config(conf_path)
         end
 
-        desc 'policy DIR TYPE [OPTIONS]', 'sets the synchronization and cleanup policy for the given target or targets'
+        desc 'policy <DIR> <TYPE> [OPTIONS]', 'sets the synchronization and cleanup policy for the given target or targets'
         long_desc <<-EOD
 This command sets the policy used to decide which snapshots to synchronize to
 the target, and which to not synchronize.
 
-Three policy types can be used: default, last and timeline
+TYPE: Three policy types can be used: default, last, or timeline
 
-The default policy takes no argument. It will synchronize all snapshots present in the source, and do no cleanup
+The 'default' policy takes no argument. It will synchronize all snapshots present in the source, and do no cleanup
 
-The last policy takes no argument. It will synchronize (and keep) only the last snapshot
+The 'last' policy takes no argument. It will synchronize (and keep) only the last snapshot
 
-The timeline policy takes periods of time as argument (as e.g. day 10 or month 20). It will keep at least
+The 'timeline' policy takes periods of time as argument (as e.g. day 10 or month 20). It will keep at least
 one snapshot for each period, and for the duration specified (day 10 tells to keep one snapshot per day
-for 10 days). snapsync understands the following period names: year month day hour.
+for 10 days). snapsync understands the following period names: year, month, week, day, hour.
+
+OPTIONS := { year {int} | month {int} | week {int} | day {int} | hour {int} }
         EOD
         def policy(dir, type, *options)
             handle_class_options
+            dir = Snapsync::path(dir)
+
             # Parse the policy early to avoid breaking later
             policy = normalize_policy([type, *options])
             each_target(dir) do |_, target|
@@ -270,14 +280,15 @@ for 10 days). snapsync understands the following period names: year month day ho
             end
         end
 
-        desc 'destroy DIR', 'destroys a snapsync target'
+        desc 'destroy <DIR>', 'destroys a snapsync target'
         long_desc <<-EOD
 While it can easily be done manually, this command makes sure that the snapshots are properly deleted
         EOD
         def destroy(dir)
             handle_class_options
-            target_dir = Pathname.new(dir)
-            target = LocalTarget.new(target_dir, create_if_needed: false)
+            target_dir = Snapsync::path(dir)
+
+            target = SyncTarget.new(target_dir, create_if_needed: false)
             snapshots = target.each_snapshot.to_a
             snapshots.sort_by(&:num).each do |s|
                 target.delete(s)
@@ -292,8 +303,7 @@ While it can easily be done manually, this command makes sure that the snapshots
             default: '/etc/snapsync.conf'
         def auto_sync
             handle_class_options
-            auto = AutoSync.new(SnapperConfig.default_config_dir)
-            auto.load_config(Pathname.new(options[:config_file]))
+            auto = AutoSync.new(SnapperConfig.default_config_dir, snapsync_config_file: Pathname.new(options[:config_file]))
             if options[:one_shot]
                 auto.sync
             else
@@ -304,18 +314,39 @@ While it can easily be done manually, this command makes sure that the snapshots
         desc 'list [DIR]', 'list the snapshots present on DIR. If DIR is omitted, tries to access all targets defined as auto-sync targets'
         def list(dir = nil)
             handle_class_options
-            each_target(dir) do |_, target|
+            each_target(dir) do |config, target|
                 puts "== #{target.dir}"
                 puts "UUID: #{target.uuid}"
                 puts "Enabled: #{target.enabled?}"
                 puts "Autoclean: #{target.autoclean?}"
+                puts "Snapper config: #{config.name}"
                 print "Policy: "
                 pp target.sync_policy
 
+                snapshots_seen = Set.new
+
+                # @type [Snapshot]
+                last_snapshot = nil
                 puts "Snapshots:"
                 target.each_snapshot do |s|
+                    snapshots_seen.add(s.num)
+                    last_snapshot = s
                     puts "  #{s.num} #{s.to_time}"
                 end
+
+                puts " [transferrable:]"
+                config.each_snapshot do |s|
+                    if not snapshots_seen.include? s.num
+                        delta = s.size_diff_from(last_snapshot)
+                        puts "  #{s.num} #{s.to_time} => from: #{last_snapshot.num} delta: " \
+                            +"#{Snapsync.human_readable_size(delta)}"
+
+                        # If the delta was 0, then the data already exists on remote.
+                        if delta > 0
+                            last_snapshot = s
+                        end
+                    end
+                end
             end
         end
     end

data/lib/snapsync/partitions_monitor.rb

@@ -4,11 +4,46 @@ module Snapsync
 
         attr_reader :dirty
 
+        # @return [Set<String>]
         attr_reader :monitored_partitions
+        # @return [Hash<String, Dev>]
         attr_reader :known_partitions
 
-        def initialize
-            dbus = DBus::SystemBus.instance
+        # @return [Hash<String, DBus::ProxyObjectInterface>]
+        attr_reader :partition_table
+
+        # @param [RemotePathname] machine Remote machine to connect to
+        def initialize(machine = nil)
+            if machine.nil?
+                dbus = DBus::SystemBus.instance
+            else
+                sock_path = '/tmp/snapsync_%04d_remote.sock' % rand(10000)
+
+
+                ready = Concurrent::AtomicBoolean.new(false)
+                @ssh_thr = Thread.new do
+                    machine.dup_ssh do |ssh|
+                        @ssh = ssh
+                        if Snapsync.SSH_DEBUG
+                            log = Logger.new(STDOUT)
+                            log.level = Logger::DEBUG
+                            ssh.logger = log
+                            ssh.logger.sev_threshold=Logger::Severity::DEBUG
+                        end
+                        ssh.forward.local_socket(sock_path, '/var/run/dbus/system_bus_socket')
+                        ObjectSpace.define_finalizer(@ssh, proc {
+                            File.delete sock_path
+                        })
+                        ready.make_true
+                        ssh.loop { true }
+                    end
+                end
+                while ready.false?
+                    sleep 0.001
+                end
+
+                dbus = DBus::RemoteBus.new "unix:path=#{sock_path}"
+            end
             @udisk = dbus.service('org.freedesktop.UDisks2')
             udisk.introspect
 
@@ -22,12 +57,30 @@ module Snapsync
 
             @monitored_partitions = Set.new
             @known_partitions = Hash.new
+            @partition_table = Hash.new
+        end
+
+        # @param [DBus::ProxyObjectInterface] fs
+        # @return [Array<String>]
+        def mountpoints(fs)
+            raise "Not mounted?" if fs.nil?
+            mount_points = fs['MountPoints'].map do |str|
+                str[0..-2].pack("U*")
+            end
+            return mount_points
+        end
+
+        def mountpoint_of_uuid(partition_uuid)
+            mounts = mountpoints(known_partitions[partition_uuid])
+            raise "Ambiguous mountpoints: #{mounts}" if mounts.length > 1
+            mounts[0]
         end
 
         def monitor_for(partition_uuid)
             monitored_partitions << partition_uuid.to_str
         end
 
+        # @return [String, Snapsync::Path, Pathname] uuid dir rel
         def partition_of(dir)
             rel = Pathname.new("")
             dir = dir.expand_path
@@ -36,16 +89,22 @@ module Snapsync
                 dir = dir.dirname
             end
 
+            # Collect partitions list from udisk
+            parts = []
             each_partition_with_filesystem do |name, dev|
                 partition = dev['org.freedesktop.UDisks2.Block']
                 uuid = partition['IdUUID']
-
                 fs = dev['org.freedesktop.UDisks2.Filesystem']
                 mount_points = fs['MountPoints'].map do |str|
                     str[0..-2].pack("U*")
                 end
-                if mount_points.include?(dir.to_s)
-                    return uuid, rel
+                parts.push([name, uuid, mount_points])
+            end
+
+            # Find any partition that is a parent of the folder we are looking at
+            parts.each do |name, uuid, mount_points|
+                if mount_points.include?(dir.path_part)
+                    return uuid, dir, rel
                 end
             end
             raise ArgumentError, "cannot guess the partition UUID of the mountpoint #{dir} for #{dir + rel}"
@@ -81,11 +140,19 @@ module Snapsync
 
             all = Hash.new
             each_partition_with_filesystem do |name, dev|
+                # @type [DBus::ProxyObjectInterface]
                 partition = dev['org.freedesktop.UDisks2.Block']
+                # @type [String]
                 uuid = partition['IdUUID']
 
                 if monitored_partitions.include?(uuid)
-                    all[uuid] = dev['org.freedesktop.UDisks2.Filesystem']
+                    fs = dev['org.freedesktop.UDisks2.Filesystem']
+
+                    # If it is a btrfs raid, it will have multiple partitions with the same uuid, but only one will be
+                    # mounted.
+                    next if all.has_key?(uuid) and all[uuid]['MountPoints'].size > 0
+
+                    all[uuid] = fs
                 end
             end
 
@@ -116,7 +183,7 @@ module Snapsync
             return enum_for(__method__) if !block_given?
             udisk.root['org']['freedesktop']['UDisks2']['block_devices'].each do |device_name, _|
                 dev = udisk.object("/org/freedesktop/UDisks2/block_devices/#{device_name}")
-                if dev['org.freedesktop.UDisks2.Block'] && dev['org.freedesktop.UDisks2.Filesystem']
+                if dev.has_iface?('org.freedesktop.UDisks2.Block') && dev.has_iface?('org.freedesktop.UDisks2.Filesystem')
                     yield(device_name, dev)
                 end
             end