snapsync 0.1.0

data/lib/snapsync/default_sync_policy.rb

@@ -0,0 +1,42 @@
+module Snapsync
+    # Exception thrown when performing sanity checks on the values returned by
+    # the policy. Snapsync usually aborts in these cases, given how this is
+    # critical
+    class InvalidPolicy < RuntimeError; end
+
+    # Default synchronization policy
+    #
+    # Synchronization policy objects are used by the synchronization passes to
+    # decide which snapshots to copy and which to not copy. They have to provide
+    # {#filter_snapshots_to_sync}.
+    #
+    # This default policy is to copy everything but the snapsync-created
+    # synchronization points that are not involving the current target
+    class DefaultSyncPolicy
+        def self.from_config(config)
+            new
+        end
+
+        def to_config
+            Array.new
+        end
+
+        # Returns the snapshots that should be synchronized according to this
+        # policy
+        #
+        # @param [#uuid] target the target object
+        # @param [Array<Snapshot>] the snapshot candidates
+        # @return [Array<Snapshot>] the snapshots that should be copied
+        def filter_snapshots_to_sync(target, snapshots)
+            # Filter out any snapsync-generated snapshot
+            snapshots.find_all { |s| !s.synchronization_point? }
+        end
+
+        # Pretty prints this policy
+        #
+        # This is used by the CLI to give information about a target to the user
+        def pretty_print(pp)
+            pp.text "default policy"
+        end
+    end
+end

data/lib/snapsync/exceptions.rb

@@ -0,0 +1,8 @@
+module Snapsync
+    # Exception raised when a snapshot directory is given to {Snapshot} that
+    # does not look like a snapshot at all
+    class InvalidSnapshot < RuntimeError; end
+    # Exception raised when a snapshot directory is given to {Snapshot} but
+    # snapshot_dir/info.xml does not look like a valid snapper info file.
+    class InvalidInfoFile < InvalidSnapshot; end
+end

data/lib/snapsync/local_sync.rb

@@ -0,0 +1,223 @@
+module Snapsync
+    # Synchronization between local file systems
+    class LocalSync
+        # The snapper configuration we should synchronize
+        # 
+        # @return [SnapperConfig]
+        attr_reader :config
+        # The target directory into which to synchronize
+        #
+        # @return [LocalTarget]
+        attr_reader :target
+        
+        def initialize(config, target)
+            @config, @target = config, target
+        end
+
+        def create_synchronization_point
+            config.create(
+                description: "synchronization snapshot for snapsync",
+                user_data: Hash['important' => 'yes', 'snapsync' => target.uuid])
+        end
+
+        def remove_synchronization_points(except_last: true)
+            synchronization_points = config.each_snapshot.find_all do |snapshot|
+                snapshot.synchronization_point_for?(target)
+            end
+            if except_last
+                synchronization_points = synchronization_points.sort_by(&:num)
+                synchronization_points.pop
+            end
+            synchronization_points.each do |snapshot|
+                config.delete(snapshot)
+            end
+        end
+
+        def copy_stream(send_io, receive_io, chunk_length: (1 << 20), estimated_size: 0)
+            longest_message_length = 0
+            counter = 0
+            start = Time.now
+            while !send_io.eof?
+                if buffer = send_io.read(chunk_length) # 1MB buffer
+                    receive_io.write(buffer)
+
+                    counter += buffer.size
+                    rate = counter / (Time.now - start)
+                    remaining =
+                        if estimated_size > counter
+                            human_readable_time((estimated_size - counter) / rate)
+                        elsif counter - estimated_size < 100 * 1024**2
+                            human_readable_time(0)
+                        else
+                            '?'
+                        end
+
+                    msg = "#{human_readable_size(counter)} (#{human_readable_size(rate)}/s), #{remaining} remaining"
+                    longest_message_length = [longest_message_length, msg.length].max
+                    print "\r%-#{longest_message_length}s" % [msg]
+                end
+            end
+            print "\r#{" " * longest_message_length}\r"
+            counter
+        end
+
+        def synchronize_snapshot(target_snapshot_dir, src, parent: nil)
+            partial_marker_path = Snapshot.partial_marker_path(target_snapshot_dir)
+
+            # Verify first if the snapshot is already present and/or partially
+            # synchronized
+            begin
+                snapshot = Snapshot.new(target_snapshot_dir)
+                if snapshot.partial?
+                    Snapsync.warn "target snapshot directory #{target_snapshot_dir} looks like an aborted snapsync synchronization, I will attempt to refresh it"
+                else
+                    return true
+                end
+            rescue InvalidSnapshot
+                if target_snapshot_dir.exist?
+                    Snapsync.warn "target snapshot directory #{target_snapshot_dir} already exists, but does not seem to be a valid snapper snapshot. I will attempt to refresh it"
+                else
+                    target_snapshot_dir.mkdir
+                end
+                FileUtils.touch(partial_marker_path.to_s)
+            end
+
+            if copy_snapshot(target_snapshot_dir, src, parent: parent)
+                partial_marker_path.unlink
+                IO.popen(["sudo", "btrfs", "filesystem", "sync", target_snapshot_dir.to_s, err: '/dev/null']).read
+                Snapsync.info "Successfully synchronized #{src.snapshot_dir}"
+                true
+            end
+        end
+
+        def copy_snapshot(target_snapshot_dir, src, parent: nil)
+            # This variable is used in the 'ensure' block. Make sure it is
+            # initialized properly
+            success = false
+
+            File.open(target_snapshot_dir + "info.xml", 'w') do |io|
+                io.write (src.snapshot_dir + "info.xml").read
+            end
+
+            if parent
+                parent_opt = ['-p', parent.subvolume_dir.to_s]
+                estimated_size = src.size_diff_from(parent)
+            else
+                parent_opt = []
+                estimated_size = src.size
+            end
+
+            Snapsync.info "Estimating transfer for #{src.snapshot_dir} to be #{human_readable_size(estimated_size)}"
+
+            start = Time.now
+            bytes_transferred = nil
+            receive_status, send_status = nil
+            err_send_pipe_r, err_send_pipe_w = IO.pipe
+            err_receive_pipe_r, err_receive_pipe_w = IO.pipe
+            IO.popen(['sudo', 'btrfs', 'send', *parent_opt, src.subvolume_dir.to_s, err: err_send_pipe_w]) do |send_io|
+                err_send_pipe_w.close
+                IO.popen(['sudo', 'btrfs', 'receive', target_snapshot_dir.to_s, err: err_receive_pipe_w, out: '/dev/null'], 'w') do |receive_io|
+                    err_receive_pipe_w.close
+                    receive_io.sync = true
+                    bytes_transferred = copy_stream(send_io, receive_io, estimated_size: estimated_size)
+                end
+                receive_status = $?
+            end
+            send_status = $?
+
+            success = (receive_status.success? && send_status.success?)
+            if !send_status.success?
+                Snapsync.warn "btrfs send reported an error"
+                err_send_pipe_w.readlines.each do |line|
+                    Snapsync.warn "  #{line.chomp}"
+                end
+            end
+
+            if !receive_status.success?
+                Snapsync.warn "btrfs receive reported an error"
+                err_receive_pipe_w.readlines.each do |line|
+                    Snapsync.warn "  #{line.chomp}"
+                end
+            end
+
+            if success
+                Snapsync.info "Flushing data to disk"
+                IO.popen(["sudo", "btrfs", "filesystem", "sync", target_snapshot_dir.to_s, err: '/dev/null']).read
+                duration = Time.now - start
+                rate = bytes_transferred / duration
+                Snapsync.info "Transferred #{human_readable_size(bytes_transferred)} in #{human_readable_time(duration)} (#{human_readable_size(rate)}/s)"
+                Snapsync.info "Successfully synchronized #{src.snapshot_dir}"
+                true
+            end
+
+        ensure
+            if !success
+                Snapsync.warn "Failed to synchronize #{src.snapshot_dir}, deleting target directory"
+                subvolume_dir = target_snapshot_dir + "snapshot"
+                if subvolume_dir.directory?
+                    IO.popen(["sudo", "btrfs", "subvolume", "delete", subvolume_dir.to_s, err: '/dev/null']).read
+                end
+                target_snapshot_dir.rmtree
+            end
+        end
+
+        def sync
+            STDOUT.sync = true
+
+            # First, create a snapshot and protect it against cleanup, to use as
+            # synchronization point
+            #
+            # We remove old synchronization points on successful synchronization
+            source_snapshots = config.each_snapshot.sort_by(&:num)
+            sync_snapshot = source_snapshots.reverse.find do |snapshot|
+                if snapshot.synchronization_point_for?(target)
+                    true
+                elsif !snapshot.synchronization_point?
+                    break
+                end
+            end
+            sync_snapshot ||= create_synchronization_point
+
+            target_snapshots = target.each_snapshot.sort_by(&:num)
+
+            last_common_snapshot = source_snapshots.find do |s|
+                target_snapshots.find { |src| src.num == s.num }
+            end
+            if !last_common_snapshot
+                Snapsync.warn "no common snapshot found, will have to synchronize the first snapshot fully"
+            end
+
+            snapshots_to_sync = target.sync_policy.filter_snapshots_to_sync(target, source_snapshots)
+            snapshots_to_sync.each do |src|
+                if synchronize_snapshot(target.dir + src.num.to_s, src, parent: last_common_snapshot)
+                    last_common_snapshot = src
+                end
+            end
+
+            if synchronize_snapshot(target.dir + sync_snapshot.num.to_s, sync_snapshot, parent: last_common_snapshot)
+                Snapsync.debug "successfully copied last synchronization point #{sync_snapshot.num}, removing old ones"
+                remove_synchronization_points
+            end
+
+            last_common_snapshot
+        end
+
+        def human_readable_time(time)
+            hrs = time / 3600
+            min = (time / 60) % 60
+            sec = time % 60
+            "%02i:%02i:%02i" % [hrs, min, sec]
+        end
+
+        def human_readable_size(size, digits: 1)
+            order = ['B', 'kB', 'MB', 'GB']
+            magnitude =
+                if size > 0
+                    Integer(Math.log2(size) / 10)
+                else 0
+                end
+            "%.#{digits}f#{order[magnitude]}" % [Float(size) / (1024 ** magnitude)]
+        end
+    end
+end
+

data/lib/snapsync/local_target.rb

@@ -0,0 +1,150 @@
+module Snapsync
+    class LocalTarget
+        # The target's UUID
+        #
+        # @return [String]
+        attr_reader :uuid
+
+        # This target's directory
+        attr_reader :dir
+
+        # The target sync policy
+        attr_reader :sync_policy
+
+        # The cleanup object
+        attr_reader :cleanup
+
+        # Whether this target is enabled or not
+        def enabled?; @enabled end
+
+        # Enable this target, i.e. add it to the auto synchronization and
+        # cleanup commands
+        def enable; @enabled = true; self end
+
+        # Disable this target, i.e. remove it from the auto synchronization and
+        # cleanup commands
+        def disable; @enabled = false; self end
+
+        # Whether the target should be autocleaned on synchronization
+        #
+        # Defaults to true
+        def autoclean?; !!@autoclean end
+
+        class InvalidUUIDError < RuntimeError; end
+        class NoUUIDError < InvalidUUIDError; end
+
+        def initialize(dir, create_if_needed: true)
+            if !dir.directory?
+                raise ArgumentError, "#{dir} does not exist"
+            end
+            @dir = dir
+
+            begin
+                read_config
+            rescue NoUUIDError
+                if !create_if_needed
+                    raise
+                end
+                @uuid = SecureRandom.uuid
+                @sync_policy = DefaultSyncPolicy.new
+                @cleanup = nil
+                @enabled = true
+                @autoclean = true
+            end
+            write_config
+        end
+
+        def each_snapshot(&block)
+            Snapshot.each(dir, &block)
+        end
+
+        def write_config
+            config = Hash['uuid' => uuid, 'policy' => Hash.new]
+            config['policy']['type'] =
+                case sync_policy
+                when TimelineSyncPolicy then 'timeline'
+                when SyncLastPolicy then 'last'
+                when DefaultSyncPolicy then 'default'
+                end
+            config['policy']['options'] =
+                sync_policy.to_config
+            config['enabled'] = enabled?
+            config['autoclean'] = autoclean?
+
+            File.open(config_path, 'w') do |io|
+                io.write YAML.dump(config)
+            end
+        end
+
+        def read_config
+            begin
+                raw_config = YAML.load(config_path.read)
+            rescue Errno::ENOENT => e
+                raise NoUUIDError, e.message, e.backtrace
+            end
+            parse_config(raw_config)
+        end
+
+        def parse_config(config)
+            uuid = config['uuid']
+            if uuid.length != 36
+                raise InvalidUUIDError, "uuid in #{uuid_path} was expected to be 36 characters long, but is #{uuid.length}"
+            end
+            @uuid = uuid
+
+            @enabled = config.fetch('enabled', true)
+            @autoclean = config.fetch('autoclean', true)
+
+            if policy_config = config['policy']
+                change_policy(policy_config['type'], policy_config['options'] || Array.new)
+            else
+                @sync_policy = DefaultSyncPolicy.new
+                @cleanup = nil
+            end
+        end
+
+        # Path to the target's configuration file
+        #
+        # @return [Pathname]
+        def config_path
+            dir + "snapsync.config"
+        end
+        
+        def change_policy(type, options)
+            case type
+            when 'default'
+                sync_policy = DefaultSyncPolicy
+                cleanup = nil
+            when 'timeline'
+                sync_policy = TimelineSyncPolicy
+                cleanup = TimelineSyncPolicy
+            when 'last'
+                sync_policy = SyncLastPolicy
+                cleanup = SyncLastPolicy
+            else
+                raise InvalidConfiguration, "synchronization policy #{type} does not exist"
+            end
+            @sync_policy = sync_policy.from_config(options)
+            @cleanup =
+                if cleanup
+                    Cleanup.new(cleanup.from_config(options))
+                end
+        end
+
+        def delete(s, dry_run: false)
+            Snapsync.info "Removing snapshot #{s.num} #{s.date.to_time} at #{s.subvolume_dir}"
+            return if dry_run
+
+            IO.popen(["sudo", "btrfs", "subvolume", "delete", s.subvolume_dir.to_s, err: '/dev/null']) do |io|
+                io.read
+            end
+            if $?.success?
+                s.snapshot_dir.rmtree
+                Snapsync.info "Flushing data to disk"
+                IO.popen(["sudo", "btrfs", "filesystem", "sync", s.snapshot_dir.to_s, err: '/dev/null']).read
+            else
+                Snapsync.warn "failed to remove snapshot at #{s.subvolume_dir}, keeping the rest of the snapshot"
+            end
+        end
+    end
+end

data/lib/snapsync/partitions_monitor.rb

@@ -0,0 +1,105 @@
+module Snapsync
+    class PartitionsMonitor
+        attr_reader :udisk
+
+        attr_reader :dirty
+
+        attr_reader :monitored_partitions
+        attr_reader :known_partitions
+
+        def initialize
+            dbus = DBus::SystemBus.instance
+            @udisk = dbus.service('org.freedesktop.UDisks2')
+            udisk.introspect
+
+            @dirty = Concurrent::AtomicBoolean.new(false)
+            # udisk.on_signal('InterfacesAdded') do
+            #     dirty!
+            # end
+            # udisk.on_signal('InterfacesRemoved') do
+            #     dirty!
+            # end
+
+            @monitored_partitions = Set.new
+            @known_partitions = Hash.new
+        end
+
+        def monitor_for(partition_uuid)
+            monitored_partitions << partition_uuid.to_str
+        end
+
+        def dirty!
+            dirty.set
+        end
+
+        def dirty?
+            dirty.set?
+        end
+
+        def partition_uuid_for_dir(dir)
+            dir = dir.expand_path
+            # Find the dir's mountpoint
+            while !dir.mountpoint?
+                dir = dir.parent
+            end
+            dir = dir.to_s
+
+            each_partition_with_filesystem.find do |name, dev|
+                fs = dev['org.freedesktop.UDisks2.Filesystem']
+                mp = fs['MountPoints']
+                if !mp.empty?
+                    binding.pry
+                end
+                # .map { |str| Pathname.new(str) }
+                mp.include?(dir)
+            end
+        end
+
+        def poll
+            udisk.introspect
+            dirty.make_false
+
+            all = Hash.new
+            each_partition_with_filesystem do |name, dev|
+                partition = dev['org.freedesktop.UDisks2.Block']
+                uuid = partition['IdUUID']
+
+                if monitored_partitions.include?(uuid)
+                    all[uuid] = dev['org.freedesktop.UDisks2.Filesystem']
+                end
+            end
+
+            added = Hash.new
+            (all.keys - known_partitions.keys).each do |uuid|
+                fs = added[uuid] = all[uuid]
+                emit_added(uuid, fs)
+            end
+            removed = (known_partitions.keys - all.keys)
+            removed.each { |uuid| emit_removed(uuid) }
+
+            @known_partitions = all
+            return added, removed
+        end
+
+        def emit_added(uuid, fs)
+        end
+
+        def emit_removed(uuid)
+        end
+
+        # Yields the udev objects representing block devices that support an
+        # underlying filesystem
+        #
+        # @yieldparam [String] the block device name (e.g. sda3)
+        # @yieldparam the block device's udev object
+        def each_partition_with_filesystem
+            return enum_for(__method__) if !block_given?
+            udisk.root['org']['freedesktop']['UDisks2']['block_devices'].each do |device_name, _|
+                dev = udisk.object("/org/freedesktop/UDisks2/block_devices/#{device_name}")
+                if dev['org.freedesktop.UDisks2.Partition'] && dev['org.freedesktop.UDisks2.Filesystem']
+                    yield(device_name, dev)
+                end
+            end
+        end
+    end
+end