snapsync 0.3.8 → 0.4.0

data/lib/snapsync/remote_pathname.rb

@@ -0,0 +1,330 @@
+require 'weakref'
+
+class Pathname
+  class NonZeroExitCode < RuntimeError
+  end
+
+  def path_part
+    to_s
+  end
+
+  def touch
+    FileUtils.touch(to_s)
+  end
+
+  def findmnt
+    # An optimization to quickly get the mountpoint.
+    # 3 levels are needed to get from `.snapshots/<id>/snapshot` to a cached entry.
+    # Will probably be fine.
+    cached = Snapsync._mountpointCache.fetch(self.to_s, nil)
+    cached = Snapsync._mountpointCache.fetch(self.parent.to_s, nil) unless cached
+    cached = Snapsync._mountpointCache.fetch(self.parent.parent.to_s, nil) unless cached
+    return cached.dup if cached
+
+    Snapsync.debug "Pathname ('#{self}').findmnt"
+
+    proc = IO.popen(Shellwords.join ['findmnt','-n','-o','TARGET','-T', self.to_s])
+    path = proc.read.strip
+    proc.close
+    if not $?.success?
+      raise NonZeroExitCode, "findmnt failed for #{self}"
+    end
+
+    raise "findmnt failed" unless path
+    p = Pathname.new path
+    # Update cache
+    p2 = self.dup
+    while p != p2
+      Snapsync._mountpointCache[p2.to_s] = p
+      p2 = p2.parent
+    end
+    p
+  end
+end
+
+module Snapsync
+  class << self
+    # @return [Hash]
+    attr_accessor :_mountpointCache
+  end
+  self._mountpointCache = {}
+
+  class AgnosticPath
+    def exist?
+      raise NotImplementedError
+    end
+    def file?
+      raise NotImplementedError
+    end
+    def directory?
+      raise NotImplementedError
+    end
+    def mountpoint?
+      raise NotImplementedError
+    end
+    def basename
+      raise NotImplementedError
+    end
+    def dirname
+      raise NotImplementedError
+    end
+    def parent
+      raise NotImplementedError
+    end
+    def each_child
+      raise NotImplementedError
+    end
+    def expand_path
+      raise NotImplementedError
+    end
+    def cleanpath
+      raise NotImplementedError
+    end
+    def mkdir
+      raise NotImplementedError
+    end
+    def mkpath
+      raise NotImplementedError
+    end
+    def rmtree
+      raise NotImplementedError
+    end
+    def unlink
+      raise NotImplementedError
+    end
+    # @return [AgnosticPath]
+    def +(path)
+      raise NotImplementedError
+    end
+    def read
+      raise NotImplementedError
+    end
+    def open(flags, &block)
+      raise NotImplementedError
+    end
+    def touch
+      raise NotImplementedError
+    end
+
+    def findmnt
+      raise NotImplementedError
+    end
+
+    # @return [String]
+    def path_part
+      raise NotImplementedError
+    end
+  end
+
+  # Ideally this would also inherit from AgnosticPath...
+  class LocalPathname < Pathname
+  end
+
+  class RemotePathname < AgnosticPath
+
+    # @return [URI::SshGit::Generic]
+    attr_reader :uri
+
+    # @return [Net::SSH::Connection::Session]
+    attr_reader :ssh
+
+    # @return [Net::SFTP::Session]
+    attr_reader :sftp
+
+    # @return [Net::SFTP::Operations::FileFactory]
+    attr_reader :sftp_f
+
+    # @param [String] dir
+    def initialize(dir)
+      if dir.instance_of? RemotePathname
+        @uri = dir.uri.dup
+        @ssh = dir.ssh
+        @sftp = dir.sftp
+        @sftp_f = dir.sftp_f
+      else
+        @uri = URI::SshGit.parse(dir)
+
+        raise RuntimeError.new('Host cannot be nil for remote pathname') if uri.host.nil?
+
+        Snapsync.debug "Opening new ssh session: "+uri.to_s
+        @ssh = Net::SSH.start(uri.host, uri.user, password: uri.password, non_interactive: true)
+
+        @sftp = @ssh.sftp
+        @sftp_f = @sftp.file
+
+        # # FIXME: these probably don't work
+        # @ssh_thr = Thread.new {
+        #   ssh = WeakRef.new(@ssh)
+        #   while ssh.weakref_alive?
+        #     ssh.process 0.1
+        #   end
+        # }
+        #
+        # @sftp_thr = Thread.new {
+        #   sftp = WeakRef.new(@sftp)
+        #   sftp.loop do
+        #     sftp.weakref_alive?
+        #   end
+        # }
+      end
+    end
+
+    def initialize_dup(other)
+      super
+      @uri = @uri.dup
+    end
+
+    # Duplicates a new ssh session with same connection options
+    # @return [Net::SSH::Connection::Session]
+    # @yieldparam ssh [Net::SSH::Connection::Session]
+    def dup_ssh(&block)
+      Snapsync.debug "Opening new ssh session: "+uri.to_s
+      Net::SSH.start(uri.host, uri.user, password: uri.password, non_interactive: true, &block)
+    end
+
+    def exist?
+      begin
+        sftp_f.open(uri.path).close
+        return true
+      rescue Net::SFTP::StatusException
+        return directory?
+      end
+    end
+
+    def file?
+      begin
+        sftp_f.open(uri.path).close
+        return true
+      rescue Net::SFTP::StatusException
+        return false
+      end
+    end
+
+    def directory?
+      begin
+        sftp_f.directory? uri.path
+      rescue Net::SFTP::StatusException
+        return false
+      end
+    end
+
+    def findmnt
+      cached = Snapsync._mountpointCache.fetch(self.to_s, nil)
+      cached = Snapsync._mountpointCache.fetch(self.parent.to_s, nil) unless cached
+      return cached if cached
+
+      Snapsync.debug "RemotePathname ('#{uri}').findmnt"
+
+      path = ssh.exec!(Shellwords.join ['findmnt','-n','-o','TARGET','-T',uri.path]).strip
+      p = self.dup
+      p.uri.path = path
+
+      # Update cache
+      p2 = self.dup
+      while p.uri.path != p2.uri.path
+        Snapsync._mountpointCache[p2.to_s] = p
+        p2 = p2.parent
+      end
+
+      p
+    end
+
+    def mountpoint?
+      Snapsync.debug "RemotePathname ('#{uri}').mountpoint?"
+      ssh.exec!(Shellwords.join ['mountpoint','-q',uri.path]).exitstatus == 0
+    end
+
+    def basename
+      Pathname.new(uri.path).basename
+    end
+
+    def dirname
+      o = self.dup
+      o.uri.path = Pathname.new(uri.path).dirname.to_s
+      o
+    end
+
+    def parent
+      o = self.dup
+      if o.uri.path == '/'
+        raise "Trying to get parent of root directory"
+      end
+      o.uri.path = Pathname.new(uri.path).parent.to_s
+      o
+    end
+
+    def each_child(with_directory=true, &block)
+      raise 'Only supports default value for with_directory' if not with_directory
+
+      sftp.dir.foreach(uri.path) do |entry|
+        next if entry.name == '.' or entry.name == '..'
+
+        o = self.dup
+        o.uri.path = (Pathname.new(o.uri.path) + entry.name).to_s
+        yield o
+      end
+    end
+
+    def expand_path
+      o = self.dup
+      o.uri.path = ssh.exec!(Shellwords.join ['readlink', '-f', uri.path]).chomp
+      o
+    end
+
+    def cleanpath
+      o = self.dup
+      o.uri.path = Pathname.new(uri.path).cleanpath.to_s
+      o
+    end
+
+    def mkdir
+      sftp.mkdir!(uri.path)
+    end
+
+    def mkpath
+      sftp.mkdir!(uri.path)
+    end
+
+    def rmtree
+      raise 'Failed' unless ssh.exec!(Shellwords.join ['rm','-r', uri.path]).exitstatus == 0
+    end
+
+    def unlink
+      raise 'Failed' unless ssh.exec!(Shellwords.join ['rm', uri.path]).exitstatus == 0
+    end
+
+    def +(path)
+      o = self.dup
+      o.uri.path = (Pathname.new(uri.path) + path).to_s
+      o
+    end
+
+    def read
+      begin
+        sftp_f.open(uri.path).read
+      rescue Net::SFTP::StatusException => e
+        raise Errno::ENOENT, e.message, e.backtrace
+      end
+    end
+
+    def open(flags, &block)
+      sftp_f.open uri.path, flags, &block
+    end
+
+    def touch
+      raise 'Failed' unless ssh.exec!(Shellwords.join ['touch', uri.path]).exitstatus == 0
+    end
+
+    def path_part
+      uri.path
+    end
+
+    def to_s
+      uri.to_s
+    end
+
+    def inspect
+      uri.to_s
+    end
+  end
+end

data/lib/snapsync/snapper_config.rb

@@ -10,6 +10,7 @@ module Snapsync
         attr_reader :subvolume
 
         # The filesystem type
+        # @return [String]
         attr_reader :fstype
 
         def initialize(name)
@@ -75,6 +76,11 @@ module Snapsync
             system("snapper", "-c", name, "delete", snapshot.num.to_s)
         end
 
+        def cleanup
+            Snapsync.debug "SnapperConfig.cleanup"
+            system('snapper', '-c', name, 'cleanup', 'all')
+        end
+
         # Create a SnapperConfig object from the data in a configuration file
         #
         # @param [#readlines] path the file

data/lib/snapsync/snapshot.rb

@@ -1,11 +1,24 @@
 module Snapsync
     # Representation of a single Snapper snapshot
     class Snapshot
+        class SnapshotCompareError < RuntimeError
+        end
+
         # The path to the snapshot directory
         #
         # @return [Pathname]
         attr_reader :snapshot_dir
 
+        # @return [Btrfs]
+        attr_reader :btrfs
+
+        # @return [SubvolumeInfo]
+        def info
+            # Load btrfs subvolume info
+            @info = SubvolumeInfo.new(subvolume_dir) unless @info
+            @info
+        end
+
         # The path to the snapshot's subvolume
         #
         # @return [Pathname]
@@ -59,8 +72,11 @@ module Snapsync
             user_data['snapsync'] == target.uuid
         end
 
+
+        # @param [AgnosticPath] snapshot_dir
         def initialize(snapshot_dir)
             @snapshot_dir = snapshot_dir
+            @btrfs = Btrfs.get(snapshot_dir)
 
             if !snapshot_dir.directory?
                 raise InvalidSnapshot, "#{snapshot_dir} does not exist"
@@ -78,12 +94,21 @@ module Snapsync
         # This is an estimate of the size required to send this snapshot using
         # the given snapshot as parent
         #
-        # @param [Snapshot] a reference snapshot, which would be used as parent
+        # @param [Snapshot] snapshot a reference snapshot, which would be used as parent
         #   in the 'send' operation
         # @return [Integer] the size in bytes of the difference between the
         #   given snapshot and the current subvolume's state
         def size_diff_from(snapshot)
-            snapshot_gen = Btrfs.generation_of(snapshot.subvolume_dir)
+            if btrfs.mountpoint != snapshot.btrfs.mountpoint
+                recv_uuid = snapshot.info.received_uuid
+                local_snapshot = btrfs.subvolume_table.find do |s|
+                    s.uuid == recv_uuid
+                end
+                raise "Cannot find snapshot with uuid #{recv_uuid} locally." if local_snapshot.nil?
+                snapshot_gen = local_snapshot.cgen
+            else
+                snapshot_gen = snapshot.info.gen_at_creation
+            end
             size_diff_from_gen(snapshot_gen)
         end
 
@@ -103,7 +128,7 @@ module Snapsync
         # @return [Integer] size in bytes
         # @see size_diff_from size
         def size_diff_from_gen(gen)
-            Btrfs.find_new(subvolume_dir, gen).inject(0) do |size, line|
+            btrfs.find_new(subvolume_dir, gen).inject(0) do |size, line|
                 if line =~ /len (\d+)/
                     size + Integer($1)
                 else size
@@ -111,6 +136,9 @@ module Snapsync
             end
         end
 
+        # @yieldparam path [AgnosticPath]
+        # @yieldparam snapshot [Snapshot, nil]
+        # @yieldparam err [InvalidSnapshot, nil]
         def self.each_snapshot_raw(snapshot_dir)
             return enum_for(__method__, snapshot_dir) if !block_given?
             snapshot_dir.each_child do |path|
@@ -130,6 +158,7 @@ module Snapsync
         # The directory is supposed to be maintained in a snapper-compatible
         # foramt, meaning that the snapshot directory name must be the
         # snapshot's number
+        # @yieldparam snapshot [Snapshot]
         def self.each(snapshot_dir, with_partial: false)
             return enum_for(__method__, snapshot_dir, with_partial: with_partial) if !block_given?
             each_snapshot_raw(snapshot_dir) do |path, snapshot, error|

data/lib/snapsync/{local_sync.rb → snapshot_transfer.rb}

@@ -1,17 +1,26 @@
 module Snapsync
-    # Synchronization between local file systems
-    class LocalSync
+    # Snapshot transfer between two btrfs filesystems
+    class SnapshotTransfer
         # The snapper configuration we should synchronize
         # 
         # @return [SnapperConfig]
         attr_reader :config
         # The target directory into which to synchronize
         #
-        # @return [LocalTarget]
+        # @return [SyncTarget]
         attr_reader :target
+
+        # @return [Btrfs] src filesystem
+        attr_reader :btrfs_src
+
+        # @return [Btrfs] dest filesystem
+        attr_reader :btrfs_dest
         
         def initialize(config, target)
             @config, @target = config, target
+
+            @btrfs_src = Btrfs.get(config.subvolume)
+            @btrfs_dest = Btrfs.get(@target.dir)
         end
 
         def create_synchronization_point
@@ -61,6 +70,7 @@ module Snapsync
             counter
         end
 
+        # @param [AgnosticPath] target_snapshot_dir
         def synchronize_snapshot(target_snapshot_dir, src, parent: nil)
             partial_marker_path = Snapshot.partial_marker_path(target_snapshot_dir)
 
@@ -79,23 +89,26 @@ module Snapsync
                 else
                     target_snapshot_dir.mkdir
                 end
-                FileUtils.touch(partial_marker_path.to_s)
+                partial_marker_path.touch
             end
 
             if copy_snapshot(target_snapshot_dir, src, parent: parent)
                 partial_marker_path.unlink
-                Btrfs.run("filesystem", "sync", target_snapshot_dir.to_s)
+                btrfs_dest.run("filesystem", "sync", target_snapshot_dir.path_part)
                 Snapsync.info "Successfully synchronized #{src.snapshot_dir}"
                 true
             end
         end
 
+        # @param [AgnosticPath] target_snapshot_dir
+        # @param [Snapshot] src
+        # @param [Snapshot, nil] parent
         def copy_snapshot(target_snapshot_dir, src, parent: nil)
             # This variable is used in the 'ensure' block. Make sure it is
             # initialized properly
             success = false
 
-            File.open(target_snapshot_dir + "info.xml", 'w') do |io|
+            (target_snapshot_dir + "info.xml").open('w') do |io|
                 io.write (src.snapshot_dir + "info.xml").read
             end
 
@@ -112,15 +125,15 @@ module Snapsync
             start = Time.now
             bytes_transferred = nil
             bytes_transferred =
-                Btrfs.popen('send', *parent_opt, src.subvolume_dir.to_s) do |send_io|
-                    Btrfs.popen('receive', target_snapshot_dir.to_s, mode: 'w', out: '/dev/null') do |receive_io|
+                btrfs_src.popen('send', *parent_opt, src.subvolume_dir.to_s) do |send_io|
+                    btrfs_dest.popen('receive', target_snapshot_dir.path_part, mode: 'w', out: '/dev/null') do |receive_io|
                         receive_io.sync = true
                         copy_stream(send_io, receive_io, estimated_size: estimated_size)
                     end
                 end
 
             Snapsync.info "Flushing data to disk"
-            Btrfs.run("filesystem", "sync", target_snapshot_dir.to_s)
+            btrfs_dest.run("filesystem", "sync", target_snapshot_dir.path_part)
             duration = Time.now - start
             rate = bytes_transferred / duration
             Snapsync.info "Transferred #{human_readable_size(bytes_transferred)} in #{human_readable_time(duration)} (#{human_readable_size(rate)}/s)"
@@ -128,10 +141,10 @@ module Snapsync
             true
 
         rescue Exception => e
-            Snapsync.warn "Failed to synchronize #{src.snapshot_dir}, deleting target directory"
             subvolume_dir = target_snapshot_dir + "snapshot"
+            Snapsync.warn "Failed to synchronize #{src.snapshot_dir}, deleting target directory #{subvolume_dir}"
             if subvolume_dir.directory?
-                Btrfs.run("subvolume", "delete", subvolume_dir.to_s)
+                btrfs_dest.run("subvolume", "delete", subvolume_dir.path_part)
             end
             if target_snapshot_dir.directory?
                 target_snapshot_dir.rmtree
@@ -143,6 +156,9 @@ module Snapsync
         def sync
             STDOUT.sync = true
 
+            # Do a snapper cleanup before syncing
+            config.cleanup
+
             # First, create a snapshot and protect it against cleanup, to use as
             # synchronization point
             #
@@ -191,23 +207,6 @@ module Snapsync
 
             last_common_snapshot
         end
-
-        def human_readable_time(time)
-            hrs = time / 3600
-            min = (time / 60) % 60
-            sec = time % 60
-            "%02i:%02i:%02i" % [hrs, min, sec]
-        end
-
-        def human_readable_size(size, digits: 1)
-            order = ['B', 'kB', 'MB', 'GB']
-            magnitude =
-                if size > 0
-                    Integer(Math.log2(size) / 10)
-                else 0
-                end
-            "%.#{digits}f#{order[magnitude]}" % [Float(size) / (1024 ** magnitude)]
-        end
     end
 end
 

data/lib/snapsync/ssh_popen.rb

@@ -0,0 +1,107 @@
+module Snapsync
+    class SSHPopen
+        class NonZeroExitCode < RuntimeError
+        end
+        class ExitSignal < RuntimeError
+        end
+
+        # @return [IO]
+        attr_reader :read_buffer
+
+        # @return [IO]
+        attr_reader :write_buffer
+
+        # @param machine [RemotePathname]
+        # @param [Array] command
+        # @param options [Hash]
+        def initialize(machine, command, **options)
+            @read_buffer, read_buffer_in = IO.pipe
+            write_buffer_out, @write_buffer = IO.pipe
+
+            if options[:out]
+                read_buffer_in = File.open(options[:out], "w")
+            end
+
+            ready = Concurrent::AtomicBoolean.new(false)
+            @ssh_thr = Thread.new do
+                machine.dup_ssh do |ssh|
+                    ready.make_true
+                    if Snapsync.SSH_DEBUG
+                        log = Logger.new(STDOUT)
+                        log.level = Logger::DEBUG
+                        ssh.logger = log
+                        ssh.logger.sev_threshold = Logger::Severity::DEBUG
+                    end
+                    # @type [Net::SSH::Connection::Channel]
+                    channel = ssh.open_channel do |channel|
+                        Snapsync.debug "SSHPopen channel opened: #{channel}"
+
+                        channel.on_data do |ch, data|
+                            read_buffer_in.write(data)
+                        end
+                        channel.on_extended_data do |ch, data|
+                            data = data.chomp
+                            if data.length > 0
+                                Snapsync.error data
+                            end
+                        end
+
+                        channel.on_request("exit-status") do |ch2, data|
+                            code = data.read_long
+                            if code != 0
+                                raise NonZeroExitCode, "Exited with code: #{code}"
+                            else
+                                Snapsync.debug "SSHPopen command finished."
+                            end
+                        end
+
+                        channel.on_request("exit-signal") do |ch2, data|
+                            exit_signal = data.read_long
+                            raise ExitSignal, "Exited due to signal: #{exit_signal}"
+                        end
+
+                        channel.on_process do
+                            begin
+                                channel.send_data(write_buffer_out.read_nonblock(2 << 20))
+                            rescue IO::EAGAINWaitReadable
+                            end
+                        end
+
+                        channel.exec(Shellwords.join command)
+                    end
+
+                    ssh.loop(0.001) {
+                        if write_buffer_out.closed?
+                            channel.close
+                        end
+
+                        channel.active?
+                    }
+
+                    Snapsync.debug "SSHPopen session closed"
+                    read_buffer_in.close
+                    write_buffer_out.close
+                end
+            end
+            while ready.false?
+                sleep 0.001
+            end
+        end
+
+        def sync=(sync)
+            # ignore
+        end
+
+        def read(nbytes = nil)
+            read_buffer.read nbytes
+        end
+
+        def write(data)
+            write_buffer.write data
+        end
+
+        def close
+            write_buffer.close
+        end
+    end
+end