snapsync 0.3.8 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: e55815742cc612afdb33329e9525e6bcf3a9f153
-  data.tar.gz: 557aa91321084a513e65f36fb8da919e2389ddbc
+SHA256:
+  metadata.gz: c87326d0c598d929dd5d82c91fb635ef9e9542c6b55add928967f4979eb9d86f
+  data.tar.gz: ff4fc4dec816b21d5a1f9944f137b4ef43a48f5198e3b9b23951c7320d5f1777
 SHA512:
-  metadata.gz: ef8db4c76dcc301c75dde68728fcf485fe0f73b5032702af3d94e935b5928e7b0a6acb646f7af500dd0e9cfd523761c87bc8f470036d7eefbe9715a51ee9a2bd
-  data.tar.gz: 726e09e82549925643c11c2c308e7a1a1e72f88da0110376d1b42ece06dd142cf891c7c79b8f09508de46bdc1efb09d61b68ef5504094def33e639798e818b09
+  metadata.gz: d5483525d7f72e59e42f88c00c5ced3473c3d2bbd6682f390fc8a7b0fa68494f2197ebc654d47ef97821654844f9d26abf897919ab2b4cee108bd883e18efc82
+  data.tar.gz: 70f52a2ba5d403d23bc06a580a80bcb410ee5cecdb1e08d414463f03e1461b43d5970fb84a7acd8ea2baa6c5c43327e84597a6d1a6b8f0dda3975b68208887ca

data/README.md

@@ -7,7 +7,7 @@ snapper snapshot directory to a different location using btrfs send and
 receive.
 
 It can be used in two modes:
- - in manual mode, you run snapsync 
+ - in manual mode, you run snapsync
 
 ## Installation
 
@@ -15,7 +15,7 @@ You need to make sure that you've installed Ruby's bundler. On Ubuntu, run
     $ apt install bundler
 
 Then, the following will install snapsync in /opt/snapsync
-    
+
     $ wget https://raw.githubusercontent.com/doudou/snapsync/master/install.sh
     $ sh install.sh
 
@@ -39,6 +39,10 @@ configurations, it will create /path/to/the/drive/snapsync/root and
 /path/to/the/drive/snapsync/home). The 'default' synchronization policy is used
 (see below for other options).
 
+[EXPERIMENTAL] Snapsync targets can also be remote (ssh-reachable) filesystems by using the following scp-like target format:
+
+    $ snapsync init [user[:password]@]host:/path/to/drive/snapsync
+
 If you use systemd, the background systemd job will from now on synchronize the
 new target whenever it is present (i.e. as soon as it is plugged in). If you
 don't, or if you decided to disable the service's auto-start, run (and keep on
@@ -64,6 +68,15 @@ Policies can be set at initialization time by passing additional arguments to
 'snapsync init', or later with 'snapsync policy'. Run
 'snapsync help init' and 'snapsync help policy' for more information.
 
+E.g. If you want to keep the most recent 23 hourly, 6 daily, 3 weekly, 11
+monthly, and 10 yearly snapshots:
+
+    $ snapsync policy /path/to/the/drive/snapsync/config_dir hour 23 day 6 week 3 month 11 year 10
+
+If you only want the most 10 most recent day's snapshots:
+
+    $ snapsync policy /path/to/the/drive/snapsync/config_dir day 10
+
 ## Manual usage
 
 If you prefer using snapsync manually, or use different automation that the one

data/lib/snapsync/auto_sync.rb

@@ -5,47 +5,89 @@ module Snapsync
     # partition availability, and will run sync-all on each (declared) targets
     # when they are available, optionally auto-mounting them
     class AutoSync
-        AutoSyncTarget = Struct.new :partition_uuid, :path, :automount, :name
+        AutoSyncTarget = Struct.new :partition_uuid, :mountpoint, :relative, :automount, :name
 
         attr_reader :config_dir
+        # @return [Hash<String,Array<AutoSyncTarget>>]
         attr_reader :targets
         attr_reader :partitions
 
         DEFAULT_CONFIG_PATH = Pathname.new('/etc/snapsync.conf')
 
-        def self.load_default
-            result = new
-            result.load_config
-            result
-        end
-
-        def initialize(config_dir = SnapperConfig.default_config_dir)
+        def initialize(config_dir = SnapperConfig.default_config_dir, snapsync_config_file: DEFAULT_CONFIG_PATH)
             @config_dir = config_dir
             @targets = Hash.new
             @partitions = PartitionsMonitor.new
+            partitions.poll
+
+            if snapsync_config_file.exist?
+                load_config snapsync_config_file
+            end
         end
 
-        def load_config(path = DEFAULT_CONFIG_PATH)
+        private def load_config(path = DEFAULT_CONFIG_PATH)
             conf = YAML.load(path.read) || Array.new
+            parse_config_migrate_if_needed(path, conf)
+        end
+
+        private def parse_config_migrate_if_needed(path, conf)
+            migrated = false
+            if conf.is_a? Array
+                # Version 1
+                Snapsync.info "Migrating config from v1 to v2"
+                conf = config_migrate_v1_v2(conf)
+                migrated = true
+            elsif conf['version'] != 2
+                raise 'Unknown snapsync config version: %d ' % [conf.version]
+            end
             parse_config(conf)
+            if migrated
+                write_config(path)
+            end
         end
 
-        def parse_config(conf)
-            conf.each do |hash|
+        private def config_migrate_v1_v2(conf)
+            Snapsync.info "Migrating config from version 1 to version 2"
+            targets = conf.map do |target|
+                target['relative'] = target['path']
+                target.delete('path')
+                begin
+                    target['mountpoint'] = partitions.mountpoint_of_uuid(target['partition_uuid'])
+                rescue
+                    raise "Could not migrate config."
+                end
+
+                target
+            end
+            {
+              'version' => 2,
+              'targets' => targets
+            }
+        end
+
+        private def parse_config(conf)
+            conf['targets'].each do |hash|
                 target = AutoSyncTarget.new
                 hash.each { |k, v| target[k] = v }
-                target.path = Pathname.new(target.path)
+                target.mountpoint = Snapsync::path(target.mountpoint)
+                target.relative = Snapsync::path(target.relative)
                 add(target)
             end
         end
 
         def write_config(path)
-            data = each_autosync_target.map do |target|
-                Hash['partition_uuid' => target.partition_uuid,
-                     'path' => target.path.to_s,
-                     'automount' => !!target.automount,
-                     'name' => target.name]
-            end
+            data = {
+              'version' => 2,
+              'targets' => each_autosync_target.map do |target|
+                  target.to_h do |k,v|
+                      if v.is_a? Snapsync::RemotePathname or v.is_a? Pathname
+                          [k.to_s,v.to_s]
+                      else
+                          [k.to_s,v]
+                      end
+                  end
+              end
+            }
             File.open(path, 'w') do |io|
                 YAML.dump(data, io)
             end
@@ -53,7 +95,7 @@ module Snapsync
 
         # Enumerates the declared autosync targets
         #
-        # @yieldparam [AutoSync] target
+        # @yieldparam [AutoSyncTarget] target
         # @return [void]
         def each_autosync_target
             return enum_for(__method__) if !block_given?
@@ -72,41 +114,34 @@ module Snapsync
         # @return [void]
         def each_available_autosync_target
             return enum_for(__method__) if !block_given?
-            partitions.poll
-
-            partitions.known_partitions.each do |uuid, fs|
-                autosync_targets = targets[uuid]
-                next if autosync_targets.empty?
-
-                mp = fs['MountPoints'].first
-                if mp
-                    mp = Pathname.new(mp[0..-2].pack("U*"))
-                end
 
+            each_autosync_target do |target|
+                # @type [RemotePathname]
                 begin
                     mounted = false
-
-                    if !mp
-                        if !autosync_targets.any?(&:automount)
+                    mountpoint = target.mountpoint
+                    if not mountpoint.mountpoint?
+                        if not target.automount
                             Snapsync.info "partition #{uuid} is present, but not mounted and automount is false. Ignoring"
                             next
                         end
 
                         Snapsync.info "partition #{uuid} is present, but not mounted, automounting"
                         begin
-                            mp = fs.Mount([]).first
+                            if mountpoint.is_a? RemotePathname
+                                # TODO: automounting of remote paths
+                                raise 'TODO'
+                            else
+                                fs.Mount([]).first
+                            end
+                            mounted = true
                         rescue Exception => e
                             Snapsync.warn "failed to mount, ignoring this target"
                             next
                         end
-                        mp = Pathname.new(mp)
-                        mounted = true
-                    end
-
-                    autosync_targets.each do |target|
-                        yield(mp + target.path, target)
                     end
 
+                    yield(target.mountpoint + target.relative, target)
                 ensure
                     if mounted
                         fs.Unmount([])
@@ -131,6 +166,7 @@ module Snapsync
             end
         end
 
+        # @param [AutoSyncTarget] target
         def add(target)
             targets[target.partition_uuid] ||= Array.new
             targets[target.partition_uuid] << target

data/lib/snapsync/btrfs.rb

@@ -1,9 +1,16 @@
 module Snapsync
-    module Btrfs
+    class Btrfs
+        class << self
+            # @return [Hash]
+            attr_accessor :_mountpointCache
+        end
+        self._mountpointCache = {}
+
         class Error < RuntimeError
             attr_reader :error_lines
             def initialize(error_lines = Array.new)
                 @error_lines = error_lines
+                super error_lines
             end
 
             def pretty_print(pp)
@@ -20,49 +27,72 @@ module Snapsync
         class UnexpectedBtrfsOutput < Error
         end
 
-        def self.btrfs_prog
+        include Comparable
+        def <=>(other)
+            mountpoint.to_s <=> other.mountpoint.to_s
+        end
+
+        # @return [AgnosticPath]
+        attr_reader :mountpoint
+
+        # @return [Array<SubvolumeMinimalInfo>]
+        attr_reader :subvolume_table
+        
+        # @param [AgnosticPath] mountpoint
+        def initialize(mountpoint)
+            raise "Trying to create Btrfs wrapper on non-mountpoint #{mountpoint}" unless mountpoint.mountpoint?
+
+            Snapsync.debug "Creating Btrfs wrapper at #{mountpoint}"
+            @mountpoint = mountpoint
+
+            @subvolume_table = read_subvolume_table
+        end
+
+        # @param [AgnosticPath] mountpoint
+        # @return [Btrfs]
+        def self.get(mountpoint)
+            mountpoint = mountpoint.findmnt
+
+            self._mountpointCache.fetch(mountpoint.to_s) do
+                btrfs = Btrfs.new mountpoint
+                self._mountpointCache[mountpoint.to_s] = btrfs
+                btrfs
+            end
+        end
+
+        def btrfs_prog
             ENV['BTRFS_PROG'] || 'btrfs'
         end
 
         # @api private
         #
         # A IO.popen-like API to btrfs subcommands
-        def self.popen(*args, mode: 'r', raise_on_error: true, **options)
-            err_r, err_w = IO.pipe
-
-            block_error, block_result = nil
-            IO.popen([btrfs_prog, *args, err: err_w, **options], mode) do |io|
-                err_w.close
+        # @yieldparam [IO] io
+        def popen(*args, mode: 'r', raise_on_error: true, **options)
+            Snapsync.debug "Btrfs(\"#{mountpoint}\").popen: #{args}"
 
+            if @mountpoint.is_a? RemotePathname
+                proc = SSHPopen.new(@mountpoint, [btrfs_prog, *args], **options)
+                return yield(proc)
+            else
+                # @type [IO,IO]
+                err_r, err_w = IO.pipe
                 begin
-                    block_result = yield(io)
-                rescue Error
-                    raise
-                rescue Exception => block_error
+                    IO.popen([btrfs_prog, *args], err: err_w, mode: mode) do |io|
+                        err_w.close
+                        return yield(io)
+                    end
+                    if not $?.success?
+                        raise Error.new, err_r.read.lines
+                    end
+                ensure err_r.close
                 end
-            end
 
-            if $?.success? && !block_error
-                block_result
-            elsif raise_on_error
-                if block_error
-                    raise Error.new, block_error.message
-                else
-                    raise Error.new, "btrfs failed"
-                end
             end
-
-        rescue Error => e
-            prefix = args.join(" ")
-            lines = err_r.readlines.map do |line|
-                "#{prefix}: #{line.chomp}"
-            end
-            raise Error.new(e.error_lines + lines), e.message, e.backtrace
-
-        ensure err_r.close
         end
 
-        def self.run(*args, **options)
+        # @return [String]
+        def run(*args, **options)
             popen(*args, **options) do |io|
                 io.read.encode('utf-8', undef: :replace, invalid: :replace)
             end
@@ -72,12 +102,13 @@ module Snapsync
         #
         # @param [Pathname] path the subvolume path
         # @return [Integer] the subvolume's generation
-        def self.generation_of(path)
-            info = Btrfs.run('subvolume', 'show', path.to_s)
+        def generation_of(path)
+            info = run('subvolume', 'show', path.to_s)
             if info =~ /Generation[^:]*:\s+(\d+)/
                 Integer($1)
             else
-                raise UnexpectedBtrfsOutput, "unexpected output for 'btrfs subvolume show', expected #{info} to contain a Generation: line"
+                raise UnexpectedBtrfsOutput, "unexpected output for 'btrfs subvolume show'," \
+                    +" expected '#{info}' to contain a 'Generation:' line"
             end
         end
 
@@ -94,9 +125,66 @@ module Snapsync
         #
         # @overload find_new(subvolume_dir, last_gen)
         #   @return [#each] an enumeration of the lines of the find-new output
-        def self.find_new(subvolume_dir, last_gen, &block)
+        def find_new(subvolume_dir, last_gen, &block)
             run('subvolume', 'find-new', subvolume_dir.to_s, last_gen.to_s).
                 each_line(&block)
         end
+
+        # @return [Array<SubvolumeMinimalInfo>]
+        def read_subvolume_table
+            table = []
+
+            text = run('subvolume', 'list','-pcgquR', mountpoint.path_part)
+            text.each_line do |l|
+                item = SubvolumeMinimalInfo.new
+                l.gsub!('top level', 'top_level')
+                l = l.split
+                l.each_slice(2) do |kv|
+                    k,v = kv
+                    if v == '-'
+                        v = nil
+                    else
+                        begin
+                            v = Integer(v)
+                        rescue
+                            # ignore
+                        end
+                    end
+                    item.instance_variable_set '@'+k, v
+                end
+                table.push item
+            end
+
+            table
+        end
+
+        # @param [AgnosticPath] subvolume_dir
+        # @return [Hash<String>]
+        def subvolume_show(subvolume_dir)
+            # @type [String]
+            info = run('subvolume', 'show', subvolume_dir.path_part)
+
+            data = {}
+
+            data['absolute_dir'] = info.lines[0].strip
+
+            lines = info.lines[1..-1]
+            lines.each_index do |i|
+                l = lines[i]
+                k, _, v = l.partition ':'
+                k = k.strip.downcase.gsub ' ', '_'
+
+                if k == 'snapshot(s)'
+                    data['snapshots'] = lines[i+1..-1].map do |s|
+                        s.strip
+                    end
+                    break
+                else
+                    data[k] = v.strip
+                end
+            end
+
+            data
+        end
     end
 end

data/lib/snapsync/btrfs_subvolume.rb

@@ -0,0 +1,103 @@
+module Snapsync
+  # Output of `btrfs subvolume list`
+  class SubvolumeMinimalInfo
+    attr_reader :id
+    attr_reader :uuid
+    attr_reader :path
+
+    attr_reader :gen
+    attr_reader :cgen
+
+    attr_reader :parent
+    attr_reader :top_level
+
+    # @return [String,nil]
+    attr_reader :parent_uuid
+    # @return [String, nil]
+    attr_reader :received_uuid
+  end
+
+  class SubvolumeInfo
+
+    # @return [Btrfs]
+    attr_reader :btrfs
+
+    # @return [AgnosticPath]
+    attr_reader :subvolume_dir
+
+    # The absolute path in the btrfs filesystem
+    # @return [String]
+    attr_reader :absolute_dir
+
+    # @return [String]
+    attr_reader :name
+
+    # @return [String]
+    attr_reader :uuid
+
+    # Denotes a subvolume that's a direct parent in the snapshot's timeline.
+    # I.e. [parent -> self] difference possible
+    # @return [String]
+    attr_reader :parent_uuid
+
+    # Denotes the UUID of the subvolume sent by 'btrfs send'
+    # @return [String]
+    attr_reader :received_uuid
+
+    # @return [String]
+    attr_reader :creation_time
+
+    # @return [Integer]
+    attr_reader :subvolume_id
+
+    # @return [Integer]
+    attr_reader :generation
+
+    # @return [Integer]
+    attr_reader :gen_at_creation
+
+    # @return [Integer]
+    attr_reader :parent_id
+
+    # @return [Integer]
+    attr_reader :top_level_id
+
+    # @return [String]
+    attr_reader :flags
+
+    # A transaction id in the sending btrfs filesystem for the `btrfs send` action.
+    # Does not correspond to anything in subvolumes.
+    # @return [Integer]
+    attr_reader :send_transid
+
+    # @return [String]
+    attr_reader :send_time
+
+
+    # The transaction of id of the start of the receive. The next transaction_id holds actual data and changes.
+    # It is +1 of the subvolume's, created by btrfs receive, gen_at_creation
+    # @return [Integer]
+    attr_reader :receive_transid
+
+    # @return [String]
+    attr_reader :receive_time
+
+    # @return [Array<String>]
+    attr_reader :snapshots
+
+    # @param [AgnosticPath] subvolume_dir
+    def initialize(subvolume_dir)
+      @subvolume_dir = subvolume_dir
+      @btrfs = Btrfs.get(subvolume_dir)
+
+      integers = Set[:subvolume_id, :generation, :gen_at_creation, :parent_id, :top_level_id, :send_transid, :receive_transid]
+      btrfs.subvolume_show(subvolume_dir).each do |k, v|
+        if integers.include? k.to_sym
+          instance_variable_set '@'+k, Integer(v)
+        else
+          instance_variable_set '@'+k, v
+        end
+      end
+    end
+  end
+end

data/lib/snapsync/cleanup.rb

@@ -8,6 +8,7 @@ module Snapsync
             @policy = policy
         end
 
+        # @param [SyncTarget] target
         def cleanup(target, dry_run: false)
             snapshots = target.each_snapshot.to_a
             filtered_snapshots = policy.filter_snapshots(snapshots).to_set