snapsync 0.1.0

data/lib/snapsync/snapper_config.rb

@@ -0,0 +1,125 @@
+module Snapsync
+    # A snapper configuration
+    class SnapperConfig
+        # The configuration name
+        attr_reader :name
+
+        # Path to the subvolume
+        #
+        # @return [Pathname]
+        attr_reader :subvolume
+
+        # The filesystem type
+        attr_reader :fstype
+
+        def initialize(name)
+            @name = name.to_str
+            @subvolume, @fstype = nil
+        end
+
+        # The directory containing the snapshots
+        def snapshot_dir
+            subvolume + ".snapshots"
+        end
+
+        # Enumerate the valid snapshots in this configuration
+        #
+        # @yieldparam [Snapshot] snapshot
+        def each_snapshot(&block)
+            Snapshot.each(snapshot_dir, &block)
+        end
+
+        def self.default_config_dir
+            Pathname.new('/etc/snapper/configs')
+        end
+
+        # Enumerates the valid snapper configurations present in a directory
+        def self.each_in_dir(path = default_config_dir)
+            path.each_entry do |config_file|
+                config_name = config_file.to_s
+                config_file = path + config_file
+                next if !config_file.file?
+                begin
+                    config = SnapperConfig.load(config_file)
+                rescue Interrupt
+                    raise
+                rescue Exception => e
+                    Snapsync.warn "cannot load #{config_file}: #{e.message}"
+                    e.backtrace.each do |line|
+                        Snapsync.debug "  #{line}"
+                    end
+                    next
+                end
+
+                yield(config)
+            end
+        end
+
+        # Create a new snapshot
+        #
+        # @return [Snapshot]
+        def create(type: 'single', description: '', user_data: Hash.new)
+            user_data = user_data.map { |k,v| "#{k}=#{v}" }.join(",")
+            snapshot_id = IO.popen(["snapper", "-c", name, "create",
+                     "--type", type,
+                     "--print-number",
+                     "--description", description,
+                     "--userdata", user_data]) do |io|
+                Integer(io.read.strip)
+            end
+            Snapshot.new(snapshot_dir + snapshot_id.to_s)
+        end
+
+        # Delete one of this configuration's snapshots
+        def delete(snapshot)
+            system("snapper", "-c", name, "delete", snapshot.num.to_s)
+        end
+
+        # Create a SnapperConfig object from the data in a configuration file
+        #
+        # @param [#readlines] path the file
+        # @param [String] name the configuration name
+        # @return [SnapperConfig]
+        # @raise (see #load)
+        def self.load(path, name: path.basename.to_s)
+            config = new(name)
+            config.load(path)
+            config
+        end
+
+        # @api private
+        #
+        # Extract the key and value from a snapper configuration file
+        #
+        # @return [(String,String)] the key and value pair, or nil if it is an
+        #   empty or comment line
+        def parse_line(line)
+            line = line.strip.gsub(/#.*/, '')
+            if !line.empty?
+                if line =~ /^(\w+)="?([^"]*)"?$/
+                    return $1, $2
+                else
+                    raise ArgumentError, "cannot parse #{line}"
+                end
+            end
+        end
+
+        # Load the information from a configuration file into this object
+        #
+        # @see SnapperConfig.load
+        def load(path)
+            path.readlines.each do |line|
+                key, value = parse_line(line)
+                case key
+                when NilClass then next
+                else
+                    instance_variable_set("@#{key.downcase}", value)
+                end
+            end
+            if @subvolume
+                @subvolume = Pathname.new(subvolume)
+            end
+        end
+    end
+end
+

data/lib/snapsync/snapshot.rb

@@ -0,0 +1,164 @@
+module Snapsync
+    # Representation of a single Snapper snapshot
+    class Snapshot
+        # The path to the snapshot directory
+        #
+        # @return [Pathname]
+        attr_reader :snapshot_dir
+
+        # The path to the snapshot's subvolume
+        #
+        # @return [Pathname]
+        def subvolume_dir; snapshot_dir + "snapshot" end
+
+        # The snapshot's date
+        #
+        # @return [DateTime]
+        attr_reader :date
+
+        # The snapshot number
+        attr_reader :num
+
+        # The snapshot's user data
+        #
+        # @return [Hash<String,String>]
+        attr_reader :user_data
+
+        PARTIAL_MARKER = "snapsync-partial"
+
+        def self.partial_marker_path(snapshot_dir)
+            snapshot_dir + PARTIAL_MARKER
+        end
+
+        # A file that is used to mark the snapshot has having only been
+        # partially synchronized
+        #
+        # @return [Pathname]
+        def partial_marker_path
+            self.class.partial_marker_path(snapshot_dir)
+        end
+
+        # Whether this snapshot has only been partially synchronized
+        def partial?
+            partial_marker_path.exist?
+        end
+
+        # This snapshot's reference time
+        def to_time
+            date.to_time
+        end
+
+        # Whether this snapshot is one of snapsync's synchronization points
+        def synchronization_point?
+            user_data['snapsync']
+        end
+
+        # Whether this snapshot is one of snapsync's synchronization points for
+        # the given target
+        def synchronization_point_for?(target)
+            user_data['snapsync'] == target.uuid
+        end
+
+        def initialize(snapshot_dir)
+            @snapshot_dir = snapshot_dir
+
+            if !snapshot_dir.directory?
+                raise InvalidSnapshot, "#{snapshot_dir} does not exist"
+            elsif !subvolume_dir.directory?
+                raise InvalidSnapshot, "#{snapshot_dir}'s subvolume directory does not exist (#{subvolume_dir})"
+            end
+
+            # This loads the information and also validates that snapshot_dir is
+            # indeed a snapper snapshot
+            load_info
+        end
+
+        # Compute the size difference between the given snapshot and self
+        #
+        # This is an estimate of the size required to send this snapshot using
+        # the given snapshot as parent
+        def size_diff_from(snapshot)
+            info = `sudo btrfs subvolume show '#{snapshot.subvolume_dir}'`
+            info =~ /Generation[^:]*:\s+(\d+)/
+            size_diff_from_gen(Integer($1))
+        end
+
+        # Compute the size of the snapshot
+        def size
+            size_diff_from_gen(0)
+        end
+
+        def size_diff_from_gen(gen)
+            new = `sudo btrfs subvolume find-new '#{subvolume_dir}' #{gen}`
+            new.split("\n").inject(0) do |size, line|
+                if line.strip =~ /len (\d+)/
+                    size + Integer($1)
+                else size
+                end
+            end
+        end
+
+        # Enumerate the snapshots from the given directory
+        #
+        # The directory is supposed to be maintained in a snapper-compatible
+        # foramt, meaning that the snapshot directory name must be the
+        # snapshot's number
+        def self.each(snapshot_dir, with_partial: false)
+            return enum_for(__method__, snapshot_dir, with_partial: with_partial) if !block_given?
+            snapshot_dir.each_child do |path|
+                if path.directory? && path.basename.to_s =~ /^\d+$/
+                    begin
+                        snapshot = Snapshot.new(path)
+                    rescue InvalidSnapshot => e
+                        Snapsync.warn "ignored #{path} in #{self}: #{e}"
+                    end
+                    if snapshot
+                        if snapshot.num != Integer(path.basename.to_s)
+                            Snapsync.warn "ignored #{path} in #{self}: the snapshot reports num=#{snapshot.num} but its directory is called #{path.basename}"
+                        elsif !with_partial && snapshot.partial?
+                            Snapsync.warn "ignored #{path} in #{self}: this is a partial snapshot"
+                        else
+                            yield snapshot
+                        end
+                    end
+                end
+            end
+        end
+
+        # Loads snapper's info.xml, validates it and assigns the information to
+        # the relevant attributes
+        def load_info
+            info_xml = snapshot_dir + "info.xml"
+            if !info_xml.file?
+                raise InvalidSnapshot, "#{snapshot_dir}/info.xml does not exist, is this really a snapper snapshot ?"
+            end
+
+            xml = REXML::Document.new(info_xml.read)
+            if xml.root.name != 'snapshot'
+                raise InvalidInfoFile, "#{snapshot_dir}/info.xml does not look like a snapper info file (root is not 'snapshot')"
+            end
+
+            date = xml.root.elements.to_a('date')
+            if date.empty?
+                raise InvalidInfoFile, "#{snapshot_dir}/info.xml does not have a date element"
+            else
+                @date = DateTime.parse(date.first.text)
+            end
+
+            num = xml.root.elements.to_a('num')
+            if num.empty?
+                raise InvalidInfoFile, "#{snapshot_dir}/info.xml does not have a num element"
+            else
+                @num = Integer(num.first.text)
+            end
+
+            @user_data = Hash.new
+            xml.root.elements.to_a('userdata').each do |node|
+                k = node.elements['key'].text
+                v = node.elements['value'].text
+                user_data[k] = v
+            end
+        end
+    end
+end
+

data/lib/snapsync/sync_all.rb

@@ -0,0 +1,67 @@
+module Snapsync
+    # Synchronizes all snapshots to a directory
+    #
+    # A snapshot will be synchronized if (1) the target directory has a
+    # subdirectory of the config's name and (2) this directory is not
+    # disabled through its config file
+    class SyncAll
+        # The path to the directory containing snapper configuration files
+        attr_reader :config_dir
+        # The path to the root directory to which we should sync
+        attr_reader :target_dir
+
+        # Creates a sync-all operation for the given target directory
+        #
+        # @param [Pathname] target_dir the target directory
+        # @param [Boolean,nil] autoclean if true or false, will control
+        #   whether the targets should be cleaned of obsolete snapshots
+        #   after synchronization. If nil (the default), the target's own
+        #   autoclean flag will be used to determine this
+        def initialize(target_dir, config_dir: Pathname.new('/etc/snapper/configs'), autoclean: nil)
+            @config_dir = config_dir
+            @target_dir = target_dir
+            @autoclean  = autoclean
+        end
+
+        # Whether the given target should be cleaned after synchronization.
+        # 
+        # This is determined either by {#autoclean?} if {.new} was called with
+        # true or false, or by the target's own autoclean flag if {.new} was
+        # called with nil
+        def should_autoclean_target?(target)
+            if @autoclean.nil?
+                target.autoclean?
+            else
+                @autoclean
+            end
+        end
+
+        def run
+            SnapperConfig.each_in_dir(config_dir) do |config|
+                dir = target_dir + config.name
+                if !dir.exist?
+                    Snapsync.warn "not synchronizing #{config.name}, there are no corresponding directory in #{target_dir}. Call snapsync init to create a proper target directory"
+                else
+                    target = LocalTarget.new(dir)
+                    if !target.enabled?
+                        Snapsync.warn "not synchronizing #{config.name}, it is disabled"
+                        next
+                    end
+
+                    LocalSync.new(config, target).sync
+                    if should_autoclean_target?(target)
+                        if target.cleanup
+                            Snapsync.info "running cleanup"
+                            target.cleanup.cleanup(target)
+                        else
+                            Snapsync.info "#{target.sync_policy.class.name} policy set, no cleanup to do"
+                        end
+                    else
+                        Snapsync.info "autoclean not set"
+                    end
+                end
+            end
+        end
+    end
+end
+

data/lib/snapsync/sync_last_policy.rb

@@ -0,0 +1,24 @@
+module Snapsync
+    # A simple policy that synchronizes only the last snapshot (that is,
+    # snapsync's own synchronization point)
+    class SyncLastPolicy
+        def self.from_config(config)
+            new
+        end
+
+        def to_config
+            Array.new
+        end
+
+        def pretty_print(pp)
+            pp.text "will keep only the latest snapshot"
+        end
+
+        # (see DefaultSyncPolicy#filter_snapshots_to_sync)
+        def filter_snapshots_to_sync(target, snapshots)
+            last = snapshots.sort_by(&:num).reverse.
+                find { |s| !s.synchronization_point? }
+            [last]
+        end
+    end
+end

data/lib/snapsync/test.rb

@@ -0,0 +1,62 @@
+# simplecov must be loaded FIRST. Only the files required after it gets loaded
+# will be profiled !!!
+if ENV['TEST_ENABLE_COVERAGE'] == '1'
+    begin
+        require 'simplecov'
+        SimpleCov.start
+    rescue LoadError
+        require 'snapsync'
+        Snapsync.warn "coverage is disabled because the 'simplecov' gem cannot be loaded"
+    rescue Exception => e
+        require 'snapsync'
+        Snapsync.warn "coverage is disabled: #{e.message}"
+    end
+end
+
+require 'minitest/autorun'
+require 'snapsync'
+require 'flexmock/test_unit'
+require 'minitest/spec'
+
+if ENV['TEST_ENABLE_PRY'] != '0'
+    begin
+        require 'pry'
+    rescue Exception
+        Snapsync.warn "debugging is disabled because the 'pry' gem cannot be loaded"
+    end
+end
+
+module Snapsync
+    # This module is the common setup for all tests
+    #
+    # It should be included in the toplevel describe blocks
+    #
+    # @example
+    #   require 'snapsync/test'
+    #   describe Snapsync do
+    #     include Snapsync::SelfTest
+    #   end
+    #
+    module SelfTest
+        if defined? FlexMock
+            include FlexMock::ArgumentTypes
+            include FlexMock::MockContainer
+        end
+
+        def setup
+            # Setup code for all the tests
+        end
+
+        def teardown
+            if defined? FlexMock
+                flexmock_teardown
+            end
+            super
+            # Teardown code for all the tests
+        end
+    end
+end
+
+Minitest::Test.include Snapsync::SelfTest
+
+

data/lib/snapsync/timeline_sync_policy.rb

@@ -0,0 +1,163 @@
+module Snapsync
+    class TimelineSyncPolicy < DefaultSyncPolicy
+        attr_reader :reference
+        attr_reader :timeline
+
+        attr_reader :periods
+
+        def initialize(reference: Time.now)
+            @reference = reference
+            @timeline = Array.new
+            @periods = Array.new
+        end
+
+        def self.from_config(config)
+            policy = new
+            policy.parse_config(config)
+            policy
+        end
+
+        def parse_config(config)
+            config.each_slice(2) do |period, count|
+                add(period.to_sym, Integer(count))
+            end
+        end
+
+        def to_config
+            periods.flatten
+        end
+
+        def pretty_print(pp)
+            pp.text "timeline policy"
+            pp.nest(2) do
+                pp.seplist(periods) do |pair|
+                    pp.breakable
+                    pp.text "#{pair[0]}: #{pair[1]}"
+                end
+            end
+        end
+
+        # Add an element to the timeline
+        #
+        # @param [Symbol] period the period (:year, :month, :week, :day, :hour)
+        # @param [Integer] count how many units of this period should be kept
+        # 
+        # @example keep one snapshot every day for the last 10 days
+        #   cleanup.add(:day, 10)
+        #
+        def add(period, count)
+            beginning_of_day   = reference.to_date
+            beginning_of_week  = beginning_of_day.prev_day(beginning_of_day.wday + 1)
+            beginning_of_month = beginning_of_day.prev_day(beginning_of_day.mday - 1)
+            beginning_of_year  = beginning_of_day.prev_day(beginning_of_day.yday - 1)
+            beginning_of_hour  = beginning_of_day.to_time + (reference.hour * 3600)
+
+            timeline = self.timeline.dup
+            if period == :year
+                count.times do
+                    timeline << beginning_of_year.to_time
+                    beginning_of_year = beginning_of_year.prev_year
+                end
+            elsif period == :month
+                count.times do
+                    timeline << begining_of_month.to_time
+                    begining_of_month = begining_of_month.prev_month
+                end
+            elsif period == :week
+                count.times do
+                    timeline << beginning_of_week.to_time
+                    beginning_of_week = beginning_of_week.prev_day(7)
+                end
+            elsif period == :day
+                count.times do
+                    timeline << beginning_of_day.to_time
+                    beginning_of_day = beginning_of_day.prev_day
+                end
+            elsif period == :hour
+                count.times do |i|
+                    timeline << beginning_of_hour
+                    beginning_of_hour = beginning_of_hour - 3600
+                end
+            else
+                raise ArgumentError, "unknown period name #{period}"
+            end
+
+            periods << [period, count]
+            @timeline = timeline.sort.uniq
+        end
+
+        # Given a list of snapshots, computes those that should be kept to honor
+        # the timeline constraints
+        def compute_required_snapshots(target_snapshots)
+            keep_flags = Hash.new { |h,k| h[k] = [false, []] }
+
+            # Mark all important snapshots as kept
+            target_snapshots.each do |s|
+                if s.user_data['important'] == 'yes'
+                    keep_flags[s.num][0] = true
+                    keep_flags[s.num][1] << "marked as important"
+                end
+            end
+
+            # For each timepoint in the timeline, find the newest snapshot that
+            # is not before the timepoint
+            merged_timelines = (target_snapshots.to_a + timeline).sort_by do |s|
+                s.to_time
+            end
+            matching_snapshots = [target_snapshots.first]
+            merged_timelines.each do |obj|
+                if obj.kind_of?(Snapshot)
+                    matching_snapshots[-1] = obj
+                else
+                    s = matching_snapshots.last
+                    matching_snapshots[-1] = [s, obj]
+                    matching_snapshots << s
+                end
+            end
+            matching_snapshots.pop
+            matching_snapshots.each do |(s, timepoint)|
+                keep_flags[s.num][0] = true
+                keep_flags[s.num][1] << "timeline(#{timepoint})"
+            end
+
+            # Finally, guard against race conditions. Always keep all snapshots
+            # between the last-to-keep and the last
+            target_snapshots.sort_by(&:num).reverse.each do |s|
+                break if keep_flags[s.num][0]
+                keep_flags[s.num][0] = true
+                keep_flags[s.num][1] << "last snapshot"
+            end
+            keep_flags
+        end
+
+        def filter_snapshots_to_sync(target, source_snapshots)
+            Snapsync.debug do
+                Snapsync.debug "Filtering snapshots according to timeline"
+                timeline.each do |t|
+                    Snapsync.debug "  #{t}"
+                end
+                break
+            end
+
+            default_policy = DefaultSyncPolicy.new
+            source_snapshots  = default_policy.filter_snapshots_to_sync(target, source_snapshots)
+
+            keep_flags = compute_required_snapshots(source_snapshots)
+            source_snapshots.sort_by(&:num).find_all do |s|
+                keep, reason = keep_flags.fetch(s.num, nil)
+                if keep
+                    Snapsync.debug "Timeline: selected snapshot #{s.num} #{s.date.to_time}"
+                    reason.each do |r|
+                        Snapsync.debug "  #{r}"
+                    end
+                else
+                    Snapsync.debug "Timeline: not selected snapshot #{s.num} #{s.date.to_time}"
+                end
+
+                keep
+            end
+        end
+    end
+end
+
+