snapsync 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0f6542443b1ff40bd7406fb68c42ce183562ad68
-  data.tar.gz: d181ac3b9658ce0c309f13fcff710b0c7003bb69
+  metadata.gz: 1f9224076ddb7de222c830cb7ad068742631d46b
+  data.tar.gz: b3ced85dbdc080938b61ec158e8d803f72ebeed1
 SHA512:
-  metadata.gz: 88057a0d14f43499e074f42f3c7cbbd84b09411df4e53e57ca0e4654e43bcc0647102d32164f462fc3cc35560337aab645fa21cd6f5305151dda6936603dc3d1
-  data.tar.gz: 2b8d3cfbf41498a2ccc0baa55c2cb6651228c68f4ca78e78a93af6edd677082f0084fd5907e94f3b3188c280f0e595052eff37aac2589274d62eb2a694f45534
+  metadata.gz: c26502e714514d0d8f686c62a3d237ded6c85f0cd1a665b2d55d340d357c40aa139a36d5040f371fc719d80aa1c7010654199b6de47b3f2534bae1a3871bb56d
+  data.tar.gz: 0c67490d86f8ad9fc007e88a7c9054883be9f45ee94410ca18dd55dbc6e422620b90f33406693ff44817a13c70f262253cc9283f568035a50b2b029b2ae56aa8

data/.gitignore

@@ -8,3 +8,4 @@
 /spec/reports/
 /tmp/
 /vendor/
+.*.sw?

data/Gemfile

@@ -2,3 +2,6 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in snapsync.gemspec
 gemspec
+
+gem 'pry'
+gem 'pry-byebug'

data/lib/snapsync.rb

@@ -19,6 +19,7 @@ require 'snapsync/timeline_sync_policy'
 require 'snapsync/sync_last_policy'
 
 require 'snapsync/partitions_monitor'
+require 'snapsync/sync'
 require 'snapsync/sync_all'
 require 'snapsync/auto_sync'
 

data/lib/snapsync/auto_sync.rb

@@ -11,13 +11,21 @@ module Snapsync
         attr_reader :targets
         attr_reader :partitions
 
+        DEFAULT_CONFIG_PATH = Pathname.new('/etc/snapsync.conf')
+
+        def self.load_default
+            result = new
+            result.load_config
+            result
+        end
+
         def initialize(config_dir = SnapperConfig.default_config_dir)
             @config_dir = config_dir
             @targets = Hash.new
             @partitions = PartitionsMonitor.new
         end
 
-        def load_config(path)
+        def load_config(path = DEFAULT_CONFIG_PATH)
             conf = YAML.load(path.read) || Array.new
             parse_config(conf)
         end
@@ -32,7 +40,7 @@ module Snapsync
         end
 
         def write_config(path)
-            data = each_target.map do |target|
+            data = each_autosync_target.map do |target|
                 Hash['partition_uuid' => target.partition_uuid,
                      'path' => target.path.to_s,
                      'automount' => !!target.automount,
@@ -43,13 +51,86 @@ module Snapsync
             end
         end
 
-        def each_target
+        # Enumerates the declared autosync targets
+        #
+        # @yieldparam [AutoSync] target
+        # @return [void]
+        def each_autosync_target
             return enum_for(__method__) if !block_given?
             targets.each_value do |targets|
                 targets.each { |t| yield(t) }
             end
         end
 
+        # Enumerates the available autosync targets
+        #
+        # It may mount partitions as needed
+        #
+        # @yieldparam [Pathname] path the path to the target's base dir
+        #   (suitable to be processed by e.g. AutoSync)
+        # @yieldparam [AutoSyncTarget] target the target located at 'path'
+        # @return [void]
+        def each_available_autosync_target
+            return enum_for(__method__) if !block_given?
+            partitions.poll
+
+            partitions.known_partitions.each do |uuid, fs|
+                autosync_targets = targets[uuid]
+                next if autosync_targets.empty?
+
+                mp = fs['MountPoints'].first
+                if mp
+                    mp = Pathname.new(mp[0..-2].pack("U*"))
+                end
+
+                begin
+                    mounted = false
+
+                    if !mp
+                        if !autosync_targets.any?(&:automount)
+                            Snapsync.info "partition #{uuid} is present, but not mounted and automount is false. Ignoring"
+                            next
+                        end
+
+                        Snapsync.info "partition #{uuid} is present, but not mounted, automounting"
+                        begin
+                            mp = fs.Mount([]).first
+                        rescue Exception => e
+                            Snapsync.warn "failed to mount, ignoring this target"
+                            next
+                        end
+                        mp = Pathname.new(mp)
+                        mounted = true
+                    end
+
+                    autosync_targets.each do |target|
+                        yield(mp + target.path, target)
+                    end
+
+                ensure
+                    if mounted
+                        fs.Unmount([])
+                    end
+                end
+            end
+        end
+
+        # Enumerates the available synchronization targets
+        #
+        # It may mount partitions as needed
+        #
+        # @yieldparam [LocalTarget] target the available target
+        # @return [void]
+        def each_available_target
+            return enum_for(__method__) if !block_given?
+            each_available_autosync_target do |path, t|
+                op = SyncAll.new(path, config_dir: config_dir)
+                op.each_target do |target|
+                    yield(target)
+                end
+            end
+        end
+
         def add(target)
             targets[target.partition_uuid] ||= Array.new
             targets[target.partition_uuid] << target
@@ -67,37 +148,10 @@ module Snapsync
 
         def run(period: 60)
             while true
-                partitions.poll
-                partitions.known_partitions.each do |uuid, fs|
-                    mp = fs['MountPoints'].first
-                    targets[uuid].each do |t|
-                        if !mp
-                            if t.automount
-                                Snapsync.info "partition #{t.partition_uuid} is present, but not mounted, automounting"
-                                begin
-                                    mp = fs.Mount([]).first
-                                rescue Exception => e
-                                    Snapsync.warn "failed to mount, ignoring this target"
-                                    next
-                                end
-                                mp = Pathname.new(mp)
-                                mounted = true
-                            else
-                                Snapsync.info "partition #{t.partition_uuid} is present, but not mounted and automount is false. Ignoring"
-                                next
-                            end
-                        else
-                            mp = Pathname.new(mp[0..-2].pack("U*"))
-                        end
-
-                        full_path = mp + t.path
-                        Snapsync.info "sync-all on #{mp + t.path} (partition #{t.partition_uuid})"
-                        op = SyncAll.new(mp + t.path, config_dir: config_dir)
-                        op.run
-                        if mounted
-                            fs.Unmount([])
-                        end
-                    end
+                each_available_autosync_target do |path, t|
+                    Snapsync.info "sync-all on #{path} (partition #{t.partition_uuid})"
+                    op = SyncAll.new(path, config_dir: config_dir)
+                    op.run
                 end
                 Snapsync.info "done all declared autosync partitions, sleeping #{period}s"
                 sleep period

data/lib/snapsync/cleanup.rb

@@ -10,7 +10,7 @@ module Snapsync
 
         def cleanup(target, dry_run: false)
             snapshots = target.each_snapshot.to_a
-            filtered_snapshots = policy.filter_snapshots_to_sync(target, snapshots).to_set
+            filtered_snapshots = policy.filter_snapshots(snapshots).to_set
 
             if filtered_snapshots.any? { |s| s.synchronization_point? }
                 raise InvalidPolicy, "#{policy} returned a snapsync synchronization point in its results"
@@ -23,9 +23,6 @@ module Snapsync
             last_sync_point = snapshots.
                 sort_by(&:num).reverse.
                 find { |s| s.synchronization_point_for?(target) }
-            if !last_sync_point
-                binding.pry
-            end
             filtered_snapshots << last_sync_point
             filtered_snapshots = filtered_snapshots.to_set
 
@@ -34,6 +31,9 @@ module Snapsync
                     target.delete(s, dry_run: dry_run)
                 end
             end
+
+            Snapsync.info "Waiting for subvolumes to be deleted"
+            IO.popen(["btrfs", "subvolume", "sync", err: '/dev/null']).read
         end
     end
 end

data/lib/snapsync/cli.rb

@@ -22,15 +22,48 @@ module Snapsync
                     Snapsync.logger.level = 'DEBUG'
                 end
             end
+
+            # Resolves a path (or nil) into a list of snapsync targets and
+            # yields them
+            #
+            # @param [String,nil] dir the path the user gave, or nil if all
+            #   available auto-sync paths should be processed. If the directory is
+            #   a target, it is yield as-is. It can also be the root of a sync-all
+            #   target (with proper snapsync target as subdirectories whose name
+            #   matches the snapper configurations)
+            #
+            # @yieldparam [LocalTarget] target
+            def each_target(dir = nil)
+                return enum_for(__method__) if !block_given?
+                if dir
+                    dir = Pathname.new(dir)
+                    begin
+                        return yield(LocalTarget.new(dir, create_if_needed: false))
+                    rescue LocalTarget::InvalidTargetPath
+                    end
+
+                    SyncAll.new(dir).each_target do |target|
+                        yield(target)
+                    end
+                else
+                    autosync = AutoSync.load_default
+                    autosync.each_available_target do |target|
+                        yield(target)
+                    end
+                end
+            end
         end
 
         desc 'sync CONFIG DIR', 'synchronizes the snapper configuration CONFIG with the snapsync target DIR'
+        option :autoclean, type: :boolean, default: nil,
+            desc: 'whether the target should be cleaned of obsolete snapshots',
+            long_desc: "The default is to use the value specified in the target's configuration file. This command line option allows to override the default"
         def sync(config_name, dir)
             handle_class_options
 
             config = config_from_name(config_name)
             target = LocalTarget.new(Pathname.new(dir))
-            LocalSync.new(config, target).sync
+            Sync.new(config, target, autoclean: options[:autoclean]).run
         end
 
         desc 'sync-all DIR', 'synchronizes all snapper configurations into corresponding subdirectories of DIR'
@@ -120,7 +153,7 @@ policy for more information
 
             autosync = AutoSync.new
             autosync.load_config(conf_path)
-            exists = autosync.each_target.find do |t|
+            exists = autosync.each_autosync_target.find do |t|
                 t.partition_uuid == uuid && t.path.cleanpath == relative.cleanpath
             end
             if exists
@@ -163,15 +196,10 @@ for 10 days). snapsync understands the following period names: year month day ho
         EOD
         def policy(dir, type, *options)
             handle_class_options
-
-            dir = Pathname.new(dir)
-            if !dir.exist?
-                dir.mkpath
+            each_target(dir) do |target|
+                target.change_policy(type, options)
+                target.write_config
             end
-
-            target = LocalTarget.new(dir)
-            target.change_policy(type, options)
-            target.write_config
         end
 
         desc 'destroy DIR', 'destroys a snapsync target'
@@ -193,10 +221,29 @@ While it can easily be done manually, this command makes sure that the snapshots
         option :config_file, desc: "path to the config file (defaults to /etc/snapsync.conf)",
             default: '/etc/snapsync.conf'
         def auto_sync
+            handle_class_options
             auto = AutoSync.new(SnapperConfig.default_config_dir)
             auto.load_config(Pathname.new(options[:config_file]))
             auto.run
         end
+
+        desc 'list [DIR]', 'list the snapshots present on DIR. If DIR is omitted, tries to access all targets defined as auto-sync targets'
+        def list(dir = nil)
+            handle_class_options
+            each_target(dir) do |target|
+                puts "== #{target.dir}"
+                puts "UUID: #{target.uuid}"
+                puts "Enabled: #{target.enabled?}"
+                puts "Autoclean: #{target.autoclean?}"
+                print "Policy: "
+                pp target.sync_policy
+
+                puts "Snapshots:"
+                target.each_snapshot do |s|
+                    puts "  #{s.num} #{s.to_time}"
+                end
+            end
+        end
     end
 end
 

data/lib/snapsync/default_sync_policy.rb

@@ -8,7 +8,7 @@ module Snapsync
     #
     # Synchronization policy objects are used by the synchronization passes to
     # decide which snapshots to copy and which to not copy. They have to provide
-    # {#filter_snapshots_to_sync}.
+    # {#filter_snapshots}.
     #
     # This default policy is to copy everything but the snapsync-created
     # synchronization points that are not involving the current target
@@ -27,7 +27,7 @@ module Snapsync
         # @param [#uuid] target the target object
         # @param [Array<Snapshot>] the snapshot candidates
         # @return [Array<Snapshot>] the snapshots that should be copied
-        def filter_snapshots_to_sync(target, snapshots)
+        def filter_snapshots(snapshots)
             # Filter out any snapsync-generated snapshot
             snapshots.find_all { |s| !s.synchronization_point? }
         end

data/lib/snapsync/local_sync.rb

@@ -179,17 +179,28 @@ module Snapsync
             sync_snapshot ||= create_synchronization_point
 
             target_snapshots = target.each_snapshot.sort_by(&:num)
+            nums_on_target = target_snapshots.map(&:num).to_set
 
             last_common_snapshot = source_snapshots.find do |s|
-                target_snapshots.find { |src| src.num == s.num }
+                nums_on_target.include?(s.num)
             end
             if !last_common_snapshot
                 Snapsync.warn "no common snapshot found, will have to synchronize the first snapshot fully"
             end
 
-            snapshots_to_sync = target.sync_policy.filter_snapshots_to_sync(target, source_snapshots)
-            snapshots_to_sync.each do |src|
-                if synchronize_snapshot(target.dir + src.num.to_s, src, parent: last_common_snapshot)
+            # Merge source and target snapshots to find out which are needed on
+            # the target, and then remove the ones that are already present.
+            all_snapshots = source_snapshots.find_all { |s| !nums_on_target.include?(s.num) } +
+                target_snapshots
+            nums_required = target.sync_policy.filter_snapshots(all_snapshots).
+                map(&:num).to_set
+            source_snapshots.each do |src|
+                if !nums_required.include?(src.num)
+                    if nums_on_target.include?(src.num)
+                        last_common_snapshot = src
+                    end
+                    next
+                elsif synchronize_snapshot(target.dir + src.num.to_s, src, parent: last_common_snapshot)
                     last_common_snapshot = src
                 end
             end

data/lib/snapsync/local_target.rb

@@ -30,7 +30,8 @@ module Snapsync
         # Defaults to true
         def autoclean?; !!@autoclean end
 
-        class InvalidUUIDError < RuntimeError; end
+        class InvalidTargetPath < RuntimeError; end
+        class InvalidUUIDError < InvalidTargetPath; end
         class NoUUIDError < InvalidUUIDError; end
 
         def initialize(dir, create_if_needed: true)

data/lib/snapsync/sync.rb

@@ -0,0 +1,50 @@
+module Snapsync
+    # Single-target synchronization
+    class Sync
+        attr_reader :config
+
+        attr_reader :target
+
+        def initialize(config, target, autoclean: nil)
+            @config = config
+            @target = target
+            @autoclean =
+                if autoclean.nil? then target.autoclean?
+                else autoclean
+                end
+        end
+
+        # Whether the target should be cleaned after synchronization.
+        # 
+        # This is determined either by {#autoclean?} if {.new} was called with
+        # true or false, or by the target's own autoclean flag if {.new} was
+        # called with nil
+        def autoclean?
+            @autoclean
+        end
+
+        # The method that performs synchronization
+        #
+        # One usually wants to call {#run}, which also takes care of running
+        # cleanup if {#autoclean?} is true
+        def sync
+            LocalSync.new(config, target).sync
+        end
+
+        def run
+            sync
+
+            if autoclean?
+                if target.cleanup
+                    Snapsync.info "running cleanup for #{target.dir}"
+                    target.cleanup.cleanup(target)
+                else
+                    Snapsync.info "#{target.sync_policy.class.name} policy set, no cleanup to do for #{target.dir}"
+                end
+            else
+                Snapsync.info "autoclean not set on #{target.dir}"
+            end
+        end
+    end
+end
+

data/lib/snapsync/sync_all.rb

@@ -23,43 +23,35 @@ module Snapsync
             @autoclean  = autoclean
         end
 
-        # Whether the given target should be cleaned after synchronization.
-        # 
-        # This is determined either by {#autoclean?} if {.new} was called with
-        # true or false, or by the target's own autoclean flag if {.new} was
-        # called with nil
-        def should_autoclean_target?(target)
-            if @autoclean.nil?
-                target.autoclean?
-            else
-                @autoclean
-            end
+        # Whether the target should be forced to autoclean(true), force to not
+        # run cleanup (false) or use their own config file to decide (nil)
+        #
+        # The default is nil
+        #
+        # @return [Boolean,nil]
+        def autoclean?
+            @autoclean
         end
 
-        def run
+        # Enumerate the targets available under {#target_dir}
+        def each_target
             SnapperConfig.each_in_dir(config_dir) do |config|
                 dir = target_dir + config.name
                 if !dir.exist?
-                    Snapsync.warn "not synchronizing #{config.name}, there are no corresponding directory in #{target_dir}. Call snapsync init to create a proper target directory"
+                    Snapsync.warn "no directory for configuration #{config.name} in #{target_dir}"
                 else
-                    target = LocalTarget.new(dir)
-                    if !target.enabled?
-                        Snapsync.warn "not synchronizing #{config.name}, it is disabled"
-                        next
-                    end
+                    yield(LocalTarget.new(dir))
+                end
+            end
+        end
 
-                    LocalSync.new(config, target).sync
-                    if should_autoclean_target?(target)
-                        if target.cleanup
-                            Snapsync.info "running cleanup for #{config.name}"
-                            target.cleanup.cleanup(target)
-                        else
-                            Snapsync.info "#{target.sync_policy.class.name} policy set, no cleanup to do for #{config.name}"
-                        end
-                    else
-                        Snapsync.info "autoclean not set on #{config.name}"
-                    end
+        def run
+            each_target do |target|
+                if !target.enabled?
+                    Snapsync.warn "not synchronizing to #{target.dir}, it is disabled"
+                    next
                 end
+                Sync.new(config, target, autoclean: autoclean?).run
             end
         end
     end

data/lib/snapsync/sync_last_policy.rb

@@ -14,8 +14,8 @@ module Snapsync
             pp.text "will keep only the latest snapshot"
         end
 
-        # (see DefaultSyncPolicy#filter_snapshots_to_sync)
-        def filter_snapshots_to_sync(target, snapshots)
+        # (see DefaultSyncPolicy#filter_snapshots)
+        def filter_snapshots(snapshots)
             last = snapshots.sort_by(&:num).reverse.
                 find { |s| !s.synchronization_point? }
             [last]

data/lib/snapsync/timeline_sync_policy.rb

@@ -91,6 +91,8 @@ module Snapsync
         def compute_required_snapshots(target_snapshots)
             keep_flags = Hash.new { |h,k| h[k] = [false, []] }
 
+            target_snapshots = target_snapshots.sort_by(&:num)
+
             # Mark all important snapshots as kept
             target_snapshots.each do |s|
                 if s.user_data['important'] == 'yes'
@@ -122,7 +124,7 @@ module Snapsync
 
             # Finally, guard against race conditions. Always keep all snapshots
             # between the last-to-keep and the last
-            target_snapshots.sort_by(&:num).reverse.each do |s|
+            target_snapshots.reverse.each do |s|
                 break if keep_flags[s.num][0]
                 keep_flags[s.num][0] = true
                 keep_flags[s.num][1] << "last snapshot"
@@ -130,9 +132,10 @@ module Snapsync
             keep_flags
         end
 
-        def filter_snapshots_to_sync(target, source_snapshots)
+        def filter_snapshots(snapshots)
             Snapsync.debug do
                 Snapsync.debug "Filtering snapshots according to timeline"
+                Snapsync.debug "Snapshots: #{snapshots.map(&:num).sort.join(", ")}"
                 timeline.each do |t|
                     Snapsync.debug "  #{t}"
                 end
@@ -140,10 +143,10 @@ module Snapsync
             end
 
             default_policy = DefaultSyncPolicy.new
-            source_snapshots  = default_policy.filter_snapshots_to_sync(target, source_snapshots)
+            snapshots  = default_policy.filter_snapshots(snapshots)
 
-            keep_flags = compute_required_snapshots(source_snapshots)
-            source_snapshots.sort_by(&:num).find_all do |s|
+            keep_flags = compute_required_snapshots(snapshots)
+            snapshots.sort_by(&:num).find_all do |s|
                 keep, reason = keep_flags.fetch(s.num, nil)
                 if keep
                     Snapsync.debug "Timeline: selected snapshot #{s.num} #{s.date.to_time}"

data/lib/snapsync/version.rb

@@ -1,3 +1,3 @@
 module Snapsync
-  VERSION = "0.1.5"
+    VERSION = "0.1.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: snapsync
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Sylvain Joyeux
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-04 00:00:00.000000000 Z
+date: 2015-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logging
@@ -186,6 +186,7 @@ files:
 - lib/snapsync/partitions_monitor.rb
 - lib/snapsync/snapper_config.rb
 - lib/snapsync/snapshot.rb
+- lib/snapsync/sync.rb
 - lib/snapsync/sync_all.rb
 - lib/snapsync/sync_last_policy.rb
 - lib/snapsync/test.rb