bitferry 0.0.3 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b3cf2cfbd34a736bdab0ca41d188a732b5e848be7a746ae9d351424e305ac59d
-  data.tar.gz: 42c91f3371a2c61afdc813870550d19341b2b2e0e8096b5ec2b41adb05fd0670
+  metadata.gz: bb10ff66d68e1981c2011621d1d216cfe759e14378844f5101310b78e5938974
+  data.tar.gz: 0341d89784f9acaa607f9267ddcbd494901efa80ca9954d60d84484d177c63ff
 SHA512:
-  metadata.gz: 3c4f17e69351732f951a411bfc2f62f5052838abfa81d79655e9f6e5b9ee7826dc98a0b40005437d613155071fb71d028dcdddfb7bdcf8e29bd3e6a0ce022bcc
-  data.tar.gz: 64e48fa1d13e51f28c598ff57022d9d300b901a9db2b98af0b446bcfe540d6dd218e8da8d1b08a4ec7b2700af63b6e1d693d340685d49674f69b4004a30fd6d6
+  metadata.gz: 2ac8f19ed6ef8607082b8943d0cab8ae4fa973694bfac1ddc219fe34e2421c05116d9d4ea20213440b192a7a1eae50cc11a485f489976dd182ff5ea1dc5bf1d2
+  data.tar.gz: cf4dabac47d7b46ed13129b749d82c92cd216f91b3fb41c6a8d2db9114738ae875f38db99d0112d26a7176db639d5cba68ebdeaee8e9e6f61f7f21dc658de568

data/CHANGES.md

@@ -1,3 +1,15 @@
+## 0.0.5
+
+- Disable cache usage in Restic check profiles
+- Show stale tasks in verbose mode only
+- Pick the innermost Bitferry volume in case of volumes nesting
+- Fix an options profile application bug
+
+## 0.0.4
+
+- Include/exclude path filters
+- Include user's home in the Bitferry volume lookup list
+
 ## 0.0.3
 
 - Windows bundle

data/README.md

@@ -6,7 +6,7 @@ The [Bitferry](https://github.com/okhlybov/bitferry) is aimed at establishing th
 
 The intended usage ranges from maintaining simple directory copy to another location (disk, mount point) to complex many-to-many (online/offline) data replication/backup solution employing portable media as additional data storage and a means of data propagation between the offsites.
 
-The core idea that drives Bitferry is the conversion of full (absolute) endpoint's paths into the volume-relative ones, where the volume is a data file which is put along the endpoint's data and denotes the root of the directory hierarchy. This leads to the important location independence property meaning that Bitferry is then able to restore the tasks' source-destination endpoint connections in spite of the volume location changes, which is a likely scenario in case of portable storage (different UNIX mount points, Windows drives etc.).
+The core idea that drives Bitferry is the conversion of full (absolute) endpoints' paths into the volume-relative ones, where the volume is a data file which is put along the endpoint's data and denotes the root of the directory hierarchy. This makes data position-independent which means that Bitferry is then able to restore the tasks' source-destination endpoint connections in spite of the volume location changes, which is a likely scenario in case of portable storage (different UNIX mount points, Windows drives etc.).
 
 Bitferry is effectively a frontend to the [Rclone](https://rclone.org) and [Restic](https://restic.net) utilities.
 
@@ -211,7 +211,7 @@ Parameters:
 Options:
     -e                       Encrypt files in destination using default profile (alias for -E default)
     -d                       Decrypt source files using default profile (alias for -D default)
-    -x                       Use extended encryption profile options (applies to -e, -d)
+    -u                       Use extended encryption profile options (applies to -e, -d)
     --process, -X OPTIONS    Extra task processing profile/options
     --encrypt, -E OPTIONS    Encrypt files in destination using specified profile/options
     --decrypt, -D OPTIONS    Decrypt source files using specified profile/options

data/lib/bitferry/cli.rb

@@ -22,6 +22,31 @@ Encryption = %{
 }
 
 
+$process = nil
+$encryption = nil
+$include = []
+$exclude = []
+
+
+def ext_globs(exts) = exts.split(',').collect { |ext| "*.#{ext}" }
+
+
+def setup_task(x, include: true)
+  x.option ['-i'], 'EXTS', 'Include file extensions (comma-separated list)', multivalued: true, attribute_name: :include_exts do |exts|
+    $include << ext_globs(exts)
+  end if include
+  x.option ['-x'], 'EXTS', 'Exclude file extensions (comma-separated list)', multivalued: true, attribute_name: :exclude_exts do |exts|
+    $exclude << ext_globs(exts)
+  end
+  x.option ['--include'], 'GLOBS', 'Include path specifications (comma-separated list)', multivalued: true, attribute_name: :include do |globs|
+    $include << globs.split(',')
+  end if include
+  x.option ['--exclude'], 'GLOBS', 'Exclude path specifications (comma-separated list)', multivalued: true, attribute_name: :exclude do |globs|
+    $exclude << globs.split(',')
+  end
+end
+
+
 def setup_rclone_task(x)
   x.parameter 'SOURCE', 'Source endpoint specifier'
   x.parameter 'DESTINATION', 'Destination endpoint specifier'
@@ -33,7 +58,7 @@ def setup_rclone_task(x)
     $encryption = Bitferry::Rclone::Decrypt
     $profile = :default
   end
-  x.option '-x', :flag, 'Use extended encryption profile options (applies to -e, -d)', attribute_name: :x do
+  x.option '-u', :flag, 'Apply extended (unicode) encryption profile options (alias for -E extended / -D extended)', attribute_name: :u do
     $extended = true
   end
   x.option ['--process', '-X'], 'OPTIONS', 'Extra task processing profile/options' do |opts|
@@ -47,6 +72,7 @@ def setup_rclone_task(x)
     $encryption = Bitferry::Rclone::Decrypt
     $profile = opts
   end
+  setup_task(x)
 end
 
 
@@ -54,6 +80,7 @@ def create_rclone_task(task, *args, **opts)
   task.new(*args,
     process: $process,
     encryption: $encryption&.new(obtain_password, process: $extended ? :extended : $profile),
+    include: $include.flatten.uniq, exclude: $exclude.flatten.uniq,
     **opts
   )
 end
@@ -131,7 +158,7 @@ Clamp do
           puts "  #{task.tag}    #{task.show_status}"
         end
       end
-      unless (xs = Bitferry::Task.stale).empty?
+      if !(xs = Bitferry::Task.stale).empty? && Bitferry.verbosity == :verbose
         puts
         puts '# Stale tasks'
         puts
@@ -251,13 +278,14 @@ Clamp do
         }
         option '--force', :flag, 'Force overwriting existing repository' do $format = true end
         option ['--attach', '-a'], :flag, 'Attach to existing repository' do $format = false end
-        option '-f', :flag, 'Rig for application of the snapshot retention policy (alias for -F default)', attribute_name: :f do $forget = :default end
-        option '-c', :flag, 'Rig for repository checking (alias for -C default)', attribute_name: :c do $check = :default end
+        option '-f', :flag, 'Apply default snapshot retention policy options (alias for -F default)', attribute_name: :f do $forget = :default end
+        option '-c', :flag, 'Apply default repository checking options (alias for -C default)', attribute_name: :c do $check = :default end
         option ['--process', '-X'], 'OPTIONS', 'Extra task processing profile/options' do |opts| $process = opts end
-        option ['--forget', '-F'], 'OPTIONS', 'Rig for snapshot retention policy with profile/options' do |opts| $forget = opts end
-        option ['--check', '-C'], 'OPTIONS', 'Rig for repository checking with profile/options' do |opts| $check = opts end
+        option ['--forget', '-F'], 'OPTIONS', 'Snapshot retention policy with profile/options' do |opts| $forget = opts end
+        option ['--check', '-C'], 'OPTIONS', 'Repository checking with profile/options' do |opts| $check = opts end
         parameter 'SOURCE', 'Source endpoint specifier'
         parameter 'REPOSITORY', 'Destination repository endpoint specifier'
+        setup_task(self, include: false)
         def execute
           bitferry {
             Bitferry::Restic::Backup.new(
@@ -265,7 +293,8 @@ Clamp do
               format: $format,
               process: $process,
               check: $check,
-              forget: $forget
+              forget: $forget,
+              exclude: $exclude.flatten.uniq
             )
           }
         end
@@ -280,11 +309,13 @@ Clamp do
         option ['--process', '-X'], 'OPTIONS', 'Extra task processing profile/options' do |opts| $process = opts end
         parameter 'REPOSITORY', 'Source repository endpoint specifier'
         parameter 'DESTINATION', 'Destination endpoint specifier'
+        setup_task(self)
         def execute
           bitferry {
             Bitferry::Restic::Restore.new(
               destination, repository, obtain_password,
               process: $process,
+              include: $include.flatten.uniq, exclude: $exclude.flatten.uniq
             )
           }
         end

data/lib/bitferry.rb

@@ -12,7 +12,7 @@ require 'shellwords'
 module Bitferry
 
 
-  VERSION = '0.0.3'
+  VERSION = '0.0.5'
 
 
   module Logging
@@ -39,7 +39,7 @@ module Bitferry
     reset
     log.info('restoring volumes')
     result = true
-    roots = (environment_mounts + system_mounts).uniq
+    roots = (environment_mounts + system_mounts + [Dir.home]).uniq
     log.info("distilled volume search path: #{roots.join(', ')}")
     roots.each do |root|
       if File.exist?(File.join(root, Volume::STORAGE))
@@ -341,12 +341,10 @@ module Bitferry
 
     def self.endpoint(root)
       path = Pathname.new(root).realdirpath
-      # FIXME select innermost or outermost volume in case of nested volumes?
-      intact.sort { |v1, v2| v2.root.size <=> v1.root.size }.each do |volume|
+      intact.sort { |v1, v2| v2.root.to_s.size <=> v1.root.to_s.size }.each do |volume|
         begin
-          # FIXME chop trailing slashes
-          stem = path.relative_path_from(volume.root)
-          case stem.to_s
+          stem = path.relative_path_from(volume.root).to_s #.chomp('/')
+          case stem
             when '.' then return volume.endpoint
             when /^[^\.].*/ then return volume.endpoint(stem)
           end
@@ -477,7 +475,7 @@ module Bitferry
     when Array then option # Array is passed verbatim
     when '-' then nil # Disable adding any options with -
     when /^-/ then option.split(',') # Split comma-separated string into array --foo,bar --> [--foo, bar]
-    else route.fetch(option) # Obtain array from the database
+    else route.fetch(option.nil? ? nil : option.to_sym) # Obtain options from the profile database
     end
   end
 
@@ -498,6 +496,9 @@ module Bitferry
     attr_reader :modified
 
 
+    attr_reader :include, :exclude
+
+
     def process_options = @process_options.nil? ? [] : @process_options # As a mandatory option it should never be nil
 
 
@@ -530,10 +531,12 @@ module Bitferry
     end
 
 
-    def initialize(tag: Bitferry.tag, modified: DateTime.now)
+    def initialize(tag: Bitferry.tag, modified: DateTime.now, include: [], exclude: [])
       @tag = tag
       @generation = 0
-      @modified = modified
+      @include = include
+      @exclude = exclude
+      @modified = modified.is_a?(DateTime) ? modified : DateTime.parse(modified)
       # FIXME handle process_options at this level
     end
 
@@ -546,6 +549,8 @@ module Bitferry
 
 
     def restore(hash)
+      @include = hash.fetch(:include, [])
+      @exclude = hash.fetch(:exclude, [])
       @state = :intact
       log.info("restored task #{tag}")
     end
@@ -558,7 +563,9 @@ module Bitferry
     def externalize
       {
         task: tag,
-        modified: @modified
+        modified: modified,
+        include: include.empty? ? nil : include,
+        exclude: exclude.empty? ? nil : exclude
       }.compact
     end
 
@@ -584,6 +591,14 @@ module Bitferry
     end
 
 
+    def show_filters
+      xs = []
+      xs << 'include: ' + include.join(',') unless include.empty?
+      xs << 'exclude: ' + exclude.join(',') unless exclude.empty?
+      xs.join(' ').to_s
+    end
+
+
     def self.[](tag) = @@registry[tag]
 
 
@@ -609,8 +624,16 @@ module Bitferry
     def self.reset = @@registry = {}
 
 
-    def self.register(task) = @@registry[task.tag] = task # TODO settle on task with the latest timestamp
-
+    def self.register(task)
+      # Task with newer timestamp replaces already registered task, if any
+      if (xtag = @@registry[task.tag]).nil?
+        @@registry[task.tag] = task
+      elsif xtag.modified < task.modified
+        @@registry[task.tag] = task
+      else
+        xtag
+      end
+    end
 
     def self.intact = live.filter { |task| task.intact? }
 
@@ -795,7 +818,7 @@ module Bitferry
       end
 
 
-      def show_status = "#{show_operation} #{source.show_status} #{show_direction} #{destination.show_status}"
+      def show_status = "#{show_operation} #{source.show_status} #{show_direction} #{destination.show_status} #{show_filters}"
 
 
       def show_operation = encryption.nil? ? '' : encryption.show_operation
@@ -833,8 +856,14 @@ module Bitferry
       end
 
 
+      def include_filters = include.collect { |x| ['--filter', "+ #{x}"]}.flatten
+
+
+      def exclude_filters = ([Volume::STORAGE, Volume::STORAGE_] + exclude).collect { |x| ['--filter', "- #{x}"]}.flatten
+
+
       def process_arguments
-        ['--filter', "- #{Volume::STORAGE}", '--filter', "- #{Volume::STORAGE_}"] + common_options + process_options + (
+        include_filters + exclude_filters + common_options + process_options + (
           encryption.nil? ? [source.root.to_s, destination.root.to_s] : encryption.arguments(self)
         )
       end
@@ -846,7 +875,7 @@ module Bitferry
         puts cms if Bitferry.verbosity == :verbose
         log.info(cms)
         status = Open3.pipeline(cmd).first
-        raise "rclone exit code #{status.exitstatus}" unless status.success?
+        raise RuntimeError, "rclone exit code #{status.exitstatus}" unless status.success?
         status.success?
       end
 
@@ -1011,6 +1040,9 @@ module Bitferry
       def format = nil
 
 
+      def include_filters = include.collect { |x| ['--include', x]}.flatten
+
+
       def common_options
         [
           case Bitferry.verbosity
@@ -1037,7 +1069,8 @@ module Bitferry
           begin
             Dir.chdir(chdir) unless chdir.nil?
             status = Open3.pipeline(cmd).first
-            raise "restic exit code #{status.exitstatus}" unless status.success?
+            raise RuntimeError, "restic exit code #{status.exitstatus}" unless status.success?
+            status.success?
           ensure
             Dir.chdir(wd) unless chdir.nil?
           end
@@ -1077,14 +1110,14 @@ module Bitferry
 
 
       FORGET = {
-        default: ['--prune', '--keep-within-hourly', '24h', '--keep-within-daily', '7d', '--keep-within-weekly', '30d', '--keep-within-monthly', '1y', '--keep-within-yearly', '100y']
+        default: ['--prune', '--no-cache', '--keep-within-hourly', '24h', '--keep-within-daily', '7d', '--keep-within-weekly', '30d', '--keep-within-monthly', '1y', '--keep-within-yearly', '100y']
       }
       FORGET[nil] = nil # Skip processing retention policy by default
 
 
       CHECK = {
-        default: [],
-        full: ['--read-data']
+        default: ['--no-cache'],
+        full: ['--no-cache', '--read-data']
       }
       CHECK[nil] = nil # Skip integrity checking by default
 
@@ -1102,7 +1135,10 @@ module Bitferry
       end
 
 
-      def show_status = "#{show_operation} #{directory.show_status} #{show_direction} #{repository.show_status}"
+      def exclude_filters = ([Volume::STORAGE, Volume::STORAGE_] + exclude).collect { |x| ['--exclude', x]}.flatten
+
+
+      def show_status = "#{show_operation} #{directory.show_status} #{show_direction} #{repository.show_status} #{show_filters}"
 
 
       def show_operation = 'encrypt+backup'
@@ -1114,7 +1150,7 @@ module Bitferry
       def process
         begin
           log.info("processing task #{tag}")
-          execute('backup', '.', '--tag', "bitferry,#{tag}", '--exclude', Volume::STORAGE, '--exclude', Volume::STORAGE_, *process_options, *common_options_simulate, chdir: directory.root)
+          execute('backup', '.', '--tag', "bitferry,#{tag}", *exclude_filters, *process_options, *common_options_simulate, chdir: directory.root)
           unless check_options.nil?
             log.info("checking repository in #{repository.root}")
             execute('check', *check_options, *common_options)
@@ -1199,7 +1235,10 @@ module Bitferry
       end
 
 
-      def show_status = "#{show_operation} #{repository.show_status} #{show_direction} #{directory.show_status}"
+      def exclude_filters = exclude.collect { |x| ['--exclude', x]}.flatten
+
+
+      def show_status = "#{show_operation} #{repository.show_status} #{show_direction} #{directory.show_status} #{show_filters}"
 
 
       def show_operation = 'decrypt+restore'
@@ -1230,7 +1269,7 @@ module Bitferry
         log.info("processing task #{tag}")
         begin
           # FIXME restore specifically tagged latest snapshot
-          execute('restore', 'latest', '--target', '.', *process_options, *common_options, simulate: Bitferry.simulate?, chdir: directory.root)
+          execute('restore', 'latest', '--target', directory.root.to_s, *include_filters, *exclude_filters, *process_options, *common_options, simulate: Bitferry.simulate?)
           true
         rescue
           false
@@ -1326,7 +1365,7 @@ module Bitferry
 
       def restore(hash)
         @volume_tag = hash.fetch(:volume)
-        @path = Pathname.new(hash.fetch(:path))
+        @path = Pathname.new(hash.fetch(:path, ''))
       end
 
 
@@ -1334,8 +1373,8 @@ module Bitferry
         {
           endpoint: :bitferry,
           volume: volume_tag,
-          path: path
-        }
+          path: path.to_s.empty? ? nil : path
+        }.compact
       end
 
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bitferry
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.5
 platform: ruby
 authors:
 - Oleg A. Khlybov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-03-07 00:00:00.000000000 Z
+date: 2024-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake