bitferry 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b3cf2cfbd34a736bdab0ca41d188a732b5e848be7a746ae9d351424e305ac59d
-  data.tar.gz: 42c91f3371a2c61afdc813870550d19341b2b2e0e8096b5ec2b41adb05fd0670
+  metadata.gz: 17d40f40cac1390daae5cb6500acddf6f47bc83cc99b089530b7babf76adfb67
+  data.tar.gz: 4a5652c86fc09e7b6a23fe1bf53b8ffd891ca544782fe73df9eeb4472cdd4495
 SHA512:
-  metadata.gz: 3c4f17e69351732f951a411bfc2f62f5052838abfa81d79655e9f6e5b9ee7826dc98a0b40005437d613155071fb71d028dcdddfb7bdcf8e29bd3e6a0ce022bcc
-  data.tar.gz: 64e48fa1d13e51f28c598ff57022d9d300b901a9db2b98af0b446bcfe540d6dd218e8da8d1b08a4ec7b2700af63b6e1d693d340685d49674f69b4004a30fd6d6
+  metadata.gz: 95f3e41305bd871b1ca3ce8b8d5172c54006e8b3ca4125ac60dd36a1d4f9b816853f2f30454c7269ef084c596c520c491c0f9202169ef9e2cf34f82a06f3d33f
+  data.tar.gz: 0cc18872c913a6583d72d487191c26bac65f35f09fa051005b6fdc39293eb8d428a151e9643486bd9bb241dbd2cf705081345638c274ce49a2f13ebddd26daed

data/CHANGES.md

@@ -1,3 +1,8 @@
+## 0.0.4
+
+- Include/exclude path filters
+- Include user's home in the Bitferry volume lookup list
+
 ## 0.0.3
 
 - Windows bundle

data/README.md

@@ -6,7 +6,7 @@ The [Bitferry](https://github.com/okhlybov/bitferry) is aimed at establishing th
 
 The intended usage ranges from maintaining simple directory copy to another location (disk, mount point) to complex many-to-many (online/offline) data replication/backup solution employing portable media as additional data storage and a means of data propagation between the offsites.
 
-The core idea that drives Bitferry is the conversion of full (absolute) endpoint's paths into the volume-relative ones, where the volume is a data file which is put along the endpoint's data and denotes the root of the directory hierarchy. This leads to the important location independence property meaning that Bitferry is then able to restore the tasks' source-destination endpoint connections in spite of the volume location changes, which is a likely scenario in case of portable storage (different UNIX mount points, Windows drives etc.).
+The core idea that drives Bitferry is the conversion of full (absolute) endpoints' paths into the volume-relative ones, where the volume is a data file which is put along the endpoint's data and denotes the root of the directory hierarchy. This makes data position-independent which means that Bitferry is then able to restore the tasks' source-destination endpoint connections in spite of the volume location changes, which is a likely scenario in case of portable storage (different UNIX mount points, Windows drives etc.).
 
 Bitferry is effectively a frontend to the [Rclone](https://rclone.org) and [Restic](https://restic.net) utilities.
 

data/lib/bitferry/cli.rb

@@ -22,6 +22,31 @@ Encryption = %{
 }
 
 
+$process = nil
+$encryption = nil
+$include = []
+$exclude = []
+
+
+def ext_globs(exts) = exts.split(',').collect { |ext| "*.#{ext}" }
+
+
+def setup_task(x, include: true)
+  x.option ['-i'], 'EXTS', 'Include file extensions (comma-separated list)', multivalued: true, attribute_name: :include_exts do |exts|
+    $include << ext_globs(exts)
+  end if include
+  x.option ['-x'], 'EXTS', 'Exclude file extensions (comma-separated list)', multivalued: true, attribute_name: :exclude_exts do |exts|
+    $exclude << ext_globs(exts)
+  end
+  x.option ['--include'], 'GLOBS', 'Include path specifications (comma-separated list)', multivalued: true, attribute_name: :include do |globs|
+    $include << globs.split(',')
+  end if include
+  x.option ['--exclude'], 'GLOBS', 'Exclude path specifications (comma-separated list)', multivalued: true, attribute_name: :exclude do |globs|
+    $exclude << globs.split(',')
+  end
+end
+
+
 def setup_rclone_task(x)
   x.parameter 'SOURCE', 'Source endpoint specifier'
   x.parameter 'DESTINATION', 'Destination endpoint specifier'
@@ -33,7 +58,7 @@ def setup_rclone_task(x)
     $encryption = Bitferry::Rclone::Decrypt
     $profile = :default
   end
-  x.option '-x', :flag, 'Use extended encryption profile options (applies to -e, -d)', attribute_name: :x do
+  x.option '-u', :flag, 'Apply extended (unicode) encryption profile options (alias for -E extended / -D extended)', attribute_name: :u do
     $extended = true
   end
   x.option ['--process', '-X'], 'OPTIONS', 'Extra task processing profile/options' do |opts|
@@ -47,6 +72,7 @@ def setup_rclone_task(x)
     $encryption = Bitferry::Rclone::Decrypt
     $profile = opts
   end
+  setup_task(x)
 end
 
 
@@ -54,6 +80,7 @@ def create_rclone_task(task, *args, **opts)
   task.new(*args,
     process: $process,
     encryption: $encryption&.new(obtain_password, process: $extended ? :extended : $profile),
+    include: $include.flatten.uniq, exclude: $exclude.flatten.uniq,
     **opts
   )
 end
@@ -251,13 +278,14 @@ Clamp do
         }
         option '--force', :flag, 'Force overwriting existing repository' do $format = true end
         option ['--attach', '-a'], :flag, 'Attach to existing repository' do $format = false end
-        option '-f', :flag, 'Rig for application of the snapshot retention policy (alias for -F default)', attribute_name: :f do $forget = :default end
-        option '-c', :flag, 'Rig for repository checking (alias for -C default)', attribute_name: :c do $check = :default end
+        option '-f', :flag, 'Apply default snapshot retention policy options (alias for -F default)', attribute_name: :f do $forget = :default end
+        option '-c', :flag, 'Apply default repository checking options (alias for -C default)', attribute_name: :c do $check = :default end
         option ['--process', '-X'], 'OPTIONS', 'Extra task processing profile/options' do |opts| $process = opts end
-        option ['--forget', '-F'], 'OPTIONS', 'Rig for snapshot retention policy with profile/options' do |opts| $forget = opts end
-        option ['--check', '-C'], 'OPTIONS', 'Rig for repository checking with profile/options' do |opts| $check = opts end
+        option ['--forget', '-F'], 'OPTIONS', 'Snapshot retention policy with profile/options' do |opts| $forget = opts end
+        option ['--check', '-C'], 'OPTIONS', 'Repository checking with profile/options' do |opts| $check = opts end
         parameter 'SOURCE', 'Source endpoint specifier'
         parameter 'REPOSITORY', 'Destination repository endpoint specifier'
+        setup_task(self, include: false)
         def execute
           bitferry {
             Bitferry::Restic::Backup.new(
@@ -265,7 +293,8 @@ Clamp do
               format: $format,
               process: $process,
               check: $check,
-              forget: $forget
+              forget: $forget,
+              exclude: $exclude.flatten.uniq
             )
           }
         end
@@ -280,11 +309,13 @@ Clamp do
         option ['--process', '-X'], 'OPTIONS', 'Extra task processing profile/options' do |opts| $process = opts end
         parameter 'REPOSITORY', 'Source repository endpoint specifier'
         parameter 'DESTINATION', 'Destination endpoint specifier'
+        setup_task(self)
         def execute
           bitferry {
             Bitferry::Restic::Restore.new(
               destination, repository, obtain_password,
               process: $process,
+              include: $include.flatten.uniq, exclude: $exclude.flatten.uniq
             )
           }
         end

data/lib/bitferry.rb

@@ -12,7 +12,7 @@ require 'shellwords'
 module Bitferry
 
 
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
 
 
   module Logging
@@ -39,7 +39,7 @@ module Bitferry
     reset
     log.info('restoring volumes')
     result = true
-    roots = (environment_mounts + system_mounts).uniq
+    roots = (environment_mounts + system_mounts + [Dir.home]).uniq
     log.info("distilled volume search path: #{roots.join(', ')}")
     roots.each do |root|
       if File.exist?(File.join(root, Volume::STORAGE))
@@ -498,6 +498,9 @@ module Bitferry
     attr_reader :modified
 
 
+    attr_reader :include, :exclude
+
+
     def process_options = @process_options.nil? ? [] : @process_options # As a mandatory option it should never be nil
 
 
@@ -530,10 +533,12 @@ module Bitferry
     end
 
 
-    def initialize(tag: Bitferry.tag, modified: DateTime.now)
+    def initialize(tag: Bitferry.tag, modified: DateTime.now, include: [], exclude: [])
       @tag = tag
       @generation = 0
-      @modified = modified
+      @include = include
+      @exclude = exclude
+      @modified = modified.is_a?(DateTime) ? modified : DateTime.parse(modified)
       # FIXME handle process_options at this level
     end
 
@@ -546,6 +551,8 @@ module Bitferry
 
 
     def restore(hash)
+      @include = hash.fetch(:include, [])
+      @exclude = hash.fetch(:exclude, [])
       @state = :intact
       log.info("restored task #{tag}")
     end
@@ -558,7 +565,9 @@ module Bitferry
     def externalize
       {
         task: tag,
-        modified: @modified
+        modified: modified,
+        include: include.empty? ? nil : include,
+        exclude: exclude.empty? ? nil : exclude
       }.compact
     end
 
@@ -584,6 +593,14 @@ module Bitferry
     end
 
 
+    def show_filters
+      xs = []
+      xs << 'include: ' + include.join(',') unless include.empty?
+      xs << 'exclude: ' + exclude.join(',') unless exclude.empty?
+      xs.join(' ').to_s
+    end
+
+
     def self.[](tag) = @@registry[tag]
 
 
@@ -609,8 +626,16 @@ module Bitferry
     def self.reset = @@registry = {}
 
 
-    def self.register(task) = @@registry[task.tag] = task # TODO settle on task with the latest timestamp
-
+    def self.register(task)
+      # Task with newer timestamp replaces already registered task, if any
+      if (xtag = @@registry[task.tag]).nil?
+        @@registry[task.tag] = task
+      elsif xtag.modified < task.modified
+        @@registry[task.tag] = task
+      else
+        xtag
+      end
+    end
 
     def self.intact = live.filter { |task| task.intact? }
 
@@ -795,7 +820,7 @@ module Bitferry
       end
 
 
-      def show_status = "#{show_operation} #{source.show_status} #{show_direction} #{destination.show_status}"
+      def show_status = "#{show_operation} #{source.show_status} #{show_direction} #{destination.show_status} #{show_filters}"
 
 
       def show_operation = encryption.nil? ? '' : encryption.show_operation
@@ -833,8 +858,14 @@ module Bitferry
       end
 
 
+      def include_filters = include.collect { |x| ['--filter', "+ #{x}"]}.flatten
+
+        
+      def exclude_filters = ([Volume::STORAGE, Volume::STORAGE_] + exclude).collect { |x| ['--filter', "- #{x}"]}.flatten
+
+
       def process_arguments
-        ['--filter', "- #{Volume::STORAGE}", '--filter', "- #{Volume::STORAGE_}"] + common_options + process_options + (
+        include_filters + exclude_filters + common_options + process_options + (
           encryption.nil? ? [source.root.to_s, destination.root.to_s] : encryption.arguments(self)
         )
       end
@@ -846,7 +877,7 @@ module Bitferry
         puts cms if Bitferry.verbosity == :verbose
         log.info(cms)
         status = Open3.pipeline(cmd).first
-        raise "rclone exit code #{status.exitstatus}" unless status.success?
+        raise RuntimeError, "rclone exit code #{status.exitstatus}" unless status.success?
         status.success?
       end
 
@@ -1011,6 +1042,9 @@ module Bitferry
       def format = nil
 
 
+      def include_filters = include.collect { |x| ['--include', x]}.flatten
+
+      
       def common_options
         [
           case Bitferry.verbosity
@@ -1037,7 +1071,8 @@ module Bitferry
           begin
             Dir.chdir(chdir) unless chdir.nil?
             status = Open3.pipeline(cmd).first
-            raise "restic exit code #{status.exitstatus}" unless status.success?
+            raise RuntimeError, "restic exit code #{status.exitstatus}" unless status.success?
+            status.success?
           ensure
             Dir.chdir(wd) unless chdir.nil?
           end
@@ -1077,7 +1112,7 @@ module Bitferry
 
 
       FORGET = {
-        default: ['--prune', '--keep-within-hourly', '24h', '--keep-within-daily', '7d', '--keep-within-weekly', '30d', '--keep-within-monthly', '1y', '--keep-within-yearly', '100y']
+        default: ['--prune', '--no-cache', '--keep-within-hourly', '24h', '--keep-within-daily', '7d', '--keep-within-weekly', '30d', '--keep-within-monthly', '1y', '--keep-within-yearly', '100y']
       }
       FORGET[nil] = nil # Skip processing retention policy by default
 
@@ -1102,7 +1137,10 @@ module Bitferry
       end
 
 
-      def show_status = "#{show_operation} #{directory.show_status} #{show_direction} #{repository.show_status}"
+      def exclude_filters = ([Volume::STORAGE, Volume::STORAGE_] + exclude).collect { |x| ['--exclude', x]}.flatten
+  
+  
+      def show_status = "#{show_operation} #{directory.show_status} #{show_direction} #{repository.show_status} #{show_filters}"
 
 
       def show_operation = 'encrypt+backup'
@@ -1114,7 +1152,7 @@ module Bitferry
       def process
         begin
           log.info("processing task #{tag}")
-          execute('backup', '.', '--tag', "bitferry,#{tag}", '--exclude', Volume::STORAGE, '--exclude', Volume::STORAGE_, *process_options, *common_options_simulate, chdir: directory.root)
+          execute('backup', '.', '--tag', "bitferry,#{tag}", *exclude_filters, *process_options, *common_options_simulate, chdir: directory.root)
           unless check_options.nil?
             log.info("checking repository in #{repository.root}")
             execute('check', *check_options, *common_options)
@@ -1199,7 +1237,10 @@ module Bitferry
       end
 
 
-      def show_status = "#{show_operation} #{repository.show_status} #{show_direction} #{directory.show_status}"
+      def exclude_filters = exclude.collect { |x| ['--exclude', x]}.flatten
+  
+  
+      def show_status = "#{show_operation} #{repository.show_status} #{show_direction} #{directory.show_status} #{show_filters}"
 
 
       def show_operation = 'decrypt+restore'
@@ -1230,7 +1271,7 @@ module Bitferry
         log.info("processing task #{tag}")
         begin
           # FIXME restore specifically tagged latest snapshot
-          execute('restore', 'latest', '--target', '.', *process_options, *common_options, simulate: Bitferry.simulate?, chdir: directory.root)
+          execute('restore', 'latest', '--target', directory.root.to_s, *include_filters, *exclude_filters, *process_options, *common_options, simulate: Bitferry.simulate?)
           true
         rescue
           false
@@ -1326,7 +1367,7 @@ module Bitferry
 
       def restore(hash)
         @volume_tag = hash.fetch(:volume)
-        @path = Pathname.new(hash.fetch(:path))
+        @path = Pathname.new(hash.fetch(:path, ''))
       end
 
 
@@ -1334,8 +1375,8 @@ module Bitferry
         {
           endpoint: :bitferry,
           volume: volume_tag,
-          path: path
-        }
+          path: path.to_s.empty? ? nil : path
+        }.compact
       end
 
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bitferry
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Oleg A. Khlybov
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-03-07 00:00:00.000000000 Z
+date: 2024-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake