zfs_mgmt 0.3.6 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e8c17652083ffa638f535ffc29e7229d2b1a86fa4839772e27c723816825b56
-  data.tar.gz: 1984744b4a11758b0733d8494145fe896a97da3e89cc93ff29afb9310cbed5b5
+  metadata.gz: 96ec8b7a9e8ba4111f9a8e5d58d347de8a62d00c3d5a681f427abcb21e3560a5
+  data.tar.gz: 2b11bd3aaa99cc664a3a4308ad33e40a640db631c86490ebb2e571718aa5e5c5
 SHA512:
-  metadata.gz: 692848f7054ba6ebe65d2343d3e74e11e4408ff0415bbb55ba127b94a6a97ffc111ec8e455e2d9ae731be73292d1a276f95f5f963b47237e1d8bdd9fe69a1da5
-  data.tar.gz: ab6171067da56aa19590f9275cd885f4e1beed09bd03916642f9390e5e2409a29979b2c64e9f4339d31594fbbf6a4ee6a841725d6467cd88bb5700130990f168
+  metadata.gz: 0dd229c996513d93439e82e08cfeeefc712202c3af28e72df3256c0a58254f8c525c47e4b1457dc0c50a8d3f3a871cebe9e3c5d9a34974c03adec921e8ed870c
+  data.tar.gz: 1a659593c4efa623cb41a8d1586e5d3f34516f960527a1829635b26baad33fcfe8a988585d0be78749692b4b5d6c05cb23dbc4c30be160de8a1c303ed975c13c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    zfs_mgmt (0.3.6)
+    zfs_mgmt (0.4.0)
 
 GEM
   remote: https://rubygems.org/

data/bin/zfsmgr

@@ -5,10 +5,23 @@ require "zfs_mgmt/zfs_mgr"
 require "zfs_mgmt/zfs_mgr/restic"
 
 class ZfsMgr < Thor
+  class_option :zfs_binary, :type => :string, :default => 'zfs',
+               :desc => 'zfs binary'
+  class_option :zpool_binary, :type => :string, :default => 'zpool',
+               :desc => 'zpool binary'
+  class_option :mbuffer_binary, :type => :string, :default => 'mbuffer',
+               :desc => 'mbuffer binary'
+  class_option :pv_binary, :type => :string, :default => 'pv',
+               :desc => 'pv binary'
+  class_option :loglevel, :type => :string, :default => ( $stdout.isatty ? 'info' : 'warn' ),
+               :enum => ['debug','error','fatal','info','warn'],
+               :desc => 'set logging level to specified severity'
   desc "zfsget [ZFS]", "execute zfs get for the given properties and types and parse the output into a nested hash"
   method_option :properties, :type => :array, :default => ['name'], :desc => "List of properties passed to zfs get"
   method_option :types,      :type => :array, :default => ['filesystem','volume'], enum: ['filesystem','volume','snapshot'], :desc => "list of types"
   def zfsget(zfs)
+    ZfsMgmt.set_log_level(options[:loglevel])
+    ZfsMgmt.global_options = options
     pp ZfsMgmt.zfsget(properties: options[:properties],
                       types: options[:types],
                       zfs: zfs)
@@ -19,6 +32,8 @@ class ZfsMgr < Thor
   subcommand "list", ZfsMgmt::ZfsMgr::List
   desc "restic SUBCOMMAND ...ARGS", "backup zfs to restic"
   subcommand "restic", ZfsMgmt::ZfsMgr::Restic
+  desc "send SUBCOMMAND ...ARGS", "send zfs"
+  subcommand "send", ZfsMgmt::ZfsMgr::Send
 end
 
 ZfsMgr.start(ARGV)

data/lib/zfs_mgmt.rb

@@ -9,7 +9,7 @@ require 'text-table'
 require 'open3'
 require 'filesize'
 
-$logger = Logger.new(STDERR)
+$logger = Logger.new(STDERR, progname: 'zfs_mgmt')
 
 $date_patterns = {
   'hourly' => '%F Hour %H',
@@ -38,6 +38,11 @@ $properties_xlate = {
 }
 
 module ZfsMgmt
+  class << self
+    attr_accessor :global_options
+  end
+  class ZfsGetError < StandardError
+  end
   def self.custom_properties()
     return [
       'policy',
@@ -50,6 +55,9 @@ module ZfsMgmt
       'snapshot',
       'snap_prefix',
       'snap_timestamp',
+      'send',
+      'remote',
+      'destination',
     ].map do |p|
       ['zfsmgmt',p].join(':')
     end
@@ -67,7 +75,7 @@ module ZfsMgmt
   end
 
   def self.zfs_holds(snapshot)
-    com = ['zfs', 'holds', '-H', snapshot]
+    com = [global_options['zfs_binary'], 'holds', '-H', snapshot]
     $logger.debug("#{com.join(' ')}")
     out = %x(#{com.join(' ')})
     unless $?.success?
@@ -83,9 +91,8 @@ module ZfsMgmt
   end
 
   def self.zfs_hold(hold,snapshot)
-    com = ['zfs', 'hold', hold, snapshot]
-    $logger.debug("#{com.join(' ')}")
-    system(com.join(' '))
+    com = [global_options['zfs_binary'], 'hold', hold, snapshot]
+    system_com(com)
     unless $?.success?
       errstr = "unable to set hold: #{hold} for snapshot: #{snapshot}"
       $logger.error(errstr)
@@ -94,9 +101,8 @@ module ZfsMgmt
   end
 
   def self.zfs_release(hold,snapshot)
-    com = ['zfs', 'release', hold, snapshot]
-    $logger.debug("#{com.join(' ')}")
-    system(com.join(' '))
+    com = [@global_options['zfs_binary'], 'release', hold, snapshot]
+    system_com(com)
     unless $?.success?
       errstr = "unable to release hold: #{hold} for snapshot: #{snapshot}"
       $logger.error(errstr)
@@ -104,19 +110,20 @@ module ZfsMgmt
     end
   end
 
-  def self.zfsget(properties: ['name'],types: ['filesystem','volume'],zfs: '')
+  def self.zfsget(properties: ['all'],types: ['filesystem','volume'],zfs: '', command_prefix: [])
     results={}
-    com = ['zfs', 'get', '-Hp', properties.join(','), '-t', types.join(','), zfs]
-    so,se,status = Open3.capture3(com.join(' '))
+    com = [ZfsMgmt.global_options[:zfs_binary], 'get', '-Hp', properties.join(','), '-t', types.join(','), zfs]
+    $logger.debug((command_prefix+com).join(' '))
+    so,se,status = Open3.capture3((command_prefix+com).join(' '))
     if status.signaled?
       $logger.error("process was signalled \"#{com.join(' ')}\", termsig #{status.termsig}")
-      raise 'ZfsGetError'
+      raise ZfsGetError, "process was signalled \"#{com.join(' ')}\", termsig #{status.termsig}"
     end
     unless status.success?
       $logger.error("failed to execute \"#{com.join(' ')}\", exit status #{status.exitstatus}")
       so.split("\n").each { |l| $logger.debug("stdout: #{l}") }
       se.split("\n").each { |l| $logger.error("stderr: #{l}") }
-      raise 'ZfsGetError'
+      raise ZfsGetError, "failed to execute \"#{com.join(' ')}\", exit status #{status.exitstatus}"
     end
     so.split("\n").each do |line|
       params = line.split("\t")
@@ -231,7 +238,7 @@ module ZfsMgmt
     }
     return saved,saved_snaps,deleteme
   end
-  def self.zfs_managed_list(filter: '.+', properties: custom_properties(), property_match: { 'zfsmgmt:manage' => 'true' } )
+  def self.zfs_managed_list(filter: '.+', properties: ['all'], property_match: { 'zfsmgmt:manage' => 'true' } )
     zfss = [] # array of arrays
     zfsget(properties: properties).each do |zfs,props|
       unless /#{filter}/ =~ zfs
@@ -254,20 +261,20 @@ module ZfsMgmt
     end
     return zfss
   end
-  def self.snapshot_policy(verbopt: false, debugopt: false, filter: '.+')
-    if debugopt
-      $logger.level = Logger::DEBUG
-    else
-      $logger.level = Logger::INFO
-    end
-    zfs_managed_list(filter: filter).each do |zdata|
-      (zfs,props,snaps) = zdata
-      unless props.has_key?('zfsmgmt:policy') and policy_parser(props['zfsmgmt:policy'])
-        $logger.error("zfs_mgmt is configured to manage #{zfs}, but there is no valid policy configuration, skipping")
+  def self.snapshot_policy(filter: '.+')
+    zfs_managed_list(filter: filter).each do |zfs,props,snaps|
+      unless props.has_key?('zfsmgmt:policy')
+        $logger.error("zfs_mgmt is configured to manage #{zfs}, but there is no policy configuration in zfsmgmt:policy, skipping")
         next # zfs
       end
-      # call the function that decides who to save and who to delete
-      (saved,saved_snaps,deleteme) = snapshot_destroy_policy(zfs,props,snaps)
+
+      begin
+        # call the function that decides who to save and who to delete
+        (saved,saved_snaps,deleteme) = snapshot_destroy_policy(zfs,props,snaps)
+      rescue ArgumentError
+        $logger.error("zfs_mgmt is configured to manage #{zfs}, but there is no valid policy configuration, skipping")
+        next
+      end
 
       if saved_snaps.length == 0
         $logger.info("no snapshots marked as saved by policy for #{zfs}")
@@ -283,21 +290,20 @@ module ZfsMgmt
       print table.to_s
     end
   end
-  def self.snapshot_destroy(noop: false, verbopt: false, debugopt: false, filter: '.+')
-    if debugopt
-      $logger.level = Logger::DEBUG
-    else
-      $logger.level = Logger::INFO
-    end
-    zfs_managed_list(filter: filter).each do |zdata|
-      (zfs,props,snaps) = zdata
-      unless props.has_key?('zfsmgmt:policy') and policy_parser(props['zfsmgmt:policy'])
-        $logger.error("zfs_mgmt is configured to manage #{zfs}, but there is no valid policy configuration, skipping")
+  def self.snapshot_destroy(noop: false, verbose: false, filter: '.+')
+    zfs_managed_list(filter: filter).each do |zfs,props,snaps|
+      unless props.has_key?('zfsmgmt:policy')
+        $logger.error("zfs_mgmt is configured to manage #{zfs}, but there is no policy configuration in zfsmgmt:policy, skipping")
         next # zfs
       end
 
-      # call the function that decides who to save and who to delete
-      (saved,saved_snaps,deleteme) = snapshot_destroy_policy(zfs,props,snaps)
+      begin
+        # call the function that decides who to save and who to delete
+        (saved,saved_snaps,deleteme) = snapshot_destroy_policy(zfs,props,snaps)
+      rescue ArgumentError
+        $logger.error("zfs_mgmt is configured to manage #{zfs}, but there is no valid policy configuration, skipping")
+        next
+      end
     
       $logger.info("deleting #{deleteme.length} snapshots for #{zfs}")
       deleteme.reverse! # oldest first for removal
@@ -305,32 +311,22 @@ module ZfsMgmt
         $logger.debug("delete: #{snap_name} #{local_epoch_to_datetime(snaps[snap_name]['creation']).strftime('%F %T')}")
       end
 
-      com_base = "zfs destroy -p"
-      if deleteme.length > 0
-        com_base = "#{com_base}d"
-      end
-      if noop
-        com_base = "#{com_base}n"
-      end
-      if verbopt
-        com_base = "#{com_base}v"
-      end
+      com_base = ['zfs', 'destroy']
+      com_base.push('-d') if deleteme.length > 0 # why?
+      com_base.push('-n') if noop
+      com_base.push('-v') if verbose
       while deleteme.length > 0
         for i in 0..(deleteme.length - 1) do
           max = deleteme.length - 1 - i
           $logger.debug("attempting to remove snaps 0 through #{max} out of #{deleteme.length} snapshots")
           bigarg = "#{zfs}@#{deleteme[0..max].map { |s| s.split('@')[1] }.join(',')}"
-          com = "#{com_base} #{bigarg}"
+          com = com_base + [bigarg]
           $logger.debug("size of bigarg: #{bigarg.length} size of com: #{com.length}")
           if bigarg.length >= 131072 or com.length >= (2097152-10000)
             next
           end
-          $logger.info(com)
           deleteme = deleteme - deleteme[0..max]
-          system(com)
-          if $?.exitstatus != 0
-            $logger.error("zfs exited with non-zero status: #{$?.exitstatus}")
-          end
+          system_com(com) # pass -n, always run the command though
           break
         end
       end
@@ -344,7 +340,7 @@ module ZfsMgmt
     end
     p = str.scan(/\d+[#{$time_pattern_map.keys.join('')}]/i)
     unless p.length > 0
-      raise "unable to parse the policy configuration #{str}"
+      raise ArgumentError.new("unable to parse the policy configuration #{str}")
     end
     p.each do |pi|
       scn = /(\d+)([#{$time_pattern_map.keys.join('')}])/i.match(pi)
@@ -352,33 +348,309 @@ module ZfsMgmt
     end
     res
   end
-  def self.snapshot_create(noop: false, verbopt: false, debugopt: false, filter: '.+')
-    if debugopt
-      $logger.level = Logger::DEBUG
+  # snapshot all filesystems configured for snapshotting
+  def self.snapshot_create(noop: false, verbose: false, filter: '.+')
+    dt = DateTime.now
+    zfsget.select { |zfs,props|
+      # must match filter
+      match_filter?(zfs: zfs, filter: filter) and
+        # snapshot must be on or true
+        (
+          key_comp?(props,'zfsmgmt:snapshot') or
+          # or snapshot can be recursive and local, but only if the source is local or received
+          ( key_comp?(props,'zfsmgmt:snapshot',['recursive','local']) and key_comp?(props,'zfsmgmt:snapshot@source',['local','received']) )
+        )
+    }.each do |zfs,props|
+      prefix = ( props.has_key?('zfsmgmt:snap_prefix') ? props['zfsmgmt:snap_prefix'] : 'zfsmgmt' )
+      ts = ( props.has_key?('zfsmgmt:snap_timestamp') ? props['zfsmgmt:snap_timestamp'] : '%FT%T%z' )
+      com = [global_options['zfs_binary'],'snapshot']
+      if key_comp?(props,'zfsmgmt:snapshot','recursive') and key_comp?(props,'zfsmgmt:snapshot@source',['local','received'])
+        com.push('-r')
+      end
+      com.push('-v') if verbose
+      com.push("#{zfs}@#{[prefix,dt.strftime(ts)].join('-')}")
+      system_com(com,noop)
+    end
+  end
+  def self.system_com(com, noop = false)
+    comstr = com.join(' ')
+    $logger.info(comstr)
+    unless noop
+      system(comstr)
+      unless $?.success?
+        $logger.error("command failed: #{$?.exitstatus}")
+      end
+    end
+  end
+  def self.zfs_send(options,zfs,props,snaps)
+    sorted = snaps.keys.sort { |a,b| snaps[a]['creation'] <=> snaps[b]['creation'] }
+    # compute the zfs "path"
+    # ternary operator 4eva
+    destination_path = ( options[:destination] ? options[:destination] : props['zfsmgmt:destination'] )
+    if props['zfsmgmt:destination@source'] == 'local'
+      destination_path = File.join( destination_path,
+                                    File.basename(zfs)
+                                  )
+    elsif m = /inherited from (.+)/.match(props['zfsmgmt:destination@source'])
+      destination_path = File.join( destination_path,
+                                    File.basename(m[1]),
+                                    zfs.sub(m[1],'')
+                                  )
     else
-      $logger.level = Logger::INFO
+      $logger.error("fatal error: #{props['zfsmgmt:destination']} source: #{props['zfsmgmt:destination@source']}")
+      exit(1)
     end
-    dt = DateTime.now
-    zfsget(properties: custom_properties()).each do |zfs,props|
-      unless /#{filter}/ =~ zfs
-        next
+    # does the destination zfs already exist?
+    remote_zfs_state = ''
+    begin
+      recv_zfs = zfsget(zfs: destination_path,
+                        command_prefix: recv_command_prefix(options,props),
+                        #properties: ['receive_resume_token'],
+                       )
+    rescue ZfsGetError
+      $logger.debug("recv filesystem doesn't exist: #{destination_path}")
+      remote_zfs_state = 'missing'
+    else
+      if recv_zfs[destination_path].has_key?('receive_resume_token')
+        remote_zfs_state = recv_zfs[destination_path]['receive_resume_token']
+      else
+        remote_zfs_state = 'present'
+      end
+    end
+
+    if remote_zfs_state == 'missing'
+      # the zfs does not exist, send initial (oldest?) snapshot
+      com = []
+      source = sorted[0]
+      if options[:initial_snapshot] == 'newest' or
+        ( options.has_key?('replicate') and options['replicate'] == true ) or
+        ( props.has_key?('zfsmgmt:send_replicate') and props['zfsmgmt:send_replicate'] == 'true' )
+        source = sorted[-1]
       end
-      # zfs must have snapshot set to true or recursive
-      if props.has_key?('zfsmgmt:snapshot') and
-        props['zfsmgmt:snapshot'] == 'true' or
-        ( props['zfsmgmt:snapshot'] == 'recursive' and props['zfsmgmt:snapshot@source'] == 'local' ) or
-        ( props['zfsmgmt:snapshot'] == 'local' and props['zfsmgmt:snapshot@source'] == 'local' )
-        
-        prefix = ( props.has_key?('zfsmgmt:snap_prefix') ? props['zfsmgmt:snap_prefix'] : 'zfsmgmt' )
-        ts = ( props.has_key?('zfsmgmt:snap_timestamp') ? props['zfsmgmt:snap_timestamp'] : '%FT%T%z' )
-        com = ['zfs','snapshot']
-        if props['zfsmgmt:snapshot'] == 'recursive' and props['zfsmgmt:snapshot@source'] == 'local'
-          com.push('-r')
+      com += zfs_send_com(options,
+                          props,
+                          [],
+                          source,
+                         )
+      e = zfs_send_estimate(com) if options[:verbose] == 'pv'
+      com += mbuffer_command(options) if options[:mbuffer]
+      com += pv_command(options,e) if options[:verbose] == 'pv'
+      com += zfs_recv_com(options,[],props,destination_path)
+ 
+      system_com(com)
+      unless $?.success?
+        return
+      end
+
+    elsif remote_zfs_state != 'present'
+      # should be resumable!
+      com = [ ]
+      com.push( ZfsMgmt.global_options[:zfs_binary], 'send', '-t', remote_zfs_state )
+      com.push('-v','-P') if options[:verbose] and options[:verbose] == 'send'
+      com.push('|')
+      e = zfs_send_estimate(com) if options[:verbose] == 'pv'
+      com += mbuffer_command(options) if options[:mbuffer]
+      com += pv_command(options,e) if options[:verbose] == 'pv'
+
+      recv = [ ZfsMgmt.global_options[:zfs_binary], 'recv', '-s' ]
+      recv.push('-n') if options[:noop]
+      recv.push('-u') if options[:unmount]
+      recv.push('-v') if options[:verbose] and ( options[:verbose] == 'receive' or options[:verbose] == 'recv' )
+      recv.push(dq(destination_path))
+
+      if options[:remote] or props['zfsmgmt:remote']
+        if options[:mbuffer]
+          recv = mbuffer_command(options) + recv
         end
-        com.push("#{zfs}@#{[prefix,dt.strftime(ts)].join('-')}")
-        $logger.info(com)
-        system(com.join(' '))
+        recv = recv_command_prefix(options,props) + [ sq(recv.join(' ')) ]
       end
+
+      com += recv
+
+      system_com(com)
+      unless $?.success?
+        return
+      end
+    end
+    
+    # the zfs already exists, so update with incremental?
+    begin
+      remote_snaps = zfsget(zfs: destination_path,
+                            types: ['snapshot'],
+                            command_prefix: recv_command_prefix(options,props),
+                            properties: ['creation','userrefs'],
+                           )
+    rescue ZfsGetError
+      $logger.error("unable to get remote snapshot information for #{destination_path}")
+      return
+    end
+    unless remote_snaps and remote_snaps.keys.length > 0
+      $logger.error("receiving filesystem has NO snapshots, it must be destroyed: #{destination_path}")
+      return
+    end
+    if remote_snaps.has_key?(sorted[-1].sub(zfs,destination_path))
+      $logger.info("the most recent local snapshot (#{sorted[-1]}) already exists on the remote side (#{sorted[-1].sub(zfs,destination_path)})")
+      return
+    end
+    remote_snaps.sort_by { |k,v| -v['creation'] }.each do |rsnap,v|
+      # oldest first
+      #pp rsnap,rsnap.sub(destination_path,zfs)
+      #pp snaps
+      if snaps.has_key?(rsnap.sub(destination_path,zfs))
+        $logger.debug("process #{rsnap} to #{sorted[-1]}")
+        com = []
+        com += zfs_send_com(options,props,[(options[:intermediary] ? '-I' : '-i'),dq(rsnap.split('@')[1])],sorted[-1])
+        e = zfs_send_estimate(com) if options[:verbose] == 'pv'
+        com += mbuffer_command(options) if options[:mbuffer]
+        com += pv_command(options,e) if options[:verbose] == 'pv'
+        com += zfs_recv_com(options,[],props,destination_path)
+
+        system_com(com)
+        return
+      end
+      $logger.debug("skipping remote snapshot #{rsnap} because the same snapshot doesn't exist locally #{rsnap.sub(destination_path,zfs)}")
+    end
+    $logger.error("receiving filesystem has no snapshots that still exists on the sending side, it must be destroyed: #{destination_path}")
+    
+  end
+  def self.mbuffer_command(options)
+    mbuffer_command = [ ZfsMgmt.global_options[:mbuffer_binary] ]
+    mbuffer_command.push('-q') unless options[:verbose] == 'mbuffer'
+    mbuffer_command.push('-m',options[:mbuffer_size]) if options[:mbuffer_size]
+    mbuffer_command.push('|')
+    mbuffer_command
+  end
+  def self.zfs_send_com(options,props,extra_opts,target)
+    zfs_send_com = [ ZfsMgmt.global_options[:zfs_binary], 'send' ]
+    zfs_send_com.push('-v','-P') if options[:verbose] and options[:verbose] == 'send'
+    send_opts = {
+      'backup'      => '-b',
+      'compressed'  => '-c',
+      'embed'       => '-e',
+      'holds'       => '-h',
+      'large_block' => '-L',
+      'props'       => '-p',
+      'raw'         => '-w',
+      'replicate'   => '-R',
+    }
+    send_opts.each do |p,o|
+      if options.has_key?(p)
+        zfs_send_com.push(o) if options[p] == true
+      elsif props.has_key?("zfsmgmt:send_#{p}")
+        zfs_send_com.push(o) if props["zfsmgmt:send_#{p}"] == 'true'
+      end
+    end
+    zfs_send_com + extra_opts + [dq(target),'|']
+  end
+  def self.zfs_recv_com(options,extra_opts,props,target)
+    zfs_recv_com = [ ZfsMgmt.global_options[:zfs_binary], 'recv', '-F', '-s' ]
+    recv_opts = {
+      'noop'       => '-n',
+      'drop_holds' => '-h',
+      'unmount'    => '-u',
+    }
+    recv_opts.each do |p,o|
+      if options.has_key?(p)
+        zfs_recv_com.push(o) if options[p] == true
+      elsif props.has_key?("zfsmgmt:recv_#{p}")
+        zfs_recv_com.push(o) if props["zfsmgmt:recv_#{p}"] == 'true'
+      end
+    end
+    zfs_recv_com.push('-v') if options[:verbose] and ( options[:verbose] == 'receive' or options[:verbose] == 'recv' )
+    if options[:exclude]
+      options[:exclude].each do |x|
+        zfs_recv_com.push('-x',x)
+      end
+    end
+    if options[:option]
+      options[:option].each do |x|
+        zfs_recv_com.push('-o',x)
+      end
+    end
+    zfs_recv_com += extra_opts
+    zfs_recv_com.push(dq(target))
+
+    if options[:remote] or props['zfsmgmt:remote']
+      if options[:mbuffer]
+        zfs_recv_com = mbuffer_command(options) + zfs_recv_com
+      end
+      zfs_recv_com = recv_command_prefix(options,props) + [ sq(zfs_recv_com.join(' ')) ]
+    end
+    zfs_recv_com
+  end
+  def self.recv_command_prefix(options,props)
+    ( (options[:remote] or props['zfsmgmt:remote']) ?
+        [ 'ssh', ( options[:remote] ? options[:remote] : props['zfsmgmt:remote'] ) ] :
+        [] )
+  end
+  def self.zfs_send_estimate(com)
+    lcom = com.dup
+    lcom.pop() # remove the pipe symbol
+    precom = [ lcom.shift, lcom.shift ]
+    lcom.unshift('-P') unless lcom.include?('-P')
+    lcom.unshift('-n')
+    lcom.push('2>&1')
+    lcom = precom + lcom
+    $logger.debug(lcom.join(' '))
+    %x[#{lcom.join(' ')}].each_line do |l|
+      if m = /(incremental|size).*\s+(\d+)$/.match(l)
+        return m[2].to_i
+      end
+    end
+    $logger.error("no estimate available")
+    return nil
+  end
+  def self.pv_command(options,estimate)
+    a = []
+    a += [options[:pv_binary], '-prb' ]
+    if estimate
+      a += ['-e', '-s', estimate ]
+    end
+    a.push('|')
+    a
+  end
+    
+  def self.sq(s)
+    "'#{s}'"
+  end
+  def self.dq(s)
+    "\"#{s}\""
+  end
+  def self.prop_on?(v)
+    ['true','on'].include?(v)
+  end
+  def self.match_filter?(zfs:, filter:)
+    /#{filter}/ =~ zfs
+  end
+  def self.key_comp?(h,p,v = method(:prop_on?))
+    #$logger.debug("p:#{p}\th[p]:#{h[p]}\tv:#{v}")
+    return false unless h.has_key?(p)
+    if v.kind_of?(Array)
+      return v.include?(h[p])
+    elsif v.kind_of?(Hash)
+      return v.keys.include?(h[p])
+    elsif v.kind_of?(String)
+      return h[p] == v
+    elsif v.kind_of?(Method)
+      return v.call(h[p])
+    elsif v.kind_of?(Regexp)
+      return v =~ h[p]
+    else
+      raise ArgumentError
+    end
+  end
+  def self.set_log_level(sev)
+    case sev
+    when 'debug'
+      $logger.level = Logger::DEBUG
+    when 'info'
+      $logger.level = Logger::INFO
+    when 'warn'
+      $logger.level = Logger::WARN
+    when 'error'
+      $logger.level = Logger::ERROR
+    when 'fatal'
+      $logger.level = Logger::FATAL
     end
   end
 end

data/lib/zfs_mgmt/restic.rb

@@ -9,11 +9,16 @@ module ZfsMgmt::Restic
             '--tag', 'zfsmgmt',
             '--path', "/#{zfs}",
           ]
-    if props.has_key?('zfsmgmt:restic_repository')
-      com.push( '-r', props['zfsmgmt:restic_repository'] )
+    if options.has_key?('password_file')
+      com.push('-p',options['password_file'])
     end
-    
-    $logger.debug("#{com.join(' ')}")
+    if options.has_key?('repo')
+      com.push('--repo', options['repo'])
+    elsif props.has_key?('zfsmgmt:restic_repository')
+      com.push( '--repo', props['zfsmgmt:restic_repository'] )
+    end
+
+    $logger.info("#{com.join(' ')}")
     restic_output = %x(#{com.join(' ')})
     unless $?.success?
       $logger.error("unable to query the restic database")
@@ -102,7 +107,7 @@ module ZfsMgmt::Restic
                "zfsmgmt:snapshot=#{last_zfs_snapshot}",
                "zfsmgmt:zfs=#{zfs}",
                "zfsmgmt:level=#{level}" ]
-      com = [ options[:zfs_binary], 'send', '-L', '-w', '-h', '-p' ]
+      com = [ ZfsMgmt.global_options['zfs_binary'], 'send', '-w', '-h', '-p' ]
       if level > 0
         if options[:intermediary]
           com.push('-I')
@@ -119,8 +124,16 @@ module ZfsMgmt::Restic
       tags.each do |tag|
         com.push( '--tag', "\"#{tag}\"" )
       end
-      if props.has_key?('zfsmgmt:restic_repository')
-        com.push( '-r', props['zfsmgmt:restic_repository'] )
+      if options.has_key?('limit_upload')
+        com.push('--limit-upload', options['limit_upload'])
+      end
+      if options.has_key?('password_file')
+        com.push('-p',options['password_file'])
+      end
+      if options.has_key?('repo')
+        com.push('--repo', options['repo'])
+      elsif props.has_key?('zfsmgmt:restic_repository')
+        com.push( '--repo', props['zfsmgmt:restic_repository'] )
       end
       if options[:verbose]
         com.push('--verbose',options[:verbose])
@@ -130,8 +143,7 @@ module ZfsMgmt::Restic
       unless ZfsMgmt.zfs_holds(last_zfs_snapshot).include?('zfsmgmt_restic')
         ZfsMgmt.zfs_hold('zfsmgmt_restic',last_zfs_snapshot)
       end
-      $logger.info("#{com.join(' ')}")
-      system(com.join(' '))
+      ZfsMgmt.system_com(com)
       chain_snaps = chain.map do |rsnap|
         rsnap['zfsmgmt:snapshot']
       end

data/lib/zfs_mgmt/version.rb

@@ -1,3 +1,3 @@
 module ZfsMgmt
-  VERSION = "0.3.6"
+  VERSION = "0.4.0"
 end

data/lib/zfs_mgmt/zfs_mgr.rb

@@ -3,4 +3,5 @@ module ZfsMgmt::ZfsMgr
   require "zfs_mgmt/zfs_mgr/list"
   require "zfs_mgmt/zfs_mgr/restic"
   require "zfs_mgmt/zfs_mgr/snapshot"
+  require "zfs_mgmt/zfs_mgr/send"
 end

data/lib/zfs_mgmt/zfs_mgr/list.rb

@@ -5,20 +5,53 @@ class ZfsMgmt::ZfsMgr::List < Thor
                :desc => 'only act on zfs matching this regexp'
   desc "stale", "list all zfs with stale snapshots"
   method_option :age, :desc => "timeframe outside of which the zfs will be considered stale", :default => '1d'
+  method_option :format, :desc => "output format", :type => :string, :enum => ['table','tab'], :default => 'table'
   def stale()
+    ZfsMgmt.set_log_level(options[:loglevel])
+    ZfsMgmt.global_options = options
     cutoff = Time.at(Time.now.to_i -  ZfsMgmt.timespec_to_seconds(options[:age]))
     table = Text::Table.new
     table.head = ['zfs','snapshot','age']
     table.rows = []
-    ZfsMgmt.zfs_managed_list(filter: options[:filter]).each do |blob|
-      zfs,props,snaps = blob
+    ZfsMgmt.zfs_managed_list(filter: options[:filter]).each do |zfs,props,snap|
       last = snaps.keys.sort { |a,b| snaps[a]['creation'] <=> snaps[b]['creation'] }.last
       snap_time = Time.at(snaps[last]['creation'])
       if snap_time < cutoff
-        table.rows << [zfs,last.split('@')[1],snap_time]
+        line = [zfs,last.split('@')[1],snap_time]
+        table.rows << line
+        if options[:format] == 'tab'
+          print line.join("\t"),"\n"
+        end
       end
     end
-    if table.rows.count > 0
+    if options[:format] == 'table' and table.rows.count > 0
+      print table.to_s
+    end
+  end
+  desc "holds", "list all holds on snapshots"
+  method_option :format, :desc => "output format", :type => :string, :enum => ['table','tab','release'], :default => 'table'
+  def holds()
+    ZfsMgmt.set_log_level(options[:loglevel])
+    ZfsMgmt.global_options = options
+    table = Text::Table.new
+    table.head = ['snapshot','userrefs','holds']
+    table.rows = []
+    ZfsMgmt.zfs_managed_list(filter: options[:filter], property_match: {} ).each do |zfs,props,snaps|
+      snaps.sort_by { |x,y| y['creation'] }.each do |snap,d|
+        if d['userrefs'] > 0
+          line = [snap,d['userrefs'].to_s,ZfsMgmt.zfs_holds(snap).join(',')]
+          table.rows << line
+          if options[:format] == 'tab'
+            print line.join("\t"),"\n"
+          elsif options[:format] == 'release'
+            ZfsMgmt.zfs_holds(snap).each do |hold|
+              print "zfs release #{hold} #{snap}\n"
+            end
+          end
+        end
+      end
+    end
+    if options[:format] == 'table' and table.rows.count > 0
       print table.to_s
     end
   end

data/lib/zfs_mgmt/zfs_mgr/restic.rb

@@ -5,25 +5,35 @@ class ZfsMgmt::ZfsMgr::Backup < Thor
                :desc => 'only act on zfs matching this regexp'
   class_option :restic_binary, :type => :string, :default => 'restic',
                :desc => 'restic binary'
-  class_option :zfs_binary, :type => :string, :default => 'zfs',
-               :desc => 'zfs binary'
   class_option :verbose, :alias => '-v', :type => :numeric,
                :desc => 'verbosity level for restic'
   class_option :buffer, :type => :string, :default => '256m',
                :desc => 'buffer size for mbuffer'
+  class_option :password_file, :alias => '-p', :type => :string,
+               :desc => 'passed to restic'
+  class_option :limit_upload, :type => :numeric,
+               :desc => 'passed to restic'
+  class_option :repo, :type => :string,
+               :desc => 'passed to restic'
   desc "incremental", "perform incremental backup"
   method_option :level, :desc => "backup level in integer form", :default => 2, :type => :numeric
   method_option :intermediary, :alias => '-I', :desc => "pass -I (intermediary) option to zfs send", :default => false, :type => :boolean
   def incremental()
+    ZfsMgmt.set_log_level(options[:loglevel])
+    ZfsMgmt.global_options = options
     ZfsMgmt::Restic.backup(backup_level: options[:level], options: options)
   end
   desc "differential", "perform differential backup"
   method_option :intermediary, :alias => '-I', :desc => "pass -I (intermediary) option to zfs send", :default => false, :type => :boolean
   def differential()
+    ZfsMgmt.set_log_level(options[:loglevel])
+    ZfsMgmt.global_options = options
     ZfsMgmt::Restic.backup(backup_level: 1, options: options)
   end
   desc "full", "perform full backup"
   def full()
+    ZfsMgmt.set_log_level(options[:loglevel])
+    ZfsMgmt.global_options = options
     ZfsMgmt::Restic.backup(backup_level: 0, options: options)
   end
 end

data/lib/zfs_mgmt/zfs_mgr/send.rb

@@ -0,0 +1,63 @@
+# zfs send stuff
+
+class ZfsMgmt::ZfsMgr::Send < Thor
+  class_option :filter, :type => :string, :default => '.+',
+               :desc => 'only act on zfs matching this regexp'
+  desc "all", "send all zfs configured via user properties"
+  method_option :remote, :type => :string,
+                :desc => 'remote specification like root@otherhost or localhost'
+  method_option :destination, :type => :string,
+                :desc => 'destination path like otherpool/ourpool'
+  method_option :verbose, :type => :string, :aliases => :'-v', :enum => ['send','receive','recv','mbuffer','pv'],
+                :desc => 'enable verbose output on the specified element of the pipe'
+  method_option :initial_snapshot, :type => :string, :enum => ['oldest','newest'], :default => 'oldest',
+                :desc => 'when sending the initial snapshot use the oldest or most recent snapshot'
+
+  method_option :intermediary, :aliases => :'-I', :desc => "pass -I option to zfs send", :type => :boolean
+  method_option :backup, :aliases => :'-p', :desc => "pass -b (--backup) option to zfs send", :type => :boolean
+  method_option :compressed, :aliases => :'-c', :desc => "pass -c (compressed) option to zfs send", :type => :boolean
+  method_option :embed, :aliases => :'-e', :desc => "pass -e (--embed) option to zfs send", :type => :boolean
+  method_option :holds, :aliases => :'-h', :desc => "pass the -h (--holds) option to zfs send", :type => :boolean
+  method_option :large_block, :aliases => :'-L', :desc => "pass -L (--large-block) option to zfs send", :type => :boolean
+  method_option :props, :aliases => :'-p', :desc => "pass -p (--props) option to zfs send", :type => :boolean
+  method_option :raw, :aliases => :'-w', :desc => "pass -w (--raw) option to zfs send", :type => :boolean
+  method_option :replicate, :aliases => :'-R', :desc => "pass -R (--replicate) option to zfs send", :type => :boolean
+
+  method_option :noop, :aliases => :'-n', :desc => "pass -n (noop) option to zfs send", :type => :boolean
+  method_option :unmount, :aliases => :'-u', :desc => "pass -u (unmount) option to zfs receive", :type => :boolean
+  method_option :exclude, :aliases => :'-x', :desc => "passed to -x option of receive side", :type => :array
+  method_option :option, :aliases => :'-o', :desc => "passed to -o option of receive side", :type => :array
+  method_option :drop_holds, :desc => "pass the -h option to zfs recv, indicating holds should be ignored", :type => :boolean
+
+  method_option :mbuffer, :desc => "insert mbuffer between send and recv", :default => true, :type => :boolean
+  method_option :mbuffer_size, :desc => "passed to mbuffer -s option", :type => :string
+  def all()
+    ZfsMgmt.set_log_level(options[:loglevel])
+    ZfsMgmt.global_options = options
+
+    [
+      { 'zfsmgmt:send' => 'true' },
+      # {
+      #   'zfsmgmt:send' => 'replicate',
+      #   'zfsmgmt:send@source' => 'local'
+      # },
+    ].each do |match|
+      ZfsMgmt.zfs_managed_list(filter: options[:filter],
+                               property_match: match).each do |zfs,props,snaps|
+        if props['zfsmgmt:send@source'] == 'received'
+          $logger.debug("skipping received filesystem: #{zfs}")
+          next
+        end
+        if props.has_key?('zfsmgmt:send_replicate') and props['zfsmgmt:send_replicate'] == 'true' and props['zfsmgmt:send_replicate@source'] != 'local'
+          $logger.debug("skipping descendant of replicated filesystems: #{zfs}")
+          next
+        end
+        unless props['zfsmgmt:destination']
+          $logger.error("#{zfs}: you must specify a destination zfs path via the user property zfsmgmt:destination, even if using --destination on the command line, skipping")
+          next
+        end
+        ZfsMgmt.zfs_send(options,zfs,props,snaps)
+      end
+    end
+  end
+end

data/lib/zfs_mgmt/zfs_mgr/snapshot.rb

@@ -1,24 +1,32 @@
 # implement snapshot management
 
 class ZfsMgmt::ZfsMgr::Snapshot < Thor
-  class_option :noop, :type => :boolean, :default => false,
-               :desc => 'pass -n option to zfs commands'
-  class_option :verbose, :type => :boolean, :default => false,
-               :desc => 'pass -v option to zfs commands'
-  class_option :debug, :type => :boolean, :default => false,
-               :desc => 'set logging level to debug'
   class_option :filter, :type => :string, :default => '.+',
                :desc => 'only act on zfs matching this regexp'
   desc "destroy", "apply the snapshot destroy policy to zfs"
+  method_option :noop, :type => :boolean, :default => false,
+                :desc => 'pass -n option to zfs commands'
+  method_option :verbose, :type => :boolean, :default => false,
+                :desc => 'pass -v option to zfs commands'
   def destroy()
-    ZfsMgmt.snapshot_destroy(noop: options[:noop], verbopt: options[:verbose], debugopt: options[:debug], filter: options[:filter])
+    ZfsMgmt.set_log_level(options[:loglevel])
+    ZfsMgmt.global_options = options
+    ZfsMgmt.snapshot_destroy(noop: options[:noop], verbose: options[:verbose], filter: options[:filter])
   end
   desc "policy", "print the policy table for zfs"
   def policy()
-    ZfsMgmt.snapshot_policy(verbopt: options[:verbose], debugopt: options[:debug], filter: options[:filter])
+    ZfsMgmt.set_log_level(options[:loglevel])
+    ZfsMgmt.global_options = options
+    ZfsMgmt.snapshot_policy(filter: options[:filter])
   end
   desc "create", "execute zfs snapshot based on zfs properties"
+  method_option :noop, :type => :boolean, :default => false,
+                :desc => 'log snapshot commands without running zfs snapshot'
+  method_option :verbose, :type => :boolean, :default => false,
+                :desc => 'pass -v option to zfs commands'
   def create()
-    ZfsMgmt.snapshot_create(verbopt: options[:verbose], debugopt: options[:debug], filter: options[:filter])
+    ZfsMgmt.set_log_level(options[:loglevel])
+    ZfsMgmt.global_options = options
+    ZfsMgmt.snapshot_create(noop: options[:noop], verbose: options[:verbose], filter: options[:filter])
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zfs_mgmt
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.4.0
 platform: ruby
 authors:
 - Aran Cox
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-18 00:00:00.000000000 Z
+date: 2021-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -130,6 +130,7 @@ files:
 - lib/zfs_mgmt/zfs_mgr.rb
 - lib/zfs_mgmt/zfs_mgr/list.rb
 - lib/zfs_mgmt/zfs_mgr/restic.rb
+- lib/zfs_mgmt/zfs_mgr/send.rb
 - lib/zfs_mgmt/zfs_mgr/snapshot.rb
 - zfs_mgmt.gemspec
 homepage: https://github.com/aranc23/zfs_mgmt