zfs_mgmt 0.2.4

data/Gemfile.lock

@@ -0,0 +1,41 @@
+PATH
+  remote: .
+  specs:
+    zfs_mgmt (0.2.4)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.3)
+    filesize (0.2.0)
+    rake (13.0.1)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.1)
+      rspec-support (~> 3.9.1)
+    rspec-expectations (3.9.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.2)
+    text-table (1.2.4)
+    thor (1.0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  filesize
+  rake (>= 12.3.3)
+  rspec (~> 3.0)
+  text-table
+  thor
+  zfs_mgmt!
+
+BUNDLED WITH
+   1.16.6

data/README.md

@@ -0,0 +1,191 @@
+# ZfsMgmt
+
+zfs_mgmt aims to provide some useful helpers for managing zfs snapshots, and eventually send/recv duties via the zfsmgr script in bin/.
+
+Currently only snapshot destruction is implemented by a policy specification stored in zfs properties.
+
+## Installation
+
+Currently zfs_mgmt is only useful for it's zfsmgr binary, although
+eventually the library might be useful for writing other applications
+around managing zfs.
+
+Therefore, building the gem and installing, or running ruby inside the src/ directory would be most useful:
+
+    $ ruby -I lib bin/zfsmgr
+
+## Usage
+
+The most common usage pattern would be to set zfs properties as explained below, then use **zfsmgr snapshot policy** to print a table of what would be kept and for what reason.  Then use **zfsmgr snapshot destroy --noop** to see what would be destroyed, and finally **zfsmgr snapshot destroy** without the --noop option to actually remove snapshots.
+
+    Commands:
+      zfsmgr help [COMMAND]               # Describe available commands or one specific command
+      zfsmgr snapshot SUBCOMMAND ...ARGS  # manage snapshots
+      zfsmgr zfsget [ZFS]                 # execute zfs get for the given properties and types and parse the output into a nested hash
+    
+    
+      zfsmgr snapshot create          # execute zfs snapshot based on zfs properties
+      zfsmgr snapshot destroy         # apply the snapshot destroy policy to zfs
+      zfsmgr snapshot help [COMMAND]  # Describe subcommands or one specific subcommand
+      zfsmgr snapshot policy          # print the policy table for zfs
+    
+    Options:
+      [--noop], [--no-noop]        # pass -n option to zfs commands
+      [--verbose], [--no-verbose]  # pass -v option to zfs commands
+      [--debug], [--no-debug]      # set logging level to debug
+      [--filter=FILTER]            # only act on zfs matching this regexp
+                                   # Default: .+
+
+
+## Example output
+    [aranc23@beast:~/src/zfs_mgmt] (master)$ zfs get all | egrep 'zfsmgmt.+local'
+    backup                      zfsmgmt:manage        true                                          local
+    backup                      zfsmgmt:policy        10y60m104w365d168h                            local
+    backup                      zfsmgmt:minage        7D                                            local
+    backup                      zfsmgmt:ignoresnaps   ^syncoid_                                     local
+    backup/beast/data/archive   zfsmgmt:policy        1h                                            local
+    backup/beast/data/archive   zfsmgmt:minage        1s                                            local
+    backup/beast/data/archive   zfsmgmt:matchsnaps    archive                                       local
+
+    [aranc23@beast:~/src/zfs_mgmt] (master)$ ruby -I lib bin/zfsmgr snapshot policy --filter pics                                                                 
+    +------------------------------------------------------------+---------------------------+--------------------+------------+--------------+---------+--------+
+    |                            snap                            |         creation          |       hourly       |   daily    |    weekly    | monthly | yearly |
+    +------------------------------------------------------------+---------------------------+--------------------+------------+--------------+---------+--------+
+    | backup/beast/data/pics@autosnap-2020-02-27T12:17:01-0600   | 2020-02-27T12:17:01-06:00 | 2020-02-27 Hour 12 |            |              |         |        |
+    | backup/beast/data/pics@autosnap-2020-02-27T11:17:01-0600   | 2020-02-27T11:17:01-06:00 | 2020-02-27 Hour 11 |            |              |         |        |
+    | backup/beast/data/pics@autosnap-2020-02-27T10:17:01-0600   | 2020-02-27T10:17:01-06:00 | 2020-02-27 Hour 10 |            |              |         |        |
+    | backup/beast/data/pics@autosnap-2020-02-27T09:17:01-0600   | 2020-02-27T09:17:02-06:00 | 2020-02-27 Hour 09 |            |              |         |        |
+    | backup/beast/data/pics@autosnap-2020-02-27T08:17:01-0600   | 2020-02-27T08:17:01-06:00 | 2020-02-27 Hour 08 |            |              |         |        |
+    | backup/beast/data/pics@autosnap-2020-02-27T07:17:01-0600   | 2020-02-27T07:17:01-06:00 | 2020-02-27 Hour 07 |            |              |         |        |
+    | backup/beast/data/pics@autosnap-2020-02-27T06:17:01-0600   | 2020-02-27T06:17:01-06:00 | 2020-02-27 Hour 06 |            |              |         |        |
+    | backup/beast/data/pics@autosnap-2020-02-27T05:17:01-0600   | 2020-02-27T05:17:01-06:00 | 2020-02-27 Hour 05 |            |              |         |        |
+    | backup/beast/data/pics@autosnap-2020-02-27T04:17:01-0600   | 2020-02-27T04:17:02-06:00 | 2020-02-27 Hour 04 |            |              |         |        |
+    | backup/beast/data/pics@autosnap-2020-02-27T03:17:01-0600   | 2020-02-27T03:17:01-06:00 | 2020-02-27 Hour 03 |            |              |         |        |
+    | backup/beast/data/pics@autosnap-2020-02-27T02:17:01-0600   | 2020-02-27T02:17:01-06:00 | 2020-02-27 Hour 02 |            |              |         |        |
+    | backup/beast/data/pics@autosnap-2020-02-27T01:17:01-0600   | 2020-02-27T01:17:02-06:00 | 2020-02-27 Hour 01 |            |              |         |        |
+    | backup/beast/data/pics@autosnap-2020-02-27T00:17:01-0600   | 2020-02-27T00:17:01-06:00 | 2020-02-27 Hour 00 | 2020-02-27 |              |         |        |
+    ...
+    | backup/beast/data/pics@zfssendman-20140604092215           | 2014-06-04T09:22:43-05:00 |                    | 2014-06-04 | 2014 Week 22 | 2014-06 |        |
+    | backup/beast/data/pics@migrate3                            | 2014-05-26T08:17:31-05:00 |                    | 2014-05-26 |              |         |        |
+    | backup/beast/data/pics@migrate2                            | 2014-05-25T21:57:28-05:00 |                    | 2014-05-25 | 2014 Week 21 |         |        |
+    | backup/beast/data/pics@migrate1                            | 2014-05-24T10:31:56-05:00 |                    | 2014-05-24 | 2014 Week 20 | 2014-05 | 2014   |
+    | backup/beast/data/pics@20131108144154                      | 2013-11-08T14:41:57-06:00 |                    | 2013-11-08 | 2013 Week 44 | 2013-11 | 2013   |
+    +------------------------------------------------------------+---------------------------+--------------------+------------+--------------+---------+--------+
+
+    [aranc23@beast:~/src/zfs_mgmt] (master)$ ruby -I lib bin/zfsmgr snapshot destroy --filter pics --noop
+    I, [2020-02-27T16:27:33.381645 #4914]  INFO -- : deleting 21 snapshots for backup/beast/data/pics
+    I, [2020-02-27T16:27:33.381731 #4914]  INFO -- : zfs destroy -pn backup/beast/data/pics@autosnap_2020-02-19_21:00:05_hourly,autosnap_2020-02-19_22:00:05_hourly,autosnap_2020-02-19_23:00:01_hourly,autosnap_2020-02-20_00:00:05_daily,autosnap_2020-02-20_01:00:04_hourly,autosnap_2020-02-20_02:00:04_hourly,autosnap_2020-02-20_03:00:04_hourly,autosnap_2020-02-20_04:00:05_hourly,autosnap_2020-02-20_05:00:05_hourly,autosnap_2020-02-20_07:00:04_hourly,autosnap_2020-02-20_08:00:01_hourly,autosnap_2020-02-20_09:00:05_hourly,autosnap_2020-02-20_10:00:05_hourly,autosnap_2020-02-20_11:00:05_hourly,autosnap_2020-02-20_12:00:05_hourly,autosnap_2020-02-20_13:00:01_hourly,autosnap_2020-02-20_14:00:05_hourly,autosnap_2020-02-20_15:00:05_hourly,autosnap_2020-02-20_16:00:05_hourly,autosnap_2020-02-20_17:00:05_hourly,autosnap_2020-02-20_18:00:05_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-19_21:00:05_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-19_22:00:05_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-19_23:00:01_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-20_00:00:05_daily
+    destroy backup/beast/data/pics@autosnap_2020-02-20_01:00:04_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-20_02:00:04_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-20_03:00:04_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-20_04:00:05_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-20_05:00:05_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-20_07:00:04_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-20_08:00:01_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-20_09:00:05_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-20_10:00:05_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-20_11:00:05_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-20_12:00:05_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-20_13:00:01_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-20_14:00:05_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-20_15:00:05_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-20_16:00:05_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-20_17:00:05_hourly
+    destroy backup/beast/data/pics@autosnap_2020-02-20_18:00:05_hourly
+    reclaim 0
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/aranc23/zfs_mgmt.
+
+## zfs user properties
+
+Destruction of zfs snapshots is based on the following zfs user properties:
+
+### zfsmgmt:manage
+manage snapshots for this filesystem if this property is 'true' (string literal)
+ 
+### zfsmgmt:policy
+
+A policy specification consisting of the number of snapshots of a
+certain time frame to keep.  A zfs must have a valid policy
+specification or zfs_mgmt will not destroy any snapshots.
+
+Examples:
+- 30d ( 30 daily snapshots )
+- 8w15d (8 weekly, and 15 daily snapshots)
+- 1y1m1y1d1h (1 of each time frame worth of snapshots)
+- 72h (72 hourly snapshots)
+
+The order in which each timeframe is listed in does not matter, and the supported specs are as follows:
+
+- h - hourly
+- d - daily
+- w - weekly (Sunday)
+- m - monthly
+- y - yearly
+
+### zfsmgmt:minage
+The minimum age of a snapshot before it will be considered for
+deletion, as specified in seconds, or using a multiplier of:
+
+- s (seconds, same as not specifiying a multiplier)
+- m (minutes, x60)
+- h (hours, x60x60)
+- d (days, x24x60x60)
+- w (weeks, x7x24x60x60)
+
+The intended purpose of minage is to keep recent snapshots regardless
+of policy, possibly to ensure zfs send/recv has recent snapshots to
+work with, or simply out of paranoia.
+
+### zfsmgmt:matchsnaps
+If this property is set, the snapshot portion of a snapshot name
+(right of the @) must match this as interpreted as a regular
+expression in order to match the policy as specified above.  The
+intended use is to match application specific snapshots, (ie: ^backup-
+) in an environment where automatic snapshots are still created but
+there is no need to keep them. Snapshots matching this pattern can and
+will still be deleted if they aren't marked to be saved by the policy
+in place for the zfs.
+
+### zfsmgmt:ignoresnaps
+Ignore snapshots matching this regexp pattern.  They are neither used
+to match the specified policy for the zfs, nor will they be deleted.
+The intended use is match zfs send/recv snapshots or hand-created
+snapshots, etc.  ie: ^syncoid_
+
+### zfsmgmt:snapshot
+If this property is 'true' then create a snapshot in the format of
+zfsmgmt-%FT%T%z.  If this property is 'recursive' then create a
+recursive snapshot of this zfs.
+
+### zfsmgmt:snap_prefix
+Change the zfsmgmt portion of created snapshots, ie: 'autosnap' would
+create snapshots called autosnap-%FT%T%z.
+
+### zfsmgmt:snap_timestamp
+strftime format string used when creating snapshot names, default
+being %FT%T%z.
+
+## Snapshot Management / zfs destroy
+When destroying snapshots according to a given policy, all snapshots
+should be considered for deletion and all snapshots should be
+considered as potentially satisfying the retention policy regardless
+of the name of the snapshot.  Only the creation property really
+matters unless the user configures zfsmgmt otherwise.  If the user
+wants to preserve a given snapshot it should be preserved using the
+zfs hold mechanism or excluded by the ignoresnaps property.  This
+allows zfs_mgmt to manage snapshots indepentantly of the mechanism
+used to create them.
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "zfs_mgmt"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/readsnaps

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'zfs_mgmt'
+print ZfsMgmt.readsnaps()

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bin/zfs-list-snapshots

@@ -0,0 +1,10 @@
+#! /bin/bash -e
+
+timeout=1800
+output=/etc/zfs-list-snapshots.txt
+lock=/etc/zfs-list-snapshots.lock
+tempfile=$(mktemp)
+
+flock -w $timeout $lock /usr/sbin/zfs list -Hprt snapshot -o name,creation,used,written,refer,logicalreferenced,logicalused,zfssnapman:destroy,zfssnapman:snap -s creation > $tempfile
+mv -f $tempfile $output
+chmod 0644 $output

data/bin/zfsfuncs

@@ -0,0 +1,126 @@
+#-*- mode: sh ; -*-
+
+function zfssendrecv {
+    local OPTIND OPTARG opt
+    local zfs
+    local snap
+    local dest
+    local inc=''
+    local hold=0
+    while getopts "z:s:d:i:I:h" opt; do
+        case $opt in
+            z)
+                zfs=$OPTARG
+                ;;
+            s)
+                snap=$OPTARG
+                ;;
+            d)
+                dest=$OPTARG
+                ;;
+            i)
+                inc="-i ${OPTARG}"
+                ;;
+            I)
+                inc="-I ${OPTARG}"
+                ;;
+            h)
+                hold=1
+                ;;
+        esac
+    done
+    shift $((OPTIND-1))
+    local zfs_normal=$( echo $zfs|sed 's/[\:\|\/\\ ]/_/g' )
+    local lock="${LOCK_DIR}/${zfs_normal}.lock"
+    local zfs_recv_status
+    local zfs_send_status
+    local pipe_status
+    (
+        if ! $FLOCK; then
+            $ulog "unable to lock ${lock}"
+            return -2
+        fi
+        if [[ $TEST == 0 && $hold == 1 ]]; then
+            zfs hold -r zfsrecvman $snap 2>&1 | $ulog
+            local hold_status="${PIPESTATUS[0]}"
+            if [[ $hold_status != 0 ]]; then
+                $ulog "unable to place a hold on our snapshots: ${snap}"
+                return -3
+            fi
+        fi
+        $ulog "estimating size of sending ${snap}"
+        local size=$( zfs $SEND -v -n $inc $snap 2>&1 | tail -1 | cut -d" " -f 5 )
+        # could be 0 or 400 or 4K or 9.3g, etc.
+        local suf=$( echo $size | sed -E 's/[0-9]+\.?[0-9]*//' | tr '[:lower:]' '[:upper:]' )
+        size=$( echo $size | sed -E 's/[pPtTgGmMkKB]$//' ) # remove known suffixes
+	if [[ $suf != 'B' ]]; then
+        	size=$( echo "${size} * 1024" | bc | sed -E 's/\.[0-9]+//' ) # use bc to multiply decimals, sed to make ceil()
+	fi
+        case $suf in
+            B)
+		suf=''
+		;;
+            K)
+                suf=''
+                ;;
+            M)
+                suf='K'
+                ;;
+            G)
+                suf='M'
+                ;;
+            T)
+                suf='G'
+                ;;
+            P)
+                suf='T'
+                ;;
+        esac
+        $ulog "estimated size of sending ${snap} is ${size}${suf}"
+        local pv_more="-s ${size}${suf}"
+        if [[ $USE_MBUFFER == 'yes' ]]; then
+            ssh "${USER}@${REMOTE}" "mbuffer ${MBUFFER} -q -I ${PORT} | zfs ${RECV} ${dest}" 2>&1 | $ulog &
+            sleep 5
+            zfs $SEND $inc $snap 2> >($ulog)|
+            mbuffer $MBUFFER $MBUFFER_SEND_OPTS -O ${REMOTE}:${PORT}
+            zfs_send_status="${PIPESTATUS[0]}"
+            $ulog "zfs send exited with status: ${zfs_send_status}"
+            $ulog "about to wait on zfs send (this may take a while and appear to have hung)"
+            wait
+            zfs_recv_status="${PIPESTATUS[0]}"
+            $ulog "zfs recv exited with status: ${zfs_recv_status}"
+        else
+            zfs $SEND $inc $snap 2> >($ulog) | pv $PV_OPTS $pv_more | ssh "${USER}@${REMOTE}" "zfs ${RECV} ${dest}" 2>&1 | $ulog
+            pipe_status=("${PIPESTATUS[@]}")
+            zfs_send_status="${pipe_status[0]}"
+            zfs_recv_status="${pipe_status[2]}"
+            $ulog "zfs send exited with status: ${zfs_send_status}"
+            $ulog "zfs recv exited with status: ${zfs_recv_status}"
+        fi
+        if [[ $zfs_send_status != 0 ]]; then
+            return $zfs_send_status
+        elif [[ $zfs_recv_status != 0 ]]; then
+            return $zfs_recv_status
+        else
+            # both must be zero
+            return 0
+        fi
+    ) 9>$lock
+}
+
+function terminal_options {
+if [ -t 1 ]; then
+    ISTERM=1
+    LOGGER_EXTRA='-s' # enable logger output to stderr
+    PV_OPTS='-perb' # enable all the magic output from pv
+    MBUFFER_SEND_OPTS='' # don't do quiet mode, we have a term
+    ulog="logger ${LOGGER_EXTRA} -p user.notice -t "$(basename $0 2>/dev/null)"[${$}]"
+else
+    ISTERM=0
+    LOGGER_EXTRA='' # don't enable stderr output
+    PV_OPTS='-q' # make pv quiet
+    MBUFFER_SEND_OPTS='-q' # enable send side -q, no terminal
+    ulog="logger ${LOGGER_EXTRA} -p user.notice -t "$(basename $0 2>/dev/null)"[${$}]"
+fi
+}
+

data/bin/zfsmgr

@@ -0,0 +1,40 @@
+require "thor"
+require "zfs_mgmt"
+
+class Snapshot < Thor
+  class_option :noop, :type => :boolean, :default => false,
+               :desc => 'pass -n option to zfs commands'
+  class_option :verbose, :type => :boolean, :default => false,
+               :desc => 'pass -v option to zfs commands'
+  class_option :debug, :type => :boolean, :default => false,
+               :desc => 'set logging level to debug'
+  class_option :filter, :type => :string, :default => '.+',
+               :desc => 'only act on zfs matching this regexp'
+  desc "destroy", "apply the snapshot destroy policy to zfs"
+  def destroy()
+    ZfsMgmt.snapshot_destroy(noop: options[:noop], verbopt: options[:verbose], debugopt: options[:debug], filter: options[:filter])
+  end
+  desc "policy", "print the policy table for zfs"
+  def policy()
+    ZfsMgmt.snapshot_policy(verbopt: options[:verbose], debugopt: options[:debug], filter: options[:filter])
+  end
+  desc "create", "execute zfs snapshot based on zfs properties"
+  def create()
+    ZfsMgmt.snapshot_create(verbopt: options[:verbose], debugopt: options[:debug], filter: options[:filter])
+  end
+end
+
+class ZfsMgr < Thor
+  desc "zfsget [ZFS]", "execute zfs get for the given properties and types and parse the output into a nested hash"
+  method_option :properties, :type => :array, :default => ['name'], :desc => "List of properties passed to zfs get"
+  method_option :types,      :type => :array, :default => ['filesystem','volume'], enum: ['filesystem','volume','snapshot'], :desc => "list of types"
+  def zfsget(zfs)
+    pp ZfsMgmt.zfsget(properties: options[:properties],
+                      types: options[:types],
+                      zfs: zfs)
+  end
+  desc "snapshot SUBCOMMAND ...ARGS", "manage snapshots"
+  subcommand "snapshot", Snapshot
+end
+
+ZfsMgr.start(ARGV)

data/bin/zfsrecvman

@@ -0,0 +1,154 @@
+#! /bin/bash
+
+export PATH=$PATH:/sbin
+
+FILTER='.'
+SENDER='beast'
+REMOTE='blob'
+RECVER='blob'
+USER='root'
+DEST="blob"/$(hostname -s)
+SEND='send -R'
+RECV='recv -u -e -F'
+FLOCK='/usr/bin/flock -w 60 -n 9'
+PORT='1337'
+MBUFFER='-s 128k -m 1G -4'
+USE_MBUFFER='no'
+LOCK_DIR='/var/run/'$(basename $0)
+TEST=0
+VERB=0
+
+test -f $HOME/.keychain/$HOSTNAME-sh && . $HOME/.keychain/$HOSTNAME-sh
+
+. /usr/bin/zfsfuncs
+
+terminal_options
+
+if [[ `hostname -s` == $SENDER ]]; then
+    SENDER_PREF=""
+    RECVER_PREF="ssh ${USER}@${SENDER}"
+elif [[ `hostname -s` == $RECVER ]]; then
+    SENDER_PREF="ssh ${USER}@${SENDER}"
+    RECVER_PREF=""
+else
+    $ulog "can only be run on ${SENDER} or ${RECVER}"
+    exit -1
+fi
+
+while getopts ":p:f:L:mnv" opt; do
+    case $opt in
+        p)
+            PORT=$OPTARG
+            ;;
+        f)
+            FILTER=$OPTARG
+            ;;
+        L)
+            PV_OPTS="${PV_OPTS} -L ${OPTARG}"
+            ;;
+        m)
+            USE_MBUFFER='yes'
+            ;;
+        n)
+            RECV="${RECV} -n"
+            TEST=1
+            VERB=1
+            PV_OPTS='-q' # make pv quiet
+            MBUFFER_SEND_OPTS='-q' # enable send side -q, no terminal
+            ;;
+        v)
+            VERB=1
+            ;;
+    esac
+done
+
+if [[ $VERB == 1 ]]; then
+    echo $RECV | grep -q -- -v || RECV="${RECV} -v"
+fi
+
+for zpool in $( zpool list -H -o name | egrep "$FILTER" | sort ); do
+    target="${DEST}/${zpool}"
+    target_dir=$( dirname $target )
+    # recv_last is the last snapshot on the recv side of this zfs
+    if ! ssh "${USER}@${REMOTE}" zfs get written $target >/dev/null 2>/dev/null; then
+        $ulog sending initial snapshot of $zpool to $target_dir on $REMOTE
+        snap=$( zfs list -t snapshot -o name -s creation -d 1 -H $zpool | grep @zfssnapman- | tail -1 )
+        result='-1'
+        zfssendrecv -z $zpool \
+            -s $snap \
+            -d $target_dir \
+            -h # create hold
+        result=$?
+        if [[ $TEST == 0 ]]; then
+            echo "${zpool}:${REMOTE}:${snap}:${result}" >> ~/.zfssendrecv.log
+            if [[ $result != 0 ]]; then
+                zfs release -r zfsrecvman $snap || $ulog "unable to remove hold on our source snapshot: ${snap}"
+            fi
+        fi
+    fi
+
+    # last known good snapshot sent
+    pattern="^${zpool}:${REMOTE}:.+:0$"
+    if ! egrep -q "${pattern}" ~/.zfssendrecv.log; then
+        $ulog "no known good snapshot logged for ${zpool} on ${REMOTE}, unable to continue"
+        continue;
+    fi
+    last_snap=$( egrep "${pattern}" ~/.zfssendrecv.log | tail -1 | cut -d: -f 3)
+    
+    remote_snaps=$( mktemp )
+    ssh "${USER}@${REMOTE}" zfs list -t snapshot -o name -s creation -d 1 -H $target > $remote_snaps
+    if [[ $? != 0 ]]; then 
+        $ulog "unable to retrieve list of remote snapshots for ${zpool} on ${REMOTE}"
+        continue;
+    fi
+
+    if ! egrep -q "^${target_dir}/${last_snap}" $remote_snaps; then
+        $ulog "${last_snap} does not exist on ${REMOTE}, you must destroy the filesystem: ${target}"
+        continue;
+    fi
+    if ! tail -1 $remote_snaps | egrep -q "^${target_dir}/${last_snap}"; then
+        $ulog "${last_snap} is not the most recent snapshot on ${REMOTE}, rollback will occur on ${target}"
+    fi
+    
+    rm -f $remote_snaps
+    # grab the most recent local recursive snapshot
+    current=$( zfs list -t snapshot -o name -s creation -d 1 -H $zpool | grep @zfssnapman- | tail -1 )
+    if [[ "${last_snap}" == "${current}" ]]; then
+        $ulog "${zpool} is in sync on source and destination (${target})"
+        continue
+    fi
+    $ulog sending $last_snap through $current to $target
+    result='-1'
+    zfssendrecv -z $zpool \
+        -I $last_snap \
+        -s $current \
+        -d $target_dir \
+        -h
+    result=$?
+    if [[ $TEST == 1 ]]; then
+        continue
+    fi
+    echo "${zpool}:${REMOTE}:${current}:${result}" >> ~/.zfssendrecv.log
+    if [[ $result == 0 ]]; then
+        $ulog "${zpool} is in sync on source and destination (${target})"
+        zfs get -t filesystem -H -r all $zpool | ssh "${USER}@${REMOTE}" "cat > ~/.zpool-properties-${zpool}" ||
+          $ulog "unable to write zpool properties backup for ${zpool}" 
+        zfs release -r zfsrecvman $last_snap ||
+          $ulog "unable to release old snapshot: ${last_snap}"
+        ssh "${USER}@${REMOTE}" "zfs release -r zfsrecvman ${target_dir}/${last_snap}" ||
+          $ulog "unable to release old snapshot on remote side: ${target_dir}/${last_snap}"
+        ssh "${USER}@${REMOTE}" "zfs hold -r zfsrecvman ${target_dir}/${current}" ||
+          $ulog "unable to hold snapshot on remote side: ${target_dir}/${current}"
+        com=''
+        for zfs in $(ssh "${USER}@${REMOTE}" "zfs list -H -o name -t filesystem -r ${target}" | sort -r ); do
+            for prop in 'canmount=off' 'sharenfs=off' 'sharesmb=off' 'mountpoint=none'; do
+                com="${com}zfs set ${prop} ${zfs};"
+            done
+        done
+        $ulog "fixing zfs properties for ${zpool} (this may take a while)"
+        ssh "${USER}@${REMOTE}" "${com}" 2>&1 | $ulog
+    else
+        zfs release -r zfsrecvman $current || $ulog "unable to release current snapshot: ${current}"
+        $ulog zfs exited with $result while sending $send through $current to $target
+    fi
+done