zfs-tools 0.2.0

data/HISTORY

@@ -0,0 +1,24 @@
+= 0.2.0
+ 
+ . first public release
+ - zfs_incremental_sync deleted. Not the general solution of the problem we'd
+   hoped for. 
+
+== 0.1.8
+ 
+ ! Pool name can now contain spaces.
+ * Update to the newest activesupport gem, works with Ruby 1.9.2
+ * Dismiss the use of pfexec to run commands that need special privileges, in
+   favor of sudo.
+ * Reworked tools to use the library instead of issuing commands themselves.
+ * [zfs_incremental_sync] Fixed: full synch.
+ * [zfs_incremental_sync] Fixed: Missing snapshots on the target 
+   would be the cause for a failed synch. I hope this is fixed now.
+ * [zfs_list_obsolete_snapshots] Works with datasets that have no snapshots
+ * [zfs_incremental_sync] Fixed: Bug where sync would abort because there 
+   were no initial snapshots. 
+ * zfs_incremental_sync doesn't use sudo anymore. All commands that need 
+   special privileges are run through pfexec.
+ * Now gives incremental sync snapshots special names. This avoids collision
+   with the ones done hourly and it also provides information about where
+   the snapshot went (to which host).

data/LICENSE

@@ -0,0 +1,23 @@
+
+ Copyright (c) 2012,2013 Kaspar Schiess
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,30 @@
+
+A few ZFS tools, mostly related to snapshotting, cleaning up and synching.
+
+SYNOPSIS
+
+  # Snapshot any dataset, snapshot name is the current date/time in a 
+  # normalized format: 
+  zfs_snapshot pool1/my/dataset
+  
+  # List snapshots that are obsolete, given some rules about what is 
+  # considered obsolete: 
+  echo pool1 | zfs_list_obsolete_snapshots
+
+  # Safely destroy some dataset (recursively). This will always ask for
+  # permission!
+  zfs_safe_destroy pool1/my/dataset
+
+STATUS
+
+  This is useful in production; the code needs to be cleaned up.
+
+LICENSE
+  
+  MIT license, see LICENSE file.
+
+HACKING
+
+  To run the specs, type `rspec`. 
+
+  Pull requests welcome. 

data/bin/zfs_list_obsolete_snapshots

@@ -0,0 +1,64 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'snapshot_set'
+require 'zfs'
+
+default = {
+  1.hour    => 24, 
+  1.day     => 7, 
+  1.month   => 12, 
+  1.year    => 2
+}
+options = {
+}
+
+require 'optparse'
+OptionParser.new { |o|
+  o.banner = %Q{Usage: #{$0} [options]
+
+  Interprets each line on stdin as a zfs dataset. Outputs all snapshots on the
+  dataset that are obsolete by the given rule. Example:
+
+    echo 'pool1/backup' | zfs_list_obsolete_snapshots --daily=3
+
+  When called without arguments, it will assumes a default of
+
+    --hourly=24 --daily=7 --monthly=12 --yearly=2
+    
+}
+  o.separator %Q{  Options are:}
+
+  o.on('--hourly [N]', Integer, "keep N hourly backups (doubles as switch for this help)") do |v|
+    if v
+      options[1.hour] = v
+    else
+      puts o
+      exit 0
+    end
+  end
+  {
+    :daily => 1.day, 
+    :weekly => 1.week, 
+    :monthly => 1.month, 
+    :quarterly => 3.months, 
+    :semianually => 6.months, 
+    :yearly => 1.year
+  }.each do |name, period|
+    o.on("--#{name} N", Integer, "keep N #{name} backups") do |v|
+      options[period] = v
+    end
+  end
+}.parse!(ARGV)
+
+
+opts = options.empty? ? default : options
+while line=gets
+  dataset = ZFS::Dataset.new(line.chomp)
+  snapshots = dataset.snapshots
+  
+  keep = SnapshotSet.new(snapshots).gpc(opts).to_a
+  remove = (snapshots - keep).sort
+  
+  puts remove.map { |sn| "#{dataset.name}@#{sn}" }
+end

data/bin/zfs_safe_destroy

@@ -0,0 +1,80 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'mixlib/shellout'
+
+if ARGV.empty? 
+  puts %Q{
+    Usage: zfs_safe_destroy DATASET
+
+    Scans the given dataset and prints a summary of what zfs objects 
+    (filesystems, volumes and snapshots) would be affected if one was to 
+    recursively destroy the given dataset. If you type 'yes' at the prompt, 
+    it will then perform a recursive destroy command on the dataset. 
+
+    CAUTION: By typing yes at the prompt, you will loose data. If this is 
+    your goal, this is your command. 
+}
+  exit 0
+end
+
+def enumerate_objects dataset
+  zfs = Mixlib::ShellOut.new(
+    'zfs', 'list -r -t all -Ho name,type ', dataset)
+  zfs.run_command
+  zfs.error!
+
+  summary = Hash.new(0)
+
+  zfs.stdout.lines.map do |line|
+    name, _, type = line.chomp.rpartition(' ')
+    name.strip!
+
+    next if name.size == 0
+
+    summary[type] += 1
+  end
+
+  summary
+rescue => ex
+  warn ex.to_s
+  warn "Aborting."
+  exit(3)
+end
+def zfs_recursive_destroy dataset
+  destroy = Mixlib::ShellOut.new('zfs', 'destroy -r ', dataset)
+  destroy.run_command
+  destroy.error!
+
+rescue => ex
+  warn ex.to_s
+  warn "Aborting."
+  exit(4)
+end
+
+# ----------------------------------------------------------------- main logic
+
+dataset = ARGV.first
+before = enumerate_objects(dataset)
+
+puts "You're about to permanently destroy: "
+before.each do |type, count|
+  printf "%5d %ss\n", count, type
+end
+
+puts "\n'zfs destroy -r #{dataset}'"
+print "Please confirm the operation by typing 'yes': "
+confirmation = $stdin.gets
+exit(1) unless confirmation.chomp == 'yes'
+
+after = enumerate_objects(dataset)
+if before != after
+  warn "The data that would have been destroyed by the command has changed."
+  exit(2)
+end
+
+$stdout.sync = true
+print "Destroying..."
+zfs_recursive_destroy dataset
+puts ' Done.'

data/bin/zfs_snapshot

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+require 'zfs'
+
+if ARGV.empty? 
+  puts %Q{
+    Usage: zfs_snapshot DATASET [COMMENT]
+
+    Snapshots a zfs dataset recursively using a timestamp as snapshot name. If
+    you give a comment after the dataset, it will be sanititzed and appended
+    to the snapshot name.
+    
+    The name of the created snapshot is output to STDOUT. 
+}
+  exit 0
+end
+
+dataset_name, comment = ARGV.first(2)
+
+dataset = ZFS::Dataset.new(dataset_name)
+snapshot_name = dataset.snapshot_with_timestamp(comment)
+
+puts snapshot_name

data/lib/snapshot_set.rb

@@ -0,0 +1,113 @@
+
+require 'active_support/core_ext/numeric/time'
+require 'active_support/core_ext/integer/time'
+require 'active_support/core_ext/date/calculations'
+require 'time'
+require 'set'
+
+# A set of snapshots. This class has methods that allow to determine which 
+# snapshots must be deleted in order to clean up space. 
+#
+class SnapshotSet
+  attr_reader :snapshots
+  
+  # Keeps metadata for each snapshot. Original name is stored as just +name+, 
+  # extracted timestamp is +time+.
+  #
+  class Snapshot
+    attr_reader :name, :time
+    
+    def initialize(name, default_time=Time.now)
+      @time = default_time
+      @name = name
+      extract_timestamp(name)
+    end
+    
+    def extract_timestamp(name)
+      if md=name.match(/(\d+)-(\d+)-(\d+)-(\d{2})(\d{2})(\d{2})(.*)/)
+        @time = Time.parse(md.captures.join) 
+      elsif md=name.match(/(\d+)-(\d+)-(\d+)_(\d+)-(\d+)(.*)/)
+        @time = Time.parse(md.captures.join) 
+      else
+        @time = Time.parse(name)
+      end
+    rescue ArgumentError
+      # no time information in "foobar"
+      return nil
+    end
+    def to_s
+      name
+    end
+  end
+  
+  # Initialize this class with an unordered set of snapshot names. Names that
+  # are of the form 'YYYYMMDDHHMM*' or 'YYYYMMDD*' will be treated as
+  # timestamps to which time based rules apply. 
+  #
+  def initialize(snapshot_names)
+    @snapshots = snapshot_names.
+      map { |name| Snapshot.new(name) }.
+      sort_by { |snapshot| snapshot.time }
+  end
+
+  # Returns the size of this set. 
+  #
+  def size
+    snapshots.size
+  end
+
+  # Returns the set as an array of snapshot names.
+  #
+  def to_a
+    snapshots.map(&:name)
+  end
+  
+  # Computes snapshots to keep according to grandparent-parent-child
+  # algorithm. If called with
+  #
+  #   set.gpc(1.day: 3, 1.week: 3)
+  #
+  # it will return a snapshot set that contains (ideally) 6 snapshots, 3 in
+  # the current week starting one day ago, spaced out by one day. The other
+  # three will be on week boundaries starting one week ago and going back 3
+  # weeks. 
+  #
+  # The algorithm will also return all the snapshots that are less than one
+  # day ago. 
+  #
+  def gpc(keep_specification, now=Time.now)
+    keep_snapshot_names = Set.new
+
+    # No snapshots, nothing to keep
+    if snapshots.empty?
+      return self.class.new([])
+    end
+
+    # Filter snapshots that we need to keep according to keep_specification
+    keep_specification.each do |interval, keep_number|
+      next if keep_number <= 0
+      
+      # We would like to sample the existing snapshots at regular offsets
+      # (n * interval). 
+      sampling_points = Array(1..keep_number).map { |offset| now - (offset*interval) }
+      
+      # For all sampling points, we'll compute the best snapshot to keep. 
+      winners = sampling_points.map { |sp| 
+        snapshots.map { |sh| [(sh.time-sp).abs, sh] }.    # <score, snapshot>
+          sort_by { |score, sh| score }.                  # sort by score
+          first.                                          # best match
+          last                                            # snapshot
+      }
+      
+      keep_snapshot_names += winners.map(&:name)
+    end
+    
+    # Add snapshots that are within [now, smallest_interval]
+    smallest_interval = keep_specification.map { |i,c| i }.min
+    keep_snapshot_names += snapshots.
+      select { |snapshot| snapshot.time > now-smallest_interval }.
+      map(&:name)
+    
+    self.class.new(keep_snapshot_names.to_a)
+  end
+end

data/lib/zfs/dataset.rb

@@ -0,0 +1,80 @@
+
+require 'shellwords'
+
+# A zfs dataset such as 'pool1/backup'
+#
+class ZFS::Dataset
+  attr_reader :name
+  def initialize(name)
+    @name = name
+  end
+  
+  # Returns an array of snapshots on the dataset. This does not include
+  # snapshots on the datasets children.
+  #
+  def snapshots
+    list(name).
+      lines.
+      select { |l| l.index('@') }.      # only snapshots
+      map { |l| l.chomp.split('@') }.   # <path, snapshot_name>
+      select { |path, sn| path==name }. # only direct snapshots
+      map { |path,sn| sn }              # only snapshot names
+  end
+
+  # Snapshots the dataset with the aid of 'zfs snapshot'. +name+ is sanitized
+  # to something zfs accepts, which means that spaces and non-word chars
+  # are replaced with '_'. 
+  #
+  def snapshot(name, recursive=false)
+    arguments = []
+    
+    snapshot_name = sanitize_snapshot_name(name)
+    
+    arguments << '-r' if recursive
+    arguments << "'#{self.name}@#{snapshot_name}'"
+    
+    zfs_snapshot(*arguments)
+    
+    return snapshot_name
+  end
+  
+  # Snapshots the dataset with a timestamp that looks like 201101011345 (year,
+  # month, day, hour and minute). If a +comment+ is provided, the snapshot
+  # will have the comment appended to it, separated by a dash
+  # (201101011345-my_example_comment). Note that the sanitizing that #snapshot
+  # does applies to the comment as well. 
+  #
+  # This snapshotting is always recursive.
+  #
+  def snapshot_with_timestamp(comment=nil, time=Time.now)
+    name = time.strftime("%Y%m%d%H%M")
+    name << "-#{comment}" if comment
+    
+    snapshot(name, true)
+  end
+  
+private
+
+  # Sanitizes a name to be used as a snapshot name. 
+  #
+  def sanitize_snapshot_name(name)
+    name.gsub(/[^-_a-z0-9]/i, '_')
+  end
+
+  # Raw command 'zfs snapshot', args are passed as command line arguments.
+  #
+  def zfs_snapshot(*args)
+    zfs "snapshot", *args
+  end
+  
+  # Raw command 'zfs', args are passed as command line arguments. 
+  # 
+  def zfs(*args)
+    arguments = args.join(' ')
+    `sudo /sbin/zfs #{arguments}`
+  end
+  
+  def list(dataset)
+    zfs 'list', %w(-rH -oname -tall), Shellwords.escape(dataset)
+  end
+end

data/lib/zfs/snapshot_list.rb

@@ -0,0 +1,29 @@
+
+
+# Represents a list of snapshots like the tool 'zfs list' outputs it. This 
+# class allows extracting information from that list. 
+#
+# Example: 
+#
+#   snl = ZFS::SnapshotList.new(`zfs list -H -o name`)
+#   snl.datasets # => array of dataset names
+#   snl.snapshots('pool1') # => array of snapshots on the pool1
+#
+class ZFS::SnapshotList
+  def initialize(tool_output)
+    @list = Hash.new { |h,k| h[k] = [] }
+    
+    tool_output.lines. 
+      map { |l| l.chomp.strip.split('@').first(2) }.  # extracts path/snapshot tuples
+      each { |path, snapshot|
+        snapshots = @list[path]
+        snapshots << snapshot if snapshot }
+        
+  end
+  def datasets
+    @list.keys
+  end
+  def snapshots(dataset_name)
+    @list[dataset_name]
+  end
+end

data/lib/zfs.rb

@@ -0,0 +1,7 @@
+
+# A small collection of zfs classes that rely on zfs utilities.
+#
+module ZFS; end
+
+require 'zfs/snapshot_list'
+require 'zfs/dataset'

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: zfs-tools
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+  prerelease: 
+platform: ruby
+authors:
+- Kaspar Schiess
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mixlib-shellout
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: i18n
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.6'
+description: 
+email: kaspar.schiess@absurd.li
+executables:
+- zfs_safe_destroy
+- zfs_list_obsolete_snapshots
+- zfs_snapshot
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- HISTORY
+- README
+- lib/snapshot_set.rb
+- lib/zfs/dataset.rb
+- lib/zfs/snapshot_list.rb
+- lib/zfs.rb
+- bin/zfs_list_obsolete_snapshots
+- bin/zfs_safe_destroy
+- bin/zfs_snapshot
+homepage: http://bitbucket.org/kschiess/zfs-tools
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: A few ZFS tools, mostly related to snapshotting, cleaning up and synching.
+test_files: []
+has_rdoc: