marius-zsnap 0.1.2

data/INSTALL

@@ -0,0 +1,39 @@
+FreeBSD Installation
+--------------------
+
+FreeBSD users please use the sysutils/zfs-snapshot-mgmt port:
+
+  cd /usr/ports/sysutils/zfs-snapshot-mgmt
+  make install clean
+
+and follow the instructions printed after installation.
+
+See the manpage for details:
+
+  man zfs-snapshot-mgmt
+
+
+
+Manual Installation
+-------------------
+
+Requirements:
+  Ruby
+  (/usr/local/bin/ruby, edit the first line of script for a different path)
+
+Copy the zfs-snapshot-mgmt script to /usr/local/bin.
+Copy the zfs-snapshot-mgmt.conf.sample file to
+/usr/local/etc/zfs-snapshot-mgmt.conf (this path is hardcoded in the script,
+sorry. You may change it manually (grep for CONFIG_FILE)).
+Copy the zfs-snapshot-mgmt.8 file to man8 directory on your system (usually
+/usr/local/share/man/man8/ or just /usr/share/man/man8/).
+
+Read the manual page (man zfs-snapshot-mgmt).
+
+Edit the zfs-snapshot-mgmt.conf file.
+Add the script to crontab (usually /etc/crontab but may differ on your system),
+append a line like this:
+
+*/5  *  *  *  *         root            /usr/local/bin/zfs-snapshot-mgmt
+
+From now on the snapshots will be automatically created.

data/bin/zfs-snapshot-mgmt

@@ -0,0 +1,36 @@
+#!/usr/local/bin/ruby -w
+# Copyright (c) 2008, Marcin Simonides
+# Copyright (c) 2009, Marius Nuennerich
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR
+# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+require 'zsnap'
+
+config = Zsnap::Config.new(YAML::load(File.open(Zsnap::CONFIG_FILE_NAME).read))
+
+now_minutes = Time.now.to_i / 60
+
+config.filesystems.each do |fs|
+  unless config.busy_pools.include? fs.pool
+    fs.create_snapshot(now_minutes, config.snapshot_prefix)
+    fs.remove_snapshots(now_minutes, config.snapshot_prefix)
+  end
+end

data/conf/zfs-snapshot-mgmt.conf.sample

@@ -0,0 +1,32 @@
+# Automatic ZFS snapshot management configuration file
+#
+# This is a YAML file (see http://www.yaml.org)
+# Use exactly 2 spaces for each indentation level
+#
+snapshot_prefix: auto-
+filesystems:
+  tank/usr/home:
+    # Create snapshots recursively for all filesystems mounted under this one
+    recursive: true
+    # Create snapshots every 10 minutes, starting at midnight
+    creation_rule:
+      at_multiple: 10
+      offset: 0
+    # Keep all snapshots for the first 90 minutes,
+    # then only those that were created at 30 minute intervals for 12 hours
+    # (after snapshot creation),
+    # then only those that were created at 3 hour intervals, counting at 2:00
+    # (i.e. 2:00, 5:00, 8:00, 11:00, 14:00, 17:00, 20:00, 23:00)
+    # for 7 days
+    preservation_rules:
+      - { for_minutes:    90, at_multiple:    0, offset:    0 }
+      - { for_minutes:   720, at_multiple:   30, offset:    0 }
+      - { for_minutes: 10080, at_multiple:  180, offset:  120 }
+  tank/usr:
+    # Create snapshots every 24 hours, starting at 20:00.
+    creation_rule:
+      at_multiple: 1440
+      offset: 1200
+    # Keep daily snapshots created at 20:00 (in this case all).
+    preservation_rules:
+      - { for_minutes: 5760, at_multiple: 1440, offset: 1200 }

data/doc/zfs-snapshot-mgmt.8

@@ -0,0 +1,164 @@
+.Dd June 6, 2008
+.Dt ZFS-SNAPSHOT-MGMT 8
+.Os
+.Sh NAME
+.Nm zfs-snapshot-mgmt
+.Nd automate creation of new and removal of stale ZFS snapshots
+.Sh SYNOPSIS
+.Nm
+.Sh DESCRIPTION
+The utility creates ZFS snapshots and removes snapshots it has created that
+are stale.
+Rules for creating and removing are defined in the configuration file.
+.Pp
+.Nm
+is designed to be run by the
+.Xr cron 8
+utility.
+.Sh CONFIGURATION
+.Nm
+reads settings from the 
+.Pa /usr/local/etc/zfs-snapshot-mgmt.conf
+file.
+.Ss General Structure
+The file is in the YAML format: it contains keys and values separated by
+a colon (:). Each value may span multiple lines and be a list or a map with
+key-value pairs. The indentation is always a multiple of two spaces.
+.Pp
+There are two top-level options:
+.Bl -tag -width "1234"
+.It snapshot_prefix
+prefix for snapshots created and recognised by this tool.
+All created snapshots
+will have names consisting of this prefix and current date and time.
+Only snapshots beginning with this prefix will be considered for removal.
+.It filesystems
+the main part of configuration - specifies rules for creating and removing
+snapshots at each filesystem.
+The value for this key contains keys for each filesystem
+(as seen by ZFS, e.g. tank/home)
+with snapshot creation and preservation settings.
+.El
+.Pp
+There is an example configuration in the
+.Sx EXAMPLES
+section.
+.Ss Creation Rules
+For each filesystem there is one snapshot
+.Li creation_rule
+defined.
+It specifies when snapshots for the particular filesystem should be created.
+It has two parameters.
+All values are specified in minutes.
+.Bl -tag -width "1234"
+.It at_multiple
+defines how often should a snapshot be created.
+E.g. a value of 60 means that a snapshot is to be created every hour.  
+.It offset
+defines an offset from midnight that is to be applied to the
+.Ix at_multiple
+parameter.
+E.g. specifying a value of 30 means that the first snapshot will be taken 30
+minutes after midnight.
+.El
+.Ss Preservation Rules
+These rules specify which of the created snapshots should be preserved.
+All that do not match the any of the rules and whose names begin with
+.Li snapshot_prefix
+are destroyed.
+.Pp
+Each of the rules has three parameters:
+.Bl -tag -width "1234"
+.It for_minutes
+specifies how long after snapshot creation is the rule applicable.
+.It at_multiple
+analogous to the same parameter for
+.Li creation_rule
+- snapshots whose creation time 
+(in minutes since midnight)
+is a multiple of this value are retained.
+.It offset
+analogous to the same parameter for
+.Li creation_rule
+- applies to
+.Li at_multiple .
+.El
+.Sh EXAMPLES
+.Ss Crontab Configuration
+To invoke the program every five minutes you may add the following line to
+the
+.Pa /etc/crontab
+configuration file:
+.Pp
+.D1 */5 * * * *         root    /usr/local/bin/zfs-snapshot-mgmt
+.Pp
+Bear in mind that this effectively limits the resolution to 5 minutes.
+.Ss zfs-snapshot-mgmt.conf
+Here is an example configuration.
+.Bd -literal -offset indent
+snapshot_prefix: auto-
+filesystems:
+  tank/usr/home:
+    creation_rule:
+      at_multiple: 5
+      offset: 0
+    preservation_rules:
+      - { for_minutes:    90, at_multiple:    0, offset:    0 }
+      - { for_minutes:   720, at_multiple:   30, offset:    0 }
+      - { for_minutes: 10080, at_multiple:  180, offset:  120 }
+  tank/usr:
+    recursive: true
+    creation_rule:
+      at_multiple: 1440
+      offset: 1200
+    preservation_rules:
+      - { for_minutes: 5760, at_multiple: 1440, offset: 1200 }
+.Ed
+.Pp
+which specifies the following settings:
+.Bl -bullet
+.It
+names of created snapshots start with
+.Li auto-
+and only such snapshots are considered for removal.
+.It
+there are two filesystems:
+.Li tank/usr/home
+and
+.Li tank/usr .
+.It
+snapshots for
+.Li tank/usr/home
+are created every five minutes, starting at midnight.
+.It
+all snapshots for
+.Li tank/usr/home
+are kept for 90 minutes after creation.
+.It 
+only snapshots created af full and half hour are retained for 12 hours
+(720 minutes) after creation for
+.Li tank/usr/home .
+.It
+snapshots created at multiple of three hours starting with 2 a.m. are retained
+for a week (10080 minutes) after creation for
+.Li tank/usr/home .
+This means: 2 a.m., 5 a.m., 8 a.m. and so on.
+.It
+snapshots on
+.Li tank/usr
+are created once a day at 20 p.m. and retained for 4 days.
+.It
+snapshots are taken recursively on
+.Li tank/usr
+filesystem and all filesystems mounted under it.
+.El
+.Sh FILES
+.Pa /usr/local/etc/zfs-snapshot-mgmt.conf
+The configuration file
+.Sh SEE ALSO
+.Xr cron 8 ,
+.Xr zfs 1M
+.Sh AUTHORS
+.An Marcin Simonides Aq marcin@studio4plus.com
+.Sh BUGS
+There is no way to use alternative path for the configuration file.

data/lib/zsnap.rb

@@ -0,0 +1,178 @@
+#!/usr/local/bin/ruby -w
+# Copyright (c) 2008, Marcin Simonides
+# Copyright (c) 2009, Marius Nuennerich
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
+# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR
+# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+require 'yaml'
+require 'time'
+
+module Zsnap
+
+CONFIG_FILE_NAME = '/usr/local/etc/zfs-snapshot-mgmt.conf'
+
+class Rule
+  def initialize(args = {})
+    args = { 'offset' => 0, 'at_multiple' => 60 }.merge(args)
+    @at_multiple = args['at_multiple'].to_i
+    @offset = args['offset'].to_i
+  end
+
+  def condition_met?(time_minutes)
+    divisor = @at_multiple
+    (divisor == 0) or ((time_minutes - @offset) % divisor) == 0
+  end
+end
+
+class PreservationRule < Rule
+  def initialize(args = {})
+    super(args)
+    args = { 'for_minutes' => 240 }.merge(args)
+    @for_minutes = args['for_minutes'].to_i
+  end
+
+  def applies?(now_minutes, creation_time_minutes)
+    (now_minutes - creation_time_minutes) < @for_minutes
+  end
+
+  def condition_met_for_snapshot?(now_minutes, snapshot)
+    creation_time_minutes = snapshot.creation_time_minutes
+    applies?(now_minutes, creation_time_minutes) and
+      condition_met?(creation_time_minutes)
+  end
+end
+
+class SnapshotInfo
+  def initialize(name, fs_name, snapshot_prefix)
+    @name = name
+    @fs_name = fs_name
+    @creation_time = parse_timestamp(name[snapshot_prefix.length .. -1])
+  end
+
+  def creation_time_minutes
+    @creation_time.to_i / 60
+  end
+
+  # Returns canonical name of the snapshot and FS (as accepted by zfs command)
+  # e.g.: /tank/usr@snapshot
+  def canonical_name
+    if @fs_name and @name
+      @fs_name + '@' + @name
+    else
+      raise "SnapshotInfo doesn't contain name and/or fs_name"
+    end
+  end
+
+  private
+
+  def parse_timestamp(time_string)
+    date, time = time_string.split('_')
+    year, month, day = date.split('-')
+    hour, minute = time.split('.')
+    Time.mktime(year, month, day, hour, minute)
+  end
+end
+
+class FSInfo
+  def initialize(fs_name, values)
+    @name = fs_name
+    raise "Filesystem #{fs_name} has no creation rule" unless values['creation_rule']
+    raise "Filesystem #{fs_name} has no preservation rules" unless values['preservation_rules']
+
+    @creation_rule = Rule.new(values['creation_rule'])
+    @preservation_rules = values['preservation_rules'].map do |value|
+      PreservationRule.new(value)
+    end
+    @is_recursive = values['recursive'] ? true : false
+  end
+
+  def create?(now_minutes)
+    @creation_rule.condition_met?(now_minutes)
+  end
+
+  def snapshots(prefix)
+    path = File.join(mount_point, '.zfs', 'snapshot')
+    Dir.open(path).select do |name|
+      name[0, prefix.length] == prefix
+    end.map { |name| SnapshotInfo.new(name, @name, prefix) }
+  end
+
+  def snapshots_to_remove(now_minutes, prefix)
+    snapshots(prefix).reject do |snapshot|
+      @preservation_rules.any? do |rule|
+        rule.condition_met_for_snapshot?(now_minutes, snapshot)
+      end
+    end
+  end
+
+  def remove_snapshots(now_minutes, prefix)
+    snapshots_to_remove(now_minutes, prefix).each do |snapshot|
+      remove_snapshot(snapshot)
+    end
+  end
+
+  def create_snapshot(now_minutes, prefix)
+    if create?(now_minutes)
+      create_snapshot_from_info(SnapshotInfo.new(prefix + Time.now.strftime('%Y-%m-%d_%H.%M'), @name, prefix))
+    end
+  end
+
+  def pool
+    # More or less according to ZFS Component Naming Requirements
+    # http://docs.sun.com/app/docs/doc/819-5461/gbcpt
+    @name[/\A[a-zA-Z_:.-]+/]
+  end
+
+  private
+
+  def remove_snapshot(snapshot_info)
+    arguments = @is_recursive ? '-r ' : ''
+    system 'zfs destroy ' + arguments + snapshot_info.canonical_name
+  end
+
+  def create_snapshot_from_info(snapshot_info)
+    arguments = @is_recursive ? '-r ' : ''
+    system 'zfs snapshot ' + arguments + snapshot_info.canonical_name
+  end
+
+  def mount_point
+    `zfs mount`.collect { |line| line.split(' ') }.
+      select { |item| item.first == @name }.collect { |item| item.last }.first
+  end
+end
+
+class Config
+  attr_reader :snapshot_prefix, :filesystems
+
+  def initialize(value)
+    @snapshot_prefix = value['snapshot_prefix']
+    @filesystems = value['filesystems'].map { |key, val| FSInfo.new(key, val) }
+    @pools = @filesystems.map { |fs| fs.pool }.uniq
+  end
+
+  def busy_pools
+    @busy_pools ||= @pools.select do |pool|
+      `zpool status #{pool}`.any? { |line| line =~ /(scrub|resilver) in progress/ }
+    end
+  end
+end
+
+end

data/test/zsnap_test.rb

@@ -0,0 +1,73 @@
+require 'test/unit'
+require 'lib/zsnap'
+
+include Zsnap
+
+class RuleTest < Test::Unit::TestCase
+  def test_rule
+    r = Rule.new
+    2.times do
+      assert r.condition_met?(0)
+      assert r.condition_met?(60)
+      assert r.condition_met?(120)
+      (1..59).each do |i|
+       assert !r.condition_met?(i)
+      end
+      r = Rule.new 'at_multiple' => 60
+    end
+  end
+
+  def test_rule_with_offset
+    r = Rule.new 'at_multiple' => 60, 'offset' => 4
+    assert r.condition_met?(4)
+    assert r.condition_met?(64)
+    assert r.condition_met?(124)
+    (5..63).each do |i|
+     assert !r.condition_met?(i)
+    end
+  end
+end
+
+class PreservationRuleTest < Test::Unit::TestCase
+  def test_create
+    p = PreservationRule.new
+    assert p.applies?(239, 0)
+    assert !p.applies?(240, 0)
+    assert !p.applies?(241, 0)
+  end
+
+  def test_condition_met
+    p = PreservationRule.new
+    s = SnapshotInfo.new 'foo-2009-08-05_21.00', 'bar', 'foo-'
+    assert p.condition_met_for_snapshot? 17, s
+
+  end
+end
+
+class SnapshotInfoTest < Test::Unit::TestCase
+  def test_create
+    s = SnapshotInfo.new 'foo-2009-08-05_21.42', 'bar', 'foo-'
+    assert_equal 20825022, s.creation_time_minutes
+    assert_equal 'bar@foo-2009-08-05_21.42', s.canonical_name
+  end
+
+  def test_raises_error
+    s = SnapshotInfo.new 'foo-2009-08-05_21.42', nil, 'foo-'
+    assert_raise(RuntimeError) { s.canonical_name }
+  end
+end
+
+class FSInfoTest < Test::Unit::TestCase
+  def test_create
+    rules = {
+              'creation_rule' => {'at_multiple' => 60, 'offset' => 0 },
+              'preservation_rules' => [
+                { 'for_minutes' => 240, 'at_multiple' => 0, 'offset' => 0 }
+              ]
+            }
+    f = FSInfo.new 'tank/foo', rules
+    assert f.create?(60)
+# Mock here
+#    assert_equal 'ff', f.send(:get_mount_point, 'tank/foo')
+  end
+end

metadata

@@ -0,0 +1,60 @@
+--- !ruby/object:Gem::Specification 
+name: marius-zsnap
+version: !ruby/object:Gem::Version 
+  version: 0.1.2
+platform: ruby
+authors: 
+- Marcin Simonides
+- Marius Nuennerich
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-05-16 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: 
+email: marius@nuenneri.ch
+executables: 
+- zfs-snapshot-mgmt
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- INSTALL
+- test/zsnap_test.rb
+- lib/zsnap.rb
+- conf/zfs-snapshot-mgmt.conf.sample
+- bin/zfs-snapshot-mgmt
+- doc/zfs-snapshot-mgmt.8
+has_rdoc: false
+homepage: http://github.com/marius/zsnap
+licenses: 
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 2
+summary: A script to automatically create and delete zfs snapshots from cron
+test_files: 
+- test/zsnap_test.rb