lsync 1.2.1

data/bin/lsync

@@ -0,0 +1,152 @@
+#!/usr/bin/env ruby
+
+require 'ftools'
+require 'lsync'
+require 'lsync/version'
+require 'optparse'
+
+OPTIONS = {
+  :ConfigFiles => [],
+  :Plan => nil,
+  :LogPath => "/var/log/lsync/backup.log"
+}
+
+ARGV.options do |o|
+  script_name = File.basename($0)
+
+  o.set_summary_indent('  ')
+  o.banner = "Usage: #{script_name} [options] [directory | config]"
+  o.define_head "This script is used to run backups. If you specify a directory, files ending in .conf will all be processed."
+
+  o.separator ""
+  o.separator "Help and Copyright information"
+
+  o.on("-p plan", String, "Run the specified backup plan") { |plan| OPTIONS[:Plan] = plan }
+  o.on("-l log_path", String, "Set the directory for backup logs") { |log_path| OPTIONS[:LogPath] = log_path }
+
+  o.on_tail("--copy", "Display copyright information") do
+    puts "#{script_name} v#{LSync::VERSION::STRING}. Copyright (c) 2008-2009 Samuel Williams. Released under the GPLv3."
+    puts "See http://www.oriontransfer.co.nz/ for more information."
+
+    exit
+  end
+
+  o.on_tail("-h", "--help", "Show this help message.") { puts o; exit }
+end.parse!
+
+# ============================== Main Scripts / Plans ==============================
+
+class LogPathChecker
+  def initialize(directory, filename)
+    full_path = File.join(directory, filename)
+
+    @directory_writable = File.directory?(directory) && File.writable?(directory)
+    @file_not_writable = File.exist?(full_path) && !File.writable?(full_path)
+
+    @accessable = @directory_writable && !@file_not_writable
+    @full_path = full_path
+  end
+  
+  attr :accessable
+  attr :full_path
+end
+
+def setup_logging
+  # Console log output
+  $console_log = Logger.new($stdout)
+  $console_log.formatter = MinimalLogFormat.new
+  
+  # File log output
+  filename = File.basename(OPTIONS[:LogPath])
+  directory = File.dirname(OPTIONS[:LogPath])
+  
+  if filename == nil || filename == ""
+    filename = "backup.log"
+  end
+  
+  check = LogPathChecker.new(directory, filename)
+  unless check.accessable
+    $console_log.warn "Could not write to #{check.full_path.dump} - changing log path to #{Dir.pwd.dump}"
+    
+    directory = Dir.pwd
+    check = LogPathChecker.new(directory, filename)
+  end
+  
+  if check.accessable
+    $backup_log = Logger.new(check.full_path, 'weekly')
+  else
+    $console_log.warn "Could not write to #{check.full_path.dump} - not logging to disk"
+    $backup_log = nil
+  end
+  
+  $logger = TeeLogger.new($console_log, $backup_log)
+end
+
+def process_config_files
+  ARGV.each do |p|
+    unless File.exists? p
+      $logger.error "Could not process #{p}"
+      exit(20)
+    end
+  
+    if File.directory? p
+      OPTIONS[:ConfigFiles] += Dir[File.join(p, "*.conf")]
+    else
+      OPTIONS[:ConfigFiles] << p
+    end
+  end
+end
+
+def validate_options
+  if OPTIONS[:Plan] && OPTIONS[:ConfigFiles].size > 0
+    $logger.error "Please specify either a backup plan or a set of backup scripts, but not both in one run."
+    exit(27)
+  end
+end
+
+def run_backups
+  OPTIONS[:ConfigFiles].each do |c|
+    config = LSync::BackupScript.load_from_file(c)
+    config.logger = $logger
+    config.run_backup
+  end
+
+  if OPTIONS[:Plan]
+    config = LSync::BackupPlan.load_from_file(OPTIONS[:Plan])
+    config.logger = $logger
+    config.run_backup
+  end
+end
+
+def dump_exception_backtrace ex
+  ex.backtrace.each do |bt|
+    $logger.error bt
+  end
+end
+
+setup_logging
+
+process_config_files
+
+validate_options
+
+$logger.info "  Backup beginning at #{Time.now.to_s}  ".center(96, "=")
+
+begin
+  run_backups
+rescue LSync::BackupError
+  $logger.error "#{$!.class.name}: #{$!.to_s}. Dumping backtrace:"
+  dump_exception_backtrace($!)
+  exit(230)
+rescue Interrupt
+  $logger.error "Backup process recevied interrupt (Ctrl-C). Dumping backtrace:"
+  dump_exception_backtrace($!)
+  exit(240)
+rescue
+  $logger.error "Unknown exception #{$!.class.name} thrown: #{$!.to_s}. Dumping backtrace:"
+  dump_exception_backtrace($!)
+  exit(250)
+ensure
+  $logger.info "  Backup finishing at #{Time.now.to_s}  ".center(96, "=")
+end
+

data/lib/lsync.rb

@@ -0,0 +1,40 @@
+# Copyright (c) 2007 Samuel Williams. Released under the GNU GPLv2.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'rubygems'
+
+gem 'termios'
+gem 'net-ssh'
+gem 'ruleby'
+
+require 'yaml'
+require 'socket'
+require 'set'
+require 'logger'
+
+require 'lsync/version'
+require 'lsync/extensions'
+require 'lsync/backup_script'
+require 'lsync/backup_plan'
+require 'lsync/tee_logger'
+
+require 'fileutils'
+require 'optparse'
+
+require 'open-uri'
+
+module LSync
+
+end

data/lib/lsync/action.rb

@@ -0,0 +1,100 @@
+
+require 'pathname'
+require 'lsync/run'
+require 'lsync/backup_error'
+
+module LSync
+  
+  class AbortBackupException < Exception
+    
+  end
+  
+  class Action
+    def initialize(function)
+      @function = function
+      
+      if @function.match(/^\%([a-z]+)(\s+.*)?$/)
+        @script_name = $1
+        @arguments = $2
+      else
+        @script_name = nil
+      end
+    end
+    
+    def to_s
+      @function
+    end
+    
+    def run_on_server(server, logger)
+      logger.info "Running #{@function} on #{server}"
+      
+      if server.is_local?
+        run_locally(server, logger)
+      else
+        run_remotely(server, logger)
+      end
+    end
+    
+  private
+    def run_locally(server, logger)
+      command = nil
+      
+      if @script_name
+        uname = `uname`.chomp.downcase
+        
+        local_path = Action.script_path(uname, @script_name)
+        command = local_path.to_cmd + @arguments
+      else
+        command = @function
+      end
+      
+      ret = nil
+      Dir.chdir(server.root_path) do
+        ret = LSync.run_command(command, logger)
+      end
+      
+      case(ret)
+      when 0
+        return
+      when 1
+        raise AbortBackupException
+      else
+        raise BackupActionError
+      end
+    end
+    
+    def run_remotely(server, logger)
+      conn = server.connect
+      conn.send_object([:set_working_dir, server.root_path])
+      
+      if @script_name
+        uname = `uname`.chomp.downcase
+
+        local_path = Action.script_path(uname, @script_name)
+        
+        logger.info("Sending run_script #{@script_name}...")
+        conn.send_object([:run_script, @script_name, Pathname.new(local_path).read, @arguments])
+      else
+        logger.info("Sending run_command #{@function}...")
+        conn.send_object([:run_command, @function])
+      end
+      
+      conn.run do |message|
+        break if message == :done
+        
+        logger.send(*message)
+      end
+    end
+    
+    def self.script_path(platform, name)
+      exact_script_path(platform, name) || exact_script_path("generic", name)
+    end
+    
+    private
+    def self.exact_script_path(platform, name)
+      path = (Pathname.new(__FILE__).dirname + "actions" + platform + name).expand_path
+      path.exist? ? path : nil
+    end
+  end
+
+end

data/lib/lsync/actions/darwin/disk

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+DISKUTIL = "diskutil"
+action = ARGV[0]
+disk_name = ARGV[1]
+
+def get_disk_id(name)
+  begin
+	  `diskutil list`.match(/#{name}\s*\*?[0-9]+\.[0-9]+ .B\s+(disk[0-9]s[0-9])$/)[1]
+	rescue
+	  exit 5
+  end
+end
+
+if (action == 'mountpoint')
+	puts File.join('', 'Volumes', disk_name, ARGV[2..-1])
+else
+  system DISKUTIL, action, get_disk_id(disk_name)
+end

data/lib/lsync/actions/darwin/terminal

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require 'base64'
+
+cmd = Base64.decode64(ARGV[0])
+
+exec cmd
+

data/lib/lsync/actions/generic/prune

@@ -0,0 +1,251 @@
+#!/usr/bin/env ruby
+
+# This script takes a specified directory, and removes any directories that
+# don't match the supplied policy. Thanks to Scott Lu and his "snapfilter"
+# command which made me realise how complicated my first attempt at doing this
+# was.
+
+require 'pathname'
+require 'fileutils'
+require 'optparse'
+
+require 'set'
+require 'time'
+# Required for strptime
+require 'date'
+
+class Rotation
+	def initialize(path, time)
+		@path = path
+		@time = time
+	end
+
+	attr :time
+	attr :path
+
+	# Sort in reverse order by default
+	def <=> other
+		return other.time <=> @time
+	end
+end
+
+class Period
+	KeepOldest = Proc.new do |t1, t2|
+		t1 > t2
+	end
+
+	KeepYoungest = Proc.new do |t1, t2|
+		t1 < t2
+	end
+
+	def initialize(count)
+		@count = count
+	end
+
+	def filter(values, options = {})
+		slots = {}
+
+		keep = (options[:keep] == :youngest) ? KeepYoungest : KeepOldest
+
+		values.each do |value|
+			time = value.time
+
+			k = key(time)
+
+			# We want to keep the newest backup if possible (<).
+			next if slots.key?(k) and keep.call(value.time, slots[k].time)
+
+			slots[k] = value
+		end
+
+		sorted_values = slots.values.sort
+
+		return sorted_values[0...@count]
+	end
+
+	def key(t)
+		raise ArgumentError
+	end
+
+	def mktime(year, month=1, day=1, hour=0, minute=0, second=0)
+		return Time.gm(year, month, day, hour, minute, second)
+	end
+
+	attr :count
+end
+
+class Hourly < Period
+	def key(t)
+		mktime(t.year, t.month, t.day, t.hour)
+	end
+end
+
+class Daily < Period
+	def key(t)
+		mktime(t.year, t.month, t.day)
+	end
+end
+
+class Weekly < Period
+	def key(t)
+		mktime(t.year, t.month, t.day) - (t.wday * 3600 * 24)
+	end
+end
+
+class Monthly < Period
+	def key(t)
+		mktime(t.year, t.month)
+	end
+end
+
+class Quarterly < Period
+	def key(t)
+		mktime(t.year, (t.month - 1) / 3 * 3 + 1)
+	end
+end
+
+class Yearly < Period
+	def key(t)
+		mktime(t.year)
+	end
+end
+
+class Policy
+	def initialize
+		@periods = {}
+	end
+
+	def <<(period)
+		@periods[period.class] = period
+	end
+
+	def filter(values, options = {})
+		filtered_values = Set.new
+		@periods.values.each do |period|
+			filtered_values += period.filter(values, options)
+		end
+
+		return filtered_values.to_a, (Set.new(values) - filtered_values).to_a
+	end
+
+	attr :periods
+end
+
+OPTIONS = {
+	:Format => "%Y.%m.%d-%H.%M.%S",
+	:Destination => "./*",
+	:Policy => Policy.new,
+	:PolicyOptions => {},
+	:Wet => true
+}
+
+ARGV.options do |o|
+	script_name = File.basename($0)
+
+	o.set_summary_indent('  ')
+	o.banner = "Usage: #{script_name} [options] [directory]"
+	o.define_head "This script is used to prune old backups."
+
+	o.separator ""
+	o.separator "Help and Copyright information"
+
+	o.on("-f format", String, "Set the format of the rotated directory names. See Time$strftime") do |format| 
+		OPTIONS[:Format] = format
+	end
+
+	o.on("-d destination", String, "Set the directory that contains backups to prune.") do |destination|
+		OPTIONS[:Destination] = destination
+	end
+
+	o.on("--dry-run", "Print out what would be deleted, but don't actually delete anything.") do
+		OPTIONS[:Wet] = false
+	end
+
+	o.separator ""
+
+	o.on("--default-policy", "Sets up a typical policy retaining a reasonable number of rotations for up to 20 years.") do
+		OPTIONS[:Policy] << Hourly.new(24)
+		OPTIONS[:Policy] << Daily.new(7*4)
+		OPTIONS[:Policy] << Weekly.new(52)
+		OPTIONS[:Policy] << Monthly.new(12*3)
+		OPTIONS[:Policy] << Quarterly.new(4*10)
+		OPTIONS[:Policy] << Yearly.new(20)
+	end
+
+	o.separator ""
+
+	o.on("--keep-oldest", "Keep older backups within the same period divsion (the default).") do
+		OPTIONS[:PolicyOptions][:keep] = :oldest
+	end
+
+	o.on("--keep-youngest", "Keep younger backups within the same period division.") do
+		OPTIONS[:PolicyOptions][:keep] = :youngest
+	end
+
+	o.on("--hourly count", Integer, "Set the number of hourly backups to keep.") do |count|
+		OPTIONS[:Policy] << Hourly.new(count)
+	end
+
+	o.on("--daily count", Integer, "Set the number of daily backups to keep.") do |count|
+		OPTIONS[:Policy] << Daily.new(count)
+	end
+
+	o.on("--weekly count", Integer, "Set the number of weekly backups to keep.") do |count|
+		OPTIONS[:Policy] << Weekly.new(count)
+	end
+
+	o.on("--monthly count", Integer, "Set the number of monthly backups to keep.") do |count|
+		OPTIONS[:Policy] << Monthly.new(count)
+	end
+
+	o.on("--quaterly count", Integer, "Set the number of monthly backups to keep.") do |count|
+		OPTIONS[:Policy] << Quarterly.new(count)
+	end
+
+	o.on("--yearly count", Integer, "Set the number of monthly backups to keep.") do |count|
+		OPTIONS[:Policy] << Yearly.new(count)
+	end
+
+	o.separator ""
+
+	o.on_tail("--copy", "Display copyright information") do
+		puts "#{script_name}. Copyright (c) 2008-2009 Samuel Williams. Released under the GPLv3."
+		puts "See http://www.oriontransfer.co.nz/ for more information."
+
+		exit
+	end
+
+	o.on_tail("-h", "--help", "Show this help message.") do
+		puts o
+		exit
+	end
+end.parse!
+
+backups = []
+
+Dir[OPTIONS[:Destination]].each do |path|
+	next if path.match("latest")
+	date_string = File.basename(path)
+
+	begin
+		backups << Rotation.new(path, DateTime.strptime(date_string, OPTIONS[:Format]))
+	rescue ArgumentError
+		puts "Skipping #{path}, error parsing #{date_string}: #{$!}"
+	end
+end
+
+keep, erase = OPTIONS[:Policy].filter(backups)
+
+if OPTIONS[:Wet]
+	erase.sort.each do |backup|
+		puts "Erasing #{backup.path}..."
+		$stdout.flush
+		FileUtils.rm_rf(backup.path)
+	end
+else
+	puts "*** Dry Run ***"
+	puts "\tKeeping:"
+	keep.sort.each { |backup| puts "\t\t#{backup.path}" }
+	puts "\tErasing:"
+	erase.sort.each { |backup| puts "\t\t#{backup.path}" }
+end