BackupMan 0.1.0

data/README.rdoc

@@ -0,0 +1,138 @@
+= BackupMan
+
+* Homepage: http://github.com/oowoo/backupman
+
+== DESCRIPTION:
+
+A tool for system administrators to easily configure pull-over-SSH backups.
+Install it on your backup server and pull your backups over SSH from there.
+
+== BACKGROUND
+
+When you have servers on the Internet or in a DMZ they are kind of "untrusted"
+because they are vulnerable to become hacked/cracked. It is therefore a best
+practice to <b>pull</b> backups from there instead of letting these servers
+push their data to the backup servers. Advantage: When doing SSH-pull you do
+not have to store any kind of authentication information on your "untrusted"
+remote servers.
+
+== FEATURES/PROBLEMS:
+
+Currently these types of backups are configurable:
+
+* tar - the tar becomes compressed on the source and is <b>streamed</b> over SSH to
+  the backup server; no tar files are created on the source side.
+
+* rsync - changes on the source side become synchronized in a very efficient
+  way to a copy on the backup server; you get a 1:1 copy of your data on the
+  backup server
+
+* mysqldump - this simply does a full dump of all your databases, compresses
+  it and streams it over to the backup server
+
+== SYNOPSIS:
+
+To run a fully configured backup configured in <tt>/etc/backup_man/somehost</tt> run
+
+  backup_man somehost
+  
+To run a backup with custom log destination (default is
+<tt>/var/log/backup_man.log</tt>) and with a custom configuration file path
+use
+
+  backup_man -l /full/path/to/logfile /full/path/to/configfile
+  
+Full synopsis:
+
+  Usage: backup_man [options] {configname | configpath}
+
+  Options are:
+      -l, --logpath=PATH               Path to the log file.
+                                       This can NOT be configured in the config file.
+                                       Default: /var/log/backup_man.log
+      -d, --debug                      Debug mode.
+                                       Much output on screen and in the log file.
+      -h, --help                       Show this help message.
+  
+
+== REQUIREMENTS:
+
+* gems: log4r
+
+== INSTALL / CONFIGURATION ON THE BACKUP SERVER:
+
+* <tt>sudo gem install backup_man</tt>
+* remove the password from your ssh private key, e.g. do +ssh-keygen -p+
+* create an configuration file in <tt>/etc/backup_man</tt>, copy this example and adapt to your needs:
+
+   ############################################################################
+   
+   # = Global settings
+   # These settings are valid for all latter backup definitions
+   
+   # the default destination directory; if you do not set this it defaults to
+   # /var/backups/backup_man
+   DESTDIR = "/Users/srm/Documents/Backups"
+   
+   # where the lockfiles go (defaults to /var/lock/backup_man/_backupname_);
+   # backup_man ensures, that each backup task can only run once at a time and
+   # uses lockfiles for this
+   LOCKDIR = "#{DESTDIR}"
+   
+   # ssh_app defines the call to ssh with command line parameters, e.g. if you
+   # want to use a special identity file; SSH_APP defaults to 'ssh'
+   SSH_APP='ssh -i /Users/srm/.ssh/id_rsa'
+   
+   ############################################################################
+   
+   # = Helper variables
+   # These are just used in this file to ease the backup definitions
+   
+   dirs_to_backup = [ "/etc", "/home", "/root", "/srv", "/var/backups",
+   "/var/cache/git", "/var/log", "/var/mail", "/var/rails", "/var/svn",
+   "/var/www" ]
+   
+   ############################################################################
+   
+   # = Backup defintions
+   # Currently there are these types of backups:
+   # * Tar - as the name says; simple tar of a bunch of directories into a tgz
+   # * Rsync - rsync of a bunch of directories
+   # * Mysql - does a simple mysqldump of all databases
+   #       
+   # == Common settings
+   # * backup: an array of directories to backup
+   # * to: the destination directory on the local machine (defaults to DESTDIR)
+   # * onlyif: specify a condition (as Ruby code) which is checked before the
+   #   actual backup run; it must evaluate true to have the backup run - see the
+   #   homepage for details on how to write conditions
+   #
+   Tar.new( 'akeso' ) do |b|
+     b.backup  dirs_to_backup
+     # in this example, this job only runs on saturdays and only if the backup
+     # does not exist already
+     b.onlyif  'Date.today.cwday == 6 && !exists?'
+   end
+   
+   Rsync.new( 'akeso' ) do |b|
+     b.backup  dirs_to_backup
+     # rsync backups go to _DESTDIR_/rsync per default; to override use the "to"
+     # setting: 
+     ## b.to '/var/backup/custom-rsync-destination'
+   end
+   
+   Mysql.new( 'akeso' ) do |b|
+     # in this case, we simply do not run the backup if we already have one
+     b.onlyif    '!exists?'
+   end
+
+
+== CONFIGURATION TASKS ON THE REMOTES
+
+* configure passwordless ssh login via certificates, e.g. do a +ssh-copy-id user@remotehostname+
+
+== LICENSE:
+
+Author:: Markus Strauss (mailto:markus@itstrauss.eu)
+Copyright:: Copyright (c)2009 Markus Strauss
+License:: GPLv3 (http://www.gnu.org/licenses/gpl-3.0.txt)

data/Rakefile

@@ -0,0 +1,49 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "BackupMan"
+    gem.summary = %Q{A tool for system administrators to easily configure pull-over-SSH backups.}
+    gem.description = %Q{A tool for system administrators to easily configure pull-over-SSH backups. Install this gem on your backup server. Configure your backups definitions in /etc/backup_man and run backup_man from cron to securely pull your data over SSH.}
+    gem.email = "Markus@ITstrauss.eu"
+    gem.homepage = "http://github.com/oowoo/BackupMan"
+    gem.authors = ["Markus Strauss"]
+    gem.rubyforge_project = "backupman"
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.add_development_dependency "yard", ">= 0"
+    gem.add_dependency "log4r", ">= 1.1.2"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::RubyforgeTasks.new do |rubyforge|
+    rubyforge.doc_task = "yardoc"
+  end
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+begin
+  require 'yard'
+  YARD::Rake::YardocTask.new
+rescue LoadError
+  task :yardoc do
+    abort "YARD is not available. In order to run yardoc, you must: sudo gem install yard"
+  end
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/backup_man

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+#
+#  Created on 2009-11-15.
+#  Copyright (c) 2009. All rights reserved.
+
+require 'rubygems'
+require File.expand_path(File.dirname(__FILE__) + "/../lib/backup_man")
+require "backup_man/cli"
+
+BackupMan::CLI.execute(STDOUT, ARGV)

data/examples/example-host-config

@@ -0,0 +1,60 @@
+############################################################################
+
+# = Global settings
+# These settings are valid for all latter backup definitions
+
+# the default destination directory; if you do not set this it defaults to
+# /var/backups/backup_man
+DESTDIR = "/Users/srm/Documents/Backups"
+
+# where the lockfiles go (defaults to /var/lock/backup_man/_backupname_);
+# backup_man ensures, that each backup task can only run once at a time and
+# uses lockfiles for this
+LOCKDIR = "#{DESTDIR}"
+
+# ssh_app defines the call to ssh with command line parameters, e.g. if you
+# want to use a special identity file; SSH_APP defaults to 'ssh'
+SSH_APP='ssh -i /Users/srm/.ssh/id_rsa'
+
+############################################################################
+
+# = Helper variables
+# These are just used in this file to ease the backup definitions
+
+dirs_to_backup = [ "/etc", "/home", "/root", "/srv", "/var/backups",
+"/var/cache/git", "/var/log", "/var/mail", "/var/rails", "/var/svn",
+"/var/www" ]
+
+############################################################################
+
+# = Backup defintions
+# Currently there are these types of backups:
+# * Tar - as the name says; simple tar of a bunch of directories into a tgz
+# * Rsync - rsync of a bunch of directories
+# * Mysql - does a simple mysqldump of all databases
+#       
+# == Common settings
+# * backup: an array of directories to backup
+# * to: the destination directory on the local machine (defaults to DESTDIR)
+# * onlyif: specify a condition (as Ruby code) which is checked before the
+#   actual backup run; it must evaluate true to have the backup run - see the
+#   homepage for details on how to write conditions
+#
+Tar.new( 'akeso' ) do |b|
+  b.backup  dirs_to_backup
+  # in this example, this job only runs on saturdays and only if the backup
+  # does not exist already
+  b.onlyif  'Date.today.cwday == 6 && !exists?'
+end
+
+Rsync.new( 'akeso' ) do |b|
+  b.backup  dirs_to_backup
+  # rsync backups go to _DESTDIR_/rsync per default; to override use the "to"
+  # setting: 
+  ## b.to '/var/backup/custom-rsync-destination'
+end
+
+Mysql.new( 'akeso' ) do |b|
+  # in this case, we simply do not run the backup if we already have one
+  b.onlyif    '!exists?'
+end

data/lib/backup_man/backup.rb

@@ -0,0 +1,110 @@
+require 'backup_man/backup_man'
+require 'backup_man/dsl'
+
+module BackupMan
+
+  def self.log_end_of_operation
+    Log.instance.info( "Finished #{self}." )
+  end
+
+  # we wanna log when the program ends
+  at_exit { self.log_end_of_operation }
+
+
+  # References: _Design Patterns in Ruby_ by Russ Olsen
+  class Backup
+    
+    # the name of our backup set; this is used for generating a default
+    # backup_directory; e.g. use the hostname of the machine to backup
+    attr_reader   :name
+    
+    # where shall our backup data go (directory path)
+    attr_reader   :backup_directory
+    
+    # user name of the remote machine (defaults to 'root')
+    attr_accessor :user
+    
+    # hostname of the remote machine (defaults to job definition name)
+    attr_accessor :host
+    
+    # DSL for conditional execution
+    include DSL
+    
+    def_dsl :onlyif
+    def_dsl :backup, :data_sources
+    def_dsl :to, :backup_directory
+    def_dsl :user
+    def_dsl :host
+    
+    
+    # this method sets all the default values but does not overwrite existing
+    # settings; this method cannot be called in the initializer of Backup
+    # because the default values are not available at that time;
+    def set_defaults
+      @data_sources = [] unless @data_sources
+      @backup_directory = "#{BackupMan.instance.destdir}/#{@name}" unless @backup_directory
+      @onlyif = "true" if @onlyif.nil?
+      @user = 'root' unless @user
+      @host = @name unless @host
+    end
+
+    # yields the block which comes from the DSL configuration file; also
+    # registers the new backup configuration with {BackupMan}
+    def initialize( name )
+      @name = name      
+      yield(self) if block_given?
+      BackupMan.instance.register_backup( self )
+    end
+    
+    # calling this actually runs the backup; DO NOT override this; override
+    # _run instead
+    def run
+      log_begin_of_run
+      set_defaults
+      debug_log_dsl_info
+      onlyif = eval( @onlyif )
+      Log.instance.debug( "onlyif = { #{@onlyif} } evaluates #{onlyif}" )
+      if onlyif
+        unless @backup_directory
+          Log.instance.error( "#{self}: No backup directory. Don't know where to store all this stuff.")
+        else
+          FileUtils.mkdir_p @backup_directory
+          _run
+        end
+      else
+        Log.instance.info( "#{self}: Preconditions for backup run not fulfilled.")
+      end
+      log_end_of_run
+    end
+    
+    # @return [String]
+    def to_s
+      "#{self.class} #{self.name}"
+    end
+
+
+    
+    private
+    
+    # @abstract override this to implement the actual backup commands
+    def _run
+      throw "Hey. Cannot run just 'Backup'."
+    end
+    
+    # not used acutally
+    def log_begin_of_run
+      Log.instance.info( "Starting #{self.class} run for #{@name}." )
+    end
+    
+    # simply logs that the program terminates
+    def log_end_of_run
+      Log.instance.info( "Finished #{self.class} run for #{@name}." )
+    end
+
+    # @return [String] the ssh command string including user@host
+    def ssh_connect_cmd
+      "#{BackupMan.instance.ssh_app} #{@user}@#{@host}"  
+    end
+
+  end
+end

data/lib/backup_man/backup_man.rb

@@ -0,0 +1,45 @@
+require 'singleton'
+
+module BackupMan
+  # this is a singleton class managing the application; it holds some
+  # important global settings, like the global destination directory;
+  class BackupMan
+
+    include Singleton
+    
+    attr_accessor :destdir, :ssh_app, :logfile, :lockdir
+
+    def initialize
+      @backups = []
+    end
+
+    def register_backup( backup )
+      @backups << backup
+    end
+    
+    def run
+      @backups.each do |backup|
+        begin
+          backup.run
+        rescue Interrupt
+          Log.instance.warn( "Interrupt: Cancelling remaining operations.")
+          return
+        end
+      end        
+    end
+    
+    def build_lockfile_path( filename )
+      "#{@lockdir}/#{filename}"
+    end
+    
+    def set_default( symbol, value )
+      const_s = symbol.to_s.upcase
+      if CLI.const_defined?( const_s )
+        self.instance_variable_set( "@#{symbol.to_s}", CLI.const_get( const_s ) )
+      else
+        self.instance_variable_set( "@#{symbol.to_s}", value )
+      end
+    end
+
+  end
+end

data/lib/backup_man/cli.rb

@@ -0,0 +1,101 @@
+require 'optparse'
+require 'date'
+require 'pathname'
+
+require_relative 'log'
+require_relative 'tar'
+require_relative 'rsync'
+require_relative 'mysql'
+
+
+module BackupMan
+  class CLI
+    def self.execute(stdout, arguments=[])
+
+      # NOTE: the option -p/--path= is given as an example, and should be replaced in your application.
+
+      options = {
+        :debug     => false,
+        :logpath   => '/var/log/backup_man.log'
+      }
+      mandatory_options = %w(  )
+
+      parser = OptionParser.new do |opts|
+        opts.banner = <<-BANNER.gsub(/^          /,'')
+          BackupMan handles your SSH-pull-style backups. Call it via cron or any other scheduler.
+
+          Usage: #{File.basename($0)} [options] {configname | configpath}
+
+          Options are:
+        BANNER
+        opts.separator ""
+        opts.on("-l", "--logpath=PATH", String,
+                "Path to the log file. This can NOT be configured in the config file.",
+                "Default: /var/log/backup_man.log") { |arg| options[:logpath] = arg }
+        opts.on("-d", "--debug", "Debug mode.", "Much output on screen and in the log file." ) {
+          options[:debug] = true
+        }
+        opts.on("-h", "--help",
+                "Show this help message.") { stdout.puts opts; exit }
+        opts.parse!(arguments)
+
+        if mandatory_options && mandatory_options.find { |option| options[option.to_sym].nil? }
+          stdout.puts opts; exit
+        end
+      end
+
+      # doing our stuff here
+      
+      # first we check if our logfile is writeable; if not, we give a warning
+      logfile = Pathname.new( options[:logpath] )
+      if (logfile.exist? && logfile.writable?) || logfile.parent.writable?
+        BackupMan.instance.logfile = logfile.to_s
+      else
+        Log.instance.warn( "Log file is not writeable: #{logfile}.")
+      end
+      
+      # root-warning
+      Log.instance.warn( "Please do not run this program as root.") if `id -u`.strip == '0'
+
+      # reconfigure our Logger for debugging if necessary
+      if options[:debug]
+        Log.instance.enable_debugmode
+        Log.instance.debug( "Debugging mode enabled.")
+      end
+      
+
+      unless ARGV[0]
+        Log.instance.fatal( "No config file given." )
+        stdout.puts parser
+        exit 1
+      else
+        config_filepath = Pathname.new( ARGV[0] )
+        config_filepath = Pathname.new "/etc/backup_man/#{config_filepath}" unless config_filepath.file?
+        
+        unless config_filepath.file?
+          Log.instance.fatal( "Config file not found [#{config_filepath}].")
+          exit 2
+        end
+
+        # get and evaluate the config file; errors in the config file may be
+        # difficult to debug, so be careful
+        Log.instance.debug( "Reading file '#{config_filepath}'.")
+        eval( File.read( config_filepath ) )
+
+        # configure global defaults
+        BackupMan.instance.set_default( :destdir, '/var/backups/backup_man')
+        BackupMan.instance.set_default( :lockdir, '/var/lock/backup_man')
+        BackupMan.instance.set_default( :ssh_app, 'ssh')
+        
+        Log.instance.debug( "Global settings:")
+        Log.instance.debug( "  DESTDIR = #{BackupMan.instance.destdir}")
+        Log.instance.debug( "  LOCKDIR = #{BackupMan.instance.lockdir}")
+        Log.instance.debug( "  SSH_APP = #{BackupMan.instance.ssh_app}")
+
+        # run the whole thing
+        BackupMan.instance.run
+      end
+      
+    end
+  end
+end

data/lib/backup_man/command.rb

@@ -0,0 +1,74 @@
+
+require 'digest/md5'
+
+module BackupMan
+  # = Command class
+  # This class allows to run commands and makes sure the same command cannot
+  # run two times at the same time.
+  #  
+  # This ensured by creating lockfiles in the working directory. To override
+  # define LOCKDIR.
+
+  class Command
+
+    attr_reader :cmd
+
+    def initialize( cmd )
+      @cmd = cmd      
+    end
+
+    # returns true if the command is locked (= currently running)
+    def locked?
+      File.exists?( self.lockfile )
+    end
+
+    # returns the path to the lockfile for this command
+    def lockfile
+      hash = Digest::MD5.hexdigest(cmd.to_s)
+      BackupMan.instance.build_lockfile_path( "#{hash}.lock.txt" )
+    end
+
+    # returns a nice string representation of the command
+    def to_s
+      self.cmd.to_s
+    end
+
+    def run
+      unless locked?
+        lock
+        begin
+          Log.instance.info( "#{self}: Running.")
+          print `#{cmd}` unless TESTING
+        rescue Interrupt
+          Log.instance.warn( "#{self}: Operation interrupted.")
+          raise
+        ensure
+          unlock
+        end
+      else
+        Log.instance.warn( "Command already running: #{self}." )
+      end
+    end
+
+
+    private
+
+    def lock
+      Log.instance.debug( "Locking command " + self.cmd )
+      unless locked?
+        f = File.new( self.lockfile, "w" )
+        # FIXME: command output shall go into the correct logging lvl (eg ERROR)
+        f.write(self.cmd)
+        f.close
+      else
+        Log.instance.info( "Lockfile exists: " + lockfile )
+      end
+    end
+
+    def unlock
+      Log.instance.debug( "Unlocking command " + self.cmd )
+      FileUtils.remove_file( self.lockfile )
+    end
+
+  end
+end

data/lib/backup_man/dsl.rb

@@ -0,0 +1,47 @@
+module BackupMan
+
+  # use "include DSL" to use this module
+  module DSL
+
+    # extend host class with class methods when we're included
+    def self.included(host_class)
+      host_class.extend(ClassMethods)
+    end
+    
+    def debug_log_dsl_info
+      Log.instance.debug( "Job settings:")
+      self.class.dsl_methods.each do |method, var|
+        Log.instance.debug( "  #{method} = #{self.instance_variable_get("@#{var}")}" )
+      end
+    end
+    
+    module ClassMethods
+                  
+      def def_dsl( name, var = name )
+        class_eval( %Q{
+          def #{name}( #{var} )
+            @#{var} = #{var}
+          end
+          })
+        register_dsl( name, var )
+      end
+
+      def register_dsl( name, var )
+        @dsl_methods = [] if @dsl_methods.nil?
+        @dsl_methods << [name, var]
+        puts "#{self} #{self.dsl_methods.join ","}"
+      end
+      
+      # returns an array of all dsl methods of this and all superclasses
+      def dsl_methods
+        dsl_methods = []
+        if self.superclass != Object
+          dsl_methods = dsl_methods | self.superclass.dsl_methods
+        end
+        dsl_methods | @dsl_methods
+      end
+
+
+    end # ClassMethods
+  end #DSL
+end #BackupMan

data/lib/backup_man/log.rb

@@ -0,0 +1,48 @@
+require 'singleton'
+require 'delegate'
+require 'rubygems'
+require 'log4r'
+
+module BackupMan
+
+  # = Singleton logger
+  #
+  # This one logs INFO and higher to a file and WARN and higher to STDERR. It
+  # does this by creating a new Log4r::Logger object and configuring it.
+  # SimpleDelegator wrapps this.
+  #  
+  # By default, the logfile goes into workdir/backup_man.log. To override use
+  # LOGFILE in the backup case definition.
+  #
+  # == References
+  # * http://www.5dollarwhitebox.org/drupal/ruby_dual_output_logging
+  # * http://rubyforge.org/snippet/detail.php?type=snippet&id=146
+  #   
+  class Log < SimpleDelegator
+    include Singleton
+    include Log4r
+
+    def initialize
+      @logger = Logger.new( "BackupMan" )
+      super( @logger )
+
+      console_format = PatternFormatter.new(:pattern => "%l:\t %m")
+      stderr_outputter = Log4r::StderrOutputter.new( 'console', :formatter => console_format )
+      stderr_outputter.level = WARN
+      @logger.add stderr_outputter
+
+      if filename = BackupMan.instance.logfile
+        file_format = PatternFormatter.new(:pattern => "[ %d ] %l\t %m")
+        file_outputter = FileOutputter.new('fileOutputter', :filename => filename, :trunc => false, :formatter => file_format )
+        file_outputter.level = DEBUG
+        @logger.add file_outputter
+      end
+    end
+    
+    def enable_debugmode
+      Log4r::Outputter.each_outputter { |outputter| outputter.level = DEBUG }
+    end
+
+  end
+
+end