backup-agoddard 3.0.27

data/backup.gemspec

@@ -0,0 +1,32 @@
+# encoding: utf-8
+
+require File.expand_path('lib/backup/version')
+
+Gem::Specification.new do |gem|
+
+  ##
+  # General configuration / information
+  gem.name        = 'backup-agoddard'
+  gem.version     = Backup::Version.current
+  gem.platform    = Gem::Platform::RUBY
+  gem.authors     = 'Michael van Rooijen; Anthony Goddard; Dane O\'Connor'
+  gem.email       = 'meskyanichi@gmail.com; agoddard@eol.org'
+  gem.homepage    = 'http://github.com/agoddard/backup'
+  gem.summary     = 'Backup is a RubyGem, written for UNIX-like operating systems, that allows you to easily perform backup operations on both your remote and local environments. It provides you with an elegant DSL in Ruby for modeling your backups. Backup has built-in support for various databases, storage protocols/services, syncers, compressors, encryptors and notifiers which you can mix and match. It was built with modularity, extensibility and simplicity in mind.'
+
+  ##
+  # Files and folder that need to be compiled in to the Ruby Gem
+  gem.files         = %x[git ls-files].split("\n")
+  gem.test_files    = %x[git ls-files -- {spec}/*].split("\n")
+  gem.require_path  = 'lib'
+
+  ##
+  # The Backup CLI executable
+  gem.executables   = ['backup']
+
+  ##
+  # Gem dependencies
+  gem.add_dependency 'thor',  ['>= 0.15.4', '< 2']
+  gem.add_dependency 'open4', ['~> 1.3.0']
+
+end

data/bin/backup

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+# Load the Backup core library
+require File.expand_path("../../lib/backup", __FILE__)
+
+# Load the Backup command line interface utility
+require File.expand_path("../../lib/backup/cli/utility", __FILE__)
+
+# Initialize the Backup command line utility
+Backup::CLI::Utility.start

data/lib/backup.rb

@@ -0,0 +1,133 @@
+# encoding: utf-8
+
+# Load Ruby Core Libraries
+require 'rubygems'
+require 'fileutils'
+require 'tempfile'
+require 'yaml'
+require 'etc'
+
+require 'open4'
+require 'thor'
+
+##
+# The Backup Ruby Gem
+module Backup
+
+  ##
+  # Backup's internal paths
+  LIBRARY_PATH       = File.join(File.dirname(__FILE__), 'backup')
+  CLI_PATH           = File.join(LIBRARY_PATH, 'cli')
+  STORAGE_PATH       = File.join(LIBRARY_PATH, 'storage')
+  DATABASE_PATH      = File.join(LIBRARY_PATH, 'database')
+  COMPRESSOR_PATH    = File.join(LIBRARY_PATH, 'compressor')
+  ENCRYPTOR_PATH     = File.join(LIBRARY_PATH, 'encryptor')
+  NOTIFIER_PATH      = File.join(LIBRARY_PATH, 'notifier')
+  SYNCER_PATH        = File.join(LIBRARY_PATH, 'syncer')
+  TEMPLATE_PATH      = File.expand_path('../../templates', __FILE__)
+
+  ##
+  # Autoload Backup CLI files
+  module CLI
+    autoload :Helpers, File.join(CLI_PATH, 'helpers')
+    autoload :Utility, File.join(CLI_PATH, 'utility')
+  end
+
+  ##
+  # Autoload Backup storage files
+  module Storage
+    autoload :Base,       File.join(STORAGE_PATH, 'base')
+    autoload :Cycler,     File.join(STORAGE_PATH, 'cycler')
+    autoload :S3,         File.join(STORAGE_PATH, 's3')
+    autoload :CloudFiles, File.join(STORAGE_PATH, 'cloudfiles')
+    autoload :Ninefold,   File.join(STORAGE_PATH, 'ninefold')
+    autoload :Dropbox,    File.join(STORAGE_PATH, 'dropbox')
+    autoload :FTP,        File.join(STORAGE_PATH, 'ftp')
+    autoload :SFTP,       File.join(STORAGE_PATH, 'sftp')
+    autoload :SCP,        File.join(STORAGE_PATH, 'scp')
+    autoload :RSync,      File.join(STORAGE_PATH, 'rsync')
+    autoload :Local,      File.join(STORAGE_PATH, 'local')
+  end
+
+  ##
+  # Autoload Backup syncer files
+  module Syncer
+    autoload :Base, File.join(SYNCER_PATH, 'base')
+    module Cloud
+      autoload :Base,       File.join(SYNCER_PATH, 'cloud', 'base')
+      autoload :CloudFiles, File.join(SYNCER_PATH, 'cloud', 'cloud_files')
+      autoload :S3,         File.join(SYNCER_PATH, 'cloud', 's3')
+    end
+    module RSync
+      autoload :Base,  File.join(SYNCER_PATH, 'rsync', 'base')
+      autoload :Local, File.join(SYNCER_PATH, 'rsync', 'local')
+      autoload :Push,  File.join(SYNCER_PATH, 'rsync', 'push')
+      autoload :Pull,  File.join(SYNCER_PATH, 'rsync', 'pull')
+    end
+  end
+
+  ##
+  # Autoload Backup database files
+  module Database
+    autoload :Base,       File.join(DATABASE_PATH, 'base')
+    autoload :MySQL,      File.join(DATABASE_PATH, 'mysql')
+    autoload :PostgreSQL, File.join(DATABASE_PATH, 'postgresql')
+    autoload :MongoDB,    File.join(DATABASE_PATH, 'mongodb')
+    autoload :Redis,      File.join(DATABASE_PATH, 'redis')
+    autoload :Riak,       File.join(DATABASE_PATH, 'riak')
+  end
+
+  ##
+  # Autoload compressor files
+  module Compressor
+    autoload :Base,   File.join(COMPRESSOR_PATH, 'base')
+    autoload :Gzip,   File.join(COMPRESSOR_PATH, 'gzip')
+    autoload :Bzip2,  File.join(COMPRESSOR_PATH, 'bzip2')
+    autoload :Custom, File.join(COMPRESSOR_PATH, 'custom')
+    autoload :Pbzip2, File.join(COMPRESSOR_PATH, 'pbzip2')
+    autoload :Lzma,   File.join(COMPRESSOR_PATH, 'lzma')
+  end
+
+  ##
+  # Autoload encryptor files
+  module Encryptor
+    autoload :Base,    File.join(ENCRYPTOR_PATH, 'base')
+    autoload :OpenSSL, File.join(ENCRYPTOR_PATH, 'open_ssl')
+    autoload :GPG,     File.join(ENCRYPTOR_PATH, 'gpg')
+  end
+
+  ##
+  # Autoload notification files
+  module Notifier
+    autoload :Base,      File.join(NOTIFIER_PATH, 'base')
+    autoload :Binder,    File.join(NOTIFIER_PATH, 'binder')
+    autoload :Mail,      File.join(NOTIFIER_PATH, 'mail')
+    autoload :Twitter,   File.join(NOTIFIER_PATH, 'twitter')
+    autoload :Campfire,  File.join(NOTIFIER_PATH, 'campfire')
+    autoload :Prowl,     File.join(NOTIFIER_PATH, 'prowl')
+    autoload :Hipchat,   File.join(NOTIFIER_PATH, 'hipchat')
+    autoload :Pushover,  File.join(NOTIFIER_PATH, 'pushover')
+  end
+
+  ##
+  # Require Backup base files
+  %w{
+    archive
+    binder
+    cleaner
+    config
+    configuration
+    dependency
+    errors
+    hooks
+    logger
+    model
+    package
+    packager
+    pipeline
+    splitter
+    template
+    version
+  }.each {|lib| require File.join(LIBRARY_PATH, lib) }
+
+end

data/lib/backup/archive.rb

@@ -0,0 +1,117 @@
+# encoding: utf-8
+
+module Backup
+  class Archive
+    include Backup::CLI::Helpers
+
+    ##
+    # Stores the name of the archive
+    attr_accessor :name
+
+    ##
+    # Stores an array of different paths/files to store
+    attr_accessor :paths
+
+    ##
+    # Stores an array of different paths/files to exclude
+    attr_accessor :excludes
+
+    ##
+    # String of additional arguments for the `tar` command
+    attr_accessor :tar_args
+
+    ##
+    # Takes the name of the archive and the configuration block
+    def initialize(model, name, &block)
+      @model    = model
+      @name     = name.to_s
+      @paths    = Array.new
+      @excludes = Array.new
+      @tar_args = ''
+
+      instance_eval(&block) if block_given?
+    end
+
+    ##
+    # Adds new paths to the @paths instance variable array
+    def add(path)
+      path = File.expand_path(path)
+      if File.exist?(path)
+        @paths << path
+      else
+        Logger.warn Errors::Archive::NotFoundError.new(<<-EOS)
+          The following path was not found:
+          #{ path }
+          This path will be omitted from the '#{ name }' Archive.
+        EOS
+      end
+    end
+
+    ##
+    # Adds new paths to the @excludes instance variable array
+    def exclude(path)
+      @excludes << File.expand_path(path)
+    end
+
+    ##
+    # Adds the given String of +options+ to the `tar` command.
+    # e.g. '-h --xattrs'
+    def tar_options(options)
+      @tar_args = options
+    end
+
+    ##
+    # Archives all the provided paths in to a single .tar file
+    # and places that .tar file in the folder which later will be packaged
+    # If the model is configured with a Compressor, the tar command output
+    # will be piped through the Compressor command and the file extension
+    # will be adjusted to indicate the type of compression used.
+    def perform!
+      Logger.message "#{ self.class } has started archiving:\n" +
+          paths.map {|path| "  #{path}" }.join("\n")
+
+      archive_path = File.join(Config.tmp_path, @model.trigger, 'archives')
+      FileUtils.mkdir_p(archive_path)
+
+      archive_ext = 'tar'
+      pipeline = Pipeline.new
+
+      pipeline << "#{ utility(:tar) } #{ tar_args } -cPf - " +
+          "#{ paths_to_exclude } #{ paths_to_package }"
+
+      if @model.compressor
+        @model.compressor.compress_with do |command, ext|
+          pipeline << command
+          archive_ext << ext
+        end
+      end
+
+      pipeline << "cat > '#{ File.join(archive_path, "#{name}.#{archive_ext}") }'"
+      pipeline.run
+      if pipeline.success?
+        Logger.message "#{ self.class } Complete!"
+      else
+        raise Errors::Archive::PipelineError,
+            "Failed to Create Backup Archive\n" +
+            pipeline.error_messages
+      end
+    end
+
+    private
+
+    ##
+    # Returns a "tar-ready" string of all the specified paths combined
+    def paths_to_package
+      paths.map {|path| "'#{path}'" }.join(' ')
+    end
+
+    ##
+    # Returns a "tar-ready" string of all the specified excludes combined
+    def paths_to_exclude
+      if excludes.any?
+        excludes.map {|path| "--exclude='#{path}'" }.join(' ')
+      end
+    end
+
+  end
+end

data/lib/backup/binder.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+
+module Backup
+  class Binder
+
+    ##
+    # Creates a new Backup::Notifier::Binder instance. Loops through the provided
+    # Hash to set instance variables
+    def initialize(key_and_values)
+      key_and_values.each do |key, value|
+        instance_variable_set("@#{ key }", value)
+      end
+    end
+
+    ##
+    # Returns the binding (needs a wrapper method because #binding is a private method)
+    def get_binding
+      binding
+    end
+
+  end
+end

data/lib/backup/cleaner.rb

@@ -0,0 +1,121 @@
+# encoding: utf-8
+
+module Backup
+  module Cleaner
+    class << self
+
+      ##
+      # Logs warnings if any temporary files still exist
+      # from the last time this model/trigger was run,
+      # then removes the files.
+      def prepare(model)
+        @model = model
+
+        messages = []
+        if packaging_folder_dirty?
+          messages << <<-EOS
+            The temporary backup folder still contains files!
+            '#{ File.join(Config.tmp_path, @model.trigger) }'
+            These files will now be removed.
+          EOS
+          FileUtils.rm_rf(File.join(Config.tmp_path, @model.trigger))
+        end
+
+        package_files = tmp_path_package_files
+        unless package_files.empty?
+          # the chances that tmp_path would be dirty
+          # AND package files exist are practically nil
+          messages << ('-' * 74) unless messages.empty?
+
+          messages << <<-EOS
+            The temporary backup folder '#{ Config.tmp_path }'
+            appears to contain the package files from the previous backup!
+            #{ package_files.join("\n") }
+            These files will now be removed.
+          EOS
+          package_files.each {|file| FileUtils.rm_f(file) }
+        end
+
+        unless messages.empty?
+          Logger.warn Errors::CleanerError.new(<<-EOS)
+            Cleanup Warning
+            #{ messages.join("\n") }
+            Please check the log for messages and/or your notifications
+            concerning this backup: '#{ @model.label } (#{ @model.trigger })'
+            The temporary files which had to be removed should not have existed.
+          EOS
+        end
+      end
+
+      ##
+      # Remove the temporary folder used during packaging
+      def remove_packaging(model)
+        Logger.message "Cleaning up the temporary files..."
+        FileUtils.rm_rf(File.join(Config.tmp_path, model.trigger))
+      end
+
+      ##
+      # Remove the final package files from tmp_path
+      # Note: 'force' is used, since a Local Storage may *move* these files.
+      def remove_package(package)
+        Logger.message "Cleaning up the package files..."
+        package.filenames.each do |file|
+          FileUtils.rm_f(File.join(Config.tmp_path, file))
+        end
+      end
+
+      ##
+      # Logs warnings if any temporary files still exist
+      # when errors occur during the backup
+      def warnings(model)
+        @model = model
+
+        messages = []
+        if packaging_folder_dirty?
+          messages << <<-EOS
+            The temporary backup folder still contains files!
+            '#{ File.join(Config.tmp_path, @model.trigger) }'
+            This folder may contain completed Archives and/or Database backups.
+          EOS
+        end
+
+        package_files = tmp_path_package_files
+        unless package_files.empty?
+          # the chances that tmp_path would be dirty
+          # AND package files exist are practically nil
+          messages << ('-' * 74) unless messages.empty?
+
+          messages << <<-EOS
+            The temporary backup folder '#{ Config.tmp_path }'
+            appears to contain the backup files which were to be stored:
+            #{ package_files.join("\n") }
+          EOS
+        end
+
+        unless messages.empty?
+          Logger.warn Errors::CleanerError.new(<<-EOS)
+            Cleanup Warning
+            #{ messages.join("\n") }
+            Make sure you check these files before the next scheduled backup for
+            '#{ @model.label } (#{ @model.trigger })'
+            These files will be removed at that time!
+          EOS
+        end
+      end
+
+      private
+
+      def packaging_folder_dirty?
+        !Dir[File.join(Config.tmp_path, @model.trigger, '*')].empty?
+      end
+
+      def tmp_path_package_files
+        Dir[File.join(
+          Config.tmp_path,
+          "????.??.??.??.??.??.#{ @model.trigger }.tar{,[.-]*}"
+        )]
+      end
+
+    end
+  end
+end

data/lib/backup/cli/helpers.rb

@@ -0,0 +1,93 @@
+# encoding: utf-8
+
+module Backup
+  module CLI
+    module Helpers
+      UTILITY = {}
+
+      private
+
+      ##
+      # Runs a system command
+      #
+      # All messages generated by the command will be logged.
+      # Messages on STDERR will be logged as warnings.
+      #
+      # If the command fails to execute, or returns a non-zero exit status
+      # an Error will be raised.
+      #
+      # Returns STDOUT
+      def run(command)
+        name = command_name(command)
+        Logger.message "Running system utility '#{ name }'..."
+
+        begin
+          out, err = '', ''
+          ps = Open4.popen4(command) do |pid, stdin, stdout, stderr|
+            stdin.close
+            out, err = stdout.read.strip, stderr.read.strip
+          end
+        rescue Exception => e
+          raise Errors::CLI::SystemCallError.wrap(e, <<-EOS)
+            Failed to execute system command on #{ RUBY_PLATFORM }
+            Command was: #{ command }
+          EOS
+        end
+
+        if ps.success?
+          unless out.empty?
+            Logger.message(
+              out.lines.map {|line| "#{ name }:STDOUT: #{ line }" }.join
+            )
+          end
+
+          unless err.empty?
+            Logger.warn(
+              err.lines.map {|line| "#{ name }:STDERR: #{ line }" }.join
+            )
+          end
+
+          return out
+        else
+          raise Errors::CLI::SystemCallError, <<-EOS
+            '#{ name }' Failed on #{ RUBY_PLATFORM }
+            The following information should help to determine the problem:
+            Command was: #{ command }
+            Exit Status: #{ ps.exitstatus }
+            STDOUT Messages: #{ out.empty? ? 'None' : "\n#{ out }" }
+            STDERR Messages: #{ err.empty? ? 'None' : "\n#{ err }" }
+          EOS
+        end
+      end
+
+
+      ##
+      # Returns the full path to the specified utility.
+      # Raises an error if utility can not be found in the system's $PATH
+      def utility(name)
+        name = name.to_s.strip
+        raise Errors::CLI::UtilityNotFoundError,
+            'Utility Name Empty' if name.empty?
+
+        path = UTILITY[name] || %x[which #{ name } 2>/dev/null].chomp
+        if path.empty?
+          raise Errors::CLI::UtilityNotFoundError, <<-EOS
+            Could not locate '#{ name }'.
+            Make sure the specified utility is installed
+            and available in your system's $PATH.
+          EOS
+        end
+        UTILITY[name] = path
+      end
+
+      ##
+      # Returns the name of the command name from the given command line
+      def command_name(command)
+        i = command =~ /\s/
+        command = command.slice(0, i) if i
+        command.split('/')[-1]
+      end
+
+    end
+  end
+end