backup-agoddard 3.0.27

data/lib/backup/encryptor/open_ssl.rb

@@ -0,0 +1,72 @@
+# encoding: utf-8
+
+module Backup
+  module Encryptor
+    class OpenSSL < Base
+
+      ##
+      # The password that'll be used to encrypt the backup. This
+      # password will be required to decrypt the backup later on.
+      attr_accessor :password
+
+      ##
+      # The password file to use to encrypt the backup.
+      attr_accessor :password_file
+
+      ##
+      # Determines whether the 'base64' should be used or not
+      attr_accessor :base64
+
+      ##
+      # Determines whether the 'salt' flag should be used
+      attr_accessor :salt
+
+      ##
+      # Creates a new instance of Backup::Encryptor::OpenSSL and
+      # sets the password attribute to what was provided
+      def initialize(&block)
+        super
+
+        @base64        ||= false
+        @salt          ||= true
+        @password_file ||= nil
+
+        instance_eval(&block) if block_given?
+      end
+
+      ##
+      # This is called as part of the procedure run by the Packager.
+      # It sets up the needed options to pass to the openssl command,
+      # then yields the command to use as part of the packaging procedure.
+      # Once the packaging procedure is complete, it will return
+      # so that any clean-up may be performed after the yield.
+      def encrypt_with
+        log!
+        yield "#{ utility(:openssl) } #{ options }", '.enc'
+      end
+
+      private
+
+      ##
+      # Uses the 256bit AES encryption cipher, which is what the
+      # US Government uses to encrypt information at the "Top Secret" level.
+      #
+      # The -base64 option will make the encrypted output base64 encoded,
+      # this makes the encrypted file readable using text editors
+      #
+      # The -salt option adds strength to the encryption
+      #
+      # Always sets a password option, if even no password is given,
+      # but will prefer the password_file option if both are given.
+      def options
+        opts = ['aes-256-cbc']
+        opts << '-base64' if @base64
+        opts << '-salt'   if @salt
+        opts << ( @password_file.to_s.empty? ?
+                "-k '#{@password}'" : "-pass file:#{@password_file}" )
+        opts.join(' ')
+      end
+
+    end
+  end
+end

data/lib/backup/errors.rb

@@ -0,0 +1,124 @@
+# encoding: utf-8
+
+module Backup
+  ##
+  # - automatically defines module namespaces referenced under Backup::Errors
+  # - any constant name referenced that ends with 'Error' will be created
+  #   as a subclass of Backup::Errors::Error
+  # e.g.
+  #   err = Backup::Errors::Foo::Bar::FooError.new('error message')
+  #   err.message => "Foo::Bar::FooError: error message"
+  #
+  module ErrorsHelper
+    def const_missing(const)
+      if const.to_s.end_with?('Error')
+        module_eval("class #{const} < Backup::Errors::Error; end")
+      else
+        module_eval("module #{const}; extend Backup::ErrorsHelper; end")
+      end
+      const_get(const)
+    end
+  end
+
+  ##
+  # provides cascading errors with formatted messages
+  # see the specs for details
+  #
+  # e.g.
+  # module Backup
+  #   begin
+  #     begin
+  #       begin
+  #         raise Errors::ZoneAError, 'an error occurred in Zone A'
+  #       rescue => err
+  #         raise Errors::ZoneBError.wrap(err, <<-EOS)
+  #           an error occurred in Zone B
+  #
+  #           the following error should give a reason
+  #         EOS
+  #       end
+  #     rescue => err
+  #       raise Errors::ZoneCError.wrap(err)
+  #     end
+  #   rescue => err
+  #     puts Errors::ZoneDError.wrap(err, 'an error occurred in Zone D')
+  #   end
+  # end
+  #
+  # Outputs:
+  #   ZoneDError: an error occurred in Zone D
+  #     Reason: ZoneCError
+  #     ZoneBError: an error occurred in Zone B
+  #
+  #     the following error should give a reason
+  #     Reason: ZoneAError
+  #     an error occurred in Zone A
+  #
+  module Errors
+    extend ErrorsHelper
+
+    class Error < StandardError
+
+      def self.wrap(orig_err, msg = nil)
+        new(msg, orig_err)
+      end
+
+      def initialize(msg = nil, orig_err = nil)
+        super(msg)
+        set_backtrace(orig_err.backtrace) if @orig_err = orig_err
+      end
+
+      def to_s
+        return @to_s if @to_s
+        orig_to_s = super()
+
+        if orig_to_s == self.class.to_s
+          msg = orig_err_msg ?
+              "#{orig_err_class}: #{orig_err_msg}" : orig_err_class
+        else
+          msg = format_msg(orig_to_s)
+          msg << "\n  Reason: #{orig_err_class}" + (orig_err_msg ?
+              "\n  #{orig_err_msg}" : ' (no message given)') if @orig_err
+        end
+
+        @to_s = msg ? msg_prefix + msg : class_name
+      end
+
+      private
+
+      def msg_prefix
+        @msg_prefix ||= class_name + ': '
+      end
+
+      def orig_msg
+        @orig_msg ||= to_s.sub(msg_prefix, '')
+      end
+
+      def class_name
+        @class_name ||= self.class.to_s.sub('Backup::Errors::', '')
+      end
+
+      def orig_err_class
+        return unless @orig_err
+
+        @orig_err_class ||= @orig_err.is_a?(Errors::Error) ?
+            @orig_err.send(:class_name) : @orig_err.class.to_s
+      end
+
+      def orig_err_msg
+        return unless @orig_err
+        return @orig_err_msg unless @orig_err_msg.nil?
+
+        msg = @orig_err.is_a?(Errors::Error) ?
+            @orig_err.send(:orig_msg) : @orig_err.to_s
+        @orig_err_msg = (msg == orig_err_class) ?
+            false : format_msg(msg)
+      end
+
+      def format_msg(msg)
+        msg.gsub(/^ */, '  ').strip
+      end
+    end
+
+  end
+end

data/lib/backup/hooks.rb

@@ -0,0 +1,68 @@
+
+# encoding: utf-8
+
+module Backup
+  class Hooks
+    include Backup::CLI::Helpers
+
+    ##
+    # Stores a block to be run before the backup
+    attr_accessor :before_proc
+
+    ##
+    # Stores a block to be run before the backup
+    attr_accessor :after_proc
+
+    ##
+    # The model
+    attr_reader :model
+
+    def initialize(model, &block)
+      @model = model
+      @before_proc = Proc.new { } # noop
+      @after_proc = Proc.new { } # noop
+      instance_eval(&block) if block_given?
+    end
+
+    def before(&code)
+      @before_proc = code
+    end
+
+    def after(&code)
+      @after_proc = code
+    end
+
+    def before!
+      begin
+        Logger.message "Performing Before Hook"
+        @before_proc.call(model)
+        Logger.message "Before Hook Completed Successfully"
+      rescue => err
+        raise Errors::Hooks::BeforeHookError.wrap(
+          err, "Before Hook Failed!"
+        )
+      end
+    end
+
+    def after!
+      begin
+        Logger.message "Performing After Hook"
+        @after_proc.call(model)
+        Logger.message "After Hook Completed Successfully"
+      rescue => err
+        raise Errors::Hooks::AfterHookError.wrap(
+          err, "After Hook Failed!"
+        )
+      end
+    end
+
+    def perform!(hook)
+      case hook
+      when :before
+        before!
+      when :after
+        after!
+      end
+    end
+  end
+end

data/lib/backup/logger.rb

@@ -0,0 +1,152 @@
+# encoding: utf-8
+
+module Backup
+  module Logger
+    class << self
+
+      attr_accessor :quiet
+
+      ##
+      # Outputs a messages to the console and writes it to the backup.log
+      def message(string)
+        to_console  loggify(string, :message, :green)
+        to_file     loggify(string, :message)
+      end
+
+      ##
+      # Outputs an error to the console and writes it to the backup.log
+      # Called when an Exception has caused the backup process to abort.
+      def error(string)
+        to_console  loggify(string, :error,   :red), true
+        to_file     loggify(string, :error)
+      end
+
+      ##
+      # Outputs a notice to the console and writes it to the backup.log
+      # Sets #has_warnings? true so :on_warning notifications will be sent
+      def warn(string)
+        @has_warnings = true
+        to_console  loggify(string, :warning, :yellow), true
+        to_file     loggify(string, :warning)
+      end
+
+      # Outputs the data as if it were a regular 'puts' command,
+      # but also logs it to the backup.log
+      def normal(string)
+        to_console  loggify(string)
+        to_file     loggify(string)
+      end
+
+      ##
+      # Silently logs data to the log file
+      def silent(string)
+        to_file     loggify(string, :silent)
+      end
+
+      ##
+      # Returns an Array of all messages written to the log file for this session
+      def messages
+        @messages ||= []
+      end
+
+      ##
+      # Returns true if any warnings have been issued
+      def has_warnings?
+        @has_warnings ||= false
+      end
+
+      def clear!
+        messages.clear
+        @has_warnings = false
+      end
+
+      def truncate!(max_bytes = 500_000)
+        log_file = File.join(Config.log_path, 'backup.log')
+        return unless File.exist?(log_file)
+
+        if File.stat(log_file).size > max_bytes
+          FileUtils.mv(log_file, log_file + '~')
+          File.open(log_file + '~', 'r') do |io_in|
+            File.open(log_file, 'w') do |io_out|
+              io_in.seek(-max_bytes, IO::SEEK_END) && io_in.gets
+              while line = io_in.gets
+                io_out.puts line
+              end
+            end
+          end
+          FileUtils.rm_f(log_file + '~')
+        end
+      end
+
+      private
+
+      ##
+      # Returns the time in [YYYY/MM/DD HH:MM:SS] format
+      def time
+        Time.now.strftime("%Y/%m/%d %H:%M:%S")
+      end
+
+      ##
+      # Receives a String, or an Object that responds to #to_s (e.g. an
+      # Exception), from one of the messaging methods and converts it into an
+      # Array of Strings, split on newline separators. Each line is then
+      # formatted into a log format based on the given options, and the Array
+      # returned to be passed to to_console() and/or to_file().
+      def loggify(string, type = false, color = false)
+        lines = string.to_s.split("\n")
+        if type
+          type = send(color, type) if color
+          time_now = time
+          lines.map {|line| "[#{time_now}][#{type}] #{line}" }
+        else
+          lines
+        end
+      end
+
+      ##
+      # Receives an Array of Strings to be written to the console.
+      def to_console(lines, stderr = false)
+        return if quiet
+        lines.each {|line| stderr ? Kernel.warn(line) : puts(line) }
+      end
+
+      ##
+      # Receives an Array of Strings to be written to the log file.
+      def to_file(lines)
+        File.open(File.join(Config.log_path, 'backup.log'), 'a') do |file|
+          lines.each {|line| file.puts line }
+        end
+        messages.push(*lines)
+      end
+
+      ##
+      # Invokes the #colorize method with the provided string
+      # and the color code "32" (for green)
+      def green(string)
+        colorize(string, 32)
+      end
+
+      ##
+      # Invokes the #colorize method with the provided string
+      # and the color code "33" (for yellow)
+      def yellow(string)
+        colorize(string, 33)
+      end
+
+      ##
+      # Invokes the #colorize method the with provided string
+      # and the color code "31" (for red)
+      def red(string)
+        colorize(string, 31)
+      end
+
+      ##
+      # Wraps the provided string in colorizing tags to provide
+      # easier to view output to the client
+      def colorize(string, code)
+        "\e[#{code}m#{string}\e[0m"
+      end
+
+    end
+  end
+end

data/lib/backup/model.rb

@@ -0,0 +1,409 @@
+# encoding: utf-8
+
+module Backup
+  class Model
+    include Backup::CLI::Helpers
+
+    class << self
+      ##
+      # The Backup::Model.all class method keeps track of all the models
+      # that have been instantiated. It returns the @all class variable,
+      # which contains an array of all the models
+      def all
+        @all ||= []
+      end
+
+      ##
+      # Return the first model matching +trigger+.
+      # Raises Errors::MissingTriggerError if no matches are found.
+      def find(trigger)
+        trigger = trigger.to_s
+        all.each do |model|
+          return model if model.trigger == trigger
+        end
+        raise Errors::Model::MissingTriggerError,
+            "Could not find trigger '#{trigger}'."
+      end
+
+      ##
+      # Find and return an Array of all models matching +trigger+
+      # Used to match triggers using a wildcard (*)
+      def find_matching(trigger)
+        regex = /^#{ trigger.to_s.gsub('*', '(.*)') }$/
+        all.select {|model| regex =~ model.trigger }
+      end
+    end
+
+    ##
+    # The trigger (stored as a String) is used as an identifier
+    # for initializing the backup process
+    attr_reader :trigger
+
+    ##
+    # The label (stored as a String) is used for a more friendly user output
+    attr_reader :label
+
+    ##
+    # The databases attribute holds an array of database objects
+    attr_reader :databases
+
+    ##
+    # The archives attr_accessor holds an array of archive objects
+    attr_reader :archives
+
+    ##
+    # The notifiers attr_accessor holds an array of notifier objects
+    attr_reader :notifiers
+
+    ##
+    # The storages attribute holds an array of storage objects
+    attr_reader :storages
+
+    ##
+    # The syncers attribute holds an array of syncer objects
+    attr_reader :syncers
+
+    ##
+    # Holds the configured Compressor
+    attr_reader :compressor
+
+    ##
+    # Holds the configured Encryptor
+    attr_reader :encryptor
+
+    ##
+    # Holds the configured Splitter
+    attr_reader :splitter
+
+    ##
+    # The final backup Package this model will create.
+    attr_reader :package
+
+    ##
+    # The time when the backup initiated (in format: 2011.02.20.03.29.59)
+    attr_reader :time
+
+    ##
+    # Hooks which can be run before or after the backup process
+    attr_reader :hooks
+
+    ##
+    # Takes a trigger, label and the configuration block.
+    # After the instance has evaluated the configuration block
+    # to configure the model, it will be appended to Model.all
+    def initialize(trigger, label, &block)
+      @trigger = trigger.to_s
+      @label   = label.to_s
+
+      # default noop hooks
+      @hooks   = Hooks.new(self)
+
+      procedure_instance_variables.each do |variable|
+        instance_variable_set(variable, Array.new)
+      end
+
+      instance_eval(&block) if block_given?
+      Model.all << self
+    end
+
+    ##
+    # Adds an archive to the array of archives
+    # to store during the backup process
+    def archive(name, &block)
+      @archives << Archive.new(self, name, &block)
+    end
+
+    ##
+    # Adds a database to the array of databases
+    # to dump during the backup process
+    def database(name, &block)
+      @databases << get_class_from_scope(Database, name).new(self, &block)
+    end
+
+    ##
+    # Adds a storage method to the array of storage
+    # methods to use during the backup process
+    def store_with(name, storage_id = nil, &block)
+      @storages << get_class_from_scope(Storage, name).new(self, storage_id, &block)
+    end
+
+    ##
+    # Adds a syncer method to the array of syncer
+    # methods to use during the backup process
+    def sync_with(name, &block)
+      ##
+      # Warn user of DSL changes
+      case name.to_s
+      when 'Backup::Config::RSync'
+        Logger.warn Errors::ConfigError.new(<<-EOS)
+          Configuration Update Needed for Syncer::RSync
+          The RSync Syncer has been split into three separate modules:
+          RSync::Local, RSync::Push and RSync::Pull
+          Please update your configuration.
+          i.e. 'sync_with RSync' is now 'sync_with RSync::Push'
+        EOS
+        name = 'RSync::Push'
+      when /(Backup::Config::S3|Backup::Config::CloudFiles)/
+        syncer = $1.split('::')[2]
+        Logger.warn Errors::ConfigError.new(<<-EOS)
+          Configuration Update Needed for '#{ syncer }' Syncer.
+          This Syncer is now referenced as Cloud::#{ syncer }
+          i.e. 'sync_with #{ syncer }' is now 'sync_with Cloud::#{ syncer }'
+        EOS
+        name = "Cloud::#{ syncer }"
+      end
+      @syncers << get_class_from_scope(Syncer, name).new(&block)
+    end
+
+    ##
+    # Adds a notifier to the array of notifiers
+    # to use during the backup process
+    def notify_by(name, &block)
+      @notifiers << get_class_from_scope(Notifier, name).new(self, &block)
+    end
+
+    ##
+    # Adds an encryptor to use during the backup process
+    def encrypt_with(name, &block)
+      @encryptor = get_class_from_scope(Encryptor, name).new(&block)
+    end
+
+    ##
+    # Adds a compressor to use during the backup process
+    def compress_with(name, &block)
+      @compressor = get_class_from_scope(Compressor, name).new(&block)
+    end
+
+    ##
+    # Run a block of ruby code before the backup process
+    def before(&block)
+      @hooks.before &block
+    end
+
+    ##
+    # Run a block of ruby code after the backup process
+    def after(&block)
+      @hooks.after &block
+    end
+
+    ##
+    # Adds a method that allows the user to configure this backup model
+    # to use a Splitter, with the given +chunk_size+
+    # The +chunk_size+ (in megabytes) will later determine
+    # in how many chunks the backup needs to be split into
+    def split_into_chunks_of(chunk_size)
+      if chunk_size.is_a?(Integer)
+        @splitter = Splitter.new(self, chunk_size)
+      else
+        raise Errors::Model::ConfigurationError, <<-EOS
+          Invalid Chunk Size for Splitter
+          Argument to #split_into_chunks_of() must be an Integer
+        EOS
+      end
+    end
+
+    ##
+    # Ensure DATA_PATH and DATA_PATH/TRIGGER are created
+    # if they do not yet exist
+    #
+    # Clean any temporary files and/or package files left over
+    # from the last time this model/trigger was performed.
+    # Logs warnings if files exist and are cleaned.
+    def prepare!
+      FileUtils.mkdir_p(File.join(Config.data_path, trigger))
+      Cleaner.prepare(self)
+    end
+
+    ##
+    # Performs the backup process
+    ##
+    # [Databases]
+    # Runs all (if any) database objects to dump the databases
+    ##
+    # [Archives]
+    # Runs all (if any) archive objects to package all their
+    # paths in to a single tar file and places it in the backup folder
+    ##
+    # [Packaging]
+    # After all the database dumps and archives are placed inside
+    # the folder, it'll make a single .tar package (archive) out of it
+    ##
+    # [Encryption]
+    # Optionally encrypts the packaged file with the configured encryptor
+    ##
+    # [Compression]
+    # Optionally compresses the each Archive and Database dump with the configured compressor
+    ##
+    # [Splitting]
+    # Optionally splits the backup file in to multiple smaller chunks before transferring them
+    ##
+    # [Storages]
+    # Runs all (if any) storage objects to store the backups to remote locations
+    # and (if configured) it'll cycle the files on the remote location to limit the
+    # amount of backups stored on each individual location
+    ##
+    # [Syncers]
+    # Runs all (if any) sync objects to store the backups to remote locations.
+    # A Syncer does not go through the process of packaging, compressing, encrypting backups.
+    # A Syncer directly transfers data from the filesystem to the remote location
+    ##
+    # [Notifiers]
+    # Runs all (if any) notifier objects when a backup proces finished with or without
+    # any errors.
+    ##
+    # [Cleaning]
+    # Once the final Packaging is complete, the temporary folder used will be removed.
+    # Then, once all Storages have run, the final packaged files will be removed.
+    # If any errors occur during the backup process, all temporary files will be left in place.
+    # If the error occurs before Packaging, then the temporary folder (tmp_path/trigger)
+    # will remain and may contain all or some of the configured Archives and/or Database dumps.
+    # If the error occurs after Packaging, but before the Storages complete, then the final
+    # packaged files (located in the root of tmp_path) will remain.
+    # *** Important *** If an error occurs and any of the above mentioned temporary files remain,
+    # those files *** will be removed *** before the next scheduled backup for the same trigger.
+    #
+    def perform!
+      @started_at = Time.now
+      @time = @started_at.strftime("%Y.%m.%d.%H.%M.%S")
+      log!(:started)
+
+      @hooks.perform!(:before)
+
+      if databases.any? or archives.any?
+        procedures.each do |procedure|
+          (procedure.call; next) if procedure.is_a?(Proc)
+          procedure.each(&:perform!)
+        end
+      end
+
+      syncers.each(&:perform!)
+      notifiers.each(&:perform!)
+
+      @hooks.perform!(:after)
+
+      log!(:finished)
+
+    rescue Exception => err
+      fatal = !err.is_a?(StandardError)
+
+      err = Errors::ModelError.wrap(err, <<-EOS)
+        Backup for #{label} (#{trigger}) Failed!
+        An Error occured which has caused this Backup to abort before completion.
+      EOS
+      Logger.error err
+      Logger.error "\nBacktrace:\n\s\s" + err.backtrace.join("\n\s\s") + "\n\n"
+
+      Cleaner.warnings(self)
+
+      if fatal
+        Logger.error Errors::ModelError.new(<<-EOS)
+          This Error was Fatal and Backup will now exit.
+          If you have other Backup jobs (triggers) configured to run,
+          they will not be processed.
+        EOS
+      else
+        Logger.message Errors::ModelError.new(<<-EOS)
+          If you have other Backup jobs (triggers) configured to run,
+          Backup will now attempt to continue...
+        EOS
+      end
+
+      notifiers.each do |n|
+        begin
+          n.perform!(true)
+        rescue Exception; end
+      end
+
+      exit(1) if fatal
+    end
+
+    private
+
+    ##
+    # After all the databases and archives have been dumped and sorted,
+    # these files will be bundled in to a .tar archive (uncompressed),
+    # which may be optionally Encrypted and/or Split into multiple "chunks".
+    # All information about this final archive is stored in the @package.
+    # Once complete, the temporary folder used during packaging is removed.
+    def package!
+      @package = Package.new(self)
+      Packager.package!(self)
+      Cleaner.remove_packaging(self)
+    end
+
+    ##
+    # Removes the final package file(s) once all configured Storages have run.
+    def clean!
+      Cleaner.remove_package(@package)
+    end
+
+    ##
+    # Returns an array of procedures
+    def procedures
+      [databases, archives, lambda { package! }, storages, lambda { clean! }]
+    end
+
+    ##
+    # Returns an Array of the names (String) of the procedure instance variables
+    def procedure_instance_variables
+      [:@databases, :@archives, :@storages, :@notifiers, :@syncers]
+    end
+
+    ##
+    # Returns the class/model specified by +name+ inside of +scope+.
+    # +scope+ should be a Class/Module.
+    # +name+ may be Class/Module or String representation
+    # of any namespace which exists under +scope+.
+    #
+    # The 'Backup::Config::' namespace is stripped from +name+,
+    # since this is the namespace where we define module namespaces
+    # for use with Model's DSL methods.
+    #
+    # Examples:
+    #   get_class_from_scope(Backup::Database, 'MySQL')
+    #     returns the class Backup::Database::MySQL
+    #
+    #   get_class_from_scope(Backup::Syncer, Backup::Config::RSync::Local)
+    #     returns the class Backup::Syncer::RSync::Local
+    #
+    def get_class_from_scope(scope, name)
+      klass = scope
+      name = name.to_s.sub(/^Backup::Config::/, '')
+      name.split('::').each do |chunk|
+        klass = klass.const_get(chunk)
+      end
+      klass
+    end
+
+    ##
+    # Logs messages when the backup starts and finishes
+    def log!(action)
+      case action
+      when :started
+        Logger.message "Performing Backup for '#{label} (#{trigger})'!\n" +
+            "[ backup #{ Version.current } : #{ RUBY_DESCRIPTION } ]"
+
+      when :finished
+        msg = "Backup for '#{ label } (#{ trigger })' " +
+              "Completed %s in #{ elapsed_time }"
+        if Logger.has_warnings?
+          Logger.warn msg % 'Successfully (with Warnings)'
+        else
+          Logger.message msg % 'Successfully'
+        end
+      end
+    end
+
+    ##
+    # Returns a string representing the elapsed time since the backup started.
+    def elapsed_time
+      duration  = Time.now.to_i - @started_at.to_i
+      hours     = duration / 3600
+      remainder = duration - (hours * 3600)
+      minutes   = remainder / 60
+      seconds   = remainder - (minutes * 60)
+      '%02d:%02d:%02d' % [hours, minutes, seconds]
+    end
+
+  end
+end