backup 3.0.20 → 3.0.21

data/lib/backup/package.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+
+module Backup
+  class Package
+
+    ##
+    # The time when the backup initiated (in format: 2011.02.20.03.29.59)
+    attr_reader :time
+
+    ##
+    # The trigger which initiated the backup process
+    attr_reader :trigger
+
+    ##
+    # Extension for the final archive file(s)
+    attr_accessor :extension
+
+    ##
+    # Set by the Splitter if the final archive was "chunked"
+    attr_accessor :chunk_suffixes
+
+    ##
+    # The version of Backup used to create the package
+    attr_reader :version
+
+    def initialize(model)
+      @time = model.time
+      @trigger = model.trigger
+      @extension = 'tar'
+      @chunk_suffixes = Array.new
+      @version = Backup::Version.current
+    end
+
+    def filenames
+      if chunk_suffixes.empty?
+        [basename]
+      else
+        chunk_suffixes.map {|suffix| "#{ basename }-#{ suffix }" }
+      end
+    end
+
+    def basename
+      "#{ time }.#{ trigger }.#{ extension }"
+    end
+
+  end
+end

data/lib/backup/packager.rb

@@ -1,25 +1,90 @@
 # encoding: utf-8
 
 module Backup
-  class Packager
-    include Backup::CLI::Helpers
+  module Packager
+    class << self
+      include Backup::CLI::Helpers
 
-    ##
-    # Holds an instance of the current Backup model
-    attr_accessor :model
+      ##
+      # Build the final package for the backup model.
+      def package!(model)
+        @package   = model.package
+        @encryptor = model.encryptor
+        @splitter  = model.splitter
 
-    ##
-    # Creates a new instance of the Backup::Packager class
-    def initialize(model)
-      @model = model
-    end
+        Logger.message "Packaging the backup files..."
+        procedure.call
+        Logger.message "Packaging Complete!"
+      end
 
-    ##
-    # Packages the current state of the backup in to a single archived file.
-    def package!
-      Logger.message "#{ self.class } started packaging the backup files."
-      run("#{ utility(:tar) } -c -f '#{ File.join(Backup::TMP_PATH, "#{ Backup::TIME }.#{ Backup::TRIGGER }.tar") }' -C '#{ Backup::TMP_PATH }' '#{ Backup::TRIGGER }'")
-    end
+      private
+
+      ##
+      # Builds a chain of nested Procs which assemble and execute
+      # the final command to package the backup.
+      # This is done so that the Encryptor and Splitter have the ability
+      # to perform actions before and after the final command is executed.
+      # No Encryptors currently utilize this, however the Splitter does.
+      def procedure
+        stack = []
+
+        ##
+        # Initial `tar` command to package the temporary backup folder.
+        # The command's output will then be either piped to the Encryptor
+        # or the Splitter (if no Encryptor), or redirected into the final
+        # output file if neither are configured.
+        @package_command = "#{ utility(:tar) } -cf - " +
+            "-C '#{ Config.tmp_path }' '#{ @package.trigger }'"
+
+        ##
+        # If an Encryptor was configured, it will be called first
+        # to amend the command to be piped through the encryption utility.
+        # It's output will then be either piped into a Splitter, or sent
+        # directly to the final output file.
+        if @encryptor
+          stack << lambda do
+            @encryptor.encrypt_with do |command, ext|
+              @package_command << " | #{command}"
+              @package.extension << ext
+              stack.shift.call
+            end
+          end
+        end
 
+        ##
+        # If a Splitter was configured, the command will be piped through
+        # the `split` command. Once the Proc executing the final command
+        # has completed and returns back to the Splitter, it will check the
+        # final output files to determine if the backup was indeed split.
+        # If so, it will set the package's chunk_suffixes. If not, it will
+        # remove the '-aa' suffix from the only file created by `split`.
+        #
+        # If no Splitter was configured, the command output will be
+        # redirected directly into the final output file.
+        if @splitter
+          stack << lambda do
+            @splitter.split_with do |command|
+              @package_command << " | #{command}"
+              stack.shift.call
+            end
+          end
+        else
+          stack << lambda do
+            outfile = File.join(Config.tmp_path, @package.basename)
+            @package_command << " > #{ outfile }"
+            stack.shift.call
+          end
+        end
+
+        ##
+        # Last Proc to be called runs the command the procedure built.
+        # Once complete, the call stack will unwind back through the
+        # preceeding Procs in the stack (if any)
+        stack << lambda { run(@package_command) }
+
+        stack.shift
+      end
+
+    end
   end
 end

data/lib/backup/splitter.rb

@@ -4,58 +4,71 @@ module Backup
   class Splitter
     include Backup::CLI::Helpers
 
-    ##
-    # Separates the end of the file from the chunk extension name
-    SUFFIX_SEPARATOR = "-"
+    def initialize(model, chunk_size)
+      @model = model
+      @chunk_size = chunk_size
+    end
 
     ##
-    # Holds an instance of the current Backup model
-    attr_accessor :model
+    # This is called as part of the procedure used to build the final
+    # backup package file(s). It yields it's portion of the command line
+    # for this procedure, which will split the data being piped into it
+    # into multiple files, based on the @chunk_size.
+    # Once the packaging procedure is complete, it will return and
+    # @package.chunk_suffixes will be set based on the resulting files.
+    def split_with
+      before_packaging
+      yield @split_command
+      after_packaging
+    end
+
+    private
 
     ##
-    # Instantiates a new instance of Backup::Splitter and takes
-    # a Backup model as an argument.
-    # Also, (re)set the Backup::Model.chunk_suffixes to an empty array.
-    def initialize(model)
-      @model = model
-      Backup::Model.chunk_suffixes = Array.new
+    # The `split` command reads from $stdin and will store it's output in
+    # multiple files, based on the @chunk_size. The files will be
+    # written using the given `prefix`, which is the full path to the
+    # final @package.basename, plus a '-' separator. This `prefix` will then
+    # be suffixed using 'aa', 'ab', and so on... for each file.
+    def before_packaging
+      @package = @model.package
+      Logger.message "Splitter configured with a chunk size of " +
+          "#{ @chunk_size }MB."
+
+      @split_command = "#{ utility(:split) } -b #{ @chunk_size }m - " +
+          "'#{ File.join(Config.tmp_path, @package.basename + '-') }'"
     end
 
     ##
-    # Splits the file in multiple chunks if necessary, and it's necessary
-    # when the requested chunk size is smaller than the actual backup file
-    def split!
-      return unless model.chunk_size.is_a?(Integer)
-
-      if File.size(model.file) > bytes_representation_of(model.chunk_size)
-        Logger.message "#{ self.class } started splitting the packaged archive in to chunks of #{ model.chunk_size } megabytes."
-        run("#{ utility(:split) } -b #{ model.chunk_size }m '#{ model.file }' '#{ model.file + SUFFIX_SEPARATOR }'")
-        Backup::Model.chunk_suffixes = chunk_suffixes
+    # Finds the resulting files from the packaging procedure
+    # and stores an Array of suffixes used in @package.chunk_suffixes.
+    # If the @chunk_size was never reached and only one file
+    # was written, that file will be suffixed with '-aa'.
+    # In which case, it will simply remove the suffix from the filename.
+    def after_packaging
+      suffixes = chunk_suffixes
+      if suffixes == ['aa']
+        FileUtils.mv(
+          File.join(Config.tmp_path, @package.basename + '-aa'),
+          File.join(Config.tmp_path, @package.basename)
+        )
+      else
+        @package.chunk_suffixes = suffixes
       end
     end
 
-  private
-
     ##
-    # Returns an array of suffixes for each chunk.
-    # For example: [aa, ab, ac, ad, ae] - Chunk suffixes are sorted on alphabetical order
+    # Returns an array of suffixes for each chunk, in alphabetical order.
+    # For example: [aa, ab, ac, ad, ae]
     def chunk_suffixes
-      chunks.map do |chunk|
-        File.extname(chunk).split("-").last
-      end.sort
+      chunks.map {|chunk| File.extname(chunk).split('-').last }.sort
     end
 
     ##
     # Returns an array of full paths to the backup chunks.
-    # Chunks aresorted on alphabetical order
+    # Chunks are sorted in alphabetical order.
     def chunks
-      Dir["#{model.file}-*"].sort
-    end
-
-    ##
-    # Converts the provided megabytes to a bytes representation
-    def bytes_representation_of(megabytes)
-      megabytes * 1024 * 1024
+      Dir[File.join(Config.tmp_path, @package.basename + '-*')].sort
     end
 
   end

data/lib/backup/storage/base.rb

@@ -5,107 +5,36 @@ module Backup
     class Base
       include Backup::Configuration::Helpers
 
-      ##
-      # The time when the backup initiated (in format: 2011.02.20.03.29.59)
-      attr_accessor :time
-
       ##
       # Sets the limit to how many backups to keep in the remote location.
       # If exceeded, the oldest will be removed to make room for the newest
       attr_accessor :keep
 
-      # Temporarily holds the configuration block used to instantiate the
-      # storage object. Used for updating storage objects loaded from YAML
-      # during backup rotation in #cycle!
-      attr_accessor :configure_block
-
-      ##
-      # Contains an array of chunk suffixes (if any)
-      # If none are set, this will be an empty array, in which case Backup assumes
-      # we haven't been splitting the backup in to multiple chunks. The storage object
-      # will only attempt to transfer/remove chunks if this array contains chunk suffixes.
-      attr_accessor :chunk_suffixes
-
-      ##
-      # Super method for the child classes' perform! method. "super" should
-      # always be invoked from the child classes' perform! method to ensure that the
-      # @chunk_suffixes array gets set to the storage object, which will be used to transfer all the
-      # chunks to the remote location, rather than the single backup file. Also, this will be persisted
-      # and loaded back in during the cycling process, so it gets properly deleted from the remote location.
-
       ##
       # (Optional)
-      # User-defined string used to uniquely identify multiple storages of the same type.
-      # This will be appended to the YAML storage file used for cycling backups.
+      # User-defined string used to uniquely identify multiple storages of the
+      # same type. This will be appended to the YAML storage file used for
+      # cycling backups.
       attr_accessor :storage_id
 
-      ##
-      # Set to Backup::Version.current just before the object is stored
-      # in the YAML file for cycling. This way, we know when the object
-      # is loaded from the YAML file, which version of Backup stored it.
-      attr_reader :version
-
-      def perform!
-        @chunk_suffixes ||= Backup::Model.chunk_suffixes
-      end
-
       ##
       # Creates a new instance of the storage object
-      def initialize(storage_id = nil, &block)
-        @configure_block = block
+      # * Called with super(model, storage_id) from each subclass
+      def initialize(model, storage_id = nil)
+        load_defaults!
+        @model = model
         @storage_id = storage_id
-        configure!
       end
 
       ##
-      # Return the storage name, with optional storage_id
-      def storage_name
-        self.class.to_s.sub('Backup::', '') +
-            (storage_id ? " (#{storage_id})" : '')
-      end
-
-      ##
-      # Returns the full filename of the processed backup file
-      def filename
-        @filename ||= File.basename(Backup::Model.file)
-      end
-
-      ##
-      # Returns the local path
-      def local_path
-        TMP_PATH
-      end
-
-      ##
-      # Returns an array of backup chunks
-      def chunks
-        chunk_suffixes.map do |chunk_suffix|
-          "#{ filename }-#{ chunk_suffix }"
-        end.sort
-      end
-
-      ##
-      # Returns a block with two arguments: "local_file, remote_file"
-      # The local_file is the full file name: "2011.08.30.11.00.02.backup.tar.gz.enc"
-      # The remote_file is the full file name, minus the timestamp: "backup.tar.gz.enc"
-      def files_to_transfer
-        if chunks?
-          chunks.each do |chunk|
-            yield chunk, chunk[20..-1]
-          end
-        else
-          yield filename, filename[20..-1]
-        end
+      # Performs the backup transfer
+      def perform!
+        @package = @model.package
+        transfer!
+        cycle!
       end
 
-      alias :transferred_files :files_to_transfer
-
-      ##
-      # Returns true if we're working with chunks
-      # that were splitted by Backup
-      def chunks?
-        chunk_suffixes.is_a?(Array) and chunk_suffixes.count > 0
-      end
+      private
 
       ##
       # Provider defaults to false. Overridden when using a service-based
@@ -114,112 +43,55 @@ module Backup
         false
       end
 
-    private
-
-      ##
-      # Configure the storage object, using optional configuration block
-      # Uses #pre_configure to set defaults (if any exist) and then evaluates
-      # the optional configuration block which may overwrite these defaults.
-      # Then uses #post_configure to adjust the configuration as needed.
-      #
-      # This method is also used to update storage objects loaded from the
-      # YAML data storage file used for backup rotation in #cycle!
-      def configure!
-        pre_configure
-        instance_eval(&@configure_block) if @configure_block
-        post_configure
-        self
-      end
-
       ##
-      # Set configuration defaults before evaluating configuration block.
-      # Each subclass may perform additional actions after calling super()
-      def pre_configure
-        load_defaults!
-      end
+      # Each subclass must define a +path+ where remote files will be stored
+      def path; end
 
       ##
-      # Adjust configuration after evaluating configuration block.
-      # Each subclass may perform additional actions after calling super()
-      def post_configure
-        @time ||= TIME
+      # Return the storage name, with optional storage_id
+      def storage_name
+        self.class.to_s.sub('Backup::', '') +
+            (storage_id ? " (#{storage_id})" : '')
       end
 
       ##
-      # Update the configuration with the given +configure_block+
-      # This is to update the configuration for this storage object
-      # once it's been loaded from the YAML storage file (within #cycle!)
-      # so that cycling operations can be performed using the latest
-      # configuration from the current backup job.
-      def update!(configure_block)
-        upgrade_if_needed!
-        instance_exec(configure_block) do |block|
-          @configure_block = block; configure!
-        end
+      # Returns the local path
+      # This is where any Package to be transferred is located.
+      def local_path
+        Config.tmp_path
       end
 
       ##
-      # Upgrades the format of an object loaded from the YAML storage file
-      # if it was stored with a previous, incompatible version of Backup,
-      # before it is updated using the new configure_block in #update!
-      def upgrade_if_needed!
-        return if version == Backup::Version.current
-        case
-        when version.nil? # <= 3.0.19
-          @filename = @remote_file
-          @chunk_suffixes = []
-          clean!
-        else; # upgrade not required
-        end
+      # Returns the remote path for the given Package
+      # This is where the Package will be stored, or was previously stored.
+      def remote_path_for(package)
+        File.join(path, package.trigger, package.time)
       end
 
       ##
-      # Clear all attributes except those which need to be stored
-      # in the YAML storage file for cycling, and version stamp it.
-      def clean!
-        stored_attrs = [:@filename, :@time, :@chunk_suffixes]
-        (instance_variables.map(&:to_sym) - stored_attrs).each do |var|
-          remove_instance_variable var
+      # Yields two arguments to the given block: "local_file, remote_file"
+      # The local_file is the full file name:
+      # e.g. "2011.08.30.11.00.02.backup.tar.enc"
+      # The remote_file is the full file name, minus the timestamp:
+      # e.g. "backup.tar.enc"
+      def files_to_transfer_for(package)
+        package.filenames.each do |filename|
+          yield filename, filename[20..-1]
         end
-        @version = Backup::Version.current
       end
+      alias :transferred_files_for :files_to_transfer_for
 
       ##
-      # Checks the persisted storage data by type (S3, CloudFiles, SCP, etc)
-      # to see if the amount of stored backups is greater than the amount of
-      # backups allowed. If this is the case it'll invoke the #remove! method
-      # on each of the oldest backups that exceed the storage limit (specified
-      # by @keep). After that it'll re-assign the objects variable with an
-      # array of objects that still remain after the removal of the older
-      # objects and files (that exceeded the @keep range). And finally these
-      # remaining objects will be converted to YAML format and are written back
-      # to the YAML file.
-      # Each remaining storage object's attributes will be updated using the
-      # defaults and configuration block defined for the current backup job
-      # in case the storage location is changed or credentials are updated.
+      # Adds the current package being stored to the YAML cycle data file
+      # and will remove any old Package file(s) when the storage limit
+      # set by #keep is exceeded. Any errors raised while attempting to
+      # remove older packages will be rescued and a warning will be logged
+      # containing the original error message.
       def cycle!
         return unless keep.to_i > 0
-        type           = self.class.name.split("::").last
-        storage_object = Backup::Storage::Object.new(type, storage_id)
-        objects        = storage_object.load
-        objects.each {|object| object.send(:update!, @configure_block) }
-        objects.unshift(self)
-        if objects.count > keep
-          objects_to_remove = objects[keep..-1]
-          objects_to_remove.each do |object|
-            Logger.message "#{storage_name} started removing (cycling) " +
-                "'#{ object.filename }'."
-            begin
-              object.send(:remove!)
-            rescue => err
-              Logger.warn Errors::Storage::CycleError.wrap(err,
-                  "#{storage_name} failed to remove '#{object.filename}'")
-            end
-          end
-          objects = objects - objects_to_remove
-        end
-        objects.each {|object| object.send(:clean!) }
-        storage_object.write(objects)
+        Logger.message "#{ storage_name }: Cycling Started..."
+        Cycler.cycle!(self, @package)
+        Logger.message "#{ storage_name }: Cycling Complete!"
       end
 
     end