backup 3.0.19 → 3.0.20

data/lib/backup/packager.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+
+module Backup
+  class Packager
+    include Backup::CLI::Helpers
+
+    ##
+    # Holds an instance of the current Backup model
+    attr_accessor :model
+
+    ##
+    # Creates a new instance of the Backup::Packager class
+    def initialize(model)
+      @model = model
+    end
+
+    ##
+    # Packages the current state of the backup in to a single archived file.
+    def package!
+      Logger.message "#{ self.class } started packaging the backup files."
+      run("#{ utility(:tar) } -c -f '#{ File.join(Backup::TMP_PATH, "#{ Backup::TIME }.#{ Backup::TRIGGER }.tar") }' -C '#{ Backup::TMP_PATH }' '#{ Backup::TRIGGER }'")
+    end
+
+  end
+end

data/lib/backup/splitter.rb

@@ -0,0 +1,62 @@
+# encoding: utf-8
+
+module Backup
+  class Splitter
+    include Backup::CLI::Helpers
+
+    ##
+    # Separates the end of the file from the chunk extension name
+    SUFFIX_SEPARATOR = "-"
+
+    ##
+    # Holds an instance of the current Backup model
+    attr_accessor :model
+
+    ##
+    # Instantiates a new instance of Backup::Splitter and takes
+    # a Backup model as an argument.
+    # Also, (re)set the Backup::Model.chunk_suffixes to an empty array.
+    def initialize(model)
+      @model = model
+      Backup::Model.chunk_suffixes = Array.new
+    end
+
+    ##
+    # Splits the file in multiple chunks if necessary, and it's necessary
+    # when the requested chunk size is smaller than the actual backup file
+    def split!
+      return unless model.chunk_size.is_a?(Integer)
+
+      if File.size(model.file) > bytes_representation_of(model.chunk_size)
+        Logger.message "#{ self.class } started splitting the packaged archive in to chunks of #{ model.chunk_size } megabytes."
+        run("#{ utility(:split) } -b #{ model.chunk_size }m '#{ model.file }' '#{ model.file + SUFFIX_SEPARATOR }'")
+        Backup::Model.chunk_suffixes = chunk_suffixes
+      end
+    end
+
+  private
+
+    ##
+    # Returns an array of suffixes for each chunk.
+    # For example: [aa, ab, ac, ad, ae] - Chunk suffixes are sorted on alphabetical order
+    def chunk_suffixes
+      chunks.map do |chunk|
+        File.extname(chunk).split("-").last
+      end.sort
+    end
+
+    ##
+    # Returns an array of full paths to the backup chunks.
+    # Chunks aresorted on alphabetical order
+    def chunks
+      Dir["#{model.file}-*"].sort
+    end
+
+    ##
+    # Converts the provided megabytes to a bytes representation
+    def bytes_representation_of(megabytes)
+      megabytes * 1024 * 1024
+    end
+
+  end
+end

data/lib/backup/storage/base.rb

@@ -11,9 +11,65 @@ module Backup
 
       ##
       # Sets the limit to how many backups to keep in the remote location.
-      # If the limit exceeds it will remove the oldest backup to make room for the newest
+      # If exceeded, the oldest will be removed to make room for the newest
       attr_accessor :keep
 
+      # Temporarily holds the configuration block used to instantiate the
+      # storage object. Used for updating storage objects loaded from YAML
+      # during backup rotation in #cycle!
+      attr_accessor :configure_block
+
+      ##
+      # Contains an array of chunk suffixes (if any)
+      # If none are set, this will be an empty array, in which case Backup assumes
+      # we haven't been splitting the backup in to multiple chunks. The storage object
+      # will only attempt to transfer/remove chunks if this array contains chunk suffixes.
+      attr_accessor :chunk_suffixes
+
+      ##
+      # Super method for the child classes' perform! method. "super" should
+      # always be invoked from the child classes' perform! method to ensure that the
+      # @chunk_suffixes array gets set to the storage object, which will be used to transfer all the
+      # chunks to the remote location, rather than the single backup file. Also, this will be persisted
+      # and loaded back in during the cycling process, so it gets properly deleted from the remote location.
+
+      ##
+      # (Optional)
+      # User-defined string used to uniquely identify multiple storages of the same type.
+      # This will be appended to the YAML storage file used for cycling backups.
+      attr_accessor :storage_id
+
+      ##
+      # Set to Backup::Version.current just before the object is stored
+      # in the YAML file for cycling. This way, we know when the object
+      # is loaded from the YAML file, which version of Backup stored it.
+      attr_reader :version
+
+      def perform!
+        @chunk_suffixes ||= Backup::Model.chunk_suffixes
+      end
+
+      ##
+      # Creates a new instance of the storage object
+      def initialize(storage_id = nil, &block)
+        @configure_block = block
+        @storage_id = storage_id
+        configure!
+      end
+
+      ##
+      # Return the storage name, with optional storage_id
+      def storage_name
+        self.class.to_s.sub('Backup::', '') +
+            (storage_id ? " (#{storage_id})" : '')
+      end
+
+      ##
+      # Returns the full filename of the processed backup file
+      def filename
+        @filename ||= File.basename(Backup::Model.file)
+      end
+
       ##
       # Returns the local path
       def local_path
@@ -21,44 +77,148 @@ module Backup
       end
 
       ##
-      # Returns the local archive filename
-      def local_file
-        @local_file ||= File.basename(Backup::Model.file)
+      # Returns an array of backup chunks
+      def chunks
+        chunk_suffixes.map do |chunk_suffix|
+          "#{ filename }-#{ chunk_suffix }"
+        end.sort
+      end
+
+      ##
+      # Returns a block with two arguments: "local_file, remote_file"
+      # The local_file is the full file name: "2011.08.30.11.00.02.backup.tar.gz.enc"
+      # The remote_file is the full file name, minus the timestamp: "backup.tar.gz.enc"
+      def files_to_transfer
+        if chunks?
+          chunks.each do |chunk|
+            yield chunk, chunk[20..-1]
+          end
+        else
+          yield filename, filename[20..-1]
+        end
       end
 
+      alias :transferred_files :files_to_transfer
+
       ##
-      # Returns the name of the file that's stored on the remote location
-      def remote_file
-        @remote_file ||= local_file
+      # Returns true if we're working with chunks
+      # that were splitted by Backup
+      def chunks?
+        chunk_suffixes.is_a?(Array) and chunk_suffixes.count > 0
       end
 
       ##
-      # Provider defaults to false and will be overridden when using
-      # a service-based storage such as Amazon S3, Rackspace Cloud Files or Dropbox
+      # Provider defaults to false. Overridden when using a service-based
+      # storage such as Amazon S3, Rackspace Cloud Files or Dropbox
       def provider
         false
       end
 
+    private
+
+      ##
+      # Configure the storage object, using optional configuration block
+      # Uses #pre_configure to set defaults (if any exist) and then evaluates
+      # the optional configuration block which may overwrite these defaults.
+      # Then uses #post_configure to adjust the configuration as needed.
+      #
+      # This method is also used to update storage objects loaded from the
+      # YAML data storage file used for backup rotation in #cycle!
+      def configure!
+        pre_configure
+        instance_eval(&@configure_block) if @configure_block
+        post_configure
+        self
+      end
+
+      ##
+      # Set configuration defaults before evaluating configuration block.
+      # Each subclass may perform additional actions after calling super()
+      def pre_configure
+        load_defaults!
+      end
+
+      ##
+      # Adjust configuration after evaluating configuration block.
+      # Each subclass may perform additional actions after calling super()
+      def post_configure
+        @time ||= TIME
+      end
+
+      ##
+      # Update the configuration with the given +configure_block+
+      # This is to update the configuration for this storage object
+      # once it's been loaded from the YAML storage file (within #cycle!)
+      # so that cycling operations can be performed using the latest
+      # configuration from the current backup job.
+      def update!(configure_block)
+        upgrade_if_needed!
+        instance_exec(configure_block) do |block|
+          @configure_block = block; configure!
+        end
+      end
+
+      ##
+      # Upgrades the format of an object loaded from the YAML storage file
+      # if it was stored with a previous, incompatible version of Backup,
+      # before it is updated using the new configure_block in #update!
+      def upgrade_if_needed!
+        return if version == Backup::Version.current
+        case
+        when version.nil? # <= 3.0.19
+          @filename = @remote_file
+          @chunk_suffixes = []
+          clean!
+        else; # upgrade not required
+        end
+      end
+
+      ##
+      # Clear all attributes except those which need to be stored
+      # in the YAML storage file for cycling, and version stamp it.
+      def clean!
+        stored_attrs = [:@filename, :@time, :@chunk_suffixes]
+        (instance_variables.map(&:to_sym) - stored_attrs).each do |var|
+          remove_instance_variable var
+        end
+        @version = Backup::Version.current
+      end
+
       ##
       # Checks the persisted storage data by type (S3, CloudFiles, SCP, etc)
       # to see if the amount of stored backups is greater than the amount of
       # backups allowed. If this is the case it'll invoke the #remove! method
-      # on each of the oldest backups that exceed the storage limit (specified by @keep).
-      # After that it'll re-assign the objects variable with an array of objects that still remain
-      # after the removal of the older objects and files (that exceeded the @keep range). And finally
-      # these remaining objects will be converted to YAML format and are written back to the YAML file
+      # on each of the oldest backups that exceed the storage limit (specified
+      # by @keep). After that it'll re-assign the objects variable with an
+      # array of objects that still remain after the removal of the older
+      # objects and files (that exceeded the @keep range). And finally these
+      # remaining objects will be converted to YAML format and are written back
+      # to the YAML file.
+      # Each remaining storage object's attributes will be updated using the
+      # defaults and configuration block defined for the current backup job
+      # in case the storage location is changed or credentials are updated.
       def cycle!
+        return unless keep.to_i > 0
         type           = self.class.name.split("::").last
-        storage_object = Backup::Storage::Object.new(type)
-        objects        = [self] + storage_object.load
-        if keep.is_a?(Integer) and keep > 0 and objects.count > keep
+        storage_object = Backup::Storage::Object.new(type, storage_id)
+        objects        = storage_object.load
+        objects.each {|object| object.send(:update!, @configure_block) }
+        objects.unshift(self)
+        if objects.count > keep
           objects_to_remove = objects[keep..-1]
           objects_to_remove.each do |object|
-            Logger.message "#{ self.class } started removing (cycling) \"#{ object.remote_file }\"."
-            object.send(:remove!)
+            Logger.message "#{storage_name} started removing (cycling) " +
+                "'#{ object.filename }'."
+            begin
+              object.send(:remove!)
+            rescue => err
+              Logger.warn Errors::Storage::CycleError.wrap(err,
+                  "#{storage_name} failed to remove '#{object.filename}'")
+            end
           end
           objects = objects - objects_to_remove
         end
+        objects.each {|object| object.send(:clean!) }
         storage_object.write(objects)
       end
 

data/lib/backup/storage/cloudfiles.rb

@@ -13,27 +13,17 @@ module Backup
       attr_accessor :username, :api_key, :auth_url
 
       ##
-      # Rackspace Cloud Files container name and path
-      attr_accessor :container, :path
+      # Rackspace Service Net (Allows for LAN-based transfers to avoid charges and improve performance)
+      attr_accessor :servicenet
 
       ##
-      # Creates a new instance of the Rackspace Cloud Files storage object
-      # First it sets the defaults (if any exist) and then evaluates
-      # the configuration block which may overwrite these defaults
-      def initialize(&block)
-        load_defaults!
-
-        @path ||= 'backups'
-
-        instance_eval(&block) if block_given?
-
-        @time = TIME
-      end
+      # Rackspace Cloud Files container name and path
+      attr_accessor :container, :path
 
       ##
       # This is the remote path to where the backup files will be stored
       def remote_path
-        File.join(path, TRIGGER)
+        File.join(path, TRIGGER, @time)
       end
 
       ##
@@ -45,12 +35,29 @@ module Backup
       ##
       # Performs the backup transfer
       def perform!
+        super
         transfer!
         cycle!
       end
 
     private
 
+      ##
+      # Set configuration defaults before evaluating configuration block,
+      # after setting defaults from Storage::Base
+      def pre_configure
+        super
+        @servicenet ||= false
+        @path       ||= 'backups'
+      end
+
+      ##
+      # Adjust configuration after evaluating configuration block,
+      # after adjustments from Storage::Base
+      def post_configure
+        super
+      end
+
       ##
       # Establishes a connection to Rackspace Cloud Files and returns the Fog object.
       # Not doing any instance variable caching because this object gets persisted in YAML
@@ -58,37 +65,36 @@ module Backup
       # gets invoked once per object for a #transfer! and once for a remove! Backups run in the
       # background anyway so even if it were a bit slower it shouldn't matter.
       def connection
-        Fog::Storage.new(
-          :provider           => provider,
-          :rackspace_username => username,
-          :rackspace_api_key  => api_key,
-          :rackspace_auth_url => auth_url
+        @connection ||= Fog::Storage.new(
+          :provider             => provider,
+          :rackspace_username   => username,
+          :rackspace_api_key    => api_key,
+          :rackspace_auth_url   => auth_url,
+          :rackspace_servicenet => servicenet
         )
       end
 
       ##
       # Transfers the archived file to the specified Cloud Files container
       def transfer!
-        begin
-          Logger.message("#{ self.class } started transferring \"#{ remote_file }\".")
+        files_to_transfer do |local_file, remote_file|
+          Logger.message "#{storage_name} started transferring '#{ local_file }'."
           connection.put_object(
             container,
             File.join(remote_path, remote_file),
             File.open(File.join(local_path, local_file))
           )
-        rescue Excon::Errors::SocketError => e
-          puts "\nAn error occurred while trying to transfer the backup."
-          puts "Make sure the container exists and try again.\n\n"
-          exit
         end
       end
 
       ##
       # Removes the transferred archive file from the Cloud Files container
       def remove!
-        begin
+        transferred_files do |local_file, remote_file|
+          Logger.message "#{storage_name} started removing '#{ local_file }'" +
+              "from container '#{ container }'"
           connection.delete_object(container, File.join(remote_path, remote_file))
-        rescue Excon::Errors::SocketError; end
+        end
       end
 
     end

data/lib/backup/storage/dropbox.rb

@@ -24,36 +24,38 @@ module Backup
       # Dropbox connection timeout
       attr_accessor :timeout
 
-      ##
-      # Creates a new instance of the Dropbox storage object
-      # First it sets the defaults (if any exist) and then evaluates
-      # the configuration block which may overwrite these defaults
-      def initialize(&block)
-        load_defaults!(:except => ['password', 'email'])
-
-        @path ||= 'backups'
-
-        instance_eval(&block) if block_given?
-
-        @timeout ||= 300
-        @time = TIME
-      end
-
       ##
       # This is the remote path to where the backup files will be stored
       def remote_path
-        File.join(path, TRIGGER)
+        File.join(path, TRIGGER, @time)
       end
 
       ##
       # Performs the backup transfer
       def perform!
+        super
         transfer!
         cycle!
       end
 
     private
 
+      ##
+      # Set configuration defaults before evaluating configuration block,
+      # after setting defaults from Storage::Base
+      def pre_configure
+        super
+        @path ||= 'backups'
+      end
+
+      ##
+      # Adjust configuration after evaluating configuration block,
+      # after adjustments from Storage::Base
+      def post_configure
+        super
+        @timeout ||= 300
+      end
+
       ##
       # The initial connection to Dropbox will provide the user with an authorization url.
       # The user must open this URL and confirm that the authorization successfully took place.
@@ -62,15 +64,20 @@ module Backup
       # in Backup::CACHE_PATH. The cached file will be used from that point on to re-establish a connection with
       # Dropbox at a later time. This allows the user to avoid having to go to a new Dropbox URL to authorize over and over again.
       def connection
+        return @connection if @connection
+
         if cache_exists?
           begin
             cached_session = ::Dropbox::Session.deserialize(File.read(cached_file))
             if cached_session.authorized?
               Logger.message "Session data loaded from cache!"
-              return cached_session
+              return @connection = cached_session
             end
-          rescue ArgumentError => error
-            Logger.warn "Could not read session data from cache. Cache data might be corrupt."
+          rescue ArgumentError => err
+            Logger.warn Errors::Storage::Dropbox::ConnectionError.wrap(err, <<-EOS)
+              Could not read session data from cache.
+              Cache data might be corrupt.
+            EOS
           end
         end
 
@@ -81,54 +88,37 @@ module Backup
       ##
       # Transfers the archived file to the specified Dropbox folder
       def transfer!
-        Logger.message("#{ self.class } started transferring \"#{ remote_file }\".")
-        connection.upload(File.join(local_path, local_file), remote_path, :timeout => timeout)
+        files_to_transfer do |local_file, remote_file|
+          Logger.message "#{storage_name} started transferring '#{ local_file }'."
+          connection.upload(
+            File.join(local_path, local_file),
+            remote_path,
+            :as      => remote_file,
+            :timeout => timeout
+          )
+        end
+
+        remove_instance_variable(:@connection) if instance_variable_defined?(:@connection)
       end
 
       ##
       # Removes the transferred archive file from the Dropbox folder
       def remove!
-        begin
-          connection.delete(File.join(remote_path, remote_file))
-        rescue ::Dropbox::FileNotFoundError
-          Logger.warn "File \"#{ File.join(remote_path, remote_file) }\" does not exist, skipping removal."
-        end
-      end
-
-      ##
-      # Create a new session, write a serialized version of it to the
-      # .cache directory, and return the session object
-      def create_write_and_return_new_session!
-        session      = ::Dropbox::Session.new(api_key, api_secret)
-        session.mode = :dropbox
-        Logger.message "Open the following URL in a browser to authorize a session for your Dropbox account:"
-        Logger.message ""
-        Logger.message "\s\s#{session.authorize_url}"
-        Logger.message ""
-        Logger.message "Once Dropbox says you're authorized, hit enter to proceed."
-        Timeout::timeout(180) { STDIN.gets }
-        begin
-          session.authorize
-        rescue OAuth::Unauthorized => error
-          Logger.error "Authorization failed!"
-          raise error
+        messages = []
+        transferred_files do |local_file, remote_file|
+          messages << "#{storage_name} started removing '#{ local_file }' from Dropbox."
         end
-        Logger.message "Authorized!"
-
-        Logger.message "Caching session data to file: #{cached_file}.."
-        write_cache!(session)
-        Logger.message "Cache data written! You will no longer need to manually authorize this Dropbox account via an URL on this machine."
-        Logger.message "Note: If you run Backup with this Dropbox account on other machines, you will need to either authorize them the same way,"
-        Logger.message "\s\sor simply copy over #{cached_file} to the cache directory"
-        Logger.message "\s\son your other machines to use this Dropbox account there as well."
+        Logger.message messages.join("\n")
 
-        session
+        connection.delete(remote_path)
+      ensure
+        remove_instance_variable(:@connection) if instance_variable_defined?(:@connection)
       end
 
       ##
       # Returns the path to the cached file
       def cached_file
-        File.join(Backup::CACHE_PATH, "#{api_key + api_secret}")
+        File.join(Backup::CACHE_PATH, api_key + api_secret)
       end
 
       ##
@@ -145,22 +135,29 @@ module Backup
         end
       end
 
-    public # DEPRECATED METHODS #############################################
+      ##
+      # Create a new session, write a serialized version of it to the
+      # .cache directory, and return the session object
+      def create_write_and_return_new_session!
+        session      = ::Dropbox::Session.new(api_key, api_secret)
+        session.mode = :dropbox
 
-      def email
-        Logger.warn "[DEPRECATED] Backup::Storage::Dropbox.email is deprecated and will be removed at some point."
-      end
+        template = Backup::Template.new({:session => session, :cached_file => cached_file})
+        template.render("storage/dropbox/authorization_url.erb")
 
-      def email=(value)
-        Logger.warn "[DEPRECATED] Backup::Storage::Dropbox.email= is deprecated and will be removed at some point."
-      end
+        begin
+          Timeout::timeout(180) { STDIN.gets }
+          session.authorize
+        rescue OAuth::Unauthorized => err
+          raise Errors::Storage::Dropbox::ConnectionError.
+              wrap(err, 'Authorization Failed!')
+        end
 
-      def password
-        Logger.warn "[DEPRECATED] Backup::Storage::Dropbox.password is deprecated and will be removed at some point."
-      end
+        template.render("storage/dropbox/authorized.erb")
+        write_cache!(session)
+        template.render("storage/dropbox/cache_file_written.erb")
 
-      def password=(value)
-        Logger.warn "[DEPRECATED] Backup::Storage::Dropbox.password= is deprecated and will be removed at some point."
+        session
       end
 
     end