backup_checksum 3.0.23

data/lib/backup/splitter.rb

@@ -0,0 +1,75 @@
+# encoding: utf-8
+
+module Backup
+  class Splitter
+    include Backup::CLI::Helpers
+
+    def initialize(model, chunk_size)
+      @model = model
+      @chunk_size = chunk_size
+    end
+
+    ##
+    # This is called as part of the procedure used to build the final
+    # backup package file(s). It yields it's portion of the command line
+    # for this procedure, which will split the data being piped into it
+    # into multiple files, based on the @chunk_size.
+    # Once the packaging procedure is complete, it will return and
+    # @package.chunk_suffixes will be set based on the resulting files.
+    def split_with
+      before_packaging
+      yield @split_command
+      after_packaging
+    end
+
+    private
+
+    ##
+    # The `split` command reads from $stdin and will store it's output in
+    # multiple files, based on the @chunk_size. The files will be
+    # written using the given `prefix`, which is the full path to the
+    # final @package.basename, plus a '-' separator. This `prefix` will then
+    # be suffixed using 'aa', 'ab', and so on... for each file.
+    def before_packaging
+      @package = @model.package
+      Logger.message "Splitter configured with a chunk size of " +
+          "#{ @chunk_size }MB."
+
+      @split_command = "#{ utility(:split) } -b #{ @chunk_size }m - " +
+          "'#{ File.join(Config.tmp_path, @package.basename + '-') }'"
+    end
+
+    ##
+    # Finds the resulting files from the packaging procedure
+    # and stores an Array of suffixes used in @package.chunk_suffixes.
+    # If the @chunk_size was never reached and only one file
+    # was written, that file will be suffixed with '-aa'.
+    # In which case, it will simply remove the suffix from the filename.
+    def after_packaging
+      suffixes = chunk_suffixes
+      if suffixes == ['aa']
+        FileUtils.mv(
+          File.join(Config.tmp_path, @package.basename + '-aa'),
+          File.join(Config.tmp_path, @package.basename)
+        )
+      else
+        @package.chunk_suffixes = suffixes
+      end
+    end
+
+    ##
+    # Returns an array of suffixes for each chunk, in alphabetical order.
+    # For example: [aa, ab, ac, ad, ae]
+    def chunk_suffixes
+      chunks.map {|chunk| File.extname(chunk).split('-').last }.sort
+    end
+
+    ##
+    # Returns an array of full paths to the backup chunks.
+    # Chunks are sorted in alphabetical order.
+    def chunks
+      Dir[File.join(Config.tmp_path, @package.basename + '-*')].sort
+    end
+
+  end
+end

data/lib/backup/storage/base.rb

@@ -0,0 +1,119 @@
+# encoding: utf-8
+
+module Backup
+  module Storage
+    class Base
+      include Backup::Configuration::Helpers
+
+      ##
+      # Sets the limit to how many backups to keep in the remote location.
+      # If exceeded, the oldest will be removed to make room for the newest
+      attr_accessor :keep
+
+      ##
+      # (Optional)
+      # User-defined string used to uniquely identify multiple storages of the
+      # same type. This will be appended to the YAML storage file used for
+      # cycling backups.
+      attr_accessor :storage_id
+
+      ##
+      # Creates a new instance of the storage object
+      # * Called with super(model, storage_id) from each subclass
+      def initialize(model, storage_id = nil)
+        load_defaults!
+        @model = model
+        @storage_id = storage_id
+      end
+
+      ##
+      # Performs the backup transfer
+      def perform!
+        @package = @model.package
+        @checksum_creator = @model.checksum_creator
+        if @checksum_creator
+          @checksum_creator.process_checksum_file_before_transfer(file_names_as_hash, @package.checksum_name, local_path)
+        end
+        transfer!
+        cycle!
+      end
+
+      private
+
+      ##
+      # Provider defaults to false. Overridden when using a service-based
+      # storage such as Amazon S3, Rackspace Cloud Files or Dropbox
+      def provider
+        false
+      end
+
+      ##
+      # Each subclass must define a +path+ where remote files will be stored
+      def path; end
+
+      ##
+      # Return the storage name, with optional storage_id
+      def storage_name
+        self.class.to_s.sub('Backup::', '') +
+            (storage_id ? " (#{storage_id})" : '')
+      end
+
+      ##
+      # Returns the local path
+      # This is where any Package to be transferred is located.
+      def local_path
+        Config.tmp_path
+      end
+
+      ##
+      # Returns the remote path for the given Package
+      # This is where the Package will be stored, or was previously stored.
+      def remote_path_for(package)
+        File.join(path, package.trigger, package.time)
+      end
+
+      def file_names_as_hash(package = @package)
+        hash = {}
+        files_to_transfer_without_checksum package do |local, remote|
+           hash[local] = remote
+        end
+        hash
+      end
+
+      def files_to_transfer_without_checksum(package)
+        package.filenames.each do |filename|
+          yield filename, filename[20..-1]
+        end
+      end
+
+      ##
+      # Yields two arguments to the given block: "local_file, remote_file"
+      # The local_file is the full file name:
+      # e.g. "2011.08.30.11.00.02.backup.tar.enc"
+      # The remote_file is the full file name, minus the timestamp:
+      # e.g. "backup.tar.enc"
+      def files_to_transfer_for(package)
+        file_names_as_hash(package).each {|local, remote| yield local, remote }
+
+        if @model.checksum_creator
+          yield package.checksum_name, package.checksum_name[20..-1]
+        end
+      end
+      alias :transferred_files_for :files_to_transfer_for
+
+      ##
+      # Adds the current package being stored to the YAML cycle data file
+      # and will remove any old Package file(s) when the storage limit
+      # set by #keep is exceeded. Any errors raised while attempting to
+      # remove older packages will be rescued and a warning will be logged
+      # containing the original error message.
+      def cycle!
+        return unless keep.to_i > 0
+        Logger.message "#{ storage_name }: Cycling Started..."
+        Cycler.cycle!(self, @package)
+        Logger.message "#{ storage_name }: Cycling Complete!"
+      end
+
+    end
+  end
+end

data/lib/backup/storage/cloudfiles.rb

@@ -0,0 +1,87 @@
+# encoding: utf-8
+
+##
+# Only load the Fog gem when the Backup::Storage::CloudFiles class is loaded
+Backup::Dependency.load('fog')
+
+module Backup
+  module Storage
+    class CloudFiles < Base
+
+      ##
+      # Rackspace Cloud Files Credentials
+      attr_accessor :username, :api_key, :auth_url
+
+      ##
+      # Rackspace Service Net
+      # (LAN-based transfers to avoid charges and improve performance)
+      attr_accessor :servicenet
+
+      ##
+      # Rackspace Cloud Files container name and path
+      attr_accessor :container, :path
+
+      ##
+      # Creates a new instance of the storage object
+      def initialize(model, storage_id = nil, &block)
+        super(model, storage_id)
+
+        @servicenet ||= false
+        @path       ||= 'backups'
+
+        instance_eval(&block) if block_given?
+      end
+
+      private
+
+      ##
+      # This is the provider that Fog uses for the Cloud Files Storage
+      def provider
+        'Rackspace'
+      end
+
+      ##
+      # Establishes a connection to Rackspace Cloud Files
+      def connection
+        @connection ||= Fog::Storage.new(
+          :provider             => provider,
+          :rackspace_username   => username,
+          :rackspace_api_key    => api_key,
+          :rackspace_auth_url   => auth_url,
+          :rackspace_servicenet => servicenet
+        )
+      end
+
+      ##
+      # Transfers the archived file to the specified Cloud Files container
+      def transfer!
+        remote_path = remote_path_for(@package)
+
+        files_to_transfer_for(@package) do |local_file, remote_file|
+          Logger.message "#{storage_name} started transferring '#{ local_file }'."
+
+          File.open(File.join(local_path, local_file), 'r') do |file|
+            connection.put_object(
+              container, File.join(remote_path, remote_file), file
+            )
+          end
+        end
+      end
+
+      ##
+      # Removes the transferred archive file(s) from the storage location.
+      # Any error raised will be rescued during Cycling
+      # and a warning will be logged, containing the error message.
+      def remove!(package)
+        remote_path = remote_path_for(package)
+
+        transferred_files_for(package) do |local_file, remote_file|
+          Logger.message "#{storage_name} started removing '#{ local_file }' " +
+              "from container '#{ container }'."
+          connection.delete_object(container, File.join(remote_path, remote_file))
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/storage/cycler.rb

@@ -0,0 +1,117 @@
+# encoding: utf-8
+
+module Backup
+  module Storage
+    module Cycler
+      class << self
+
+        ##
+        # Adds the given +package+ to the YAML storage file corresponding
+        # to the given +storage+ and Package#trigger (Model#trigger).
+        # Then, calls the +storage+ to remove the files for any older
+        # packages that were removed from the YAML storage file.
+        def cycle!(storage, package)
+          @storage, @package = storage, package
+          @storage_file = storage_file
+
+          update_storage_file!
+          remove_packages!
+        end
+
+        private
+
+        ##
+        # Updates the YAML data file according to the #keep setting
+        # for the storage and sets the @packages_to_remove
+        def update_storage_file!
+          packages = yaml_load.unshift(@package)
+          excess = packages.count - @storage.keep
+          @packages_to_remove = (excess > 0) ? packages.pop(excess) : []
+          yaml_save(packages)
+        end
+
+        ##
+        # Calls the @storage to remove any old packages
+        # which were cycled out of the storage file.
+        def remove_packages!
+          @packages_to_remove.each do |pkg|
+            begin
+              @storage.send(:remove!, pkg)
+            rescue => err
+              Logger.warn Errors::Storage::CyclerError.wrap(err, <<-EOS)
+                There was a problem removing the following package:
+                Trigger: #{pkg.trigger} :: Dated: #{pkg.time}
+                Package included the following #{ pkg.filenames.count } file(s):
+                #{ pkg.filenames.join("\n") }
+              EOS
+            end
+          end
+        end
+
+        ##
+        # Return full path to the YAML data file,
+        # based on the current values of @storage and @package
+        def storage_file
+          type     = @storage.class.to_s.split('::').last
+          suffix   = @storage.storage_id.to_s.strip.gsub(/[\W\s]/, '_')
+          filename = suffix.empty? ? type : "#{type}-#{suffix}"
+          File.join(Config.data_path, @package.trigger, "#{filename}.yml")
+        end
+
+        ##
+        # Load Package objects from YAML file.
+        # Returns an Array, sorted by @time descending.
+        # i.e. most recent is objects[0]
+        def yaml_load
+          packages = []
+          if File.exist?(@storage_file) && !File.zero?(@storage_file)
+            packages = check_upgrade(
+              YAML.load_file(@storage_file).sort do |a, b|
+                b.instance_variable_get(:@time) <=> a.instance_variable_get(:@time)
+              end
+            )
+          end
+          packages
+        end
+
+        ##
+        # Store the given package objects to the YAML data file.
+        def yaml_save(packages)
+          File.open(@storage_file, 'w') do |file|
+            file.write(packages.to_yaml)
+          end
+        end
+
+        ##
+        # Upgrade the objects loaded from the YAML file, if needed.
+        def check_upgrade(objects)
+          if objects.any? {|obj| obj.class.to_s =~ /Backup::Storage/ }
+            # Version <= 3.0.20
+            model = @storage.instance_variable_get(:@model)
+            v3_0_20 = objects.any? {|obj| obj.instance_variable_defined?(:@version) }
+            objects.map! do |obj|
+              if v3_0_20 # Version == 3.0.20
+                filename = obj.instance_variable_get(:@filename)[20..-1]
+                chunk_suffixes = obj.instance_variable_get(:@chunk_suffixes)
+              else # Version <= 3.0.19
+                filename = obj.instance_variable_get(:@remote_file)[20..-1]
+                chunk_suffixes = []
+              end
+              time = obj.instance_variable_get(:@time)
+              extension = filename.match(/\.(tar.*)$/)[1]
+
+              package = Backup::Package.new(model)
+              package.instance_variable_set(:@time, time)
+              package.extension = extension
+              package.chunk_suffixes = chunk_suffixes
+
+              package
+            end
+          end
+          objects
+        end
+
+      end
+    end
+  end
+end

data/lib/backup/storage/dropbox.rb

@@ -0,0 +1,181 @@
+# encoding: utf-8
+
+##
+# Only load the Dropbox gem when the Backup::Storage::Dropbox class is loaded
+Backup::Dependency.load('dropbox-sdk')
+
+module Backup
+  module Storage
+    class Dropbox < Base
+
+      ##
+      # Dropbox API credentials
+      attr_accessor :api_key, :api_secret
+
+      ##
+      # Dropbox Access Type
+      # Valid values are:
+      #   :app_folder (default)
+      #   :dropbox (full access)
+      attr_accessor :access_type
+
+      ##
+      # Path to where the backups will be stored
+      attr_accessor :path
+
+      # Deprecated as of v3.0.21 - for move to official 'dropbox-sdk' gem (v1.1)
+      def timeout=(value)
+        if value
+          Logger.warn "[DEPRECATED] Backup::Storage::Dropbox.timeout=\n" +
+              "  is deprecated and will be removed at some point."
+        end
+      end
+
+      ##
+      # Creates a new instance of the storage object
+      def initialize(model, storage_id = nil, &block)
+        super(model, storage_id)
+
+        @path ||= 'backups'
+        @access_type ||= :app_folder
+
+        instance_eval(&block) if block_given?
+      end
+
+      private
+
+      ##
+      # The initial connection to Dropbox will provide the user with an
+      # authorization url. The user must open this URL and confirm that the
+      # authorization successfully took place. If this is the case, then the
+      # user hits 'enter' and the session will be properly established.
+      # Immediately after establishing the session, the session will be
+      # serialized and written to a cache file in Backup::Config.cache_path.
+      # The cached file will be used from that point on to re-establish a
+      # connection with Dropbox at a later time. This allows the user to avoid
+      # having to go to a new Dropbox URL to authorize over and over again.
+      def connection
+        return @connection if @connection
+
+        unless session = cached_session
+          Logger.message "Creating a new session!"
+          session = create_write_and_return_new_session!
+        end
+
+        # will raise an error if session not authorized
+        @connection = DropboxClient.new(session, access_type)
+
+      rescue => err
+        raise Errors::Storage::Dropbox::ConnectionError.wrap(err)
+      end
+
+      ##
+      # Attempt to load a cached session
+      def cached_session
+        session = false
+        if cache_exists?
+          begin
+            session = DropboxSession.deserialize(File.read(cached_file))
+            Logger.message "Session data loaded from cache!"
+
+          rescue => err
+            Logger.warn Errors::Storage::Dropbox::CacheError.wrap(err, <<-EOS)
+              Could not read session data from cache.
+              Cache data might be corrupt.
+            EOS
+          end
+        end
+        session
+      end
+
+      ##
+      # Transfers the archived file to the specified Dropbox folder
+      def transfer!
+        remote_path = remote_path_for(@package)
+
+        files_to_transfer_for(@package) do |local_file, remote_file|
+          Logger.message "#{storage_name} started transferring '#{ local_file }'."
+          File.open(File.join(local_path, local_file), 'r') do |file|
+            connection.put_file(File.join(remote_path, remote_file), file)
+          end
+        end
+      end
+
+      ##
+      # Removes the transferred archive file(s) from the storage location.
+      # Any error raised will be rescued during Cycling
+      # and a warning will be logged, containing the error message.
+      def remove!(package)
+        remote_path = remote_path_for(package)
+
+        messages = []
+        transferred_files_for(package) do |local_file, remote_file|
+          messages << "#{storage_name} started removing " +
+              "'#{ local_file }' from Dropbox."
+        end
+        Logger.message messages.join("\n")
+
+        connection.file_delete(remote_path)
+      end
+
+      ##
+      # Returns the path to the cached file
+      def cached_file
+        File.join(Config.cache_path, api_key + api_secret)
+      end
+
+      ##
+      # Checks to see if the cache file exists
+      def cache_exists?
+        File.exist?(cached_file)
+      end
+
+      ##
+      # Serializes and writes the Dropbox session to a cache file
+      def write_cache!(session)
+        File.open(cached_file, "w") do |cache_file|
+          cache_file.write(session.serialize)
+        end
+      end
+
+      ##
+      # Create a new session, write a serialized version of it to the
+      # .cache directory, and return the session object
+      def create_write_and_return_new_session!
+        require 'timeout'
+
+        session = DropboxSession.new(api_key, api_secret)
+
+        # grab the request token for session
+        session.get_request_token
+
+        template = Backup::Template.new(
+          {:session => session, :cached_file => cached_file}
+        )
+        template.render("storage/dropbox/authorization_url.erb")
+
+        # wait for user to hit 'return' to continue
+        Timeout::timeout(180) { STDIN.gets }
+
+        # this will raise an error if the user did not
+        # visit the authorization_url and grant access
+        #
+        # get the access token from the server
+        # this will be stored with the session in the cache file
+        session.get_access_token
+
+        template.render("storage/dropbox/authorized.erb")
+        write_cache!(session)
+        template.render("storage/dropbox/cache_file_written.erb")
+
+        session
+
+        rescue => err
+          raise Errors::Storage::Dropbox::AuthenticationError.wrap(
+            err, 'Could not create or authenticate a new session'
+          )
+      end
+
+    end
+  end
+end