backup 3.0.20 → 3.0.21

data/lib/backup/storage/cloudfiles.rb

@@ -13,7 +13,8 @@ module Backup
       attr_accessor :username, :api_key, :auth_url
 
       ##
-      # Rackspace Service Net (Allows for LAN-based transfers to avoid charges and improve performance)
+      # Rackspace Service Net
+      # (LAN-based transfers to avoid charges and improve performance)
       attr_accessor :servicenet
 
       ##
@@ -21,49 +22,26 @@ module Backup
       attr_accessor :container, :path
 
       ##
-      # This is the remote path to where the backup files will be stored
-      def remote_path
-        File.join(path, TRIGGER, @time)
-      end
-
-      ##
-      # This is the provider that Fog uses for the Cloud Files Storage
-      def provider
-        'Rackspace'
-      end
-
-      ##
-      # Performs the backup transfer
-      def perform!
-        super
-        transfer!
-        cycle!
-      end
+      # Creates a new instance of the storage object
+      def initialize(model, storage_id = nil, &block)
+        super(model, storage_id)
 
-    private
-
-      ##
-      # Set configuration defaults before evaluating configuration block,
-      # after setting defaults from Storage::Base
-      def pre_configure
-        super
         @servicenet ||= false
         @path       ||= 'backups'
+
+        instance_eval(&block) if block_given?
       end
 
+      private
+
       ##
-      # Adjust configuration after evaluating configuration block,
-      # after adjustments from Storage::Base
-      def post_configure
-        super
+      # This is the provider that Fog uses for the Cloud Files Storage
+      def provider
+        'Rackspace'
       end
 
       ##
-      # Establishes a connection to Rackspace Cloud Files and returns the Fog object.
-      # Not doing any instance variable caching because this object gets persisted in YAML
-      # format to a file and will issues. This, however has no impact on performance since it only
-      # gets invoked once per object for a #transfer! and once for a remove! Backups run in the
-      # background anyway so even if it were a bit slower it shouldn't matter.
+      # Establishes a connection to Rackspace Cloud Files
       def connection
         @connection ||= Fog::Storage.new(
           :provider             => provider,
@@ -77,22 +55,29 @@ module Backup
       ##
       # Transfers the archived file to the specified Cloud Files container
       def transfer!
-        files_to_transfer do |local_file, remote_file|
+        remote_path = remote_path_for(@package)
+
+        files_to_transfer_for(@package) do |local_file, remote_file|
           Logger.message "#{storage_name} started transferring '#{ local_file }'."
-          connection.put_object(
-            container,
-            File.join(remote_path, remote_file),
-            File.open(File.join(local_path, local_file))
-          )
+
+          File.open(File.join(local_path, local_file), 'r') do |file|
+            connection.put_object(
+              container, File.join(remote_path, remote_file), file
+            )
+          end
         end
       end
 
       ##
-      # Removes the transferred archive file from the Cloud Files container
-      def remove!
-        transferred_files do |local_file, remote_file|
-          Logger.message "#{storage_name} started removing '#{ local_file }'" +
-              "from container '#{ container }'"
+      # Removes the transferred archive file(s) from the storage location.
+      # Any error raised will be rescued during Cycling
+      # and a warning will be logged, containing the error message.
+      def remove!(package)
+        remote_path = remote_path_for(package)
+
+        transferred_files_for(package) do |local_file, remote_file|
+          Logger.message "#{storage_name} started removing '#{ local_file }' " +
+              "from container '#{ container }'."
           connection.delete_object(container, File.join(remote_path, remote_file))
         end
       end

data/lib/backup/storage/cycler.rb

@@ -0,0 +1,117 @@
+# encoding: utf-8
+
+module Backup
+  module Storage
+    module Cycler
+      class << self
+
+        ##
+        # Adds the given +package+ to the YAML storage file corresponding
+        # to the given +storage+ and Package#trigger (Model#trigger).
+        # Then, calls the +storage+ to remove the files for any older
+        # packages that were removed from the YAML storage file.
+        def cycle!(storage, package)
+          @storage, @package = storage, package
+          @storage_file = storage_file
+
+          update_storage_file!
+          remove_packages!
+        end
+
+        private
+
+        ##
+        # Updates the YAML data file according to the #keep setting
+        # for the storage and sets the @packages_to_remove
+        def update_storage_file!
+          packages = yaml_load.unshift(@package)
+          excess = packages.count - @storage.keep
+          @packages_to_remove = (excess > 0) ? packages.pop(excess) : []
+          yaml_save(packages)
+        end
+
+        ##
+        # Calls the @storage to remove any old packages
+        # which were cycled out of the storage file.
+        def remove_packages!
+          @packages_to_remove.each do |pkg|
+            begin
+              @storage.send(:remove!, pkg)
+            rescue => err
+              Logger.warn Errors::Storage::CyclerError.wrap(err, <<-EOS)
+                There was a problem removing the following package:
+                Trigger: #{pkg.trigger} :: Dated: #{pkg.time}
+                Package included the following #{ pkg.filenames.count } file(s):
+                #{ pkg.filenames.join("\n") }
+              EOS
+            end
+          end
+        end
+
+        ##
+        # Return full path to the YAML data file,
+        # based on the current values of @storage and @package
+        def storage_file
+          type     = @storage.class.to_s.split('::').last
+          suffix   = @storage.storage_id.to_s.strip.gsub(/[\W\s]/, '_')
+          filename = suffix.empty? ? type : "#{type}-#{suffix}"
+          File.join(Config.data_path, @package.trigger, "#{filename}.yml")
+        end
+
+        ##
+        # Load Package objects from YAML file.
+        # Returns an Array, sorted by @time descending.
+        # i.e. most recent is objects[0]
+        def yaml_load
+          packages = []
+          if File.exist?(@storage_file) && !File.zero?(@storage_file)
+            packages = check_upgrade(
+              YAML.load_file(@storage_file).sort do |a, b|
+                b.instance_variable_get(:@time) <=> a.instance_variable_get(:@time)
+              end
+            )
+          end
+          packages
+        end
+
+        ##
+        # Store the given package objects to the YAML data file.
+        def yaml_save(packages)
+          File.open(@storage_file, 'w') do |file|
+            file.write(packages.to_yaml)
+          end
+        end
+
+        ##
+        # Upgrade the objects loaded from the YAML file, if needed.
+        def check_upgrade(objects)
+          if objects.any? {|obj| obj.class.to_s =~ /Backup::Storage/ }
+            # Version <= 3.0.20
+            model = @storage.instance_variable_get(:@model)
+            v3_0_20 = objects.any? {|obj| obj.instance_variable_defined?(:@version) }
+            objects.map! do |obj|
+              if v3_0_20 # Version == 3.0.20
+                filename = obj.instance_variable_get(:@filename)[20..-1]
+                chunk_suffixes = obj.instance_variable_get(:@chunk_suffixes)
+              else # Version <= 3.0.19
+                filename = obj.instance_variable_get(:@remote_file)[20..-1]
+                chunk_suffixes = []
+              end
+              time = obj.instance_variable_get(:@time)
+              extension = filename.match(/\.(tar.*)$/)[1]
+
+              package = Backup::Package.new(model)
+              package.instance_variable_set(:@time, time)
+              package.extension = extension
+              package.chunk_suffixes = chunk_suffixes
+
+              package
+            end
+          end
+          objects
+        end
+
+      end
+    end
+  end
+end

data/lib/backup/storage/dropbox.rb

@@ -2,11 +2,7 @@
 
 ##
 # Only load the Dropbox gem when the Backup::Storage::Dropbox class is loaded
-Backup::Dependency.load('dropbox')
-
-##
-# Only load the timeout library when the Backup::Storage::Dropbox class is loaded
-require 'timeout'
+Backup::Dependency.load('dropbox-sdk')
 
 module Backup
   module Storage
@@ -17,108 +13,115 @@ module Backup
       attr_accessor :api_key, :api_secret
 
       ##
-      # Path to where the backups will be stored
-      attr_accessor :path
+      # Dropbox Access Type
+      # Valid values are:
+      #   :app_folder (default)
+      #   :dropbox (full access)
+      attr_accessor :access_type
 
       ##
-      # Dropbox connection timeout
-      attr_accessor :timeout
+      # Path to where the backups will be stored
+      attr_accessor :path
 
-      ##
-      # This is the remote path to where the backup files will be stored
-      def remote_path
-        File.join(path, TRIGGER, @time)
+      # Deprecated as of v3.0.21 - for move to official 'dropbox-sdk' gem (v1.1)
+      def timeout=(value)
+        if value
+          Logger.warn "[DEPRECATED] Backup::Storage::Dropbox.timeout=\n" +
+              "  is deprecated and will be removed at some point."
+        end
       end
 
       ##
-      # Performs the backup transfer
-      def perform!
-        super
-        transfer!
-        cycle!
-      end
+      # Creates a new instance of the storage object
+      def initialize(model, storage_id = nil, &block)
+        super(model, storage_id)
 
-    private
-
-      ##
-      # Set configuration defaults before evaluating configuration block,
-      # after setting defaults from Storage::Base
-      def pre_configure
-        super
         @path ||= 'backups'
-      end
+        @access_type ||= :app_folder
 
-      ##
-      # Adjust configuration after evaluating configuration block,
-      # after adjustments from Storage::Base
-      def post_configure
-        super
-        @timeout ||= 300
+        instance_eval(&block) if block_given?
       end
 
+      private
+
       ##
-      # The initial connection to Dropbox will provide the user with an authorization url.
-      # The user must open this URL and confirm that the authorization successfully took place.
-      # If this is the case, then the user hits 'enter' and the session will be properly established.
-      # Immediately after establishing the session, the session will be serialized and written to a cache file
-      # in Backup::CACHE_PATH. The cached file will be used from that point on to re-establish a connection with
-      # Dropbox at a later time. This allows the user to avoid having to go to a new Dropbox URL to authorize over and over again.
+      # The initial connection to Dropbox will provide the user with an
+      # authorization url. The user must open this URL and confirm that the
+      # authorization successfully took place. If this is the case, then the
+      # user hits 'enter' and the session will be properly established.
+      # Immediately after establishing the session, the session will be
+      # serialized and written to a cache file in Backup::Config.cache_path.
+      # The cached file will be used from that point on to re-establish a
+      # connection with Dropbox at a later time. This allows the user to avoid
+      # having to go to a new Dropbox URL to authorize over and over again.
       def connection
         return @connection if @connection
 
+        unless session = cached_session
+          Logger.message "Creating a new session!"
+          session = create_write_and_return_new_session!
+        end
+
+        # will raise an error if session not authorized
+        @connection = DropboxClient.new(session, access_type)
+
+      rescue => err
+        raise Errors::Storage::Dropbox::ConnectionError.wrap(err)
+      end
+
+      ##
+      # Attempt to load a cached session
+      def cached_session
+        session = false
         if cache_exists?
           begin
-            cached_session = ::Dropbox::Session.deserialize(File.read(cached_file))
-            if cached_session.authorized?
-              Logger.message "Session data loaded from cache!"
-              return @connection = cached_session
-            end
-          rescue ArgumentError => err
-            Logger.warn Errors::Storage::Dropbox::ConnectionError.wrap(err, <<-EOS)
+            session = DropboxSession.deserialize(File.read(cached_file))
+            Logger.message "Session data loaded from cache!"
+
+          rescue => err
+            Logger.warn Errors::Storage::Dropbox::CacheError.wrap(err, <<-EOS)
               Could not read session data from cache.
               Cache data might be corrupt.
             EOS
           end
         end
-
-        Logger.message "Creating a new session!"
-        create_write_and_return_new_session!
+        session
       end
 
       ##
       # Transfers the archived file to the specified Dropbox folder
       def transfer!
-        files_to_transfer do |local_file, remote_file|
+        remote_path = remote_path_for(@package)
+
+        files_to_transfer_for(@package) do |local_file, remote_file|
           Logger.message "#{storage_name} started transferring '#{ local_file }'."
-          connection.upload(
-            File.join(local_path, local_file),
-            remote_path,
-            :as      => remote_file,
-            :timeout => timeout
-          )
+          File.open(File.join(local_path, local_file), 'r') do |file|
+            connection.put_file(File.join(remote_path, remote_file), file)
+          end
         end
-
-        remove_instance_variable(:@connection) if instance_variable_defined?(:@connection)
       end
 
       ##
-      # Removes the transferred archive file from the Dropbox folder
-      def remove!
+      # Removes the transferred archive file(s) from the storage location.
+      # Any error raised will be rescued during Cycling
+      # and a warning will be logged, containing the error message.
+      def remove!(package)
+        remote_path = remote_path_for(package)
+
         messages = []
-        transferred_files do |local_file, remote_file|
-          messages << "#{storage_name} started removing '#{ local_file }' from Dropbox."
+        transferred_files_for(package) do |local_file, remote_file|
+          messages << "#{storage_name} started removing " +
+              "'#{ local_file }' from Dropbox."
         end
         Logger.message messages.join("\n")
 
-        connection.delete(remote_path)
-      ensure
-        remove_instance_variable(:@connection) if instance_variable_defined?(:@connection)
+        connection.file_delete(remote_path)
       end
 
       ##
       # Returns the path to the cached file
       def cached_file
-        File.join(Backup::CACHE_PATH, api_key + api_secret)
+        File.join(Config.cache_path, api_key + api_secret)
       end
 
       ##
@@ -139,25 +142,38 @@ module Backup
       # Create a new session, write a serialized version of it to the
       # .cache directory, and return the session object
       def create_write_and_return_new_session!
-        session      = ::Dropbox::Session.new(api_key, api_secret)
-        session.mode = :dropbox
+        require 'timeout'
+
+        session = DropboxSession.new(api_key, api_secret)
+
+        # grab the request token for session
+        session.get_request_token
 
-        template = Backup::Template.new({:session => session, :cached_file => cached_file})
+        template = Backup::Template.new(
+          {:session => session, :cached_file => cached_file}
+        )
         template.render("storage/dropbox/authorization_url.erb")
 
-        begin
-          Timeout::timeout(180) { STDIN.gets }
-          session.authorize
-        rescue OAuth::Unauthorized => err
-          raise Errors::Storage::Dropbox::ConnectionError.
-              wrap(err, 'Authorization Failed!')
-        end
+        # wait for user to hit 'return' to continue
+        Timeout::timeout(180) { STDIN.gets }
+
+        # this will raise an error if the user did not
+        # visit the authorization_url and grant access
+        #
+        # get the access token from the server
+        # this will be stored with the session in the cache file
+        session.get_access_token
 
         template.render("storage/dropbox/authorized.erb")
         write_cache!(session)
         template.render("storage/dropbox/cache_file_written.erb")
 
         session
+
+        rescue => err
+          raise Errors::Storage::Dropbox::AuthenticationError.wrap(
+            err, 'Could not create or authenticate a new session'
+          )
       end
 
     end