ext_backup 5.0.0.beta.2.1

data/lib/backup/storage/base.rb

@@ -0,0 +1,66 @@
+module Backup
+  module Storage
+    class Base
+      include Config::Helpers
+
+      ##
+      # Base path on the remote where backup package files will be stored.
+      attr_accessor :path
+
+      ##
+      # Number of backups to keep or time until which to keep.
+      #
+      # If an Integer is given it sets the limit to how many backups to keep in
+      # the remote location. If exceeded, the oldest will be removed to make
+      # room for the newest.
+      #
+      # If a Time object is given it will remove backups _older_ than the given
+      # date.
+      #
+      # @!attribute [rw] keep
+      #   @param [Integer|Time]
+      #   @return [Integer|Time]
+      attr_accessor :keep
+
+      attr_reader :model, :package, :storage_id
+
+      ##
+      # +storage_id+ is a user-defined string used to uniquely identify
+      # multiple storages of the same type. If multiple storages of the same
+      # type are added to a single backup model, this identifier must be set.
+      # This will be appended to the YAML storage file used for cycling backups.
+      def initialize(model, storage_id = nil, &block)
+        @model = model
+        @package = model.package
+        @storage_id = storage_id.to_s.gsub(/\W/, "_") if storage_id
+
+        load_defaults!
+        instance_eval(&block) if block_given?
+      end
+
+      def perform!
+        Logger.info "#{storage_name} Started..."
+        transfer!
+        if respond_to?(:cycle!, true) && (keep.to_i > 0 || keep.is_a?(Time))
+          cycle!
+        end
+        Logger.info "#{storage_name} Finished!"
+      end
+
+      private
+
+      ##
+      # Return the remote path for the current or given package.
+      def remote_path(pkg = package)
+        path.empty? ? File.join(pkg.trigger, pkg.time) :
+                      File.join(path, pkg.trigger, pkg.time)
+      end
+      alias :remote_path_for :remote_path
+
+      def storage_name
+        @storage_name ||= self.class.to_s.sub("Backup::", "") +
+          (storage_id ? " (#{storage_id})" : "")
+      end
+    end
+  end
+end

data/lib/backup/storage/cloud_files.rb

@@ -0,0 +1,156 @@
+require "backup/cloud_io/cloud_files"
+
+module Backup
+  module Storage
+    class CloudFiles < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Rackspace CloudFiles Credentials
+      attr_accessor :username, :api_key
+
+      ##
+      # Rackspace Auth URL (optional)
+      attr_accessor :auth_url
+
+      ##
+      # Rackspace Service Net
+      # (LAN-based transfers to avoid charges and improve performance)
+      attr_accessor :servicenet
+
+      ##
+      # Rackspace Region (optional)
+      attr_accessor :region
+
+      ##
+      # Rackspace Container Name
+      attr_accessor :container
+
+      ##
+      # Rackspace Container Name for SLO Segments
+      # Required if #segment_size is set. Must be different from #container.
+      attr_accessor :segments_container
+
+      ##
+      # SLO Segment size, specified in MiB.
+      #
+      # Each package file larger than +segment_size+
+      # will be uploaded as a Static Large Objects (SLO).
+      #
+      # Defaults to 0 for backward compatibility (pre v.3.7.0),
+      # since #segments_container would be required.
+      #
+      # Minimum: 1 (0 disables SLO support)
+      # Maximum: 5120 (5 GiB)
+      attr_accessor :segment_size
+
+      ##
+      # If set, all backup package files (including SLO segments) will be
+      # scheduled for automatic removal by the server.
+      #
+      # The `keep` option should not be used if this is set,
+      # unless you're transitioning from the `keep` option.
+      attr_accessor :days_to_keep
+
+      ##
+      # Number of times to retry failed operations.
+      #
+      # Default: 10
+      attr_accessor :max_retries
+
+      ##
+      # Time in seconds to pause before each retry.
+      #
+      # Default: 30
+      attr_accessor :retry_waitsec
+
+      ##
+      # Additional options to pass along to fog.
+      # e.g. Fog::Storage.new({ :provider => 'Rackspace' }.merge(fog_options))
+      attr_accessor :fog_options
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @servicenet         ||= false
+        @segment_size       ||= 0
+        @max_retries        ||= 10
+        @retry_waitsec      ||= 30
+
+        @path ||= "backups"
+        path.sub!(/^\//, "")
+
+        check_configuration
+      end
+
+      private
+
+      def cloud_io
+        @cloud_io ||= CloudIO::CloudFiles.new(
+          username: username,
+          api_key: api_key,
+          auth_url: auth_url,
+          region: region,
+          servicenet: servicenet,
+          container: container,
+          segments_container: segments_container,
+          segment_size: segment_size,
+          days_to_keep: days_to_keep,
+          max_retries: max_retries,
+          retry_waitsec: retry_waitsec,
+          fog_options: fog_options
+        )
+      end
+
+      def transfer!
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{container}/#{dest}'..."
+          cloud_io.upload(src, dest)
+        end
+
+        package.no_cycle = true if days_to_keep
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{package.time}..."
+
+        remote_path = remote_path_for(package)
+        objects = cloud_io.objects(remote_path)
+
+        raise Error, "Package at '#{remote_path}' not found" if objects.empty?
+
+        slo_objects, objects = objects.partition(&:slo?)
+        cloud_io.delete_slo(slo_objects)
+        cloud_io.delete(objects)
+      end
+
+      def check_configuration
+        required = %w[username api_key container]
+        raise Error, <<-EOS if required.map { |name| send(name) }.any?(&:nil?)
+          Configuration Error
+          #{required.map { |name| "##{name}" }.join(", ")} are all required
+        EOS
+
+        raise Error, <<-EOS if segment_size > 0 && segments_container.to_s.empty?
+          Configuration Error
+          #segments_container is required if #segment_size is > 0
+        EOS
+
+        raise Error, <<-EOS if container == segments_container
+          Configuration Error
+          #container and #segments_container must not be the same container.
+        EOS
+
+        raise Error, <<-EOS if segment_size > 5120
+          Configuration Error
+          #segment_size is too large (max 5120)
+        EOS
+      end
+    end
+  end
+end

data/lib/backup/storage/cycler.rb

@@ -0,0 +1,70 @@
+module Backup
+  module Storage
+    module Cycler
+      class Error < Backup::Error; end
+
+      private
+
+      # Adds the current package being stored to the YAML cycle data file
+      # and will remove any old package file(s) when the storage limit
+      # set by #keep is exceeded.
+      def cycle!
+        Logger.info "Cycling Started..."
+
+        packages = yaml_load.unshift(package)
+        cycled_packages = []
+
+        if keep.is_a?(Date) || keep.is_a?(Time)
+          cycled_packages = packages.select do |p|
+            p.time_as_object < keep.to_time
+          end
+        else
+          excess = packages.count - keep.to_i
+          cycled_packages = packages.last(excess) if excess > 0
+        end
+
+        saved_packages = packages - cycled_packages
+        cycled_packages.each { |package| delete_package package }
+
+        yaml_save(saved_packages)
+      end
+
+      def delete_package(package)
+        remove!(package) unless package.no_cycle
+      rescue => err
+        Logger.warn Error.wrap(err, <<-EOS)
+            There was a problem removing the following package:
+            Trigger: #{package.trigger} :: Dated: #{package.time}
+            Package included the following #{package.filenames.count} file(s):
+            #{package.filenames.join("\n")}
+          EOS
+      end
+
+      # Returns path to the YAML data file.
+      def yaml_file
+        @yaml_file ||= begin
+          filename = self.class.to_s.split("::").last
+          filename << "-#{storage_id}" if storage_id
+          File.join(Config.data_path, package.trigger, "#{filename}.yml")
+        end
+      end
+
+      # Returns stored Package objects, sorted by #time descending (oldest last).
+      def yaml_load
+        if File.exist?(yaml_file) && !File.zero?(yaml_file)
+          YAML.load_file(yaml_file).sort_by!(&:time).reverse!
+        else
+          []
+        end
+      end
+
+      # Stores the given package objects to the YAML data file.
+      def yaml_save(packages)
+        FileUtils.mkdir_p(File.dirname(yaml_file))
+        File.open(yaml_file, "w") do |file|
+          file.write(packages.to_yaml)
+        end
+      end
+    end
+  end
+end

data/lib/backup/storage/dropbox.rb

@@ -0,0 +1,206 @@
+require "dropbox_sdk"
+
+module Backup
+  module Storage
+    class Dropbox < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Dropbox API credentials
+      attr_accessor :api_key, :api_secret
+
+      ##
+      # Path to store cached authorized session.
+      #
+      # Relative paths will be expanded using Config.root_path,
+      # which by default is ~/Backup unless --root-path was used
+      # on the command line or set in config.rb.
+      #
+      # By default, +cache_path+ is '.cache', which would be
+      # '~/Backup/.cache/' if using the default root_path.
+      attr_accessor :cache_path
+
+      ##
+      # Dropbox Access Type
+      # Valid values are:
+      #   :app_folder (default)
+      #   :dropbox (full access)
+      attr_accessor :access_type
+
+      ##
+      # Chunk size, specified in MiB, for the ChunkedUploader.
+      attr_accessor :chunk_size
+
+      ##
+      # Number of times to retry failed operations.
+      #
+      # Default: 10
+      attr_accessor :max_retries
+
+      ##
+      # Time in seconds to pause before each retry.
+      #
+      # Default: 30
+      attr_accessor :retry_waitsec
+
+      ##
+      # Creates a new instance of the storage object
+      def initialize(model, storage_id = nil)
+        super
+
+        @path           ||= "backups"
+        @cache_path     ||= ".cache"
+        @access_type    ||= :app_folder
+        @chunk_size     ||= 4 # MiB
+        @max_retries    ||= 10
+        @retry_waitsec  ||= 30
+        path.sub!(/^\//, "")
+      end
+
+      private
+
+      ##
+      # The initial connection to Dropbox will provide the user with an
+      # authorization url. The user must open this URL and confirm that the
+      # authorization successfully took place. If this is the case, then the
+      # user hits 'enter' and the session will be properly established.
+      # Immediately after establishing the session, the session will be
+      # serialized and written to a cache file in +cache_path+.
+      # The cached file will be used from that point on to re-establish a
+      # connection with Dropbox at a later time. This allows the user to avoid
+      # having to go to a new Dropbox URL to authorize over and over again.
+      def connection
+        return @connection if @connection
+
+        unless session = cached_session
+          Logger.info "Creating a new session!"
+          session = create_write_and_return_new_session!
+        end
+
+        # will raise an error if session not authorized
+        @connection = DropboxClient.new(session, access_type)
+      rescue => err
+        raise Error.wrap(err, "Authorization Failed")
+      end
+
+      ##
+      # Attempt to load a cached session
+      def cached_session
+        session = false
+        if File.exist?(cached_file)
+          begin
+            session = DropboxSession.deserialize(File.read(cached_file))
+            Logger.info "Session data loaded from cache!"
+          rescue => err
+            Logger.warn Error.wrap(err, <<-EOS)
+              Could not read session data from cache.
+              Cache data might be corrupt.
+            EOS
+          end
+        end
+        session
+      end
+
+      ##
+      # Transfer each of the package files to Dropbox in chunks of +chunk_size+.
+      # Each chunk will be retried +chunk_retries+ times, pausing +retry_waitsec+
+      # between retries, if errors occur.
+      def transfer!
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{dest}'..."
+
+          uploader = nil
+          File.open(src, "r") do |file|
+            uploader = connection.get_chunked_uploader(file, file.stat.size)
+            while uploader.offset < uploader.total_size
+              with_retries do
+                uploader.upload(1024**2 * chunk_size)
+              end
+            end
+          end
+
+          with_retries do
+            uploader.finish(dest)
+          end
+        end
+      rescue => err
+        raise Error.wrap(err, "Upload Failed!")
+      end
+
+      def with_retries
+        retries = 0
+        begin
+          yield
+        rescue StandardError => err
+          retries += 1
+          raise if retries > max_retries
+
+          Logger.info Error.wrap(err, "Retry ##{retries} of #{max_retries}.")
+          sleep(retry_waitsec)
+          retry
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{package.time}..."
+
+        connection.file_delete(remote_path_for(package))
+      end
+
+      def cached_file
+        path = cache_path.start_with?("/") ?
+               cache_path : File.join(Config.root_path, cache_path)
+        File.join(path, api_key + api_secret)
+      end
+
+      ##
+      # Serializes and writes the Dropbox session to a cache file
+      def write_cache!(session)
+        FileUtils.mkdir_p File.dirname(cached_file)
+        File.open(cached_file, "w") do |cache_file|
+          cache_file.write(session.serialize)
+        end
+      end
+
+      ##
+      # Create a new session, write a serialized version of it to the
+      # .cache directory, and return the session object
+      def create_write_and_return_new_session!
+        require "timeout"
+
+        session = DropboxSession.new(api_key, api_secret)
+
+        # grab the request token for session
+        session.get_request_token
+
+        template = Backup::Template.new(
+          session: session, cached_file: cached_file
+        )
+        template.render("storage/dropbox/authorization_url.erb")
+
+        # wait for user to hit 'return' to continue
+        Timeout.timeout(180) { STDIN.gets }
+
+        # this will raise an error if the user did not
+        # visit the authorization_url and grant access
+        #
+        # get the access token from the server
+        # this will be stored with the session in the cache file
+        session.get_access_token
+
+        template.render("storage/dropbox/authorized.erb")
+        write_cache!(session)
+        template.render("storage/dropbox/cache_file_written.erb")
+
+        session
+      rescue => err
+        raise Error.wrap(err, "Could not create or authenticate a new session")
+      end
+    end
+  end
+end