backup-ssh 4.1.10

data/lib/backup/storage/cloud_files.rb

@@ -0,0 +1,158 @@
+# encoding: utf-8
+require 'backup/cloud_io/cloud_files'
+
+module Backup
+  module Storage
+    class CloudFiles < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Rackspace CloudFiles Credentials
+      attr_accessor :username, :api_key
+
+      ##
+      # Rackspace Auth URL (optional)
+      attr_accessor :auth_url
+
+      ##
+      # Rackspace Service Net
+      # (LAN-based transfers to avoid charges and improve performance)
+      attr_accessor :servicenet
+
+      ##
+      # Rackspace Region (optional)
+      attr_accessor :region
+
+      ##
+      # Rackspace Container Name
+      attr_accessor :container
+
+      ##
+      # Rackspace Container Name for SLO Segments
+      # Required if #segment_size is set. Must be different from #container.
+      attr_accessor :segments_container
+
+      ##
+      # SLO Segment size, specified in MiB.
+      #
+      # Each package file larger than +segment_size+
+      # will be uploaded as a Static Large Objects (SLO).
+      #
+      # Defaults to 0 for backward compatibility (pre v.3.7.0),
+      # since #segments_container would be required.
+      #
+      # Minimum: 1 (0 disables SLO support)
+      # Maximum: 5120 (5 GiB)
+      attr_accessor :segment_size
+
+      ##
+      # If set, all backup package files (including SLO segments) will be
+      # scheduled for automatic removal by the server.
+      #
+      # The `keep` option should not be used if this is set,
+      # unless you're transitioning from the `keep` option.
+      attr_accessor :days_to_keep
+
+      ##
+      # Number of times to retry failed operations.
+      #
+      # Default: 10
+      attr_accessor :max_retries
+
+      ##
+      # Time in seconds to pause before each retry.
+      #
+      # Default: 30
+      attr_accessor :retry_waitsec
+
+      ##
+      # Additional options to pass along to fog.
+      # e.g. Fog::Storage.new({ :provider => 'Rackspace' }.merge(fog_options))
+      attr_accessor :fog_options
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @servicenet         ||= false
+        @segment_size       ||= 0
+        @max_retries        ||= 10
+        @retry_waitsec      ||= 30
+
+        @path ||= 'backups'
+        path.sub!(/^\//, '')
+
+        check_configuration
+      end
+
+      private
+
+      def cloud_io
+        @cloud_io ||= CloudIO::CloudFiles.new(
+          :username           => username,
+          :api_key            => api_key,
+          :auth_url           => auth_url,
+          :region             => region,
+          :servicenet         => servicenet,
+          :container          => container,
+          :segments_container => segments_container,
+          :segment_size       => segment_size,
+          :days_to_keep       => days_to_keep,
+          :max_retries        => max_retries,
+          :retry_waitsec      => retry_waitsec,
+          :fog_options        => fog_options
+        )
+      end
+
+      def transfer!
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{ container }/#{ dest }'..."
+          cloud_io.upload(src, dest)
+        end
+
+        package.no_cycle = true if days_to_keep
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{ package.time }..."
+
+        remote_path = remote_path_for(package)
+        objects = cloud_io.objects(remote_path)
+
+        raise Error, "Package at '#{ remote_path }' not found" if objects.empty?
+
+        slo_objects, objects = objects.partition(&:slo?)
+        cloud_io.delete_slo(slo_objects)
+        cloud_io.delete(objects)
+      end
+
+      def check_configuration
+        required = %w{ username api_key container }
+        raise Error, <<-EOS if required.map {|name| send(name) }.any?(&:nil?)
+          Configuration Error
+          #{ required.map {|name| "##{ name }"}.join(', ') } are all required
+        EOS
+
+        raise Error, <<-EOS if segment_size > 0 && segments_container.to_s.empty?
+          Configuration Error
+          #segments_container is required if #segment_size is > 0
+        EOS
+
+        raise Error, <<-EOS if container == segments_container
+          Configuration Error
+          #container and #segments_container must not be the same container.
+        EOS
+
+        raise Error, <<-EOS if segment_size > 5120
+          Configuration Error
+          #segment_size is too large (max 5120)
+        EOS
+      end
+
+    end
+  end
+end

data/lib/backup/storage/cycler.rb

@@ -0,0 +1,65 @@
+# encoding: utf-8
+
+module Backup
+  module Storage
+    module Cycler
+      class Error < Backup::Error; end
+
+      private
+
+      # Adds the current package being stored to the YAML cycle data file
+      # and will remove any old package file(s) when the storage limit
+      # set by #keep is exceeded.
+      def cycle!
+        Logger.info 'Cycling Started...'
+
+        packages = yaml_load.unshift(package)
+        excess = packages.count - keep.to_i
+
+        if excess > 0
+          packages.pop(excess).each do |pkg|
+            begin
+              remove!(pkg) unless pkg.no_cycle
+            rescue => err
+              Logger.warn Error.wrap(err, <<-EOS)
+                There was a problem removing the following package:
+                Trigger: #{pkg.trigger} :: Dated: #{pkg.time}
+                Package included the following #{ pkg.filenames.count } file(s):
+                #{ pkg.filenames.join("\n") }
+              EOS
+            end
+          end
+        end
+
+        yaml_save(packages)
+      end
+
+      # Returns path to the YAML data file.
+      def yaml_file
+        @yaml_file ||= begin
+          filename = self.class.to_s.split('::').last
+          filename << "-#{ storage_id }" if storage_id
+          File.join(Config.data_path, package.trigger, "#{ filename }.yml")
+        end
+      end
+
+      # Returns stored Package objects, sorted by #time descending (oldest last).
+      def yaml_load
+        if File.exist?(yaml_file) && !File.zero?(yaml_file)
+          YAML.load_file(yaml_file).sort_by!(&:time).reverse!
+        else
+          []
+        end
+      end
+
+      # Stores the given package objects to the YAML data file.
+      def yaml_save(packages)
+        FileUtils.mkdir_p(File.dirname(yaml_file))
+        File.open(yaml_file, 'w') do |file|
+          file.write(packages.to_yaml)
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/storage/dropbox.rb

@@ -0,0 +1,236 @@
+# encoding: utf-8
+require 'dropbox_sdk'
+
+module Backup
+  module Storage
+    class Dropbox < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Dropbox API credentials
+      attr_accessor :api_key, :api_secret
+
+      ##
+      # Path to store cached authorized session.
+      #
+      # Relative paths will be expanded using Config.root_path,
+      # which by default is ~/Backup unless --root-path was used
+      # on the command line or set in config.rb.
+      #
+      # By default, +cache_path+ is '.cache', which would be
+      # '~/Backup/.cache/' if using the default root_path.
+      attr_accessor :cache_path
+
+      ##
+      # Dropbox Access Type
+      # Valid values are:
+      #   :app_folder (default)
+      #   :dropbox (full access)
+      attr_accessor :access_type
+
+      ##
+      # Chunk size, specified in MiB, for the ChunkedUploader.
+      attr_accessor :chunk_size
+
+      ##
+      # Number of times to retry failed operations.
+      #
+      # Default: 10
+      attr_accessor :max_retries
+
+      ##
+      # Time in seconds to pause before each retry.
+      #
+      # Default: 30
+      attr_accessor :retry_waitsec
+
+      ##
+      # Creates a new instance of the storage object
+      def initialize(model, storage_id = nil)
+        super
+
+        @path           ||= 'backups'
+        @cache_path     ||= '.cache'
+        @access_type    ||= :app_folder
+        @chunk_size     ||= 4 # MiB
+        @max_retries    ||= 10
+        @retry_waitsec  ||= 30
+        path.sub!(/^\//, '')
+      end
+
+      private
+
+      ##
+      # The initial connection to Dropbox will provide the user with an
+      # authorization url. The user must open this URL and confirm that the
+      # authorization successfully took place. If this is the case, then the
+      # user hits 'enter' and the session will be properly established.
+      # Immediately after establishing the session, the session will be
+      # serialized and written to a cache file in +cache_path+.
+      # The cached file will be used from that point on to re-establish a
+      # connection with Dropbox at a later time. This allows the user to avoid
+      # having to go to a new Dropbox URL to authorize over and over again.
+      def connection
+        return @connection if @connection
+
+        unless session = cached_session
+          Logger.info "Creating a new session!"
+          session = create_write_and_return_new_session!
+        end
+
+        # will raise an error if session not authorized
+        @connection = DropboxClient.new(session, access_type)
+
+      rescue => err
+        raise Error.wrap(err, 'Authorization Failed')
+      end
+
+      ##
+      # Attempt to load a cached session
+      def cached_session
+        session = false
+        if File.exist?(cached_file)
+          begin
+            session = DropboxSession.deserialize(File.read(cached_file))
+            Logger.info "Session data loaded from cache!"
+
+          rescue => err
+            Logger.warn Error.wrap(err, <<-EOS)
+              Could not read session data from cache.
+              Cache data might be corrupt.
+            EOS
+          end
+        end
+        session
+      end
+
+      ##
+      # Transfer each of the package files to Dropbox in chunks of +chunk_size+.
+      # Each chunk will be retried +chunk_retries+ times, pausing +retry_waitsec+
+      # between retries, if errors occur.
+      def transfer!
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{ dest }'..."
+
+          uploader = nil
+          File.open(src, 'r') do |file|
+            uploader = connection.get_chunked_uploader(file, file.stat.size)
+            while uploader.offset < uploader.total_size
+              with_retries do
+                uploader.upload(1024**2 * chunk_size)
+              end
+            end
+          end
+
+          with_retries do
+            uploader.finish(dest)
+          end
+        end
+
+      rescue => err
+        raise Error.wrap(err, 'Upload Failed!')
+      end
+
+      def with_retries
+        retries = 0
+        begin
+          yield
+        rescue StandardError => err
+          retries += 1
+          raise if retries > max_retries
+
+          Logger.info Error.wrap(err, "Retry ##{ retries } of #{ max_retries }.")
+          sleep(retry_waitsec)
+          retry
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{ package.time }..."
+
+        connection.file_delete(remote_path_for(package))
+      end
+
+      def cached_file
+        path = cache_path.start_with?('/') ?
+               cache_path : File.join(Config.root_path, cache_path)
+        File.join(path, api_key + api_secret)
+      end
+
+      ##
+      # Serializes and writes the Dropbox session to a cache file
+      def write_cache!(session)
+        FileUtils.mkdir_p File.dirname(cached_file)
+        File.open(cached_file, "w") do |cache_file|
+          cache_file.write(session.serialize)
+        end
+      end
+
+      ##
+      # Create a new session, write a serialized version of it to the
+      # .cache directory, and return the session object
+      def create_write_and_return_new_session!
+        require 'timeout'
+
+        session = DropboxSession.new(api_key, api_secret)
+
+        # grab the request token for session
+        session.get_request_token
+
+        template = Backup::Template.new(
+          {:session => session, :cached_file => cached_file}
+        )
+        template.render("storage/dropbox/authorization_url.erb")
+
+        # wait for user to hit 'return' to continue
+        Timeout::timeout(180) { STDIN.gets }
+
+        # this will raise an error if the user did not
+        # visit the authorization_url and grant access
+        #
+        # get the access token from the server
+        # this will be stored with the session in the cache file
+        session.get_access_token
+
+        template.render("storage/dropbox/authorized.erb")
+        write_cache!(session)
+        template.render("storage/dropbox/cache_file_written.erb")
+
+        session
+
+      rescue => err
+        raise Error.wrap(err, 'Could not create or authenticate a new session')
+      end
+
+    end
+  end
+end
+
+# Patch for dropbox-ruby-sdk-1.5.1
+class DropboxClient
+  class ChunkedUploader
+    def upload(chunk_size = 1024**2 * 4)
+      while @offset < @total_size
+        @file_obj.seek(@offset) unless @file_obj.pos == @offset
+        data = @file_obj.read(chunk_size)
+
+        begin
+          resp = @client.parse_response(
+            @client.partial_chunked_upload(data, @upload_id, @offset)
+          )
+        rescue DropboxError => err
+          resp = JSON.parse(err.http_response.body) rescue {}
+          raise err unless resp['offset']
+        end
+
+        @offset = resp['offset']
+        @upload_id ||= resp['upload_id']
+      end
+    end
+  end
+end