backup 3.6.0 → 3.7.0

data/lib/backup/storage/cycler.rb

@@ -3,6 +3,8 @@
 module Backup
   module Storage
     module Cycler
+      class Error < Backup::Error; end
+
       class << self
 
         ##
@@ -37,9 +39,9 @@ module Backup
         def remove_packages!
           @packages_to_remove.each do |pkg|
             begin
-              @storage.send(:remove!, pkg)
+              @storage.send(:remove!, pkg) unless pkg.no_cycle
             rescue => err
-              Logger.warn Errors::Storage::CyclerError.wrap(err, <<-EOS)
+              Logger.warn Error.wrap(err, <<-EOS)
                 There was a problem removing the following package:
                 Trigger: #{pkg.trigger} :: Dated: #{pkg.time}
                 Package included the following #{ pkg.filenames.count } file(s):

data/lib/backup/storage/dropbox.rb

@@ -4,6 +4,7 @@ require 'dropbox_sdk'
 module Backup
   module Storage
     class Dropbox < Base
+      class Error < Backup::Error; end
 
       ##
       # Dropbox API credentials
@@ -21,23 +22,26 @@ module Backup
       attr_accessor :chunk_size
 
       ##
-      # Number of times to retry a failed chunk.
-      attr_accessor :chunk_retries
+      # Number of times to retry failed operations.
+      #
+      # Default: 10
+      attr_accessor :max_retries
 
       ##
-      # Seconds to wait between chunk retries.
+      # Time in seconds to pause before each retry.
+      #
+      # Default: 30
       attr_accessor :retry_waitsec
 
       ##
       # Creates a new instance of the storage object
-      def initialize(model, storage_id = nil, &block)
+      def initialize(model, storage_id = nil)
         super
-        instance_eval(&block) if block_given?
 
         @path           ||= 'backups'
         @access_type    ||= :app_folder
         @chunk_size     ||= 4 # MiB
-        @chunk_retries  ||= 10
+        @max_retries    ||= 10
         @retry_waitsec  ||= 30
         path.sub!(/^\//, '')
       end
@@ -66,7 +70,7 @@ module Backup
         @connection = DropboxClient.new(session, access_type)
 
       rescue => err
-        raise Errors::Storage::Dropbox::ConnectionError.wrap(err)
+        raise Error.wrap(err, 'Authorization Failed')
       end
 
       ##
@@ -79,7 +83,7 @@ module Backup
             Logger.info "Session data loaded from cache!"
 
           rescue => err
-            Logger.warn Errors::Storage::Dropbox::CacheError.wrap(err, <<-EOS)
+            Logger.warn Error.wrap(err, <<-EOS)
               Could not read session data from cache.
               Cache data might be corrupt.
             EOS
@@ -114,7 +118,7 @@ module Backup
         end
 
       rescue => err
-        raise Errors::Storage::Dropbox::TransferError.wrap(err, 'Upload Failed!')
+        raise Error.wrap(err, 'Upload Failed!')
       end
 
       # Timeout::Error is not a StandardError under ruby-1.8.7
@@ -124,10 +128,9 @@ module Backup
           yield
         rescue StandardError, Timeout::Error => err
           retries += 1
-          raise if retries > chunk_retries
+          raise if retries > max_retries
 
-          Logger.info Errors::Storage::Dropbox::TransferError.
-              wrap(err, "Retry ##{ retries } of #{ chunk_retries }.")
+          Logger.info Error.wrap(err, "Retry ##{ retries } of #{ max_retries }.")
           sleep(retry_waitsec)
           retry
         end
@@ -186,16 +189,17 @@ module Backup
 
         session
 
-        rescue => err
-          raise Errors::Storage::Dropbox::AuthenticationError.wrap(
-            err, 'Could not create or authenticate a new session'
-          )
+      rescue => err
+        raise Error.wrap(err, 'Could not create or authenticate a new session')
       end
 
       attr_deprecate :email,    :version => '3.0.17'
       attr_deprecate :password, :version => '3.0.17'
       attr_deprecate :timeout,  :version => '3.0.21'
 
+      attr_deprecate :chunk_retries, :version => '3.7.0',
+                     :message => 'Use #max_retries instead.',
+                     :action => lambda {|klass, val| klass.max_retries = val }
     end
   end
 end

data/lib/backup/storage/ftp.rb

@@ -17,9 +17,8 @@ module Backup
       # use passive mode?
       attr_accessor :passive_mode
 
-      def initialize(model, storage_id = nil, &block)
+      def initialize(model, storage_id = nil)
         super
-        instance_eval(&block) if block_given?
 
         @port         ||= 21
         @path         ||= 'backups'

data/lib/backup/storage/local.rb

@@ -3,10 +3,10 @@
 module Backup
   module Storage
     class Local < Base
+      class Error < Backup::Error; end
 
-      def initialize(model, storage_id = nil, &block)
+      def initialize(model, storage_id = nil)
         super
-        instance_eval(&block) if block_given?
 
         @path ||= '~/backups'
       end
@@ -47,7 +47,7 @@ module Backup
         if self == model.storages.last
           true
         else
-          Logger.warn Errors::Storage::Local::TransferError.new(<<-EOS)
+          Logger.warn Error.new(<<-EOS)
             Local File Copy Warning!
             The final backup file(s) for '#{ model.label }' (#{ model.trigger })
             will be *copied* to '#{ remote_path }'

data/lib/backup/storage/ninefold.rb

@@ -4,14 +4,14 @@ require 'fog'
 module Backup
   module Storage
     class Ninefold < Base
+      class Error < Backup::Error; end
 
       ##
       # Ninefold Credentials
       attr_accessor :storage_token, :storage_secret
 
-      def initialize(model, storage_id = nil, &block)
+      def initialize(model, storage_id = nil)
         super
-        instance_eval(&block) if block_given?
 
         @path ||= 'backups'
         path.sub!(/^\//, '')
@@ -58,8 +58,7 @@ module Backup
         remote_path = remote_path_for(package)
         directory = directory_for(remote_path)
 
-        raise Errors::Storage::Ninefold::NotFoundError,
-            "Directory at '#{ remote_path }' not found" unless directory
+        raise Error, "Directory at '#{ remote_path }' not found" unless directory
 
         package.filenames.each do |filename|
           file = directory.files.get(filename)

data/lib/backup/storage/rsync.rb

@@ -132,9 +132,8 @@ module Backup
       # will also store the files directly in the +path+ given.
       attr_accessor :path
 
-      def initialize(model, storage_id = nil, &block)
+      def initialize(model, storage_id = nil)
         super
-        instance_eval(&block) if block_given?
 
         @mode ||= :ssh
         @port ||= mode == :rsync_daemon ? 873 : 22

data/lib/backup/storage/s3.rb

@@ -1,11 +1,10 @@
 # encoding: utf-8
-require 'fog'
-require 'base64'
-require 'digest/md5'
+require 'backup/cloud_io/s3'
 
 module Backup
   module Storage
     class S3 < Base
+      class Error < Backup::Error; end
 
       ##
       # Amazon Simple Storage Service (S3) Credentials
@@ -20,32 +19,19 @@ module Backup
       attr_accessor :region
 
       ##
-      # Chunk size, specified in MiB, for S3 Multipart Upload.
+      # Multipart chunk size, specified in MiB.
       #
-      # Each backup package file that is greater than +chunk_size+
-      # will be uploaded using AWS' Multipart Upload.
+      # Each package file larger than +chunk_size+
+      # will be uploaded using S3 Multipart Upload.
       #
-      # Package files less than or equal to +chunk_size+ will be
-      # uploaded via a single PUT request.
-      #
-      # Minimum allowed: 5 (but may be disabled with 0)
+      # Minimum: 5 (but may be disabled with 0)
+      # Maximum: 5120
       # Default: 5
       attr_accessor :chunk_size
 
       ##
       # Number of times to retry failed operations.
       #
-      # The retry count is reset when the failing operation succeeds,
-      # so each operation that fails will be retried this number of times.
-      # Once a single failed operation exceeds +max_retries+, the entire
-      # storage operation will fail.
-      #
-      # Operations that may fail and be retried include:
-      # - Multipart initiation requests.
-      # - Each multipart upload of +chunk_size+. (retries the chunk)
-      # - Multipart upload completion requests.
-      # - Each file uploaded not using multipart upload. (retries the file)
-      #
       # Default: 10
       attr_accessor :max_retries
 
@@ -62,8 +48,6 @@ module Backup
       #
       # - :aes256
       #
-      # @see http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
-      #
       # Default: nil
       attr_accessor :encryption
 
@@ -75,14 +59,11 @@ module Backup
       # - :standard (default)
       # - :reduced_redundancy
       #
-      # @see http://docs.aws.amazon.com/AmazonS3/latest/dev/SetStoClsOfObjUploaded.html
-      #
       # Default: :standard
       attr_accessor :storage_class
 
-      def initialize(model, storage_id = nil, &block)
+      def initialize(model, storage_id = nil)
         super
-        instance_eval(&block) if block_given?
 
         @chunk_size     ||= 5 # MiB
         @max_retries    ||= 10
@@ -90,21 +71,24 @@ module Backup
         @path           ||= 'backups'
         @storage_class  ||= :standard
         path.sub!(/^\//, '')
+
+        check_configuration
       end
 
       private
 
-      def connection
-        @connection ||= begin
-          conn = Fog::Storage.new(
-            :provider               => 'AWS',
-            :aws_access_key_id      => access_key_id,
-            :aws_secret_access_key  => secret_access_key,
-            :region                 => region
-          )
-          conn.sync_clock
-          conn
-        end
+      def cloud_io
+        @cloud_io ||= CloudIO::S3.new(
+          :access_key_id      => access_key_id,
+          :secret_access_key  => secret_access_key,
+          :region             => region,
+          :bucket             => bucket,
+          :encryption         => encryption,
+          :storage_class      => storage_class,
+          :max_retries        => max_retries,
+          :retry_waitsec      => retry_waitsec,
+          :chunk_size         => chunk_size
+        )
       end
 
       def transfer!
@@ -112,7 +96,7 @@ module Backup
           src = File.join(Config.tmp_path, filename)
           dest = File.join(remote_path, filename)
           Logger.info "Storing '#{ bucket }/#{ dest }'..."
-          Uploader.new(self, connection, src, dest).run
+          cloud_io.upload(src, dest)
         end
       end
 
@@ -122,129 +106,36 @@ module Backup
         Logger.info "Removing backup package dated #{ package.time }..."
 
         remote_path = remote_path_for(package)
-        resp = connection.get_bucket(bucket, :prefix => remote_path)
-        keys = resp.body['Contents'].map {|entry| entry['Key'] }
+        objects = cloud_io.objects(remote_path)
 
-        raise Errors::Storage::S3::NotFoundError,
-            "Package at '#{ remote_path }' not found" if keys.empty?
+        raise Error, "Package at '#{ remote_path }' not found" if objects.empty?
 
-        connection.delete_multiple_objects(bucket, keys)
+        cloud_io.delete(objects)
       end
 
-      class Uploader
-        attr_reader :connection, :bucket, :chunk_size, :max_retries,
-                    :retry_waitsec, :storage_class, :encryption,
-                    :src, :dest, :upload_id, :parts
-
-        def initialize(storage, connection, src, dest)
-          @connection     = connection
-          @bucket         = storage.bucket
-          @chunk_size     = storage.chunk_size * 1024**2
-          @max_retries    = storage.max_retries
-          @retry_waitsec  = storage.retry_waitsec
-          @encryption     = storage.encryption
-          @storage_class  = storage.storage_class
-          @src    = src
-          @dest   = dest
-          @parts  = []
-        end
-
-        def run
-          if chunk_size > 0 && File.size(src) > chunk_size
-            initiate_multipart
-            upload_parts
-            complete_multipart
-          else
-            upload
-          end
-        rescue => err
-          raise error_with(err, 'Upload Failed!')
-        end
-
-        private
-
-        def upload
-          md5 = Base64.encode64(Digest::MD5.file(src).digest).chomp
-          options = headers.merge('Content-MD5' => md5)
-          with_retries do
-            File.open(src, 'r') do |file|
-              connection.put_object(bucket, dest, file, options)
-            end
-          end
-        end
-
-        def initiate_multipart
-          with_retries do
-            resp = connection.initiate_multipart_upload(bucket, dest, headers)
-            @upload_id = resp.body['UploadId']
-          end
-        end
-
-        def upload_parts
-          File.open(src, 'r') do |file|
-            part_number = 0
-            while data = file.read(chunk_size)
-              part_number += 1
-              md5 = Base64.encode64(Digest::MD5.digest(data)).chomp
-              with_retries do
-                resp = connection.upload_part(
-                  bucket, dest, upload_id, part_number, data,
-                  { 'Content-MD5' => md5 }
-                )
-                parts << resp.headers['ETag']
-              end
-            end
-          end
-        end
-
-        def headers
-          headers = {}
-
-          val = encryption.to_s.upcase
-          headers.merge!(
-            { 'x-amz-server-side-encryption' => val }
-          ) unless val.empty?
-
-          val = storage_class.to_s.upcase
-          headers.merge!(
-            { 'x-amz-storage-class' => val }
-          ) unless val.empty? || val == 'STANDARD'
-
-          headers
-        end
-
-        def complete_multipart
-          with_retries do
-            connection.complete_multipart_upload(bucket, dest, upload_id, parts)
-          end
-        end
-
-        def with_retries
-          retries = 0
-          begin
-            yield
-          rescue => err
-            retries += 1
-            raise if retries > max_retries
-
-            Logger.info error_with(err, "Retry ##{ retries } of #{ max_retries }.")
-            sleep(retry_waitsec)
-            retry
-          end
-        end
-
-        def error_with(err, msg)
-          if err.is_a? Excon::Errors::HTTPStatusError
-            Errors::Storage::S3::UploaderError.new(<<-EOS)
-              #{ msg }
-              Reason: #{ err.class }
-              response => #{ err.response.inspect }
-            EOS
-          else
-            Errors::Storage::S3::UploaderError.wrap(err, msg)
-          end
-        end
-      end # class Uploader
+      def check_configuration
+        required = %w{ access_key_id secret_access_key bucket }
+        raise Error, <<-EOS if required.map {|name| send(name) }.any?(&:nil?)
+          Configuration Error
+          #{ required.map {|name| "##{ name }"}.join(', ') } are all required
+        EOS
+
+        raise Error, <<-EOS if chunk_size > 0 && !chunk_size.between?(5, 5120)
+          Configuration Error
+          #chunk_size must be between 5 and 5120 (or 0 to disable multipart)
+        EOS
+
+        raise Error, <<-EOS if encryption && encryption.to_s.upcase != 'AES256'
+          Configuration Error
+          #encryption must be :aes256 or nil
+        EOS
+
+        classes = ['STANDARD', 'REDUCED_REDUNDANCY']
+        raise Error, <<-EOS unless classes.include?(storage_class.to_s.upcase)
+          Configuration Error
+          #storage_class must be :standard or :reduced_redundancy
+        EOS
+      end
 
     end
   end