backup 3.3.2 → 3.4.0

data/lib/backup/storage/rsync.rb

@@ -145,46 +145,40 @@ module Backup
       private
 
       def transfer!
-        Logger.info "#{ storage_name } Started..."
+        write_password_file
+        create_remote_path
 
-        write_password_file!
-        create_dest_path!
-
-        files_to_transfer_for(@package) do |local_file, remote_file|
-          src = "'#{ File.join(local_path, local_file) }'"
-          dest = "#{ host_options }'#{ File.join(dest_path, remote_file) }'"
+        package.filenames.each do |filename|
+          src = "'#{ File.join(Config.tmp_path, filename) }'"
+          dest = "#{ host_options }'#{ File.join(remote_path, filename) }'"
           Logger.info "Syncing to #{ dest }..."
           run("#{ rsync_command } #{ src } #{ dest }")
         end
-
-        Logger.info "#{ storage_name } Finished!"
       ensure
-        remove_password_file!
+        remove_password_file
       end
 
-      ##
       # Storage::RSync doesn't cycle
       def cycle!; end
 
       ##
-      # Other storages use #remote_path_for to set the dest_path,
-      # which adds an additional timestamp directory to the path.
+      # Other storages add an additional timestamp directory to this path.
       # This is not desired here, since we need to transfer the package files
       # to the same location each time.
       #
       # Note: In v4.0, the additional trigger directory will to be dropped
-      # from dest_path for both local and :ssh mode, so the package files will
-      # be stored directly in #path.
-      def dest_path
-        @dest_path ||= begin
+      # from remote_path for both local and :ssh mode, so the package files
+      # will be stored directly in #path.
+      def remote_path
+        @remote_path ||= begin
           if host
             if mode == :ssh
-              File.join(path.sub(/^~\//, ''), @package.trigger)
+              File.join(path.sub(/^~\//, ''), package.trigger)
             else
               path.sub(/^~\//, '').sub(/\/$/, '')
             end
           else
-            File.join(File.expand_path(path), @package.trigger)
+            File.join(File.expand_path(path), package.trigger)
           end
         end
       end
@@ -197,12 +191,12 @@ module Backup
       # This is only applicable locally and in :ssh mode.
       # In :ssh_daemon and :rsync_daemon modes the `path` would include a
       # module name that must define a path on the remote that already exists.
-      def create_dest_path!
+      def create_remote_path
         if host
           run("#{ utility(:ssh) } #{ ssh_transport_args } #{ host } " +
-                  %Q["mkdir -p '#{ dest_path }'"]) if mode == :ssh
+                  %Q["mkdir -p '#{ remote_path }'"]) if mode == :ssh
         else
-          FileUtils.mkdir_p dest_path
+          FileUtils.mkdir_p(remote_path)
         end
       end
 
@@ -254,7 +248,7 @@ module Backup
         args.rstrip
       end
 
-      def write_password_file!
+      def write_password_file
         return unless host && rsync_password && mode != :ssh
 
         @password_file = Tempfile.new('backup-rsync-password')
@@ -262,7 +256,7 @@ module Backup
         @password_file.close
       end
 
-      def remove_password_file!
+      def remove_password_file
         @password_file.delete if @password_file
       end
 

data/lib/backup/storage/s3.rb

@@ -1,8 +1,8 @@
 # encoding: utf-8
 
-##
-# Only load the Fog gem when the Backup::Storage::S3 class is loaded
 Backup::Dependency.load('fog')
+require 'base64'
+require 'digest/md5'
 
 module Backup
   module Storage
@@ -13,81 +13,199 @@ module Backup
       attr_accessor :access_key_id, :secret_access_key
 
       ##
-      # Amazon S3 bucket name and path
-      attr_accessor :bucket, :path
+      # Amazon S3 bucket name
+      attr_accessor :bucket
 
       ##
       # Region of the specified S3 bucket
       attr_accessor :region
 
       ##
-      # Creates a new instance of the storage object
-      def initialize(model, storage_id = nil, &block)
-        super(model, storage_id)
+      # Chunk size, specified in MiB, for S3 Multipart Upload.
+      #
+      # Each backup package file that is greater than +chunk_size+
+      # will be uploaded using AWS' Multipart Upload.
+      #
+      # Package files less than or equal to +chunk_size+ will be
+      # uploaded via a single PUT request.
+      #
+      # Minimum allowed: 5 (but may be disabled with 0)
+      # Default: 5
+      attr_accessor :chunk_size
 
-        @path ||= 'backups'
+      ##
+      # Number of times to retry failed operations.
+      #
+      # The retry count is reset when the failing operation succeeds,
+      # so each operation that fails will be retried this number of times.
+      # Once a single failed operation exceeds +max_retries+, the entire
+      # storage operation will fail.
+      #
+      # Operations that may fail and be retried include:
+      # - Multipart initiation requests.
+      # - Each multipart upload of +chunk_size+. (retries the chunk)
+      # - Multipart upload completion requests.
+      # - Each file uploaded not using multipart upload. (retries the file)
+      #
+      # Default: 10
+      attr_accessor :max_retries
 
+      ##
+      # Time in seconds to pause before each retry.
+      #
+      # Default: 30
+      attr_accessor :retry_waitsec
+
+      def initialize(model, storage_id = nil, &block)
+        super
         instance_eval(&block) if block_given?
+
+        @chunk_size     ||= 5 # MiB
+        @max_retries    ||= 10
+        @retry_waitsec  ||= 30
+        @path           ||= 'backups'
+        path.sub!(/^\//, '')
       end
 
       private
 
-      ##
-      # This is the provider that Fog uses for the S3 Storage
-      def provider
-        'AWS'
+      def connection
+        @connection ||= begin
+          conn = Fog::Storage.new(
+            :provider               => 'AWS',
+            :aws_access_key_id      => access_key_id,
+            :aws_secret_access_key  => secret_access_key,
+            :region                 => region
+          )
+          conn.sync_clock
+          conn
+        end
       end
 
-      ##
-      # Establishes a connection to Amazon S3
-      def connection
-        @connection ||= Fog::Storage.new(
-          :provider               => provider,
-          :aws_access_key_id      => access_key_id,
-          :aws_secret_access_key  => secret_access_key,
-          :region                 => region
-        )
+      def transfer!
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{ bucket }/#{ dest }'..."
+          Uploader.new(connection, bucket, src, dest,
+                       chunk_size, max_retries, retry_waitsec).run
+        end
       end
 
-      def remote_path_for(package)
-        super(package).sub(/^\//, '')
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{ package.time }..."
+
+        remote_path = remote_path_for(package)
+        resp = connection.get_bucket(bucket, :prefix => remote_path)
+        keys = resp.body['Contents'].map {|entry| entry['Key'] }
+
+        raise Errors::Storage::S3::NotFoundError,
+            "Package at '#{ remote_path }' not found" if keys.empty?
+
+        connection.delete_multiple_objects(bucket, keys)
       end
 
-      ##
-      # Transfers the archived file to the specified Amazon S3 bucket
-      def transfer!
-        remote_path = remote_path_for(@package)
+      class Uploader
+        attr_reader :connection, :bucket, :src, :dest
+        attr_reader :chunk_size, :max_retries, :retry_waitsec
+        attr_reader :upload_id, :parts
+
+        def initialize(connection, bucket, src, dest,
+                       chunk_size, max_retries, retry_waitsec)
+          @connection = connection
+          @bucket = bucket
+          @src = src
+          @dest = dest
+          @chunk_size = 1024**2 * chunk_size
+          @max_retries = max_retries
+          @retry_waitsec = retry_waitsec
+          @parts = []
+        end
 
-        connection.sync_clock
+        def run
+          if chunk_size > 0 && File.size(src) > chunk_size
+            initiate_multipart
+            upload_parts
+            complete_multipart
+          else
+            upload
+          end
+        rescue => err
+          raise error_with(err, 'Upload Failed!')
+        end
 
-        files_to_transfer_for(@package) do |local_file, remote_file|
-          Logger.info "#{storage_name} started transferring " +
-              "'#{ local_file }' to bucket '#{ bucket }'."
+        private
 
-          File.open(File.join(local_path, local_file), 'r') do |file|
-            connection.put_object(
-              bucket, File.join(remote_path, remote_file), file
-            )
+        def upload
+          md5 = Base64.encode64(Digest::MD5.file(src).digest).chomp
+          with_retries do
+            File.open(src, 'r') do |file|
+              connection.put_object(bucket, dest, file, { 'Content-MD5' => md5 })
+            end
           end
         end
-      end
 
-      ##
-      # Removes the transferred archive file(s) from the storage location.
-      # Any error raised will be rescued during Cycling
-      # and a warning will be logged, containing the error message.
-      def remove!(package)
-        remote_path = remote_path_for(package)
+        def initiate_multipart
+          with_retries do
+            resp = connection.initiate_multipart_upload(bucket, dest)
+            @upload_id = resp.body['UploadId']
+          end
+        end
 
-        connection.sync_clock
+        def upload_parts
+          File.open(src, 'r') do |file|
+            part_number = 0
+            while data = file.read(chunk_size)
+              part_number += 1
+              md5 = Base64.encode64(Digest::MD5.digest(data)).chomp
+              with_retries do
+                resp = connection.upload_part(
+                  bucket, dest, upload_id, part_number, data,
+                  { 'Content-MD5' => md5 }
+                )
+                parts << resp.headers['ETag']
+              end
+            end
+          end
+        end
 
-        transferred_files_for(package) do |local_file, remote_file|
-          Logger.info "#{storage_name} started removing " +
-              "'#{ local_file }' from bucket '#{ bucket }'."
+        def complete_multipart
+          with_retries do
+            connection.complete_multipart_upload(bucket, dest, upload_id, parts)
+          end
+        end
 
-          connection.delete_object(bucket, File.join(remote_path, remote_file))
+        def with_retries
+          retries = 0
+          begin
+            yield
+          rescue => err
+            retries += 1
+            raise if retries > max_retries
+
+            Logger.info error_with(err, "Retry ##{ retries } of #{ max_retries }.")
+            sleep(retry_waitsec)
+            retry
+          end
         end
-      end
+
+        # Avoid wrapping Excon::Errors::HTTPStatusError since it's message
+        # includes `request.inspect`. For multipart uploads, this includes
+        # the String#inspect output of `file.read(chunk_size)`.
+        def error_with(err, msg)
+          if err.is_a? Excon::Errors::HTTPStatusError
+            Errors::Storage::S3::UploaderError.new(<<-EOS)
+              #{ msg }
+              Reason: #{ err.class }
+              response => #{ err.response.inspect }
+            EOS
+          else
+            Errors::Storage::S3::UploaderError.wrap(err, msg)
+          end
+        end
+      end # class Uploader
 
     end
   end

data/lib/backup/storage/scp.rb

@@ -14,78 +14,50 @@ module Backup
       # Server IP Address and SCP port
       attr_accessor :ip, :port
 
-      ##
-      # Path to store backups to
-      attr_accessor :path
-
-      ##
-      # Creates a new instance of the storage object
       def initialize(model, storage_id = nil, &block)
-        super(model, storage_id)
+        super
+        instance_eval(&block) if block_given?
 
         @port ||= 22
         @path ||= 'backups'
-
-        instance_eval(&block) if block_given?
-
-        @path = path.sub(/^\~\//, '')
+        path.sub!(/^~\//, '')
       end
 
       private
 
-      ##
-      # Establishes a connection to the remote server
-      # and yields the Net::SSH connection.
-      # Net::SCP will use this connection to transfer backups
       def connection
         Net::SSH.start(
           ip, username, :password => password, :port => port
         ) {|ssh| yield ssh }
       end
 
-      ##
-      # Transfers the archived file to the specified remote server
       def transfer!
-        remote_path = remote_path_for(@package)
-
         connection do |ssh|
           ssh.exec!("mkdir -p '#{ remote_path }'")
 
-          files_to_transfer_for(@package) do |local_file, remote_file|
-            Logger.info "#{storage_name} started transferring " +
-                "'#{local_file}' to '#{ip}'."
-
-            ssh.scp.upload!(
-              File.join(local_path, local_file),
-              File.join(remote_path, remote_file)
-            )
+          package.filenames.each do |filename|
+            src = File.join(Config.tmp_path, filename)
+            dest = File.join(remote_path, filename)
+            Logger.info "Storing '#{ ip }:#{ dest }'..."
+            ssh.scp.upload!(src, dest)
           end
         end
       end
 
-      ##
-      # Removes the transferred archive file(s) from the storage location.
-      # Any error raised will be rescued during Cycling
-      # and a warning will be logged, containing the error message.
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
       def remove!(package)
-        remote_path = remote_path_for(package)
-
-        messages = []
-        transferred_files_for(package) do |local_file, remote_file|
-          messages << "#{storage_name} started removing " +
-              "'#{local_file}' from '#{ip}'."
-        end
-        Logger.info messages.join("\n")
+        Logger.info "Removing backup package dated #{ package.time }..."
 
         errors = []
         connection do |ssh|
-          ssh.exec!("rm -r '#{remote_path}'") do |ch, stream, data|
+          ssh.exec!("rm -r '#{ remote_path_for(package) }'") do |ch, stream, data|
             errors << data if stream == :stderr
           end
         end
         unless errors.empty?
           raise Errors::Storage::SCP::SSHError,
-            "Net::SSH reported the following errors:\n" +
+              "Net::SSH reported the following errors:\n" +
               errors.join("\n")
         end
       end

data/lib/backup/storage/sftp.rb

@@ -14,27 +14,17 @@ module Backup
       # Server IP Address and SFTP port
       attr_accessor :ip, :port
 
-      ##
-      # Path to store backups to
-      attr_accessor :path
-
-      ##
-      # Creates a new instance of the storage object
       def initialize(model, storage_id = nil, &block)
-        super(model, storage_id)
+        super
+        instance_eval(&block) if block_given?
 
         @port ||= 22
         @path ||= 'backups'
-
-        instance_eval(&block) if block_given?
-
-        @path = path.sub(/^\~\//, '')
+        path.sub!(/^~\//, '')
       end
 
       private
 
-      ##
-      # Establishes a connection to the remote server
       def connection
         Net::SFTP.start(
           ip, username,
@@ -43,39 +33,28 @@ module Backup
         ) {|sftp| yield sftp }
       end
 
-      ##
-      # Transfers the archived file to the specified remote server
       def transfer!
-        remote_path = remote_path_for(@package)
-
         connection do |sftp|
-          create_remote_path(remote_path, sftp)
+          create_remote_path(sftp)
 
-          files_to_transfer_for(@package) do |local_file, remote_file|
-            Logger.info "#{storage_name} started transferring " +
-                "'#{ local_file }' to '#{ ip }'."
-
-            sftp.upload!(
-              File.join(local_path, local_file),
-              File.join(remote_path, remote_file)
-            )
+          package.filenames.each do |filename|
+            src = File.join(Config.tmp_path, filename)
+            dest = File.join(remote_path, filename)
+            Logger.info "Storing '#{ ip }:#{ dest }'..."
+            sftp.upload!(src, dest)
           end
         end
       end
 
-      ##
-      # Removes the transferred archive file(s) from the storage location.
-      # Any error raised will be rescued during Cycling
-      # and a warning will be logged, containing the error message.
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
       def remove!(package)
-        remote_path = remote_path_for(package)
+        Logger.info "Removing backup package dated #{ package.time }..."
 
+        remote_path = remote_path_for(package)
         connection do |sftp|
-          transferred_files_for(package) do |local_file, remote_file|
-            Logger.info "#{storage_name} started removing " +
-                "'#{ local_file }' from '#{ ip }'."
-
-            sftp.remove!(File.join(remote_path, remote_file))
+          package.filenames.each do |filename|
+            sftp.remove!(File.join(remote_path, filename))
           end
 
           sftp.rmdir!(remote_path)
@@ -90,7 +69,7 @@ module Backup
       # '/') and loop through that to create the directories one by one.
       # Net::SFTP raises an exception when the directory it's trying to create
       # already exists, so we have rescue it
-      def create_remote_path(remote_path, sftp)
+      def create_remote_path(sftp)
         path_parts = Array.new
         remote_path.split('/').each do |path_part|
           path_parts << path_part