backup-remote 0.0.2

data/lib/backup/cloud_io/base.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+
+module Backup
+  module CloudIO
+    class Error < Backup::Error; end
+    class FileSizeError < Backup::Error; end
+
+    class Base
+      attr_reader :max_retries, :retry_waitsec
+
+      def initialize(options = {})
+        @max_retries    = options[:max_retries]
+        @retry_waitsec  = options[:retry_waitsec]
+      end
+
+      private
+
+      def with_retries(operation)
+        retries = 0
+        begin
+          yield
+        rescue => err
+          retries += 1
+          raise Error.wrap(err, <<-EOS) if retries > max_retries
+            Max Retries (#{ max_retries }) Exceeded!
+            Operation: #{ operation }
+            Be sure to check the log messages for each retry attempt.
+          EOS
+
+          Logger.info Error.wrap(err, <<-EOS)
+            Retry ##{ retries } of #{ max_retries }
+            Operation: #{ operation }
+          EOS
+          sleep(retry_waitsec)
+          retry
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/cloud_io/cloud_files.rb

@@ -0,0 +1,298 @@
+# encoding: utf-8
+require 'backup/cloud_io/base'
+require 'fog'
+require 'digest/md5'
+
+module Backup
+  module CloudIO
+    class CloudFiles < Base
+      class Error < Backup::Error; end
+
+      MAX_FILE_SIZE   = 1024**3 * 5     # 5 GiB
+      MAX_SLO_SIZE    = 1024**3 * 5000  # 1000 segments @ 5 GiB
+      SEGMENT_BUFFER  = 1024**2         # 1 MiB
+
+      attr_reader :username, :api_key, :auth_url, :region, :servicenet,
+                  :container, :segments_container, :segment_size, :days_to_keep,
+                  :fog_options
+
+      def initialize(options = {})
+        super
+
+        @username           = options[:username]
+        @api_key            = options[:api_key]
+        @auth_url           = options[:auth_url]
+        @region             = options[:region]
+        @servicenet         = options[:servicenet]
+        @container          = options[:container]
+        @segments_container = options[:segments_container]
+        @segment_size       = options[:segment_size]
+        @days_to_keep       = options[:days_to_keep]
+        @fog_options        = options[:fog_options]
+      end
+
+      # The Syncer may call this method in multiple threads,
+      # but #objects is always called before this occurs.
+      def upload(src, dest)
+        create_containers
+
+        file_size = File.size(src)
+        segment_bytes = segment_size * 1024**2
+        if segment_bytes > 0 && file_size > segment_bytes
+          raise FileSizeError, <<-EOS if file_size > MAX_SLO_SIZE
+            File Too Large
+            File: #{ src }
+            Size: #{ file_size }
+            Max SLO Size is #{ MAX_SLO_SIZE } (5 GiB * 1000 segments)
+          EOS
+
+          segment_bytes = adjusted_segment_bytes(segment_bytes, file_size)
+          segments = upload_segments(src, dest, segment_bytes, file_size)
+          upload_manifest(dest, segments)
+        else
+          raise FileSizeError, <<-EOS if file_size > MAX_FILE_SIZE
+            File Too Large
+            File: #{ src }
+            Size: #{ file_size }
+            Max File Size is #{ MAX_FILE_SIZE } (5 GiB)
+          EOS
+
+          put_object(src, dest)
+        end
+      end
+
+      # Returns all objects in the container with the given prefix.
+      #
+      # - #get_container returns a max of 10000 objects per request.
+      # - Returns objects sorted using a sqlite binary collating function.
+      # - If marker is given, only objects after the marker are in the response.
+      def objects(prefix)
+        objects = []
+        resp = nil
+        prefix = prefix.chomp('/')
+        opts = { :prefix => prefix + '/' }
+
+        create_containers
+
+        while resp.nil? || resp.body.count == 10000
+          opts.merge!(:marker => objects.last.name) unless objects.empty?
+          with_retries("GET '#{ container }/#{ prefix }/*'") do
+            resp = connection.get_container(container, opts)
+          end
+          resp.body.each do |obj_data|
+            objects << Object.new(self, obj_data)
+          end
+        end
+
+        objects
+      end
+
+      # Used by Object to fetch metadata if needed.
+      def head_object(object)
+        resp = nil
+        with_retries("HEAD '#{ container }/#{ object.name }'") do
+          resp = connection.head_object(container, object.name)
+        end
+        resp
+      end
+
+      # Delete non-SLO object(s) from the container.
+      #
+      # - Called by the Storage (with objects) and the Syncer (with names)
+      # - Deletes 10,000 objects per request.
+      # - Missing objects will be ignored.
+      def delete(objects_or_names)
+        names = Array(objects_or_names).dup
+        names.map!(&:name) if names.first.is_a?(Object)
+
+        until names.empty?
+          _names = names.slice!(0, 10000)
+          with_retries('DELETE Multiple Objects') do
+            resp = connection.delete_multiple_objects(container, _names)
+            resp_status = resp.body['Response Status']
+            raise Error, <<-EOS unless resp_status == '200 OK'
+              #{ resp_status }
+              The server returned the following:
+              #{ resp.body.inspect }
+            EOS
+          end
+        end
+      end
+
+      # Delete an SLO object(s) from the container.
+      #
+      # - Used only by the Storage. The Syncer cannot use SLOs.
+      # - Removes the SLO manifest object and all associated segments.
+      # - Missing segments will be ignored.
+      def delete_slo(objects)
+        Array(objects).each do |object|
+          with_retries("DELETE SLO Manifest '#{ container }/#{ object.name }'") do
+            resp = connection.delete_static_large_object(container, object.name)
+            resp_status = resp.body['Response Status']
+            raise Error, <<-EOS unless resp_status == '200 OK'
+              #{ resp_status }
+              The server returned the following:
+              #{ resp.body.inspect }
+            EOS
+          end
+        end
+      end
+
+      private
+
+      def connection
+        @connection ||= Fog::Storage.new({
+          :provider             => 'Rackspace',
+          :rackspace_username   => username,
+          :rackspace_api_key    => api_key,
+          :rackspace_auth_url   => auth_url,
+          :rackspace_region     => region,
+          :rackspace_servicenet => servicenet
+        }.merge(fog_options || {}))
+      end
+
+      def create_containers
+        return if @containers_created
+        @containers_created = true
+
+        with_retries('Create Containers') do
+          connection.put_container(container)
+          connection.put_container(segments_container) if segments_container
+        end
+      end
+
+      def put_object(src, dest)
+        opts = headers.merge('ETag' => Digest::MD5.file(src).hexdigest)
+        with_retries("PUT '#{ container }/#{ dest }'") do
+          File.open(src, 'r') do |file|
+            connection.put_object(container, dest, file, opts)
+          end
+        end
+      end
+
+      # Each segment is uploaded using chunked transfer encoding using
+      # SEGMENT_BUFFER, and each segment's MD5 is sent to verify the transfer.
+      # Each segment's MD5 and byte_size will also be verified when the
+      # SLO manifest object is uploaded.
+      def upload_segments(src, dest, segment_bytes, file_size)
+        total_segments = (file_size / segment_bytes.to_f).ceil
+        progress = (0.1..0.9).step(0.1).map {|n| (total_segments * n).floor }
+        Logger.info "\s\sUploading #{ total_segments } SLO Segments..."
+
+        segments = []
+        File.open(src, 'r') do |file|
+          segment_number = 0
+          until file.eof?
+            segment_number += 1
+            object = "#{ dest }/#{ segment_number.to_s.rjust(4, '0') }"
+            pos = file.pos
+            md5 = segment_md5(file, segment_bytes)
+            opts = headers.merge('ETag' => md5)
+
+            with_retries("PUT '#{ segments_container }/#{ object }'") do
+              file.seek(pos)
+              offset = 0
+              connection.put_object(segments_container, object, nil, opts) do
+                # block is called to stream data until it returns ''
+                data = ''
+                if offset <= segment_bytes - SEGMENT_BUFFER
+                  data = file.read(SEGMENT_BUFFER).to_s # nil => ''
+                  offset += data.size
+                end
+                data
+              end
+            end
+
+            segments << {
+              :path => "#{ segments_container }/#{ object }",
+              :etag => md5,
+              :size_bytes => file.pos - pos
+            }
+
+            if i = progress.rindex(segment_number)
+              Logger.info "\s\s...#{ i + 1 }0% Complete..."
+            end
+          end
+        end
+        segments
+      end
+
+      def segment_md5(file, segment_bytes)
+        md5 = Digest::MD5.new
+        offset = 0
+        while offset <= segment_bytes - SEGMENT_BUFFER
+          data = file.read(SEGMENT_BUFFER)
+          break unless data
+          offset += data.size
+          md5 << data
+        end
+        md5.hexdigest
+      end
+
+      # Each segment's ETag and byte_size will be verified once uploaded.
+      # Request will raise an exception if verification fails or segments
+      # are not found. However, each segment's ETag was verified when we
+      # uploaded the segments, so this should only retry failed requests.
+      def upload_manifest(dest, segments)
+        Logger.info "\s\sStoring SLO Manifest '#{ container }/#{ dest }'"
+
+        with_retries("PUT SLO Manifest '#{ container }/#{ dest }'") do
+          connection.put_static_obj_manifest(container, dest, segments, headers)
+        end
+      end
+
+      # If :days_to_keep was set, each object will be scheduled for deletion.
+      # This includes non-SLO objects, the SLO manifest and all segments.
+      def headers
+        headers = {}
+        headers.merge!('X-Delete-At' => delete_at) if delete_at
+        headers
+      end
+
+      def delete_at
+        return unless days_to_keep
+        @delete_at ||= (Time.now.utc + days_to_keep * 60**2 * 24).to_i
+      end
+
+      def adjusted_segment_bytes(segment_bytes, file_size)
+        return segment_bytes if file_size / segment_bytes.to_f <= 1000
+
+        mb = orig_mb = segment_bytes / 1024**2
+        mb += 1 until file_size / (1024**2 * mb).to_f <= 1000
+        Logger.warn Error.new(<<-EOS)
+          Segment Size Adjusted
+          Your original #segment_size of #{ orig_mb } MiB has been adjusted
+          to #{ mb } MiB in order to satisfy the limit of 1000 segments.
+          To enforce your chosen #segment_size, you should use the Splitter.
+          e.g. split_into_chunks_of #{ mb * 1000 } (#segment_size * 1000)
+        EOS
+        1024**2 * mb
+      end
+
+      class Object
+        attr_reader :name, :hash
+
+        def initialize(cloud_io, data)
+          @cloud_io = cloud_io
+          @name = data['name']
+          @hash = data['hash']
+        end
+
+        def slo?
+          !!metadata['X-Static-Large-Object']
+        end
+
+        def marked_for_deletion?
+          !!metadata['X-Delete-At']
+        end
+
+        private
+
+        def metadata
+          @metadata ||= @cloud_io.head_object(self).headers
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/cloud_io/s3.rb

@@ -0,0 +1,260 @@
+# encoding: utf-8
+require 'backup/cloud_io/base'
+require 'fog'
+require 'digest/md5'
+require 'base64'
+require 'stringio'
+
+module Backup
+  module CloudIO
+    class S3 < Base
+      class Error < Backup::Error; end
+
+      MAX_FILE_SIZE       = 1024**3 * 5   # 5 GiB
+      MAX_MULTIPART_SIZE  = 1024**4 * 5   # 5 TiB
+
+      attr_reader :access_key_id, :secret_access_key, :use_iam_profile,
+                  :region, :bucket, :chunk_size, :encryption, :storage_class,
+                  :fog_options
+
+      def initialize(options = {})
+        super
+
+        @access_key_id      = options[:access_key_id]
+        @secret_access_key  = options[:secret_access_key]
+        @use_iam_profile    = options[:use_iam_profile]
+        @region             = options[:region]
+        @bucket             = options[:bucket]
+        @chunk_size         = options[:chunk_size]
+        @encryption         = options[:encryption]
+        @storage_class      = options[:storage_class]
+        @fog_options        = options[:fog_options]
+      end
+
+      # The Syncer may call this method in multiple threads.
+      # However, #objects is always called prior to multithreading.
+      def upload(src, dest)
+        file_size = File.size(src)
+        chunk_bytes = chunk_size * 1024**2
+        if chunk_bytes > 0 && file_size > chunk_bytes
+          raise FileSizeError, <<-EOS if file_size > MAX_MULTIPART_SIZE
+            File Too Large
+            File: #{ src }
+            Size: #{ file_size }
+            Max Multipart Upload Size is #{ MAX_MULTIPART_SIZE } (5 TiB)
+          EOS
+
+          chunk_bytes = adjusted_chunk_bytes(chunk_bytes, file_size)
+          upload_id = initiate_multipart(dest)
+          parts = upload_parts(src, dest, upload_id, chunk_bytes, file_size)
+          complete_multipart(dest, upload_id, parts)
+        else
+          raise FileSizeError, <<-EOS if file_size > MAX_FILE_SIZE
+            File Too Large
+            File: #{ src }
+            Size: #{ file_size }
+            Max File Size is #{ MAX_FILE_SIZE } (5 GiB)
+          EOS
+
+          put_object(src, dest)
+        end
+      end
+
+      # Returns all objects in the bucket with the given prefix.
+      #
+      # - #get_bucket returns a max of 1000 objects per request.
+      # - Returns objects in alphabetical order.
+      # - If marker is given, only objects after the marker are in the response.
+      def objects(prefix)
+        objects = []
+        resp = nil
+        prefix = prefix.chomp('/')
+        opts = { 'prefix' => prefix + '/' }
+
+        while resp.nil? || resp.body['IsTruncated']
+          opts.merge!('marker' => objects.last.key) unless objects.empty?
+          with_retries("GET '#{ bucket }/#{ prefix }/*'") do
+            resp = connection.get_bucket(bucket, opts)
+          end
+          resp.body['Contents'].each do |obj_data|
+            objects << Object.new(self, obj_data)
+          end
+        end
+
+        objects
+      end
+
+      # Used by Object to fetch metadata if needed.
+      def head_object(object)
+        resp = nil
+        with_retries("HEAD '#{ bucket }/#{ object.key }'") do
+          resp = connection.head_object(bucket, object.key)
+        end
+        resp
+      end
+
+      # Delete object(s) from the bucket.
+      #
+      # - Called by the Storage (with objects) and the Syncer (with keys)
+      # - Deletes 1000 objects per request.
+      # - Missing objects will be ignored.
+      def delete(objects_or_keys)
+        keys = Array(objects_or_keys).dup
+        keys.map!(&:key) if keys.first.is_a?(Object)
+
+        opts = { :quiet => true } # only report Errors in DeleteResult
+        until keys.empty?
+          _keys = keys.slice!(0, 1000)
+          with_retries('DELETE Multiple Objects') do
+            resp = connection.delete_multiple_objects(bucket, _keys, opts.dup)
+            unless resp.body['DeleteResult'].empty?
+              errors = resp.body['DeleteResult'].map do |result|
+                error = result['Error']
+                "Failed to delete: #{ error['Key'] }\n" +
+                "Reason: #{ error['Code'] }: #{ error['Message'] }"
+              end.join("\n")
+              raise Error, "The server returned the following:\n#{ errors }"
+            end
+          end
+        end
+      end
+
+      private
+
+      def connection
+        @connection ||= begin
+          opts = { :provider => 'AWS', :region => region }
+          if use_iam_profile
+            opts.merge!(:use_iam_profile => true)
+          else
+            opts.merge!(
+              :aws_access_key_id      => access_key_id,
+              :aws_secret_access_key  => secret_access_key
+            )
+          end
+          opts.merge!(fog_options || {})
+          conn = Fog::Storage.new(opts)
+          conn.sync_clock
+          conn
+        end
+      end
+
+      def put_object(src, dest)
+        md5 = Base64.encode64(Digest::MD5.file(src).digest).chomp
+        options = headers.merge('Content-MD5' => md5)
+        with_retries("PUT '#{ bucket }/#{ dest }'") do
+          File.open(src, 'r') do |file|
+            connection.put_object(bucket, dest, file, options)
+          end
+        end
+      end
+
+      def initiate_multipart(dest)
+        Logger.info "\s\sInitiate Multipart '#{ bucket }/#{ dest }'"
+
+        resp = nil
+        with_retries("POST '#{ bucket }/#{ dest }' (Initiate)") do
+          resp = connection.initiate_multipart_upload(bucket, dest, headers)
+        end
+        resp.body['UploadId']
+      end
+
+      # Each part's MD5 is sent to verify the transfer.
+      # AWS will concatenate all parts into a single object
+      # once the multipart upload is completed.
+      def upload_parts(src, dest, upload_id, chunk_bytes, file_size)
+        total_parts = (file_size / chunk_bytes.to_f).ceil
+        progress = (0.1..0.9).step(0.1).map {|n| (total_parts * n).floor }
+        Logger.info "\s\sUploading #{ total_parts } Parts..."
+
+        parts = []
+        File.open(src, 'r') do |file|
+          part_number = 0
+          while data = file.read(chunk_bytes)
+            part_number += 1
+            md5 = Base64.encode64(Digest::MD5.digest(data)).chomp
+
+            with_retries("PUT '#{ bucket }/#{ dest }' Part ##{ part_number }") do
+              resp = connection.upload_part(
+                bucket, dest, upload_id, part_number, StringIO.new(data),
+                { 'Content-MD5' => md5 }
+              )
+              parts << resp.headers['ETag']
+            end
+
+            if i = progress.rindex(part_number)
+              Logger.info "\s\s...#{ i + 1 }0% Complete..."
+            end
+          end
+        end
+        parts
+      end
+
+      def complete_multipart(dest, upload_id, parts)
+        Logger.info "\s\sComplete Multipart '#{ bucket }/#{ dest }'"
+
+        with_retries("POST '#{ bucket }/#{ dest }' (Complete)") do
+          resp = connection.complete_multipart_upload(bucket, dest, upload_id, parts)
+          raise Error, <<-EOS if resp.body['Code']
+            The server returned the following error:
+            #{ resp.body['Code'] }: #{ resp.body['Message'] }
+          EOS
+        end
+      end
+
+      def headers
+        headers = {}
+
+        enc = encryption.to_s.upcase
+        headers.merge!(
+          { 'x-amz-server-side-encryption' => enc}
+        ) unless enc.empty?
+
+        sc = storage_class.to_s.upcase
+        headers.merge!(
+          { 'x-amz-storage-class' => sc }
+        ) unless sc.empty? || sc == 'STANDARD'
+
+        headers
+      end
+
+      def adjusted_chunk_bytes(chunk_bytes, file_size)
+        return chunk_bytes if file_size / chunk_bytes.to_f <= 10_000
+
+        mb = orig_mb = chunk_bytes / 1024**2
+        mb += 1 until file_size / (1024**2 * mb).to_f <= 10_000
+        Logger.warn Error.new(<<-EOS)
+          Chunk Size Adjusted
+          Your original #chunk_size of #{ orig_mb } MiB has been adjusted
+          to #{ mb } MiB in order to satisfy the limit of 10,000 chunks.
+          To enforce your chosen #chunk_size, you should use the Splitter.
+          e.g. split_into_chunks_of #{ mb * 10_000 } (#chunk_size * 10_000)
+        EOS
+        1024**2 * mb
+      end
+
+      class Object
+        attr_reader :key, :etag, :storage_class
+
+        def initialize(cloud_io, data)
+          @cloud_io = cloud_io
+          @key  = data['Key']
+          @etag = data['ETag']
+          @storage_class = data['StorageClass']
+        end
+
+        # currently 'AES256' or nil
+        def encryption
+          metadata['x-amz-server-side-encryption']
+        end
+
+        private
+
+        def metadata
+          @metadata ||= @cloud_io.head_object(self).headers
+        end
+      end
+
+    end
+  end
+end