backedup 5.0.0.beta.3

data/lib/backup/cloud_io/gcs.rb

@@ -0,0 +1,121 @@
+# encoding: utf-8
+require "backup/cloud_io/base"
+require "fog"
+
+module Backup
+  module CloudIO
+    class GCS < Base
+      class Error < Backup::Error; end
+
+      MAX_FILE_SIZE = 1024**5 * 5 # 5 TiB
+
+      attr_reader :google_storage_access_key_id, :google_storage_secret_access_key,
+        :bucket, :fog_options
+
+      def initialize(options = {})
+        super
+
+        @google_storage_access_key_id     = options[:google_storage_access_key_id]
+        @google_storage_secret_access_key = options[:google_storage_secret_access_key]
+        @bucket                           = options[:bucket]
+        @fog_options                      = options[:fog_options]
+      end
+
+      # The Syncer may call this method in multiple threads.
+      # However, #objects is always called prior to multithreading.
+      def upload(src, dest)
+        file_size = File.size(src)
+        raise FileSizeError, <<-EOS if file_size > MAX_FILE_SIZE
+            File Too Large
+            File: #{src}
+            Size: #{file_size}
+            Max File Size is #{MAX_FILE_SIZE} (5 GiB)
+          EOS
+
+        put_object(src, dest)
+      end
+
+      # Returns all objects in the bucket with the given prefix.
+      #
+      # - #get_bucket returns a max of 1000 objects per request.
+      # - Returns objects in alphabetical order.
+      # - If marker is given, only objects after the marker are in the response.
+      def objects(prefix)
+        objects = []
+        resp = nil
+        prefix = prefix.chomp("/")
+        opts = { "prefix" => prefix + "/" }
+
+        while resp.nil? || resp.body["IsTruncated"]
+          opts["marker"] = objects.last.key unless objects.empty?
+          with_retries("GET '#{bucket}/#{prefix}/*'") do
+            resp = connection.get_bucket(bucket, opts)
+          end
+          resp.body["Contents"].each do |obj_data|
+            objects << Object.new(self, obj_data)
+          end
+        end
+
+        objects
+      end
+
+      # Delete object(s) from the bucket.
+      #
+      # - Called by the Storage (with objects) and the Syncer (with keys)
+      # - Missing objects will be ignored.
+      def delete(objects_or_keys)
+        keys = Array(objects_or_keys).dup
+        keys.map!(&:key) if keys.first.is_a?(Object)
+        keys.each do |key|
+          with_retries("DELETE object") do
+            begin
+              connection.delete(bucket, key)
+            rescue StandardError => e
+              raise Error, "The server returned the following:\n#{e.message}"
+            end
+          end
+        end
+      end
+
+      private
+
+      def connection
+        @connection ||=
+          begin
+            opts = { provider: "Google",
+                     google_storage_access_key_id: google_storage_access_key_id,
+                     google_storage_secret_access_key: google_storage_secret_access_key }
+
+            opts.merge!(fog_options || {})
+            conn = Fog::Storage.new(opts)
+            conn
+          end
+      end
+
+      def put_object(src, dest)
+        md5 = Base64.encode64(Digest::MD5.file(src).digest).chomp
+        options = { "Content-MD5" => md5 }
+        with_retries("PUT '#{bucket}/#{dest}'") do
+          File.open(src, "r") do |file|
+            begin
+              connection.put_object(bucket, dest, file, options)
+            rescue StandardError => e
+              raise Error, "The server returned the following:\n#{e.message}\n"
+            end
+          end
+        end
+      end
+
+      class Object
+        attr_reader :key, :etag, :storage_class
+
+        def initialize(cloud_io, data)
+          @cloud_io = cloud_io
+          @key  = data["Key"]
+          @etag = data["ETag"]
+          @storage_class = data["StorageClass"]
+        end
+      end
+    end
+  end
+end

data/lib/backup/cloud_io/s3.rb

@@ -0,0 +1,253 @@
+require "backup/cloud_io/base"
+require "fog"
+require "digest/md5"
+require "base64"
+require "stringio"
+
+module Backup
+  module CloudIO
+    class S3 < Base
+      class Error < Backup::Error; end
+
+      MAX_FILE_SIZE       = 1024**3 * 5   # 5 GiB
+      MAX_MULTIPART_SIZE  = 1024**4 * 5   # 5 TiB
+
+      attr_reader :access_key_id, :secret_access_key, :use_iam_profile,
+        :region, :bucket, :chunk_size, :encryption, :storage_class,
+        :fog_options
+
+      def initialize(options = {})
+        super
+
+        @access_key_id      = options[:access_key_id]
+        @secret_access_key  = options[:secret_access_key]
+        @use_iam_profile    = options[:use_iam_profile]
+        @region             = options[:region]
+        @bucket             = options[:bucket]
+        @chunk_size         = options[:chunk_size]
+        @encryption         = options[:encryption]
+        @storage_class      = options[:storage_class]
+        @fog_options        = options[:fog_options]
+      end
+
+      # The Syncer may call this method in multiple threads.
+      # However, #objects is always called prior to multithreading.
+      def upload(src, dest)
+        file_size = File.size(src)
+        chunk_bytes = chunk_size * 1024**2
+        if chunk_bytes > 0 && file_size > chunk_bytes
+          raise FileSizeError, <<-EOS if file_size > MAX_MULTIPART_SIZE
+            File Too Large
+            File: #{src}
+            Size: #{file_size}
+            Max Multipart Upload Size is #{MAX_MULTIPART_SIZE} (5 TiB)
+          EOS
+
+          chunk_bytes = adjusted_chunk_bytes(chunk_bytes, file_size)
+          upload_id = initiate_multipart(dest)
+          parts = upload_parts(src, dest, upload_id, chunk_bytes, file_size)
+          complete_multipart(dest, upload_id, parts)
+        else
+          raise FileSizeError, <<-EOS if file_size > MAX_FILE_SIZE
+            File Too Large
+            File: #{src}
+            Size: #{file_size}
+            Max File Size is #{MAX_FILE_SIZE} (5 GiB)
+          EOS
+
+          put_object(src, dest)
+        end
+      end
+
+      # Returns all objects in the bucket with the given prefix.
+      #
+      # - #get_bucket returns a max of 1000 objects per request.
+      # - Returns objects in alphabetical order.
+      # - If marker is given, only objects after the marker are in the response.
+      def objects(prefix)
+        objects = []
+        resp = nil
+        prefix = prefix.chomp("/")
+        opts = { "prefix" => prefix + "/" }
+
+        while resp.nil? || resp.body["IsTruncated"]
+          opts["marker"] = objects.last.key unless objects.empty?
+          with_retries("GET '#{bucket}/#{prefix}/*'") do
+            resp = connection.get_bucket(bucket, opts)
+          end
+          resp.body["Contents"].each do |obj_data|
+            objects << Object.new(self, obj_data)
+          end
+        end
+
+        objects
+      end
+
+      # Used by Object to fetch metadata if needed.
+      def head_object(object)
+        resp = nil
+        with_retries("HEAD '#{bucket}/#{object.key}'") do
+          resp = connection.head_object(bucket, object.key)
+        end
+        resp
+      end
+
+      # Delete object(s) from the bucket.
+      #
+      # - Called by the Storage (with objects) and the Syncer (with keys)
+      # - Deletes 1000 objects per request.
+      # - Missing objects will be ignored.
+      def delete(objects_or_keys)
+        keys = Array(objects_or_keys).dup
+        keys.map!(&:key) if keys.first.is_a?(Object)
+
+        opts = { quiet: true } # only report Errors in DeleteResult
+        until keys.empty?
+          keys_partial = keys.slice!(0, 1000)
+          with_retries("DELETE Multiple Objects") do
+            resp = connection.delete_multiple_objects(bucket, keys_partial, opts.dup)
+            unless resp.body["DeleteResult"].empty?
+              errors = resp.body["DeleteResult"].map do |result|
+                error = result["Error"]
+                "Failed to delete: #{error["Key"]}\n" \
+                  "Reason: #{error["Code"]}: #{error["Message"]}"
+              end.join("\n")
+              raise Error, "The server returned the following:\n#{errors}"
+            end
+          end
+        end
+      end
+
+      private
+
+      def connection
+        @connection ||=
+          begin
+            opts = { provider: "AWS", region: region }
+            if use_iam_profile
+              opts[:use_iam_profile] = true
+            else
+              opts[:aws_access_key_id] = access_key_id
+              opts[:aws_secret_access_key] = secret_access_key
+            end
+            opts.merge!(fog_options || {})
+            conn = Fog::Storage.new(opts)
+            conn.sync_clock
+            conn
+          end
+      end
+
+      def put_object(src, dest)
+        md5 = Base64.encode64(Digest::MD5.file(src).digest).chomp
+        options = headers.merge("Content-MD5" => md5)
+        with_retries("PUT '#{bucket}/#{dest}'") do
+          File.open(src, "r") do |file|
+            connection.put_object(bucket, dest, file, options)
+          end
+        end
+      end
+
+      def initiate_multipart(dest)
+        Logger.info "\s\sInitiate Multipart '#{bucket}/#{dest}'"
+
+        resp = nil
+        with_retries("POST '#{bucket}/#{dest}' (Initiate)") do
+          resp = connection.initiate_multipart_upload(bucket, dest, headers)
+        end
+        resp.body["UploadId"]
+      end
+
+      # Each part's MD5 is sent to verify the transfer.
+      # AWS will concatenate all parts into a single object
+      # once the multipart upload is completed.
+      def upload_parts(src, dest, upload_id, chunk_bytes, file_size)
+        total_parts = (file_size / chunk_bytes.to_f).ceil
+        progress = (0.1..0.9).step(0.1).map { |n| (total_parts * n).floor }
+        Logger.info "\s\sUploading #{total_parts} Parts..."
+
+        parts = []
+        File.open(src, "r") do |file|
+          part_number = 0
+          while data = file.read(chunk_bytes)
+            part_number += 1
+            md5 = Base64.encode64(Digest::MD5.digest(data)).chomp
+
+            with_retries("PUT '#{bucket}/#{dest}' Part ##{part_number}") do
+              resp = connection.upload_part(
+                bucket, dest, upload_id, part_number, StringIO.new(data),
+                "Content-MD5" => md5
+              )
+              parts << resp.headers["ETag"]
+            end
+
+            if i = progress.rindex(part_number)
+              Logger.info "\s\s...#{i + 1}0% Complete..."
+            end
+          end
+        end
+        parts
+      end
+
+      def complete_multipart(dest, upload_id, parts)
+        Logger.info "\s\sComplete Multipart '#{bucket}/#{dest}'"
+
+        with_retries("POST '#{bucket}/#{dest}' (Complete)") do
+          resp = connection.complete_multipart_upload(bucket, dest, upload_id, parts)
+          raise Error, <<-EOS if resp.body["Code"]
+            The server returned the following error:
+            #{resp.body["Code"]}: #{resp.body["Message"]}
+          EOS
+        end
+      end
+
+      def headers
+        headers = {}
+
+        enc = encryption.to_s.upcase
+        headers["x-amz-server-side-encryption"] = enc unless enc.empty?
+
+        sc = storage_class.to_s.upcase
+        headers["x-amz-storage-class"] = sc unless sc.empty? || sc == "STANDARD"
+
+        headers
+      end
+
+      def adjusted_chunk_bytes(chunk_bytes, file_size)
+        return chunk_bytes if file_size / chunk_bytes.to_f <= 10_000
+
+        mb = orig_mb = chunk_bytes / 1024**2
+        mb += 1 until file_size / (1024**2 * mb).to_f <= 10_000
+        Logger.warn Error.new(<<-EOS)
+          Chunk Size Adjusted
+          Your original #chunk_size of #{orig_mb} MiB has been adjusted
+          to #{mb} MiB in order to satisfy the limit of 10,000 chunks.
+          To enforce your chosen #chunk_size, you should use the Splitter.
+          e.g. split_into_chunks_of #{mb * 10_000} (#chunk_size * 10_000)
+        EOS
+        1024**2 * mb
+      end
+
+      class Object
+        attr_reader :key, :etag, :storage_class
+
+        def initialize(cloud_io, data)
+          @cloud_io = cloud_io
+          @key  = data["Key"]
+          @etag = data["ETag"]
+          @storage_class = data["StorageClass"]
+        end
+
+        # currently 'AES256' or nil
+        def encryption
+          metadata["x-amz-server-side-encryption"]
+        end
+
+        private
+
+        def metadata
+          @metadata ||= @cloud_io.head_object(self).headers
+        end
+      end
+    end
+  end
+end

data/lib/backup/cloud_io/swift.rb

@@ -0,0 +1,96 @@
+require "backup/cloud_io/base"
+require "fog/openstack"
+
+LARGE_FILE = 5 * 1024**3 - 1
+
+module Backup
+  module CloudIO
+    class Swift < Base
+      class Error < Backup::Error; end
+      # Handle fog-openstack namespace change, as they moved everything under
+      # the OpenStack namespace starting at version 1.0
+      Storage = if Fog::Storage.const_defined? :OpenStack
+                  Fog::Storage::OpenStack
+                else
+                  Fog::OpenStack::Storage
+                end
+
+      attr_reader :username, :password, :tenant, :region,
+        :container, :auth_url, :max_retries,
+        :retry_waitsec, :fog_options, :batch_size
+
+      def initialize(opts = {})
+        super
+
+        @username       = opts[:username]
+        @password       = opts[:password]
+        @tenant         = opts[:tenant_name]
+        @container      = opts[:container]
+        @auth_url       = opts[:auth_url]
+        @region         = opts[:region]
+        @max_retries    = opts[:max_retries]
+        @retry_waitsec  = opts[:retry_waitsec]
+        @batch_size     = opts[:batch_size]
+        @fog_options    = opts[:fog_options]
+      end
+
+      def upload(src, dest)
+        file_size = File.size(src)
+
+        raise FileSizeError, <<-EOS if file_size > LARGE_FILE
+          [FIXME] File Too Large
+          File: #{src}
+          Size: #{file_size}
+          Max Swift Upload Size is #{LARGE_FILE} (5 Gb) (FIXME)
+        EOS
+
+        directory.files.create key: dest, body: File.open(src)
+      end
+
+      def delete(objects_or_keys)
+        keys = Array(objects_or_keys).dup
+        keys = keys.map(&:key) unless keys.first.is_a?(String)
+
+        until keys.empty?
+          key = keys.slice!(0, batch_size)
+          with_retries("DELETE Multiple Objects") do
+            resp = connection.delete_multiple_objects(container, key)
+            if resp.data[:status] != 200
+              raise Error, <<-EOS
+                Failed to delete.
+                Status = #{resp.data[:status]}
+                Reason = #{resp.data[:reason_phrase]}
+                Body = #{resp.data[:body]}
+              EOS
+            end
+          end
+        end
+      end
+
+      def objects(prefix)
+        directory.files.all(prefix: prefix.chomp("/") + "/")
+      end
+
+      private
+
+      def directory
+        @directory ||= connection.directories.get container
+      end
+
+      def connection
+        @connection ||= begin
+          opts = {
+            openstack_auth_url: auth_url,
+            openstack_username: username,
+            openstack_api_key: password
+          }
+          opts[:openstack_region] = region unless region.nil?
+          opts[:openstack_tenant] = tenant unless tenant.nil?
+
+          opts.merge!(fog_options || {})
+          Storage.new(opts)
+        end
+      end
+    end
+  end
+end

data/lib/backup/compressor/base.rb

@@ -0,0 +1,32 @@
+module Backup
+  module Compressor
+    class Base
+      include Utilities::Helpers
+      include Config::Helpers
+
+      ##
+      # Yields to the block the compressor command and filename extension.
+      def compress_with
+        log!
+        yield @cmd, @ext
+      end
+
+      private
+
+      ##
+      # Return the compressor name, with Backup namespace removed
+      def compressor_name
+        self.class.to_s.sub("Backup::", "")
+      end
+
+      ##
+      # Logs a message to the console and log file to inform
+      # the client that Backup is using the compressor
+      def log!
+        Logger.info "Using #{compressor_name} for compression.\n" \
+          "  Command: '#{@cmd}'\n" \
+          "  Ext: '#{@ext}'"
+      end
+    end
+  end
+end

data/lib/backup/compressor/bzip2.rb

@@ -0,0 +1,35 @@
+module Backup
+  module Compressor
+    class Bzip2 < Base
+      ##
+      # Specify the level of compression to use.
+      #
+      # Values should be a single digit from 1 to 9.
+      # Note that setting the level to either extreme may or may not
+      # give the desired result. Be sure to check the documentation
+      # for the compressor being used.
+      #
+      # The default `level` is 9.
+      attr_accessor :level
+
+      ##
+      # Creates a new instance of Backup::Compressor::Bzip2
+      def initialize(&block)
+        load_defaults!
+
+        @level ||= false
+
+        instance_eval(&block) if block_given?
+
+        @cmd = "#{utility(:bzip2)}#{options}"
+        @ext = ".bz2"
+      end
+
+      private
+
+      def options
+        " -#{@level}" if @level
+      end
+    end
+  end
+end

data/lib/backup/compressor/custom.rb

@@ -0,0 +1,49 @@
+module Backup
+  module Compressor
+    class Custom < Base
+      ##
+      # Specify the system command to invoke a compressor,
+      # including any command-line arguments.
+      # e.g. @compressor.command = 'pbzip2 -p2 -4'
+      #
+      # The data to be compressed will be piped to the command's STDIN,
+      # and it should write the compressed data to STDOUT.
+      # i.e. `cat file.tar | %command% > file.tar.%extension%`
+      attr_accessor :command
+
+      ##
+      # File extension to append to the compressed file's filename.
+      # e.g. @compressor.extension = '.bz2'
+      attr_accessor :extension
+
+      ##
+      # Initializes a new custom compressor.
+      def initialize(&block)
+        load_defaults!
+
+        instance_eval(&block) if block_given?
+
+        @cmd = set_cmd
+        @ext = set_ext
+      end
+
+      private
+
+      ##
+      # Return the command line using the full path.
+      # Ensures the command exists and is executable.
+      def set_cmd
+        parts = @command.to_s.split(" ")
+        parts[0] = utility(parts[0])
+        parts.join(" ")
+      end
+
+      ##
+      # Return the extension given without whitespace.
+      # If extension was not set, return an empty string
+      def set_ext
+        @extension.to_s.strip
+      end
+    end
+  end
+end

data/lib/backup/compressor/gzip.rb

@@ -0,0 +1,73 @@
+module Backup
+  module Compressor
+    class Gzip < Base
+      class Error < Backup::Error; end
+      extend Utilities::Helpers
+
+      ##
+      # Specify the level of compression to use.
+      #
+      # Values should be a single digit from 1 to 9.
+      # Note that setting the level to either extreme may or may not
+      # give the desired result. Be sure to check the documentation
+      # for the compressor being used.
+      #
+      # The default `level` is 6.
+      attr_accessor :level
+
+      ##
+      # Use the `--rsyncable` option with `gzip`.
+      #
+      # This option directs `gzip` to compress data using an algorithm that
+      # allows `rsync` to efficiently detect changes. This is especially useful
+      # when used to compress `Archive` or `Database` backups that will be
+      # stored using Backup's `RSync` Storage option.
+      #
+      # The `--rsyncable` option is only available on patched versions of `gzip`.
+      # While most distributions apply this patch, this option may not be
+      # available on your system. If it's not available, Backup will log a
+      # warning and continue to use the compressor without this option.
+      attr_accessor :rsyncable
+
+      ##
+      # Determine if +--rsyncable+ is supported and cache the result.
+      def self.has_rsyncable?
+        return @has_rsyncable unless @has_rsyncable.nil?
+        cmd = "#{utility(:gzip)} --rsyncable --version >/dev/null 2>&1; echo $?"
+        @has_rsyncable = `#{cmd}`.chomp == "0"
+      end
+
+      ##
+      # Creates a new instance of Backup::Compressor::Gzip
+      def initialize(&block)
+        load_defaults!
+
+        @level ||= false
+        @rsyncable ||= false
+
+        instance_eval(&block) if block_given?
+
+        @cmd = "#{utility(:gzip)}#{options}"
+        @ext = ".gz"
+      end
+
+      private
+
+      def options
+        opts = ""
+        opts << " -#{@level}" if @level
+        if @rsyncable
+          if self.class.has_rsyncable?
+            opts << " --rsyncable"
+          else
+            Logger.warn Error.new(<<-EOS)
+              'rsyncable' option ignored.
+              Your system's 'gzip' does not support the `--rsyncable` option.
+            EOS
+          end
+        end
+        opts
+      end
+    end
+  end
+end

data/lib/backup/compressor/pbzip2.rb

@@ -0,0 +1,45 @@
+module Backup
+  module Compressor
+    class PBzip2 < Base
+      ##
+      # Specify the level of compression to use.
+      #
+      # Values should be a single digit from 1 to 9.
+      # Note that setting the level to either extreme may or may not
+      # give the desired result. Be sure to check the documentation
+      # for the compressor being used.
+      #
+      # The default `level` is 9.
+      attr_accessor :level
+
+      ##
+      # The number of processors to be used with Parallel BZIP2
+      # (pbzip2).
+      #
+      # Default is autodetection to use all.
+      attr_accessor :processors
+
+      ##
+      # Creates a new instance of Backup::Compressor::PBzip2
+      def initialize(&block)
+        load_defaults!
+
+        @level ||= false
+
+        instance_eval(&block) if block_given?
+
+        @cmd = "#{utility(:pbzip2)}#{options}"
+        @ext = ".bz2"
+      end
+
+      private
+
+      def options
+        o = ""
+        o << " -#{@level}" if @level
+        o << " -p#{@processors}" if @processors
+        o
+      end
+    end
+  end
+end