backup 3.6.0 → 3.7.0

data/lib/backup/cloud_io/s3.rb

@@ -0,0 +1,252 @@
+# encoding: utf-8
+require 'backup/cloud_io/base'
+require 'fog'
+require 'digest/md5'
+require 'base64'
+require 'stringio'
+
+module Backup
+  module CloudIO
+    class S3 < Base
+      class Error < Backup::Error; end
+
+      MAX_FILE_SIZE       = 1024**3 * 5   # 5 GiB
+      MAX_MULTIPART_SIZE  = 1024**4 * 5   # 5 TiB
+
+      attr_reader :access_key_id, :secret_access_key, :region, :bucket,
+                  :chunk_size, :encryption, :storage_class
+
+      def initialize(options = {})
+        super
+
+        @access_key_id      = options[:access_key_id]
+        @secret_access_key  = options[:secret_access_key]
+        @region             = options[:region]
+        @bucket             = options[:bucket]
+        @chunk_size         = options[:chunk_size]
+        @encryption         = options[:encryption]
+        @storage_class      = options[:storage_class]
+      end
+
+      # The Syncer may call this method in multiple threads.
+      # However, #objects is always called prior to multithreading.
+      def upload(src, dest)
+        file_size = File.size(src)
+        chunk_bytes = chunk_size * 1024**2
+        if chunk_bytes > 0 && file_size > chunk_bytes
+          raise FileSizeError, <<-EOS if file_size > MAX_MULTIPART_SIZE
+            File Too Large
+            File: #{ src }
+            Size: #{ file_size }
+            Max Multipart Upload Size is #{ MAX_MULTIPART_SIZE } (5 TiB)
+          EOS
+
+          chunk_bytes = adjusted_chunk_bytes(chunk_bytes, file_size)
+          upload_id = initiate_multipart(dest)
+          parts = upload_parts(src, dest, upload_id, chunk_bytes, file_size)
+          complete_multipart(dest, upload_id, parts)
+        else
+          raise FileSizeError, <<-EOS if file_size > MAX_FILE_SIZE
+            File Too Large
+            File: #{ src }
+            Size: #{ file_size }
+            Max File Size is #{ MAX_FILE_SIZE } (5 GiB)
+          EOS
+
+          put_object(src, dest)
+        end
+      end
+
+      # Returns all objects in the bucket with the given prefix.
+      #
+      # - #get_bucket returns a max of 1000 objects per request.
+      # - Returns objects in alphabetical order.
+      # - If marker is given, only objects after the marker are in the response.
+      def objects(prefix)
+        objects = []
+        resp = nil
+        prefix = prefix.chomp('/')
+        opts = { :prefix => prefix + '/' }
+
+        while resp.nil? || resp.body['IsTruncated']
+          opts.merge!(:marker => objects.last.key) unless objects.empty?
+          with_retries("GET '#{ bucket }/#{ prefix }/*'") do
+            resp = connection.get_bucket(bucket, opts)
+          end
+          resp.body['Contents'].each do |obj_data|
+            objects << Object.new(self, obj_data)
+          end
+        end
+
+        objects
+      end
+
+      # Used by Object to fetch metadata if needed.
+      def head_object(object)
+        resp = nil
+        with_retries("HEAD '#{ bucket }/#{ object.key }'") do
+          resp = connection.head_object(bucket, object.key)
+        end
+        resp
+      end
+
+      # Delete object(s) from the bucket.
+      #
+      # - Called by the Storage (with objects) and the Syncer (with keys)
+      # - Deletes 1000 objects per request.
+      # - Missing objects will be ignored.
+      def delete(objects_or_keys)
+        keys = Array(objects_or_keys).dup
+        keys.map!(&:key) if keys.first.is_a?(Object)
+
+        opts = { :quiet => true } # only report Errors in DeleteResult
+        until keys.empty?
+          _keys = keys.slice!(0, 1000)
+          with_retries('DELETE Multiple Objects') do
+            resp = connection.delete_multiple_objects(bucket, _keys, opts)
+            unless resp.body['DeleteResult'].empty?
+              errors = resp.body['DeleteResult'].map do |result|
+                error = result['Error']
+                "Failed to delete: #{ error['Key'] }\n" +
+                "Reason: #{ error['Code'] }: #{ error['Message'] }"
+              end.join("\n")
+              raise Error, "The server returned the following:\n#{ errors }"
+            end
+          end
+        end
+      end
+
+      private
+
+      def connection
+        @connection ||= begin
+          conn = Fog::Storage.new(
+            :provider               => 'AWS',
+            :aws_access_key_id      => access_key_id,
+            :aws_secret_access_key  => secret_access_key,
+            :region                 => region
+          )
+          conn.sync_clock
+          conn
+        end
+      end
+
+      def put_object(src, dest)
+        md5 = Base64.encode64(Digest::MD5.file(src).digest).chomp
+        options = headers.merge('Content-MD5' => md5)
+        with_retries("PUT '#{ bucket }/#{ dest }'") do
+          File.open(src, 'r') do |file|
+            connection.put_object(bucket, dest, file, options)
+          end
+        end
+      end
+
+      def initiate_multipart(dest)
+        Logger.info "\s\sInitiate Multipart '#{ bucket }/#{ dest }'"
+
+        resp = nil
+        with_retries("POST '#{ bucket }/#{ dest }' (Initiate)") do
+          resp = connection.initiate_multipart_upload(bucket, dest, headers)
+        end
+        resp.body['UploadId']
+      end
+
+      # Each part's MD5 is sent to verify the transfer.
+      # AWS will concatenate all parts into a single object
+      # once the multipart upload is completed.
+      def upload_parts(src, dest, upload_id, chunk_bytes, file_size)
+        total_parts = (file_size / chunk_bytes.to_f).ceil
+        progress = (0.1..0.9).step(0.1).map {|n| (total_parts * n).floor }
+        Logger.info "\s\sUploading #{ total_parts } Parts..."
+
+        parts = []
+        File.open(src, 'r') do |file|
+          part_number = 0
+          while data = file.read(chunk_bytes)
+            part_number += 1
+            md5 = Base64.encode64(Digest::MD5.digest(data)).chomp
+
+            with_retries("PUT '#{ bucket }/#{ dest }' Part ##{ part_number }") do
+              resp = connection.upload_part(
+                bucket, dest, upload_id, part_number, StringIO.new(data),
+                { 'Content-MD5' => md5 }
+              )
+              parts << resp.headers['ETag']
+            end
+
+            if i = progress.rindex(part_number)
+              Logger.info "\s\s...#{ i + 1 }0% Complete..."
+            end
+          end
+        end
+        parts
+      end
+
+      def complete_multipart(dest, upload_id, parts)
+        Logger.info "\s\sComplete Multipart '#{ bucket }/#{ dest }'"
+
+        with_retries("POST '#{ bucket }/#{ dest }' (Complete)") do
+          resp = connection.complete_multipart_upload(bucket, dest, upload_id, parts)
+          raise Error, <<-EOS if resp.body['Code']
+            The server returned the following error:
+            #{ resp.body['Code'] }: #{ resp.body['Message'] }
+          EOS
+        end
+      end
+
+      def headers
+        headers = {}
+
+        enc = encryption.to_s.upcase
+        headers.merge!(
+          { 'x-amz-server-side-encryption' => enc}
+        ) unless enc.empty?
+
+        sc = storage_class.to_s.upcase
+        headers.merge!(
+          { 'x-amz-storage-class' => sc }
+        ) unless sc.empty? || sc == 'STANDARD'
+
+        headers
+      end
+
+      def adjusted_chunk_bytes(chunk_bytes, file_size)
+        return chunk_bytes if file_size / chunk_bytes.to_f <= 10_000
+
+        mb = orig_mb = chunk_bytes / 1024**2
+        mb += 1 until file_size / (1024**2 * mb).to_f <= 10_000
+        Logger.warn Error.new(<<-EOS)
+          Chunk Size Adjusted
+          Your original #chunk_size of #{ orig_mb } MiB has been adjusted
+          to #{ mb } MiB in order to satisfy the limit of 10,000 chunks.
+          To enforce your chosen #chunk_size, you should use the Splitter.
+          e.g. split_into_chunks_of #{ mb * 10_000 } (#chunk_size * 10_000)
+        EOS
+        1024**2 * mb
+      end
+
+      class Object
+        attr_reader :key, :etag, :storage_class
+
+        def initialize(cloud_io, data)
+          @cloud_io = cloud_io
+          @key  = data['Key']
+          @etag = data['ETag']
+          @storage_class = data['StorageClass']
+        end
+
+        # currently 'AES256' or nil
+        def encryption
+          metadata['x-amz-server-side-encryption']
+        end
+
+        private
+
+        def metadata
+          @metadata ||= @cloud_io.head_object(self).headers
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/compressor/gzip.rb

@@ -3,6 +3,7 @@
 module Backup
   module Compressor
     class Gzip < Base
+      class Error < Backup::Error; end
       extend Utilities::Helpers
 
       ##
@@ -71,7 +72,7 @@ module Backup
         if self.class.has_rsyncable?
           opts << ' --rsyncable'
         else
-          Logger.warn Errors::Compressor::Gzip::RsyncableError.new(<<-EOS)
+          Logger.warn Error.new(<<-EOS)
             'rsyncable' option ignored.
             Your system's 'gzip' does not support the `--rsyncable` option.
           EOS

data/lib/backup/config.rb

@@ -2,6 +2,8 @@
 
 module Backup
   module Config
+    class Error < Backup::Error; end
+
     DEFAULTS = {
       :config_file  => 'config.rb',
       :data_path    => 'data',
@@ -10,6 +12,8 @@ module Backup
     }
 
     class << self
+      include Backup::Utilities::Helpers
+
       attr_reader :user, :root_path, :config_file,
                   :data_path, :cache_path, :tmp_path
 
@@ -28,13 +32,16 @@ module Backup
       # Tries to find and load the configuration file
       def load_config!
         unless File.exist?(@config_file)
-          raise Errors::Config::NotFoundError,
-              "Could not find configuration file: '#{@config_file}'."
+          raise Error, "Could not find configuration file: '#{@config_file}'."
         end
 
         module_eval(File.read(@config_file), @config_file)
       end
 
+      def hostname
+        @hostname ||= run(utility(:hostname))
+      end
+
       private
 
       ##
@@ -48,7 +55,7 @@ module Backup
 
         path = File.expand_path(path)
         unless File.directory?(path)
-          raise Errors::Config::NotFoundError, <<-EOS
+          raise Error, <<-EOS
             Root Path Not Found
             When specifying a --root-path, the path must exist.
             Path was: #{ path }
@@ -115,7 +122,8 @@ module Backup
               { 'RSync' => ['Push', 'Pull', 'Local'] }
             ],
             # Notifiers
-            ['Mail', 'Twitter', 'Campfire', 'Prowl', 'Hipchat', 'Pushover']
+            ['Mail', 'Twitter', 'Campfire', 'Prowl',
+             'Hipchat', 'Pushover', 'HttpPost', 'Nagios']
           ]
         )
       end
@@ -153,7 +161,7 @@ module Backup
   class << self
     def const_missing(const)
       if const.to_s == 'CONFIG_FILE'
-        Logger.warn Errors::ConfigError.new(<<-EOS)
+        Logger.warn Error.new(<<-EOS)
           Configuration File Upgrade Needed
           Your configuration file, located at #{ Config.config_file }
           needs to be upgraded for this version of Backup.

data/lib/backup/configuration.rb

@@ -14,7 +14,7 @@ module Backup
     # Pass calls on to the proper class and log a warning
     def defaults(&block)
       klass = eval(self.to_s.sub('Configuration::', ''))
-      Logger.warn Errors::ConfigurationError.new <<-EOS
+      Logger.warn Error.new(<<-EOS)
         [DEPRECATION WARNING]
         #{ self }.defaults is being deprecated.
         To set pre-configured defaults for #{ klass }, use:

data/lib/backup/configuration/helpers.rb

@@ -2,6 +2,8 @@
 
 module Backup
   module Configuration
+    class Error < Backup::Error; end
+
     module Helpers
 
       def self.included(klass)
@@ -37,7 +39,7 @@ module Backup
           msg = "#{ self }##{ name } has been deprecated as of " +
               "backup v.#{ deprecation[:version] }"
           msg << "\n#{ deprecation[:message] }" if deprecation[:message]
-          Logger.warn Backup::Errors::ConfigurationError.new <<-EOS
+          Logger.warn Error.new(<<-EOS)
             [DEPRECATION WARNING]
             #{ msg }
           EOS

data/lib/backup/database/base.rb

@@ -2,6 +2,8 @@
 
 module Backup
   module Database
+    class Error < Backup::Error; end
+
     class Base
       include Backup::Utilities::Helpers
       include Backup::Configuration::Helpers
@@ -47,7 +49,7 @@ module Backup
           unless database_id
             if model.databases.select {|d| d.class == self.class }.count > 1
               sleep 1; @database_id = Time.now.to_i.to_s[-5, 5]
-              Logger.warn Errors::Database::ConfigurationError.new(<<-EOS)
+              Logger.warn Error.new(<<-EOS)
                 Database Identifier Missing
                 When multiple Databases are configured in a single Backup Model
                 that have the same class (MySQL, PostgreSQL, etc.), the optional

data/lib/backup/database/mongodb.rb

@@ -3,6 +3,7 @@
 module Backup
   module Database
     class MongoDB < Base
+      class Error < Backup::Error; end
 
       ##
       # Name of the database that needs to get dumped
@@ -109,8 +110,7 @@ module Backup
           FileUtils.rm_rf dump_packaging_path
           log!(:finished)
         else
-          raise Errors::Database::PipelineError,
-              "#{ database_name } Dump Failed!\n" + pipeline.error_messages
+          raise Error, "Dump Failed!\n" + pipeline.error_messages
         end
       end
 

data/lib/backup/database/mysql.rb

@@ -3,6 +3,7 @@
 module Backup
   module Database
     class MySQL < Base
+      class Error < Backup::Error; end
 
       ##
       # Name of the database that needs to get dumped
@@ -66,8 +67,7 @@ module Backup
         if pipeline.success?
           log!(:finished)
         else
-          raise Errors::Database::PipelineError,
-              "#{ database_name } Dump Failed!\n" + pipeline.error_messages
+          raise Error, "Dump Failed!\n" + pipeline.error_messages
         end
       end
 

data/lib/backup/database/postgresql.rb

@@ -3,6 +3,7 @@
 module Backup
   module Database
     class PostgreSQL < Base
+      class Error < Backup::Error; end
 
       ##
       # Name of the database that needs to get dumped.
@@ -14,6 +15,10 @@ module Backup
       # Credentials for the specified database
       attr_accessor :username, :password
 
+      ##
+      # If set the pg_dump(all) command is executed as the given user
+      attr_accessor :sudo_user
+
       ##
       # Connectivity options
       attr_accessor :host, :port, :socket
@@ -64,19 +69,20 @@ module Backup
         if pipeline.success?
           log!(:finished)
         else
-          raise Errors::Database::PipelineError,
-              "#{ database_name } Dump Failed!\n" + pipeline.error_messages
+          raise Error, "Dump Failed!\n" + pipeline.error_messages
         end
       end
 
       def pgdump
         "#{ password_option }" +
+        "#{ sudo_option }" +
         "#{ utility(:pg_dump) } #{ username_option } #{ connectivity_options } " +
         "#{ user_options } #{ tables_to_dump } #{ tables_to_skip } #{ name }"
       end
 
       def pgdumpall
         "#{ password_option }" +
+        "#{ sudo_option }" +
         "#{ utility(:pg_dumpall) } #{ username_option } " +
         "#{ connectivity_options } #{ user_options }"
       end
@@ -85,6 +91,10 @@ module Backup
         "PGPASSWORD='#{ password }' " if password
       end
 
+      def sudo_option
+        "#{ utility(:sudo) } -n -u #{ sudo_user } " if sudo_user
+      end
+
       def username_option
         "--username='#{ username }'" if username
       end