backup 3.6.0 → 3.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d68dc83802f92387efeb5d0bfd6e89c7d5d32e49
-  data.tar.gz: a9c414e7eb6a1c329b49329d3c4fc5391ab78f84
+  metadata.gz: eee6bab50e1788fc4d4095ee5dee8fad3f1d5b0c
+  data.tar.gz: 4c6c6f7cab298371726c8c33faab6e46f0aaa675
 SHA512:
-  metadata.gz: 1f801689681baae58593843de6828116bc688ece381912e47d6b74e8354f61bd437a13e0b36bf106331124de108788c626dd0fa1d16e09f9c04fff4834c9a747
-  data.tar.gz: ed9473697e3a81910ada2857f7c3533b341bc8ff0a9a6863db58d16cc408a2e29042ab4284bf0e44979dea0fa7dfebe68e48de10cc8b6a41a7c13ed62a321c1d
+  metadata.gz: f401dc3a3f4a50747c3f13cc819292a6196f6d0c3913e31d079b2384f7185f7f6348f2202489a0f2341739dc96c7544478ad45ab96c1b2afaee72ddb0a0ca1c4
+  data.tar.gz: a867d269512d8ca5eb4b19bba111e13555a1a9167d689d7d16da730ddddeaf8d4e7b125571d65298adee9eb56548ac58c79be5b320ee870f69e6233bf05bc841

data/README.md

@@ -125,6 +125,8 @@ Supported notification services include:
 - Prowl
 - Hipchat
 - Pushover
+- POST Request
+- Nagios
 
 
 ## Generators

data/lib/backup.rb

@@ -7,10 +7,17 @@ require 'syslog'
 require 'yaml'
 require 'etc'
 require 'forwardable'
+require 'thread'
 
 require 'open4'
 require 'thor'
 
+require 'excon'
+# Include response.inspect in error messages.
+Excon.defaults[:debug_response] = true
+# Excon should not retry failed requests. We handle that.
+Excon.defaults[:middlewares].delete(Excon::Middleware::Idempotent)
+
 ##
 # The Backup Ruby Gem
 module Backup
@@ -19,11 +26,11 @@ module Backup
   # Backup's internal paths
   LIBRARY_PATH       = File.join(File.dirname(__FILE__), 'backup')
   STORAGE_PATH       = File.join(LIBRARY_PATH, 'storage')
+  SYNCER_PATH        = File.join(LIBRARY_PATH, 'syncer')
   DATABASE_PATH      = File.join(LIBRARY_PATH, 'database')
   COMPRESSOR_PATH    = File.join(LIBRARY_PATH, 'compressor')
   ENCRYPTOR_PATH     = File.join(LIBRARY_PATH, 'encryptor')
   NOTIFIER_PATH      = File.join(LIBRARY_PATH, 'notifier')
-  SYNCER_PATH        = File.join(LIBRARY_PATH, 'syncer')
   TEMPLATE_PATH      = File.expand_path('../../templates', __FILE__)
 
   ##
@@ -32,7 +39,7 @@ module Backup
     autoload :Base,       File.join(STORAGE_PATH, 'base')
     autoload :Cycler,     File.join(STORAGE_PATH, 'cycler')
     autoload :S3,         File.join(STORAGE_PATH, 's3')
-    autoload :CloudFiles, File.join(STORAGE_PATH, 'cloudfiles')
+    autoload :CloudFiles, File.join(STORAGE_PATH, 'cloud_files')
     autoload :Ninefold,   File.join(STORAGE_PATH, 'ninefold')
     autoload :Dropbox,    File.join(STORAGE_PATH, 'dropbox')
     autoload :FTP,        File.join(STORAGE_PATH, 'ftp')
@@ -48,6 +55,7 @@ module Backup
     autoload :Base, File.join(SYNCER_PATH, 'base')
     module Cloud
       autoload :Base,       File.join(SYNCER_PATH, 'cloud', 'base')
+      autoload :LocalFile,  File.join(SYNCER_PATH, 'cloud', 'local_file')
       autoload :CloudFiles, File.join(SYNCER_PATH, 'cloud', 'cloud_files')
       autoload :S3,         File.join(SYNCER_PATH, 'cloud', 's3')
     end
@@ -99,11 +107,15 @@ module Backup
     autoload :Prowl,     File.join(NOTIFIER_PATH, 'prowl')
     autoload :Hipchat,   File.join(NOTIFIER_PATH, 'hipchat')
     autoload :Pushover,  File.join(NOTIFIER_PATH, 'pushover')
+    autoload :HttpPost,  File.join(NOTIFIER_PATH, 'http_post')
+    autoload :Nagios,    File.join(NOTIFIER_PATH, 'nagios')
   end
 
   ##
   # Require Backup base files
   %w{
+    errors
+    logger
     utilities
     archive
     binder
@@ -111,8 +123,6 @@ module Backup
     config
     cli
     configuration
-    errors
-    logger
     model
     package
     packager

data/lib/backup/archive.rb

@@ -2,6 +2,8 @@
 
 module Backup
   class Archive
+    class Error < Backup::Error; end
+
     include Backup::Utilities::Helpers
     attr_reader :name, :options
 
@@ -88,8 +90,7 @@ module Backup
       if pipeline.success?
         Logger.info "Archive '#{ name }' Complete!"
       else
-        raise Errors::Archive::PipelineError,
-            "Failed to Create Archive '#{ name }'\n" +
+        raise Error, "Failed to Create Archive '#{ name }'\n" +
             pipeline.error_messages
       end
     end

data/lib/backup/cleaner.rb

@@ -2,6 +2,8 @@
 
 module Backup
   module Cleaner
+    class Error < Backup::Error; end
+
     class << self
 
       ##
@@ -37,7 +39,7 @@ module Backup
         end
 
         unless messages.empty?
-          Logger.warn Errors::CleanerError.new(<<-EOS)
+          Logger.warn Error.new(<<-EOS)
             Cleanup Warning
             #{ messages.join("\n") }
             Please check the log for messages and/or your notifications
@@ -93,7 +95,7 @@ module Backup
         end
 
         unless messages.empty?
-          Logger.warn Errors::CleanerError.new(<<-EOS)
+          Logger.warn Error.new(<<-EOS)
             Cleanup Warning
             #{ messages.join("\n") }
             Make sure you check these files before the next scheduled backup for

data/lib/backup/cli.rb

@@ -4,6 +4,8 @@
 # Build the Backup Command Line Interface using Thor
 module Backup
   class CLI < Thor
+    class Error < Backup::Error; end
+    class FatalError < Backup::FatalError; end
 
     ##
     # [Perform]
@@ -142,14 +144,14 @@ module Backup
           Model.find_by_trigger(trigger)
         }.flatten.uniq
 
-        raise Errors::CLIError, "No Models found for trigger(s) " +
+        raise Error, "No Models found for trigger(s) " +
             "'#{ triggers.join(',') }'." if models.empty?
 
         # Finalize Logger and begin real-time logging.
         Logger.start!
 
       rescue Exception => err
-        Logger.error Errors::CLIError.wrap(err)
+        Logger.error Error.wrap(err)
         # Logger configuration will be ignored
         # and messages will be output to the console only.
         Logger.abort!
@@ -166,7 +168,7 @@ module Backup
         when 2
           errors = true
           unless models.empty?
-            Logger.info Errors::ModelError.new(<<-EOS)
+            Logger.info Error.new(<<-EOS)
               Backup will now continue...
               The following triggers will now be processed:
               (#{ models.map {|m| m.trigger }.join(', ') })
@@ -175,7 +177,7 @@ module Backup
         when 3
           fatal = true
           unless models.empty?
-            Logger.error Errors::ModelFatalError.new(<<-EOS)
+            Logger.error FatalError.new(<<-EOS)
               Backup will now exit.
               The following triggers will not be processed:
               (#{ models.map {|m| m.trigger }.join(', ') })
@@ -227,7 +229,7 @@ module Backup
         Config.update(options)
         Config.load_config!
       rescue Exception => err
-        Logger.error Errors::CLIError.wrap(err)
+        Logger.error Error.wrap(err)
       end
 
       if Logger.has_warnings? || Logger.has_errors?

data/lib/backup/cloud_io/base.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+
+module Backup
+  module CloudIO
+    class Error < Backup::Error; end
+    class FileSizeError < Backup::Error; end
+
+    class Base
+      attr_reader :max_retries, :retry_waitsec
+
+      def initialize(options = {})
+        @max_retries    = options[:max_retries]
+        @retry_waitsec  = options[:retry_waitsec]
+      end
+
+      private
+
+      def with_retries(operation)
+        retries = 0
+        begin
+          yield
+        rescue => err
+          retries += 1
+          raise Error.wrap(err, <<-EOS) if retries > max_retries
+            Max Retries (#{ max_retries }) Exceeded!
+            Operation: #{ operation }
+            Be sure to check the log messages for each retry attempt.
+          EOS
+
+          Logger.info Error.wrap(err, <<-EOS)
+            Retry ##{ retries } of #{ max_retries }
+            Operation: #{ operation }
+          EOS
+          sleep(retry_waitsec)
+          retry
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/cloud_io/cloud_files.rb

@@ -0,0 +1,296 @@
+# encoding: utf-8
+require 'backup/cloud_io/base'
+require 'fog'
+require 'digest/md5'
+
+module Backup
+  module CloudIO
+    class CloudFiles < Base
+      class Error < Backup::Error; end
+
+      MAX_FILE_SIZE   = 1024**3 * 5     # 5 GiB
+      MAX_SLO_SIZE    = 1024**3 * 5000  # 1000 segments @ 5 GiB
+      SEGMENT_BUFFER  = 1024**2         # 1 MiB
+
+      attr_reader :username, :api_key, :auth_url, :region, :servicenet,
+                  :container, :segments_container, :segment_size, :days_to_keep
+
+      def initialize(options = {})
+        super
+
+        @username           = options[:username]
+        @api_key            = options[:api_key]
+        @auth_url           = options[:auth_url]
+        @region             = options[:region]
+        @servicenet         = options[:servicenet]
+        @container          = options[:container]
+        @segments_container = options[:segments_container]
+        @segment_size       = options[:segment_size]
+        @days_to_keep       = options[:days_to_keep]
+      end
+
+      # The Syncer may call this method in multiple threads,
+      # but #objects is always called before this occurs.
+      def upload(src, dest)
+        create_containers
+
+        file_size = File.size(src)
+        segment_bytes = segment_size * 1024**2
+        if segment_bytes > 0 && file_size > segment_bytes
+          raise FileSizeError, <<-EOS if file_size > MAX_SLO_SIZE
+            File Too Large
+            File: #{ src }
+            Size: #{ file_size }
+            Max SLO Size is #{ MAX_SLO_SIZE } (5 GiB * 1000 segments)
+          EOS
+
+          segment_bytes = adjusted_segment_bytes(segment_bytes, file_size)
+          segments = upload_segments(src, dest, segment_bytes, file_size)
+          upload_manifest(dest, segments)
+        else
+          raise FileSizeError, <<-EOS if file_size > MAX_FILE_SIZE
+            File Too Large
+            File: #{ src }
+            Size: #{ file_size }
+            Max File Size is #{ MAX_FILE_SIZE } (5 GiB)
+          EOS
+
+          put_object(src, dest)
+        end
+      end
+
+      # Returns all objects in the container with the given prefix.
+      #
+      # - #get_container returns a max of 10000 objects per request.
+      # - Returns objects sorted using a sqlite binary collating function.
+      # - If marker is given, only objects after the marker are in the response.
+      def objects(prefix)
+        objects = []
+        resp = nil
+        prefix = prefix.chomp('/')
+        opts = { :prefix => prefix + '/' }
+
+        create_containers
+
+        while resp.nil? || resp.body.count == 10000
+          opts.merge!(:marker => objects.last.name) unless objects.empty?
+          with_retries("GET '#{ container }/#{ prefix }/*'") do
+            resp = connection.get_container(container, opts)
+          end
+          resp.body.each do |obj_data|
+            objects << Object.new(self, obj_data)
+          end
+        end
+
+        objects
+      end
+
+      # Used by Object to fetch metadata if needed.
+      def head_object(object)
+        resp = nil
+        with_retries("HEAD '#{ container }/#{ object.name }'") do
+          resp = connection.head_object(container, object.name)
+        end
+        resp
+      end
+
+      # Delete non-SLO object(s) from the container.
+      #
+      # - Called by the Storage (with objects) and the Syncer (with names)
+      # - Deletes 10,000 objects per request.
+      # - Missing objects will be ignored.
+      def delete(objects_or_names)
+        names = Array(objects_or_names).dup
+        names.map!(&:name) if names.first.is_a?(Object)
+
+        until names.empty?
+          _names = names.slice!(0, 10000)
+          with_retries('DELETE Multiple Objects') do
+            resp = connection.delete_multiple_objects(container, _names)
+            resp_status = resp.body['Response Status']
+            raise Error, <<-EOS unless resp_status == '200 OK'
+              #{ resp_status }
+              The server returned the following:
+              #{ resp.body.inspect }
+            EOS
+          end
+        end
+      end
+
+      # Delete an SLO object(s) from the container.
+      #
+      # - Used only by the Storage. The Syncer cannot use SLOs.
+      # - Removes the SLO manifest object and all associated segments.
+      # - Missing segments will be ignored.
+      def delete_slo(objects)
+        Array(objects).each do |object|
+          with_retries("DELETE SLO Manifest '#{ container }/#{ object.name }'") do
+            resp = connection.delete_static_large_object(container, object.name)
+            resp_status = resp.body['Response Status']
+            raise Error, <<-EOS unless resp_status == '200 OK'
+              #{ resp_status }
+              The server returned the following:
+              #{ resp.body.inspect }
+            EOS
+          end
+        end
+      end
+
+      private
+
+      def connection
+        @connection ||= Fog::Storage.new(
+          :provider             => 'Rackspace',
+          :rackspace_username   => username,
+          :rackspace_api_key    => api_key,
+          :rackspace_auth_url   => auth_url,
+          :rackspace_region     => region,
+          :rackspace_servicenet => servicenet
+        )
+      end
+
+      def create_containers
+        return if @containers_created
+        @containers_created = true
+
+        with_retries('Create Containers') do
+          connection.put_container(container)
+          connection.put_container(segments_container) if segments_container
+        end
+      end
+
+      def put_object(src, dest)
+        opts = headers.merge('ETag' => Digest::MD5.file(src).hexdigest)
+        with_retries("PUT '#{ container }/#{ dest }'") do
+          File.open(src, 'r') do |file|
+            connection.put_object(container, dest, file, opts)
+          end
+        end
+      end
+
+      # Each segment is uploaded using chunked transfer encoding using
+      # SEGMENT_BUFFER, and each segment's MD5 is sent to verify the transfer.
+      # Each segment's MD5 and byte_size will also be verified when the
+      # SLO manifest object is uploaded.
+      def upload_segments(src, dest, segment_bytes, file_size)
+        total_segments = (file_size / segment_bytes.to_f).ceil
+        progress = (0.1..0.9).step(0.1).map {|n| (total_segments * n).floor }
+        Logger.info "\s\sUploading #{ total_segments } SLO Segments..."
+
+        segments = []
+        File.open(src, 'r') do |file|
+          segment_number = 0
+          until file.eof?
+            segment_number += 1
+            object = "#{ dest }/#{ segment_number.to_s.rjust(4, '0') }"
+            pos = file.pos
+            md5 = segment_md5(file, segment_bytes)
+            opts = headers.merge('ETag' => md5)
+
+            with_retries("PUT '#{ segments_container }/#{ object }'") do
+              file.seek(pos)
+              offset = 0
+              connection.put_object(segments_container, object, nil, opts) do
+                # block is called to stream data until it returns ''
+                data = ''
+                if offset <= segment_bytes - SEGMENT_BUFFER
+                  data = file.read(SEGMENT_BUFFER).to_s # nil => ''
+                  offset += data.size
+                end
+                data
+              end
+            end
+
+            segments << {
+              :path => "#{ segments_container }/#{ object }",
+              :etag => md5,
+              :size_bytes => file.pos - pos
+            }
+
+            if i = progress.rindex(segment_number)
+              Logger.info "\s\s...#{ i + 1 }0% Complete..."
+            end
+          end
+        end
+        segments
+      end
+
+      def segment_md5(file, segment_bytes)
+        md5 = Digest::MD5.new
+        offset = 0
+        while offset <= segment_bytes - SEGMENT_BUFFER
+          data = file.read(SEGMENT_BUFFER)
+          break unless data
+          offset += data.size
+          md5 << data
+        end
+        md5.hexdigest
+      end
+
+      # Each segment's ETag and byte_size will be verified once uploaded.
+      # Request will raise an exception if verification fails or segments
+      # are not found. However, each segment's ETag was verified when we
+      # uploaded the segments, so this should only retry failed requests.
+      def upload_manifest(dest, segments)
+        Logger.info "\s\sStoring SLO Manifest '#{ container }/#{ dest }'"
+
+        with_retries("PUT SLO Manifest '#{ container }/#{ dest }'") do
+          connection.put_static_obj_manifest(container, dest, segments, headers)
+        end
+      end
+
+      # If :days_to_keep was set, each object will be scheduled for deletion.
+      # This includes non-SLO objects, the SLO manifest and all segments.
+      def headers
+        headers = {}
+        headers.merge!('X-Delete-At' => delete_at) if delete_at
+        headers
+      end
+
+      def delete_at
+        return unless days_to_keep
+        @delete_at ||= (Time.now.utc + days_to_keep * 60**2 * 24).to_i
+      end
+
+      def adjusted_segment_bytes(segment_bytes, file_size)
+        return segment_bytes if file_size / segment_bytes.to_f <= 1000
+
+        mb = orig_mb = segment_bytes / 1024**2
+        mb += 1 until file_size / (1024**2 * mb).to_f <= 1000
+        Logger.warn Error.new(<<-EOS)
+          Segment Size Adjusted
+          Your original #segment_size of #{ orig_mb } MiB has been adjusted
+          to #{ mb } MiB in order to satisfy the limit of 1000 segments.
+          To enforce your chosen #segment_size, you should use the Splitter.
+          e.g. split_into_chunks_of #{ mb * 1000 } (#segment_size * 1000)
+        EOS
+        1024**2 * mb
+      end
+
+      class Object
+        attr_reader :name, :hash
+
+        def initialize(cloud_io, data)
+          @cloud_io = cloud_io
+          @name = data['name']
+          @hash = data['hash']
+        end
+
+        def slo?
+          !!metadata['X-Static-Large-Object']
+        end
+
+        def marked_for_deletion?
+          !!metadata['X-Delete-At']
+        end
+
+        private
+
+        def metadata
+          @metadata ||= @cloud_io.head_object(self).headers
+        end
+      end
+
+    end
+  end
+end