backup 3.0.27 → 3.1.0

data/lib/backup/storage/ninefold.rb

@@ -68,7 +68,7 @@ module Backup
         directory = directory_for(remote_path, true)
 
         files_to_transfer_for(@package) do |local_file, remote_file|
-          Logger.message "#{storage_name} started transferring '#{ local_file }'."
+          Logger.info "#{storage_name} started transferring '#{ local_file }'."
 
           File.open(File.join(local_path, local_file), 'r') do |file|
             directory.files.create(:key => remote_file, :body => file)
@@ -87,7 +87,7 @@ module Backup
           not_found = []
 
           transferred_files_for(package) do |local_file, remote_file|
-            Logger.message "#{storage_name} started removing " +
+            Logger.info "#{storage_name} started removing " +
                 "'#{ local_file }' from Ninefold."
 
             if file = directory.files.get(remote_file)

data/lib/backup/storage/rsync.rb

@@ -7,7 +7,7 @@ Backup::Dependency.load('net-ssh')
 module Backup
   module Storage
     class RSync < Base
-      include Backup::CLI::Helpers
+      include Backup::Utilities::Helpers
 
       ##
       # Server credentials
@@ -68,14 +68,14 @@ module Backup
 
         files_to_transfer_for(@package) do |local_file, remote_file|
           if local
-            Logger.message "#{storage_name} started transferring " +
+            Logger.info "#{storage_name} started transferring " +
                 "'#{ local_file }' to '#{ remote_path }'."
             run(
               "#{ utility(:rsync) } '#{ File.join(local_path, local_file) }' " +
               "'#{ File.join(remote_path, remote_file) }'"
             )
           else
-            Logger.message "#{storage_name} started transferring " +
+            Logger.info "#{storage_name} started transferring " +
                 "'#{ local_file }' to '#{ ip }'."
             run(
               "#{ utility(:rsync) } #{ rsync_options } #{ rsync_port } " +

data/lib/backup/storage/s3.rb

@@ -61,7 +61,7 @@ module Backup
         connection.sync_clock
 
         files_to_transfer_for(@package) do |local_file, remote_file|
-          Logger.message "#{storage_name} started transferring " +
+          Logger.info "#{storage_name} started transferring " +
               "'#{ local_file }' to bucket '#{ bucket }'."
 
           File.open(File.join(local_path, local_file), 'r') do |file|
@@ -82,7 +82,7 @@ module Backup
         connection.sync_clock
 
         transferred_files_for(package) do |local_file, remote_file|
-          Logger.message "#{storage_name} started removing " +
+          Logger.info "#{storage_name} started removing " +
               "'#{ local_file }' from bucket '#{ bucket }'."
 
           connection.delete_object(bucket, File.join(remote_path, remote_file))

data/lib/backup/storage/scp.rb

@@ -1,9 +1,5 @@
 # encoding: utf-8
 
-##
-# Only load the Net::SSH and Net::SCP library/gems
-# when the Backup::Storage::SCP class is loaded
-Backup::Dependency.load('net-ssh')
 Backup::Dependency.load('net-scp')
 
 module Backup
@@ -56,7 +52,7 @@ module Backup
           ssh.exec!("mkdir -p '#{ remote_path }'")
 
           files_to_transfer_for(@package) do |local_file, remote_file|
-            Logger.message "#{storage_name} started transferring " +
+            Logger.info "#{storage_name} started transferring " +
                 "'#{local_file}' to '#{ip}'."
 
             ssh.scp.upload!(
@@ -79,7 +75,7 @@ module Backup
           messages << "#{storage_name} started removing " +
               "'#{local_file}' from '#{ip}'."
         end
-        Logger.message messages.join("\n")
+        Logger.info messages.join("\n")
 
         errors = []
         connection do |ssh|

data/lib/backup/storage/sftp.rb

@@ -1,8 +1,5 @@
 # encoding: utf-8
 
-##
-# Only load the Net::SFTP library/gem when the Backup::Storage::SFTP class is loaded
-Backup::Dependency.load('net-ssh')
 Backup::Dependency.load('net-sftp')
 
 module Backup
@@ -55,7 +52,7 @@ module Backup
           create_remote_path(remote_path, sftp)
 
           files_to_transfer_for(@package) do |local_file, remote_file|
-            Logger.message "#{storage_name} started transferring " +
+            Logger.info "#{storage_name} started transferring " +
                 "'#{ local_file }' to '#{ ip }'."
 
             sftp.upload!(
@@ -75,7 +72,7 @@ module Backup
 
         connection do |sftp|
           transferred_files_for(package) do |local_file, remote_file|
-            Logger.message "#{storage_name} started removing " +
+            Logger.info "#{storage_name} started removing " +
                 "'#{ local_file }' from '#{ ip }'."
 
             sftp.remove!(File.join(remote_path, remote_file))

data/lib/backup/syncer/base.rb

@@ -3,7 +3,7 @@
 module Backup
   module Syncer
     class Base
-      include Backup::CLI::Helpers
+      include Backup::Utilities::Helpers
       include Backup::Configuration::Helpers
 
       ##

data/lib/backup/syncer/cloud/base.rb

@@ -49,7 +49,7 @@ module Backup
         ##
         # Performs the Sync operation
         def perform!
-          Logger.message(
+          Logger.info(
             "#{ syncer_name } started the syncing process:\n" +
             "\s\sConcurrency: #{ @concurrency_type } Level: #{ @concurrency_level }"
           )
@@ -60,12 +60,14 @@ module Backup
             ).sync! @mirror, @concurrency_type, @concurrency_level
           end
 
-          Logger.message("#{ syncer_name } Syncing Complete!")
+          Logger.info("#{ syncer_name } Syncing Complete!")
         end
 
         private
 
         class SyncContext
+          include Utilities::Helpers
+
           attr_reader :directory, :bucket, :path, :remote_base
 
           ##
@@ -126,9 +128,14 @@ module Backup
 
           ##
           # Returns a String of file paths and their md5 hashes.
+          #
+          # Utilities#run is not used here because this would produce too much
+          # log output, and Pipeline does not support capturing output.
           def local_hashes
-            Logger.message("\s\sGenerating checksums for '#{ @directory }'")
-            `find '#{ @directory }' -print0 | xargs -0 openssl md5 2> /dev/null`
+            Logger.info("\s\sGenerating checksums for '#{ @directory }'")
+            cmd = "#{ utility(:find) } -L '#{ @directory }' -type f -print0 | " +
+                "#{ utility(:xargs) } -0 #{ utility(:openssl) } md5 2> /dev/null"
+            %x[#{ cmd }]
           end
 
           ##
@@ -160,7 +167,7 @@ module Backup
             if local_file && File.exist?(local_file.path)
               unless remote_file && remote_file.etag == local_file.md5
                 MUTEX.synchronize {
-                  Logger.message("\s\s[transferring] '#{ remote_path }'")
+                  Logger.info("\s\s[transferring] '#{ remote_path }'")
                 }
                 File.open(local_file.path, 'r') do |file|
                   @bucket.files.create(
@@ -170,18 +177,18 @@ module Backup
                 end
               else
                 MUTEX.synchronize {
-                  Logger.message("\s\s[skipping] '#{ remote_path }'")
+                  Logger.info("\s\s[skipping] '#{ remote_path }'")
                 }
               end
             elsif remote_file
               if mirror
                 MUTEX.synchronize {
-                  Logger.message("\s\s[removing] '#{ remote_path }'")
+                  Logger.info("\s\s[removing] '#{ remote_path }'")
                 }
                 remote_file.destroy
               else
                 MUTEX.synchronize {
-                  Logger.message("\s\s[leaving] '#{ remote_path }'")
+                  Logger.info("\s\s[leaving] '#{ remote_path }'")
                 }
               end
             end

data/lib/backup/syncer/rsync/local.rb

@@ -25,7 +25,7 @@ module Backup
         # Performs the RSync::Local operation
         # debug options: -vhP
         def perform!
-          Logger.message(
+          Logger.info(
             "#{ syncer_name } started syncing the following directories:\n\s\s" +
             @directories.join("\n\s\s")
           )

data/lib/backup/syncer/rsync/pull.rb

@@ -12,7 +12,7 @@ module Backup
           write_password_file!
 
           @directories.each do |directory|
-            Logger.message("#{ syncer_name } started syncing '#{ directory }'.")
+            Logger.info("#{ syncer_name } started syncing '#{ directory }'.")
             run("#{ utility(:rsync) } #{ options } " +
                 "'#{ username }@#{ ip }:#{ directory.sub(/^\~\//, '') }' " +
                 "'#{ dest_path }'")

data/lib/backup/syncer/rsync/push.rb

@@ -47,7 +47,7 @@ module Backup
         def perform!
           write_password_file!
 
-          Logger.message(
+          Logger.info(
             "#{ syncer_name } started syncing the following directories:\n\s\s" +
             @directories.join("\n\s\s")
           )

data/lib/backup/utilities.rb

@@ -0,0 +1,211 @@
+# encoding: utf-8
+
+module Backup
+  module Utilities
+    UTILITY = {}
+    NAMES = %w{
+      tar cat split find xargs
+      gzip bzip2 lzma pbzip2
+      mongo mongodump mysqldump pg_dump redis-cli riak-admin
+      gpg openssl
+      rsync
+    }
+
+    module Helpers
+      private
+
+      ##
+      # Returns the full path to the specified utility.
+      # Raises an error if utility can not be found in the system's $PATH
+      def utility(name)
+        name = name.to_s.strip
+        raise Errors::Utilities::NotFoundError,
+            'Utility Name Empty' if name.empty?
+
+        path = UTILITY[name] || %x[which '#{ name }' 2>/dev/null].chomp
+        if path.empty?
+          raise Errors::Utilities::NotFoundError, <<-EOS
+            Could not locate '#{ name }'.
+            Make sure the specified utility is installed
+            and available in your system's $PATH.
+          EOS
+        end
+        UTILITY[name] = path
+      end
+
+      ##
+      # Returns the name of the command name from the given command line
+      def command_name(command)
+        i = command =~ /\s/
+        command = command.slice(0, i) if i
+        command.split('/')[-1]
+      end
+
+      ##
+      # Runs a system command
+      #
+      # All messages generated by the command will be logged.
+      # Messages on STDERR will be logged as warnings.
+      #
+      # If the command fails to execute, or returns a non-zero exit status
+      # an Error will be raised.
+      #
+      # Returns STDOUT
+      def run(command)
+        name = command_name(command)
+        Logger.info "Running system utility '#{ name }'..."
+
+        begin
+          out, err = '', ''
+          # popen4 doesn't work in 1.8.7 with stock versions of ruby shipped
+          # with major OSs. Hack to make it stop segfaulting.
+          # See: https://github.com/engineyard/engineyard/issues/115
+          GC.disable if RUBY_VERSION < '1.9'
+          ps = Open4.popen4(command) do |pid, stdin, stdout, stderr|
+            stdin.close
+            out, err = stdout.read.strip, stderr.read.strip
+          end
+        rescue Exception => e
+          raise Errors::Utilities::SystemCallError.wrap(e, <<-EOS)
+            Failed to execute system command on #{ RUBY_PLATFORM }
+            Command was: #{ command }
+          EOS
+        ensure
+          GC.enable if RUBY_VERSION < '1.9'
+        end
+
+        if ps.success?
+          unless out.empty?
+            Logger.info(
+              out.lines.map {|line| "#{ name }:STDOUT: #{ line }" }.join
+            )
+          end
+
+          unless err.empty?
+            Logger.warn(
+              err.lines.map {|line| "#{ name }:STDERR: #{ line }" }.join
+            )
+          end
+
+          return out
+        else
+          raise Errors::Utilities::SystemCallError, <<-EOS
+            '#{ name }' Failed on #{ RUBY_PLATFORM }
+            The following information should help to determine the problem:
+            Command was: #{ command }
+            Exit Status: #{ ps.exitstatus }
+            STDOUT Messages: #{ out.empty? ? 'None' : "\n#{ out }" }
+            STDERR Messages: #{ err.empty? ? 'None' : "\n#{ err }" }
+          EOS
+        end
+      end
+
+      def gnu_tar?
+        Utilities.gnu_tar?
+      end
+    end # Helpers
+
+    class << self
+      include Helpers
+
+      ##
+      # Configure the path to system utilities used by Backup.
+      #
+      # Backup will attempt to locate any required system utilities using a
+      # +which+ command call. If a utility can not be found, or you need to
+      # specify an alternate path for a utility, you may do so in your
+      # +config.rb+ file using this method.
+      #
+      # Backup supports both GNU and BSD utilities.
+      # While Backup uses these utilities in a manner compatible with either
+      # version, the +tar+ utility requires some special handling with respect
+      # to +Archive+s. Backup will attempt to detect if the +tar+ command
+      # found (or set here) is GNU or BSD. If for some reason this fails,
+      # this may be set using the +tar_dist+ command shown below.
+      #
+      #   Backup::Utilities.configure do
+      #     # General Utilites
+      #     tar   '/path/to/tar'
+      #     tar_dist :gnu   # or :bsd
+      #     cat   '/path/to/cat'
+      #     split '/path/to/split'
+      #     find  '/path/to/find'
+      #     xargs '/path/to/xargs'
+      #
+      #     # Compressors
+      #     gzip    '/path/to/gzip'
+      #     bzip2   '/path/to/bzip2'
+      #     lzma    '/path/to/lzma'   # deprecated. use a Custom Compressor
+      #     pbzip2  '/path/to/pbzip2' # deprecated. use a Custom Compressor
+      #
+      #     # Database Utilities
+      #     mongo       '/path/to/mongo'
+      #     mongodump   '/path/to/mongodump'
+      #     mysqldump   '/path/to/mysqldump'
+      #     pg_dump     '/path/to/pg_dump'
+      #     redis_cli   '/path/to/redis-cli'
+      #     riak_admin  '/path/to/riak-admin'
+      #
+      #     # Encryptors
+      #     gpg     '/path/to/gpg'
+      #     openssl '/path/to/openssl'
+      #
+      #     # Syncer and Storage
+      #     rsync   '/path/to/rsync'
+      #   end
+      #
+      # These paths may be set using absolute paths, or relative to the
+      # working directory when Backup is run.
+      #
+      # Note that many of Backup's components currently have their own
+      # configuration settings for utility paths. For instance, when configuring
+      # a +MySQL+ database backup, +mysqldump_utility+ may be used:
+      #
+      #   database MySQL do |db|
+      #     db.mysqldump_utility = '/path/to/mysqldump'
+      #   end
+      #
+      # Use of these configuration settings will override the path set here.
+      # (The use of these may be deprecated in the future)
+      def configure(&block)
+        instance_eval(&block)
+      end
+
+      def gnu_tar?
+        return @gnu_tar unless @gnu_tar.nil?
+        @gnu_tar = !!run("#{ utility(:tar) } --version").match(/GNU/)
+      end
+
+      private
+
+      ##
+      # Allow users to set the path for all utilities in the .configure block.
+      #
+      # Utility names with dashes ('redis-cli') will be set using method calls
+      # with an underscore ('redis_cli').
+      NAMES.each do |name|
+        define_method name.gsub('-', '_'), lambda {|val|
+          path = File.expand_path(val)
+          unless File.executable?(path)
+            raise Errors::Utilities::NotFoundError, <<-EOS
+              The path given for '#{ name }' was not found or not executable.
+              Path was: #{ path }
+            EOS
+          end
+          UTILITY[name] = path
+        }
+      end
+
+      ##
+      # Allow users to set the +tar+ distribution if needed. (:gnu or :bsd)
+      def tar_dist(val)
+        @gnu_tar = val == :gnu
+      end
+
+      def reset!
+        UTILITY.clear
+        @gnu_tar = nil
+      end
+    end
+  end
+end

data/lib/backup/version.rb

@@ -13,7 +13,7 @@ module Backup
     #  Defines the minor version
     # PATCH:
     #  Defines the patch version
-    MAJOR, MINOR, PATCH = 3, 0, 27
+    MAJOR, MINOR, PATCH = 3, 1, 0
 
     ##
     # Returns the major version ( big release based off of multiple minor releases )