backup 3.0.14 → 3.0.15

data/README.md

@@ -205,6 +205,8 @@ The __Mail__ notifier. I have not provided the SMTP options to use my Gmail acco
 
 The __Twitter__ notifier. You will require your consumer and oauth credentials, which I have also left out of this example.
 
+MongoDB backup utility (mongodump) by default does not fsync & lock the database, opening a possibility for inconsistent data dump. This is addressed by setting lock = true which causes mongodump to be wrapped with lock&fsync calls (with a lock takedown after the dump). Please check the Wiki on this subject and remember this is a very fresh feature, needing some more real-world testing. Disabled at default.
+
 Check out the Wiki for more information on all the above subjects.
 
 ### And that's it!

data/bin/backup

@@ -31,11 +31,12 @@ class BackupCLI < Thor
   # Performs the backup process. The only required option is the --trigger [-t].
   # If the other options (--config_file, --data_path, --tmp_path) aren't specified
   # it'll fallback to the (good) defaults
-  method_option :trigger,     :type => :string, :aliases => ['-t', '--triggers'], :required => true
-  method_option :config_file, :type => :string, :aliases => '-c'
-  method_option :data_path,   :type => :string, :aliases => '-d'
-  method_option :log_path,    :type => :string, :aliases => '-l'
+  method_option :trigger,     :type => :string,  :aliases => ['-t', '--triggers'], :required => true
+  method_option :config_file, :type => :string,  :aliases => '-c'
+  method_option :data_path,   :type => :string,  :aliases => '-d'
+  method_option :log_path,    :type => :string,  :aliases => '-l'
   method_option :tmp_path,    :type => :string
+  method_option :quiet,       :type => :boolean, :aliases => '-q'
   desc 'perform', "Performs the backup for the specified trigger.\n" + 
                   "You may perform multiple backups by providing multiple triggers, separated by commas.\n\n" +
                   "Example:\n\s\s$ backup perform --triggers backup1,backup2,backup3,backup4\n\n" +
@@ -76,6 +77,12 @@ class BackupCLI < Thor
       FileUtils.mkdir_p(path)
     end
 
+    ##
+    # Silence Backup::Logger from printing to STDOUT, if --quiet was specified
+    if options[:quiet]
+      Backup::Logger.send(:const_set, :QUIET, options[:quiet])
+    end
+
     ##
     # Process each trigger
     options[:trigger].split(",").map(&:strip).each do |trigger|
@@ -257,4 +264,4 @@ end
 
 ##
 # Enable the CLI for the Backup binary
-BackupCLI.start
+BackupCLI.start

data/lib/backup/cli.rb

@@ -8,6 +8,10 @@ module Backup
     # through a ruby method. This helps with test coverage and
     # improves readability.
     #
+    # It'll first remove all prefixing slashes ( / ) by using .gsub(/^\s+/, '')
+    # This allows for the EOS blocks to be indented without actually using any
+    # prefixing spaces. This cleans up the implementation code.
+    #
     # Every time the Backup::CLI#run method is invoked, it'll invoke
     # the Backup::CLI#raise_if_command_not_found method after running the
     # requested command on the OS.
@@ -17,6 +21,7 @@ module Backup
     # name (e.g. mongodump, pgdump, etc) from a command like "/usr/local/bin/mongodump <options>"
     # and pass that in to the Backup::CLI#raise_if_command_not_found
     def run(command)
+      command.gsub!(/^\s+/, '')
       %x[#{command}]
       raise_if_command_not_found!(
         command.slice(0, command.index(/\s/)).split('/')[-1]
@@ -67,7 +72,9 @@ module Backup
     # and notify the user via the built-in notifiers if these are set.
     def raise_if_command_not_found!(utility)
       if $?.to_i.eql?(32512)
-        raise Exception::CommandNotFound , "Could not find the utility \"#{utility}\" on \"#{RUBY_PLATFORM}\"."
+        raise Exception::CommandNotFound , "Could not find the utility \"#{utility}\" on \"#{RUBY_PLATFORM}\".\n" +
+                                           "If this is a database utility, try defining the 'utility_path' option in the configuration file.\n" +
+                                           "See the Database Wiki for more information about the Utility Path option."
       end
     end
 

data/lib/backup/configuration/database/mongodb.rb

@@ -30,6 +30,10 @@ module Backup
           # Additional "mongodump" options
           attr_accessor :additional_options
 
+          ##
+          # 'lock' dump meaning wrapping mongodump with fsync & lock
+          attr_accessor :lock
+
         end
       end
     end

data/lib/backup/database/mongodb.rb

@@ -28,6 +28,10 @@ module Backup
       # Additional "mongodump" options
       attr_accessor :additional_options
 
+      ##
+      # 'lock' dump meaning wrapping mongodump with fsync & lock
+      attr_accessor :lock
+
       ##
       # Creates a new instance of the MongoDB database object
       def initialize(&block)
@@ -36,6 +40,7 @@ module Backup
         @only_collections   ||= Array.new
         @additional_options ||= Array.new
         @ipv6               ||= false
+        @lock               ||= false
 
         instance_eval(&block)
         prepare!
@@ -109,10 +114,17 @@ module Backup
       def perform!
         log!
 
-        if collections_to_dump.is_a?(Array) and not collections_to_dump.empty?
-          specific_collection_dump!
-        else
-          dump!
+        begin
+          lock_database if @lock.eql?(true)
+          if collections_to_dump.is_a?(Array) and not collections_to_dump.empty?
+            specific_collection_dump!
+          else
+            dump!
+          end
+          unlock_database if @lock.eql?(true)
+        rescue => exception
+          unlock_database if @lock.eql?(true)
+          raise exception
         end
       end
 
@@ -132,6 +144,36 @@ module Backup
         end
       end
 
+      ##
+      # Builds the full mongo string based on all attributes
+      def mongo_shell
+        [utility(:mongo), database, credential_options, connectivity_options, ipv6].join(' ')
+      end
+
+      ##
+      # Locks and FSyncs the database to bring it up to sync
+      # and ensure no 'write operations' are performed during the
+      # dump process
+      def lock_database
+        lock_command = <<-EOS
+          echo 'use admin
+          db.runCommand({"fsync" : 1, "lock" : 1})' | #{mongo_shell}
+        EOS
+
+        run(lock_command)
+      end
+
+      ##
+      # Unlocks the (locked) database
+      def unlock_database
+        unlock_command = <<-EOS
+          echo 'use admin
+          db.$cmd.sys.unlock.findOne()' | #{mongo_shell}
+        EOS
+
+        run(unlock_command)
+      end
+
     end
   end
 end

data/lib/backup/logger.rb

@@ -6,21 +6,21 @@ module Backup
     ##
     # Outputs a messages to the console and writes it to the backup.log
     def self.message(string)
-      puts    loggify(:message, string, :green)
+      puts    loggify(:message, string, :green) unless quiet?
       to_file loggify(:message, string)
     end
 
     ##
     # Outputs an error to the console and writes it to the backup.log
     def self.error(string)
-      puts    loggify(:error, string, :red)
+      puts    loggify(:error, string, :red) unless quiet?
       to_file loggify(:error, string)
     end
 
     ##
     # Outputs a notice to the console and writes it to the backup.log
     def self.warn(string)
-      puts    loggify(:warning, string, :yellow)
+      puts    loggify(:warning, string, :yellow) unless quiet?
       to_file loggify(:warning, string)
     end
 
@@ -28,7 +28,7 @@ module Backup
     # Outputs the data as if it were a regular 'puts' command,
     # but also logs it to the backup.log
     def self.normal(string)
-      puts    string
+      puts    string unless quiet?
       to_file string
     end
 
@@ -90,5 +90,13 @@ module Backup
       "\e[#{code}m#{string}\e[0m"
     end
 
+    ##
+    # Returns 'true' (boolean) if the QUIET constant is defined
+    # By default it isn't defined, only when initializing Backup using
+    # the '--quite' (or '-q') option in the CLI (e.g. backup perform -t my_backup --quiet)
+    def self.quiet?
+      const_defined?(:QUIET) && QUIET
+    end
+
   end
 end

data/lib/backup/storage/rsync.rb

@@ -1,5 +1,9 @@
 # encoding: utf-8
 
+##
+# Require the tempfile Ruby library when Backup::Storage::RSync is loaded
+require 'tempfile'
+
 ##
 # Only load the Net::SSH library when the Backup::Storage::RSync class is loaded
 Backup::Dependency.load('net-ssh')
@@ -32,6 +36,7 @@ module Backup
         @path ||= 'backups'
 
         instance_eval(&block) if block_given?
+        write_password_file!
 
         @time = TIME
         @path = path.sub(/^\~\//, '')
@@ -47,6 +52,7 @@ module Backup
       # Performs the backup transfer
       def perform!
         transfer!
+        remove_password_file!
       end
 
     private
@@ -58,7 +64,7 @@ module Backup
       # gets invoked once per object for a #transfer! and once for a remove! Backups run in the
       # background anyway so even if it were a bit slower it shouldn't matter.
       def connection
-        Net::SSH.start(ip, username, :password => password, :port => port)
+        Net::SSH.start(ip, username, :password => @password, :port => port)
       end
 
       ##
@@ -66,7 +72,7 @@ module Backup
       def transfer!
         Logger.message("#{ self.class } started transferring \"#{ remote_file }\".")
         create_remote_directories!
-        run("#{ utility(:rsync) } #{ options } '#{ File.join(local_path, local_file) }' '#{ username }@#{ ip }:#{ File.join(remote_path, remote_file[20..-1]) }'")
+        run("#{ utility(:rsync) } #{ options } #{ password } '#{ File.join(local_path, local_file) }' '#{ username }@#{ ip }:#{ File.join(remote_path, remote_file[20..-1]) }'")
       end
 
       ##
@@ -94,6 +100,30 @@ module Backup
         "-z --port='#{ port }'"
       end
 
+      ##
+      # Returns Rsync syntax for using a password file
+      def password
+        "--password-file='#{@password_file.path}'" unless @password.nil?
+      end
+
+      ##
+      # Writes the provided password to a temporary file so that
+      # the rsync utility can read the password from this file
+      def write_password_file!
+        unless @password.nil?
+          @password_file = Tempfile.new('backup-rsync-password')
+          @password_file.write(@password)
+          @password_file.close
+        end
+      end
+
+      ##
+      # Removes the previously created @password_file
+      # (temporary file containing the password)
+      def remove_password_file!
+        @password_file.unlink unless @password.nil?
+      end
+
     end
   end
 end

data/lib/backup/syncer/rsync.rb

@@ -1,5 +1,9 @@
 # encoding: utf-8
 
+##
+# Require the tempfile Ruby library when Backup::Syncer::RSync is loaded
+require 'tempfile'
+
 module Backup
   module Syncer
     class RSync < Base
@@ -52,6 +56,7 @@ module Backup
         @compress           ||= false
 
         instance_eval(&block) if block_given?
+        write_password_file!
 
         @path = path.sub(/^\~\//, '')
       end
@@ -61,13 +66,17 @@ module Backup
       # debug options: -vhP
       def perform!
         Logger.message("#{ self.class } started syncing #{ directories }.")
-        Logger.silent( run("#{ utility(:rsync) } -vhP #{ options } #{ directories } '#{ username }@#{ ip }:#{ path }'") )
+        Logger.silent(
+          run("#{ utility(:rsync) } -vhP #{ options } #{ directories } '#{ username }@#{ ip }:#{ path }'")
+        )
+
+        remove_password_file!
       end
 
       ##
       # Returns all the specified Rsync options, concatenated, ready for the CLI
       def options
-        ([archive, mirror, compress, port] + additional_options).compact.join("\s")
+        ([archive, mirror, compress, port, password] + additional_options).compact.join("\s")
       end
 
       ##
@@ -94,6 +103,12 @@ module Backup
         "--port='#{@port}'"
       end
 
+      ##
+      # Returns Rsync syntax for setting a password (via a file)
+      def password
+        "--password-file='#{@password_file.path}'" unless @password.nil?
+      end
+
       ##
       # If no block has been provided, it'll return the array of @directories.
       # If a block has been provided, it'll evaluate it and add the defined paths to the @directories
@@ -112,6 +127,26 @@ module Backup
         @directories << path
       end
 
+      private
+
+      ##
+      # Writes the provided password to a temporary file so that
+      # the rsync utility can read the password from this file
+      def write_password_file!
+        unless @password.nil?
+          @password_file = Tempfile.new('backup-rsync-password')
+          @password_file.write(@password)
+          @password_file.close
+        end
+      end
+
+      ##
+      # Removes the previously created @password_file
+      # (temporary file containing the password)
+      def remove_password_file!
+        @password_file.unlink unless @password.nil?
+      end
+
     end
   end
 end

data/lib/backup/version.rb

@@ -13,7 +13,7 @@ module Backup
     #  Defines the minor version
     # PATCH:
     #  Defines the patch version
-    MAJOR, MINOR, PATCH = 3, 0, 14
+    MAJOR, MINOR, PATCH = 3, 0, 15
 
     ##
     # Returns the major version ( big release based off of multiple minor releases )

data/lib/templates/database/mongodb

@@ -7,4 +7,5 @@
     db.ipv6               = false
     db.only_collections   = ['only', 'these' 'collections']
     db.additional_options = []
+    db.lock               = false
   end

data/spec/database/mongodb_spec.rb

@@ -129,6 +129,43 @@ describe Backup::Database::MongoDB do
       db.perform!
     end
 
+    it 'should lock database before dump if lock mode is enabled' do
+      db.lock = true
+      db.expects(:lock_database)
+
+      db.perform!
+    end
+
+    it 'should not lock database before dump if lock mode is disabled' do
+      db.lock = false
+      db.expects(:lock_database).never
+
+      db.perform!
+    end
+
+    it 'should unlock database after dump if lock mode is enabled' do
+      db.lock = true
+      db.expects(:unlock_database)
+
+      db.perform!
+    end
+
+    it 'should unlock the database if an exception is raised after it was locked' do
+      db.lock = true
+      db.expects(:unlock_database)
+      db.expects(:lock_database).raises(RuntimeError, 'something went wrong')
+      db.expects(:raise)
+
+      db.perform!
+    end
+
+    it 'should not unlock database after dump if lock mode is disabled' do
+      db.lock = false
+      db.expects(:unlock_database).never
+
+      db.perform!
+    end
+
     it 'should dump only the provided collections' do
       db.only_collections = %w[users admins profiles]
       db.expects(:specific_collection_dump!)

data/spec/logger_spec.rb

@@ -43,4 +43,16 @@ describe Backup::Logger do
       Backup::Logger.silent "This has been logged."
     end
   end
+
+  context 'when quieted' do
+    it do
+      Backup::Logger.send(:const_set, :QUIET, true)
+      Backup::Logger.expects(:puts).never
+
+      Backup::Logger.message "This has been logged."
+      Backup::Logger.error   "This has been logged."
+      Backup::Logger.warn    "This has been logged."
+      Backup::Logger.normal  "This has been logged."
+    end
+  end
 end

data/spec/storage/rsync_spec.rb

@@ -19,11 +19,13 @@ describe Backup::Storage::RSync do
   end
 
   it 'should have defined the configuration properly' do
-    rsync.username.should == 'my_username'
-    rsync.password.should == 'my_password'
-    rsync.ip.should       == '123.45.678.90'
-    rsync.port.should     == 22
-    rsync.path.should     == 'backups/'
+    rsync.username.should        == 'my_username'
+    rsync.send(:password).should =~ /backup-rsync-password/
+    rsync.ip.should              == '123.45.678.90'
+    rsync.port.should            == 22
+    rsync.path.should            == 'backups/'
+
+    File.read(rsync.instance_variable_get('@password_file').path).should == 'my_password'
   end
 
   it 'should use the defaults if a particular attribute has not been defined' do
@@ -38,10 +40,12 @@ describe Backup::Storage::RSync do
       rsync.ip       = '123.45.678.90'
     end
 
-    rsync.username.should == 'my_default_username'
-    rsync.password.should == 'my_password'
-    rsync.ip.should       == '123.45.678.90'
-    rsync.port.should     == 22
+    rsync.username.should        == 'my_default_username'
+    rsync.send(:password).should =~ /backup-rsync-password/
+    rsync.ip.should              == '123.45.678.90'
+    rsync.port.should            == 22
+
+    File.read(rsync.instance_variable_get('@password_file').path).should == 'my_password'
   end
 
   it 'should have its own defaults' do
@@ -72,7 +76,19 @@ describe Backup::Storage::RSync do
 
       rsync.expects(:create_remote_directories!)
       rsync.expects(:utility).returns('rsync')
-      rsync.expects(:run).with("rsync -z --port='22' '#{ File.join(Backup::TMP_PATH, "#{ Backup::TIME }.#{ Backup::TRIGGER }.tar") }' 'my_username@123.45.678.90:backups/#{ Backup::TRIGGER }/#{ Backup::TRIGGER }.tar'")
+      rsync.expects(:run).with("rsync -z --port='22' --password-file='#{rsync.instance_variable_get('@password_file').path}' '#{ File.join(Backup::TMP_PATH, "#{ Backup::TIME }.#{ Backup::TRIGGER }.tar") }' 'my_username@123.45.678.90:backups/#{ Backup::TRIGGER }/#{ Backup::TRIGGER }.tar'")
+
+      rsync.send(:transfer!)
+    end
+
+    it 'should not provide the --password-file option' do
+      Backup::Model.new('blah', 'blah') {}
+      file = mock("Backup::Storage::RSync::File")
+
+      rsync.password = nil
+      rsync.expects(:create_remote_directories!)
+      rsync.expects(:utility).returns('rsync')
+      rsync.expects(:run).with("rsync -z --port='22'  '#{ File.join(Backup::TMP_PATH, "#{ Backup::TIME }.#{ Backup::TRIGGER }.tar") }' 'my_username@123.45.678.90:backups/#{ Backup::TRIGGER }/#{ Backup::TRIGGER }.tar'")
 
       rsync.send(:transfer!)
     end

data/spec/syncer/rsync_spec.rb

@@ -28,12 +28,14 @@ describe Backup::Syncer::RSync do
 
   it 'should have defined the configuration properly' do
     rsync.username.should == 'my_username'
-    rsync.password.should == 'my_password'
+    rsync.password.should =~ /backup-rsync-password/
     rsync.ip.should       == '123.45.678.90'
     rsync.port.should     == "--port='22'"
     rsync.path.should     == 'backups/'
     rsync.mirror.should   == "--delete"
     rsync.compress.should == "--compress"
+
+    File.read(rsync.instance_variable_get('@password_file').path).should == 'my_password'
   end
 
   it 'should use the defaults if a particular attribute has not been defined' do
@@ -51,11 +53,13 @@ describe Backup::Syncer::RSync do
     end
 
     rsync.username.should == 'my_default_username'
-    rsync.password.should == 'my_password'
+    rsync.password.should =~ /backup-rsync-password/
     rsync.ip.should       == '123.45.678.90'
     rsync.port.should     == "--port='22'"
     rsync.mirror.should   == nil
     rsync.compress.should == nil
+
+    File.read(rsync.instance_variable_get('@password_file').path).should == 'my_password'
   end
 
   it 'should have its own defaults' do
@@ -140,7 +144,30 @@ describe Backup::Syncer::RSync do
 
   describe '#options' do
     it do
-      rsync.options.should == "--archive --delete --compress --port='22'"
+      rsync.options.should == "--archive --delete --compress --port='22' " +
+                              "--password-file='#{rsync.instance_variable_get('@password_file').path}'"
+    end
+  end
+
+  describe '#password' do
+    before do
+      Backup::Logger.stubs(:message)
+      rsync.stubs(:utility).with(:rsync).returns(:rsync)
+      rsync.stubs(:run)
+    end
+
+    it do
+      rsync.password = 'my_password'
+      rsync.expects(:remove_password_file!)
+
+      rsync.perform!
+    end
+
+    it do
+      rsync.password = nil
+      rsync.expects(:remove_password_file!)
+
+      rsync.perform!
     end
   end
 
@@ -148,7 +175,19 @@ describe Backup::Syncer::RSync do
     it 'should invoke the rsync command to transfer the files and directories' do
       Backup::Logger.expects(:message).with("Backup::Syncer::RSync started syncing '/some/random/directory' '/another/random/directory'.")
       rsync.expects(:utility).with(:rsync).returns(:rsync)
-      rsync.expects(:run).with("rsync -vhP --archive --delete --compress --port='22' '/some/random/directory' '/another/random/directory' 'my_username@123.45.678.90:backups/'")
+      rsync.expects(:remove_password_file!)
+      rsync.expects(:run).with("rsync -vhP --archive --delete --compress --port='22' --password-file='#{rsync.instance_variable_get('@password_file').path}' " +
+                               "'/some/random/directory' '/another/random/directory' 'my_username@123.45.678.90:backups/'")
+      rsync.perform!
+    end
+
+    it 'should not pass in the --password-file option' do
+      Backup::Logger.expects(:message).with("Backup::Syncer::RSync started syncing '/some/random/directory' '/another/random/directory'.")
+      rsync.password = nil
+      rsync.expects(:utility).with(:rsync).returns(:rsync)
+      rsync.expects(:remove_password_file!)
+      rsync.expects(:run).with("rsync -vhP --archive --delete --compress --port='22' " +
+                               "'/some/random/directory' '/another/random/directory' 'my_username@123.45.678.90:backups/'")
       rsync.perform!
     end
   end

metadata

@@ -2,7 +2,7 @@
 name: backup
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 3.0.14
+  version: 3.0.15
 platform: ruby
 authors: 
 - Michael van Rooijen
@@ -10,8 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-02 00:00:00 +02:00
-default_executable: 
+date: 2011-05-01 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: thor
@@ -176,7 +175,6 @@ files:
 - spec/syncer/rsync_spec.rb
 - spec/syncer/s3_spec.rb
 - spec/version_spec.rb
-has_rdoc: true
 homepage: http://rubygems.org/gems/backup
 licenses: []
 
@@ -200,9 +198,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.6.1
+rubygems_version: 1.7.2
 signing_key: 
 specification_version: 3
 summary: "Backup is a RubyGem (for UNIX-like operating systems: Linux, Mac OSX) that allows you to configure and perform backups in a simple manner using an elegant Ruby DSL. It supports various databases (MySQL, PostgreSQL, MongoDB and Redis), it supports various storage locations (Amazon S3, Rackspace Cloud Files, Dropbox, any remote server through FTP, SFTP, SCP and RSync), it provide Syncers (RSync, S3) for efficient backups, it can archive files and directories, it can cycle backups, it can do incremental backups, it can compress backups, it can encrypt backups (OpenSSL or GPG), it can notify you about successful and/or failed backups (Email, Twitter and Campfire). It is very extensible and easy to add new functionality to. It's easy to use."
 test_files: []
 
+has_rdoc: