backup 3.11.0 → 4.0.0rc1

data/lib/backup/config/dsl.rb

@@ -0,0 +1,102 @@
+# encoding: utf-8
+
+module Backup
+  module Config
+    # Context for loading user config.rb and model files.
+    class DSL
+      class Error < Backup::Error; end
+      Model = Backup::Model
+
+      class << self
+        private
+
+        # List the available database, storage, syncer, compressor, encryptor
+        # and notifier constants. These are used to define constant names within
+        # Backup::Config::DSL so that users may use a constant instead of a string.
+        # Nested namespaces are represented using Hashs. Deep nesting supported.
+        #
+        # Example, instead of:
+        #  database "MySQL" do |mysql|
+        #  sync_with "RSync::Local" do |rsync|
+        #
+        # You can do:
+        #  database MySQL do |mysql|
+        #  sync_with RSync::Local do |rsync|
+        #
+        def add_dsl_constants
+          create_modules(
+            DSL,
+            [ # Databases
+              ['MySQL', 'PostgreSQL', 'MongoDB', 'Redis', 'Riak'],
+              # Storages
+              ['S3', 'CloudFiles', 'Ninefold', 'Dropbox', 'FTP',
+              'SFTP', 'SCP', 'RSync', 'Local'],
+              # Compressors
+              ['Gzip', 'Bzip2', 'Custom', 'Pbzip2', 'Lzma'],
+              # Encryptors
+              ['OpenSSL', 'GPG'],
+              # Syncers
+              [
+                { 'Cloud' => ['CloudFiles', 'S3'] },
+                { 'RSync' => ['Push', 'Pull', 'Local'] }
+              ],
+              # Notifiers
+              ['Mail', 'Twitter', 'Campfire', 'Prowl',
+              'Hipchat', 'Pushover', 'HttpPost', 'Nagios']
+            ]
+          )
+        end
+
+        def create_modules(scope, names)
+          names.flatten.each do |name|
+            if name.is_a?(Hash)
+              name.each do |key, val|
+                create_modules(get_or_create_empty_module(scope, key), [val])
+              end
+            else
+              get_or_create_empty_module(scope, name)
+            end
+          end
+        end
+
+        def get_or_create_empty_module(scope, const)
+          if scope.const_defined?(const)
+            scope.const_get(const)
+          else
+            scope.const_set(const, Module.new)
+          end
+        end
+      end
+
+      add_dsl_constants  # add constants on load
+
+      attr_reader :_config_options
+
+      def initialize
+        @_config_options = {}
+      end
+
+      # Allow users to set command line path options in config.rb
+      [:root_path, :data_path, :tmp_path].each do |name|
+        define_method name, lambda {|path| _config_options[name] = path }
+      end
+
+      # Allows users to create preconfigured models.
+      def preconfigure(name, &block)
+        unless name.is_a?(String) && name =~ /^[A-Z]/
+          raise Error, "Preconfigured model names must be given as a string " +
+                        "and start with a capital letter."
+        end
+
+        if DSL.const_defined?(name)
+          raise Error, "'#{ name }' is already in use " +
+                        "and can not be used for a preconfigured model."
+        end
+
+        DSL.const_set(name, Class.new(Model))
+        DSL.const_get(name).preconfigure(&block)
+      end
+
+    end
+  end
+end

data/lib/backup/{configuration → config}/helpers.rb

@@ -1,9 +1,8 @@
 # encoding: utf-8
+require 'ostruct'
 
 module Backup
-  module Configuration
-    class Error < Backup::Error; end
-
+  module Config
     module Helpers
 
       def self.included(klass)
@@ -12,20 +11,16 @@ module Backup
 
       module ClassMethods
 
-        ##
-        # Returns or yields the Configuration::Store
-        # for storing pre-configured defaults for the class.
         def defaults
-          @configuration ||= Configuration::Store.new
+          @defaults ||= Config::Defaults.new
 
           if block_given?
-            yield @configuration
+            yield @defaults
           else
-            @configuration
+            @defaults
           end
         end
 
-        ##
         # Used only within the specs
         def clear_defaults!
           defaults.reset!
@@ -39,7 +34,7 @@ module Backup
           msg = "#{ self }##{ name } has been deprecated as of " +
               "backup v.#{ deprecation[:version] }"
           msg << "\n#{ deprecation[:message] }" if deprecation[:message]
-          Logger.warn Error.new(<<-EOS)
+          Logger.warn Config::Error.new(<<-EOS)
             [DEPRECATION WARNING]
             #{ msg }
           EOS
@@ -85,9 +80,8 @@ module Backup
       # If a default value was set for an invalid accessor,
       # this will raise a NameError.
       def load_defaults!
-        configuration = self.class.defaults
-        configuration._attributes.each do |name|
-          val = configuration.send(name)
+        self.class.defaults._attributes.each do |name|
+          val = self.class.defaults.send(name)
           val = val.dup rescue val
           send(:"#{ name }=", val)
         end
@@ -129,6 +123,21 @@ module Backup
         end
       end
 
+    end # Helpers
+
+    # Store for pre-configured defaults.
+    class Defaults < OpenStruct
+      # Returns an Array of all attribute method names
+      # that default values were set for.
+      def _attributes
+        @table.keys
+      end
+
+      # Used only within the specs
+      def reset!
+        @table.clear
+      end
     end
+
   end
 end

data/lib/backup/database/base.rb

@@ -5,8 +5,8 @@ module Backup
     class Error < Backup::Error; end
 
     class Base
-      include Backup::Utilities::Helpers
-      include Backup::Configuration::Helpers
+      include Utilities::Helpers
+      include Config::Helpers
 
       attr_reader :model, :database_id, :dump_path
 

data/lib/backup/database/mongodb.rb

@@ -181,24 +181,6 @@ module Backup
         cmd
       end
 
-      attr_deprecate :utility_path, :version => '3.0.21',
-          :message => 'Use Backup::Utilities.configure instead.',
-          :action => lambda {|klass, val|
-            Utilities.configure { mongodump val }
-          }
-
-      attr_deprecate :mongodump_utility, :version => '3.3.0',
-          :message => 'Use Backup::Utilities.configure instead.',
-          :action => lambda {|klass, val|
-            Utilities.configure { mongodump val }
-          }
-
-      attr_deprecate :mongo_utility, :version => '3.3.0',
-          :message => 'Use Backup::Utilities.configure instead.',
-          :action => lambda {|klass, val|
-            Utilities.configure { mongo val }
-          }
-
     end
   end
 end

data/lib/backup/database/mysql.rb

@@ -32,48 +32,28 @@ module Backup
       attr_accessor :only_tables
 
       ##
-      # Additional "mysqldump" or "innobackupex (backup creation)" options
+      # Additional "mysqldump" options
       attr_accessor :additional_options
 
-      ##
-      # Additional innobackupex log preparation phase ("apply-logs") options
-      attr_accessor :prepare_options
-
-      ##
-      # Default is :mysqldump (which is built in MySQL and generates
-      # a textual SQL file), but can be changed to :innobackupex, which
-      # has more feasible restore times for large databases.
-      # See: http://www.percona.com/doc/percona-xtrabackup/
-      attr_accessor :backup_engine
-
-      ##
-      # If set the backup engine command block is executed as the given user
-      attr_accessor :sudo_user
-
-      ##
-      # If set, do not suppress innobackupdb output (useful for debugging)
-      attr_accessor :verbose
-
       def initialize(model, database_id = nil, &block)
         super
         instance_eval(&block) if block_given?
 
         @name ||= :all
-        @backup_engine ||= :mysqldump
       end
 
       ##
-      # Performs the mysqldump or innobackupex command and outputs
-      # the dump file in the +dump_path+ using +dump_filename+.
+      # Performs the mysqldump command and outputs the dump file
+      # in the +dump_path+ using +dump_filename+.
       #
-      #   <trigger>/databases/MySQL[-<database_id>].[sql|tar][.gz]
+      #   <trigger>/databases/MySQL[-<database_id>].sql[.gz]
       def perform!
         super
 
         pipeline = Pipeline.new
-        dump_ext = sql_backup? ? 'sql' : 'tar'
+        dump_ext = 'sql'
 
-        pipeline << sudo_option(sql_backup? ? mysqldump : innobackupex)
+        pipeline << mysqldump
 
         model.compressor.compress_with do |command, ext|
           pipeline << command
@@ -119,10 +99,6 @@ module Backup
         Array(additional_options).join(' ')
       end
 
-      def user_prepare_options
-        Array(prepare_options).join(' ')
-      end
-
       def name_option
         dump_all? ? '--all-databases' : name
       end
@@ -142,51 +118,6 @@ module Backup
         name == :all
       end
 
-      attr_deprecate :utility_path, :version => '3.0.21',
-          :message => 'Use Backup::Utilities.configure instead.',
-          :action => lambda {|klass, val|
-            Utilities.configure { mysqldump val }
-          }
-
-      attr_deprecate :mysqldump_utility, :version => '3.3.0',
-          :message => 'Use Backup::Utilities.configure instead.',
-          :action => lambda {|klass, val|
-            Utilities.configure { mysqldump val }
-          }
-
-      def sql_backup?
-        backup_engine.to_sym == :mysqldump
-      end
-
-      def innobackupex
-        # Creation phase
-        "#{ utility(:innobackupex) } #{ credential_options } " +
-        "#{ connectivity_options } #{ user_options } " +
-        "--no-timestamp #{ temp_dir } #{ quiet_option } && " +
-        # Log applying phase (prepare for restore)
-        "#{ utility(:innobackupex) } --apply-log #{ temp_dir } " +
-        "#{ user_prepare_options }  #{ quiet_option } && " +
-        # Move files to tar-ed stream on stdout
-        "#{ utility(:tar) } --remove-files -cf -  " +
-        "-C #{ File.dirname(temp_dir) } #{ File.basename(temp_dir) }"
-      end
-
-      def sudo_option(command_block)
-        return command_block unless sudo_user
-
-        "sudo -s -u #{ sudo_user } -- <<END_OF_SUDO\n" +
-        "#{command_block}\n" +
-        "END_OF_SUDO\n"
-      end
-
-      def quiet_option
-        verbose ? "" : " 2> /dev/null "
-      end
-
-      def temp_dir
-        File.join(dump_path, dump_filename + ".bkpdir")
-      end
-
     end
   end
 end

data/lib/backup/database/postgresql.rb

@@ -128,18 +128,6 @@ module Backup
         name == :all
       end
 
-      attr_deprecate :utility_path, :version => '3.0.21',
-          :message => 'Use Backup::Utilities.configure instead.',
-          :action => lambda {|klass, val|
-            Utilities.configure { pg_dump val }
-          }
-
-      attr_deprecate :pg_dump_utility, :version => '3.3.0',
-          :message => 'Use Backup::Utilities.configure instead.',
-          :action => lambda {|klass, val|
-            Utilities.configure { pg_dump val }
-          }
-
     end
   end
 end

data/lib/backup/database/redis.rb

@@ -5,70 +5,121 @@ module Backup
     class Redis < Base
       class Error < Backup::Error; end
 
+      MODES = [:copy, :sync]
+
       ##
-      # Name of the redis dump file.
+      # Mode of operation.
+      #
+      # [:copy]
+      #   Copies the redis dump file specified by {#rdb_path}.
+      #   This data will be current as of the last RDB Snapshot
+      #   performed by the server (per your redis.conf settings).
+      #   You may set {#invoke_save} to +true+ to have Backup issue
+      #   a +SAVE+ command to update the dump file with the current
+      #   data before performing the copy.
+      #
+      # [:sync]
+      #   Performs a dump of your redis data using +redis-cli --rdb -+.
+      #   Redis implements this internally using a +SYNC+ command.
+      #   The operation is analogous to requesting a +BGSAVE+, then having the
+      #   dump returned. This mode is capable of dumping data from a local or
+      #   remote server. Requires Redis v2.6 or better.
       #
-      # This is set in `redis.conf` as `dbfilename`.
-      # This must be set to the name of that file without the `.rdb` extension.
-      # Default: 'dump'
-      attr_accessor :name
+      # Defaults to +:copy+.
+      attr_accessor :mode
 
       ##
-      # Path to the redis dump file.
+      # Full path to the redis dump file.
       #
-      # This is set in `redis.conf` as `dir`.
-      attr_accessor :path
+      # Required when {#mode} is +:copy+.
+      attr_accessor :rdb_path
 
       ##
-      # Password for the redis-cli utility to perform the `SAVE` command
-      # if +invoke_save+ is set `true`.
-      attr_accessor :password
+      # Perform a +SAVE+ command using the +redis-cli+ utility
+      # before copying the dump file specified by {#rdb_path}.
+      #
+      # Only valid when {#mode} is +:copy+.
+      attr_accessor :invoke_save
 
       ##
-      # Connectivity options for the +invoke_save+ option.
+      # Connectivity options for the +redis-cli+ utility.
       attr_accessor :host, :port, :socket
 
       ##
-      # Determines whether Backup should invoke the `SAVE` command through
-      # the `redis-cli` utility to persist the most recent data before
-      # copying the dump file specified by +path+ and +name+.
-      attr_accessor :invoke_save
+      # Password for the +redis-cli+ utility.
+      attr_accessor :password
 
       ##
-      # Additional "redis-cli" options
+      # Additional options for the +redis-cli+ utility.
       attr_accessor :additional_options
 
       def initialize(model, database_id = nil, &block)
         super
         instance_eval(&block) if block_given?
 
-        @name ||= 'dump'
+        @mode ||= :copy
+
+        unless MODES.include?(mode)
+          raise Error, "'#{ mode }' is not a valid mode"
+        end
+
+        if mode == :copy && rdb_path.nil?
+          raise Error, '`rdb_path` must be set when `mode` is :copy'
+        end
       end
 
       ##
-      # Copies and optionally compresses the Redis dump file to the
-      # +dump_path+ using the +dump_filename+.
+      # Performs the dump based on {#mode} and stores the Redis dump file
+      # to the +dump_path+ using the +dump_filename+.
       #
       #   <trigger>/databases/Redis[-<database_id>].rdb[.gz]
-      #
-      # If +invoke_save+ is true, `redis-cli SAVE` will be invoked.
       def perform!
         super
 
-        invoke_save! if invoke_save
-        copy!
+        case mode
+        when :sync
+          # messages output by `redis-cli --rdb` on $stderr
+          Logger.configure do
+            ignore_warning(/Transfer finished with success/)
+            ignore_warning(/SYNC sent to master/)
+          end
+          sync!
+        when :copy
+          save! if invoke_save
+          copy!
+        end
 
         log!(:finished)
       end
 
       private
 
-      def invoke_save!
-        resp = run(redis_save_cmd)
+      def sync!
+        pipeline = Pipeline.new
+        dump_ext = 'rdb'
+
+        pipeline << "#{ redis_cli_cmd } --rdb -"
+
+        model.compressor.compress_with do |command, ext|
+          pipeline << command
+          dump_ext << ext
+        end if model.compressor
+
+        pipeline << "#{ utility(:cat) } > " +
+            "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
+
+        pipeline.run
+
+        unless pipeline.success?
+          raise Error, "Dump Failed!\n" + pipeline.error_messages
+        end
+      end
+
+      def save!
+        resp = run("#{ redis_cli_cmd } SAVE")
         unless resp =~ /OK$/
           raise Error, <<-EOS
-            Could not invoke the Redis SAVE command.
-            Command was: #{ redis_save_cmd }
+            Failed to invoke the `SAVE` command
             Response was: #{ resp }
           EOS
         end
@@ -84,27 +135,26 @@ module Backup
       end
 
       def copy!
-        src_path = File.join(path, name + '.rdb')
-        unless File.exist?(src_path)
+        unless File.exist?(rdb_path)
           raise Error, <<-EOS
             Redis database dump not found
-            File path was #{ src_path }
+            `rdb_path` was '#{ rdb_path }'
           EOS
         end
 
         dst_path = File.join(dump_path, dump_filename + '.rdb')
         if model.compressor
           model.compressor.compress_with do |command, ext|
-            run("#{ command } -c '#{ src_path }' > '#{ dst_path + ext }'")
+            run("#{ command } -c '#{ rdb_path }' > '#{ dst_path + ext }'")
           end
         else
-          FileUtils.cp(src_path, dst_path)
+          FileUtils.cp(rdb_path, dst_path)
         end
       end
 
-      def redis_save_cmd
+      def redis_cli_cmd
         "#{ utility('redis-cli') } #{ password_option } " +
-        "#{ connectivity_options } #{ user_options } SAVE"
+        "#{ connectivity_options } #{ user_options }"
       end
 
       def password_option
@@ -124,18 +174,6 @@ module Backup
         Array(additional_options).join(' ')
       end
 
-      attr_deprecate :utility_path, :version => '3.0.21',
-          :message => 'Use Backup::Utilities.configure instead.',
-          :action => lambda {|klass, val|
-            Utilities.configure { redis_cli val }
-          }
-
-      attr_deprecate :redis_cli_utility, :version => '3.3.0',
-          :message => 'Use Backup::Utilities.configure instead.',
-          :action => lambda {|klass, val|
-            Utilities.configure { redis_cli val }
-          }
-
     end
   end
 end