backup-remote 0.0.2

data/lib/backup/database/base.rb

@@ -0,0 +1,85 @@
+# encoding: utf-8
+
+module Backup
+  module Database
+    class Error < Backup::Error; end
+
+    class Base
+      include Utilities::Helpers
+      include Config::Helpers
+
+      attr_reader :model, :database_id, :dump_path
+
+      ##
+      # If given, +database_id+ will be appended to the #dump_filename.
+      # This is required if multiple Databases of the same class are added to
+      # the model.
+      def initialize(model, database_id = nil)
+        @model = model
+        @database_id = database_id.to_s.gsub(/\W/, '_') if database_id
+        @dump_path = File.join(Config.tmp_path, model.trigger, 'databases')
+        load_defaults!
+      end
+
+      def perform!
+        log!(:started)
+        prepare!
+      end
+
+      private
+
+      def prepare!
+        FileUtils.mkdir_p(dump_path)
+      end
+
+      ##
+      # Sets the base filename for the final dump file to be saved in +dump_path+,
+      # based on the class name. e.g. databases/MySQL.sql
+      #
+      # +database_id+ will be appended if it is defined.
+      # e.g. databases/MySQL-database_id.sql
+      #
+      # If multiple Databases of the same class are defined and no +database_id+
+      # is defined, the user will be warned and one will be auto-generated.
+      #
+      # Model#initialize calls this method *after* all defined databases have
+      # been initialized so `backup check` can report these warnings.
+      def dump_filename
+        @dump_filename ||= begin
+          unless database_id
+            if model.databases.select {|d| d.class == self.class }.count > 1
+              sleep 1; @database_id = Time.now.to_i.to_s[-5, 5]
+              Logger.warn Error.new(<<-EOS)
+                Database Identifier Missing
+                When multiple Databases are configured in a single Backup Model
+                that have the same class (MySQL, PostgreSQL, etc.), the optional
+                +database_id+ must be specified to uniquely identify each instance.
+                e.g. database MySQL, :database_id do |db|
+                This will result in an output file in your final backup package like:
+                databases/MySQL-database_id.sql
+
+                Backup has auto-generated an identifier (#{ database_id }) for this
+                database dump and will now continue.
+              EOS
+            end
+          end
+
+          self.class.name.split('::').last + (database_id ? "-#{ database_id }" : '')
+        end
+      end
+
+      def database_name
+        @database_name ||= self.class.to_s.sub('Backup::', '') +
+            (database_id ? " (#{ database_id })" : '')
+      end
+
+      def log!(action)
+        msg = case action
+              when :started then 'Started...'
+              when :finished then 'Finished!'
+              end
+        Logger.info "#{ database_name } #{ msg }"
+      end
+    end
+  end
+end

data/lib/backup/database/mongodb.rb

@@ -0,0 +1,187 @@
+# encoding: utf-8
+
+module Backup
+  module Database
+    class MongoDB < Base
+      class Error < Backup::Error; end
+
+      ##
+      # Name of the database that needs to get dumped
+      attr_accessor :name
+
+      ##
+      # Credentials for the specified database
+      attr_accessor :username, :password, :authdb
+
+      ##
+      # Connectivity options
+      attr_accessor :host, :port
+
+      ##
+      # IPv6 support (disabled by default)
+      attr_accessor :ipv6
+
+      ##
+      # Collections to dump, collections that aren't specified won't get dumped
+      attr_accessor :only_collections
+
+      ##
+      # Additional "mongodump" options
+      attr_accessor :additional_options
+
+      ##
+      # Forces mongod to flush all pending write operations to the disk and
+      # locks the entire mongod instance to prevent additional writes until the
+      # dump is complete.
+      #
+      # Note that if Profiling is enabled, this will disable it and will not
+      # re-enable it after the dump is complete.
+      attr_accessor :lock
+
+      ##
+      # Creates a dump of the database that includes an oplog, to create a
+      # point-in-time snapshot of the state of a mongod instance.
+      #
+      # If this option is used, you would not use the `lock` option.
+      #
+      # This will only work against nodes that maintain a oplog.
+      # This includes all members of a replica set, as well as master nodes in
+      # master/slave replication deployments.
+      attr_accessor :oplog
+
+      def initialize(model, database_id = nil, &block)
+        super
+        instance_eval(&block) if block_given?
+      end
+
+      def perform!
+        super
+
+        lock_database if @lock
+        dump!
+        package!
+
+      ensure
+        unlock_database if @lock
+      end
+
+      private
+
+      ##
+      # Performs all required mongodump commands, dumping the output files
+      # into the +dump_packaging_path+ directory for packaging.
+      def dump!
+        FileUtils.mkdir_p dump_packaging_path
+
+        collections = Array(only_collections)
+        if collections.empty?
+          run(mongodump)
+        else
+          collections.each do |collection|
+            run("#{ mongodump } --collection='#{ collection }'")
+          end
+        end
+      end
+
+      ##
+      # Creates a tar archive of the +dump_packaging_path+ directory
+      # and stores it in the +dump_path+ using +dump_filename+.
+      #
+      #   <trigger>/databases/MongoDB[-<database_id>].tar[.gz]
+      #
+      # If successful, +dump_packaging_path+ is removed.
+      def package!
+        pipeline = Pipeline.new
+        dump_ext = 'tar'
+
+        pipeline << "#{ utility(:tar) } -cf - " +
+            "-C '#{ dump_path }' '#{ dump_filename }'"
+
+        model.compressor.compress_with do |command, ext|
+          pipeline << command
+          dump_ext << ext
+        end if model.compressor
+
+        pipeline << "#{ utility(:cat) } > " +
+            "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
+
+        pipeline.run
+        if pipeline.success?
+          FileUtils.rm_rf dump_packaging_path
+          log!(:finished)
+        else
+          raise Error, "Dump Failed!\n" + pipeline.error_messages
+        end
+      end
+
+      def dump_packaging_path
+        File.join(dump_path, dump_filename)
+      end
+
+      def mongodump
+        "#{ utility(:mongodump) } #{ name_option } #{ credential_options } " +
+        "#{ connectivity_options } #{ ipv6_option } #{ oplog_option } " +
+        "#{ user_options } --out='#{ dump_packaging_path }'"
+      end
+
+      def name_option
+        "--db='#{ name }'" if name
+      end
+
+      def credential_options
+        opts = []
+        opts << "--username='#{ username }'" if username
+        opts << "--password='#{ password }'" if password
+        opts << "--authenticationDatabase='#{ authdb }'" if authdb
+        opts.join(' ')
+      end
+
+      def connectivity_options
+        opts = []
+        opts << "--host='#{ host }'" if host
+        opts << "--port='#{ port }'" if port
+        opts.join(' ')
+      end
+
+      def ipv6_option
+        '--ipv6' if ipv6
+      end
+
+      def oplog_option
+        '--oplog' if oplog
+      end
+
+      def user_options
+        Array(additional_options).join(' ')
+      end
+
+      def lock_database
+        lock_command = <<-EOS.gsub(/^ +/, '')
+          echo 'use admin
+          db.setProfilingLevel(0)
+          db.fsyncLock()' | #{ mongo_shell }
+        EOS
+
+        run(lock_command)
+      end
+
+      def unlock_database
+        unlock_command = <<-EOS.gsub(/^ +/, '')
+          echo 'use admin
+          db.fsyncUnlock()' | #{ mongo_shell }
+        EOS
+
+        run(unlock_command)
+      end
+
+      def mongo_shell
+        cmd = "#{ utility(:mongo) } #{ connectivity_options }".rstrip
+        cmd << " #{ credential_options }".rstrip
+        cmd << " #{ ipv6_option }".rstrip
+        cmd << " '#{ name }'" if name
+        cmd
+      end
+
+    end
+  end
+end

data/lib/backup/database/mysql.rb

@@ -0,0 +1,192 @@
+# encoding: utf-8
+
+module Backup
+  module Database
+    class MySQL < Base
+      class Error < Backup::Error; end
+
+      ##
+      # Name of the database that needs to get dumped
+      # To dump all databases, set this to `:all` or leave blank.
+      attr_accessor :name
+
+      ##
+      # Credentials for the specified database
+      attr_accessor :username, :password
+
+      ##
+      # Connectivity options
+      attr_accessor :host, :port, :socket
+
+      ##
+      # Tables to skip while dumping the database
+      #
+      # If `name` is set to :all (or not specified), these must include
+      # a database name. e.g. 'name.table'.
+      # If `name` is given, these may simply be table names.
+      attr_accessor :skip_tables
+
+      ##
+      # Tables to dump. This in only valid if `name` is specified.
+      # If none are given, the entire database will be dumped.
+      attr_accessor :only_tables
+
+      ##
+      # Additional "mysqldump" or "innobackupex (backup creation)" options
+      attr_accessor :additional_options
+
+      ##
+      # Additional innobackupex log preparation phase ("apply-logs") options
+      attr_accessor :prepare_options
+
+      ##
+      # Default is :mysqldump (which is built in MySQL and generates
+      # a textual SQL file), but can be changed to :innobackupex, which
+      # has more feasible restore times for large databases.
+      # See: http://www.percona.com/doc/percona-xtrabackup/
+      attr_accessor :backup_engine
+
+      ##
+      # If true (which is the default behaviour), the backup will be prepared
+      # after it has been successfuly created. This option is only valid if
+      # :backup_engine is set to :innobackupex.
+      attr_accessor :prepare_backup
+
+      ##
+      # If set the backup engine command block is executed as the given user
+      attr_accessor :sudo_user
+
+      ##
+      # If set, do not suppress innobackupdb output (useful for debugging)
+      attr_accessor :verbose
+
+      def initialize(model, database_id = nil, &block)
+        super
+        instance_eval(&block) if block_given?
+
+        @name ||= :all
+        @backup_engine ||= :mysqldump
+        @prepare_backup = true if @prepare_backup.nil?
+      end
+
+      ##
+      # Performs the mysqldump or innobackupex command and outputs
+      # the dump file in the +dump_path+ using +dump_filename+.
+      #
+      #   <trigger>/databases/MySQL[-<database_id>].[sql|tar][.gz]
+      def perform!
+        super
+
+        pipeline = Pipeline.new
+        dump_ext = sql_backup? ? 'sql' : 'tar'
+
+        pipeline << sudo_option(sql_backup? ? mysqldump : innobackupex)
+
+        model.compressor.compress_with do |command, ext|
+          pipeline << command
+          dump_ext << ext
+        end if model.compressor
+
+        pipeline << "#{ utility(:cat) } > " +
+            "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
+
+        pipeline.run
+        if pipeline.success?
+          log!(:finished)
+        else
+          raise Error, "Dump Failed!\n" + pipeline.error_messages
+        end
+      end
+
+      private
+
+      def mysqldump
+        "#{ utility(:mysqldump) } #{ user_options } #{ credential_options } " +
+        "#{ connectivity_options } #{ name_option } " +
+        "#{ tables_to_dump } #{ tables_to_skip }"
+      end
+
+      def credential_options
+        opts = []
+        opts << "--user=#{ Shellwords.escape(username) }" if username
+        opts << "--password=#{ Shellwords.escape(password) }" if password
+        opts.join(' ')
+      end
+
+      def connectivity_options
+        return "--socket='#{ socket }'" if socket
+
+        opts = []
+        opts << "--host='#{ host }'" if host
+        opts << "--port='#{ port }'" if port
+        opts.join(' ')
+      end
+
+      def user_options
+        Array(additional_options).join(' ')
+      end
+
+      def user_prepare_options
+        Array(prepare_options).join(' ')
+      end
+
+      def name_option
+        dump_all? ? '--all-databases' : name
+      end
+
+      def tables_to_dump
+        Array(only_tables).join(' ') unless dump_all?
+      end
+
+      def tables_to_skip
+        Array(skip_tables).map do |table|
+          table = (dump_all? || table['.']) ? table : "#{ name }.#{ table }"
+          "--ignore-table='#{ table }'"
+        end.join(' ')
+      end
+
+      def dump_all?
+        name == :all
+      end
+
+      def sql_backup?
+        backup_engine.to_sym == :mysqldump
+      end
+
+      def innobackupex
+        # Creation phase
+        "#{ utility(:innobackupex) } #{ credential_options } " +
+        "#{ connectivity_options } #{ user_options } " +
+        "--no-timestamp #{ temp_dir } #{ quiet_option } && " +
+        innobackupex_prepare +
+        # Move files to tar-ed stream on stdout
+        "#{ utility(:tar) } --remove-files -cf -  " +
+        "-C #{ File.dirname(temp_dir) } #{ File.basename(temp_dir) }"
+      end
+
+      def innobackupex_prepare
+        return "" unless @prepare_backup
+        # Log applying phase (prepare for restore)
+        "#{ utility(:innobackupex) } --apply-log #{ temp_dir } " +
+        "#{ user_prepare_options }  #{ quiet_option } && "
+      end
+
+      def sudo_option(command_block)
+        return command_block unless sudo_user
+
+        "sudo -s -u #{ sudo_user } -- <<END_OF_SUDO\n" +
+        "#{command_block}\n" +
+        "END_OF_SUDO\n"
+      end
+
+      def quiet_option
+        verbose ? "" : " 2> /dev/null "
+      end
+
+      def temp_dir
+        File.join(dump_path, dump_filename + ".bkpdir")
+      end
+
+    end
+  end
+end