nfm-backup 4.0.1a

data/lib/backup/database/mongodb.rb

@@ -0,0 +1,186 @@
+# encoding: utf-8
+
+module Backup
+  module Database
+    class MongoDB < Base
+      class Error < Backup::Error; end
+
+      ##
+      # Name of the database that needs to get dumped
+      attr_accessor :name
+
+      ##
+      # Credentials for the specified database
+      attr_accessor :username, :password
+
+      ##
+      # Connectivity options
+      attr_accessor :host, :port
+
+      ##
+      # IPv6 support (disabled by default)
+      attr_accessor :ipv6
+
+      ##
+      # Collections to dump, collections that aren't specified won't get dumped
+      attr_accessor :only_collections
+
+      ##
+      # Additional "mongodump" options
+      attr_accessor :additional_options
+
+      ##
+      # Forces mongod to flush all pending write operations to the disk and
+      # locks the entire mongod instance to prevent additional writes until the
+      # dump is complete.
+      #
+      # Note that if Profiling is enabled, this will disable it and will not
+      # re-enable it after the dump is complete.
+      attr_accessor :lock
+
+      ##
+      # Creates a dump of the database that includes an oplog, to create a
+      # point-in-time snapshot of the state of a mongod instance.
+      #
+      # If this option is used, you would not use the `lock` option.
+      #
+      # This will only work against nodes that maintain a oplog.
+      # This includes all members of a replica set, as well as master nodes in
+      # master/slave replication deployments.
+      attr_accessor :oplog
+
+      def initialize(model, database_id = nil, &block)
+        super
+        instance_eval(&block) if block_given?
+      end
+
+      def perform!
+        super
+
+        lock_database if @lock
+        dump!
+        package!
+
+      ensure
+        unlock_database if @lock
+      end
+
+      private
+
+      ##
+      # Performs all required mongodump commands, dumping the output files
+      # into the +dump_packaging_path+ directory for packaging.
+      def dump!
+        FileUtils.mkdir_p dump_packaging_path
+
+        collections = Array(only_collections)
+        if collections.empty?
+          run(mongodump)
+        else
+          collections.each do |collection|
+            run("#{ mongodump } --collection='#{ collection }'")
+          end
+        end
+      end
+
+      ##
+      # Creates a tar archive of the +dump_packaging_path+ directory
+      # and stores it in the +dump_path+ using +dump_filename+.
+      #
+      #   <trigger>/databases/MongoDB[-<database_id>].tar[.gz]
+      #
+      # If successful, +dump_packaging_path+ is removed.
+      def package!
+        pipeline = Pipeline.new
+        dump_ext = 'tar'
+
+        pipeline << "#{ utility(:tar) } -cf - " +
+            "-C '#{ dump_path }' '#{ dump_filename }'"
+
+        model.compressor.compress_with do |command, ext|
+          pipeline << command
+          dump_ext << ext
+        end if model.compressor
+
+        pipeline << "#{ utility(:cat) } > " +
+            "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
+
+        pipeline.run
+        if pipeline.success?
+          FileUtils.rm_rf dump_packaging_path
+          log!(:finished)
+        else
+          raise Error, "Dump Failed!\n" + pipeline.error_messages
+        end
+      end
+
+      def dump_packaging_path
+        File.join(dump_path, dump_filename)
+      end
+
+      def mongodump
+        "#{ utility(:mongodump) } #{ name_option } #{ credential_options } " +
+        "#{ connectivity_options } #{ ipv6_option } #{ oplog_option } " +
+        "#{ user_options } --out='#{ dump_packaging_path }'"
+      end
+
+      def name_option
+        "--db='#{ name }'" if name
+      end
+
+      def credential_options
+        opts = []
+        opts << "--username='#{ username }'" if username
+        opts << "--password='#{ password }'" if password
+        opts.join(' ')
+      end
+
+      def connectivity_options
+        opts = []
+        opts << "--host='#{ host }'" if host
+        opts << "--port='#{ port }'" if port
+        opts.join(' ')
+      end
+
+      def ipv6_option
+        '--ipv6' if ipv6
+      end
+
+      def oplog_option
+        '--oplog' if oplog
+      end
+
+      def user_options
+        Array(additional_options).join(' ')
+      end
+
+      def lock_database
+        lock_command = <<-EOS.gsub(/^ +/, '')
+          echo 'use admin
+          db.setProfilingLevel(0)
+          db.fsyncLock()' | #{ mongo_shell }
+        EOS
+
+        run(lock_command)
+      end
+
+      def unlock_database
+        unlock_command = <<-EOS.gsub(/^ +/, '')
+          echo 'use admin
+          db.fsyncUnlock()' | #{ mongo_shell }
+        EOS
+
+        run(unlock_command)
+      end
+
+      def mongo_shell
+        cmd = "#{ utility(:mongo) } #{ connectivity_options }".rstrip
+        cmd << " #{ credential_options }".rstrip
+        cmd << " #{ ipv6_option }".rstrip
+        cmd << " '#{ name }'" if name
+        cmd
+      end
+
+    end
+  end
+end

data/lib/backup/database/mysql.rb

@@ -0,0 +1,123 @@
+# encoding: utf-8
+
+module Backup
+  module Database
+    class MySQL < Base
+      class Error < Backup::Error; end
+
+      ##
+      # Name of the database that needs to get dumped
+      # To dump all databases, set this to `:all` or leave blank.
+      attr_accessor :name
+
+      ##
+      # Credentials for the specified database
+      attr_accessor :username, :password
+
+      ##
+      # Connectivity options
+      attr_accessor :host, :port, :socket
+
+      ##
+      # Tables to skip while dumping the database
+      #
+      # If `name` is set to :all (or not specified), these must include
+      # a database name. e.g. 'name.table'.
+      # If `name` is given, these may simply be table names.
+      attr_accessor :skip_tables
+
+      ##
+      # Tables to dump. This in only valid if `name` is specified.
+      # If none are given, the entire database will be dumped.
+      attr_accessor :only_tables
+
+      ##
+      # Additional "mysqldump" options
+      attr_accessor :additional_options
+
+      def initialize(model, database_id = nil, &block)
+        super
+        instance_eval(&block) if block_given?
+
+        @name ||= :all
+      end
+
+      ##
+      # Performs the mysqldump command and outputs the dump file
+      # in the +dump_path+ using +dump_filename+.
+      #
+      #   <trigger>/databases/MySQL[-<database_id>].sql[.gz]
+      def perform!
+        super
+
+        pipeline = Pipeline.new
+        dump_ext = 'sql'
+
+        pipeline << mysqldump
+
+        model.compressor.compress_with do |command, ext|
+          pipeline << command
+          dump_ext << ext
+        end if model.compressor
+
+        pipeline << "#{ utility(:cat) } > " +
+            "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
+
+        pipeline.run
+        if pipeline.success?
+          log!(:finished)
+        else
+          raise Error, "Dump Failed!\n" + pipeline.error_messages
+        end
+      end
+
+      private
+
+      def mysqldump
+        "#{ utility(:mysqldump) } #{ credential_options } " +
+        "#{ connectivity_options } #{ user_options } #{ name_option } " +
+        "#{ tables_to_dump } #{ tables_to_skip }"
+      end
+
+      def credential_options
+        opts = []
+        opts << "--user='#{ username }'" if username
+        opts << "--password='#{ password }'" if password
+        opts.join(' ')
+      end
+
+      def connectivity_options
+        return "--socket='#{ socket }'" if socket
+
+        opts = []
+        opts << "--host='#{ host }'" if host
+        opts << "--port='#{ port }'" if port
+        opts.join(' ')
+      end
+
+      def user_options
+        Array(additional_options).join(' ')
+      end
+
+      def name_option
+        dump_all? ? '--all-databases' : name
+      end
+
+      def tables_to_dump
+        Array(only_tables).join(' ') unless dump_all?
+      end
+
+      def tables_to_skip
+        Array(skip_tables).map do |table|
+          table = (dump_all? || table['.']) ? table : "#{ name }.#{ table }"
+          "--ignore-table='#{ table }'"
+        end.join(' ')
+      end
+
+      def dump_all?
+        name == :all
+      end
+
+    end
+  end
+end

data/lib/backup/database/postgresql.rb

@@ -0,0 +1,133 @@
+# encoding: utf-8
+
+module Backup
+  module Database
+    class PostgreSQL < Base
+      class Error < Backup::Error; end
+
+      ##
+      # Name of the database that needs to get dumped.
+      # To dump all databases, set this to `:all` or leave blank.
+      # +username+ must be a PostgreSQL superuser to run `pg_dumpall`.
+      attr_accessor :name
+
+      ##
+      # Credentials for the specified database
+      attr_accessor :username, :password
+
+      ##
+      # If set the pg_dump(all) command is executed as the given user
+      attr_accessor :sudo_user
+
+      ##
+      # Connectivity options
+      attr_accessor :host, :port, :socket
+
+      ##
+      # Tables to skip while dumping the database.
+      # If `name` is set to :all (or not specified), these are ignored.
+      attr_accessor :skip_tables
+
+      ##
+      # Tables to dump. This in only valid if `name` is specified.
+      # If none are given, the entire database will be dumped.
+      attr_accessor :only_tables
+
+      ##
+      # Additional "pg_dump" or "pg_dumpall" options
+      attr_accessor :additional_options
+
+      def initialize(model, database_id = nil, &block)
+        super
+        instance_eval(&block) if block_given?
+
+        @name ||= :all
+      end
+
+      ##
+      # Performs the mysqldump command and outputs the dump file
+      # in the +dump_path+ using +dump_filename+.
+      #
+      #   <trigger>/databases/PostgreSQL[-<database_id>].sql[.gz]
+      def perform!
+        super
+
+        pipeline = Pipeline.new
+        dump_ext = 'sql'
+
+        pipeline << (dump_all? ? pgdumpall : pgdump)
+
+        model.compressor.compress_with do |command, ext|
+          pipeline << command
+          dump_ext << ext
+        end if model.compressor
+
+        pipeline << "#{ utility(:cat) } > " +
+            "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
+
+        pipeline.run
+        if pipeline.success?
+          log!(:finished)
+        else
+          raise Error, "Dump Failed!\n" + pipeline.error_messages
+        end
+      end
+
+      def pgdump
+        "#{ password_option }" +
+        "#{ sudo_option }" +
+        "#{ utility(:pg_dump) } #{ username_option } #{ connectivity_options } " +
+        "#{ user_options } #{ tables_to_dump } #{ tables_to_skip } #{ name }"
+      end
+
+      def pgdumpall
+        "#{ password_option }" +
+        "#{ sudo_option }" +
+        "#{ utility(:pg_dumpall) } #{ username_option } " +
+        "#{ connectivity_options } #{ user_options }"
+      end
+
+      def password_option
+        "PGPASSWORD='#{ password }' " if password
+      end
+
+      def sudo_option
+        "#{ utility(:sudo) } -n -u #{ sudo_user } " if sudo_user
+      end
+
+      def username_option
+        "--username='#{ username }'" if username
+      end
+
+      def connectivity_options
+        return "--host='#{ socket }'" if socket
+
+        opts = []
+        opts << "--host='#{ host }'" if host
+        opts << "--port='#{ port }'" if port
+        opts.join(' ')
+      end
+
+      def user_options
+        Array(additional_options).join(' ')
+      end
+
+      def tables_to_dump
+        Array(only_tables).map do |table|
+          "--table='#{ table }'"
+        end.join(' ')
+      end
+
+      def tables_to_skip
+        Array(skip_tables).map do |table|
+          "--exclude-table='#{ table }'"
+        end.join(' ')
+      end
+
+      def dump_all?
+        name == :all
+      end
+
+    end
+  end
+end

data/lib/backup/database/redis.rb

@@ -0,0 +1,179 @@
+# encoding: utf-8
+
+module Backup
+  module Database
+    class Redis < Base
+      class Error < Backup::Error; end
+
+      MODES = [:copy, :sync]
+
+      ##
+      # Mode of operation.
+      #
+      # [:copy]
+      #   Copies the redis dump file specified by {#rdb_path}.
+      #   This data will be current as of the last RDB Snapshot
+      #   performed by the server (per your redis.conf settings).
+      #   You may set {#invoke_save} to +true+ to have Backup issue
+      #   a +SAVE+ command to update the dump file with the current
+      #   data before performing the copy.
+      #
+      # [:sync]
+      #   Performs a dump of your redis data using +redis-cli --rdb -+.
+      #   Redis implements this internally using a +SYNC+ command.
+      #   The operation is analogous to requesting a +BGSAVE+, then having the
+      #   dump returned. This mode is capable of dumping data from a local or
+      #   remote server. Requires Redis v2.6 or better.
+      #
+      # Defaults to +:copy+.
+      attr_accessor :mode
+
+      ##
+      # Full path to the redis dump file.
+      #
+      # Required when {#mode} is +:copy+.
+      attr_accessor :rdb_path
+
+      ##
+      # Perform a +SAVE+ command using the +redis-cli+ utility
+      # before copying the dump file specified by {#rdb_path}.
+      #
+      # Only valid when {#mode} is +:copy+.
+      attr_accessor :invoke_save
+
+      ##
+      # Connectivity options for the +redis-cli+ utility.
+      attr_accessor :host, :port, :socket
+
+      ##
+      # Password for the +redis-cli+ utility.
+      attr_accessor :password
+
+      ##
+      # Additional options for the +redis-cli+ utility.
+      attr_accessor :additional_options
+
+      def initialize(model, database_id = nil, &block)
+        super
+        instance_eval(&block) if block_given?
+
+        @mode ||= :copy
+
+        unless MODES.include?(mode)
+          raise Error, "'#{ mode }' is not a valid mode"
+        end
+
+        if mode == :copy && rdb_path.nil?
+          raise Error, '`rdb_path` must be set when `mode` is :copy'
+        end
+      end
+
+      ##
+      # Performs the dump based on {#mode} and stores the Redis dump file
+      # to the +dump_path+ using the +dump_filename+.
+      #
+      #   <trigger>/databases/Redis[-<database_id>].rdb[.gz]
+      def perform!
+        super
+
+        case mode
+        when :sync
+          # messages output by `redis-cli --rdb` on $stderr
+          Logger.configure do
+            ignore_warning(/Transfer finished with success/)
+            ignore_warning(/SYNC sent to master/)
+          end
+          sync!
+        when :copy
+          save! if invoke_save
+          copy!
+        end
+
+        log!(:finished)
+      end
+
+      private
+
+      def sync!
+        pipeline = Pipeline.new
+        dump_ext = 'rdb'
+
+        pipeline << "#{ redis_cli_cmd } --rdb -"
+
+        model.compressor.compress_with do |command, ext|
+          pipeline << command
+          dump_ext << ext
+        end if model.compressor
+
+        pipeline << "#{ utility(:cat) } > " +
+            "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
+
+        pipeline.run
+
+        unless pipeline.success?
+          raise Error, "Dump Failed!\n" + pipeline.error_messages
+        end
+      end
+
+      def save!
+        resp = run("#{ redis_cli_cmd } SAVE")
+        unless resp =~ /OK$/
+          raise Error, <<-EOS
+            Failed to invoke the `SAVE` command
+            Response was: #{ resp }
+          EOS
+        end
+
+      rescue Error
+        if resp =~ /save already in progress/
+          unless (attempts ||= '0').next! == '5'
+            sleep 5
+            retry
+          end
+        end
+        raise
+      end
+
+      def copy!
+        unless File.exist?(rdb_path)
+          raise Error, <<-EOS
+            Redis database dump not found
+            `rdb_path` was '#{ rdb_path }'
+          EOS
+        end
+
+        dst_path = File.join(dump_path, dump_filename + '.rdb')
+        if model.compressor
+          model.compressor.compress_with do |command, ext|
+            run("#{ command } -c '#{ rdb_path }' > '#{ dst_path + ext }'")
+          end
+        else
+          FileUtils.cp(rdb_path, dst_path)
+        end
+      end
+
+      def redis_cli_cmd
+        "#{ utility('redis-cli') } #{ password_option } " +
+        "#{ connectivity_options } #{ user_options }"
+      end
+
+      def password_option
+        "-a '#{ password }'" if password
+      end
+
+      def connectivity_options
+        return "-s '#{ socket }'" if socket
+
+        opts = []
+        opts << "-h '#{ host }'" if host
+        opts << "-p '#{ port }'" if port
+        opts.join(' ')
+      end
+
+      def user_options
+        Array(additional_options).join(' ')
+      end
+
+    end
+  end
+end