backedup 5.0.0.beta.3

data/lib/backup/database/mysql.rb

@@ -0,0 +1,191 @@
+module Backup
+  module Database
+    class MySQL < Base
+      class Error < Backup::Error; end
+
+      ##
+      # Name of the database that needs to get dumped
+      # To dump all databases, set this to `:all` or leave blank.
+      attr_accessor :name
+
+      ##
+      # Credentials for the specified database
+      attr_accessor :username, :password
+
+      ##
+      # Connectivity options
+      attr_accessor :host, :port, :socket
+
+      ##
+      # Tables to skip while dumping the database
+      #
+      # If `name` is set to :all (or not specified), these must include
+      # a database name. e.g. 'name.table'.
+      # If `name` is given, these may simply be table names.
+      attr_accessor :skip_tables
+
+      ##
+      # Tables to dump. This in only valid if `name` is specified.
+      # If none are given, the entire database will be dumped.
+      attr_accessor :only_tables
+
+      ##
+      # Additional "mysqldump" or "innobackupex (backup creation)" options
+      attr_accessor :additional_options
+
+      ##
+      # Additional innobackupex log preparation phase ("apply-logs") options
+      attr_accessor :prepare_options
+
+      ##
+      # Default is :mysqldump (which is built in MySQL and generates
+      # a textual SQL file), but can be changed to :innobackupex, which
+      # has more feasible restore times for large databases.
+      # See: http://www.percona.com/doc/percona-xtrabackup/
+      attr_accessor :backup_engine
+
+      ##
+      # If true (which is the default behaviour), the backup will be prepared
+      # after it has been successfuly created. This option is only valid if
+      # :backup_engine is set to :innobackupex.
+      attr_accessor :prepare_backup
+
+      ##
+      # If set the backup engine command block is executed as the given user
+      attr_accessor :sudo_user
+
+      ##
+      # If set, do not suppress innobackupdb output (useful for debugging)
+      attr_accessor :verbose
+
+      def initialize(model, database_id = nil, &block)
+        super
+        instance_eval(&block) if block_given?
+
+        @name ||= :all
+        @backup_engine ||= :mysqldump
+        @prepare_backup = true if @prepare_backup.nil?
+      end
+
+      ##
+      # Performs the mysqldump or innobackupex command and outputs
+      # the dump file in the +dump_path+ using +dump_filename+.
+      #
+      #   <trigger>/databases/MySQL[-<database_id>].[sql|tar][.gz]
+      def perform!
+        super
+
+        pipeline = Pipeline.new
+        dump_ext = sql_backup? ? "sql" : "tar"
+
+        pipeline << sudo_option(sql_backup? ? mysqldump : innobackupex)
+
+        if model.compressor
+          model.compressor.compress_with do |command, ext|
+            pipeline << command
+            dump_ext << ext
+          end
+        end
+
+        pipeline << "#{utility(:cat)} > " \
+          "'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
+
+        pipeline.run
+        if pipeline.success?
+          log!(:finished)
+        else
+          raise Error, "Dump Failed!\n#{pipeline.error_messages}"
+        end
+      end
+
+      private
+
+      def mysqldump
+        "#{utility(:mysqldump)} #{user_options} #{credential_options} " \
+          "#{connectivity_options} #{name_option} " \
+          "#{tables_to_dump} #{tables_to_skip}"
+      end
+
+      def credential_options
+        opts = []
+        opts << "--user=#{Shellwords.escape(username)}" if username
+        opts << "--password=#{Shellwords.escape(password)}" if password
+        opts.join(" ")
+      end
+
+      def connectivity_options
+        return "--socket='#{socket}'" if socket
+
+        opts = []
+        opts << "--host='#{host}'" if host
+        opts << "--port='#{port}'" if port
+        opts.join(" ")
+      end
+
+      def user_options
+        Array(additional_options).join(" ")
+      end
+
+      def user_prepare_options
+        Array(prepare_options).join(" ")
+      end
+
+      def name_option
+        dump_all? ? "--all-databases" : name
+      end
+
+      def tables_to_dump
+        Array(only_tables).join(" ") unless dump_all?
+      end
+
+      def tables_to_skip
+        Array(skip_tables).map do |table|
+          table = dump_all? || table["."] ? table : "#{name}.#{table}"
+          "--ignore-table='#{table}'"
+        end.join(" ")
+      end
+
+      def dump_all?
+        name == :all
+      end
+
+      def sql_backup?
+        backup_engine.to_sym == :mysqldump
+      end
+
+      def innobackupex
+        # Creation phase
+        "#{utility(:innobackupex)} #{credential_options} " \
+          "#{connectivity_options} #{user_options} " \
+          "--no-timestamp #{temp_dir} #{quiet_option} && " +
+          innobackupex_prepare +
+          # Move files to tar-ed stream on stdout
+          "#{utility(:tar)} --remove-files -cf -  " \
+          "-C #{File.dirname(temp_dir)} #{File.basename(temp_dir)}"
+      end
+
+      def innobackupex_prepare
+        return "" unless @prepare_backup
+        # Log applying phase (prepare for restore)
+        "#{utility(:innobackupex)} --apply-log #{temp_dir} " \
+          "#{user_prepare_options}  #{quiet_option} && "
+      end
+
+      def sudo_option(command_block)
+        return command_block unless sudo_user
+
+        "sudo -s -u #{sudo_user} -- <<END_OF_SUDO\n" \
+          "#{command_block}\n" \
+          "END_OF_SUDO\n"
+      end
+
+      def quiet_option
+        verbose ? "" : " 2> /dev/null "
+      end
+
+      def temp_dir
+        File.join(dump_path, "#{dump_filename}.bkpdir")
+      end
+    end
+  end
+end

data/lib/backup/database/openldap.rb

@@ -0,0 +1,93 @@
+module Backup
+  module Database
+    class OpenLDAP < Base
+      class Error < Backup::Error; end
+
+      ##
+      # Name of the ldap backup
+      attr_accessor :name
+
+      ##
+      # run slapcat under sudo if needed
+      # make sure to set SUID on a file, to let you run the file with permissions of file owner
+      # eg. sudo chmod u+s /usr/sbin/slapcat
+      attr_accessor :use_sudo
+
+      ##
+      # Stores the location of the slapd.conf or slapcat confdir
+      attr_accessor :slapcat_conf
+
+      ##
+      # Additional slapcat options
+      attr_accessor :slapcat_args
+
+      ##
+      # Path to slapcat utility (optional)
+      attr_accessor :slapcat_utility
+
+      ##
+      # Takes the name of the archive and the configuration block
+      def initialize(model, database_id = nil, &block)
+        super
+        instance_eval(&block) if block_given?
+
+        @name             ||= "ldap_backup"
+        @use_sudo         ||= false
+        @slapcat_args     ||= []
+        @slapcat_utility  ||= utility(:slapcat)
+        @slapcat_conf     ||= "/etc/ldap/slapd.d"
+      end
+
+      ##
+      # Performs the slapcat command and outputs the
+      # data to the specified path based on the 'trigger'
+      def perform!
+        super
+
+        pipeline = Pipeline.new
+        dump_ext = "ldif"
+
+        pipeline << slapcat
+        if @model.compressor
+          @model.compressor.compress_with do |command, ext|
+            pipeline << command
+            dump_ext << ext
+          end
+        end
+
+        pipeline << "#{utility(:cat)} > " \
+            "'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
+
+        pipeline.run
+        if pipeline.success?
+          log!(:finished)
+        else
+          raise Error, "Dump Failed!\n" + pipeline.error_messages
+        end
+      end
+
+      private
+
+      ##
+      # Builds the full slapcat string based on all attributes
+      def slapcat
+        command = "#{slapcat_utility} #{slapcat_conf_option} #{slapcat_conf} #{user_options}"
+        command.prepend("sudo ") if use_sudo
+        command
+      end
+
+      ##
+      # Uses different slapcat switch depending on confdir or conffile set
+      def slapcat_conf_option
+        @slapcat_conf.include?(".d") ? "-F" : "-f"
+      end
+
+      ##
+      # Builds a compatible string for the additional options
+      # specified by the user
+      def user_options
+        slapcat_args.join(" ")
+      end
+    end
+  end
+end

data/lib/backup/database/postgresql.rb

@@ -0,0 +1,164 @@
+module Backup
+  module Database
+    class PostgreSQL < Base
+      class Error < Backup::Error; end
+
+      ##
+      # Name of the database that needs to get dumped.
+      # To dump all databases, set this to `:all` or leave blank.
+      # +username+ must be a PostgreSQL superuser to run `pg_dumpall`.
+      attr_accessor :name
+
+      ##
+      # Credentials for the specified database
+      attr_accessor :username, :password
+
+      ##
+      # If set the pg_dump(all) command is executed as the given user
+      attr_accessor :sudo_user
+
+      ##
+      # Connectivity options
+      attr_accessor :host, :port, :socket
+
+      ##
+      # URL includes credentials and connectivity options
+      attr_accessor :url
+
+      ##
+      # Tables to skip while dumping the database.
+      # If `name` is set to :all (or not specified), these are ignored.
+      attr_accessor :skip_tables
+
+      ##
+      # Tables to dump. This in only valid if `name` is specified.
+      # If none are given, the entire database will be dumped.
+      attr_accessor :only_tables
+
+      ##
+      # Additional "pg_dump" or "pg_dumpall" options
+      attr_accessor :additional_options
+
+      def check_options
+        return unless url
+        ignored_options = {
+          username: username,
+          password: password,
+          host: host,
+          port: port,
+          socket: socket,
+          name: name
+        }.reject { |_, value| value.nil? }
+        return if ignored_options.empty?
+
+        Logger.warn "PostgreSQL: the options " \
+          "#{ignored_options.keys.join(", ")} are set, but will be " \
+          "ignored because the `:url` option is set."
+      end
+
+      def initialize(model, database_id = nil, &block)
+        super
+        instance_eval(&block) if block_given?
+
+        @name ||= :all
+
+        # Warn user if url option is mixed with the others
+        check_options
+      end
+
+      ##
+      # Performs the pgdump command and outputs the dump file
+      # in the +dump_path+ using +dump_filename+.
+      #
+      #   <trigger>/databases/PostgreSQL[-<database_id>].sql[.gz]
+      def perform!
+        super
+
+        pipeline = Pipeline.new
+        dump_ext = "sql"
+
+        pipeline << (dump_all? ? pgdumpall : pgdump)
+
+        if model.compressor
+          model.compressor.compress_with do |command, ext|
+            pipeline << command
+            dump_ext << ext
+          end
+        end
+
+        pipeline << "#{utility(:cat)} > " \
+          "'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
+
+        pipeline.run
+        if pipeline.success?
+          log!(:finished)
+        else
+          raise Error, "Dump Failed!\n" + pipeline.error_messages
+        end
+      end
+
+      def pgdump
+        password_option.to_s +
+          sudo_option.to_s +
+          "#{utility(:pg_dump)} #{pgdump_arguments}"
+      end
+
+      def pgdump_arguments
+        if url.nil?
+          "#{username_option} #{connectivity_options} #{user_options} #{tables_to_dump} " \
+          "#{tables_to_skip} #{name}"
+        else
+          "#{user_options} #{tables_to_dump} #{tables_to_skip} #{url}"
+        end
+      end
+
+      def pgdumpall
+        password_option.to_s +
+          sudo_option.to_s +
+          "#{utility(:pg_dumpall)} #{username_option} " \
+          "#{connectivity_options} #{user_options}"
+      end
+
+      def password_option
+        "PGPASSWORD=#{Shellwords.escape(password)} " if password
+      end
+
+      def sudo_option
+        "#{utility(:sudo)} -n -H -u #{sudo_user} " if sudo_user
+      end
+
+      def username_option
+        "--username=#{Shellwords.escape(username)}" if username
+      end
+
+      def connectivity_options
+        return "--host='#{socket}'" if socket
+
+        opts = []
+        opts << "--host='#{host}'" if host
+        opts << "--port='#{port}'" if port
+        opts.join(" ")
+      end
+
+      def user_options
+        Array(additional_options).join(" ")
+      end
+
+      def tables_to_dump
+        Array(only_tables).map do |table|
+          "--table='#{table}'"
+        end.join(" ")
+      end
+
+      def tables_to_skip
+        Array(skip_tables).map do |table|
+          "--exclude-table='#{table}'"
+        end.join(" ")
+      end
+
+      def dump_all?
+        name == :all
+      end
+    end
+  end
+end

data/lib/backup/database/redis.rb

@@ -0,0 +1,176 @@
+module Backup
+  module Database
+    class Redis < Base
+      class Error < Backup::Error; end
+
+      MODES = [:copy, :sync]
+
+      ##
+      # Mode of operation.
+      #
+      # [:copy]
+      #   Copies the redis dump file specified by {#rdb_path}.
+      #   This data will be current as of the last RDB Snapshot
+      #   performed by the server (per your redis.conf settings).
+      #   You may set {#invoke_save} to +true+ to have Backup issue
+      #   a +SAVE+ command to update the dump file with the current
+      #   data before performing the copy.
+      #
+      # [:sync]
+      #   Performs a dump of your redis data using +redis-cli --rdb -+.
+      #   Redis implements this internally using a +SYNC+ command.
+      #   The operation is analogous to requesting a +BGSAVE+, then having the
+      #   dump returned. This mode is capable of dumping data from a local or
+      #   remote server. Requires Redis v2.6 or better.
+      #
+      # Defaults to +:copy+.
+      attr_accessor :mode
+
+      ##
+      # Full path to the redis dump file.
+      #
+      # Required when {#mode} is +:copy+.
+      attr_accessor :rdb_path
+
+      ##
+      # Perform a +SAVE+ command using the +redis-cli+ utility
+      # before copying the dump file specified by {#rdb_path}.
+      #
+      # Only valid when {#mode} is +:copy+.
+      attr_accessor :invoke_save
+
+      ##
+      # Connectivity options for the +redis-cli+ utility.
+      attr_accessor :host, :port, :socket
+
+      ##
+      # Password for the +redis-cli+ utility.
+      attr_accessor :password
+
+      ##
+      # Additional options for the +redis-cli+ utility.
+      attr_accessor :additional_options
+
+      def initialize(model, database_id = nil, &block)
+        super
+        instance_eval(&block) if block_given?
+
+        @mode ||= :copy
+
+        raise Error, "'#{mode}' is not a valid mode" unless MODES.include?(mode)
+
+        if mode == :copy && rdb_path.nil?
+          raise Error, "`rdb_path` must be set when `mode` is :copy"
+        end
+      end
+
+      ##
+      # Performs the dump based on {#mode} and stores the Redis dump file
+      # to the +dump_path+ using the +dump_filename+.
+      #
+      #   <trigger>/databases/Redis[-<database_id>].rdb[.gz]
+      def perform!
+        super
+
+        case mode
+        when :sync
+          # messages output by `redis-cli --rdb` on $stderr
+          Logger.configure do
+            ignore_warning(/Transfer finished with success/)
+            ignore_warning(/SYNC sent to master/)
+          end
+          sync!
+        when :copy
+          save! if invoke_save
+          copy!
+        end
+
+        log!(:finished)
+      end
+
+      private
+
+      def sync!
+        pipeline = Pipeline.new
+        dump_ext = "rdb"
+
+        pipeline << "#{redis_cli_cmd} --rdb -"
+
+        if model.compressor
+          model.compressor.compress_with do |command, ext|
+            pipeline << command
+            dump_ext << ext
+          end
+        end
+
+        pipeline << "#{utility(:cat)} > " \
+          "'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
+
+        pipeline.run
+
+        unless pipeline.success?
+          raise Error, "Dump Failed!\n" + pipeline.error_messages
+        end
+      end
+
+      def save!
+        resp = run("#{redis_cli_cmd} SAVE")
+        unless resp =~ /OK$/
+          raise Error, <<-EOS
+            Failed to invoke the `SAVE` command
+            Response was: #{resp}
+          EOS
+        end
+      rescue Error
+        if resp =~ /save already in progress/
+          unless (attempts ||= "0").next! == "5"
+            sleep 5
+            retry
+          end
+        end
+        raise
+      end
+
+      def copy!
+        unless File.exist?(rdb_path)
+          raise Error, <<-EOS
+            Redis database dump not found
+            `rdb_path` was '#{rdb_path}'
+          EOS
+        end
+
+        dst_path = File.join(dump_path, dump_filename + ".rdb")
+        if model.compressor
+          model.compressor.compress_with do |command, ext|
+            run("#{command} -c '#{rdb_path}' > '#{dst_path + ext}'")
+          end
+        else
+          FileUtils.cp(rdb_path, dst_path)
+        end
+      end
+
+      def redis_cli_cmd
+        "#{utility("redis-cli")} #{password_option} " \
+          "#{connectivity_options} #{user_options}"
+      end
+
+      def password_option
+        return unless password
+        "-a '#{password}'"
+      end
+
+      def connectivity_options
+        return "-s '#{socket}'" if socket
+
+        opts = []
+        opts << "-h '#{host}'" if host
+        opts << "-p '#{port}'" if port
+        opts.join(" ")
+      end
+
+      def user_options
+        Array(additional_options).join(" ")
+      end
+    end
+  end
+end

data/lib/backup/database/riak.rb

@@ -0,0 +1,79 @@
+module Backup
+  module Database
+    class Riak < Base
+      ##
+      # Node is the node from which to perform the backup.
+      # Default: riak@127.0.0.1
+      attr_accessor :node
+
+      ##
+      # Cookie is the Erlang cookie/shared secret used to connect to the node.
+      # Default: riak
+      attr_accessor :cookie
+
+      ##
+      # Username for the riak instance
+      # Default: riak
+      attr_accessor :user
+
+      def initialize(model, database_id = nil, &block)
+        super
+        instance_eval(&block) if block_given?
+
+        @node   ||= "riak@127.0.0.1"
+        @cookie ||= "riak"
+        @user   ||= "riak"
+      end
+
+      ##
+      # Performs the dump using `riak-admin backup`.
+      #
+      # This will be stored in the final backup package as
+      # <trigger>/databases/<dump_filename>-<node>[.gz]
+      def perform!
+        super
+
+        dump_file = File.join(dump_path, dump_filename)
+        with_riak_owned_dump_path do
+          run("#{riakadmin} backup #{node} #{cookie} '#{dump_file}' node")
+        end
+
+        if model.compressor
+          model.compressor.compress_with do |command, ext|
+            dump_file << "-#{node}" # `riak-admin` appends `node` to the filename.
+            run("#{command} -c '#{dump_file}' > '#{dump_file + ext}'")
+            FileUtils.rm_f(dump_file)
+          end
+        end
+
+        log!(:finished)
+      end
+
+      private
+
+      ##
+      # The `riak-admin backup` command is run as the riak +user+,
+      # so +user+ must have write priviledges to the +dump_path+.
+      #
+      # Note that the riak +user+ must also have access to +dump_path+.
+      # This means Backup's +tmp_path+ can not be under the home directory of
+      # the user running Backup, since the absence of the execute bit on their
+      # home directory would deny +user+ access.
+      def with_riak_owned_dump_path
+        run "#{utility(:sudo)} -n #{utility(:chown)} #{user} '#{dump_path}'"
+        yield
+      ensure
+        # reclaim ownership
+        run "#{utility(:sudo)} -n #{utility(:chown)} -R " \
+          "#{Config.user} '#{dump_path}'"
+      end
+
+      ##
+      # `riak-admin` must be run as the riak +user+.
+      # It will do this itself, but without `-n` and emits a message on STDERR.
+      def riakadmin
+        "#{utility(:sudo)} -n -u #{user} #{utility("riak-admin")}"
+      end
+    end
+  end
+end