backup-remote 0.0.2

data/lib/backup/database/openldap.rb

@@ -0,0 +1,95 @@
+# encoding: utf-8
+
+module Backup
+  module Database
+    class OpenLDAP < Base
+      class Error < Backup::Error; end
+
+      ##
+      # Name of the ldap backup
+      attr_accessor :name
+
+      ##
+      # run slapcat under sudo if needed
+      # make sure to set SUID on a file, to let you run the file with permissions of file owner
+      # eg. sudo chmod u+s /usr/sbin/slapcat
+      attr_accessor :use_sudo
+
+      ##
+      # Stores the location of the slapd.conf or slapcat confdir
+      attr_accessor :slapcat_conf
+
+      ##
+      # Additional slapcat options
+      attr_accessor :slapcat_args
+
+      ##
+      # Path to slapcat utility (optional)
+      attr_accessor :slapcat_utility
+
+      ##
+      # Takes the name of the archive and the configuration block
+      def initialize(model, database_id = nil, &block)
+        super
+        instance_eval(&block) if block_given?
+
+        @name             ||= 'ldap_backup'
+        @use_sudo         ||= false
+        @slapcat_args     ||= Array.new
+        @slapcat_utility  ||= utility(:slapcat)
+        @slapcat_conf     ||= '/etc/ldap/slapd.d'
+      end
+
+      ##
+      # Performs the slapcat command and outputs the
+      # data to the specified path based on the 'trigger'
+      def perform!
+        super
+
+        pipeline = Pipeline.new
+        dump_ext = 'ldif'
+
+        pipeline << slapcat
+        if @model.compressor
+          @model.compressor.compress_with do |command, ext|
+            pipeline << command
+            dump_ext << ext
+          end
+        end
+
+        pipeline << "#{ utility(:cat) } > " +
+            "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
+
+        pipeline.run
+        if pipeline.success?
+          log!(:finished)
+        else
+          raise Error, "Dump Failed!\n" + pipeline.error_messages
+        end
+      end
+
+      private
+
+      ##
+      # Builds the full slapcat string based on all attributes
+      def slapcat
+        command = "#{ slapcat_utility } #{ slapcat_conf_option } #{ slapcat_conf } #{ user_options }"
+        command.prepend("sudo ") if use_sudo
+        command
+      end
+
+      ##
+      # Uses different slapcat switch depending on confdir or conffile set
+      def slapcat_conf_option
+        @slapcat_conf.include?(".d") ? "-F" : "-f"
+      end
+
+      ##
+      # Builds a compatible string for the additional options
+      # specified by the user
+      def user_options
+        slapcat_args.join(' ')
+      end
+    end
+  end
+end

data/lib/backup/database/postgresql.rb

@@ -0,0 +1,133 @@
+# encoding: utf-8
+
+module Backup
+  module Database
+    class PostgreSQL < Base
+      class Error < Backup::Error; end
+
+      ##
+      # Name of the database that needs to get dumped.
+      # To dump all databases, set this to `:all` or leave blank.
+      # +username+ must be a PostgreSQL superuser to run `pg_dumpall`.
+      attr_accessor :name
+
+      ##
+      # Credentials for the specified database
+      attr_accessor :username, :password
+
+      ##
+      # If set the pg_dump(all) command is executed as the given user
+      attr_accessor :sudo_user
+
+      ##
+      # Connectivity options
+      attr_accessor :host, :port, :socket
+
+      ##
+      # Tables to skip while dumping the database.
+      # If `name` is set to :all (or not specified), these are ignored.
+      attr_accessor :skip_tables
+
+      ##
+      # Tables to dump. This in only valid if `name` is specified.
+      # If none are given, the entire database will be dumped.
+      attr_accessor :only_tables
+
+      ##
+      # Additional "pg_dump" or "pg_dumpall" options
+      attr_accessor :additional_options
+
+      def initialize(model, database_id = nil, &block)
+        super
+        instance_eval(&block) if block_given?
+
+        @name ||= :all
+      end
+
+      ##
+      # Performs the pgdump command and outputs the dump file
+      # in the +dump_path+ using +dump_filename+.
+      #
+      #   <trigger>/databases/PostgreSQL[-<database_id>].sql[.gz]
+      def perform!
+        super
+
+        pipeline = Pipeline.new
+        dump_ext = 'sql'
+
+        pipeline << (dump_all? ? pgdumpall : pgdump)
+
+        model.compressor.compress_with do |command, ext|
+          pipeline << command
+          dump_ext << ext
+        end if model.compressor
+
+        pipeline << "#{ utility(:cat) } > " +
+            "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
+
+        pipeline.run
+        if pipeline.success?
+          log!(:finished)
+        else
+          raise Error, "Dump Failed!\n" + pipeline.error_messages
+        end
+      end
+
+      def pgdump
+        "#{ password_option }" +
+        "#{ sudo_option }" +
+        "#{ utility(:pg_dump) } #{ username_option } #{ connectivity_options } " +
+        "#{ user_options } #{ tables_to_dump } #{ tables_to_skip } #{ name }"
+      end
+
+      def pgdumpall
+        "#{ password_option }" +
+        "#{ sudo_option }" +
+        "#{ utility(:pg_dumpall) } #{ username_option } " +
+        "#{ connectivity_options } #{ user_options }"
+      end
+
+      def password_option
+        "PGPASSWORD=#{ Shellwords.escape(password) } " if password
+      end
+
+      def sudo_option
+        "#{ utility(:sudo) } -n -H -u #{ sudo_user } " if sudo_user
+      end
+
+      def username_option
+        "--username=#{ Shellwords.escape(username) }" if username
+      end
+
+      def connectivity_options
+        return "--host='#{ socket }'" if socket
+
+        opts = []
+        opts << "--host='#{ host }'" if host
+        opts << "--port='#{ port }'" if port
+        opts.join(' ')
+      end
+
+      def user_options
+        Array(additional_options).join(' ')
+      end
+
+      def tables_to_dump
+        Array(only_tables).map do |table|
+          "--table='#{ table }'"
+        end.join(' ')
+      end
+
+      def tables_to_skip
+        Array(skip_tables).map do |table|
+          "--exclude-table='#{ table }'"
+        end.join(' ')
+      end
+
+      def dump_all?
+        name == :all
+      end
+
+    end
+  end
+end

data/lib/backup/database/redis.rb

@@ -0,0 +1,179 @@
+# encoding: utf-8
+
+module Backup
+  module Database
+    class Redis < Base
+      class Error < Backup::Error; end
+
+      MODES = [:copy, :sync]
+
+      ##
+      # Mode of operation.
+      #
+      # [:copy]
+      #   Copies the redis dump file specified by {#rdb_path}.
+      #   This data will be current as of the last RDB Snapshot
+      #   performed by the server (per your redis.conf settings).
+      #   You may set {#invoke_save} to +true+ to have Backup issue
+      #   a +SAVE+ command to update the dump file with the current
+      #   data before performing the copy.
+      #
+      # [:sync]
+      #   Performs a dump of your redis data using +redis-cli --rdb -+.
+      #   Redis implements this internally using a +SYNC+ command.
+      #   The operation is analogous to requesting a +BGSAVE+, then having the
+      #   dump returned. This mode is capable of dumping data from a local or
+      #   remote server. Requires Redis v2.6 or better.
+      #
+      # Defaults to +:copy+.
+      attr_accessor :mode
+
+      ##
+      # Full path to the redis dump file.
+      #
+      # Required when {#mode} is +:copy+.
+      attr_accessor :rdb_path
+
+      ##
+      # Perform a +SAVE+ command using the +redis-cli+ utility
+      # before copying the dump file specified by {#rdb_path}.
+      #
+      # Only valid when {#mode} is +:copy+.
+      attr_accessor :invoke_save
+
+      ##
+      # Connectivity options for the +redis-cli+ utility.
+      attr_accessor :host, :port, :socket
+
+      ##
+      # Password for the +redis-cli+ utility.
+      attr_accessor :password
+
+      ##
+      # Additional options for the +redis-cli+ utility.
+      attr_accessor :additional_options
+
+      def initialize(model, database_id = nil, &block)
+        super
+        instance_eval(&block) if block_given?
+
+        @mode ||= :copy
+
+        unless MODES.include?(mode)
+          raise Error, "'#{ mode }' is not a valid mode"
+        end
+
+        if mode == :copy && rdb_path.nil?
+          raise Error, '`rdb_path` must be set when `mode` is :copy'
+        end
+      end
+
+      ##
+      # Performs the dump based on {#mode} and stores the Redis dump file
+      # to the +dump_path+ using the +dump_filename+.
+      #
+      #   <trigger>/databases/Redis[-<database_id>].rdb[.gz]
+      def perform!
+        super
+
+        case mode
+        when :sync
+          # messages output by `redis-cli --rdb` on $stderr
+          Logger.configure do
+            ignore_warning(/Transfer finished with success/)
+            ignore_warning(/SYNC sent to master/)
+          end
+          sync!
+        when :copy
+          save! if invoke_save
+          copy!
+        end
+
+        log!(:finished)
+      end
+
+      private
+
+      def sync!
+        pipeline = Pipeline.new
+        dump_ext = 'rdb'
+
+        pipeline << "#{ redis_cli_cmd } --rdb -"
+
+        model.compressor.compress_with do |command, ext|
+          pipeline << command
+          dump_ext << ext
+        end if model.compressor
+
+        pipeline << "#{ utility(:cat) } > " +
+            "'#{ File.join(dump_path, dump_filename) }.#{ dump_ext }'"
+
+        pipeline.run
+
+        unless pipeline.success?
+          raise Error, "Dump Failed!\n" + pipeline.error_messages
+        end
+      end
+
+      def save!
+        resp = run("#{ redis_cli_cmd } SAVE")
+        unless resp =~ /OK$/
+          raise Error, <<-EOS
+            Failed to invoke the `SAVE` command
+            Response was: #{ resp }
+          EOS
+        end
+
+      rescue Error
+        if resp =~ /save already in progress/
+          unless (attempts ||= '0').next! == '5'
+            sleep 5
+            retry
+          end
+        end
+        raise
+      end
+
+      def copy!
+        unless File.exist?(rdb_path)
+          raise Error, <<-EOS
+            Redis database dump not found
+            `rdb_path` was '#{ rdb_path }'
+          EOS
+        end
+
+        dst_path = File.join(dump_path, dump_filename + '.rdb')
+        if model.compressor
+          model.compressor.compress_with do |command, ext|
+            run("#{ command } -c '#{ rdb_path }' > '#{ dst_path + ext }'")
+          end
+        else
+          FileUtils.cp(rdb_path, dst_path)
+        end
+      end
+
+      def redis_cli_cmd
+        "#{ utility('redis-cli') } #{ password_option } " +
+        "#{ connectivity_options } #{ user_options }"
+      end
+
+      def password_option
+        "-a '#{ password }'" if password
+      end
+
+      def connectivity_options
+        return "-s '#{ socket }'" if socket
+
+        opts = []
+        opts << "-h '#{ host }'" if host
+        opts << "-p '#{ port }'" if port
+        opts.join(' ')
+      end
+
+      def user_options
+        Array(additional_options).join(' ')
+      end
+
+    end
+  end
+end

data/lib/backup/database/remote_mysql.rb

@@ -0,0 +1,248 @@
+# encoding: utf-8
+require 'net/ssh'
+
+require 'sshkit'
+require 'sshkit/dsl'
+require 'sshkit/sudo'
+
+
+module Backup
+  module Database
+    class RemoteMySQL < Base
+      class Error < Backup::Error; end
+
+      include SSHKit::DSL
+
+      # server options
+      attr_accessor :server_host
+      attr_accessor :server_ssh_user
+      attr_accessor :server_ssh_password
+      attr_accessor :server_ssh_key
+      attr_accessor :server_backup_path
+
+
+      ##
+      # Name of the database that needs to get dumped
+      # To dump all databases, set this to `:all` or leave blank.
+      attr_accessor :name
+
+      ##
+      # Credentials for the specified database
+      attr_accessor :username, :password
+
+      ##
+      # Connectivity options
+      attr_accessor :host, :port, :socket
+
+      ##
+      # Tables to skip while dumping the database
+      #
+      # If `name` is set to :all (or not specified), these must include
+      # a database name. e.g. 'name.table'.
+      # If `name` is given, these may simply be table names.
+      attr_accessor :skip_tables
+
+      ##
+      # Tables to dump. This in only valid if `name` is specified.
+      # If none are given, the entire database will be dumped.
+      attr_accessor :only_tables
+
+      ##
+      # Additional "mysqldump" or "innobackupex (backup creation)" options
+      attr_accessor :additional_options
+
+      ##
+      # Additional innobackupex log preparation phase ("apply-logs") options
+      attr_accessor :prepare_options
+
+      ##
+      # Default is :mysqldump (which is built in MySQL and generates
+      # a textual SQL file), but can be changed to :innobackupex, which
+      # has more feasible restore times for large databases.
+      # See: http://www.percona.com/doc/percona-xtrabackup/
+      attr_accessor :backup_engine
+
+      ##
+      # If true (which is the default behaviour), the backup will be prepared
+      # after it has been successfuly created. This option is only valid if
+      # :backup_engine is set to :innobackupex.
+      attr_accessor :prepare_backup
+
+      ##
+      # If set the backup engine command block is executed as the given user
+      attr_accessor :sudo_user
+
+      ##
+      # If set, do not suppress innobackupdb output (useful for debugging)
+      attr_accessor :verbose
+
+      def initialize(model, database_id = nil, &block)
+        super
+        instance_eval(&block) if block_given?
+
+        @name ||= :all
+        @backup_engine ||= :mysqldump
+        @prepare_backup = true if @prepare_backup.nil?
+      end
+
+      ##
+      # Performs the mysqldump or innobackupex command and outputs
+      # the dump file in the +dump_path+ using +dump_filename+.
+      #
+      #   <trigger>/databases/MySQL[-<database_id>].[sql|tar][.gz]
+      def perform!
+        super
+
+
+        #
+        pipeline = Pipeline.new
+        dump_ext = sql_backup? ? 'sql' : 'tar'
+
+        pipeline << sudo_option(sql_backup? ? mysqldump : innobackupex)
+
+        model.compressor.compress_with do |command, ext|
+          pipeline << command
+          dump_ext << ext
+        end if model.compressor
+
+        dump_remote_file = File.join('/tmp', dump_filename+"."+dump_ext)
+        pipeline << "#{ utility(:cat) } > '#{ dump_remote_file }'"
+
+
+
+
+        # generate backup on remote server
+        cmd_remote = pipeline.commands.join(" | ")
+
+        remote = Backup::Remote::Command.new
+        res_generate = remote.run_ssh_cmd(server_host, server_ssh_user, server_ssh_password, cmd_remote)
+
+        if res_generate[:res]==0
+          raise 'Cannot create backup on server'
+        end
+
+        # download backup
+        dump_file = File.join(dump_path, dump_filename+"."+dump_ext)
+
+        #puts "dump local: #{dump_file}"
+
+        res_download = remote.ssh_download_file(server_host, server_ssh_user, server_ssh_password, dump_remote_file, dump_file)
+
+        #puts "res: #{res_download}"
+
+        if res_download[:res]==0
+          raise 'Cannot download file from server'
+        end
+
+        #puts "pipe: #{pipeline.commands.inspect}"
+        #puts "cmd: #{cmd_remote}"
+
+        #exit
+
+        # download dump from server
+
+        #puts "file: #{dump_filename}"
+        #puts "path: #{dump_path}"
+
+        #exit
+
+        #pipeline.run
+        #if pipeline.success?
+        #  log!(:finished)
+        #else
+        #  raise Error, "Dump Failed!\n" + pipeline.error_messages
+        #end
+      end
+
+      private
+
+      def mysqldump
+        "#{ utility(:mysqldump) } #{ user_options } #{ credential_options } " +
+        "#{ connectivity_options } #{ name_option } " +
+        "#{ tables_to_dump } #{ tables_to_skip }"
+      end
+
+      def credential_options
+        opts = []
+        opts << "--user=#{ Shellwords.escape(username) }" if username
+        opts << "--password=#{ Shellwords.escape(password) }" if password
+        opts.join(' ')
+      end
+
+      def connectivity_options
+        return "--socket='#{ socket }'" if socket
+
+        opts = []
+        opts << "--host='#{ host }'" if host
+        opts << "--port='#{ port }'" if port
+        opts.join(' ')
+      end
+
+      def user_options
+        Array(additional_options).join(' ')
+      end
+
+      def user_prepare_options
+        Array(prepare_options).join(' ')
+      end
+
+      def name_option
+        dump_all? ? '--all-databases' : name
+      end
+
+      def tables_to_dump
+        Array(only_tables).join(' ') unless dump_all?
+      end
+
+      def tables_to_skip
+        Array(skip_tables).map do |table|
+          table = (dump_all? || table['.']) ? table : "#{ name }.#{ table }"
+          "--ignore-table='#{ table }'"
+        end.join(' ')
+      end
+
+      def dump_all?
+        name == :all
+      end
+
+      def sql_backup?
+        backup_engine.to_sym == :mysqldump
+      end
+
+      def innobackupex
+        # Creation phase
+        "#{ utility(:innobackupex) } #{ credential_options } " +
+        "#{ connectivity_options } #{ user_options } " +
+        "--no-timestamp #{ temp_dir } #{ quiet_option } && " +
+        innobackupex_prepare +
+        # Move files to tar-ed stream on stdout
+        "#{ utility(:tar) } --remove-files -cf -  " +
+        "-C #{ File.dirname(temp_dir) } #{ File.basename(temp_dir) }"
+      end
+
+      def innobackupex_prepare
+        return "" unless @prepare_backup
+        # Log applying phase (prepare for restore)
+        "#{ utility(:innobackupex) } --apply-log #{ temp_dir } " +
+        "#{ user_prepare_options }  #{ quiet_option } && "
+      end
+
+      def sudo_option(command_block)
+        return command_block unless sudo_user
+
+        "sudo -s -u #{ sudo_user } -- <<END_OF_SUDO\n" +
+        "#{command_block}\n" +
+        "END_OF_SUDO\n"
+      end
+
+      def quiet_option
+        verbose ? "" : " 2> /dev/null "
+      end
+
+      def temp_dir
+        File.join(dump_path, dump_filename + ".bkpdir")
+      end
+
+    end
+  end
+end