sliday_backup 0.1

data/lib/sliday_backup/syncer/rsync/base.rb

@@ -0,0 +1,54 @@
+# encoding: utf-8
+
+module SlidayBackup
+  module Syncer
+    module RSync
+      class Base < Syncer::Base
+
+        ##
+        # Additional String or Array of options for the rsync cli
+        attr_accessor :additional_rsync_options
+        attr_accessor :archive
+
+        def initialize(syncer_id = nil, &block)
+          super
+          instance_eval(&block) if block_given?
+
+          @path ||= '~/backups'
+          @archive = @archive.nil? ? true : @archive
+        end
+
+        private
+
+        ##
+        # Common base command for Local/Push/Pull
+        def rsync_command
+          utility(:rsync) << archive_option << mirror_option << exclude_option <<
+              " #{ Array(additional_rsync_options).join(' ') }".rstrip
+        end
+
+        def mirror_option
+          mirror ? ' --delete' : ''
+        end
+
+        def archive_option
+          archive ? ' --archive' : ''
+        end
+
+        def exclude_option
+          excludes.map {|pattern| " --exclude='#{ pattern }'" }.join
+        end
+
+        ##
+        # Each path is expanded, since these refer to local paths and are
+        # being shell-quoted. This will also remove any trailing `/` from
+        # each path, as we don't want rsync's "trailing / on source directories"
+        # behavior. This method is used by RSync::Local and RSync::Push.
+        def paths_to_push
+          directories.map {|dir| "'#{ File.expand_path(dir) }'" }.join(' ')
+        end
+
+      end
+    end
+  end
+end

data/lib/sliday_backup/syncer/rsync/local.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+
+module SlidayBackup
+  module Syncer
+    module RSync
+      class Local < Base
+
+        def perform!
+          log!(:started)
+
+          create_dest_path!
+          run("#{ rsync_command } #{ paths_to_push } '#{ dest_path }'")
+
+          log!(:finished)
+        end
+
+        private
+
+        # Expand path, since this is local and shell-quoted.
+        def dest_path
+          @dest_path ||= File.expand_path(path)
+        end
+
+        def create_dest_path!
+          FileUtils.mkdir_p dest_path
+        end
+
+      end
+    end
+  end
+end

data/lib/sliday_backup/syncer/rsync/pull.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+
+module SlidayBackup
+  module Syncer
+    module RSync
+      class Pull < Push
+
+        def perform!
+          log!(:started)
+          write_password_file!
+
+          create_dest_path!
+          run("#{ rsync_command } #{ host_options }#{ paths_to_pull } " +
+              "'#{ dest_path }'")
+
+          log!(:finished)
+        ensure
+          remove_password_file!
+        end
+
+        private
+
+        ##
+        # Returns the syntax for pulling multiple paths from the remote host.
+        # e.g.
+        #   rsync -a -e "ssh -p 22" host:'path1' :'path2' '/dest'
+        #   rsync -a rsync_user@host::'modname/path1' ::'modname/path2' '/dest'
+        #
+        # Remove any preceeding '~/', since these paths are on the remote.
+        # Also remove any trailing `/`, since we don't want rsync's
+        # "trailing / on source directories" behavior.
+        def paths_to_pull
+          sep = mode == :ssh ? ':' : '::'
+          directories.map {|dir|
+            "#{ sep }'#{ dir.sub(/^~\//, '').sub(/\/$/, '') }'"
+          }.join(' ').sub(/^#{ sep }/, '')
+        end
+
+        # Expand path, since this is local and shell-quoted.
+        def dest_path
+          @dest_path ||= File.expand_path(path)
+        end
+
+        def create_dest_path!
+          FileUtils.mkdir_p dest_path
+        end
+
+      end
+    end
+  end
+end

data/lib/sliday_backup/syncer/rsync/push.rb

@@ -0,0 +1,205 @@
+# encoding: utf-8
+
+module SlidayBackup
+  module Syncer
+    module RSync
+      class Push < Base
+
+        ##
+        # Mode of operation
+        #
+        # [:ssh (default)]
+        #   Connects to the remote via SSH.
+        #   Does not use an rsync daemon on the remote.
+        #
+        # [:ssh_daemon]
+        #   Connects to the remote via SSH.
+        #   Spawns a single-use daemon on the remote, which allows certain
+        #   daemon features (like modules) to be used.
+        #
+        # [:rsync_daemon]
+        #   Connects directly to an rsync daemon via TCP.
+        #   Data transferred is not encrypted.
+        #
+        attr_accessor :mode
+
+        ##
+        # Server Address
+        attr_accessor :host
+
+        ##
+        # SSH or RSync port
+        #
+        # For `:ssh` or `:ssh_daemon` mode, this specifies the SSH port to use
+        # and defaults to 22.
+        #
+        # For `:rsync_daemon` mode, this specifies the TCP port to use
+        # and defaults to 873.
+        attr_accessor :port
+
+        ##
+        # SSH User
+        #
+        # If the user running the backup is not the same user that needs to
+        # authenticate with the remote server, specify the user here.
+        #
+        # The user must have SSH keys setup for passphrase-less access to the
+        # remote. If the SSH User does not have passphrase-less keys, or no
+        # default keys in their `~/.ssh` directory, you will need to use the
+        # `-i` option in `:additional_ssh_options` to specify the
+        # passphrase-less key to use.
+        #
+        # Used only for `:ssh` and `:ssh_daemon` modes.
+        attr_accessor :ssh_user
+
+        ##
+        # Additional SSH Options
+        #
+        # Used to supply a String or Array of options to be passed to the SSH
+        # command in `:ssh` and `:ssh_daemon` modes.
+        #
+        # For example, if you need to supply a specific SSH key for the `ssh_user`,
+        # you would set this to: "-i '/path/to/id_rsa'". Which would produce:
+        #
+        #   rsync -e "ssh -p 22 -i '/path/to/id_rsa'"
+        #
+        # Arguments may be single-quoted, but should not contain any double-quotes.
+        #
+        # Used only for `:ssh` and `:ssh_daemon` modes.
+        attr_accessor :additional_ssh_options
+
+        ##
+        # RSync User
+        #
+        # If the user running the backup is not the same user that needs to
+        # authenticate with the rsync daemon, specify the user here.
+        #
+        # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+        attr_accessor :rsync_user
+
+        ##
+        # RSync Password
+        #
+        # If specified, SlidayBackup will write the password to a temporary file and
+        # use it with rsync's `--password-file` option for daemon authentication.
+        #
+        # Note that setting this will override `rsync_password_file`.
+        #
+        # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+        attr_accessor :rsync_password
+
+        ##
+        # RSync Password File
+        #
+        # If specified, this path will be passed to rsync's `--password-file`
+        # option for daemon authentication.
+        #
+        # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+        attr_accessor :rsync_password_file
+
+        ##
+        # Flag for compressing (only compresses for the transfer)
+        attr_accessor :compress
+
+        def initialize(syncer_id = nil)
+          super
+
+          @mode ||= :ssh
+          @port ||= mode == :rsync_daemon ? 873 : 22
+          @compress ||= false
+        end
+
+        def perform!
+          log!(:started)
+          write_password_file!
+
+          create_dest_path!
+          run("#{ rsync_command } #{ paths_to_push } " +
+              "#{ host_options }'#{ dest_path }'")
+
+          log!(:finished)
+        ensure
+          remove_password_file!
+        end
+
+        private
+
+        ##
+        # Remove any preceeding '~/', since this is on the remote,
+        # and remove any trailing `/`.
+        def dest_path
+          @dest_path ||= path.sub(/^~\//, '').sub(/\/$/, '')
+        end
+
+        ##
+        # Runs a 'mkdir -p' command on the remote to ensure the dest_path exists.
+        # This used because rsync will attempt to create the path, but will only
+        # call 'mkdir' without the '-p' option. This is only applicable in :ssh
+        # mode, and only used if the path would require this.
+        def create_dest_path!
+          return unless mode == :ssh && dest_path.index('/').to_i > 0
+
+          run "#{ utility(:ssh) } #{ ssh_transport_args } #{ host } " +
+                 %Q["mkdir -p '#{ dest_path }'"]
+        end
+
+        ##
+        # For Push, this will prepend the #dest_path.
+        # For Pull, this will prepend the first path in #paths_to_pull.
+        def host_options
+          if mode == :ssh
+            "#{ host }:"
+          else
+            user = "#{ rsync_user }@" if rsync_user
+            "#{ user }#{ host }::"
+          end
+        end
+
+        ##
+        # Common base command, plus options for Push/Pull
+        def rsync_command
+          super << compress_option << password_option << transport_options
+        end
+
+        def compress_option
+          compress ? ' --compress' : ''
+        end
+
+        def password_option
+          return '' if mode == :ssh
+
+          path = @password_file ? @password_file.path : rsync_password_file
+          path ? " --password-file='#{ File.expand_path(path) }'" : ''
+        end
+
+        def transport_options
+          if mode == :rsync_daemon
+            " --port #{ port }"
+          else
+            %Q[ -e "#{ utility(:ssh) } #{ ssh_transport_args }"]
+          end
+        end
+
+        def ssh_transport_args
+          args = "-p #{ port } "
+          args << "-l #{ ssh_user } " if ssh_user
+          args << Array(additional_ssh_options).join(' ')
+          args.rstrip
+        end
+
+        def write_password_file!
+          return unless rsync_password && mode != :ssh
+
+          @password_file = Tempfile.new('backup-rsync-password')
+          @password_file.write(rsync_password)
+          @password_file.close
+        end
+
+        def remove_password_file!
+          @password_file.delete if @password_file
+        end
+
+      end
+    end
+  end
+end

data/lib/sliday_backup/template.rb

@@ -0,0 +1,46 @@
+# encoding: utf-8
+
+require 'erb'
+
+module SlidayBackup
+  class Template
+
+    # Holds a binding object. Nil if not provided.
+    attr_accessor :binding
+
+    ##
+    # Creates a new instance of the SlidayBackup::Template class
+    # and optionally takes an argument that can be either a binding object, a Hash or nil
+    def initialize(object = nil)
+      if object.is_a?(Binding)
+        @binding = object
+      elsif object.is_a?(Hash)
+        @binding = SlidayBackup::Binder.new(object).get_binding
+      else
+        @binding = nil
+      end
+    end
+
+    ##
+    # Renders the provided file (in the context of the binding if any) to the console
+    def render(file)
+      puts result(file)
+    end
+
+    ##
+    # Returns a String object containing the contents of the file (in the context of the binding if any)
+    def result(file)
+      ERB.new(file_contents(file), nil, '<>').result(binding)
+    end
+
+  private
+
+    ##
+    # Reads and returns the contents of the provided file path,
+    # relative from the SlidayBackup::TEMPLATE_PATH
+    def file_contents(file)
+      File.read(File.join(SlidayBackup::TEMPLATE_PATH, file))
+    end
+
+  end
+end

data/lib/sliday_backup/utilities.rb

@@ -0,0 +1,224 @@
+# encoding: utf-8
+
+module SlidayBackup
+  module Utilities
+    class Error < SlidayBackup::Error; end
+
+    UTILITY = {}
+    NAMES = %w{
+      tar cat split sudo chown hostname
+      gzip bzip2
+      mongo mongodump mysqldump innobackupex
+      pg_dump pg_dumpall redis-cli riak-admin
+      gpg openssl
+      rsync ssh
+      sendmail exim
+      send_nsca
+      zabbix_sender
+    }
+
+    module DSL
+      class << self
+        ##
+        # Allow users to set the path for all utilities in the .configure block.
+        #
+        # Utility names with dashes ('redis-cli') will be set using method calls
+        # with an underscore ('redis_cli').
+        NAMES.each do |name|
+          define_method name.gsub('-', '_'), lambda {|val|
+            path = File.expand_path(val)
+            unless File.executable?(path)
+              raise Utilities::Error, <<-EOS
+                The path given for '#{ name }' was not found or not executable.
+                Path was: #{ path }
+              EOS
+            end
+            UTILITY[name] = path
+          }
+        end
+
+        ##
+        # Allow users to set the +tar+ distribution if needed. (:gnu or :bsd)
+        def tar_dist(val)
+          Utilities.tar_dist(val)
+        end
+      end
+    end
+
+    class << self
+      ##
+      # Configure the path to system utilities used by SlidayBackup.
+      #
+      # SlidayBackup will attempt to locate any required system utilities using a
+      # +which+ command call. If a utility can not be found, or you need to
+      # specify an alternate path for a utility, you may do so in your
+      # +config.rb+ file using this method.
+      #
+      # SlidayBackup supports both GNU and BSD utilities.
+      # While SlidayBackup uses these utilities in a manner compatible with either
+      # version, the +tar+ utility requires some special handling with respect
+      # to +Archive+s. SlidayBackup will attempt to detect if the +tar+ command
+      # found (or set here) is GNU or BSD. If for some reason this fails,
+      # this may be set using the +tar_dist+ command shown below.
+      #
+      #   SlidayBackup::Utilities.configure do
+      #     # General Utilites
+      #     tar      '/path/to/tar'
+      #     tar_dist :gnu   # or :bsd
+      #     cat      '/path/to/cat'
+      #     split    '/path/to/split'
+      #     sudo     '/path/to/sudo'
+      #     chown    '/path/to/chown'
+      #     hostname '/path/to/hostname'
+      #
+      #     # Compressors
+      #     gzip    '/path/to/gzip'
+      #     bzip2   '/path/to/bzip2'
+      #
+      #     # Database Utilities
+      #     mongo       '/path/to/mongo'
+      #     mongodump   '/path/to/mongodump'
+      #     mysqldump   '/path/to/mysqldump'
+      #     pg_dump     '/path/to/pg_dump'
+      #     pg_dumpall  '/path/to/pg_dumpall'
+      #     redis_cli   '/path/to/redis-cli'
+      #     riak_admin  '/path/to/riak-admin'
+      #
+      #     # Encryptors
+      #     gpg     '/path/to/gpg'
+      #     openssl '/path/to/openssl'
+      #
+      #     # Syncer and Storage
+      #     rsync   '/path/to/rsync'
+      #     ssh     '/path/to/ssh'
+      #
+      #     # Notifiers
+      #     sendmail  '/path/to/sendmail'
+      #     exim      '/path/to/exim'
+      #     send_nsca '/path/to/send_nsca'
+      #     zabbix_sender '/path/to/zabbix_sender'
+      #   end
+      #
+      # These paths may be set using absolute paths, or relative to the
+      # working directory when SlidayBackup is run.
+      def configure(&block)
+        DSL.instance_eval(&block)
+      end
+
+      def tar_dist(val)
+        # the acceptance tests need to be able to reset this to nil
+        @gnu_tar = val.nil? ? nil : val == :gnu
+      end
+
+      def gnu_tar?
+        return @gnu_tar unless @gnu_tar.nil?
+        @gnu_tar = !!run("#{ utility(:tar) } --version").match(/GNU/)
+      end
+
+      private
+
+      ##
+      # Returns the full path to the specified utility.
+      # Raises an error if utility can not be found in the system's $PATH
+      def utility(name)
+        name = name.to_s.strip
+        raise Error, 'Utility Name Empty' if name.empty?
+
+        UTILITY[name] ||= %x[which '#{ name }' 2>/dev/null].chomp
+        raise Error, <<-EOS if UTILITY[name].empty?
+          Could not locate '#{ name }'.
+          Make sure the specified utility is installed
+          and available in your system's $PATH, or specify it's location
+          in your 'config.rb' file using SlidayBackup::Utilities.configure
+        EOS
+
+        UTILITY[name].dup
+      end
+
+      ##
+      # Returns the name of the command name from the given command line.
+      # This is only used to simplify log messages.
+      def command_name(command)
+        parts = []
+        command = command.split(' ')
+        command.shift while command[0].to_s.include?('=')
+        parts << command.shift.split('/')[-1]
+        if parts[0] == 'sudo'
+          until command.empty?
+            part = command.shift
+            if part.include?('/')
+              parts << part.split('/')[-1]
+              break
+            else
+              parts << part
+            end
+          end
+        end
+        parts.join(' ')
+      end
+
+      ##
+      # Runs a system command
+      #
+      # All messages generated by the command will be logged.
+      # Messages on STDERR will be logged as warnings.
+      #
+      # If the command fails to execute, or returns a non-zero exit status
+      # an Error will be raised.
+      #
+      # Returns STDOUT
+      def run(command)
+        name = command_name(command)
+        Logger.info "Running system utility '#{ name }'..."
+
+        begin
+          out, err = '', ''
+          ps = Open4.popen4(command) do |pid, stdin, stdout, stderr|
+            stdin.close
+            out, err = stdout.read.strip, stderr.read.strip
+          end
+        rescue Exception => e
+          raise Error.wrap(e, "Failed to execute '#{ name }'")
+        end
+
+        if ps.success?
+          unless out.empty?
+            Logger.info(
+              out.lines.map {|line| "#{ name }:STDOUT: #{ line }" }.join
+            )
+          end
+
+          unless err.empty?
+            Logger.warn(
+              err.lines.map {|line| "#{ name }:STDERR: #{ line }" }.join
+            )
+          end
+
+          return out
+        else
+          raise Error, <<-EOS
+            '#{ name }' failed with exit status: #{ ps.exitstatus }
+            STDOUT Messages: #{ out.empty? ? 'None' : "\n#{ out }" }
+            STDERR Messages: #{ err.empty? ? 'None' : "\n#{ err }" }
+          EOS
+        end
+      end
+
+      def reset!
+        UTILITY.clear
+        @gnu_tar = nil
+      end
+    end
+
+    # Allows these utility methods to be included in other classes,
+    # while allowing them to be stubbed in spec_helper for all specs.
+    module Helpers
+      [:utility, :command_name, :run].each do |name|
+        define_method name, lambda {|arg| Utilities.send(name, arg) }
+        private name
+      end
+      private
+      def gnu_tar?; Utilities.gnu_tar?; end
+    end
+  end
+end