backupii 0.1.0.pre.alpha.1

data/lib/backup/syncer/rsync/push.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+module Backup
+  module Syncer
+    module RSync
+      class Push < Base
+        ##
+        # Mode of operation
+        #
+        # [:ssh (default)]
+        #   Connects to the remote via SSH.
+        #   Does not use an rsync daemon on the remote.
+        #
+        # [:ssh_daemon]
+        #   Connects to the remote via SSH.
+        #   Spawns a single-use daemon on the remote, which allows certain
+        #   daemon features (like modules) to be used.
+        #
+        # [:rsync_daemon]
+        #   Connects directly to an rsync daemon via TCP.
+        #   Data transferred is not encrypted.
+        #
+        attr_accessor :mode
+
+        ##
+        # Server Address
+        attr_accessor :host
+
+        ##
+        # SSH or RSync port
+        #
+        # For `:ssh` or `:ssh_daemon` mode, this specifies the SSH port to use
+        # and defaults to 22.
+        #
+        # For `:rsync_daemon` mode, this specifies the TCP port to use
+        # and defaults to 873.
+        attr_accessor :port
+
+        ##
+        # SSH User
+        #
+        # If the user running the backup is not the same user that needs to
+        # authenticate with the remote server, specify the user here.
+        #
+        # The user must have SSH keys setup for passphrase-less access to the
+        # remote. If the SSH User does not have passphrase-less keys, or no
+        # default keys in their `~/.ssh` directory, you will need to use the
+        # `-i` option in `:additional_ssh_options` to specify the
+        # passphrase-less key to use.
+        #
+        # Used only for `:ssh` and `:ssh_daemon` modes.
+        attr_accessor :ssh_user
+
+        ##
+        # Additional SSH Options
+        #
+        # Used to supply a String or Array of options to be passed to the SSH
+        # command in `:ssh` and `:ssh_daemon` modes.
+        #
+        # For example, if you need to supply a specific SSH key for the
+        # `ssh_user`, you would set this to: "-i '/path/to/id_rsa'". Which would
+        # produce:
+        #
+        #   rsync -e "ssh -p 22 -i '/path/to/id_rsa'"
+        #
+        # Arguments may be single-quoted, but should not contain any
+        # double-quotes.
+        #
+        # Used only for `:ssh` and `:ssh_daemon` modes.
+        attr_accessor :additional_ssh_options
+
+        ##
+        # RSync User
+        #
+        # If the user running the backup is not the same user that needs to
+        # authenticate with the rsync daemon, specify the user here.
+        #
+        # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+        attr_accessor :rsync_user
+
+        ##
+        # RSync Password
+        #
+        # If specified, Backup will write the password to a temporary file and
+        # use it with rsync's `--password-file` option for daemon
+        # authentication.
+        #
+        # Note that setting this will override `rsync_password_file`.
+        #
+        # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+        attr_accessor :rsync_password
+
+        ##
+        # RSync Password File
+        #
+        # If specified, this path will be passed to rsync's `--password-file`
+        # option for daemon authentication.
+        #
+        # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+        attr_accessor :rsync_password_file
+
+        ##
+        # Flag for compressing (only compresses for the transfer)
+        attr_accessor :compress
+
+        def initialize(syncer_id = nil)
+          super
+
+          @mode ||= :ssh
+          @port ||= mode == :rsync_daemon ? 873 : 22
+          @compress ||= false
+        end
+
+        def perform!
+          log!(:started)
+          write_password_file!
+
+          create_dest_path!
+          run("#{rsync_command} #{paths_to_push} " \
+              "#{host_options}'#{dest_path}'")
+
+          log!(:finished)
+        ensure
+          remove_password_file!
+        end
+
+        private
+
+        ##
+        # Remove any preceeding '~/', since this is on the remote,
+        # and remove any trailing `/`.
+        def dest_path
+          @dest_path ||= path.sub(%r{^~/}, "").sub(%r{/$}, "")
+        end
+
+        ##
+        # Runs a 'mkdir -p' command on the remote to ensure the dest_path
+        # exists. This used because rsync will attempt to create the path, but
+        # will only call 'mkdir' without the '-p' option. This is only
+        # applicable in :ssh mode, and only used if the path would require this.
+        def create_dest_path!
+          return unless mode == :ssh && dest_path.index("/").to_i > 0
+
+          run "#{utility(:ssh)} #{ssh_transport_args} #{host} " +
+            %("mkdir -p '#{dest_path}'")
+        end
+
+        ##
+        # For Push, this will prepend the #dest_path.
+        # For Pull, this will prepend the first path in #paths_to_pull.
+        def host_options
+          if mode == :ssh
+            "#{host}:"
+          else
+            user = "#{rsync_user}@" if rsync_user
+            "#{user}#{host}::"
+          end
+        end
+
+        ##
+        # Common base command, plus options for Push/Pull
+        def rsync_command
+          super << compress_option << password_option << transport_options
+        end
+
+        def compress_option
+          compress ? " --compress" : ""
+        end
+
+        def password_option
+          return "" if mode == :ssh
+
+          path = @password_file ? @password_file.path : rsync_password_file
+          path ? " --password-file='#{File.expand_path(path)}'" : ""
+        end
+
+        def transport_options
+          if mode == :rsync_daemon
+            " --port #{port}"
+          else
+            %( -e "#{utility(:ssh)} #{ssh_transport_args}")
+          end
+        end
+
+        def ssh_transport_args
+          args = "-p #{port} ".dup
+          args << "-l #{ssh_user} " if ssh_user
+          args << Array(additional_ssh_options).join(" ")
+          args.rstrip
+        end
+
+        def write_password_file!
+          return unless rsync_password && mode != :ssh
+
+          @password_file = Tempfile.new("backup-rsync-password")
+          @password_file.write(rsync_password)
+          @password_file.close
+        end
+
+        def remove_password_file!
+          @password_file.delete if @password_file
+        end
+      end
+    end
+  end
+end

data/lib/backup/template.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "erb"
+
+module Backup
+  class Template
+    # Holds a binding object. Nil if not provided.
+    attr_accessor :binding
+
+    ##
+    # Creates a new instance of the Backup::Template class and optionally takes
+    # an argument that can be either a binding object, a Hash or nil
+    def initialize(object = nil)
+      @binding =
+        if object.is_a?(Binding)
+          object
+        elsif object.is_a?(Hash)
+          Backup::Binder.new(object).get_binding
+        end
+    end
+
+    ##
+    # Renders the provided file (in the context of the binding if any) to the
+    # console
+    def render(file)
+      puts result(file)
+    end
+
+    ##
+    # Returns a String object containing the contents of the file (in the
+    # context of the binding if any)
+    def result(file)
+      ERB.new(file_contents(file), nil, "<>").result(binding)
+    end
+
+    private
+
+    ##
+    # Reads and returns the contents of the provided file path,
+    # relative from the Backup::TEMPLATE_PATH
+    def file_contents(file)
+      File.read(File.join(Backup::TEMPLATE_PATH, file))
+    end
+  end
+end

data/lib/backup/utilities.rb

@@ -0,0 +1,235 @@
+# frozen_string_literal: true
+
+module Backup
+  module Utilities
+    class Error < Backup::Error; end
+
+    UTILITIES_NAMES = %w[
+      tar cat split sudo chown hostname
+      gzip bzip2
+      mongo mongodump mysqldump innobackupex
+      pg_dump pg_dumpall redis-cli riak-admin
+      gpg openssl
+      rsync ssh
+      sendmail exim
+      send_nsca
+      zabbix_sender
+    ].freeze
+
+    # @api private
+    class DSL
+      def initialize(utils)
+        @utilities = utils
+      end
+
+      # Helper methods to allow users to set the path for all utilities in the
+      # .configure block.
+      #
+      # Utility names with dashes (`redis-cli`) will be set using method calls
+      # with an underscore (`redis_cli`).
+      UTILITIES_NAMES.each do |util_name|
+        define_method util_name.tr("-", "_") do |raw_path|
+          path = File.expand_path(raw_path)
+
+          unless File.executable?(path)
+            raise Utilities::Error, <<-EOS
+              The path given for '#{util_name}' was not found or not executable.
+              Path was: #{path}
+            EOS
+          end
+
+          @utilities.utilities[util_name] = path
+        end
+      end
+
+      # Allow users to set the +tar+ distribution if needed. (:gnu or :bsd)
+      def tar_dist(val)
+        Utilities.tar_dist(val)
+      end
+    end
+
+    class << self
+      ##
+      # Configure the path to system utilities used by Backup.
+      #
+      # Backup will attempt to locate any required system utilities using a
+      # +which+ command call. If a utility can not be found, or you need to
+      # specify an alternate path for a utility, you may do so in your
+      # +config.rb+ file using this method.
+      #
+      # Backup supports both GNU and BSD utilities.
+      # While Backup uses these utilities in a manner compatible with either
+      # version, the +tar+ utility requires some special handling with respect
+      # to +Archive+s. Backup will attempt to detect if the +tar+ command
+      # found (or set here) is GNU or BSD. If for some reason this fails,
+      # this may be set using the +tar_dist+ command shown below.
+      #
+      #   Backup::Utilities.configure do
+      #     # General Utilites
+      #     tar      '/path/to/tar'
+      #     tar_dist :gnu   # or :bsd
+      #     cat      '/path/to/cat'
+      #     split    '/path/to/split'
+      #     sudo     '/path/to/sudo'
+      #     chown    '/path/to/chown'
+      #     hostname '/path/to/hostname'
+      #
+      #     # Compressors
+      #     gzip    '/path/to/gzip'
+      #     bzip2   '/path/to/bzip2'
+      #
+      #     # Database Utilities
+      #     mongo       '/path/to/mongo'
+      #     mongodump   '/path/to/mongodump'
+      #     mysqldump   '/path/to/mysqldump'
+      #     pg_dump     '/path/to/pg_dump'
+      #     pg_dumpall  '/path/to/pg_dumpall'
+      #     redis_cli   '/path/to/redis-cli'
+      #     riak_admin  '/path/to/riak-admin'
+      #
+      #     # Encryptors
+      #     gpg     '/path/to/gpg'
+      #     openssl '/path/to/openssl'
+      #
+      #     # Syncer and Storage
+      #     rsync   '/path/to/rsync'
+      #     ssh     '/path/to/ssh'
+      #
+      #     # Notifiers
+      #     sendmail  '/path/to/sendmail'
+      #     exim      '/path/to/exim'
+      #     send_nsca '/path/to/send_nsca'
+      #     zabbix_sender '/path/to/zabbix_sender'
+      #   end
+      #
+      # These paths may be set using absolute paths, or relative to the
+      # working directory when Backup is run.
+      def configure(&block)
+        DSL.new(self).instance_eval(&block)
+      end
+
+      def tar_dist(val)
+        # the acceptance tests need to be able to reset this to nil
+        @gnu_tar = val.nil? ? nil : val == :gnu
+      end
+
+      def gnu_tar?
+        return @gnu_tar unless @gnu_tar.nil?
+
+        @gnu_tar = !!run("#{utility(:tar)} --version").match(%r{GNU})
+      end
+
+      def utilities
+        @utilities ||= {}
+      end
+
+      private
+
+      ##
+      # Returns the full path to the specified utility.
+      # Raises an error if utility can not be found in the system's $PATH
+      def utility(name)
+        name = name.to_s.strip
+        raise Error, "Utility Name Empty" if name.empty?
+
+        utilities[name] ||= `which '#{name}' 2>/dev/null`.chomp
+        raise Error, <<-EOS if utilities[name].empty?
+          Could not locate '#{name}'.
+          Make sure the specified utility is installed
+          and available in your system's $PATH, or specify it's location
+          in your 'config.rb' file using Backup::Utilities.configure
+        EOS
+
+        utilities[name].dup
+      end
+
+      ##
+      # Returns the name of the command name from the given command line.
+      # This is only used to simplify log messages.
+      def command_name(command)
+        parts = []
+        command = command.split(" ")
+        command.shift while command[0].to_s.include?("=")
+        parts << command.shift.split("/")[-1]
+        if parts[0] == "sudo"
+          until command.empty?
+            part = command.shift
+            if part.include?("/")
+              parts << part.split("/")[-1]
+              break
+            else
+              parts << part
+            end
+          end
+        end
+        parts.join(" ")
+      end
+
+      ##
+      # Runs a system command
+      #
+      # All messages generated by the command will be logged.
+      # Messages on STDERR will be logged as warnings.
+      #
+      # If the command fails to execute, or returns a non-zero exit status
+      # an Error will be raised.
+      #
+      # Returns STDOUT
+      def run(command)
+        name = command_name(command)
+        Logger.info "Running system utility '#{name}'..."
+
+        begin
+          out = ""
+          err = ""
+          ps = Open4.popen4(command) do |_pid, stdin, stdout, stderr|
+            stdin.close
+            out = stdout.read.strip
+            err = stderr.read.strip
+          end
+        rescue Exception => err
+          raise Error.wrap(err, "Failed to execute '#{name}'")
+        end
+
+        unless ps.success?
+          raise Error, <<-EOS
+            '#{name}' failed with exit status: #{ps.exitstatus}
+            STDOUT Messages: #{out.empty? ? "None" : "\n#{out}"}
+            STDERR Messages: #{err.empty? ? "None" : "\n#{err}"}
+          EOS
+        end
+
+        unless out.empty?
+          Logger.info(out.lines.map { |line| "#{name}:STDOUT: #{line}" }.join)
+        end
+
+        unless err.empty?
+          Logger.warn(err.lines.map { |line| "#{name}:STDERR: #{line}" }.join)
+        end
+
+        out
+      end
+
+      def reset!
+        utilities.clear
+        @gnu_tar = nil
+      end
+    end
+
+    # Allows these utility methods to be included in other classes,
+    # while allowing them to be stubbed in spec_helper for all specs.
+    module Helpers
+      private
+
+      [:utility, :command_name, :run].each do |name|
+        define_method name do |arg|
+          Utilities.send(name, arg)
+        end
+      end
+
+      def gnu_tar?
+        Utilities.gnu_tar?
+      end
+    end
+  end
+end

data/lib/backup/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Backup
+  VERSION = "0.1.0-alpha.1"
+end

data/lib/backup.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+# Load Ruby Core Libraries
+require "time"
+require "fileutils"
+require "tempfile"
+require "syslog"
+require "yaml"
+require "etc"
+require "forwardable"
+
+require "open4"
+require "thor"
+require "shellwords"
+
+require "excon"
+# Include response.inspect in error messages.
+Excon.defaults[:debug_response] = true
+# Excon should not retry failed requests. We handle that.
+Excon.defaults[:middlewares].delete(Excon::Middleware::Idempotent)
+
+##
+# The Backup Ruby Gem
+module Backup
+  ##
+  # Backup's internal paths
+  LIBRARY_PATH       = File.join(File.dirname(__FILE__), "backup")
+  STORAGE_PATH       = File.join(LIBRARY_PATH, "storage")
+  SYNCER_PATH        = File.join(LIBRARY_PATH, "syncer")
+  DATABASE_PATH      = File.join(LIBRARY_PATH, "database")
+  COMPRESSOR_PATH    = File.join(LIBRARY_PATH, "compressor")
+  ENCRYPTOR_PATH     = File.join(LIBRARY_PATH, "encryptor")
+  NOTIFIER_PATH      = File.join(LIBRARY_PATH, "notifier")
+  TEMPLATE_PATH      = File.expand_path("../templates", __dir__)
+
+  ##
+  # Autoload Backup storage files
+  module Storage
+    autoload :Base,       File.join(STORAGE_PATH, "base")
+    autoload :Cycler,     File.join(STORAGE_PATH, "cycler")
+    autoload :S3,         File.join(STORAGE_PATH, "s3")
+    autoload :CloudFiles, File.join(STORAGE_PATH, "cloud_files")
+    autoload :Ninefold,   File.join(STORAGE_PATH, "ninefold")
+    autoload :Dropbox,    File.join(STORAGE_PATH, "dropbox")
+    autoload :FTP,        File.join(STORAGE_PATH, "ftp")
+    autoload :SFTP,       File.join(STORAGE_PATH, "sftp")
+    autoload :SCP,        File.join(STORAGE_PATH, "scp")
+    autoload :RSync,      File.join(STORAGE_PATH, "rsync")
+    autoload :Local,      File.join(STORAGE_PATH, "local")
+    autoload :Qiniu,      File.join(STORAGE_PATH, "qiniu")
+  end
+
+  ##
+  # Autoload Backup syncer files
+  module Syncer
+    autoload :Base, File.join(SYNCER_PATH, "base")
+    module Cloud
+      autoload :Base,       File.join(SYNCER_PATH, "cloud", "base")
+      autoload :LocalFile,  File.join(SYNCER_PATH, "cloud", "local_file")
+      autoload :CloudFiles, File.join(SYNCER_PATH, "cloud", "cloud_files")
+      autoload :S3,         File.join(SYNCER_PATH, "cloud", "s3")
+    end
+    module RSync
+      autoload :Base,  File.join(SYNCER_PATH, "rsync", "base")
+      autoload :Local, File.join(SYNCER_PATH, "rsync", "local")
+      autoload :Push,  File.join(SYNCER_PATH, "rsync", "push")
+      autoload :Pull,  File.join(SYNCER_PATH, "rsync", "pull")
+    end
+  end
+
+  ##
+  # Autoload Backup database files
+  module Database
+    autoload :Base,       File.join(DATABASE_PATH, "base")
+    autoload :MySQL,      File.join(DATABASE_PATH, "mysql")
+    autoload :PostgreSQL, File.join(DATABASE_PATH, "postgresql")
+    autoload :MongoDB,    File.join(DATABASE_PATH, "mongodb")
+    autoload :Redis,      File.join(DATABASE_PATH, "redis")
+    autoload :Riak,       File.join(DATABASE_PATH, "riak")
+    autoload :OpenLDAP,   File.join(DATABASE_PATH, "openldap")
+    autoload :SQLite,     File.join(DATABASE_PATH, "sqlite")
+  end
+
+  ##
+  # Autoload compressor files
+  module Compressor
+    autoload :Base,   File.join(COMPRESSOR_PATH, "base")
+    autoload :Gzip,   File.join(COMPRESSOR_PATH, "gzip")
+    autoload :Bzip2,  File.join(COMPRESSOR_PATH, "bzip2")
+    autoload :Custom, File.join(COMPRESSOR_PATH, "custom")
+  end
+
+  ##
+  # Autoload encryptor files
+  module Encryptor
+    autoload :Base,    File.join(ENCRYPTOR_PATH, "base")
+    autoload :OpenSSL, File.join(ENCRYPTOR_PATH, "open_ssl")
+    autoload :GPG,     File.join(ENCRYPTOR_PATH, "gpg")
+  end
+
+  ##
+  # Autoload notification files
+  module Notifier
+    autoload :Base,      File.join(NOTIFIER_PATH, "base")
+    autoload :Mail,      File.join(NOTIFIER_PATH, "mail")
+    autoload :Twitter,   File.join(NOTIFIER_PATH, "twitter")
+    autoload :Campfire,  File.join(NOTIFIER_PATH, "campfire")
+    autoload :Prowl,     File.join(NOTIFIER_PATH, "prowl")
+    autoload :Hipchat,   File.join(NOTIFIER_PATH, "hipchat")
+    autoload :PagerDuty, File.join(NOTIFIER_PATH, "pagerduty")
+    autoload :Pushover,  File.join(NOTIFIER_PATH, "pushover")
+    autoload :Slack,     File.join(NOTIFIER_PATH, "slack")
+    autoload :HttpPost,  File.join(NOTIFIER_PATH, "http_post")
+    autoload :Nagios,    File.join(NOTIFIER_PATH, "nagios")
+    autoload :FlowDock,  File.join(NOTIFIER_PATH, "flowdock")
+    autoload :Zabbix,    File.join(NOTIFIER_PATH, "zabbix")
+    autoload :DataDog,   File.join(NOTIFIER_PATH, "datadog")
+    autoload :Ses,       File.join(NOTIFIER_PATH, "ses")
+    autoload :Command,   File.join(NOTIFIER_PATH, "command")
+  end
+
+  ##
+  # Require Backup base files
+  %w[
+    errors
+    logger
+    utilities
+    archive
+    binder
+    cleaner
+    model
+    config
+    cli
+    package
+    packager
+    pipeline
+    splitter
+    template
+    version
+  ].each { |lib| require File.join(LIBRARY_PATH, lib) }
+end

data/templates/cli/archive

@@ -0,0 +1,28 @@
+  ##
+  # Archive [Archive]
+  #
+  # Adding a file or directory (including sub-directories):
+  #   archive.add "/path/to/a/file.rb"
+  #   archive.add "/path/to/a/directory/"
+  #
+  # Excluding a file or directory (including sub-directories):
+  #   archive.exclude "/path/to/an/excluded_file.rb"
+  #   archive.exclude "/path/to/an/excluded_directory
+  #
+  # By default, relative paths will be relative to the directory
+  # where `backup perform` is executed, and they will be expanded
+  # to the root of the filesystem when added to the archive.
+  #
+  # If a `root` path is set, relative paths will be relative to the
+  # given `root` path and will not be expanded when added to the archive.
+  #
+  #   archive.root '/path/to/archive/root'
+  #
+  archive :my_archive do |archive|
+    # Run the `tar` command using `sudo`
+    # archive.use_sudo
+    archive.add "/path/to/a/file.rb"
+    archive.add "/path/to/a/folder/"
+    archive.exclude "/path/to/a/excluded_file.rb"
+    archive.exclude "/path/to/a/excluded_folder"
+  end

data/templates/cli/compressor/bzip2

@@ -0,0 +1,4 @@
+  ##
+  # Bzip2 [Compressor]
+  #
+  compress_with Bzip2

data/templates/cli/compressor/custom

@@ -0,0 +1,7 @@
+  ##
+  # Custom [Compressor]
+  #
+  compress_with Custom do |compressor|
+    compressor.command   = 'gzip'
+    compressor.extension = '.gz'
+  end

data/templates/cli/compressor/gzip

@@ -0,0 +1,4 @@
+  ##
+  # Gzip [Compressor]
+  #
+  compress_with Gzip