backedup 5.0.0.beta.3

data/lib/backup/utilities.rb

@@ -0,0 +1,234 @@
+module Backup
+  module Utilities
+    class Error < Backup::Error; end
+
+    UTILITIES_NAMES = %w[
+      tar cat split sudo chown hostname
+      gzip bzip2 pbzip2
+      mongo mongodump mysqldump innobackupex
+      pg_dump pg_dumpall redis-cli riak-admin
+      gpg openssl
+      rsync ssh
+      sendmail exim
+      send_nsca
+      zabbix_sender
+    ].freeze
+
+    # @api private
+    class DSL
+      def initialize(utils)
+        @utilities = utils
+      end
+
+      # Helper methods to allow users to set the path for all utilities in the
+      # .configure block.
+      #
+      # Utility names with dashes (`redis-cli`) will be set using method calls
+      # with an underscore (`redis_cli`).
+      UTILITIES_NAMES.each do |util_name|
+        define_method util_name.tr("-", "_") do |raw_path|
+          path = File.expand_path(raw_path)
+
+          unless File.executable?(path)
+            raise Utilities::Error, <<-EOS
+              The path given for '#{util_name}' was not found or not executable.
+              Path was: #{path}
+            EOS
+          end
+
+          @utilities.utilities[util_name] = path
+        end
+      end
+
+      # Allow users to set the +tar+ distribution if needed. (:gnu or :bsd)
+      def tar_dist(val)
+        Utilities.tar_dist(val)
+      end
+    end
+
+    class << self
+      ##
+      # Configure the path to system utilities used by Backup.
+      #
+      # Backup will attempt to locate any required system utilities using a
+      # +which+ command call. If a utility can not be found, or you need to
+      # specify an alternate path for a utility, you may do so in your
+      # +config.rb+ file using this method.
+      #
+      # Backup supports both GNU and BSD utilities.
+      # While Backup uses these utilities in a manner compatible with either
+      # version, the +tar+ utility requires some special handling with respect
+      # to +Archive+s. Backup will attempt to detect if the +tar+ command
+      # found (or set here) is GNU or BSD. If for some reason this fails,
+      # this may be set using the +tar_dist+ command shown below.
+      #
+      #   Backup::Utilities.configure do
+      #     # General Utilites
+      #     tar      '/path/to/tar'
+      #     tar_dist :gnu   # or :bsd
+      #     cat      '/path/to/cat'
+      #     split    '/path/to/split'
+      #     sudo     '/path/to/sudo'
+      #     chown    '/path/to/chown'
+      #     hostname '/path/to/hostname'
+      #
+      #     # Compressors
+      #     gzip    '/path/to/gzip'
+      #     bzip2   '/path/to/bzip2'
+      #     pbzip2  '/path/to/pbzip2'
+      #
+      #     # Database Utilities
+      #     mongo       '/path/to/mongo'
+      #     mongodump   '/path/to/mongodump'
+      #     mysqldump   '/path/to/mysqldump'
+      #     pg_dump     '/path/to/pg_dump'
+      #     pg_dumpall  '/path/to/pg_dumpall'
+      #     redis_cli   '/path/to/redis-cli'
+      #     riak_admin  '/path/to/riak-admin'
+      #
+      #     # Encryptors
+      #     gpg     '/path/to/gpg'
+      #     openssl '/path/to/openssl'
+      #
+      #     # Syncer and Storage
+      #     rsync   '/path/to/rsync'
+      #     ssh     '/path/to/ssh'
+      #
+      #     # Notifiers
+      #     sendmail  '/path/to/sendmail'
+      #     exim      '/path/to/exim'
+      #     send_nsca '/path/to/send_nsca'
+      #     zabbix_sender '/path/to/zabbix_sender'
+      #   end
+      #
+      # These paths may be set using absolute paths, or relative to the
+      # working directory when Backup is run.
+      def configure(&block)
+        DSL.new(self).instance_eval(&block)
+      end
+
+      def tar_dist(val)
+        # the acceptance tests need to be able to reset this to nil
+        @gnu_tar = val.nil? ? nil : val == :gnu
+      end
+
+      def gnu_tar?
+        return @gnu_tar unless @gnu_tar.nil?
+        @gnu_tar = !!run("#{utility(:tar)} --version").match(/GNU/)
+      end
+
+      def utilities
+        @utilities ||= {}
+      end
+
+      private
+
+      ##
+      # Returns the full path to the specified utility.
+      # Raises an error if utility can not be found in the system's $PATH
+      def utility(name)
+        name = name.to_s.strip
+        raise Error, "Utility Name Empty" if name.empty?
+
+        utilities[name] ||= `which '#{name}' 2>/dev/null`.chomp
+        raise Error, <<-EOS if utilities[name].empty?
+          Could not locate '#{name}'.
+          Make sure the specified utility is installed
+          and available in your system's $PATH, or specify it's location
+          in your 'config.rb' file using Backup::Utilities.configure
+        EOS
+
+        utilities[name].dup
+      end
+
+      ##
+      # Returns the name of the command name from the given command line.
+      # This is only used to simplify log messages.
+      def command_name(command)
+        parts = []
+        command = command.split(" ")
+        command.shift while command[0].to_s.include?("=")
+        parts << command.shift.split("/")[-1]
+        if parts[0] == "sudo"
+          until command.empty?
+            part = command.shift
+            if part.include?("/")
+              parts << part.split("/")[-1]
+              break
+            else
+              parts << part
+            end
+          end
+        end
+        parts.join(" ")
+      end
+
+      ##
+      # Runs a system command
+      #
+      # All messages generated by the command will be logged.
+      # Messages on STDERR will be logged as warnings.
+      #
+      # If the command fails to execute, or returns a non-zero exit status
+      # an Error will be raised.
+      #
+      # Returns STDOUT
+      def run(command)
+        name = command_name(command)
+        Logger.info "Running system utility '#{name}'..."
+
+        begin
+          out = ""
+          err = ""
+          ps = Open4.popen4(command) do |_pid, stdin, stdout, stderr|
+            stdin.close
+            out = stdout.read.strip
+            err = stderr.read.strip
+          end
+        rescue Exception => e
+          raise Error.wrap(e, "Failed to execute '#{name}'")
+        end
+
+        unless ps.success?
+          raise Error, <<-EOS
+            '#{name}' failed with exit status: #{ps.exitstatus}
+            STDOUT Messages: #{out.empty? ? "None" : "\n#{out}"}
+            STDERR Messages: #{err.empty? ? "None" : "\n#{err}"}
+          EOS
+        end
+
+        unless out.empty?
+          Logger.info(out.lines.map { |line| "#{name}:STDOUT: #{line}" }.join)
+        end
+
+        unless err.empty?
+          Logger.warn(err.lines.map { |line| "#{name}:STDERR: #{line}" }.join)
+        end
+
+        out
+      end
+
+      def reset!
+        utilities.clear
+        @gnu_tar = nil
+      end
+    end
+
+    # Allows these utility methods to be included in other classes,
+    # while allowing them to be stubbed in spec_helper for all specs.
+    module Helpers
+      [:utility, :command_name, :run].each do |name|
+        define_method name do |arg|
+          Utilities.send(name, arg)
+        end
+        private name
+      end
+
+      private
+
+      def gnu_tar?
+        Utilities.gnu_tar?
+      end
+    end
+  end
+end

data/lib/backup/version.rb

@@ -0,0 +1,3 @@
+module Backup
+  VERSION = "5.0.0.beta.3"
+end

data/lib/backup.rb

@@ -0,0 +1,145 @@
+# Load Ruby Core Libraries
+require "time"
+require "fileutils"
+require "tempfile"
+require "syslog"
+require "yaml"
+require "etc"
+require "forwardable"
+require "thread"
+
+require "open4"
+require "thor"
+require "shellwords"
+
+require "excon"
+# Include response.inspect in error messages.
+Excon.defaults[:debug_response] = true
+# Excon should not retry failed requests. We handle that.
+Excon.defaults[:middlewares].delete(Excon::Middleware::Idempotent)
+
+##
+# The Backup Ruby Gem
+module Backup
+  ##
+  # Backup's internal paths
+  LIBRARY_PATH       = File.join(File.dirname(__FILE__), "backup")
+  STORAGE_PATH       = File.join(LIBRARY_PATH, "storage")
+  SYNCER_PATH        = File.join(LIBRARY_PATH, "syncer")
+  DATABASE_PATH      = File.join(LIBRARY_PATH, "database")
+  COMPRESSOR_PATH    = File.join(LIBRARY_PATH, "compressor")
+  ENCRYPTOR_PATH     = File.join(LIBRARY_PATH, "encryptor")
+  NOTIFIER_PATH      = File.join(LIBRARY_PATH, "notifier")
+  TEMPLATE_PATH      = File.expand_path("../../templates", __FILE__)
+
+  ##
+  # Autoload Backup storage files
+  module Storage
+    autoload :Base,       File.join(STORAGE_PATH, "base")
+    autoload :Cycler,     File.join(STORAGE_PATH, "cycler")
+    autoload :S3,         File.join(STORAGE_PATH, "s3")
+    autoload :Swift,      File.join(STORAGE_PATH, "swift")
+    autoload :GCS,        File.join(STORAGE_PATH, "gcs")
+    autoload :CloudFiles, File.join(STORAGE_PATH, "cloud_files")
+    autoload :Ninefold,   File.join(STORAGE_PATH, "ninefold")
+    autoload :Dropbox,    File.join(STORAGE_PATH, "dropbox")
+    autoload :FTP,        File.join(STORAGE_PATH, "ftp")
+    autoload :SFTP,       File.join(STORAGE_PATH, "sftp")
+    autoload :SCP,        File.join(STORAGE_PATH, "scp")
+    autoload :RSync,      File.join(STORAGE_PATH, "rsync")
+    autoload :Local,      File.join(STORAGE_PATH, "local")
+    autoload :Qiniu,      File.join(STORAGE_PATH, "qiniu")
+    autoload :Webdav,     File.join(STORAGE_PATH, 'webdav')
+  end
+
+  ##
+  # Autoload Backup syncer files
+  module Syncer
+    autoload :Base, File.join(SYNCER_PATH, "base")
+    module Cloud
+      autoload :Base,       File.join(SYNCER_PATH, "cloud", "base")
+      autoload :LocalFile,  File.join(SYNCER_PATH, "cloud", "local_file")
+      autoload :CloudFiles, File.join(SYNCER_PATH, "cloud", "cloud_files")
+      autoload :S3,         File.join(SYNCER_PATH, "cloud", "s3")
+      autoload :GCS,        File.join(SYNCER_PATH, "cloud", "gcs")
+    end
+    module RSync
+      autoload :Base,  File.join(SYNCER_PATH, "rsync", "base")
+      autoload :Local, File.join(SYNCER_PATH, "rsync", "local")
+      autoload :Push,  File.join(SYNCER_PATH, "rsync", "push")
+      autoload :Pull,  File.join(SYNCER_PATH, "rsync", "pull")
+    end
+  end
+
+  ##
+  # Autoload Backup database files
+  module Database
+    autoload :Base,       File.join(DATABASE_PATH, "base")
+    autoload :MySQL,      File.join(DATABASE_PATH, "mysql")
+    autoload :PostgreSQL, File.join(DATABASE_PATH, "postgresql")
+    autoload :MongoDB,    File.join(DATABASE_PATH, "mongodb")
+    autoload :Redis,      File.join(DATABASE_PATH, "redis")
+    autoload :Riak,       File.join(DATABASE_PATH, "riak")
+    autoload :OpenLDAP,   File.join(DATABASE_PATH, "openldap")
+    autoload :SQLite,     File.join(DATABASE_PATH, "sqlite")
+  end
+
+  ##
+  # Autoload compressor files
+  module Compressor
+    autoload :Base,   File.join(COMPRESSOR_PATH, "base")
+    autoload :Gzip,   File.join(COMPRESSOR_PATH, "gzip")
+    autoload :Bzip2,  File.join(COMPRESSOR_PATH, "bzip2")
+    autoload :PBzip2, File.join(COMPRESSOR_PATH, "pbzip2")
+    autoload :Custom, File.join(COMPRESSOR_PATH, "custom")
+  end
+
+  ##
+  # Autoload encryptor files
+  module Encryptor
+    autoload :Base,    File.join(ENCRYPTOR_PATH, "base")
+    autoload :OpenSSL, File.join(ENCRYPTOR_PATH, "open_ssl")
+    autoload :GPG,     File.join(ENCRYPTOR_PATH, "gpg")
+  end
+
+  ##
+  # Autoload notification files
+  module Notifier
+    autoload :Base,      File.join(NOTIFIER_PATH, "base")
+    autoload :Mail,      File.join(NOTIFIER_PATH, "mail")
+    autoload :Twitter,   File.join(NOTIFIER_PATH, "twitter")
+    autoload :Campfire,  File.join(NOTIFIER_PATH, "campfire")
+    autoload :Prowl,     File.join(NOTIFIER_PATH, "prowl")
+    autoload :Hipchat,   File.join(NOTIFIER_PATH, "hipchat")
+    autoload :PagerDuty, File.join(NOTIFIER_PATH, "pagerduty")
+    autoload :Pushover,  File.join(NOTIFIER_PATH, "pushover")
+    autoload :Slack,     File.join(NOTIFIER_PATH, "slack")
+    autoload :HttpPost,  File.join(NOTIFIER_PATH, "http_post")
+    autoload :Nagios,    File.join(NOTIFIER_PATH, "nagios")
+    autoload :FlowDock,  File.join(NOTIFIER_PATH, "flowdock")
+    autoload :Zabbix,    File.join(NOTIFIER_PATH, "zabbix")
+    autoload :DataDog,   File.join(NOTIFIER_PATH, "datadog")
+    autoload :Ses,       File.join(NOTIFIER_PATH, "ses")
+    autoload :Command,   File.join(NOTIFIER_PATH, "command")
+  end
+
+  ##
+  # Require Backup base files
+  %w[
+    errors
+    logger
+    utilities
+    archive
+    binder
+    cleaner
+    model
+    config
+    cli
+    package
+    packager
+    pipeline
+    splitter
+    template
+    version
+  ].each { |lib| require File.join(LIBRARY_PATH, lib) }
+end

data/templates/cli/archive

@@ -0,0 +1,28 @@
+  ##
+  # Archive [Archive]
+  #
+  # Adding a file or directory (including sub-directories):
+  #   archive.add "/path/to/a/file.rb"
+  #   archive.add "/path/to/a/directory/"
+  #
+  # Excluding a file or directory (including sub-directories):
+  #   archive.exclude "/path/to/an/excluded_file.rb"
+  #   archive.exclude "/path/to/an/excluded_directory
+  #
+  # By default, relative paths will be relative to the directory
+  # where `backup perform` is executed, and they will be expanded
+  # to the root of the filesystem when added to the archive.
+  #
+  # If a `root` path is set, relative paths will be relative to the
+  # given `root` path and will not be expanded when added to the archive.
+  #
+  #   archive.root '/path/to/archive/root'
+  #
+  archive :my_archive do |archive|
+    # Run the `tar` command using `sudo`
+    # archive.use_sudo
+    archive.add "/path/to/a/file.rb"
+    archive.add "/path/to/a/folder/"
+    archive.exclude "/path/to/a/excluded_file.rb"
+    archive.exclude "/path/to/a/excluded_folder"
+  end

data/templates/cli/compressor/bzip2

@@ -0,0 +1,4 @@
+  ##
+  # Bzip2 [Compressor]
+  #
+  compress_with Bzip2

data/templates/cli/compressor/custom

@@ -0,0 +1,7 @@
+  ##
+  # Custom [Compressor]
+  #
+  compress_with Custom do |compressor|
+    compressor.command   = 'gzip'
+    compressor.extension = '.gz'
+  end

data/templates/cli/compressor/gzip

@@ -0,0 +1,4 @@
+  ##
+  # Gzip [Compressor]
+  #
+  compress_with Gzip

data/templates/cli/config

@@ -0,0 +1,123 @@
+# encoding: utf-8
+
+##
+# Backup v<%= Backup::VERSION.split('.').first %>.x Configuration
+#
+# Documentation: http://backup.github.io/backup
+# Issue Tracker: https://github.com/backup/backup/issues
+
+##
+# Config Options
+#
+# The options here may be overridden on the command line, but the result
+# will depend on the use of --root-path on the command line.
+#
+# If --root-path is used on the command line, then all paths set here
+# will be overridden. If a path (like --tmp-path) is not given along with
+# --root-path, that path will use it's default location _relative to --root-path_.
+#
+# If --root-path is not used on the command line, a path option (like --tmp-path)
+# given on the command line will override the tmp_path set here, but all other
+# paths set here will be used.
+#
+# Note that relative paths given on the command line without --root-path
+# are relative to the current directory. The root_path set here only applies
+# to relative paths set here.
+#
+# ---
+#
+# Sets the root path for all relative paths, including default paths.
+# May be an absolute path, or relative to the current working directory.
+#
+# root_path 'my/root'
+#
+# Sets the path where backups are processed until they're stored.
+# This must have enough free space to hold apx. 2 backups.
+# May be an absolute path, or relative to the current directory or +root_path+.
+#
+# tmp_path  'my/tmp'
+#
+# Sets the path where backup stores persistent information.
+# When Backup's Cycler is used, small YAML files are stored here.
+# May be an absolute path, or relative to the current directory or +root_path+.
+#
+# data_path 'my/data'
+
+##
+# Utilities
+#
+# If you need to use a utility other than the one Backup detects,
+# or a utility can not be found in your $PATH.
+#
+#   Utilities.configure do
+#     tar       '/usr/bin/gnutar'
+#     redis_cli '/opt/redis/redis-cli'
+#   end
+
+##
+# Logging
+#
+# Logging options may be set on the command line, but certain settings
+# may only be configured here.
+#
+#   Logger.configure do
+#     console.quiet     = true            # Same as command line: --quiet
+#     logfile.max_bytes = 2_000_000       # Default: 500_000
+#     syslog.enabled    = true            # Same as command line: --syslog
+#     syslog.ident      = 'my_app_backup' # Default: 'backup'
+#   end
+#
+# Command line options will override those set here.
+# For example, the following would override the example settings above
+# to disable syslog and enable console output.
+#   backup perform --trigger my_backup --no-syslog --no-quiet
+
+##
+# Component Defaults
+#
+# Set default options to be applied to components in all models.
+# Options set within a model will override those set here.
+#
+#   Storage::S3.defaults do |s3|
+#     s3.access_key_id     = "my_access_key_id"
+#     s3.secret_access_key = "my_secret_access_key"
+#   end
+#
+#   Notifier::Mail.defaults do |mail|
+#     mail.from                 = 'sender@email.com'
+#     mail.to                   = 'receiver@email.com'
+#     mail.address              = 'smtp.gmail.com'
+#     mail.port                 = 587
+#     mail.domain               = 'your.host.name'
+#     mail.user_name            = 'sender@email.com'
+#     mail.password             = 'my_password'
+#     mail.authentication       = 'plain'
+#     mail.encryption           = :starttls
+#   end
+
+##
+# Preconfigured Models
+#
+# Create custom models with preconfigured components.
+# Components added within the model definition will
+# +add to+ the preconfigured components.
+#
+#   preconfigure 'MyModel' do
+#     archive :user_pictures do |archive|
+#       archive.add '~/pictures'
+#     end
+#
+#     notify_by Mail do |mail|
+#       mail.to = 'admin@email.com'
+#     end
+#   end
+#
+#   MyModel.new(:john_smith, 'John Smith Backup') do
+#     archive :user_music do |archive|
+#       archive.add '~/music'
+#     end
+#
+#     notify_by Mail do |mail|
+#       mail.to = 'john.smith@email.com'
+#     end
+#   end

data/templates/cli/databases/mongodb

@@ -0,0 +1,15 @@
+  ##
+  # MongoDB [Database]
+  #
+  database MongoDB do |db|
+    db.name               = "my_database_name"
+    db.username           = "my_username"
+    db.password           = "my_password"
+    db.host               = "localhost"
+    db.port               = 27017
+    db.ipv6               = false
+    db.only_collections   = ["only", "these", "collections"]
+    db.additional_options = []
+    db.lock               = false
+    db.oplog              = false
+  end

data/templates/cli/databases/mysql

@@ -0,0 +1,18 @@
+  ##
+  # MySQL [Database]
+  #
+  database MySQL do |db|
+    # To dump all databases, set `db.name = :all` (or leave blank)
+    db.name               = "my_database_name"
+    db.username           = "my_username"
+    db.password           = "my_password"
+    db.host               = "localhost"
+    db.port               = 3306
+    db.socket             = "/tmp/mysql.sock"
+    # Note: when using `skip_tables` with the `db.name = :all` option,
+    # table names should be prefixed with a database name.
+    # e.g. ["db_name.table_to_skip", ...]
+    db.skip_tables        = ["skip", "these", "tables"]
+    db.only_tables        = ["only", "these", "tables"]
+    db.additional_options = ["--quick", "--single-transaction"]
+  end

data/templates/cli/databases/openldap

@@ -0,0 +1,24 @@
+  ##
+  # OpenLDAP [Database]
+  #
+  database OpenLDAP do |db|
+    # Name of the ldap backup
+    db.name = "my_database_name"
+
+    # Additional "slapcat" options
+    # defaults to an empty list of options
+    # db.slapcat_args      = []
+
+    # run slapcat under sudo if needed
+    # defaults to "false"
+    # db.use_sudo = true
+
+    # Stores the location of the slapd.conf,
+    # defaults to "/etc/ldap/slapd.d" confdir
+    # you can use also conf file instead of directory
+    # db.slapcat_conf = /path/to/slapd.conf
+
+    # Optional: Use to set the location of slapcat utility
+    # if it cannot be found by name in your $PATH
+    # db.slapcat_utility = "/opt/local/bin/slapcat"
+  end

data/templates/cli/databases/postgresql

@@ -0,0 +1,16 @@
+  ##
+  # PostgreSQL [Database]
+  #
+  database PostgreSQL do |db|
+    # To dump all databases, set `db.name = :all` (or leave blank)
+    db.name               = "my_database_name"
+    db.username           = "my_username"
+    db.password           = "my_password"
+    db.host               = "localhost"
+    db.port               = 5432
+    db.socket             = "/tmp/pg.sock"
+    # When dumping all databases, `skip_tables` and `only_tables` are ignored.
+    db.skip_tables        = ["skip", "these", "tables"]
+    db.only_tables        = ["only", "these", "tables"]
+    db.additional_options = ["-xc", "-E=utf8"]
+  end

data/templates/cli/databases/redis

@@ -0,0 +1,16 @@
+  ##
+  # Redis [Database]
+  #
+  database Redis do |db|
+    db.mode               = :copy # or :sync
+    # Full path to redis dump file for :copy mode.
+    db.rdb_path           = '/var/lib/redis/dump.rdb'
+    # When :copy mode is used, perform a SAVE before
+    # copying the dump file specified by `rdb_path`.
+    db.invoke_save        = false
+    db.host               = 'localhost'
+    db.port               = 6379
+    db.socket             = '/tmp/redis.sock'
+    db.password           = 'my_password'
+    db.additional_options = []
+  end

data/templates/cli/databases/riak

@@ -0,0 +1,17 @@
+  ##
+  # Riak [Database]
+  #
+  database Riak do |db|
+    ##
+    # The node from which to perform the backup.
+    # default: 'riak@127.0.0.1'
+    db.node = 'riak@hostname'
+    ##
+    # The Erlang cookie/shared secret used to connect to the node.
+    # default: 'riak'
+    db.cookie = 'cookie'
+    ##
+    # The user for the Riak instance.
+    # default: 'riak'
+    db.user = 'riak'
+  end

data/templates/cli/databases/sqlite

@@ -0,0 +1,11 @@
+  ##
+  # SQLite [Database]
+  #
+  database SQLite do |db|
+    # Path to database file
+    db.path               = "/path/to/my/sqlite/db.sqlite"
+
+    # Optional: Use to set the location of this utility
+    #   if it cannot be found by name in your $PATH
+    db.sqlitedump_utility = "/opt/local/bin/sqlite3"
+  end