backedup 5.0.0.beta.3

data/lib/backup/config/dsl.rb

@@ -0,0 +1,102 @@
+module Backup
+  module Config
+    # Context for loading user config.rb and model files.
+    class DSL
+      class Error < Backup::Error; end
+      Model = Backup::Model
+
+      class << self
+        private
+
+        # List the available database, storage, syncer, compressor, encryptor
+        # and notifier constants. These are used to define constant names within
+        # Backup::Config::DSL so that users may use a constant instead of a string.
+        # Nested namespaces are represented using Hashs. Deep nesting supported.
+        #
+        # Example, instead of:
+        #  database "MySQL" do |mysql|
+        #  sync_with "RSync::Local" do |rsync|
+        #
+        # You can do:
+        #  database MySQL do |mysql|
+        #  sync_with RSync::Local do |rsync|
+        #
+        def add_dsl_constants
+          create_modules(
+            DSL,
+            [ # Databases
+              ["MySQL", "PostgreSQL", "MongoDB", "Redis", "Riak", "OpenLDAP", "SQLite"],
+              # Storages
+              ["S3", "Swift", "GCS", "CloudFiles", "Ninefold", "Dropbox", "FTP",
+               "SFTP", "SCP", "RSync", "Local", "Qiniu", "Webdav"],
+              # Compressors
+              ["Gzip", "Bzip2", "PBzip2", "Custom"],
+              # Encryptors
+              ["OpenSSL", "GPG"],
+              # Syncers
+              [
+                { "Cloud" => ["CloudFiles", "S3"] },
+                { "RSync" => ["Push", "Pull", "Local"] }
+              ],
+              # Notifiers
+              ["Mail", "Twitter", "Campfire", "Prowl",
+               "Hipchat", "PagerDuty", "Pushover", "HttpPost", "Nagios",
+               "Slack", "FlowDock", "Zabbix", "Ses", "DataDog", "Command"]
+            ]
+          )
+        end
+
+        def create_modules(scope, names)
+          names.flatten.each do |name|
+            if name.is_a?(Hash)
+              name.each do |key, val|
+                create_modules(get_or_create_empty_module(scope, key), [val])
+              end
+            else
+              get_or_create_empty_module(scope, name)
+            end
+          end
+        end
+
+        def get_or_create_empty_module(scope, const)
+          if scope.const_defined?(const)
+            scope.const_get(const)
+          else
+            scope.const_set(const, Module.new)
+          end
+        end
+      end
+
+      add_dsl_constants # add constants on load
+
+      attr_reader :_config_options
+
+      def initialize
+        @_config_options = {}
+      end
+
+      # Allow users to set command line path options in config.rb
+      [:root_path, :data_path, :tmp_path].each do |name|
+        define_method name do |path|
+          _config_options[name] = path
+        end
+      end
+
+      # Allows users to create preconfigured models.
+      def preconfigure(name, &block)
+        unless name.is_a?(String) && name =~ /^[A-Z]/
+          raise Error, "Preconfigured model names must be given as a string " \
+                        "and start with a capital letter."
+        end
+
+        if DSL.const_defined?(name)
+          raise Error, "'#{name}' is already in use " \
+                        "and can not be used for a preconfigured model."
+        end
+
+        DSL.const_set(name, Class.new(Model))
+        DSL.const_get(name).preconfigure(&block)
+      end
+    end
+  end
+end

data/lib/backup/config/helpers.rb

@@ -0,0 +1,137 @@
+require "ostruct"
+
+module Backup
+  module Config
+    module Helpers
+      def self.included(klass)
+        klass.extend ClassMethods
+      end
+
+      module ClassMethods
+        def defaults
+          @defaults ||= Config::Defaults.new
+
+          if block_given?
+            yield @defaults
+          else
+            @defaults
+          end
+        end
+
+        # Used only within the specs
+        def clear_defaults!
+          defaults.reset!
+        end
+
+        def deprecations
+          @deprecations ||= {}
+        end
+
+        def log_deprecation_warning(name, deprecation)
+          msg = "#{self}##{name} has been deprecated as of " \
+              "backup v.#{deprecation[:version]}"
+          msg << "\n#{deprecation[:message]}" if deprecation[:message]
+          Logger.warn Config::Error.new(<<-EOS)
+            [DEPRECATION WARNING]
+            #{msg}
+          EOS
+        end
+
+        protected
+
+        ##
+        # Method to deprecate an attribute.
+        #
+        # :version
+        #   Must be set to the backup version which will first
+        #   introduce the deprecation.
+        #
+        # :action
+        #   If set, this Proc will be called with a reference to the
+        #   class instance and the value set on the deprecated accessor.
+        #   e.g. deprecation[:action].call(klass, value)
+        #   This should perform whatever action is neccessary, such as
+        #   transferring the value to a new accessor.
+        #
+        # :message
+        #   If set, this will be appended to #log_deprecation_warning
+        #
+        # Note that this replaces the `attr_accessor` method, or other
+        # method previously used to set the accessor being deprecated.
+        # #method_missing will handle any calls to `name=`.
+        #
+        def attr_deprecate(name, args = {})
+          deprecations[name] = {
+            version: nil,
+            message: nil,
+            action: nil
+          }.merge(args)
+        end
+      end # ClassMethods
+
+      private
+
+      ##
+      # Sets any pre-configured default values.
+      # If a default value was set for an invalid accessor,
+      # this will raise a NameError.
+      def load_defaults!
+        self.class.defaults._attributes.each do |name|
+          val = self.class.defaults.send(name)
+          val = val.dup rescue val
+          send(:"#{ name }=", val)
+        end
+      end
+
+      ##
+      # Check missing methods for deprecated attribute accessors.
+      #
+      # If a value is set on an accessor that has been deprecated
+      # using #attr_deprecate, a warning will be issued and any
+      # :action (Proc) specified will be called with a reference to
+      # the class instance and the value set on the deprecated accessor.
+      # See #attr_deprecate and #log_deprecation_warning
+      #
+      # Note that OpenStruct (used for setting defaults) does not allow
+      # multiple arguments when assigning values for members.
+      # So, we won't allow it here either, even though an attr_accessor
+      # will accept and convert them into an Array. Therefore, setting
+      # an option value using multiple values, whether as a default or
+      # directly on the class' accessor, should not be supported.
+      # i.e. if an option will accept being set as an Array, then it
+      # should be explicitly set as such. e.g. option = [val1, val2]
+      #
+      def method_missing(name, *args)
+        deprecation = nil
+        if method = name.to_s.chomp!("=")
+          if (len = args.count) != 1
+            raise ArgumentError,
+              "wrong number of arguments (#{len} for 1)", caller(1)
+          end
+          deprecation = self.class.deprecations[method.to_sym]
+        end
+
+        if deprecation
+          self.class.log_deprecation_warning(method, deprecation)
+          deprecation[:action].call(self, args[0]) if deprecation[:action]
+        else
+          super
+        end
+      end
+    end # Helpers
+
+    # Store for pre-configured defaults.
+    class Defaults < OpenStruct
+      # Returns an Array of all attribute method names
+      # that default values were set for.
+      def _attributes
+        @table.keys
+      end
+
+      # Used only within the specs
+      def reset!
+        @table.clear
+      end
+    end
+  end
+end

data/lib/backup/config.rb

@@ -0,0 +1,118 @@
+require "backup/config/dsl"
+require "backup/config/helpers"
+
+module Backup
+  module Config
+    class Error < Backup::Error; end
+
+    DEFAULTS = {
+      config_file: "config.rb",
+      data_path: ".data",
+      tmp_path: ".tmp"
+    }
+
+    class << self
+      include Utilities::Helpers
+
+      attr_reader :user, :root_path, :config_file, :data_path, :tmp_path
+
+      # Loads the user's +config.rb+ and all model files.
+      def load(options = {})
+        update(options) # from the command line
+
+        unless File.exist?(config_file)
+          raise Error, "Could not find configuration file: '#{config_file}'."
+        end
+
+        config = File.read(config_file)
+        version = Backup::VERSION.split(".").first
+        unless config =~ /^# Backup v#{ version }\.x Configuration$/
+          raise Error, <<-EOS
+            Invalid Configuration File
+            The configuration file at '#{config_file}'
+            does not appear to be a Backup v#{version}.x configuration file.
+            If you have upgraded to v#{version}.x from a previous version,
+            you need to upgrade your configuration file.
+            Please see the instructions for upgrading in the Backup documentation.
+          EOS
+        end
+
+        dsl = DSL.new
+        dsl.instance_eval(config, config_file)
+
+        update(dsl._config_options)  # from config.rb
+        update(options)              # command line takes precedence
+
+        Dir[File.join(File.dirname(config_file), "models", "*.rb")].each do |model|
+          dsl.instance_eval(File.read(model), model)
+        end
+      end
+
+      def hostname
+        @hostname ||= run(utility(:hostname))
+      end
+
+      private
+
+      # If :root_path is set in the options, all paths will be updated.
+      # Otherwise, only the paths given will be updated.
+      def update(options = {})
+        root_path = options[:root_path].to_s.strip
+        new_root = root_path.empty? ? false : set_root_path(root_path)
+
+        DEFAULTS.each do |name, ending|
+          set_path_variable(name, options[name], ending, new_root)
+        end
+      end
+
+      # Sets the @root_path to the given +path+ and returns it.
+      # Raises an error if the given +path+ does not exist.
+      def set_root_path(path)
+        # allows #reset! to set the default @root_path,
+        # then use #update to set all other paths,
+        # without requiring that @root_path exist.
+        return @root_path if path == @root_path
+
+        path = File.expand_path(path)
+        unless File.directory?(path)
+          raise Error, <<-EOS
+            Root Path Not Found
+            When specifying a --root-path, the path must exist.
+            Path was: #{path}
+          EOS
+        end
+        @root_path = path
+      end
+
+      def set_path_variable(name, path, ending, root_path)
+        # strip any trailing '/' in case the user supplied this as part of
+        # an absolute path, so we can match it against File.expand_path()
+        path = path.to_s.sub(/\/\s*$/, "").lstrip
+        new_path = false
+        # If no path is given, the variable will not be set/updated
+        # unless a root_path was given. In which case the value will
+        # be updated with our default ending.
+        if path.empty?
+          new_path = File.join(root_path, ending) if root_path
+        else
+          # When a path is given, the variable will be set/updated.
+          # If the path is relative, it will be joined with root_path (if given),
+          # or expanded relative to PWD.
+          new_path = File.expand_path(path)
+          unless path == new_path
+            new_path = File.join(root_path, path) if root_path
+          end
+        end
+        instance_variable_set(:"@#{name}", new_path) if new_path
+      end
+
+      def reset!
+        @user      = ENV["USER"] || Etc.getpwuid.name
+        @root_path = File.join(File.expand_path(ENV["HOME"] || ""), "Backup")
+        update(root_path: @root_path)
+      end
+    end
+
+    reset! # set defaults on load
+  end
+end

data/lib/backup/database/base.rb

@@ -0,0 +1,86 @@
+module Backup
+  module Database
+    class Error < Backup::Error; end
+
+    class Base
+      include Utilities::Helpers
+      include Config::Helpers
+
+      attr_reader :model, :database_id, :dump_path
+
+      ##
+      # If given, +database_id+ will be appended to the #dump_filename.
+      # This is required if multiple Databases of the same class are added to
+      # the model.
+      def initialize(model, database_id = nil)
+        @model = model
+        @database_id = database_id.to_s.gsub(/\W/, "_") if database_id
+        @dump_path = File.join(Config.tmp_path, model.trigger, "databases")
+        load_defaults!
+      end
+
+      def perform!
+        log!(:started)
+        prepare!
+      end
+
+      private
+
+      def prepare!
+        FileUtils.mkdir_p(dump_path)
+      end
+
+      ##
+      # Sets the base filename for the final dump file to be saved in +dump_path+,
+      # based on the class name. e.g. databases/MySQL.sql
+      #
+      # +database_id+ will be appended if it is defined.
+      # e.g. databases/MySQL-database_id.sql
+      #
+      # If multiple Databases of the same class are defined and no +database_id+
+      # is defined, the user will be warned and one will be auto-generated.
+      #
+      # Model#initialize calls this method *after* all defined databases have
+      # been initialized so `backup check` can report these warnings.
+      def dump_filename
+        @dump_filename ||=
+          begin
+            unless database_id
+              if model.databases.select { |d| d.class == self.class }.count > 1
+                sleep 1
+                @database_id = Time.now.to_i.to_s[-5, 5]
+                Logger.warn Error.new(<<-EOS)
+                Database Identifier Missing
+                When multiple Databases are configured in a single Backup Model
+                that have the same class (MySQL, PostgreSQL, etc.), the optional
+                +database_id+ must be specified to uniquely identify each instance.
+                e.g. database MySQL, :database_id do |db|
+                This will result in an output file in your final backup package like:
+                databases/MySQL-database_id.sql
+
+                Backup has auto-generated an identifier (#{database_id}) for this
+                database dump and will now continue.
+                EOS
+              end
+            end
+
+            self.class.name.split("::").last + (database_id ? "-#{database_id}" : "")
+          end
+      end
+
+      def database_name
+        @database_name ||= self.class.to_s.sub("Backup::", "") +
+          (database_id ? " (#{database_id})" : "")
+      end
+
+      def log!(action)
+        msg =
+          case action
+          when :started then "Started..."
+          when :finished then "Finished!"
+          end
+        Logger.info "#{database_name} #{msg}"
+      end
+    end
+  end
+end

data/lib/backup/database/mongodb.rb

@@ -0,0 +1,186 @@
+module Backup
+  module Database
+    class MongoDB < Base
+      class Error < Backup::Error; end
+
+      ##
+      # Name of the database that needs to get dumped
+      attr_accessor :name
+
+      ##
+      # Credentials for the specified database
+      attr_accessor :username, :password, :authdb
+
+      ##
+      # Connectivity options
+      attr_accessor :host, :port
+
+      ##
+      # IPv6 support (disabled by default)
+      attr_accessor :ipv6
+
+      ##
+      # Collections to dump, collections that aren't specified won't get dumped
+      attr_accessor :only_collections
+
+      ##
+      # Additional "mongodump" options
+      attr_accessor :additional_options
+
+      ##
+      # Forces mongod to flush all pending write operations to the disk and
+      # locks the entire mongod instance to prevent additional writes until the
+      # dump is complete.
+      #
+      # Note that if Profiling is enabled, this will disable it and will not
+      # re-enable it after the dump is complete.
+      attr_accessor :lock
+
+      ##
+      # Creates a dump of the database that includes an oplog, to create a
+      # point-in-time snapshot of the state of a mongod instance.
+      #
+      # If this option is used, you would not use the `lock` option.
+      #
+      # This will only work against nodes that maintain a oplog.
+      # This includes all members of a replica set, as well as master nodes in
+      # master/slave replication deployments.
+      attr_accessor :oplog
+
+      def initialize(model, database_id = nil, &block)
+        super
+        instance_eval(&block) if block_given?
+      end
+
+      def perform!
+        super
+
+        lock_database if @lock
+        dump!
+        package!
+      ensure
+        unlock_database if @lock
+      end
+
+      private
+
+      ##
+      # Performs all required mongodump commands, dumping the output files
+      # into the +dump_packaging_path+ directory for packaging.
+      def dump!
+        FileUtils.mkdir_p dump_packaging_path
+
+        collections = Array(only_collections)
+        if collections.empty?
+          run(mongodump)
+        else
+          collections.each do |collection|
+            run("#{mongodump} --collection='#{collection}'")
+          end
+        end
+      end
+
+      ##
+      # Creates a tar archive of the +dump_packaging_path+ directory
+      # and stores it in the +dump_path+ using +dump_filename+.
+      #
+      #   <trigger>/databases/MongoDB[-<database_id>].tar[.gz]
+      #
+      # If successful, +dump_packaging_path+ is removed.
+      def package!
+        pipeline = Pipeline.new
+        dump_ext = "tar"
+
+        pipeline << "#{utility(:tar)} -cf - " \
+            "-C '#{dump_path}' '#{dump_filename}'"
+
+        if model.compressor
+          model.compressor.compress_with do |command, ext|
+            pipeline << command
+            dump_ext << ext
+          end
+        end
+
+        pipeline << "#{utility(:cat)} > " \
+          "'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
+
+        pipeline.run
+        if pipeline.success?
+          FileUtils.rm_rf dump_packaging_path
+          log!(:finished)
+        else
+          raise Error, "Dump Failed!\n#{pipeline.error_messages}"
+        end
+      end
+
+      def dump_packaging_path
+        File.join(dump_path, dump_filename)
+      end
+
+      def mongodump
+        "#{utility(:mongodump)} #{name_option} #{credential_options} " \
+          "#{connectivity_options} #{ipv6_option} #{oplog_option} " \
+          "#{user_options} --out='#{dump_packaging_path}'"
+      end
+
+      def name_option
+        return unless name
+        "--db='#{name}'"
+      end
+
+      def credential_options
+        opts = []
+        opts << "--username='#{username}'" if username
+        opts << "--password='#{password}'" if password
+        opts << "--authenticationDatabase='#{authdb}'" if authdb
+        opts.join(" ")
+      end
+
+      def connectivity_options
+        opts = []
+        opts << "--host='#{host}'" if host
+        opts << "--port='#{port}'" if port
+        opts.join(" ")
+      end
+
+      def ipv6_option
+        "--ipv6" if ipv6
+      end
+
+      def oplog_option
+        "--oplog" if oplog
+      end
+
+      def user_options
+        Array(additional_options).join(" ")
+      end
+
+      def lock_database
+        lock_command = <<-EOS.gsub(/^ +/, "")
+          echo 'use admin
+          db.setProfilingLevel(0)
+          db.fsyncLock()' | #{mongo_shell}
+        EOS
+
+        run(lock_command)
+      end
+
+      def unlock_database
+        unlock_command = <<-EOS.gsub(/^ +/, "")
+          echo 'use admin
+          db.fsyncUnlock()' | #{mongo_shell}
+        EOS
+
+        run(unlock_command)
+      end
+
+      def mongo_shell
+        cmd = "#{utility(:mongo)} #{connectivity_options}".rstrip
+        cmd << " #{credential_options}".rstrip
+        cmd << " #{ipv6_option}".rstrip
+        cmd << " '#{name}'" if name
+        cmd
+      end
+    end
+  end
+end