backedup 5.0.0.beta.3

data/lib/backup/cli.rb

@@ -0,0 +1,370 @@
+##
+# Build the Backup Command Line Interface using Thor
+module Backup
+  class CLI < Thor
+    class Error < Backup::Error; end
+    class FatalError < Backup::FatalError; end
+
+    ##
+    # [Perform]
+    #
+    # The only required option is the --trigger [-t].
+    # If --config-file, --data-path, --tmp-path or --log-path
+    # aren't specified they will fallback to defaults.
+    # If --root-path is given, it will be used as the base path for our defaults,
+    # as well as the base path for any option specified as a relative path.
+    # Any option given as an absolute path will be used "as-is".
+    #
+    # This command will exit with one of the following status codes:
+    #
+    #   0: All triggers were successful and no warnings were issued.
+    #   1: All triggers were successful, but some had warnings.
+    #   2: All triggers were processed, but some failed.
+    #   3: A fatal error caused Backup to exit.
+    #      Some triggers may not have been processed.
+    #
+    # If the --check option is given, `backup check` will be run
+    # and no triggers will be performed.
+    desc "perform", "Performs the backup for the specified trigger(s)."
+
+    long_desc <<-EOS.gsub(/^ +/, "")
+      Performs the backup for the specified trigger(s).
+
+      You may perform multiple backups by providing multiple triggers,
+      separated by commas. Each will run in the order specified.
+
+      $ backup perform --triggers backup1,backup2,backup3,backup4
+
+      --root-path may be an absolute path or relative to the current directory.
+
+      To use the current directory, use: `--root-path .`
+
+      Relative paths given for --config-file, --data-path, --tmp-path,
+      and --log-path will be relative to --root-path.
+
+      Console log output may be forced using --no-quiet.
+
+      Logging to file or syslog may be disabled using --no-logfile or --no-syslog
+      respectively. This will override logging options set in `config.rb`.
+    EOS
+
+    method_option :trigger,
+      aliases: ["-t", "--triggers"],
+      required: true,
+      type: :string,
+      desc: "Triggers to perform. e.g. 'trigger_a,trigger_b'"
+
+    method_option :config_file,
+      aliases: "-c",
+      type: :string,
+      default: "",
+      desc: "Path to your config.rb file."
+
+    method_option :root_path,
+      aliases: "-r",
+      type: :string,
+      default: "",
+      desc: "Root path to base all relative path on."
+
+    method_option :data_path,
+      aliases: "-d",
+      type: :string,
+      default: "",
+      desc: "Path to store storage cycling data."
+
+    method_option :log_path,
+      aliases: "-l",
+      type: :string,
+      default: "",
+      desc: "Path to store Backup's log file."
+
+    method_option :tmp_path,
+      type: :string,
+      default: "",
+      desc: "Path to store temporary data during the backup."
+
+    # Note that :quiet, :syslog and :logfile are specified as :string types,
+    # so the --no-<option> usage will set the value to nil instead of false.
+    method_option :quiet,
+      aliases: "-q",
+      type: :boolean,
+      default: false,
+      banner: "",
+      desc: "Disable console log output."
+
+    method_option :syslog,
+      type: :boolean,
+      default: false,
+      banner: "",
+      desc: "Enable logging to syslog."
+
+    method_option :logfile,
+      type: :boolean,
+      default: true,
+      banner: "",
+      desc: "Enable Backup's log file."
+
+    method_option :check,
+      type: :boolean,
+      default: false,
+      desc: "Check configuration for errors or warnings."
+
+    def perform
+      check if options[:check] # this will exit()
+
+      models = nil
+      begin
+        # Set logger options
+        opts = options
+        Logger.configure do
+          console.quiet    = opts[:quiet]
+          logfile.enabled  = opts[:logfile]
+          logfile.log_path = opts[:log_path]
+          syslog.enabled   = opts[:syslog]
+        end
+
+        # Load the user's +config.rb+ file and all their Models
+        Config.load(options)
+
+        # Identify all Models to be run for the given +triggers+.
+        triggers = options[:trigger].split(",").map(&:strip)
+        models = triggers.uniq.flat_map do |trigger|
+          Model.find_by_trigger(trigger)
+        end
+
+        if models.empty?
+          raise Error, "No Models found for trigger(s) " \
+            "'#{triggers.join(",")}'."
+        end
+
+        # Finalize Logger and begin real-time logging.
+        Logger.start!
+      rescue Exception => err
+        Logger.error Error.wrap(err)
+        unless Helpers.is_backup_error? err
+          Logger.error err.backtrace.join("\n")
+        end
+        # Logger configuration will be ignored
+        # and messages will be output to the console only.
+        Logger.abort!
+        exit 3
+      end
+
+      until models.empty?
+        model = models.shift
+        model.perform!
+
+        case model.exit_status
+        when 1
+          warnings = true
+        when 2
+          errors = true
+          unless models.empty?
+            Logger.info Error.new(<<-EOS)
+              Backup will now continue...
+              The following triggers will now be processed:
+              (#{models.map(&:trigger).join(", ")})
+            EOS
+          end
+        when 3
+          fatal = true
+          unless models.empty?
+            Logger.error FatalError.new(<<-EOS)
+              Backup will now exit.
+              The following triggers will not be processed:
+              (#{models.map(&:trigger).join(", ")})
+            EOS
+          end
+        end
+
+        model.notifiers.each(&:perform!)
+        exit(3) if fatal
+        Logger.clear!
+      end
+
+      exit(errors ? 2 : 1) if errors || warnings
+    end
+
+    ##
+    # [Check]
+    #
+    # Loads the user's `config.rb` (and all Model files) and reports any Errors
+    # or Warnings. This is primarily for checking for syntax errors, missing
+    # dependencies and deprecation warnings.
+    #
+    # This may also be invoked using the `--check` option to `backup perform`.
+    #
+    # This command only requires `Config.config_file` to be correct.
+    # All other Config paths are irrelevant.
+    #
+    # All output will be sent to the console only.
+    # Logger options will be ignored.
+    #
+    # If successful, this method with exit(0).
+    # If there are Errors or Warnings, it will exit(1).
+    desc "check", "Check for configuration errors or warnings"
+
+    long_desc <<-EOS.gsub(/^ +/, "")
+      Loads your 'config.rb' file and all models and reports any
+      errors or warnings with your configuration, including missing
+      dependencies and the use of any deprecated settings.
+    EOS
+
+    method_option :config_file,
+      aliases: "-c",
+      type: :string,
+      default: "",
+      desc: "Path to your config.rb file."
+
+    def check
+      begin
+        Config.load(options)
+      rescue Exception => err
+        Logger.error Error.wrap(err)
+        unless Helpers.is_backup_error? err
+          Logger.error err.backtrace.join("\n")
+        end
+      end
+
+      if Logger.has_warnings? || Logger.has_errors?
+        Logger.error "Configuration Check Failed."
+        exit_code = 1
+      else
+        Logger.info "Configuration Check Succeeded."
+        exit_code = 0
+      end
+
+      Logger.abort!
+      exit(exit_code)
+    end
+
+    ##
+    # [Generate:Model]
+    # Generates a model configuration file based on the arguments passed in.
+    # For example:
+    #   $ backup generate:model --trigger my_backup --databases='mongodb'
+    # will generate a pre-populated model with a base MongoDB setup
+    desc "generate:model", "Generates a Backup model file."
+
+    long_desc <<-EOS.gsub(/^ +/, "")
+      Generates a Backup model file.
+
+      If your configuration file is not in the default location at
+      #{Config.config_file}
+      you must specify it's location using '--config-file'.
+      If no configuration file exists at this location, one will be created.
+
+      The model file will be created as '<config_path>/models/<trigger>.rb'
+      Your model file will be created in a 'models/' sub-directory
+      where your config file is located. The default location would be:
+      #{Config.root_path}/models/<trigger>.rb
+    EOS
+
+    method_option :trigger,
+      aliases: "-t",
+      required: true,
+      type: :string,
+      desc: "Trigger name for the Backup model"
+
+    method_option :config_file,
+      type: :string,
+      desc: "Path to your Backup configuration file"
+
+    # options with their available values
+    %w[databases storages syncers encryptor compressor notifiers].each do |name|
+      path = File.join(Backup::TEMPLATE_PATH, "cli", name)
+      opts = Dir[path + "/*"].sort.map { |p| File.basename(p) }.join(", ")
+      method_option name, type: :string, desc: "(#{opts})"
+    end
+
+    method_option :archives,
+      type: :boolean,
+      desc: "Model will include tar archives."
+
+    method_option :splitter,
+      type: :boolean,
+      default: false,
+      desc: "Add Splitter to the model"
+
+    define_method "generate:model" do
+      opts = options.merge(trigger: options[:trigger].gsub(/\W/, "_"))
+      config_file = opts[:config_file] ?
+                    File.expand_path(opts.delete(:config_file)) : Config.config_file
+      models_path = File.join(File.dirname(config_file), "models")
+      model_file  = File.join(models_path, "#{opts[:trigger]}.rb")
+
+      unless File.exist?(config_file)
+        invoke "generate:config", [], config_file: config_file
+      end
+
+      FileUtils.mkdir_p(models_path)
+      if Helpers.overwrite?(model_file)
+        File.open(model_file, "w") do |file|
+          file.write(Backup::Template.new(options: opts).result("cli/model"))
+        end
+        puts "Generated model file: '#{model_file}'."
+      end
+    end
+
+    ##
+    # [Generate:Config]
+    # Generates the main configuration file
+    desc "generate:config", "Generates the main Backup configuration file"
+
+    long_desc <<-EOS.gsub(/^ +/, "")
+      Path to the Backup configuration file to generate.
+
+      Defaults to:
+
+      #{Config.config_file}
+    EOS
+
+    method_option :config_file,
+      type: :string,
+      desc: "Path to the Backup configuration file to generate."
+
+    define_method "generate:config" do
+      config_file = options[:config_file] ?
+          File.expand_path(options[:config_file]) : Config.config_file
+
+      FileUtils.mkdir_p(File.dirname(config_file))
+      if Helpers.overwrite?(config_file)
+        File.open(config_file, "w") do |file|
+          file.write(Backup::Template.new.result("cli/config"))
+        end
+        puts "Generated configuration file: '#{config_file}'."
+      end
+    end
+
+    ##
+    # [Version]
+    # Returns the current version of the Backup gem
+    map "-v" => :version
+    desc "version", "Display installed Backup version"
+    def version
+      puts "Backup #{Backup::VERSION}"
+    end
+
+    # This is to avoid Thor's warnings when stubbing methods on the Thor class.
+    module Helpers
+      class << self
+        def overwrite?(path)
+          return true unless File.exist?(path)
+
+          $stderr.print "A file already exists at '#{path}'.\n" \
+                        "Do you want to overwrite? [y/n] "
+          /^[Yy]/ =~ $stdin.gets
+        end
+
+        def exec!(cmd)
+          puts "Launching: #{cmd}"
+          exec(cmd)
+        end
+
+        def is_backup_error?(error)
+          error.class.ancestors.include? Backup::Error
+        end
+      end
+    end
+  end
+end

data/lib/backup/cloud_io/base.rb

@@ -0,0 +1,38 @@
+module Backup
+  module CloudIO
+    class Error < Backup::Error; end
+    class FileSizeError < Backup::Error; end
+
+    class Base
+      attr_reader :max_retries, :retry_waitsec
+
+      def initialize(options = {})
+        @max_retries    = options[:max_retries]
+        @retry_waitsec  = options[:retry_waitsec]
+      end
+
+      private
+
+      def with_retries(operation)
+        retries = 0
+        begin
+          yield
+        rescue => err
+          retries += 1
+          raise Error.wrap(err, <<-EOS) if retries > max_retries
+            Max Retries (#{max_retries}) Exceeded!
+            Operation: #{operation}
+            Be sure to check the log messages for each retry attempt.
+          EOS
+
+          Logger.info Error.wrap(err, <<-EOS)
+            Retry ##{retries} of #{max_retries}
+            Operation: #{operation}
+          EOS
+          sleep(retry_waitsec)
+          retry
+        end
+      end
+    end
+  end
+end

data/lib/backup/cloud_io/cloud_files.rb

@@ -0,0 +1,296 @@
+require "backup/cloud_io/base"
+require "fog"
+require "digest/md5"
+
+module Backup
+  module CloudIO
+    class CloudFiles < Base
+      class Error < Backup::Error; end
+
+      MAX_FILE_SIZE   = 1024**3 * 5     # 5 GiB
+      MAX_SLO_SIZE    = 1024**3 * 5000  # 1000 segments @ 5 GiB
+      SEGMENT_BUFFER  = 1024**2         # 1 MiB
+
+      attr_reader :username, :api_key, :auth_url, :region, :servicenet,
+        :container, :segments_container, :segment_size, :days_to_keep,
+        :fog_options
+
+      def initialize(options = {})
+        super
+
+        @username           = options[:username]
+        @api_key            = options[:api_key]
+        @auth_url           = options[:auth_url]
+        @region             = options[:region]
+        @servicenet         = options[:servicenet]
+        @container          = options[:container]
+        @segments_container = options[:segments_container]
+        @segment_size       = options[:segment_size]
+        @days_to_keep       = options[:days_to_keep]
+        @fog_options        = options[:fog_options]
+      end
+
+      # The Syncer may call this method in multiple threads,
+      # but #objects is always called before this occurs.
+      def upload(src, dest)
+        create_containers
+
+        file_size = File.size(src)
+        segment_bytes = segment_size * 1024**2
+        if segment_bytes > 0 && file_size > segment_bytes
+          raise FileSizeError, <<-EOS if file_size > MAX_SLO_SIZE
+            File Too Large
+            File: #{src}
+            Size: #{file_size}
+            Max SLO Size is #{MAX_SLO_SIZE} (5 GiB * 1000 segments)
+          EOS
+
+          segment_bytes = adjusted_segment_bytes(segment_bytes, file_size)
+          segments = upload_segments(src, dest, segment_bytes, file_size)
+          upload_manifest(dest, segments)
+        else
+          raise FileSizeError, <<-EOS if file_size > MAX_FILE_SIZE
+            File Too Large
+            File: #{src}
+            Size: #{file_size}
+            Max File Size is #{MAX_FILE_SIZE} (5 GiB)
+          EOS
+
+          put_object(src, dest)
+        end
+      end
+
+      # Returns all objects in the container with the given prefix.
+      #
+      # - #get_container returns a max of 10000 objects per request.
+      # - Returns objects sorted using a sqlite binary collating function.
+      # - If marker is given, only objects after the marker are in the response.
+      def objects(prefix)
+        objects = []
+        resp = nil
+        prefix = prefix.chomp("/")
+        opts = { prefix: prefix + "/" }
+
+        create_containers
+
+        while resp.nil? || resp.body.count == 10_000
+          opts[:marker] = objects.last.name unless objects.empty?
+          with_retries("GET '#{container}/#{prefix}/*'") do
+            resp = connection.get_container(container, opts)
+          end
+          resp.body.each do |obj_data|
+            objects << Object.new(self, obj_data)
+          end
+        end
+
+        objects
+      end
+
+      # Used by Object to fetch metadata if needed.
+      def head_object(object)
+        resp = nil
+        with_retries("HEAD '#{container}/#{object.name}'") do
+          resp = connection.head_object(container, object.name)
+        end
+        resp
+      end
+
+      # Delete non-SLO object(s) from the container.
+      #
+      # - Called by the Storage (with objects) and the Syncer (with names)
+      # - Deletes 10,000 objects per request.
+      # - Missing objects will be ignored.
+      def delete(objects_or_names)
+        names = Array(objects_or_names).dup
+        names.map!(&:name) if names.first.is_a?(Object)
+
+        until names.empty?
+          names_partial = names.slice!(0, 10_000)
+          with_retries("DELETE Multiple Objects") do
+            resp = connection.delete_multiple_objects(container, names_partial)
+            resp_status = resp.body["Response Status"]
+            raise Error, <<-EOS unless resp_status == "200 OK"
+              #{resp_status}
+              The server returned the following:
+              #{resp.body.inspect}
+            EOS
+          end
+        end
+      end
+
+      # Delete an SLO object(s) from the container.
+      #
+      # - Used only by the Storage. The Syncer cannot use SLOs.
+      # - Removes the SLO manifest object and all associated segments.
+      # - Missing segments will be ignored.
+      def delete_slo(objects)
+        Array(objects).each do |object|
+          with_retries("DELETE SLO Manifest '#{container}/#{object.name}'") do
+            resp = connection.delete_static_large_object(container, object.name)
+            resp_status = resp.body["Response Status"]
+            raise Error, <<-EOS unless resp_status == "200 OK"
+              #{resp_status}
+              The server returned the following:
+              #{resp.body.inspect}
+            EOS
+          end
+        end
+      end
+
+      private
+
+      def connection
+        @connection ||= Fog::Storage.new({
+          provider: "Rackspace",
+          rackspace_username: username,
+          rackspace_api_key: api_key,
+          rackspace_auth_url: auth_url,
+          rackspace_region: region,
+          rackspace_servicenet: servicenet
+        }.merge(fog_options || {}))
+      end
+
+      def create_containers
+        return if @containers_created
+        @containers_created = true
+
+        with_retries("Create Containers") do
+          connection.put_container(container)
+          connection.put_container(segments_container) if segments_container
+        end
+      end
+
+      def put_object(src, dest)
+        opts = headers.merge("ETag" => Digest::MD5.file(src).hexdigest)
+        with_retries("PUT '#{container}/#{dest}'") do
+          File.open(src, "r") do |file|
+            connection.put_object(container, dest, file, opts)
+          end
+        end
+      end
+
+      # Each segment is uploaded using chunked transfer encoding using
+      # SEGMENT_BUFFER, and each segment's MD5 is sent to verify the transfer.
+      # Each segment's MD5 and byte_size will also be verified when the
+      # SLO manifest object is uploaded.
+      def upload_segments(src, dest, segment_bytes, file_size)
+        total_segments = (file_size / segment_bytes.to_f).ceil
+        progress = (0.1..0.9).step(0.1).map { |n| (total_segments * n).floor }
+        Logger.info "\s\sUploading #{total_segments} SLO Segments..."
+
+        segments = []
+        File.open(src, "r") do |file|
+          segment_number = 0
+          until file.eof?
+            segment_number += 1
+            object = "#{dest}/#{segment_number.to_s.rjust(4, "0")}"
+            pos = file.pos
+            md5 = segment_md5(file, segment_bytes)
+            opts = headers.merge("ETag" => md5)
+
+            with_retries("PUT '#{segments_container}/#{object}'") do
+              file.seek(pos)
+              offset = 0
+              connection.put_object(segments_container, object, nil, opts) do
+                # block is called to stream data until it returns ''
+                data = ""
+                if offset <= segment_bytes - SEGMENT_BUFFER
+                  data = file.read(SEGMENT_BUFFER).to_s # nil => ''
+                  offset += data.size
+                end
+                data
+              end
+            end
+
+            segments << {
+              path: "#{segments_container}/#{object}",
+              etag: md5,
+              size_bytes: file.pos - pos
+            }
+
+            if i = progress.rindex(segment_number)
+              Logger.info "\s\s...#{i + 1}0% Complete..."
+            end
+          end
+        end
+        segments
+      end
+
+      def segment_md5(file, segment_bytes)
+        md5 = Digest::MD5.new
+        offset = 0
+        while offset <= segment_bytes - SEGMENT_BUFFER
+          data = file.read(SEGMENT_BUFFER)
+          break unless data
+          offset += data.size
+          md5 << data
+        end
+        md5.hexdigest
+      end
+
+      # Each segment's ETag and byte_size will be verified once uploaded.
+      # Request will raise an exception if verification fails or segments
+      # are not found. However, each segment's ETag was verified when we
+      # uploaded the segments, so this should only retry failed requests.
+      def upload_manifest(dest, segments)
+        Logger.info "\s\sStoring SLO Manifest '#{container}/#{dest}'"
+
+        with_retries("PUT SLO Manifest '#{container}/#{dest}'") do
+          connection.put_static_obj_manifest(container, dest, segments, headers)
+        end
+      end
+
+      # If :days_to_keep was set, each object will be scheduled for deletion.
+      # This includes non-SLO objects, the SLO manifest and all segments.
+      def headers
+        headers = {}
+        headers["X-Delete-At"] = delete_at if delete_at
+        headers
+      end
+
+      def delete_at
+        return unless days_to_keep
+        @delete_at ||= (Time.now.utc + days_to_keep * 60**2 * 24).to_i
+      end
+
+      def adjusted_segment_bytes(segment_bytes, file_size)
+        return segment_bytes if file_size / segment_bytes.to_f <= 1000
+
+        mb = orig_mb = segment_bytes / 1024**2
+        mb += 1 until file_size / (1024**2 * mb).to_f <= 1000
+        Logger.warn Error.new(<<-EOS)
+          Segment Size Adjusted
+          Your original #segment_size of #{orig_mb} MiB has been adjusted
+          to #{mb} MiB in order to satisfy the limit of 1000 segments.
+          To enforce your chosen #segment_size, you should use the Splitter.
+          e.g. split_into_chunks_of #{mb * 1000} (#segment_size * 1000)
+        EOS
+        1024**2 * mb
+      end
+
+      class Object
+        attr_reader :name, :hash
+
+        def initialize(cloud_io, data)
+          @cloud_io = cloud_io
+          @name = data["name"]
+          @hash = data["hash"]
+        end
+
+        def slo?
+          !!metadata["X-Static-Large-Object"]
+        end
+
+        def marked_for_deletion?
+          !!metadata["X-Delete-At"]
+        end
+
+        private
+
+        def metadata
+          @metadata ||= @cloud_io.head_object(self).headers
+        end
+      end
+    end
+  end
+end