nfm-backup 4.0.1a

data/lib/backup/cli.rb

@@ -0,0 +1,364 @@
+# encoding: utf-8
+
+##
+# Build the Backup Command Line Interface using Thor
+module Backup
+  class CLI < Thor
+    class Error < Backup::Error; end
+    class FatalError < Backup::FatalError; end
+
+    ##
+    # [Perform]
+    #
+    # The only required option is the --trigger [-t].
+    # If --config-file, --data-path, --tmp-path or --log-path
+    # aren't specified they will fallback to defaults.
+    # If --root-path is given, it will be used as the base path for our defaults,
+    # as well as the base path for any option specified as a relative path.
+    # Any option given as an absolute path will be used "as-is".
+    #
+    # This command will exit with one of the following status codes:
+    #
+    #   0: All triggers were successful and no warnings were issued.
+    #   1: All triggers were successful, but some had warnings.
+    #   2: All triggers were processed, but some failed.
+    #   3: A fatal error caused Backup to exit.
+    #      Some triggers may not have been processed.
+    #
+    # If the --check option is given, `backup check` will be run
+    # and no triggers will be performed.
+    desc 'perform', "Performs the backup for the specified trigger(s)."
+
+    long_desc <<-EOS.gsub(/^ +/, '')
+      Performs the backup for the specified trigger(s).
+
+      You may perform multiple backups by providing multiple triggers,
+      separated by commas. Each will run in the order specified.
+
+      $ backup perform --triggers backup1,backup2,backup3,backup4
+
+      --root-path may be an absolute path or relative to the current directory.
+
+      To use the current directory, use: `--root-path .`
+
+      Relative paths given for --config-file, --data-path, --tmp-path,
+      and --log-path will be relative to --root-path.
+
+      Console log output may be forced using --no-quiet.
+
+      Logging to file or syslog may be disabled using --no-logfile or --no-syslog
+      respectively. This will override logging options set in `config.rb`.
+    EOS
+
+    method_option :trigger,
+                  :aliases  => ['-t', '--triggers'],
+                  :required => true,
+                  :type     => :string,
+                  :desc     => "Triggers to perform. e.g. 'trigger_a,trigger_b'"
+
+    method_option :config_file,
+                  :aliases  => '-c',
+                  :type     => :string,
+                  :default  => '',
+                  :desc     => 'Path to your config.rb file.'
+
+    method_option :root_path,
+                  :aliases  => '-r',
+                  :type     => :string,
+                  :default  => '',
+                  :desc     => 'Root path to base all relative path on.'
+
+    method_option :data_path,
+                  :aliases  => '-d',
+                  :type     => :string,
+                  :default  => '',
+                  :desc     => 'Path to store storage cycling data.'
+
+    method_option :log_path,
+                  :aliases  => '-l',
+                  :type     => :string,
+                  :default  => '',
+                  :desc     => "Path to store Backup's log file."
+
+    method_option :tmp_path,
+                  :type     => :string,
+                  :default  => '',
+                  :desc     => 'Path to store temporary data during the backup.'
+
+    # Note that :quiet, :syslog and :logfile are specified as :string types,
+    # so the --no-<option> usage will set the value to nil instead of false.
+    method_option :quiet,
+                  :aliases  => '-q',
+                  :type     => :string,
+                  :default  => false,
+                  :banner   => '',
+                  :desc     => 'Disable console log output.'
+
+    method_option :syslog,
+                  :type     => :string,
+                  :default  => false,
+                  :banner   => '',
+                  :desc     => 'Enable logging to syslog.'
+
+    method_option :logfile,
+                  :type     => :string,
+                  :default  => true,
+                  :banner   => '',
+                  :desc     => "Enable Backup's log file."
+
+    method_option :check,
+                  :type     => :boolean,
+                  :default  => false,
+                  :desc     => 'Check configuration for errors or warnings.'
+
+    def perform
+      check if options[:check] # this will exit()
+
+      models = nil
+      begin
+        # Set logger options
+        opts = options
+        Logger.configure do
+          console.quiet     = opts[:quiet]
+          logfile.enabled   = opts[:logfile]
+          logfile.log_path  = opts[:log_path]
+          syslog.enabled    = opts[:syslog]
+        end
+
+        # Load the user's +config.rb+ file and all their Models
+        Config.load(options)
+
+        # Identify all Models to be run for the given +triggers+.
+        triggers = options[:trigger].split(',').map(&:strip)
+        models = triggers.map {|trigger|
+          Model.find_by_trigger(trigger)
+        }.flatten.uniq
+
+        raise Error, "No Models found for trigger(s) " +
+            "'#{ triggers.join(',') }'." if models.empty?
+
+        # Finalize Logger and begin real-time logging.
+        Logger.start!
+
+      rescue Exception => err
+        Logger.error Error.wrap(err)
+        # Logger configuration will be ignored
+        # and messages will be output to the console only.
+        Logger.abort!
+        exit(3)
+      end
+
+      until models.empty?
+        model = models.shift
+        model.perform!
+
+        case model.exit_status
+        when 1
+          warnings = true
+        when 2
+          errors = true
+          unless models.empty?
+            Logger.info Error.new(<<-EOS)
+              Backup will now continue...
+              The following triggers will now be processed:
+              (#{ models.map {|m| m.trigger }.join(', ') })
+            EOS
+          end
+        when 3
+          fatal = true
+          unless models.empty?
+            Logger.error FatalError.new(<<-EOS)
+              Backup will now exit.
+              The following triggers will not be processed:
+              (#{ models.map {|m| m.trigger }.join(', ') })
+            EOS
+          end
+        end
+
+        model.notifiers.each(&:perform!)
+        exit(3) if fatal
+        Logger.clear!
+      end
+
+      exit(errors ? 2 : 1) if errors || warnings
+    end
+
+    ##
+    # [Check]
+    #
+    # Loads the user's `config.rb` (and all Model files) and reports any Errors
+    # or Warnings. This is primarily for checking for syntax errors, missing
+    # dependencies and deprecation warnings.
+    #
+    # This may also be invoked using the `--check` option to `backup perform`.
+    #
+    # This command only requires `Config.config_file` to be correct.
+    # All other Config paths are irrelevant.
+    #
+    # All output will be sent to the console only.
+    # Logger options will be ignored.
+    #
+    # If successful, this method with exit(0).
+    # If there are Errors or Warnings, it will exit(1).
+    desc 'check', 'Check for configuration errors or warnings'
+
+    long_desc <<-EOS.gsub(/^ +/, '')
+      Loads your 'config.rb' file and all models and reports any
+      errors or warnings with your configuration, including missing
+      dependencies and the use of any deprecated settings.
+    EOS
+
+    method_option :config_file,
+                  :aliases  => '-c',
+                  :type     => :string,
+                  :default  => '',
+                  :desc     => "Path to your config.rb file."
+
+    def check
+      begin
+        Config.load(options)
+      rescue Exception => err
+        Logger.error Error.wrap(err)
+      end
+
+      if Logger.has_warnings? || Logger.has_errors?
+        Logger.error 'Configuration Check Failed.'
+        exit_code = 1
+      else
+        Logger.info 'Configuration Check Succeeded.'
+        exit_code = 0
+      end
+
+      Logger.abort!
+      exit(exit_code)
+    end
+
+    ##
+    # [Generate:Model]
+    # Generates a model configuration file based on the arguments passed in.
+    # For example:
+    #   $ backup generate:model --trigger my_backup --databases='mongodb'
+    # will generate a pre-populated model with a base MongoDB setup
+    desc 'generate:model', "Generates a Backup model file."
+
+    long_desc <<-EOS.gsub(/^ +/, '')
+      Generates a Backup model file.
+
+      If your configuration file is not in the default location at
+      #{ Config.config_file }
+      you must specify it's location using '--config-file'.
+      If no configuration file exists at this location, one will be created.
+
+      The model file will be created as '<config_path>/models/<trigger>.rb'
+      Your model file will be created in a 'models/' sub-directory
+      where your config file is located. The default location would be:
+      #{ Config.root_path }/models/<trigger>.rb
+    EOS
+
+    method_option :trigger,
+                  :aliases  => '-t',
+                  :required => true,
+                  :type     => :string,
+                  :desc     => 'Trigger name for the Backup model'
+
+    method_option :config_file,
+                  :type     => :string,
+                  :desc     => 'Path to your Backup configuration file'
+
+    # options with their available values
+    %w{ databases storages syncers encryptor compressor notifiers }.each do |name|
+      path = File.join(Backup::TEMPLATE_PATH, 'cli', name)
+      opts = Dir[path + '/*'].sort.map {|p| File.basename(p) }.join(', ')
+      method_option name, :type => :string, :desc => "(#{ opts })"
+    end
+
+    method_option :archives,
+                  :type     => :boolean,
+                  :desc     => 'Model will include tar archives.'
+
+    method_option :splitter,
+                  :type     => :boolean,
+                  :default  => false,
+                  :desc     => 'Add Splitter to the model'
+
+    define_method 'generate:model' do
+      opts = options.merge(:trigger => options[:trigger].gsub(/\W/, '_'))
+      config_file = opts[:config_file] ?
+                    File.expand_path(opts.delete(:config_file)) : Config.config_file
+      models_path = File.join(File.dirname(config_file), 'models')
+      model_file  = File.join(models_path, "#{ opts[:trigger] }.rb")
+
+      unless File.exist?(config_file)
+        invoke 'generate:config', [], :config_file => config_file
+      end
+
+      FileUtils.mkdir_p(models_path)
+      if Helpers.overwrite?(model_file)
+        File.open(model_file, 'w') do |file|
+          file.write(Backup::Template.new({:options => opts}).result('cli/model'))
+        end
+        puts "Generated model file: '#{ model_file }'."
+      end
+    end
+
+    ##
+    # [Generate:Config]
+    # Generates the main configuration file
+    desc 'generate:config', 'Generates the main Backup configuration file'
+
+    long_desc <<-EOS.gsub(/^ +/, '')
+      Path to the Backup configuration file to generate.
+
+      Defaults to:
+
+      #{ Config.config_file }
+    EOS
+
+    method_option :config_file,
+                  :type => :string,
+                  :desc => 'Path to the Backup configuration file to generate.'
+
+    define_method 'generate:config' do
+      config_file = options[:config_file] ?
+          File.expand_path(options[:config_file]) : Config.config_file
+
+      FileUtils.mkdir_p(File.dirname(config_file))
+      if Helpers.overwrite?(config_file)
+        File.open(config_file, 'w') do |file|
+          file.write(Backup::Template.new.result('cli/config'))
+        end
+        puts "Generated configuration file: '#{ config_file }'."
+      end
+    end
+
+    ##
+    # [Version]
+    # Returns the current version of the Backup gem
+    map '-v' => :version
+    desc 'version', 'Display installed Backup version'
+    def version
+      puts "Backup #{ Backup::VERSION }"
+    end
+
+    # This is to avoid Thor's warnings when stubbing methods on the Thor class.
+    module Helpers
+      class << self
+
+        def overwrite?(path)
+          return true unless File.exist?(path)
+
+          $stderr.print "A file already exists at '#{ path }'.\n" +
+                        "Do you want to overwrite? [y/n] "
+          /^[Yy]/ =~ $stdin.gets
+        end
+
+        def exec!(cmd)
+          puts "Launching: #{ cmd }"
+          exec(cmd)
+        end
+
+      end
+    end
+
+  end
+end

data/lib/backup/cloud_io/base.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+
+module Backup
+  module CloudIO
+    class Error < Backup::Error; end
+    class FileSizeError < Backup::Error; end
+
+    class Base
+      attr_reader :max_retries, :retry_waitsec
+
+      def initialize(options = {})
+        @max_retries    = options[:max_retries]
+        @retry_waitsec  = options[:retry_waitsec]
+      end
+
+      private
+
+      def with_retries(operation)
+        retries = 0
+        begin
+          yield
+        rescue => err
+          retries += 1
+          raise Error.wrap(err, <<-EOS) if retries > max_retries
+            Max Retries (#{ max_retries }) Exceeded!
+            Operation: #{ operation }
+            Be sure to check the log messages for each retry attempt.
+          EOS
+
+          Logger.info Error.wrap(err, <<-EOS)
+            Retry ##{ retries } of #{ max_retries }
+            Operation: #{ operation }
+          EOS
+          sleep(retry_waitsec)
+          retry
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/cloud_io/cloud_files.rb

@@ -0,0 +1,298 @@
+# encoding: utf-8
+require 'backup/cloud_io/base'
+require 'fog'
+require 'digest/md5'
+
+module Backup
+  module CloudIO
+    class CloudFiles < Base
+      class Error < Backup::Error; end
+
+      MAX_FILE_SIZE   = 1024**3 * 5     # 5 GiB
+      MAX_SLO_SIZE    = 1024**3 * 5000  # 1000 segments @ 5 GiB
+      SEGMENT_BUFFER  = 1024**2         # 1 MiB
+
+      attr_reader :username, :api_key, :auth_url, :region, :servicenet,
+                  :container, :segments_container, :segment_size, :days_to_keep,
+                  :fog_options
+
+      def initialize(options = {})
+        super
+
+        @username           = options[:username]
+        @api_key            = options[:api_key]
+        @auth_url           = options[:auth_url]
+        @region             = options[:region]
+        @servicenet         = options[:servicenet]
+        @container          = options[:container]
+        @segments_container = options[:segments_container]
+        @segment_size       = options[:segment_size]
+        @days_to_keep       = options[:days_to_keep]
+        @fog_options        = options[:fog_options]
+      end
+
+      # The Syncer may call this method in multiple threads,
+      # but #objects is always called before this occurs.
+      def upload(src, dest)
+        create_containers
+
+        file_size = File.size(src)
+        segment_bytes = segment_size * 1024**2
+        if segment_bytes > 0 && file_size > segment_bytes
+          raise FileSizeError, <<-EOS if file_size > MAX_SLO_SIZE
+            File Too Large
+            File: #{ src }
+            Size: #{ file_size }
+            Max SLO Size is #{ MAX_SLO_SIZE } (5 GiB * 1000 segments)
+          EOS
+
+          segment_bytes = adjusted_segment_bytes(segment_bytes, file_size)
+          segments = upload_segments(src, dest, segment_bytes, file_size)
+          upload_manifest(dest, segments)
+        else
+          raise FileSizeError, <<-EOS if file_size > MAX_FILE_SIZE
+            File Too Large
+            File: #{ src }
+            Size: #{ file_size }
+            Max File Size is #{ MAX_FILE_SIZE } (5 GiB)
+          EOS
+
+          put_object(src, dest)
+        end
+      end
+
+      # Returns all objects in the container with the given prefix.
+      #
+      # - #get_container returns a max of 10000 objects per request.
+      # - Returns objects sorted using a sqlite binary collating function.
+      # - If marker is given, only objects after the marker are in the response.
+      def objects(prefix)
+        objects = []
+        resp = nil
+        prefix = prefix.chomp('/')
+        opts = { :prefix => prefix + '/' }
+
+        create_containers
+
+        while resp.nil? || resp.body.count == 10000
+          opts.merge!(:marker => objects.last.name) unless objects.empty?
+          with_retries("GET '#{ container }/#{ prefix }/*'") do
+            resp = connection.get_container(container, opts)
+          end
+          resp.body.each do |obj_data|
+            objects << Object.new(self, obj_data)
+          end
+        end
+
+        objects
+      end
+
+      # Used by Object to fetch metadata if needed.
+      def head_object(object)
+        resp = nil
+        with_retries("HEAD '#{ container }/#{ object.name }'") do
+          resp = connection.head_object(container, object.name)
+        end
+        resp
+      end
+
+      # Delete non-SLO object(s) from the container.
+      #
+      # - Called by the Storage (with objects) and the Syncer (with names)
+      # - Deletes 10,000 objects per request.
+      # - Missing objects will be ignored.
+      def delete(objects_or_names)
+        names = Array(objects_or_names).dup
+        names.map!(&:name) if names.first.is_a?(Object)
+
+        until names.empty?
+          _names = names.slice!(0, 10000)
+          with_retries('DELETE Multiple Objects') do
+            resp = connection.delete_multiple_objects(container, _names)
+            resp_status = resp.body['Response Status']
+            raise Error, <<-EOS unless resp_status == '200 OK'
+              #{ resp_status }
+              The server returned the following:
+              #{ resp.body.inspect }
+            EOS
+          end
+        end
+      end
+
+      # Delete an SLO object(s) from the container.
+      #
+      # - Used only by the Storage. The Syncer cannot use SLOs.
+      # - Removes the SLO manifest object and all associated segments.
+      # - Missing segments will be ignored.
+      def delete_slo(objects)
+        Array(objects).each do |object|
+          with_retries("DELETE SLO Manifest '#{ container }/#{ object.name }'") do
+            resp = connection.delete_static_large_object(container, object.name)
+            resp_status = resp.body['Response Status']
+            raise Error, <<-EOS unless resp_status == '200 OK'
+              #{ resp_status }
+              The server returned the following:
+              #{ resp.body.inspect }
+            EOS
+          end
+        end
+      end
+
+      private
+
+      def connection
+        @connection ||= Fog::Storage.new({
+          :provider             => 'Rackspace',
+          :rackspace_username   => username,
+          :rackspace_api_key    => api_key,
+          :rackspace_auth_url   => auth_url,
+          :rackspace_region     => region,
+          :rackspace_servicenet => servicenet
+        }.merge(fog_options || {}))
+      end
+
+      def create_containers
+        return if @containers_created
+        @containers_created = true
+
+        with_retries('Create Containers') do
+          connection.put_container(container)
+          connection.put_container(segments_container) if segments_container
+        end
+      end
+
+      def put_object(src, dest)
+        opts = headers.merge('ETag' => Digest::MD5.file(src).hexdigest)
+        with_retries("PUT '#{ container }/#{ dest }'") do
+          File.open(src, 'r') do |file|
+            connection.put_object(container, dest, file, opts)
+          end
+        end
+      end
+
+      # Each segment is uploaded using chunked transfer encoding using
+      # SEGMENT_BUFFER, and each segment's MD5 is sent to verify the transfer.
+      # Each segment's MD5 and byte_size will also be verified when the
+      # SLO manifest object is uploaded.
+      def upload_segments(src, dest, segment_bytes, file_size)
+        total_segments = (file_size / segment_bytes.to_f).ceil
+        progress = (0.1..0.9).step(0.1).map {|n| (total_segments * n).floor }
+        Logger.info "\s\sUploading #{ total_segments } SLO Segments..."
+
+        segments = []
+        File.open(src, 'r') do |file|
+          segment_number = 0
+          until file.eof?
+            segment_number += 1
+            object = "#{ dest }/#{ segment_number.to_s.rjust(4, '0') }"
+            pos = file.pos
+            md5 = segment_md5(file, segment_bytes)
+            opts = headers.merge('ETag' => md5)
+
+            with_retries("PUT '#{ segments_container }/#{ object }'") do
+              file.seek(pos)
+              offset = 0
+              connection.put_object(segments_container, object, nil, opts) do
+                # block is called to stream data until it returns ''
+                data = ''
+                if offset <= segment_bytes - SEGMENT_BUFFER
+                  data = file.read(SEGMENT_BUFFER).to_s # nil => ''
+                  offset += data.size
+                end
+                data
+              end
+            end
+
+            segments << {
+              :path => "#{ segments_container }/#{ object }",
+              :etag => md5,
+              :size_bytes => file.pos - pos
+            }
+
+            if i = progress.rindex(segment_number)
+              Logger.info "\s\s...#{ i + 1 }0% Complete..."
+            end
+          end
+        end
+        segments
+      end
+
+      def segment_md5(file, segment_bytes)
+        md5 = Digest::MD5.new
+        offset = 0
+        while offset <= segment_bytes - SEGMENT_BUFFER
+          data = file.read(SEGMENT_BUFFER)
+          break unless data
+          offset += data.size
+          md5 << data
+        end
+        md5.hexdigest
+      end
+
+      # Each segment's ETag and byte_size will be verified once uploaded.
+      # Request will raise an exception if verification fails or segments
+      # are not found. However, each segment's ETag was verified when we
+      # uploaded the segments, so this should only retry failed requests.
+      def upload_manifest(dest, segments)
+        Logger.info "\s\sStoring SLO Manifest '#{ container }/#{ dest }'"
+
+        with_retries("PUT SLO Manifest '#{ container }/#{ dest }'") do
+          connection.put_static_obj_manifest(container, dest, segments, headers)
+        end
+      end
+
+      # If :days_to_keep was set, each object will be scheduled for deletion.
+      # This includes non-SLO objects, the SLO manifest and all segments.
+      def headers
+        headers = {}
+        headers.merge!('X-Delete-At' => delete_at) if delete_at
+        headers
+      end
+
+      def delete_at
+        return unless days_to_keep
+        @delete_at ||= (Time.now.utc + days_to_keep * 60**2 * 24).to_i
+      end
+
+      def adjusted_segment_bytes(segment_bytes, file_size)
+        return segment_bytes if file_size / segment_bytes.to_f <= 1000
+
+        mb = orig_mb = segment_bytes / 1024**2
+        mb += 1 until file_size / (1024**2 * mb).to_f <= 1000
+        Logger.warn Error.new(<<-EOS)
+          Segment Size Adjusted
+          Your original #segment_size of #{ orig_mb } MiB has been adjusted
+          to #{ mb } MiB in order to satisfy the limit of 1000 segments.
+          To enforce your chosen #segment_size, you should use the Splitter.
+          e.g. split_into_chunks_of #{ mb * 1000 } (#segment_size * 1000)
+        EOS
+        1024**2 * mb
+      end
+
+      class Object
+        attr_reader :name, :hash
+
+        def initialize(cloud_io, data)
+          @cloud_io = cloud_io
+          @name = data['name']
+          @hash = data['hash']
+        end
+
+        def slo?
+          !!metadata['X-Static-Large-Object']
+        end
+
+        def marked_for_deletion?
+          !!metadata['X-Delete-At']
+        end
+
+        private
+
+        def metadata
+          @metadata ||= @cloud_io.head_object(self).headers
+        end
+      end
+
+    end
+  end
+end