nfm-backup 4.0.1a

data/lib/backup/packager.rb

@@ -0,0 +1,101 @@
+# encoding: utf-8
+
+module Backup
+  module Packager
+    class Error < Backup::Error; end
+
+    class << self
+      include Utilities::Helpers
+
+      ##
+      # Build the final package for the backup model.
+      def package!(model)
+        @package   = model.package
+        @encryptor = model.encryptor
+        @splitter  = model.splitter
+        @pipeline  = Pipeline.new
+
+        Logger.info "Packaging the backup files..."
+        procedure.call
+
+        if @pipeline.success?
+          Logger.info "Packaging Complete!"
+        else
+          raise Error, "Failed to Create Backup Package\n" +
+              @pipeline.error_messages
+        end
+      end
+
+      private
+
+      ##
+      # Builds a chain of nested Procs which adds each command to a Pipeline
+      # needed to package the final command to package the backup.
+      # This is done so that the Encryptor and Splitter have the ability
+      # to perform actions before and after the final command is executed.
+      # No Encryptors currently utilize this, however the Splitter does.
+      def procedure
+        stack = []
+
+        ##
+        # Initial `tar` command to package the temporary backup folder.
+        # The command's output will then be either piped to the Encryptor
+        # or the Splitter (if no Encryptor), or through `cat` into the final
+        # output file if neither are configured.
+        @pipeline << "#{ utility(:tar) } -cf - " +
+            "-C '#{ Config.tmp_path }' '#{ @package.trigger }'"
+
+        ##
+        # If an Encryptor was configured, it will be called first
+        # to add the encryption utility command to be piped through,
+        # and amend the final package extension.
+        # It's output will then be either piped into a Splitter,
+        # or through `cat` into the final output file.
+        if @encryptor
+          stack << lambda do
+            @encryptor.encrypt_with do |command, ext|
+              @pipeline << command
+              @package.extension << ext
+              stack.shift.call
+            end
+          end
+        end
+
+        ##
+        # If a Splitter was configured, the `split` utility command will be
+        # added to the Pipeline to split the final output into multiple files.
+        # Once the Proc executing the Pipeline has completed and returns back
+        # to the Splitter, it will check the final output files to determine
+        # if the backup was indeed split.
+        # If so, it will set the package's chunk_suffixes. If not, it will
+        # remove the '-aa' suffix from the only file created by `split`.
+        #
+        # If no Splitter was configured, the final file output will be
+        # piped through `cat` into the final output file.
+        if @splitter
+          stack << lambda do
+            @splitter.split_with do |command|
+              @pipeline << command
+              stack.shift.call
+            end
+          end
+        else
+          stack << lambda do
+            outfile = File.join(Config.tmp_path, @package.basename)
+            @pipeline << "#{ utility(:cat) } > #{ outfile }"
+            stack.shift.call
+          end
+        end
+
+        ##
+        # Last Proc to be called runs the Pipeline the procedure built.
+        # Once complete, the call stack will unwind back through the
+        # preceeding Procs in the stack (if any)
+        stack << lambda { @pipeline.run }
+
+        stack.shift
+      end
+
+    end
+  end
+end

data/lib/backup/pipeline.rb

@@ -0,0 +1,124 @@
+# encoding: utf-8
+
+module Backup
+  class Pipeline
+    class Error < Backup::Error; end
+
+    include Utilities::Helpers
+
+    attr_reader :stderr, :errors
+
+    def initialize
+      @commands = []
+      @success_codes = []
+      @errors = []
+      @stderr = ''
+    end
+
+    ##
+    # Adds a command to be executed in the pipeline.
+    # Each command will be run in the order in which it was added,
+    # with it's output being piped to the next command.
+    #
+    # +success_codes+ must be an Array of Integer exit codes that will
+    # be considered successful for the +command+.
+    def add(command, success_codes)
+      @commands << command
+      @success_codes << success_codes
+    end
+
+    ##
+    # Commands added using this method will only be considered successful
+    # if their exit status is 0.
+    #
+    # Use #add if successful exit status codes need to be specified.
+    def <<(command)
+      add(command, [0])
+    end
+
+    ##
+    # Runs the command line from `#pipeline` and collects STDOUT/STDERR.
+    # STDOUT is then parsed to determine the exit status of each command.
+    # For each command with a non-zero exit status, a SystemCallError is
+    # created and added to @errors. All STDERR output is set in @stderr.
+    #
+    # Note that there is no accumulated STDOUT from the commands themselves.
+    # Also, the last command should not attempt to write to STDOUT.
+    # Any output on STDOUT from the final command will be sent to STDERR.
+    # This in itself will not cause #run to fail, but will log warnings
+    # when all commands exit with non-zero status.
+    #
+    # Use `#success?` to determine if all commands in the pipeline succeeded.
+    # If `#success?` returns `false`, use `#error_messages` to get an error report.
+    def run
+      Open4.popen4(pipeline) do |pid, stdin, stdout, stderr|
+        pipestatus = stdout.read.gsub("\n", '').split(':').sort
+        pipestatus.each do |status|
+          index, exitstatus = status.split('|').map(&:to_i)
+          unless @success_codes[index].include?(exitstatus)
+            command = command_name(@commands[index])
+            @errors << SystemCallError.new(
+              "'#{ command }' returned exit code: #{ exitstatus }", exitstatus
+            )
+          end
+        end
+        @stderr = stderr.read.strip
+      end
+      Logger.warn(stderr_messages) if success? && stderr_messages
+    rescue Exception => err
+      raise Error.wrap(err, 'Pipeline failed to execute')
+    end
+
+    def success?
+      @errors.empty?
+    end
+
+    ##
+    # Returns a multi-line String, reporting all STDERR messages received
+    # from the commands in the pipeline (if any), along with the SystemCallError
+    # (Errno) message for each command which had a non-zero exit status.
+    def error_messages
+      @error_messages ||= (stderr_messages || '') +
+          "The following system errors were returned:\n" +
+          @errors.map {|err| "#{ err.class }: #{ err.message }" }.join("\n")
+    end
+
+    private
+
+    ##
+    # Each command is added as part of the pipeline, grouped with an `echo`
+    # command to pass along the command's index in @commands and it's exit status.
+    # The command's STDERR is redirected to FD#4, and the `echo` command to
+    # report the "index|exit status" is redirected to FD#3.
+    # Each command's STDOUT will be connected to the STDIN of the next subshell.
+    # The entire pipeline is run within a container group, which redirects
+    # FD#3 to STDOUT and FD#4 to STDERR so these can be collected.
+    # FD#1 is redirected to STDERR so that any output from the final command
+    # on STDOUT will generate warnings, since the final command should not
+    # attempt to write to STDOUT, as this would interfere with collecting
+    # the exit statuses.
+    #
+    # There is no guarantee as to the order of this output, which is why the
+    # command's index in @commands is passed along with it's exit status.
+    # And, if multiple commands output messages on STDERR, those messages
+    # may be interleaved. Interleaving of the "index|exit status" outputs
+    # should not be an issue, given the small byte size of the data being written.
+    def pipeline
+      parts = []
+      @commands.each_with_index do |command, index|
+        parts << %Q[{ #{ command } 2>&4 ; echo "#{ index }|$?:" >&3 ; }]
+      end
+      %Q[{ #{ parts.join(' | ') } } 3>&1 1>&2 4>&2]
+    end
+
+    def stderr_messages
+      @stderr_messages ||= @stderr.empty? ? false : <<-EOS.gsub(/^ +/, '  ')
+        Pipeline STDERR Messages:
+        (Note: may be interleaved if multiple commands returned error messages)
+
+        #{ @stderr }
+      EOS
+    end
+
+  end
+end

data/lib/backup/splitter.rb

@@ -0,0 +1,76 @@
+# encoding: utf-8
+
+module Backup
+  class Splitter
+    include Utilities::Helpers
+
+    attr_reader :package, :chunk_size, :suffix_length
+
+    def initialize(model, chunk_size, suffix_length)
+      @package = model.package
+      @chunk_size = chunk_size
+      @suffix_length = suffix_length
+    end
+
+    ##
+    # This is called as part of the procedure used to build the final
+    # backup package file(s). It yields it's portion of the command line
+    # for this procedure, which will split the data being piped into it
+    # into multiple files, based on the @chunk_size, using a suffix length as
+    # specified by @suffix_length.
+    # Once the packaging procedure is complete, it will return and
+    # @package.chunk_suffixes will be set based on the resulting files.
+    def split_with
+      Logger.info "Splitter configured with a chunk size of #{ chunk_size }MB " +
+                  "and suffix length of #{ suffix_length }."
+      yield split_command
+      after_packaging
+    end
+
+    private
+
+    ##
+    # The `split` command reads from $stdin and will store it's output in
+    # multiple files, based on @chunk_size and @suffix_length, using the full
+    # path to the final @package.basename, plus a '-' separator as the `prefix`.
+    def split_command
+      "#{ utility(:split) } -a #{ suffix_length } -b #{ chunk_size }m - " +
+          "'#{ File.join(Config.tmp_path, package.basename + '-') }'"
+    end
+
+    ##
+    # Finds the resulting files from the packaging procedure
+    # and stores an Array of suffixes used in @package.chunk_suffixes.
+    # If the @chunk_size was never reached and only one file
+    # was written, that file will be suffixed with '-aa' (or -a; -aaa; etc
+    # depending upon suffix_length). In which case, it will simply
+    # remove the suffix from the filename.
+    def after_packaging
+      suffixes = chunk_suffixes
+      first_suffix = 'a' * suffix_length
+      if suffixes == [first_suffix]
+        FileUtils.mv(
+          File.join(Config.tmp_path, "#{ package.basename }-#{ first_suffix }"),
+          File.join(Config.tmp_path, package.basename)
+        )
+      else
+        package.chunk_suffixes = suffixes
+      end
+    end
+
+    ##
+    # Returns an array of suffixes for each chunk, in alphabetical order.
+    # For example: [aa, ab, ac, ad, ae] or [aaa, aab, aac aad]
+    def chunk_suffixes
+      chunks.map {|chunk| File.extname(chunk).split('-').last }.sort
+    end
+
+    ##
+    # Returns an array of full paths to the backup chunks.
+    # Chunks are sorted in alphabetical order.
+    def chunks
+      Dir[File.join(Config.tmp_path, package.basename + '-*')].sort
+    end
+
+  end
+end

data/lib/backup/storage/base.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+
+module Backup
+  module Storage
+    class Base
+      include Config::Helpers
+
+      ##
+      # Base path on the remote where backup package files will be stored.
+      attr_accessor :path
+
+      ##
+      # Sets the limit to how many backups to keep in the remote location.
+      # If exceeded, the oldest will be removed to make room for the newest
+      attr_accessor :keep
+
+      attr_reader :model, :package, :storage_id
+
+      ##
+      # +storage_id+ is a user-defined string used to uniquely identify
+      # multiple storages of the same type. If multiple storages of the same
+      # type are added to a single backup model, this identifier must be set.
+      # This will be appended to the YAML storage file used for cycling backups.
+      def initialize(model, storage_id = nil, &block)
+        @model = model
+        @package = model.package
+        @storage_id = storage_id.to_s.gsub(/\W/, '_') if storage_id
+
+        load_defaults!
+        instance_eval(&block) if block_given?
+      end
+
+      def perform!
+        Logger.info "#{ storage_name } Started..."
+        transfer!
+        cycle! if respond_to?(:cycle!, true) && keep.to_i > 0
+        Logger.info "#{ storage_name } Finished!"
+      end
+
+      private
+
+      ##
+      # Return the remote path for the current or given package.
+      def remote_path(pkg = package)
+        path.empty? ? File.join(pkg.trigger, pkg.time) :
+                      File.join(path, pkg.trigger, pkg.time)
+      end
+      alias :remote_path_for :remote_path
+
+      def storage_name
+        @storage_name ||= self.class.to_s.sub('Backup::', '') +
+            (storage_id ? " (#{ storage_id })" : '')
+      end
+
+    end
+  end
+end

data/lib/backup/storage/cloud_files.rb

@@ -0,0 +1,158 @@
+# encoding: utf-8
+require 'backup/cloud_io/cloud_files'
+
+module Backup
+  module Storage
+    class CloudFiles < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Rackspace CloudFiles Credentials
+      attr_accessor :username, :api_key
+
+      ##
+      # Rackspace Auth URL (optional)
+      attr_accessor :auth_url
+
+      ##
+      # Rackspace Service Net
+      # (LAN-based transfers to avoid charges and improve performance)
+      attr_accessor :servicenet
+
+      ##
+      # Rackspace Region (optional)
+      attr_accessor :region
+
+      ##
+      # Rackspace Container Name
+      attr_accessor :container
+
+      ##
+      # Rackspace Container Name for SLO Segments
+      # Required if #segment_size is set. Must be different from #container.
+      attr_accessor :segments_container
+
+      ##
+      # SLO Segment size, specified in MiB.
+      #
+      # Each package file larger than +segment_size+
+      # will be uploaded as a Static Large Objects (SLO).
+      #
+      # Defaults to 0 for backward compatibility (pre v.3.7.0),
+      # since #segments_container would be required.
+      #
+      # Minimum: 1 (0 disables SLO support)
+      # Maximum: 5120 (5 GiB)
+      attr_accessor :segment_size
+
+      ##
+      # If set, all backup package files (including SLO segments) will be
+      # scheduled for automatic removal by the server.
+      #
+      # The `keep` option should not be used if this is set,
+      # unless you're transitioning from the `keep` option.
+      attr_accessor :days_to_keep
+
+      ##
+      # Number of times to retry failed operations.
+      #
+      # Default: 10
+      attr_accessor :max_retries
+
+      ##
+      # Time in seconds to pause before each retry.
+      #
+      # Default: 30
+      attr_accessor :retry_waitsec
+
+      ##
+      # Additional options to pass along to fog.
+      # e.g. Fog::Storage.new({ :provider => 'Rackspace' }.merge(fog_options))
+      attr_accessor :fog_options
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @servicenet         ||= false
+        @segment_size       ||= 0
+        @max_retries        ||= 10
+        @retry_waitsec      ||= 30
+
+        @path ||= 'backups'
+        path.sub!(/^\//, '')
+
+        check_configuration
+      end
+
+      private
+
+      def cloud_io
+        @cloud_io ||= CloudIO::CloudFiles.new(
+          :username           => username,
+          :api_key            => api_key,
+          :auth_url           => auth_url,
+          :region             => region,
+          :servicenet         => servicenet,
+          :container          => container,
+          :segments_container => segments_container,
+          :segment_size       => segment_size,
+          :days_to_keep       => days_to_keep,
+          :max_retries        => max_retries,
+          :retry_waitsec      => retry_waitsec,
+          :fog_options        => fog_options
+        )
+      end
+
+      def transfer!
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{ container }/#{ dest }'..."
+          cloud_io.upload(src, dest)
+        end
+
+        package.no_cycle = true if days_to_keep
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{ package.time }..."
+
+        remote_path = remote_path_for(package)
+        objects = cloud_io.objects(remote_path)
+
+        raise Error, "Package at '#{ remote_path }' not found" if objects.empty?
+
+        slo_objects, objects = objects.partition(&:slo?)
+        cloud_io.delete_slo(slo_objects)
+        cloud_io.delete(objects)
+      end
+
+      def check_configuration
+        required = %w{ username api_key container }
+        raise Error, <<-EOS if required.map {|name| send(name) }.any?(&:nil?)
+          Configuration Error
+          #{ required.map {|name| "##{ name }"}.join(', ') } are all required
+        EOS
+
+        raise Error, <<-EOS if segment_size > 0 && segments_container.to_s.empty?
+          Configuration Error
+          #segments_container is required if #segment_size is > 0
+        EOS
+
+        raise Error, <<-EOS if container == segments_container
+          Configuration Error
+          #container and #segments_container must not be the same container.
+        EOS
+
+        raise Error, <<-EOS if segment_size > 5120
+          Configuration Error
+          #segment_size is too large (max 5120)
+        EOS
+      end
+
+    end
+  end
+end