backup 3.0.23 → 3.0.24

data/lib/backup/pipeline.rb

@@ -0,0 +1,110 @@
+# encoding: utf-8
+
+module Backup
+  class Pipeline
+    include Backup::CLI::Helpers
+
+    attr_reader :stderr, :errors
+
+    def initialize
+      @commands = []
+      @errors = []
+      @stderr = ''
+    end
+
+    ##
+    # Adds a command to be executed in the pipeline.
+    # Each command will be run in the order in which it was added,
+    # with it's output being piped to the next command.
+    def <<(command)
+      @commands << command
+    end
+
+    ##
+    # Runs the command line from `#pipeline` and collects STDOUT/STDERR.
+    # STDOUT is then parsed to determine the exit status of each command.
+    # For each command with a non-zero exit status, a SystemCallError is
+    # created and added to @errors. All STDERR output is set in @stderr.
+    #
+    # Note that there is no accumulated STDOUT from the commands themselves.
+    # Also, the last command should not attempt to write to STDOUT.
+    # Any output on STDOUT from the final command will be sent to STDERR.
+    # This in itself will not cause #run to fail, but will log warnings
+    # when all commands exit with non-zero status.
+    #
+    # Use `#success?` to determine if all commands in the pipeline succeeded.
+    # If `#success?` returns `false`, use `#error_messages` to get an error report.
+    def run
+      Open4.popen4(pipeline) do |pid, stdin, stdout, stderr|
+        pipestatus = stdout.read.gsub("\n", '').split(':').sort
+        pipestatus.each do |status|
+          index, exitstatus = status.split('|').map(&:to_i)
+          if exitstatus > 0
+            command = command_name(@commands[index])
+            @errors << SystemCallError.new(
+              "'#{ command }' returned exit code: #{ exitstatus }", exitstatus
+            )
+          end
+        end
+        @stderr = stderr.read.strip
+      end
+      Logger.warn(stderr_messages) if success? && stderr_messages
+    rescue Exception => e
+      raise Errors::Pipeline::ExecutionError.wrap(e)
+    end
+
+    def success?
+      @errors.empty?
+    end
+
+    ##
+    # Returns a multi-line String, reporting all STDERR messages received
+    # from the commands in the pipeline (if any), along with the SystemCallError
+    # (Errno) message for each command which had a non-zero exit status.
+    #
+    # Each error is wrapped by Backup::Errors to provide formatting.
+    def error_messages
+      @error_messages ||= (stderr_messages || '') +
+          "The following system errors were returned:\n" +
+          @errors.map {|err| Errors::Error.wrap(err).message }.join("\n")
+    end
+
+    private
+
+    ##
+    # Each command is added as part of the pipeline, grouped with an `echo`
+    # command to pass along the command's index in @commands and it's exit status.
+    # The command's STDERR is redirected to FD#4, and the `echo` command to
+    # report the "index|exit status" is redirected to FD#3.
+    # Each command's STDOUT will be connected to the STDIN of the next subshell.
+    # The entire pipeline is run within a container group, which redirects
+    # FD#3 to STDOUT and FD#4 to STDERR so these can be collected.
+    # FD#1 is redirected to STDERR so that any output from the final command
+    # on STDOUT will generate warnings, since the final command should not
+    # attempt to write to STDOUT, as this would interfere with collecting
+    # the exit statuses.
+    #
+    # There is no guarantee as to the order of this output, which is why the
+    # command's index in @commands is passed along with it's exit status.
+    # And, if multiple commands output messages on STDERR, those messages
+    # may be interleaved. Interleaving of the "index|exit status" outputs
+    # should not be an issue, given the small byte size of the data being written.
+    def pipeline
+      parts = []
+      @commands.each_with_index do |command, index|
+        parts << %Q[{ #{ command } 2>&4 ; echo "#{ index }|$?:" >&3 ; }]
+      end
+      %Q[{ #{ parts.join(' | ') } } 3>&1 1>&2 4>&2]
+    end
+
+    def stderr_messages
+      @stderr_messages ||= @stderr.empty? ? false : <<-EOS.gsub(/^ +/, '  ')
+        Pipeline STDERR Messages:
+        (Note: may be interleaved if multiple commands returned error messages)
+
+        #{ @stderr }
+      EOS
+    end
+
+  end
+end

data/lib/backup/storage/dropbox.rb

@@ -23,13 +23,10 @@ module Backup
       # Path to where the backups will be stored
       attr_accessor :path
 
-      # Deprecated as of v3.0.21 - for move to official 'dropbox-sdk' gem (v1.1)
-      def timeout=(value)
-        if value
-          Logger.warn "[DEPRECATED] Backup::Storage::Dropbox.timeout=\n" +
-              "  is deprecated and will be removed at some point."
-        end
-      end
+      attr_deprecate :email,    :version => '3.0.17'
+      attr_deprecate :password, :version => '3.0.17'
+
+      attr_deprecate :timeout, :version => '3.0.21'
 
       ##
       # Creates a new instance of the storage object

data/lib/backup/syncer/base.rb

@@ -6,10 +6,6 @@ module Backup
       include Backup::CLI::Helpers
       include Backup::Configuration::Helpers
 
-      ##
-      # Directories to sync
-      attr_accessor :directories
-
       ##
       # Path to store the synced files/directories to
       attr_accessor :path
@@ -18,6 +14,14 @@ module Backup
       # Flag for mirroring the files/directories
       attr_accessor :mirror
 
+      def initialize
+        load_defaults!
+
+        @path               ||= 'backups'
+        @mirror             ||= false
+        @directories          = Array.new
+      end
+
       ##
       # Syntactical suger for the DSL for adding directories
       def directories(&block)

data/lib/backup/syncer/cloud/base.rb

@@ -0,0 +1,247 @@
+# encoding: utf-8
+
+##
+# Only load the Fog gem, along with the Parallel gem, when the
+# Backup::Syncer::Cloud class is loaded
+Backup::Dependency.load('fog')
+Backup::Dependency.load('parallel')
+
+module Backup
+  module Syncer
+    module Cloud
+      class Base < Syncer::Base
+
+        ##
+        # Create a Mutex to synchronize certain parts of the code
+        # in order to prevent race conditions or broken STDOUT.
+        MUTEX = Mutex.new
+
+        ##
+        # Concurrency setting - defaults to false, but can be set to:
+        # - :threads
+        # - :processes
+        attr_accessor :concurrency_type
+
+        ##
+        # Concurrency level - the number of threads or processors to use.
+        # Defaults to 2.
+        attr_accessor :concurrency_level
+
+        ##
+        # Instantiates a new Cloud Syncer object for either
+        # the Cloud::S3 or Cloud::CloudFiles Syncer.
+        #
+        # Pre-configured defaults specified in either
+        # Configuration::Syncer::Cloud::S3 or
+        # Configuration::Syncer::Cloud::CloudFiles
+        # are set via a super() call to Syncer::Base.
+        #
+        # If not specified in the pre-configured defaults,
+        # the Cloud specific defaults are set here before evaluating
+        # any block provided in the user's configuration file.
+        def initialize
+          super
+
+          @concurrency_type  ||= false
+          @concurrency_level ||= 2
+        end
+
+        ##
+        # Performs the Sync operation
+        def perform!
+          Logger.message("#{ syncer_name } started the syncing process:")
+
+          @directories.each do |directory|
+            SyncContext.new(
+              File.expand_path(directory), repository_object, @path
+            ).sync! @mirror, @concurrency_type, @concurrency_level
+          end
+
+          Logger.message("#{ syncer_name } Syncing Complete!")
+        end
+
+        private
+
+        class SyncContext
+          attr_reader :directory, :bucket, :path, :remote_base
+
+          ##
+          # Creates a new SyncContext object which handles a single directory
+          # from the Syncer::Base @directories array.
+          def initialize(directory, bucket, path)
+            @directory, @bucket, @path = directory, bucket, path
+            @remote_base = File.join(path, File.basename(directory))
+          end
+
+          ##
+          # Performs the sync operation using the provided techniques
+          # (mirroring/concurrency).
+          def sync!(mirror = false, concurrency_type = false, concurrency_level = 2)
+            block = Proc.new { |relative_path| sync_file relative_path, mirror }
+
+            case concurrency_type
+            when FalseClass
+              all_file_names.each &block
+            when :threads
+              Parallel.each all_file_names,
+                  :in_threads => concurrency_level, &block
+            when :processes
+              Parallel.each all_file_names,
+                  :in_processes => concurrency_level, &block
+            else
+              raise Errors::Syncer::Cloud::ConfigurationError,
+                  "Unknown concurrency_type setting: #{ concurrency_type.inspect }"
+            end
+          end
+
+          private
+
+          ##
+          # Gathers all the relative paths to the local files
+          # and merges them with the , removing
+          # duplicate keys if any, and sorts the in alphabetical order.
+          def all_file_names
+            @all_file_names ||= (local_files.keys | remote_files.keys).sort
+          end
+
+          ##
+          # Returns a Hash of local files, validated to ensure the path
+          # does not contain invalid UTF-8 byte sequences.
+          # The keys are the filesystem paths, relative to @directory.
+          # The values are the LocalFile objects for that given file.
+          def local_files
+            @local_files ||= begin
+              hash = {}
+              local_hashes.lines.map do |line|
+                LocalFile.new(@directory, line)
+              end.compact.each do |file|
+                hash.merge!(file.relative_path => file)
+              end
+              hash
+            end
+          end
+
+          ##
+          # Returns a String of file paths and their md5 hashes.
+          def local_hashes
+            MUTEX.synchronize {
+              Logger.message("\s\sGenerating checksums for '#{ @directory }'")
+            }
+            `find #{ @directory } -print0 | xargs -0 openssl md5 2> /dev/null`
+          end
+
+          ##
+          # Returns a Hash of remote files
+          # The keys are the remote paths, relative to @remote_base
+          # The values are the Fog file objects for that given file
+          def remote_files
+            @remote_files ||= begin
+              hash = {}
+              @bucket.files.all(:prefix => @remote_base).each do |file|
+                hash.merge!(file.key.sub("#{ @remote_base }/", '') => file)
+              end
+              hash
+            end
+          end
+
+          ##
+          # Performs a sync operation on a file. When mirroring is enabled
+          # and a local file has been removed since the last sync, it will also
+          # remove it from the remote location. It will no upload files that
+          # have not changed since the last sync. Checks are done using an md5
+          # hash. If a file has changed, or has been newly added, the file will
+          # be transferred/overwritten.
+          def sync_file(relative_path, mirror)
+            local_file  = local_files[relative_path]
+            remote_file = remote_files[relative_path]
+            remote_path = File.join(@remote_base, relative_path)
+
+            if local_file && File.exist?(local_file.path)
+              unless remote_file && remote_file.etag == local_file.md5
+                MUTEX.synchronize {
+                  Logger.message("\s\s[transferring] '#{ remote_path }'")
+                }
+                File.open(local_file.path, 'r') do |file|
+                  @bucket.files.create(
+                    :key  => remote_path,
+                    :body => file
+                  )
+                end
+              else
+                MUTEX.synchronize {
+                  Logger.message("\s\s[skipping] '#{ remote_path }'")
+                }
+              end
+            elsif remote_file
+              if mirror
+                MUTEX.synchronize {
+                  Logger.message("\s\s[removing] '#{ remote_path }'")
+                }
+                remote_file.destroy
+              else
+                MUTEX.synchronize {
+                  Logger.message("\s\s[leaving] '#{ remote_path }'")
+                }
+              end
+            end
+          end
+        end # class SyncContext
+
+        class LocalFile
+          attr_reader :path, :relative_path, :md5
+
+          ##
+          # Return a new LocalFile object if it's valid.
+          # Otherwise, log a warning and return nil.
+          def self.new(*args)
+            local_file = super(*args)
+            if local_file.invalid?
+              MUTEX.synchronize {
+                Logger.warn(
+                  "\s\s[skipping] #{ local_file.path }\n" +
+                  "\s\sPath Contains Invalid UTF-8 byte sequences"
+                )
+              }
+              return nil
+            end
+            local_file
+          end
+
+          ##
+          # Creates a new LocalFile object using the given directory and line
+          # from the md5 hash checkup. This object figures out the path,
+          # relative_path and md5 hash for the file.
+          def initialize(directory, line)
+            @invalid = false
+            @directory = sanitize(directory)
+            @path, @md5 = sanitize(line).chomp.
+                match(/^MD5\(([^\)]+)\)= (\w+)$/).captures
+            @relative_path = @path.sub(@directory + '/', '')
+          end
+
+          def invalid?
+            @invalid
+          end
+
+          private
+
+          ##
+          # Sanitize string and replace any invalid UTF-8 characters.
+          # If replacements are made, flag the LocalFile object as invalid.
+          def sanitize(str)
+            str.each_char.map do |char|
+              begin
+                char if !!char.unpack('U')
+              rescue
+                @invalid = true
+                "\xEF\xBF\xBD" # => "\uFFFD"
+              end
+            end.join
+          end
+
+        end # class LocalFile
+
+      end # class Base < Syncer::Base
+    end # module Cloud
+  end
+end

data/lib/backup/syncer/cloud/cloud_files.rb

@@ -0,0 +1,78 @@
+# encoding: utf-8
+
+module Backup
+  module Syncer
+    module Cloud
+      class CloudFiles < Base
+
+        ##
+        # Rackspace CloudFiles Credentials
+        attr_accessor :api_key, :username
+
+        ##
+        # Rackspace CloudFiles Container
+        attr_accessor :container
+
+        ##
+        # Rackspace AuthURL allows you to connect
+        # to a different Rackspace datacenter
+        # - https://auth.api.rackspacecloud.com     (Default: US)
+        # - https://lon.auth.api.rackspacecloud.com (UK)
+        attr_accessor :auth_url
+
+        ##
+        # Improve performance and avoid data transfer costs
+        # by setting @servicenet to `true`
+        # This only works if Backup runs on a Rackspace server
+        attr_accessor :servicenet
+
+        ##
+        # Instantiates a new Cloud::CloudFiles Syncer.
+        #
+        # Pre-configured defaults specified in
+        # Configuration::Syncer::Cloud::CloudFiles
+        # are set via a super() call to Cloud::Base,
+        # which in turn will invoke Syncer::Base.
+        #
+        # Once pre-configured defaults and Cloud specific defaults are set,
+        # the block from the user's configuration file is evaluated.
+        def initialize(&block)
+          super
+
+          instance_eval(&block) if block_given?
+          @path = path.sub(/^\//, '')
+        end
+
+        private
+
+        ##
+        # Established and creates a new Fog storage object for CloudFiles.
+        def connection
+          @connection ||= Fog::Storage.new(
+            :provider             => provider,
+            :rackspace_username   => username,
+            :rackspace_api_key    => api_key,
+            :rackspace_auth_url   => auth_url,
+            :rackspace_servicenet => servicenet
+          )
+        end
+
+        ##
+        # Creates a new @repository_object (container).
+        # Fetches it from Cloud Files if it already exists,
+        # otherwise it will create it first and fetch use that instead.
+        def repository_object
+          @repository_object ||= connection.directories.get(container) ||
+            connection.directories.create(:key => container)
+        end
+
+        ##
+        # This is the provider that Fog uses for the Cloud Files
+        def provider
+          "Rackspace"
+        end
+
+      end # class Cloudfiles < Base
+    end # module Cloud
+  end
+end