backup 3.6.0 → 3.7.0

data/lib/backup/storage/scp.rb

@@ -4,6 +4,7 @@ require 'net/scp'
 module Backup
   module Storage
     class SCP < Base
+      class Error < Backup::Error; end
 
       ##
       # Server credentials
@@ -13,9 +14,8 @@ module Backup
       # Server IP Address and SCP port
       attr_accessor :ip, :port
 
-      def initialize(model, storage_id = nil, &block)
+      def initialize(model, storage_id = nil)
         super
-        instance_eval(&block) if block_given?
 
         @port ||= 22
         @path ||= 'backups'
@@ -55,8 +55,7 @@ module Backup
           end
         end
         unless errors.empty?
-          raise Errors::Storage::SCP::SSHError,
-              "Net::SSH reported the following errors:\n" +
+          raise Error, "Net::SSH reported the following errors:\n" +
               errors.join("\n")
         end
       end

data/lib/backup/storage/sftp.rb

@@ -13,9 +13,8 @@ module Backup
       # Server IP Address and SFTP port
       attr_accessor :ip, :port
 
-      def initialize(model, storage_id = nil, &block)
+      def initialize(model, storage_id = nil)
         super
-        instance_eval(&block) if block_given?
 
         @port ||= 22
         @path ||= 'backups'

data/lib/backup/syncer/base.rb

@@ -25,7 +25,6 @@ module Backup
 
         load_defaults!
 
-        @path   ||= '~/backups'
         @mirror ||= false
         @directories = Array.new
       end

data/lib/backup/syncer/cloud/base.rb

@@ -1,257 +1,178 @@
 # encoding: utf-8
 
-##
-# Only load the Fog gem, along with the Parallel gem, when the
-# Backup::Syncer::Cloud class is loaded
-# Backup::Dependency.load('fog')
-# Backup::Dependency.load('parallel')
-require 'fog'
-require 'parallel'
-
 module Backup
   module Syncer
     module Cloud
-      class Base < Syncer::Base
+      class Error < Backup::Error; end
 
-        ##
-        # Create a Mutex to synchronize certain parts of the code
-        # in order to prevent race conditions or broken STDOUT.
+      class Base < Syncer::Base
         MUTEX = Mutex.new
 
         ##
-        # Concurrency setting - defaults to false, but can be set to:
-        # - :threads
-        # - :processes
-        attr_accessor :concurrency_type
+        # Number of threads to use for concurrency.
+        #
+        # Default: 0 (no concurrency)
+        attr_accessor :thread_count
 
         ##
-        # Concurrency level - the number of threads or processors to use.
-        # Defaults to 2.
-        attr_accessor :concurrency_level
+        # Number of times to retry failed operations.
+        #
+        # Default: 10
+        attr_accessor :max_retries
 
         ##
-        # Instantiates a new Cloud Syncer object for either
-        # the Cloud::S3 or Cloud::CloudFiles Syncer.
+        # Time in seconds to pause before each retry.
         #
-        # Pre-configured defaults specified in either
-        # Configuration::Syncer::Cloud::S3 or
-        # Configuration::Syncer::Cloud::CloudFiles
-        # are set via a super() call to Syncer::Base.
-        #
-        # If not specified in the pre-configured defaults,
-        # the Cloud specific defaults are set here before evaluating
-        # any block provided in the user's configuration file.
-        def initialize(syncer_id = nil)
+        # Default: 30
+        attr_accessor :retry_waitsec
+
+        def initialize(syncer_id = nil, &block)
           super
+          instance_eval(&block) if block_given?
+
+          @thread_count   ||= 0
+          @max_retries    ||= 10
+          @retry_waitsec  ||= 30
 
-          @path = path.sub(/^~\//, '')
-          @concurrency_type  ||= false
-          @concurrency_level ||= 2
+          @path ||= 'backups'
+          @path = path.sub(/^\//, '')
         end
 
-        ##
-        # Performs the Sync operation
         def perform!
           log!(:started)
-          Logger.info(
-            "\s\sConcurrency: #{ @concurrency_type } Level: #{ @concurrency_level }"
-          )
-
-          @directories.each do |directory|
-            SyncContext.new(
-              File.expand_path(directory), repository_object, @path
-            ).sync! @mirror, @concurrency_type, @concurrency_level
-          end
-
+          @transfer_count = 0
+          @unchanged_count = 0
+          @skipped_count = 0
+          @orphans = thread_count > 0 ? Queue.new : []
+
+          directories.each {|dir| sync_directory(dir) }
+          orphans_result = process_orphans
+
+          Logger.info "\nSummary:"
+          Logger.info "\s\sTransferred Files: #{ @transfer_count }"
+          Logger.info "\s\s#{ orphans_result }"
+          Logger.info "\s\sUnchanged Files: #{ @unchanged_count }"
+          Logger.warn "\s\sSkipped Files: #{ @skipped_count }" if @skipped_count > 0
           log!(:finished)
         end
 
         private
 
-        class SyncContext
-          include Utilities::Helpers
-
-          attr_reader :directory, :bucket, :path, :remote_base
-
-          ##
-          # Creates a new SyncContext object which handles a single directory
-          # from the Syncer::Base @directories array.
-          def initialize(directory, bucket, path)
-            @directory, @bucket, @path = directory, bucket, path
-            @remote_base = File.join(path, File.basename(directory))
-          end
-
-          ##
-          # Performs the sync operation using the provided techniques
-          # (mirroring/concurrency).
-          def sync!(mirror = false, concurrency_type = false, concurrency_level = 2)
-            block = Proc.new { |relative_path| sync_file relative_path, mirror }
+        def sync_directory(dir)
+          remote_base = File.join(path, File.basename(dir))
+          Logger.info "Gathering remote data for '#{ remote_base }'..."
+          remote_files = get_remote_files(remote_base)
+
+          Logger.info("Gathering local data for '#{ File.expand_path(dir) }'...")
+          local_files = LocalFile.find(dir)
+
+          relative_paths = (local_files.keys | remote_files.keys).sort
+          if relative_paths.empty?
+            Logger.info 'No local or remote files found'
+          else
+            Logger.info 'Syncing...'
+            sync_block = Proc.new do |relative_path|
+              local_file  = local_files[relative_path]
+              remote_md5  = remote_files[relative_path]
+              remote_path = File.join(remote_base, relative_path)
+              sync_file(local_file, remote_path, remote_md5)
+            end
 
-            case concurrency_type
-            when FalseClass
-              all_file_names.each(&block)
-            when :threads
-              Parallel.each all_file_names,
-                  :in_threads => concurrency_level, &block
-            when :processes
-              Parallel.each all_file_names,
-                  :in_processes => concurrency_level, &block
+            if thread_count > 0
+              sync_in_threads(relative_paths, sync_block)
             else
-              raise Errors::Syncer::Cloud::ConfigurationError,
-                  "Unknown concurrency_type setting: #{ concurrency_type.inspect }"
+              relative_paths.each(&sync_block)
             end
           end
+        end
 
-          private
-
-          ##
-          # Gathers all the relative paths to the local files
-          # and merges them with the , removing
-          # duplicate keys if any, and sorts the in alphabetical order.
-          def all_file_names
-            @all_file_names ||= (local_files.keys | remote_files.keys).sort
-          end
-
-          ##
-          # Returns a Hash of local files, validated to ensure the path
-          # does not contain invalid UTF-8 byte sequences.
-          # The keys are the filesystem paths, relative to @directory.
-          # The values are the LocalFile objects for that given file.
-          def local_files
-            @local_files ||= begin
-              hash = {}
-              local_hashes.lines.map do |line|
-                LocalFile.new(@directory, line)
-              end.compact.each do |file|
-                hash.merge!(file.relative_path => file)
+        def sync_in_threads(relative_paths, sync_block)
+          queue = Queue.new
+          queue << relative_paths.shift until relative_paths.empty?
+          num_threads = [thread_count, queue.size].min
+          Logger.info "\s\sUsing #{ num_threads } Threads"
+          threads = num_threads.times.map do
+            Thread.new do
+              loop do
+                path = queue.shift(true) rescue nil
+                path ? sync_block.call(path) : break
               end
-              hash
             end
           end
 
-          ##
-          # Returns a String of file paths and their md5 hashes.
-          #
-          # Utilities#run is not used here because this would produce too much
-          # log output, and Pipeline does not support capturing output.
-          def local_hashes
-            Logger.info("\s\sGenerating checksums for '#{ @directory }'")
-            cmd = "#{ utility(:find) } -L '#{ @directory }' -type f -print0 | " +
-                "#{ utility(:xargs) } -0 #{ utility(:openssl) } md5 2> /dev/null"
-            %x[#{ cmd }]
-          end
-
-          ##
-          # Returns a Hash of remote files
-          # The keys are the remote paths, relative to @remote_base
-          # The values are the Fog file objects for that given file
-          def remote_files
-            @remote_files ||= begin
-              hash = {}
-              @bucket.files.all(:prefix => @remote_base).each do |file|
-                hash.merge!(file.key.sub("#{ @remote_base }/", '') => file)
-              end
-              hash
+          # abort if any thread raises an exception
+          while threads.any?(&:alive?)
+            if threads.any? {|thr| thr.status.nil? }
+              threads.each(&:kill)
+              Thread.pass while threads.any?(&:alive?)
+              break
             end
+            sleep num_threads * 0.1
           end
+          threads.each(&:join)
+        end
 
-          ##
-          # Performs a sync operation on a file. When mirroring is enabled
-          # and a local file has been removed since the last sync, it will also
-          # remove it from the remote location. It will no upload files that
-          # have not changed since the last sync. Checks are done using an md5
-          # hash. If a file has changed, or has been newly added, the file will
-          # be transferred/overwritten.
-          def sync_file(relative_path, mirror)
-            local_file  = local_files[relative_path]
-            remote_file = remote_files[relative_path]
-            remote_path = File.join(@remote_base, relative_path)
-
-            if local_file && File.exist?(local_file.path)
-              unless remote_file && remote_file.etag == local_file.md5
-                MUTEX.synchronize {
-                  Logger.info("\s\s[transferring] '#{ remote_path }'")
-                }
-                File.open(local_file.path, 'r') do |file|
-                  @bucket.files.create(
-                    :key  => remote_path,
-                    :body => file
-                  )
-                end
-              else
-                MUTEX.synchronize {
-                  Logger.info("\s\s[skipping] '#{ remote_path }'")
-                }
-              end
-            elsif remote_file
-              if mirror
-                MUTEX.synchronize {
-                  Logger.info("\s\s[removing] '#{ remote_path }'")
-                }
-                remote_file.destroy
-              else
-                MUTEX.synchronize {
-                  Logger.info("\s\s[leaving] '#{ remote_path }'")
-                }
+        # If an exception is raised in multiple threads, only the exception
+        # raised in the first thread that Thread#join is called on will be
+        # handled. So all exceptions are logged first with their details,
+        # then a generic exception is raised.
+        def sync_file(local_file, remote_path, remote_md5)
+          if local_file && File.exist?(local_file.path)
+            if local_file.md5 == remote_md5
+              MUTEX.synchronize { @unchanged_count += 1 }
+            else
+              Logger.info("\s\s[transferring] '#{ remote_path }'")
+              begin
+                cloud_io.upload(local_file.path, remote_path)
+                MUTEX.synchronize { @transfer_count += 1 }
+              rescue CloudIO::FileSizeError => err
+                MUTEX.synchronize { @skipped_count += 1 }
+                Logger.warn Error.wrap(err, "Skipping '#{ remote_path }'")
+              rescue => err
+                Logger.error(err)
+                raise Error, <<-EOS
+                  Syncer Failed!
+                  See the Retry [info] and [error] messages (if any)
+                  for details on each failed operation.
+                EOS
               end
             end
+          elsif remote_md5
+            @orphans << remote_path
           end
-        end # class SyncContext
-
-        class LocalFile
-          attr_reader :path, :relative_path, :md5
-
-          ##
-          # Return a new LocalFile object if it's valid.
-          # Otherwise, log a warning and return nil.
-          def self.new(*args)
-            local_file = super(*args)
-            if local_file.invalid?
-              Logger.warn(
-                "\s\s[skipping] #{ local_file.path }\n" +
-                "\s\sPath Contains Invalid UTF-8 byte sequences"
-              )
-              return nil
-            end
-            local_file
-          end
+        end
 
-          ##
-          # Creates a new LocalFile object using the given directory and line
-          # from the md5 hash checkup. This object figures out the path,
-          # relative_path and md5 hash for the file.
-          def initialize(directory, line)
-            @invalid = false
-            @directory = sanitize(directory)
-            line = sanitize(line).chomp
-            @path = line.slice(4..-36)
-            @md5 = line.slice(-32..-1)
-            @relative_path = @path.sub(@directory + '/', '')
+        def process_orphans
+          if @orphans.empty?
+            return mirror ? 'Deleted Files: 0' : 'Orphaned Files: 0'
           end
 
-          def invalid?
-            @invalid
+          if @orphans.is_a?(Queue)
+            @orphans = @orphans.size.times.map { @orphans.shift }
           end
 
-          private
-
-          ##
-          # Sanitize string and replace any invalid UTF-8 characters.
-          # If replacements are made, flag the LocalFile object as invalid.
-          def sanitize(str)
-            str.each_char.map do |char|
-              begin
-                char if !!char.unpack('U')
-              rescue
-                @invalid = true
-                "\xEF\xBF\xBD" # => "\uFFFD"
-              end
-            end.join
+          if mirror
+            Logger.info @orphans.map {|path|
+              "\s\s[removing] '#{ path }'"
+            }.join("\n")
+
+            begin
+              cloud_io.delete(@orphans)
+              "Deleted Files: #{ @orphans.count }"
+            rescue => err
+              Logger.warn Error.wrap(err, 'Delete Operation Failed')
+              "Attempted to Delete: #{ @orphans.count } " +
+              "(See log messages for actual results)"
+            end
+          else
+            Logger.info @orphans.map {|path|
+              "\s\s[orphaned] '#{ path }'"
+            }.join("\n")
+            "Orphaned Files: #{ @orphans.count }"
           end
+        end
 
-        end # class LocalFile
-
-      end # class Base < Syncer::Base
-    end # module Cloud
+      end
+    end
   end
 end

data/lib/backup/syncer/cloud/cloud_files.rb

@@ -1,77 +1,92 @@
 # encoding: utf-8
+require 'backup/cloud_io/cloud_files'
 
 module Backup
   module Syncer
     module Cloud
       class CloudFiles < Base
+        class Error < Backup::Error; end
 
         ##
         # Rackspace CloudFiles Credentials
-        attr_accessor :api_key, :username
+        attr_accessor :username, :api_key
 
         ##
         # Rackspace CloudFiles Container
         attr_accessor :container
 
         ##
-        # Rackspace AuthURL allows you to connect
-        # to a different Rackspace datacenter
-        # - https://auth.api.rackspacecloud.com     (Default: US)
-        # - https://lon.auth.api.rackspacecloud.com (UK)
+        # Rackspace AuthURL (optional)
         attr_accessor :auth_url
 
         ##
-        # Improve performance and avoid data transfer costs
-        # by setting @servicenet to `true`
-        # This only works if Backup runs on a Rackspace server
-        attr_accessor :servicenet
+        # Rackspace Region (optional)
+        attr_accessor :region
 
         ##
-        # Instantiates a new Cloud::CloudFiles Syncer.
-        #
-        # Pre-configured defaults specified in
-        # Configuration::Syncer::Cloud::CloudFiles
-        # are set via a super() call to Cloud::Base,
-        # which in turn will invoke Syncer::Base.
-        #
-        # Once pre-configured defaults and Cloud specific defaults are set,
-        # the block from the user's configuration file is evaluated.
-        def initialize(syncer_id = nil, &block)
+        # Rackspace Service Net
+        # (LAN-based transfers to avoid charges and improve performance)
+        attr_accessor :servicenet
+
+        def initialize(syncer_id = nil)
           super
 
-          instance_eval(&block) if block_given?
-          @path = path.sub(/^\//, '')
+          @servicenet ||= false
+
+          check_configuration
         end
 
         private
 
-        ##
-        # Established and creates a new Fog storage object for CloudFiles.
-        def connection
-          @connection ||= Fog::Storage.new(
-            :provider             => provider,
-            :rackspace_username   => username,
-            :rackspace_api_key    => api_key,
-            :rackspace_auth_url   => auth_url,
-            :rackspace_servicenet => servicenet
+        def cloud_io
+          @cloud_io ||= CloudIO::CloudFiles.new(
+            :username           => username,
+            :api_key            => api_key,
+            :auth_url           => auth_url,
+            :region             => region,
+            :servicenet         => servicenet,
+            :container          => container,
+            :max_retries        => max_retries,
+            :retry_waitsec      => retry_waitsec,
+            # Syncer can not use SLOs.
+            :segments_container => nil,
+            :segment_size       => 0
           )
         end
 
-        ##
-        # Creates a new @repository_object (container).
-        # Fetches it from Cloud Files if it already exists,
-        # otherwise it will create it first and fetch use that instead.
-        def repository_object
-          @repository_object ||= connection.directories.get(container) ||
-            connection.directories.create(:key => container)
+        def get_remote_files(remote_base)
+          hash = {}
+          cloud_io.objects(remote_base).each do |object|
+            relative_path = object.name.sub(remote_base + '/', '')
+            hash[relative_path] = object.hash
+          end
+          hash
         end
 
-        ##
-        # This is the provider that Fog uses for the Cloud Files
-        def provider
-          "Rackspace"
+        def check_configuration
+          required = %w{ username api_key container }
+          raise Error, <<-EOS if required.map {|name| send(name) }.any?(&:nil?)
+            Configuration Error
+            #{ required.map {|name| "##{ name }"}.join(', ') } are all required
+          EOS
         end
 
+        attr_deprecate :concurrency_type, :version => '3.7.0',
+                       :message => 'Use #thread_count instead.',
+                       :action => lambda {|klass, val|
+                         if val == :threads
+                           klass.thread_count = 2 unless klass.thread_count
+                         else
+                           klass.thread_count = 0
+                         end
+                       }
+
+        attr_deprecate :concurrency_level, :version => '3.7.0',
+                       :message => 'Use #thread_count instead.',
+                       :action => lambda {|klass, val|
+                         klass.thread_count = val unless klass.thread_count == 0
+                       }
+
       end # class Cloudfiles < Base
     end # module Cloud
   end