backupii 0.1.0.pre.alpha.1

data/lib/backup/syncer/cloud/base.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+module Backup
+  module Syncer
+    module Cloud
+      class Error < Backup::Error; end
+
+      class Base < Syncer::Base
+        MUTEX = Mutex.new
+
+        ##
+        # Number of threads to use for concurrency.
+        #
+        # Default: 0 (no concurrency)
+        attr_accessor :thread_count
+
+        ##
+        # Number of times to retry failed operations.
+        #
+        # Default: 10
+        attr_accessor :max_retries
+
+        ##
+        # Time in seconds to pause before each retry.
+        #
+        # Default: 30
+        attr_accessor :retry_waitsec
+
+        def initialize(syncer_id = nil, &block)
+          super
+          instance_eval(&block) if block_given?
+
+          @thread_count   ||= 0
+          @max_retries    ||= 10
+          @retry_waitsec  ||= 30
+
+          @path ||= "backups"
+          @path = path.sub(%r{^/}, "")
+        end
+
+        def perform!
+          log!(:started)
+          @transfer_count = 0
+          @unchanged_count = 0
+          @skipped_count = 0
+          @orphans = thread_count > 0 ? Queue.new : []
+
+          directories.each { |dir| sync_directory(dir) }
+          orphans_result = process_orphans
+
+          Logger.info "\nSummary:"
+          Logger.info "\s\sTransferred Files: #{@transfer_count}"
+          Logger.info "\s\s#{orphans_result}"
+          Logger.info "\s\sUnchanged Files: #{@unchanged_count}"
+          if @skipped_count > 0
+            Logger.warn "\s\sSkipped Files: #{@skipped_count}"
+          end
+          log!(:finished)
+        end
+
+        private
+
+        def sync_directory(dir)
+          remote_base = path.empty? ? File.basename(dir) :
+                                      File.join(path, File.basename(dir))
+          Logger.info "Gathering remote data for '#{remote_base}'..."
+          remote_files = get_remote_files(remote_base)
+
+          Logger.info("Gathering local data for '#{File.expand_path(dir)}'...")
+          local_files = LocalFile.find(dir, excludes)
+
+          relative_paths = (local_files.keys | remote_files.keys).sort
+          if relative_paths.empty?
+            Logger.info "No local or remote files found"
+          else
+            Logger.info "Syncing..."
+            sync_block = proc do |relative_path|
+              local_file  = local_files[relative_path]
+              remote_md5  = remote_files[relative_path]
+              remote_path = File.join(remote_base, relative_path)
+              sync_file(local_file, remote_path, remote_md5)
+            end
+
+            if thread_count > 0
+              sync_in_threads(relative_paths, sync_block)
+            else
+              relative_paths.each(&sync_block)
+            end
+          end
+        end
+
+        def sync_in_threads(relative_paths, sync_block)
+          queue = Queue.new
+          queue << relative_paths.shift until relative_paths.empty?
+          num_threads = [thread_count, queue.size].min
+          Logger.info "\s\sUsing #{num_threads} Threads"
+          threads = Array.new(num_threads) do
+            Thread.new do
+              loop do
+                path = queue.shift(true) rescue nil
+                path ? sync_block.call(path) : break
+              end
+            end
+          end
+
+          # abort if any thread raises an exception
+          while threads.any?(&:alive?)
+            if threads.any? { |thr| thr.status.nil? }
+              threads.each(&:kill)
+              Thread.pass while threads.any?(&:alive?)
+              break
+            end
+            sleep num_threads * 0.1
+          end
+          threads.each(&:join)
+        end
+
+        # If an exception is raised in multiple threads, only the exception
+        # raised in the first thread that Thread#join is called on will be
+        # handled. So all exceptions are logged first with their details,
+        # then a generic exception is raised.
+        def sync_file(local_file, remote_path, remote_md5)
+          if local_file && File.exist?(local_file.path)
+            if local_file.md5 == remote_md5
+              MUTEX.synchronize { @unchanged_count += 1 }
+            else
+              Logger.info("\s\s[transferring] '#{remote_path}'")
+              begin
+                cloud_io.upload(local_file.path, remote_path)
+                MUTEX.synchronize { @transfer_count += 1 }
+              rescue CloudIO::FileSizeError => err
+                MUTEX.synchronize { @skipped_count += 1 }
+                Logger.warn Error.wrap(err, "Skipping '#{remote_path}'")
+              rescue => err
+                Logger.error(err)
+                raise Error, <<-EOS
+                  Syncer Failed!
+                  See the Retry [info] and [error] messages (if any)
+                  for details on each failed operation.
+                EOS
+              end
+            end
+          elsif remote_md5
+            @orphans << remote_path
+          end
+        end
+
+        def process_orphans
+          if @orphans.empty?
+            return mirror ? "Deleted Files: 0" : "Orphaned Files: 0"
+          end
+
+          if @orphans.is_a?(Queue)
+            @orphans = Array.new(@orphans.size) { @orphans.shift }
+          end
+
+          if mirror
+            Logger.info @orphans.map { |path|
+              "\s\s[removing] '#{path}'"
+            }.join("\n")
+
+            begin
+              cloud_io.delete(@orphans)
+              "Deleted Files: #{@orphans.count}"
+            rescue => err
+              Logger.warn Error.wrap(err, "Delete Operation Failed")
+              "Attempted to Delete: #{@orphans.count} " \
+              "(See log messages for actual results)"
+            end
+          else
+            Logger.info @orphans.map { |path|
+              "\s\s[orphaned] '#{path}'"
+            }.join("\n")
+            "Orphaned Files: #{@orphans.count}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/backup/syncer/cloud/cloud_files.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "backup/cloud_io/cloud_files"
+
+module Backup
+  module Syncer
+    module Cloud
+      class CloudFiles < Base
+        class Error < Backup::Error; end
+
+        ##
+        # Rackspace CloudFiles Credentials
+        attr_accessor :username, :api_key
+
+        ##
+        # Rackspace CloudFiles Container
+        attr_accessor :container
+
+        ##
+        # Rackspace AuthURL (optional)
+        attr_accessor :auth_url
+
+        ##
+        # Rackspace Region (optional)
+        attr_accessor :region
+
+        ##
+        # Rackspace Service Net
+        # (LAN-based transfers to avoid charges and improve performance)
+        attr_accessor :servicenet
+
+        ##
+        # Additional options to pass along to fog.
+        # e.g. Fog::Storage.new({ :provider => 'Rackspace' }.merge(fog_options))
+        attr_accessor :fog_options
+
+        def initialize(syncer_id = nil)
+          super
+
+          @servicenet ||= false
+
+          check_configuration
+        end
+
+        private
+
+        def cloud_io
+          @cloud_io ||= CloudIO::CloudFiles.new(
+            username: username,
+            api_key: api_key,
+            auth_url: auth_url,
+            region: region,
+            servicenet: servicenet,
+            container: container,
+            max_retries: max_retries,
+            retry_waitsec: retry_waitsec,
+            # Syncer can not use SLOs.
+            segments_container: nil,
+            segment_size: 0,
+            fog_options: fog_options
+          )
+        end
+
+        def get_remote_files(remote_base)
+          hash = {}
+          cloud_io.objects(remote_base).each do |object|
+            relative_path = object.name.sub(remote_base + "/", "")
+            hash[relative_path] = object.hash
+          end
+          hash
+        end
+
+        def check_configuration
+          required = %w[username api_key container]
+          raise Error, <<-EOS if required.map { |name| send(name) }.any?(&:nil?)
+            Configuration Error
+            #{required.map { |name| "##{name}" }.join(", ")} are all required
+          EOS
+        end
+      end # class Cloudfiles < Base
+    end # module Cloud
+  end
+end

data/lib/backup/syncer/cloud/local_file.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require "digest/md5"
+
+module Backup
+  module Syncer
+    module Cloud
+      class LocalFile
+        attr_reader :path
+        attr_accessor :md5
+
+        class << self
+          # Returns a Hash of LocalFile objects for each file within +dir+,
+          # except those matching any of the +excludes+.
+          # Hash keys are the file's path relative to +dir+.
+          def find(dir, excludes = [])
+            dir = File.expand_path(dir)
+            hash = {}
+            find_md5(dir, excludes).each do |file|
+              hash[file.path.sub(dir + "/", "")] = file
+            end
+            hash
+          end
+
+          # Return a new LocalFile object if it's valid.
+          # Otherwise, log a warning and return nil.
+          def new(*args)
+            file = super
+            if file.invalid?
+              Logger.warn("\s\s[skipping] #{file.path}\n" \
+                          "\s\sPath Contains Invalid UTF-8 byte sequences")
+              file = nil
+            end
+            file
+          end
+
+          private
+
+          # Returns an Array of file paths and their md5 hashes.
+          def find_md5(dir, excludes)
+            found = []
+            (Dir.entries(dir) - %w[. ..]).map { |e| File.join(dir, e) }.each do |path|
+              if File.directory?(path)
+                unless exclude?(excludes, path)
+                  found += find_md5(path, excludes)
+                end
+              elsif File.file?(path)
+                if (file = new(path))
+                  unless exclude?(excludes, file.path)
+                    file.md5 = Digest::MD5.file(file.path).hexdigest
+                    found << file
+                  end
+                end
+              end
+            end
+            found
+          end
+
+          # Returns true if +path+ matches any of the +excludes+.
+          # Note this can not be called if +path+ includes invalid UTF-8.
+          def exclude?(excludes, path)
+            excludes.any? do |ex|
+              if ex.is_a?(String)
+                File.fnmatch?(ex, path)
+              elsif ex.is_a?(Regexp)
+                ex.match(path)
+              end
+            end
+          end
+        end
+
+        # If +path+ contains invalid UTF-8, it will be sanitized
+        # and the LocalFile object will be flagged as invalid.
+        # This is done so @file.path may be logged.
+        def initialize(path)
+          @path = sanitize(path)
+        end
+
+        def invalid?
+          !!@invalid
+        end
+
+        private
+
+        def sanitize(str)
+          str.each_char.map do |char|
+            begin
+              char.unpack("U")
+              char
+            rescue
+              @invalid = true
+              "\xEF\xBF\xBD" # => "\uFFFD"
+            end
+          end.join
+        end
+      end
+    end
+  end
+end

data/lib/backup/syncer/cloud/s3.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require "backup/cloud_io/s3"
+
+module Backup
+  module Syncer
+    module Cloud
+      class S3 < Base
+        class Error < Backup::Error; end
+
+        ##
+        # Amazon Simple Storage Service (S3) Credentials
+        attr_accessor :access_key_id, :secret_access_key, :use_iam_profile
+
+        ##
+        # Amazon S3 bucket name
+        attr_accessor :bucket
+
+        ##
+        # Region of the specified S3 bucket
+        attr_accessor :region
+
+        ##
+        # Encryption algorithm to use for Amazon Server-Side Encryption
+        #
+        # Supported values:
+        #
+        # - :aes256
+        #
+        # Default: nil
+        attr_accessor :encryption
+
+        ##
+        # Storage class to use for the S3 objects uploaded
+        #
+        # Supported values:
+        #
+        # - :standard (default)
+        # - :reduced_redundancy
+        #
+        # Default: :standard
+        attr_accessor :storage_class
+
+        ##
+        # Additional options to pass along to fog.
+        # e.g. Fog::Storage.new({ :provider => 'AWS' }.merge(fog_options))
+        attr_accessor :fog_options
+
+        def initialize(syncer_id = nil)
+          super
+
+          @storage_class ||= :standard
+
+          check_configuration
+        end
+
+        private
+
+        def cloud_io
+          @cloud_io ||= CloudIO::S3.new(
+            access_key_id: access_key_id,
+            secret_access_key: secret_access_key,
+            use_iam_profile: use_iam_profile,
+            bucket: bucket,
+            region: region,
+            encryption: encryption,
+            storage_class: storage_class,
+            max_retries: max_retries,
+            retry_waitsec: retry_waitsec,
+            # Syncer can not use multipart upload.
+            chunk_size: 0,
+            fog_options: fog_options
+          )
+        end
+
+        def get_remote_files(remote_base)
+          hash = {}
+          cloud_io.objects(remote_base).each do |object|
+            relative_path = object.key.sub(remote_base + "/", "")
+            hash[relative_path] = object.etag
+          end
+          hash
+        end
+
+        def check_configuration
+          required =
+            if use_iam_profile
+              %w[bucket]
+            else
+              %w[access_key_id secret_access_key bucket]
+            end
+
+          if required.map { |name| send(name) }.any?(&:nil?)
+            raise Error, <<-EOS
+              Configuration Error
+              #{required.map { |name| "##{name}" }.join(", ")} are all required
+            EOS
+          end
+
+          if encryption && encryption.to_s.upcase != "AES256"
+            raise Error, <<-EOS
+              Configuration Error
+              #encryption must be :aes256 or nil
+            EOS
+          end
+          classes = %w[STANDARD REDUCED_REDUNDANCY]
+
+          unless classes.include?(storage_class.to_s.upcase)
+            raise Error, <<-EOS
+              Configuration Error
+              #storage_class must be :standard or :reduced_redundancy
+            EOS
+          end
+        end
+      end # Class S3 < Base
+    end # module Cloud
+  end
+end

data/lib/backup/syncer/rsync/base.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Backup
+  module Syncer
+    module RSync
+      class Base < Syncer::Base
+        ##
+        # Additional String or Array of options for the rsync cli
+        attr_accessor :additional_rsync_options
+        attr_accessor :archive
+
+        def initialize(syncer_id = nil, &block)
+          super
+          instance_eval(&block) if block_given?
+
+          @path ||= "~/backups"
+          @archive = @archive.nil? ? true : @archive
+        end
+
+        private
+
+        ##
+        # Common base command for Local/Push/Pull
+        def rsync_command
+          utility(:rsync).dup <<
+            archive_option <<
+            mirror_option <<
+            exclude_option <<
+            " #{Array(additional_rsync_options).join(" ")}".rstrip
+        end
+
+        def mirror_option
+          mirror ? " --delete" : ""
+        end
+
+        def archive_option
+          archive ? " --archive" : ""
+        end
+
+        def exclude_option
+          excludes.map { |pattern| " --exclude='#{pattern}'" }.join
+        end
+
+        ##
+        # Each path is expanded, since these refer to local paths and are
+        # being shell-quoted. This will also remove any trailing `/` from
+        # each path, as we don't want rsync's "trailing / on source directories"
+        # behavior. This method is used by RSync::Local and RSync::Push.
+        def paths_to_push
+          directories.map { |dir| "'#{File.expand_path(dir)}'" }.join(" ")
+        end
+      end
+    end
+  end
+end

data/lib/backup/syncer/rsync/local.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Backup
+  module Syncer
+    module RSync
+      class Local < Base
+        def perform!
+          log!(:started)
+
+          create_dest_path!
+          run("#{rsync_command} #{paths_to_push} '#{dest_path}'")
+
+          log!(:finished)
+        end
+
+        private
+
+        # Expand path, since this is local and shell-quoted.
+        def dest_path
+          @dest_path ||= File.expand_path(path)
+        end
+
+        def create_dest_path!
+          FileUtils.mkdir_p dest_path
+        end
+      end
+    end
+  end
+end

data/lib/backup/syncer/rsync/pull.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Backup
+  module Syncer
+    module RSync
+      class Pull < Push
+        def perform!
+          log!(:started)
+          write_password_file!
+
+          create_dest_path!
+          run("#{rsync_command} #{host_options}#{paths_to_pull} " \
+              "'#{dest_path}'")
+
+          log!(:finished)
+        ensure
+          remove_password_file!
+        end
+
+        private
+
+        ##
+        # Returns the syntax for pulling multiple paths from the remote host.
+        # e.g.
+        #   rsync -a -e "ssh -p 22" host:'path1' :'path2' '/dest'
+        #   rsync -a rsync_user@host::'modname/path1' ::'modname/path2' '/dest'
+        #
+        # Remove any preceeding '~/', since these paths are on the remote.
+        # Also remove any trailing `/`, since we don't want rsync's
+        # "trailing / on source directories" behavior.
+        def paths_to_pull
+          sep = mode == :ssh ? ":" : "::"
+          directories.map do |dir|
+            "#{sep}'#{dir.sub(%r{^~/}, "").sub(%r{/$}, "")}'"
+          end.join(" ").sub(%r{^#{sep}}, "")
+        end
+
+        # Expand path, since this is local and shell-quoted.
+        def dest_path
+          @dest_path ||= File.expand_path(path)
+        end
+
+        def create_dest_path!
+          FileUtils.mkdir_p dest_path
+        end
+      end
+    end
+  end
+end