backup-agoddard 3.0.27

data/lib/backup/storage/s3.rb

@@ -0,0 +1,94 @@
+# encoding: utf-8
+
+##
+# Only load the Fog gem when the Backup::Storage::S3 class is loaded
+Backup::Dependency.load('fog')
+
+module Backup
+  module Storage
+    class S3 < Base
+
+      ##
+      # Amazon Simple Storage Service (S3) Credentials
+      attr_accessor :access_key_id, :secret_access_key
+
+      ##
+      # Amazon S3 bucket name and path
+      attr_accessor :bucket, :path
+
+      ##
+      # Region of the specified S3 bucket
+      attr_accessor :region
+
+      ##
+      # Creates a new instance of the storage object
+      def initialize(model, storage_id = nil, &block)
+        super(model, storage_id)
+
+        @path ||= 'backups'
+
+        instance_eval(&block) if block_given?
+      end
+
+      private
+
+      ##
+      # This is the provider that Fog uses for the S3 Storage
+      def provider
+        'AWS'
+      end
+
+      ##
+      # Establishes a connection to Amazon S3
+      def connection
+        @connection ||= Fog::Storage.new(
+          :provider               => provider,
+          :aws_access_key_id      => access_key_id,
+          :aws_secret_access_key  => secret_access_key,
+          :region                 => region
+        )
+      end
+
+      def remote_path_for(package)
+        super(package).sub(/^\//, '')
+      end
+
+      ##
+      # Transfers the archived file to the specified Amazon S3 bucket
+      def transfer!
+        remote_path = remote_path_for(@package)
+
+        connection.sync_clock
+
+        files_to_transfer_for(@package) do |local_file, remote_file|
+          Logger.message "#{storage_name} started transferring " +
+              "'#{ local_file }' to bucket '#{ bucket }'."
+
+          File.open(File.join(local_path, local_file), 'r') do |file|
+            connection.put_object(
+              bucket, File.join(remote_path, remote_file), file
+            )
+          end
+        end
+      end
+
+      ##
+      # Removes the transferred archive file(s) from the storage location.
+      # Any error raised will be rescued during Cycling
+      # and a warning will be logged, containing the error message.
+      def remove!(package)
+        remote_path = remote_path_for(package)
+
+        connection.sync_clock
+
+        transferred_files_for(package) do |local_file, remote_file|
+          Logger.message "#{storage_name} started removing " +
+              "'#{ local_file }' from bucket '#{ bucket }'."
+
+          connection.delete_object(bucket, File.join(remote_path, remote_file))
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/storage/scp.rb

@@ -0,0 +1,99 @@
+# encoding: utf-8
+
+##
+# Only load the Net::SSH and Net::SCP library/gems
+# when the Backup::Storage::SCP class is loaded
+Backup::Dependency.load('net-ssh')
+Backup::Dependency.load('net-scp')
+
+module Backup
+  module Storage
+    class SCP < Base
+
+      ##
+      # Server credentials
+      attr_accessor :username, :password
+
+      ##
+      # Server IP Address and SCP port
+      attr_accessor :ip, :port
+
+      ##
+      # Path to store backups to
+      attr_accessor :path
+
+      ##
+      # Creates a new instance of the storage object
+      def initialize(model, storage_id = nil, &block)
+        super(model, storage_id)
+
+        @port ||= 22
+        @path ||= 'backups'
+
+        instance_eval(&block) if block_given?
+
+        @path = path.sub(/^\~\//, '')
+      end
+
+      private
+
+      ##
+      # Establishes a connection to the remote server
+      # and yields the Net::SSH connection.
+      # Net::SCP will use this connection to transfer backups
+      def connection
+        Net::SSH.start(
+          ip, username, :password => password, :port => port
+        ) {|ssh| yield ssh }
+      end
+
+      ##
+      # Transfers the archived file to the specified remote server
+      def transfer!
+        remote_path = remote_path_for(@package)
+
+        connection do |ssh|
+          ssh.exec!("mkdir -p '#{ remote_path }'")
+
+          files_to_transfer_for(@package) do |local_file, remote_file|
+            Logger.message "#{storage_name} started transferring " +
+                "'#{local_file}' to '#{ip}'."
+
+            ssh.scp.upload!(
+              File.join(local_path, local_file),
+              File.join(remote_path, remote_file)
+            )
+          end
+        end
+      end
+
+      ##
+      # Removes the transferred archive file(s) from the storage location.
+      # Any error raised will be rescued during Cycling
+      # and a warning will be logged, containing the error message.
+      def remove!(package)
+        remote_path = remote_path_for(package)
+
+        messages = []
+        transferred_files_for(package) do |local_file, remote_file|
+          messages << "#{storage_name} started removing " +
+              "'#{local_file}' from '#{ip}'."
+        end
+        Logger.message messages.join("\n")
+
+        errors = []
+        connection do |ssh|
+          ssh.exec!("rm -r '#{remote_path}'") do |ch, stream, data|
+            errors << data if stream == :stderr
+          end
+        end
+        unless errors.empty?
+          raise Errors::Storage::SCP::SSHError,
+            "Net::SSH reported the following errors:\n" +
+              errors.join("\n")
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/storage/sftp.rb

@@ -0,0 +1,108 @@
+# encoding: utf-8
+
+##
+# Only load the Net::SFTP library/gem when the Backup::Storage::SFTP class is loaded
+Backup::Dependency.load('net-ssh')
+Backup::Dependency.load('net-sftp')
+
+module Backup
+  module Storage
+    class SFTP < Base
+
+      ##
+      # Server credentials
+      attr_accessor :username, :password
+
+      ##
+      # Server IP Address and SFTP port
+      attr_accessor :ip, :port
+
+      ##
+      # Path to store backups to
+      attr_accessor :path
+
+      ##
+      # Creates a new instance of the storage object
+      def initialize(model, storage_id = nil, &block)
+        super(model, storage_id)
+
+        @port ||= 22
+        @path ||= 'backups'
+
+        instance_eval(&block) if block_given?
+
+        @path = path.sub(/^\~\//, '')
+      end
+
+      private
+
+      ##
+      # Establishes a connection to the remote server
+      def connection
+        Net::SFTP.start(
+          ip, username,
+          :password => password,
+          :port     => port
+        ) {|sftp| yield sftp }
+      end
+
+      ##
+      # Transfers the archived file to the specified remote server
+      def transfer!
+        remote_path = remote_path_for(@package)
+
+        connection do |sftp|
+          create_remote_path(remote_path, sftp)
+
+          files_to_transfer_for(@package) do |local_file, remote_file|
+            Logger.message "#{storage_name} started transferring " +
+                "'#{ local_file }' to '#{ ip }'."
+
+            sftp.upload!(
+              File.join(local_path, local_file),
+              File.join(remote_path, remote_file)
+            )
+          end
+        end
+      end
+
+      ##
+      # Removes the transferred archive file(s) from the storage location.
+      # Any error raised will be rescued during Cycling
+      # and a warning will be logged, containing the error message.
+      def remove!(package)
+        remote_path = remote_path_for(package)
+
+        connection do |sftp|
+          transferred_files_for(package) do |local_file, remote_file|
+            Logger.message "#{storage_name} started removing " +
+                "'#{ local_file }' from '#{ ip }'."
+
+            sftp.remove!(File.join(remote_path, remote_file))
+          end
+
+          sftp.rmdir!(remote_path)
+        end
+      end
+
+      ##
+      # Creates (if they don't exist yet) all the directories on the remote
+      # server in order to upload the backup file. Net::SFTP does not support
+      # paths to directories that don't yet exist when creating new
+      # directories. Instead, we split the parts up in to an array (for each
+      # '/') and loop through that to create the directories one by one.
+      # Net::SFTP raises an exception when the directory it's trying to create
+      # already exists, so we have rescue it
+      def create_remote_path(remote_path, sftp)
+        path_parts = Array.new
+        remote_path.split('/').each do |path_part|
+          path_parts << path_part
+          begin
+            sftp.mkdir!(path_parts.join('/'))
+          rescue Net::SFTP::StatusException; end
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/syncer/base.rb

@@ -0,0 +1,46 @@
+# encoding: utf-8
+
+module Backup
+  module Syncer
+    class Base
+      include Backup::CLI::Helpers
+      include Backup::Configuration::Helpers
+
+      ##
+      # Path to store the synced files/directories to
+      attr_accessor :path
+
+      ##
+      # Flag for mirroring the files/directories
+      attr_accessor :mirror
+
+      def initialize
+        load_defaults!
+
+        @path               ||= 'backups'
+        @mirror             ||= false
+        @directories          = Array.new
+      end
+
+      ##
+      # Syntactical suger for the DSL for adding directories
+      def directories(&block)
+        return @directories unless block_given?
+        instance_eval(&block)
+      end
+
+      ##
+      # Adds a path to the @directories array
+      def add(path)
+        @directories << path
+      end
+
+      private
+
+      def syncer_name
+        self.class.to_s.sub('Backup::', '')
+      end
+
+    end
+  end
+end

data/lib/backup/syncer/cloud/base.rb

@@ -0,0 +1,247 @@
+# encoding: utf-8
+
+##
+# Only load the Fog gem, along with the Parallel gem, when the
+# Backup::Syncer::Cloud class is loaded
+Backup::Dependency.load('fog')
+Backup::Dependency.load('parallel')
+
+module Backup
+  module Syncer
+    module Cloud
+      class Base < Syncer::Base
+
+        ##
+        # Create a Mutex to synchronize certain parts of the code
+        # in order to prevent race conditions or broken STDOUT.
+        MUTEX = Mutex.new
+
+        ##
+        # Concurrency setting - defaults to false, but can be set to:
+        # - :threads
+        # - :processes
+        attr_accessor :concurrency_type
+
+        ##
+        # Concurrency level - the number of threads or processors to use.
+        # Defaults to 2.
+        attr_accessor :concurrency_level
+
+        ##
+        # Instantiates a new Cloud Syncer object for either
+        # the Cloud::S3 or Cloud::CloudFiles Syncer.
+        #
+        # Pre-configured defaults specified in either
+        # Configuration::Syncer::Cloud::S3 or
+        # Configuration::Syncer::Cloud::CloudFiles
+        # are set via a super() call to Syncer::Base.
+        #
+        # If not specified in the pre-configured defaults,
+        # the Cloud specific defaults are set here before evaluating
+        # any block provided in the user's configuration file.
+        def initialize
+          super
+
+          @concurrency_type  ||= false
+          @concurrency_level ||= 2
+        end
+
+        ##
+        # Performs the Sync operation
+        def perform!
+          Logger.message(
+            "#{ syncer_name } started the syncing process:\n" +
+            "\s\sConcurrency: #{ @concurrency_type } Level: #{ @concurrency_level }"
+          )
+
+          @directories.each do |directory|
+            SyncContext.new(
+              File.expand_path(directory), repository_object, @path
+            ).sync! @mirror, @concurrency_type, @concurrency_level
+          end
+
+          Logger.message("#{ syncer_name } Syncing Complete!")
+        end
+
+        private
+
+        class SyncContext
+          attr_reader :directory, :bucket, :path, :remote_base
+
+          ##
+          # Creates a new SyncContext object which handles a single directory
+          # from the Syncer::Base @directories array.
+          def initialize(directory, bucket, path)
+            @directory, @bucket, @path = directory, bucket, path
+            @remote_base = File.join(path, File.basename(directory))
+          end
+
+          ##
+          # Performs the sync operation using the provided techniques
+          # (mirroring/concurrency).
+          def sync!(mirror = false, concurrency_type = false, concurrency_level = 2)
+            block = Proc.new { |relative_path| sync_file relative_path, mirror }
+
+            case concurrency_type
+            when FalseClass
+              all_file_names.each(&block)
+            when :threads
+              Parallel.each all_file_names,
+                  :in_threads => concurrency_level, &block
+            when :processes
+              Parallel.each all_file_names,
+                  :in_processes => concurrency_level, &block
+            else
+              raise Errors::Syncer::Cloud::ConfigurationError,
+                  "Unknown concurrency_type setting: #{ concurrency_type.inspect }"
+            end
+          end
+
+          private
+
+          ##
+          # Gathers all the relative paths to the local files
+          # and merges them with the , removing
+          # duplicate keys if any, and sorts the in alphabetical order.
+          def all_file_names
+            @all_file_names ||= (local_files.keys | remote_files.keys).sort
+          end
+
+          ##
+          # Returns a Hash of local files, validated to ensure the path
+          # does not contain invalid UTF-8 byte sequences.
+          # The keys are the filesystem paths, relative to @directory.
+          # The values are the LocalFile objects for that given file.
+          def local_files
+            @local_files ||= begin
+              hash = {}
+              local_hashes.lines.map do |line|
+                LocalFile.new(@directory, line)
+              end.compact.each do |file|
+                hash.merge!(file.relative_path => file)
+              end
+              hash
+            end
+          end
+
+          ##
+          # Returns a String of file paths and their md5 hashes.
+          def local_hashes
+            Logger.message("\s\sGenerating checksums for '#{ @directory }'")
+            `find '#{ @directory }' -print0 | xargs -0 openssl md5 2> /dev/null`
+          end
+
+          ##
+          # Returns a Hash of remote files
+          # The keys are the remote paths, relative to @remote_base
+          # The values are the Fog file objects for that given file
+          def remote_files
+            @remote_files ||= begin
+              hash = {}
+              @bucket.files.all(:prefix => @remote_base).each do |file|
+                hash.merge!(file.key.sub("#{ @remote_base }/", '') => file)
+              end
+              hash
+            end
+          end
+
+          ##
+          # Performs a sync operation on a file. When mirroring is enabled
+          # and a local file has been removed since the last sync, it will also
+          # remove it from the remote location. It will no upload files that
+          # have not changed since the last sync. Checks are done using an md5
+          # hash. If a file has changed, or has been newly added, the file will
+          # be transferred/overwritten.
+          def sync_file(relative_path, mirror)
+            local_file  = local_files[relative_path]
+            remote_file = remote_files[relative_path]
+            remote_path = File.join(@remote_base, relative_path)
+
+            if local_file && File.exist?(local_file.path)
+              unless remote_file && remote_file.etag == local_file.md5
+                MUTEX.synchronize {
+                  Logger.message("\s\s[transferring] '#{ remote_path }'")
+                }
+                File.open(local_file.path, 'r') do |file|
+                  @bucket.files.create(
+                    :key  => remote_path,
+                    :body => file
+                  )
+                end
+              else
+                MUTEX.synchronize {
+                  Logger.message("\s\s[skipping] '#{ remote_path }'")
+                }
+              end
+            elsif remote_file
+              if mirror
+                MUTEX.synchronize {
+                  Logger.message("\s\s[removing] '#{ remote_path }'")
+                }
+                remote_file.destroy
+              else
+                MUTEX.synchronize {
+                  Logger.message("\s\s[leaving] '#{ remote_path }'")
+                }
+              end
+            end
+          end
+        end # class SyncContext
+
+        class LocalFile
+          attr_reader :path, :relative_path, :md5
+
+          ##
+          # Return a new LocalFile object if it's valid.
+          # Otherwise, log a warning and return nil.
+          def self.new(*args)
+            local_file = super(*args)
+            if local_file.invalid?
+              Logger.warn(
+                "\s\s[skipping] #{ local_file.path }\n" +
+                "\s\sPath Contains Invalid UTF-8 byte sequences"
+              )
+              return nil
+            end
+            local_file
+          end
+
+          ##
+          # Creates a new LocalFile object using the given directory and line
+          # from the md5 hash checkup. This object figures out the path,
+          # relative_path and md5 hash for the file.
+          def initialize(directory, line)
+            @invalid = false
+            @directory = sanitize(directory)
+            line = sanitize(line).chomp
+            @path = line.slice(4..-36)
+            @md5 = line.slice(-32..-1)
+            @relative_path = @path.sub(@directory + '/', '')
+          end
+
+          def invalid?
+            @invalid
+          end
+
+          private
+
+          ##
+          # Sanitize string and replace any invalid UTF-8 characters.
+          # If replacements are made, flag the LocalFile object as invalid.
+          def sanitize(str)
+            str.each_char.map do |char|
+              begin
+                char if !!char.unpack('U')
+              rescue
+                @invalid = true
+                "\xEF\xBF\xBD" # => "\uFFFD"
+              end
+            end.join
+          end
+
+        end # class LocalFile
+
+      end # class Base < Syncer::Base
+    end # module Cloud
+  end
+end