backup_checksum 3.0.23

data/lib/backup/storage/s3.rb

@@ -0,0 +1,94 @@
+# encoding: utf-8
+
+##
+# Only load the Fog gem when the Backup::Storage::S3 class is loaded
+Backup::Dependency.load('fog')
+
+module Backup
+  module Storage
+    class S3 < Base
+
+      ##
+      # Amazon Simple Storage Service (S3) Credentials
+      attr_accessor :access_key_id, :secret_access_key
+
+      ##
+      # Amazon S3 bucket name and path
+      attr_accessor :bucket, :path
+
+      ##
+      # Region of the specified S3 bucket
+      attr_accessor :region
+
+      ##
+      # Creates a new instance of the storage object
+      def initialize(model, storage_id = nil, &block)
+        super(model, storage_id)
+
+        @path ||= 'backups'
+
+        instance_eval(&block) if block_given?
+      end
+
+      private
+
+      ##
+      # This is the provider that Fog uses for the S3 Storage
+      def provider
+        'AWS'
+      end
+
+      ##
+      # Establishes a connection to Amazon S3
+      def connection
+        @connection ||= Fog::Storage.new(
+          :provider               => provider,
+          :aws_access_key_id      => access_key_id,
+          :aws_secret_access_key  => secret_access_key,
+          :region                 => region
+        )
+      end
+
+      def remote_path_for(package)
+        super(package).sub(/^\//, '')
+      end
+
+      ##
+      # Transfers the archived file to the specified Amazon S3 bucket
+      def transfer!
+        remote_path = remote_path_for(@package)
+
+        connection.sync_clock
+
+        files_to_transfer_for(@package) do |local_file, remote_file|
+          Logger.message "#{storage_name} started transferring " +
+              "'#{ local_file }' to bucket '#{ bucket }'."
+
+          File.open(File.join(local_path, local_file), 'r') do |file|
+            connection.put_object(
+              bucket, File.join(remote_path, remote_file), file
+            )
+          end
+        end
+      end
+
+      ##
+      # Removes the transferred archive file(s) from the storage location.
+      # Any error raised will be rescued during Cycling
+      # and a warning will be logged, containing the error message.
+      def remove!(package)
+        remote_path = remote_path_for(package)
+
+        connection.sync_clock
+
+        transferred_files_for(package) do |local_file, remote_file|
+          Logger.message "#{storage_name} started removing " +
+              "'#{ local_file }' from bucket '#{ bucket }'."
+
+          connection.delete_object(bucket, File.join(remote_path, remote_file))
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/storage/scp.rb

@@ -0,0 +1,99 @@
+# encoding: utf-8
+
+##
+# Only load the Net::SSH and Net::SCP library/gems
+# when the Backup::Storage::SCP class is loaded
+Backup::Dependency.load('net-ssh')
+Backup::Dependency.load('net-scp')
+
+module Backup
+  module Storage
+    class SCP < Base
+
+      ##
+      # Server credentials
+      attr_accessor :username, :password
+
+      ##
+      # Server IP Address and SCP port
+      attr_accessor :ip, :port
+
+      ##
+      # Path to store backups to
+      attr_accessor :path
+
+      ##
+      # Creates a new instance of the storage object
+      def initialize(model, storage_id = nil, &block)
+        super(model, storage_id)
+
+        @port ||= 22
+        @path ||= 'backups'
+
+        instance_eval(&block) if block_given?
+
+        @path = path.sub(/^\~\//, '')
+      end
+
+      private
+
+      ##
+      # Establishes a connection to the remote server
+      # and yields the Net::SSH connection.
+      # Net::SCP will use this connection to transfer backups
+      def connection
+        Net::SSH.start(
+          ip, username, :password => password, :port => port
+        ) {|ssh| yield ssh }
+      end
+
+      ##
+      # Transfers the archived file to the specified remote server
+      def transfer!
+        remote_path = remote_path_for(@package)
+
+        connection do |ssh|
+          ssh.exec!("mkdir -p '#{ remote_path }'")
+
+          files_to_transfer_for(@package) do |local_file, remote_file|
+            Logger.message "#{storage_name} started transferring " +
+                "'#{local_file}' to '#{ip}'."
+
+            ssh.scp.upload!(
+              File.join(local_path, local_file),
+              File.join(remote_path, remote_file)
+            )
+          end
+        end
+      end
+
+      ##
+      # Removes the transferred archive file(s) from the storage location.
+      # Any error raised will be rescued during Cycling
+      # and a warning will be logged, containing the error message.
+      def remove!(package)
+        remote_path = remote_path_for(package)
+
+        messages = []
+        transferred_files_for(package) do |local_file, remote_file|
+          messages << "#{storage_name} started removing " +
+              "'#{local_file}' from '#{ip}'."
+        end
+        Logger.message messages.join("\n")
+
+        errors = []
+        connection do |ssh|
+          ssh.exec!("rm -r '#{remote_path}'") do |ch, stream, data|
+            errors << data if stream == :stderr
+          end
+        end
+        unless errors.empty?
+          raise Errors::Storage::SCP::SSHError,
+            "Net::SSH reported the following errors:\n" +
+              errors.join("\n")
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/storage/sftp.rb

@@ -0,0 +1,108 @@
+# encoding: utf-8
+
+##
+# Only load the Net::SFTP library/gem when the Backup::Storage::SFTP class is loaded
+Backup::Dependency.load('net-ssh')
+Backup::Dependency.load('net-sftp')
+
+module Backup
+  module Storage
+    class SFTP < Base
+
+      ##
+      # Server credentials
+      attr_accessor :username, :password
+
+      ##
+      # Server IP Address and SFTP port
+      attr_accessor :ip, :port
+
+      ##
+      # Path to store backups to
+      attr_accessor :path
+
+      ##
+      # Creates a new instance of the storage object
+      def initialize(model, storage_id = nil, &block)
+        super(model, storage_id)
+
+        @port ||= 22
+        @path ||= 'backups'
+
+        instance_eval(&block) if block_given?
+
+        @path = path.sub(/^\~\//, '')
+      end
+
+      private
+
+      ##
+      # Establishes a connection to the remote server
+      def connection
+        Net::SFTP.start(
+          ip, username,
+          :password => password,
+          :port     => port
+        ) {|sftp| yield sftp }
+      end
+
+      ##
+      # Transfers the archived file to the specified remote server
+      def transfer!
+        remote_path = remote_path_for(@package)
+
+        connection do |sftp|
+          create_remote_path(remote_path, sftp)
+
+          files_to_transfer_for(@package) do |local_file, remote_file|
+            Logger.message "#{storage_name} started transferring " +
+                "'#{ local_file }' to '#{ ip }'."
+
+            sftp.upload!(
+              File.join(local_path, local_file),
+              File.join(remote_path, remote_file)
+            )
+          end
+        end
+      end
+
+      ##
+      # Removes the transferred archive file(s) from the storage location.
+      # Any error raised will be rescued during Cycling
+      # and a warning will be logged, containing the error message.
+      def remove!(package)
+        remote_path = remote_path_for(package)
+
+        connection do |sftp|
+          transferred_files_for(package) do |local_file, remote_file|
+            Logger.message "#{storage_name} started removing " +
+                "'#{ local_file }' from '#{ ip }'."
+
+            sftp.remove!(File.join(remote_path, remote_file))
+          end
+
+          sftp.rmdir!(remote_path)
+        end
+      end
+
+      ##
+      # Creates (if they don't exist yet) all the directories on the remote
+      # server in order to upload the backup file. Net::SFTP does not support
+      # paths to directories that don't yet exist when creating new
+      # directories. Instead, we split the parts up in to an array (for each
+      # '/') and loop through that to create the directories one by one.
+      # Net::SFTP raises an exception when the directory it's trying to create
+      # already exists, so we have rescue it
+      def create_remote_path(remote_path, sftp)
+        path_parts = Array.new
+        remote_path.split('/').each do |path_part|
+          path_parts << path_part
+          begin
+            sftp.mkdir!(path_parts.join('/'))
+          rescue Net::SFTP::StatusException; end
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/syncer/base.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+
+module Backup
+  module Syncer
+    class Base
+      include Backup::CLI::Helpers
+      include Backup::Configuration::Helpers
+
+      ##
+      # Directories to sync
+      attr_accessor :directories
+
+      ##
+      # Path to store the synced files/directories to
+      attr_accessor :path
+
+      ##
+      # Flag for mirroring the files/directories
+      attr_accessor :mirror
+
+      ##
+      # Syntactical suger for the DSL for adding directories
+      def directories(&block)
+        return @directories unless block_given?
+        instance_eval(&block)
+      end
+
+      ##
+      # Adds a path to the @directories array
+      def add(path)
+        @directories << path
+      end
+
+      private
+
+      def syncer_name
+        self.class.to_s.sub('Backup::', '')
+      end
+
+    end
+  end
+end

data/lib/backup/syncer/cloud.rb

@@ -0,0 +1,190 @@
+# encoding: utf-8
+
+##
+# Only load the Fog gem, along with the Parallel gem, when the Backup::Syncer::Cloud class is loaded
+Backup::Dependency.load('fog')
+Backup::Dependency.load('parallel')
+
+module Backup
+  module Syncer
+    class Cloud < Base
+
+      ##
+      # Create a Mutex to synchronize certain parts of the code
+      # in order to prevent race conditions or broken STDOUT.
+      MUTEX = Mutex.new
+
+      ##
+      # Concurrency setting - defaults to false, but can be set to:
+      # - :threads
+      # - :processes
+      attr_accessor :concurrency_type
+
+      ##
+      # Concurrency level - the number of threads or processors to use. Defaults to 2.
+      attr_accessor :concurrency_level
+
+      ##
+      # Instantiates a new Cloud Syncer object and sets the default
+      # configuration specified in the Backup::Configuration::Syncer::S3. Then
+      # it sets the object defaults if particular properties weren't set.
+      # Finally it'll evaluate the users configuration file and overwrite
+      # anything that's been defined.
+      def initialize(&block)
+        load_defaults!
+
+        @path               ||= 'backups'
+        @directories        ||= Array.new
+        @mirror             ||= false
+        @concurrency_type     = false
+        @concurrency_level    = 2
+
+        instance_eval(&block) if block_given?
+
+        @path = path.sub(/^\//, '')
+      end
+
+      ##
+      # Performs the Sync operation
+      def perform!
+        Logger.message("#{ syncer_name } started the syncing process:")
+
+        directories.each do |directory|
+          SyncContext.new(directory, repository_object, path).
+            sync! mirror, concurrency_type, concurrency_level
+        end
+      end
+
+      private
+
+      class SyncContext
+        attr_reader :directory, :bucket, :path
+
+        ##
+        # Creates a new SyncContext object which handles a single directory
+        # from the Syncer::Base @directories array.
+        def initialize(directory, bucket, path)
+          @directory, @bucket, @path = directory, bucket, path
+        end
+
+        ##
+        # Performs the sync operation using the provided techniques (mirroring/concurrency).
+        def sync!(mirror = false, concurrency_type = false, concurrency_level = 2)
+          block = Proc.new { |relative_path| sync_file relative_path, mirror }
+
+          case concurrency_type
+          when FalseClass
+            all_file_names.each &block
+          when :threads
+            Parallel.each all_file_names, :in_threads => concurrency_level, &block
+          when :processes
+            Parallel.each all_file_names, :in_processes => concurrency_level, &block
+          else
+            raise Errors::Syncer::Cloud::ConfigurationError,
+                "Unknown concurrency_type setting: #{concurrency_type.inspect}"
+          end
+        end
+
+        private
+
+        ##
+        # Gathers all the remote and local file name and merges them together, removing
+        # duplicate keys if any, and sorts the in alphabetical order.
+        def all_file_names
+          @all_file_names ||= (local_files.keys | remote_files.keys).sort
+        end
+
+        ##
+        # Returns a Hash of local files (the keys are the filesystem paths,
+        # the values are the LocalFile objects for that given file)
+        def local_files
+          @local_files ||= begin
+            local_hashes.split("\n").collect { |line|
+              LocalFile.new directory, line
+            }.inject({}) { |hash, file|
+              hash[file.relative_path] = file
+              hash
+            }
+          end
+        end
+
+        ##
+        # Returns a String of file paths and their md5 hashes.
+        def local_hashes
+          MUTEX.synchronize { Logger.message("\s\sGenerating checksums for #{ directory }") }
+          `find #{directory} -print0 | xargs -0 openssl md5 2> /dev/null`
+        end
+
+        ##
+        # Returns a Hash of remote files (the keys are the remote paths,
+        # the values are the Fog file objects for that given file)
+        def remote_files
+          @remote_files ||= bucket.files.to_a.select { |file|
+            file.key[%r{^#{remote_base}/}]
+          }.inject({}) { |hash, file|
+            key = file.key.gsub(/^#{remote_base}\//,
+              "#{directory.split('/').last}/")
+            hash[key] = file
+            hash
+          }
+        end
+
+        ##
+        # Creates and returns a String that represents the base remote storage path
+        def remote_base
+          @remote_base ||= [path, directory.split('/').last].select { |part|
+            part && part.strip.length > 0
+          }.join('/')
+        end
+
+        ##
+        # Performs a sync operation on a file. When mirroring is enabled
+        # and a local file has been removed since the last sync, it will also
+        # remove it from the remote location. It will no upload files that
+        # have not changed since the last sync. Checks are done using an md5 hash.
+        # If a file has changed, or has been newly added, the file will be transferred/overwritten.
+        def sync_file(relative_path, mirror)
+          local_file  = local_files[relative_path]
+          remote_file = remote_files[relative_path]
+
+          if local_file && File.exist?(local_file.path)
+            unless remote_file && remote_file.etag == local_file.md5
+              MUTEX.synchronize { Logger.message("\s\s[transferring] #{relative_path}") }
+              File.open(local_file.path, 'r') do |file|
+                bucket.files.create(
+                  :key  => "#{path}/#{relative_path}".gsub(/^\//, ''),
+                  :body => file
+                )
+              end
+            else
+              MUTEX.synchronize { Logger.message("\s\s[skipping] #{relative_path}") }
+            end
+          elsif remote_file && mirror
+            MUTEX.synchronize { Logger.message("\s\s[removing] #{relative_path}") }
+            remote_file.destroy
+          end
+        end
+      end
+
+      class LocalFile
+        attr_reader :directory, :path, :md5
+
+        ##
+        # Creates a new LocalFile object using the given directory and line
+        # from the md5 hash checkup. This object figures out the path, relative_path and md5 hash
+        # for the file.
+        def initialize(directory, line)
+          @directory  = directory
+          @path, @md5 = *line.chomp.match(/^MD5\(([^\)]+)\)= (\w+)$/).captures
+        end
+
+        ##
+        # Returns the relative path to the file.
+        def relative_path
+          @relative_path ||= path.gsub %r{^#{directory}},
+            directory.split('/').last
+        end
+      end
+    end
+  end
+end