cm-backup 1.0.0

data/lib/backup/storage/s3.rb

@@ -0,0 +1,156 @@
+# encoding: utf-8
+require 'backup/cloud_io/s3'
+
+module Backup
+  module Storage
+    class S3 < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Amazon Simple Storage Service (S3) Credentials
+      attr_accessor :access_key_id, :secret_access_key, :use_iam_profile
+
+      ##
+      # Amazon S3 bucket name
+      attr_accessor :bucket
+
+      ##
+      # Region of the specified S3 bucket
+      attr_accessor :region
+
+      ##
+      # Multipart chunk size, specified in MiB.
+      #
+      # Each package file larger than +chunk_size+
+      # will be uploaded using S3 Multipart Upload.
+      #
+      # Minimum: 5 (but may be disabled with 0)
+      # Maximum: 5120
+      # Default: 5
+      attr_accessor :chunk_size
+
+      ##
+      # Number of times to retry failed operations.
+      #
+      # Default: 10
+      attr_accessor :max_retries
+
+      ##
+      # Time in seconds to pause before each retry.
+      #
+      # Default: 30
+      attr_accessor :retry_waitsec
+
+      ##
+      # Encryption algorithm to use for Amazon Server-Side Encryption
+      #
+      # Supported values:
+      #
+      # - :aes256
+      #
+      # Default: nil
+      attr_accessor :encryption
+
+      ##
+      # Storage class to use for the S3 objects uploaded
+      #
+      # Supported values:
+      #
+      # - :standard (default)
+      # - :standard_ia
+      # - :reduced_redundancy
+      #
+      # Default: :standard
+      attr_accessor :storage_class
+
+      ##
+      # Additional options to pass along to fog.
+      # e.g. Fog::Storage.new({ :provider => 'AWS' }.merge(fog_options))
+      attr_accessor :fog_options
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @chunk_size     ||= 5 # MiB
+        @max_retries    ||= 10
+        @retry_waitsec  ||= 30
+        @path           ||= 'backups'
+        @storage_class  ||= :standard
+
+        @path = @path.sub(/^\//, '')
+
+        check_configuration
+      end
+
+      private
+
+      def cloud_io
+        @cloud_io ||= CloudIO::S3.new(
+          :access_key_id      => access_key_id,
+          :secret_access_key  => secret_access_key,
+          :use_iam_profile    => use_iam_profile,
+          :region             => region,
+          :bucket             => bucket,
+          :encryption         => encryption,
+          :storage_class      => storage_class,
+          :max_retries        => max_retries,
+          :retry_waitsec      => retry_waitsec,
+          :chunk_size         => chunk_size,
+          :fog_options        => fog_options
+        )
+      end
+
+      def transfer!
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{ bucket }/#{ dest }'..."
+          cloud_io.upload(src, dest)
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{ package.time }..."
+
+        remote_path = remote_path_for(package)
+        objects = cloud_io.objects(remote_path)
+
+        raise Error, "Package at '#{ remote_path }' not found" if objects.empty?
+
+        cloud_io.delete(objects)
+      end
+
+      def check_configuration
+        if use_iam_profile
+          required = %w{ bucket }
+        else
+          required = %w{ access_key_id secret_access_key bucket }
+        end
+        raise Error, <<-EOS if required.map {|name| send(name) }.any?(&:nil?)
+          Configuration Error
+          #{ required.map {|name| "##{ name }"}.join(', ') } are all required
+        EOS
+
+        raise Error, <<-EOS if chunk_size > 0 && !chunk_size.between?(5, 5120)
+          Configuration Error
+          #chunk_size must be between 5 and 5120 (or 0 to disable multipart)
+        EOS
+
+        raise Error, <<-EOS if encryption && encryption.to_s.upcase != 'AES256'
+          Configuration Error
+          #encryption must be :aes256 or nil
+        EOS
+
+        classes = ['STANDARD', 'STANDARD_IA', 'REDUCED_REDUNDANCY']
+        raise Error, <<-EOS unless classes.include?(storage_class.to_s.upcase)
+          Configuration Error
+          #storage_class must be :standard or :standard_ia or :reduced_redundancy
+        EOS
+      end
+
+    end
+  end
+end

data/lib/backup/storage/scp.rb

@@ -0,0 +1,67 @@
+# encoding: utf-8
+require 'net/scp'
+
+module Backup
+  module Storage
+    class SCP < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Server credentials
+      attr_accessor :username, :password, :ssh_options
+
+      ##
+      # Server IP Address and SCP port
+      attr_accessor :ip, :port
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @port ||= 22
+        @path ||= 'backups'
+        @ssh_options ||= {}
+        path.sub!(/^~\//, '')
+      end
+
+      private
+
+      def connection
+        Net::SSH.start(
+          ip, username, { :password => password, :port => port }.merge(ssh_options)
+        ) {|ssh| yield ssh }
+      end
+
+      def transfer!
+        connection do |ssh|
+          ssh.exec!("mkdir -p '#{ remote_path }'")
+
+          package.filenames.each do |filename|
+            src = File.join(Config.tmp_path, filename)
+            dest = File.join(remote_path, filename)
+            Logger.info "Storing '#{ ip }:#{ dest }'..."
+            ssh.scp.upload!(src, dest)
+          end
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{ package.time }..."
+
+        errors = []
+        connection do |ssh|
+          ssh.exec!("rm -r '#{ remote_path_for(package) }'") do |ch, stream, data|
+            errors << data if stream == :stderr
+          end
+        end
+        unless errors.empty?
+          raise Error, "Net::SSH reported the following errors:\n" +
+              errors.join("\n")
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/storage/sftp.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+require 'net/sftp'
+
+module Backup
+  module Storage
+    class SFTP < Base
+      include Storage::Cycler
+
+      ##
+      # Server credentials
+      attr_accessor :username, :password, :ssh_options
+
+      ##
+      # Server IP Address and SFTP port
+      attr_accessor :ip, :port
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @ssh_options ||= {}
+        @port        ||= 22
+        @path        ||= 'backups'
+        path.sub!(/^~\//, '')
+      end
+
+      private
+
+      def connection
+        Net::SFTP.start(
+          ip, username, { :password => password, :port => port }.merge(ssh_options)
+        ) {|sftp| yield sftp }
+      end
+
+      def transfer!
+        connection do |sftp|
+          create_remote_path(sftp)
+
+          package.filenames.each do |filename|
+            src = File.join(Config.tmp_path, filename)
+            dest = File.join(remote_path, filename)
+            Logger.info "Storing '#{ ip }:#{ dest }'..."
+            sftp.upload!(src, dest)
+          end
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{ package.time }..."
+
+        remote_path = remote_path_for(package)
+        connection do |sftp|
+          package.filenames.each do |filename|
+            sftp.remove!(File.join(remote_path, filename))
+          end
+
+          sftp.rmdir!(remote_path)
+        end
+      end
+
+      ##
+      # Creates (if they don't exist yet) all the directories on the remote
+      # server in order to upload the backup file. Net::SFTP does not support
+      # paths to directories that don't yet exist when creating new
+      # directories. Instead, we split the parts up in to an array (for each
+      # '/') and loop through that to create the directories one by one.
+      # Net::SFTP raises an exception when the directory it's trying to create
+      # already exists, so we have rescue it
+      def create_remote_path(sftp)
+        path_parts = Array.new
+        remote_path.split('/').each do |path_part|
+          path_parts << path_part
+          begin
+            sftp.mkdir!(path_parts.join('/'))
+          rescue Net::SFTP::StatusException; end
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/syncer/base.rb

@@ -0,0 +1,70 @@
+# encoding: utf-8
+
+module Backup
+  module Syncer
+    class Base
+      include Utilities::Helpers
+      include Config::Helpers
+
+      ##
+      # Path to store the synced files/directories to
+      attr_accessor :path
+
+      ##
+      # Flag for mirroring the files/directories
+      attr_accessor :mirror
+
+      ##
+      # Optional user-defined identifier to differentiate multiple syncers
+      # defined within a single backup model. Currently this is only used
+      # in the log messages.
+      attr_reader :syncer_id
+
+      attr_reader :excludes
+
+      def initialize(syncer_id = nil)
+        @syncer_id = syncer_id
+
+        load_defaults!
+
+        @mirror ||= false
+        @directories ||= []
+        @excludes ||= []
+      end
+
+      ##
+      # Syntactical suger for the DSL for adding directories
+      def directories(&block)
+        return @directories unless block_given?
+        instance_eval(&block)
+      end
+
+      def add(path)
+        directories << path
+      end
+
+      # For Cloud Syncers, +pattern+ can be a string (with shell-style
+      # wildcards) or a regex.
+      # For RSync, each +pattern+ will be passed to rsync's --exclude option.
+      def exclude(pattern)
+        excludes << pattern
+      end
+
+      private
+
+      def syncer_name
+        @syncer_name ||= self.class.to_s.sub('Backup::', '') +
+            (syncer_id ? " (#{ syncer_id })" : '')
+      end
+
+      def log!(action)
+        msg = case action
+              when :started then 'Started...'
+              when :finished then 'Finished!'
+              end
+        Logger.info "#{ syncer_name } #{ msg }"
+      end
+
+    end
+  end
+end

data/lib/backup/syncer/cloud/base.rb

@@ -0,0 +1,179 @@
+# encoding: utf-8
+
+module Backup
+  module Syncer
+    module Cloud
+      class Error < Backup::Error; end
+
+      class Base < Syncer::Base
+        MUTEX = Mutex.new
+
+        ##
+        # Number of threads to use for concurrency.
+        #
+        # Default: 0 (no concurrency)
+        attr_accessor :thread_count
+
+        ##
+        # Number of times to retry failed operations.
+        #
+        # Default: 10
+        attr_accessor :max_retries
+
+        ##
+        # Time in seconds to pause before each retry.
+        #
+        # Default: 30
+        attr_accessor :retry_waitsec
+
+        def initialize(syncer_id = nil, &block)
+          super
+          instance_eval(&block) if block_given?
+
+          @thread_count   ||= 0
+          @max_retries    ||= 10
+          @retry_waitsec  ||= 30
+
+          @path ||= 'backups'
+          @path = path.sub(/^\//, '')
+        end
+
+        def perform!
+          log!(:started)
+          @transfer_count = 0
+          @unchanged_count = 0
+          @skipped_count = 0
+          @orphans = thread_count > 0 ? Queue.new : []
+
+          directories.each {|dir| sync_directory(dir) }
+          orphans_result = process_orphans
+
+          Logger.info "\nSummary:"
+          Logger.info "\s\sTransferred Files: #{ @transfer_count }"
+          Logger.info "\s\s#{ orphans_result }"
+          Logger.info "\s\sUnchanged Files: #{ @unchanged_count }"
+          Logger.warn "\s\sSkipped Files: #{ @skipped_count }" if @skipped_count > 0
+          log!(:finished)
+        end
+
+        private
+
+        def sync_directory(dir)
+          remote_base = path.empty? ? File.basename(dir) :
+                                      File.join(path, File.basename(dir))
+          Logger.info "Gathering remote data for '#{ remote_base }'..."
+          remote_files = get_remote_files(remote_base)
+
+          Logger.info("Gathering local data for '#{ File.expand_path(dir) }'...")
+          local_files = LocalFile.find(dir, excludes)
+
+          relative_paths = (local_files.keys | remote_files.keys).sort
+          if relative_paths.empty?
+            Logger.info 'No local or remote files found'
+          else
+            Logger.info 'Syncing...'
+            sync_block = Proc.new do |relative_path|
+              local_file  = local_files[relative_path]
+              remote_md5  = remote_files[relative_path]
+              remote_path = File.join(remote_base, relative_path)
+              sync_file(local_file, remote_path, remote_md5)
+            end
+
+            if thread_count > 0
+              sync_in_threads(relative_paths, sync_block)
+            else
+              relative_paths.each(&sync_block)
+            end
+          end
+        end
+
+        def sync_in_threads(relative_paths, sync_block)
+          queue = Queue.new
+          queue << relative_paths.shift until relative_paths.empty?
+          num_threads = [thread_count, queue.size].min
+          Logger.info "\s\sUsing #{ num_threads } Threads"
+          threads = num_threads.times.map do
+            Thread.new do
+              loop do
+                path = queue.shift(true) rescue nil
+                path ? sync_block.call(path) : break
+              end
+            end
+          end
+
+          # abort if any thread raises an exception
+          while threads.any?(&:alive?)
+            if threads.any? {|thr| thr.status.nil? }
+              threads.each(&:kill)
+              Thread.pass while threads.any?(&:alive?)
+              break
+            end
+            sleep num_threads * 0.1
+          end
+          threads.each(&:join)
+        end
+
+        # If an exception is raised in multiple threads, only the exception
+        # raised in the first thread that Thread#join is called on will be
+        # handled. So all exceptions are logged first with their details,
+        # then a generic exception is raised.
+        def sync_file(local_file, remote_path, remote_md5)
+          if local_file && File.exist?(local_file.path)
+            if local_file.md5 == remote_md5
+              MUTEX.synchronize { @unchanged_count += 1 }
+            else
+              Logger.info("\s\s[transferring] '#{ remote_path }'")
+              begin
+                cloud_io.upload(local_file.path, remote_path)
+                MUTEX.synchronize { @transfer_count += 1 }
+              rescue CloudIO::FileSizeError => err
+                MUTEX.synchronize { @skipped_count += 1 }
+                Logger.warn Error.wrap(err, "Skipping '#{ remote_path }'")
+              rescue => err
+                Logger.error(err)
+                raise Error, <<-EOS
+                  Syncer Failed!
+                  See the Retry [info] and [error] messages (if any)
+                  for details on each failed operation.
+                EOS
+              end
+            end
+          elsif remote_md5
+            @orphans << remote_path
+          end
+        end
+
+        def process_orphans
+          if @orphans.empty?
+            return mirror ? 'Deleted Files: 0' : 'Orphaned Files: 0'
+          end
+
+          if @orphans.is_a?(Queue)
+            @orphans = @orphans.size.times.map { @orphans.shift }
+          end
+
+          if mirror
+            Logger.info @orphans.map {|path|
+              "\s\s[removing] '#{ path }'"
+            }.join("\n")
+
+            begin
+              cloud_io.delete(@orphans)
+              "Deleted Files: #{ @orphans.count }"
+            rescue => err
+              Logger.warn Error.wrap(err, 'Delete Operation Failed')
+              "Attempted to Delete: #{ @orphans.count } " +
+              "(See log messages for actual results)"
+            end
+          else
+            Logger.info @orphans.map {|path|
+              "\s\s[orphaned] '#{ path }'"
+            }.join("\n")
+            "Orphaned Files: #{ @orphans.count }"
+          end
+        end
+
+      end
+    end
+  end
+end