cm-backup 1.0.0

data/lib/backup/syncer/cloud/cloud_files.rb

@@ -0,0 +1,83 @@
+# encoding: utf-8
+require 'backup/cloud_io/cloud_files'
+
+module Backup
+  module Syncer
+    module Cloud
+      class CloudFiles < Base
+        class Error < Backup::Error; end
+
+        ##
+        # Rackspace CloudFiles Credentials
+        attr_accessor :username, :api_key
+
+        ##
+        # Rackspace CloudFiles Container
+        attr_accessor :container
+
+        ##
+        # Rackspace AuthURL (optional)
+        attr_accessor :auth_url
+
+        ##
+        # Rackspace Region (optional)
+        attr_accessor :region
+
+        ##
+        # Rackspace Service Net
+        # (LAN-based transfers to avoid charges and improve performance)
+        attr_accessor :servicenet
+
+        ##
+        # Additional options to pass along to fog.
+        # e.g. Fog::Storage.new({ :provider => 'Rackspace' }.merge(fog_options))
+        attr_accessor :fog_options
+
+        def initialize(syncer_id = nil)
+          super
+
+          @servicenet ||= false
+
+          check_configuration
+        end
+
+        private
+
+        def cloud_io
+          @cloud_io ||= CloudIO::CloudFiles.new(
+            :username           => username,
+            :api_key            => api_key,
+            :auth_url           => auth_url,
+            :region             => region,
+            :servicenet         => servicenet,
+            :container          => container,
+            :max_retries        => max_retries,
+            :retry_waitsec      => retry_waitsec,
+            # Syncer can not use SLOs.
+            :segments_container => nil,
+            :segment_size       => 0,
+            :fog_options        => fog_options
+          )
+        end
+
+        def get_remote_files(remote_base)
+          hash = {}
+          cloud_io.objects(remote_base).each do |object|
+            relative_path = object.name.sub(remote_base + '/', '')
+            hash[relative_path] = object.hash
+          end
+          hash
+        end
+
+        def check_configuration
+          required = %w{ username api_key container }
+          raise Error, <<-EOS if required.map {|name| send(name) }.any?(&:nil?)
+            Configuration Error
+            #{ required.map {|name| "##{ name }"}.join(', ') } are all required
+          EOS
+        end
+
+      end # class Cloudfiles < Base
+    end # module Cloud
+  end
+end

data/lib/backup/syncer/cloud/local_file.rb

@@ -0,0 +1,100 @@
+# encoding: utf-8
+require 'digest/md5'
+
+module Backup
+  module Syncer
+    module Cloud
+      class LocalFile
+        attr_reader :path
+        attr_accessor :md5
+
+        class << self
+
+          # Returns a Hash of LocalFile objects for each file within +dir+,
+          # except those matching any of the +excludes+.
+          # Hash keys are the file's path relative to +dir+.
+          def find(dir, excludes = [])
+            dir = File.expand_path(dir)
+            hash = {}
+            find_md5(dir, excludes).each do |file|
+              hash[file.path.sub(dir + '/', '')] = file
+            end
+            hash
+          end
+
+          # Return a new LocalFile object if it's valid.
+          # Otherwise, log a warning and return nil.
+          def new(*args)
+            file = super
+            if file.invalid?
+              Logger.warn("\s\s[skipping] #{ file.path }\n" +
+                          "\s\sPath Contains Invalid UTF-8 byte sequences")
+              file = nil
+            end
+            file
+          end
+
+          private
+
+          # Returns an Array of file paths and their md5 hashes.
+          def find_md5(dir, excludes)
+            found = []
+            (Dir.entries(dir) - %w{. ..}).map {|e| File.join(dir, e) }.each do |path|
+              if File.directory?(path)
+                unless exclude?(excludes, path)
+                  found += find_md5(path, excludes)
+                end
+              elsif File.file?(path)
+                if file = new(path)
+                  unless exclude?(excludes, file.path)
+                    file.md5 = Digest::MD5.file(file.path).hexdigest
+                    found << file
+                  end
+                end
+              end
+            end
+            found
+          end
+
+          # Returns true if +path+ matches any of the +excludes+.
+          # Note this can not be called if +path+ includes invalid UTF-8.
+          def exclude?(excludes, path)
+            excludes.any? do |ex|
+              if ex.is_a?(String)
+                File.fnmatch?(ex, path)
+              elsif ex.is_a?(Regexp)
+                ex.match(path)
+              end
+            end
+          end
+        end
+
+        # If +path+ contains invalid UTF-8, it will be sanitized
+        # and the LocalFile object will be flagged as invalid.
+        # This is done so @file.path may be logged.
+        def initialize(path)
+          @path = sanitize(path)
+        end
+
+        def invalid?
+          !!@invalid
+        end
+
+        private
+
+        def sanitize(str)
+          str.each_char.map do |char|
+            begin
+              char.unpack('U')
+              char
+            rescue
+              @invalid = true
+              "\xEF\xBF\xBD" # => "\uFFFD"
+            end
+          end.join
+        end
+
+      end
+    end
+  end
+end

data/lib/backup/syncer/cloud/s3.rb

@@ -0,0 +1,110 @@
+# encoding: utf-8
+require 'backup/cloud_io/s3'
+
+module Backup
+  module Syncer
+    module Cloud
+      class S3 < Base
+        class Error < Backup::Error; end
+
+        ##
+        # Amazon Simple Storage Service (S3) Credentials
+        attr_accessor :access_key_id, :secret_access_key, :use_iam_profile
+
+        ##
+        # Amazon S3 bucket name
+        attr_accessor :bucket
+
+        ##
+        # Region of the specified S3 bucket
+        attr_accessor :region
+
+        ##
+        # Encryption algorithm to use for Amazon Server-Side Encryption
+        #
+        # Supported values:
+        #
+        # - :aes256
+        #
+        # Default: nil
+        attr_accessor :encryption
+
+        ##
+        # Storage class to use for the S3 objects uploaded
+        #
+        # Supported values:
+        #
+        # - :standard (default)
+        # - :reduced_redundancy
+        #
+        # Default: :standard
+        attr_accessor :storage_class
+
+        ##
+        # Additional options to pass along to fog.
+        # e.g. Fog::Storage.new({ :provider => 'AWS' }.merge(fog_options))
+        attr_accessor :fog_options
+
+        def initialize(syncer_id = nil)
+          super
+
+          @storage_class ||= :standard
+
+          check_configuration
+        end
+
+        private
+
+        def cloud_io
+          @cloud_io ||= CloudIO::S3.new(
+            :access_key_id      => access_key_id,
+            :secret_access_key  => secret_access_key,
+            :use_iam_profile    => use_iam_profile,
+            :bucket             => bucket,
+            :region             => region,
+            :encryption         => encryption,
+            :storage_class      => storage_class,
+            :max_retries        => max_retries,
+            :retry_waitsec      => retry_waitsec,
+            # Syncer can not use multipart upload.
+            :chunk_size         => 0,
+            :fog_options        => fog_options
+          )
+        end
+
+        def get_remote_files(remote_base)
+          hash = {}
+          cloud_io.objects(remote_base).each do |object|
+            relative_path = object.key.sub(remote_base + '/', '')
+            hash[relative_path] = object.etag
+          end
+          hash
+        end
+
+        def check_configuration
+          if use_iam_profile
+            required = %w{ bucket }
+          else
+            required = %w{ access_key_id secret_access_key bucket }
+          end
+          raise Error, <<-EOS if required.map {|name| send(name) }.any?(&:nil?)
+            Configuration Error
+            #{ required.map {|name| "##{ name }"}.join(', ') } are all required
+          EOS
+
+          raise Error, <<-EOS if encryption && encryption.to_s.upcase != 'AES256'
+            Configuration Error
+            #encryption must be :aes256 or nil
+          EOS
+
+          classes = ['STANDARD', 'REDUCED_REDUNDANCY']
+          raise Error, <<-EOS unless classes.include?(storage_class.to_s.upcase)
+            Configuration Error
+            #storage_class must be :standard or :reduced_redundancy
+          EOS
+        end
+
+      end # Class S3 < Base
+    end # module Cloud
+  end
+end

data/lib/backup/syncer/rsync/base.rb

@@ -0,0 +1,54 @@
+# encoding: utf-8
+
+module Backup
+  module Syncer
+    module RSync
+      class Base < Syncer::Base
+
+        ##
+        # Additional String or Array of options for the rsync cli
+        attr_accessor :additional_rsync_options
+        attr_accessor :archive
+
+        def initialize(syncer_id = nil, &block)
+          super
+          instance_eval(&block) if block_given?
+
+          @path ||= '~/backups'
+          @archive = @archive.nil? ? true : @archive
+        end
+
+        private
+
+        ##
+        # Common base command for Local/Push/Pull
+        def rsync_command
+          utility(:rsync) << archive_option << mirror_option << exclude_option <<
+              " #{ Array(additional_rsync_options).join(' ') }".rstrip
+        end
+
+        def mirror_option
+          mirror ? ' --delete' : ''
+        end
+
+        def archive_option
+          archive ? ' --archive' : ''
+        end
+
+        def exclude_option
+          excludes.map {|pattern| " --exclude='#{ pattern }'" }.join
+        end
+
+        ##
+        # Each path is expanded, since these refer to local paths and are
+        # being shell-quoted. This will also remove any trailing `/` from
+        # each path, as we don't want rsync's "trailing / on source directories"
+        # behavior. This method is used by RSync::Local and RSync::Push.
+        def paths_to_push
+          directories.map {|dir| "'#{ File.expand_path(dir) }'" }.join(' ')
+        end
+
+      end
+    end
+  end
+end

data/lib/backup/syncer/rsync/local.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+
+module Backup
+  module Syncer
+    module RSync
+      class Local < Base
+
+        def perform!
+          log!(:started)
+
+          create_dest_path!
+          run("#{ rsync_command } #{ paths_to_push } '#{ dest_path }'")
+
+          log!(:finished)
+        end
+
+        private
+
+        # Expand path, since this is local and shell-quoted.
+        def dest_path
+          @dest_path ||= File.expand_path(path)
+        end
+
+        def create_dest_path!
+          FileUtils.mkdir_p dest_path
+        end
+
+      end
+    end
+  end
+end

data/lib/backup/syncer/rsync/pull.rb

@@ -0,0 +1,51 @@
+# encoding: utf-8
+
+module Backup
+  module Syncer
+    module RSync
+      class Pull < Push
+
+        def perform!
+          log!(:started)
+          write_password_file!
+
+          create_dest_path!
+          run("#{ rsync_command } #{ host_options }#{ paths_to_pull } " +
+              "'#{ dest_path }'")
+
+          log!(:finished)
+        ensure
+          remove_password_file!
+        end
+
+        private
+
+        ##
+        # Returns the syntax for pulling multiple paths from the remote host.
+        # e.g.
+        #   rsync -a -e "ssh -p 22" host:'path1' :'path2' '/dest'
+        #   rsync -a rsync_user@host::'modname/path1' ::'modname/path2' '/dest'
+        #
+        # Remove any preceeding '~/', since these paths are on the remote.
+        # Also remove any trailing `/`, since we don't want rsync's
+        # "trailing / on source directories" behavior.
+        def paths_to_pull
+          sep = mode == :ssh ? ':' : '::'
+          directories.map {|dir|
+            "#{ sep }'#{ dir.sub(/^~\//, '').sub(/\/$/, '') }'"
+          }.join(' ').sub(/^#{ sep }/, '')
+        end
+
+        # Expand path, since this is local and shell-quoted.
+        def dest_path
+          @dest_path ||= File.expand_path(path)
+        end
+
+        def create_dest_path!
+          FileUtils.mkdir_p dest_path
+        end
+
+      end
+    end
+  end
+end

data/lib/backup/syncer/rsync/push.rb

@@ -0,0 +1,205 @@
+# encoding: utf-8
+
+module Backup
+  module Syncer
+    module RSync
+      class Push < Base
+
+        ##
+        # Mode of operation
+        #
+        # [:ssh (default)]
+        #   Connects to the remote via SSH.
+        #   Does not use an rsync daemon on the remote.
+        #
+        # [:ssh_daemon]
+        #   Connects to the remote via SSH.
+        #   Spawns a single-use daemon on the remote, which allows certain
+        #   daemon features (like modules) to be used.
+        #
+        # [:rsync_daemon]
+        #   Connects directly to an rsync daemon via TCP.
+        #   Data transferred is not encrypted.
+        #
+        attr_accessor :mode
+
+        ##
+        # Server Address
+        attr_accessor :host
+
+        ##
+        # SSH or RSync port
+        #
+        # For `:ssh` or `:ssh_daemon` mode, this specifies the SSH port to use
+        # and defaults to 22.
+        #
+        # For `:rsync_daemon` mode, this specifies the TCP port to use
+        # and defaults to 873.
+        attr_accessor :port
+
+        ##
+        # SSH User
+        #
+        # If the user running the backup is not the same user that needs to
+        # authenticate with the remote server, specify the user here.
+        #
+        # The user must have SSH keys setup for passphrase-less access to the
+        # remote. If the SSH User does not have passphrase-less keys, or no
+        # default keys in their `~/.ssh` directory, you will need to use the
+        # `-i` option in `:additional_ssh_options` to specify the
+        # passphrase-less key to use.
+        #
+        # Used only for `:ssh` and `:ssh_daemon` modes.
+        attr_accessor :ssh_user
+
+        ##
+        # Additional SSH Options
+        #
+        # Used to supply a String or Array of options to be passed to the SSH
+        # command in `:ssh` and `:ssh_daemon` modes.
+        #
+        # For example, if you need to supply a specific SSH key for the `ssh_user`,
+        # you would set this to: "-i '/path/to/id_rsa'". Which would produce:
+        #
+        #   rsync -e "ssh -p 22 -i '/path/to/id_rsa'"
+        #
+        # Arguments may be single-quoted, but should not contain any double-quotes.
+        #
+        # Used only for `:ssh` and `:ssh_daemon` modes.
+        attr_accessor :additional_ssh_options
+
+        ##
+        # RSync User
+        #
+        # If the user running the backup is not the same user that needs to
+        # authenticate with the rsync daemon, specify the user here.
+        #
+        # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+        attr_accessor :rsync_user
+
+        ##
+        # RSync Password
+        #
+        # If specified, Backup will write the password to a temporary file and
+        # use it with rsync's `--password-file` option for daemon authentication.
+        #
+        # Note that setting this will override `rsync_password_file`.
+        #
+        # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+        attr_accessor :rsync_password
+
+        ##
+        # RSync Password File
+        #
+        # If specified, this path will be passed to rsync's `--password-file`
+        # option for daemon authentication.
+        #
+        # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+        attr_accessor :rsync_password_file
+
+        ##
+        # Flag for compressing (only compresses for the transfer)
+        attr_accessor :compress
+
+        def initialize(syncer_id = nil)
+          super
+
+          @mode ||= :ssh
+          @port ||= mode == :rsync_daemon ? 873 : 22
+          @compress ||= false
+        end
+
+        def perform!
+          log!(:started)
+          write_password_file!
+
+          create_dest_path!
+          run("#{ rsync_command } #{ paths_to_push } " +
+              "#{ host_options }'#{ dest_path }'")
+
+          log!(:finished)
+        ensure
+          remove_password_file!
+        end
+
+        private
+
+        ##
+        # Remove any preceeding '~/', since this is on the remote,
+        # and remove any trailing `/`.
+        def dest_path
+          @dest_path ||= path.sub(/^~\//, '').sub(/\/$/, '')
+        end
+
+        ##
+        # Runs a 'mkdir -p' command on the remote to ensure the dest_path exists.
+        # This used because rsync will attempt to create the path, but will only
+        # call 'mkdir' without the '-p' option. This is only applicable in :ssh
+        # mode, and only used if the path would require this.
+        def create_dest_path!
+          return unless mode == :ssh && dest_path.index('/').to_i > 0
+
+          run "#{ utility(:ssh) } #{ ssh_transport_args } #{ host } " +
+                 %Q["mkdir -p '#{ dest_path }'"]
+        end
+
+        ##
+        # For Push, this will prepend the #dest_path.
+        # For Pull, this will prepend the first path in #paths_to_pull.
+        def host_options
+          if mode == :ssh
+            "#{ host }:"
+          else
+            user = "#{ rsync_user }@" if rsync_user
+            "#{ user }#{ host }::"
+          end
+        end
+
+        ##
+        # Common base command, plus options for Push/Pull
+        def rsync_command
+          super << compress_option << password_option << transport_options
+        end
+
+        def compress_option
+          compress ? ' --compress' : ''
+        end
+
+        def password_option
+          return '' if mode == :ssh
+
+          path = @password_file ? @password_file.path : rsync_password_file
+          path ? " --password-file='#{ File.expand_path(path) }'" : ''
+        end
+
+        def transport_options
+          if mode == :rsync_daemon
+            " --port #{ port }"
+          else
+            %Q[ -e "#{ utility(:ssh) } #{ ssh_transport_args }"]
+          end
+        end
+
+        def ssh_transport_args
+          args = "-p #{ port } "
+          args << "-l #{ ssh_user } " if ssh_user
+          args << Array(additional_ssh_options).join(' ')
+          args.rstrip
+        end
+
+        def write_password_file!
+          return unless rsync_password && mode != :ssh
+
+          @password_file = Tempfile.new('backup-rsync-password')
+          @password_file.write(rsync_password)
+          @password_file.close
+        end
+
+        def remove_password_file!
+          @password_file.delete if @password_file
+        end
+
+      end
+    end
+  end
+end