ext_backup 5.0.0.beta.2.1

data/lib/backup/syncer/cloud/cloud_files.rb

@@ -0,0 +1,81 @@
+require "backup/cloud_io/cloud_files"
+
+module Backup
+  module Syncer
+    module Cloud
+      class CloudFiles < Base
+        class Error < Backup::Error; end
+
+        ##
+        # Rackspace CloudFiles Credentials
+        attr_accessor :username, :api_key
+
+        ##
+        # Rackspace CloudFiles Container
+        attr_accessor :container
+
+        ##
+        # Rackspace AuthURL (optional)
+        attr_accessor :auth_url
+
+        ##
+        # Rackspace Region (optional)
+        attr_accessor :region
+
+        ##
+        # Rackspace Service Net
+        # (LAN-based transfers to avoid charges and improve performance)
+        attr_accessor :servicenet
+
+        ##
+        # Additional options to pass along to fog.
+        # e.g. Fog::Storage.new({ :provider => 'Rackspace' }.merge(fog_options))
+        attr_accessor :fog_options
+
+        def initialize(syncer_id = nil)
+          super
+
+          @servicenet ||= false
+
+          check_configuration
+        end
+
+        private
+
+        def cloud_io
+          @cloud_io ||= CloudIO::CloudFiles.new(
+            username: username,
+            api_key: api_key,
+            auth_url: auth_url,
+            region: region,
+            servicenet: servicenet,
+            container: container,
+            max_retries: max_retries,
+            retry_waitsec: retry_waitsec,
+            # Syncer can not use SLOs.
+            segments_container: nil,
+            segment_size: 0,
+            fog_options: fog_options
+          )
+        end
+
+        def get_remote_files(remote_base)
+          hash = {}
+          cloud_io.objects(remote_base).each do |object|
+            relative_path = object.name.sub(remote_base + "/", "")
+            hash[relative_path] = object.hash
+          end
+          hash
+        end
+
+        def check_configuration
+          required = %w[username api_key container]
+          raise Error, <<-EOS if required.map { |name| send(name) }.any?(&:nil?)
+            Configuration Error
+            #{required.map { |name| "##{name}" }.join(", ")} are all required
+          EOS
+        end
+      end # class Cloudfiles < Base
+    end # module Cloud
+  end
+end

data/lib/backup/syncer/cloud/local_file.rb

@@ -0,0 +1,97 @@
+require "digest/md5"
+
+module Backup
+  module Syncer
+    module Cloud
+      class LocalFile
+        attr_reader :path
+        attr_accessor :md5
+
+        class << self
+          # Returns a Hash of LocalFile objects for each file within +dir+,
+          # except those matching any of the +excludes+.
+          # Hash keys are the file's path relative to +dir+.
+          def find(dir, excludes = [])
+            dir = File.expand_path(dir)
+            hash = {}
+            find_md5(dir, excludes).each do |file|
+              hash[file.path.sub(dir + "/", "")] = file
+            end
+            hash
+          end
+
+          # Return a new LocalFile object if it's valid.
+          # Otherwise, log a warning and return nil.
+          def new(*args)
+            file = super
+            if file.invalid?
+              Logger.warn("\s\s[skipping] #{file.path}\n" \
+                          "\s\sPath Contains Invalid UTF-8 byte sequences")
+              file = nil
+            end
+            file
+          end
+
+          private
+
+          # Returns an Array of file paths and their md5 hashes.
+          def find_md5(dir, excludes)
+            found = []
+            (Dir.entries(dir) - %w[. ..]).map { |e| File.join(dir, e) }.each do |path|
+              if File.directory?(path)
+                unless exclude?(excludes, path)
+                  found += find_md5(path, excludes)
+                end
+              elsif File.file?(path)
+                if file = new(path)
+                  unless exclude?(excludes, file.path)
+                    file.md5 = Digest::MD5.file(file.path).hexdigest
+                    found << file
+                  end
+                end
+              end
+            end
+            found
+          end
+
+          # Returns true if +path+ matches any of the +excludes+.
+          # Note this can not be called if +path+ includes invalid UTF-8.
+          def exclude?(excludes, path)
+            excludes.any? do |ex|
+              if ex.is_a?(String)
+                File.fnmatch?(ex, path)
+              elsif ex.is_a?(Regexp)
+                ex.match(path)
+              end
+            end
+          end
+        end
+
+        # If +path+ contains invalid UTF-8, it will be sanitized
+        # and the LocalFile object will be flagged as invalid.
+        # This is done so @file.path may be logged.
+        def initialize(path)
+          @path = sanitize(path)
+        end
+
+        def invalid?
+          !!@invalid
+        end
+
+        private
+
+        def sanitize(str)
+          str.each_char.map do |char|
+            begin
+              char.unpack("U")
+              char
+            rescue
+              @invalid = true
+              "\xEF\xBF\xBD" # => "\uFFFD"
+            end
+          end.join
+        end
+      end
+    end
+  end
+end

data/lib/backup/syncer/cloud/s3.rb

@@ -0,0 +1,109 @@
+require "backup/cloud_io/s3"
+
+module Backup
+  module Syncer
+    module Cloud
+      class S3 < Base
+        class Error < Backup::Error; end
+
+        ##
+        # Amazon Simple Storage Service (S3) Credentials
+        attr_accessor :access_key_id, :secret_access_key, :use_iam_profile
+
+        ##
+        # Amazon S3 bucket name
+        attr_accessor :bucket
+
+        ##
+        # Region of the specified S3 bucket
+        attr_accessor :region
+
+        ##
+        # Encryption algorithm to use for Amazon Server-Side Encryption
+        #
+        # Supported values:
+        #
+        # - :aes256
+        #
+        # Default: nil
+        attr_accessor :encryption
+
+        ##
+        # Storage class to use for the S3 objects uploaded
+        #
+        # Supported values:
+        #
+        # - :standard (default)
+        # - :reduced_redundancy
+        #
+        # Default: :standard
+        attr_accessor :storage_class
+
+        ##
+        # Additional options to pass along to fog.
+        # e.g. Fog::Storage.new({ :provider => 'AWS' }.merge(fog_options))
+        attr_accessor :fog_options
+
+        def initialize(syncer_id = nil)
+          super
+
+          @storage_class ||= :standard
+
+          check_configuration
+        end
+
+        private
+
+        def cloud_io
+          @cloud_io ||= CloudIO::S3.new(
+            access_key_id: access_key_id,
+            secret_access_key: secret_access_key,
+            use_iam_profile: use_iam_profile,
+            bucket: bucket,
+            region: region,
+            encryption: encryption,
+            storage_class: storage_class,
+            max_retries: max_retries,
+            retry_waitsec: retry_waitsec,
+            # Syncer can not use multipart upload.
+            chunk_size: 0,
+            fog_options: fog_options
+          )
+        end
+
+        def get_remote_files(remote_base)
+          hash = {}
+          cloud_io.objects(remote_base).each do |object|
+            relative_path = object.key.sub(remote_base + "/", "")
+            hash[relative_path] = object.etag
+          end
+          hash
+        end
+
+        def check_configuration
+          required =
+            if use_iam_profile
+              %w[bucket]
+            else
+              %w[access_key_id secret_access_key bucket]
+            end
+          raise Error, <<-EOS if required.map { |name| send(name) }.any?(&:nil?)
+            Configuration Error
+            #{required.map { |name| "##{name}" }.join(", ")} are all required
+          EOS
+
+          raise Error, <<-EOS if encryption && encryption.to_s.upcase != "AES256"
+            Configuration Error
+            #encryption must be :aes256 or nil
+          EOS
+
+          classes = ["STANDARD", "REDUCED_REDUNDANCY"]
+          raise Error, <<-EOS unless classes.include?(storage_class.to_s.upcase)
+            Configuration Error
+            #storage_class must be :standard or :reduced_redundancy
+          EOS
+        end
+      end # Class S3 < Base
+    end # module Cloud
+  end
+end

data/lib/backup/syncer/rsync/base.rb

@@ -0,0 +1,50 @@
+module Backup
+  module Syncer
+    module RSync
+      class Base < Syncer::Base
+        ##
+        # Additional String or Array of options for the rsync cli
+        attr_accessor :additional_rsync_options
+        attr_accessor :archive
+
+        def initialize(syncer_id = nil, &block)
+          super
+          instance_eval(&block) if block_given?
+
+          @path ||= "~/backups"
+          @archive = @archive.nil? ? true : @archive
+        end
+
+        private
+
+        ##
+        # Common base command for Local/Push/Pull
+        def rsync_command
+          utility(:rsync) << archive_option << mirror_option << exclude_option <<
+            " #{Array(additional_rsync_options).join(" ")}".rstrip
+        end
+
+        def mirror_option
+          mirror ? " --delete" : ""
+        end
+
+        def archive_option
+          archive ? " --archive" : ""
+        end
+
+        def exclude_option
+          excludes.map { |pattern| " --exclude='#{pattern}'" }.join
+        end
+
+        ##
+        # Each path is expanded, since these refer to local paths and are
+        # being shell-quoted. This will also remove any trailing `/` from
+        # each path, as we don't want rsync's "trailing / on source directories"
+        # behavior. This method is used by RSync::Local and RSync::Push.
+        def paths_to_push
+          directories.map { |dir| "'#{File.expand_path(dir)}'" }.join(" ")
+        end
+      end
+    end
+  end
+end

data/lib/backup/syncer/rsync/local.rb

@@ -0,0 +1,27 @@
+module Backup
+  module Syncer
+    module RSync
+      class Local < Base
+        def perform!
+          log!(:started)
+
+          create_dest_path!
+          run("#{rsync_command} #{paths_to_push} '#{dest_path}'")
+
+          log!(:finished)
+        end
+
+        private
+
+        # Expand path, since this is local and shell-quoted.
+        def dest_path
+          @dest_path ||= File.expand_path(path)
+        end
+
+        def create_dest_path!
+          FileUtils.mkdir_p dest_path
+        end
+      end
+    end
+  end
+end

data/lib/backup/syncer/rsync/pull.rb

@@ -0,0 +1,47 @@
+module Backup
+  module Syncer
+    module RSync
+      class Pull < Push
+        def perform!
+          log!(:started)
+          write_password_file!
+
+          create_dest_path!
+          run("#{rsync_command} #{host_options}#{paths_to_pull} " \
+              "'#{dest_path}'")
+
+          log!(:finished)
+        ensure
+          remove_password_file!
+        end
+
+        private
+
+        ##
+        # Returns the syntax for pulling multiple paths from the remote host.
+        # e.g.
+        #   rsync -a -e "ssh -p 22" host:'path1' :'path2' '/dest'
+        #   rsync -a rsync_user@host::'modname/path1' ::'modname/path2' '/dest'
+        #
+        # Remove any preceeding '~/', since these paths are on the remote.
+        # Also remove any trailing `/`, since we don't want rsync's
+        # "trailing / on source directories" behavior.
+        def paths_to_pull
+          sep = mode == :ssh ? ":" : "::"
+          directories.map do |dir|
+            "#{sep}'#{dir.sub(/^~\//, "").sub(/\/$/, "")}'"
+          end.join(" ").sub(/^#{ sep }/, "")
+        end
+
+        # Expand path, since this is local and shell-quoted.
+        def dest_path
+          @dest_path ||= File.expand_path(path)
+        end
+
+        def create_dest_path!
+          FileUtils.mkdir_p dest_path
+        end
+      end
+    end
+  end
+end

data/lib/backup/syncer/rsync/push.rb

@@ -0,0 +1,201 @@
+module Backup
+  module Syncer
+    module RSync
+      class Push < Base
+        ##
+        # Mode of operation
+        #
+        # [:ssh (default)]
+        #   Connects to the remote via SSH.
+        #   Does not use an rsync daemon on the remote.
+        #
+        # [:ssh_daemon]
+        #   Connects to the remote via SSH.
+        #   Spawns a single-use daemon on the remote, which allows certain
+        #   daemon features (like modules) to be used.
+        #
+        # [:rsync_daemon]
+        #   Connects directly to an rsync daemon via TCP.
+        #   Data transferred is not encrypted.
+        #
+        attr_accessor :mode
+
+        ##
+        # Server Address
+        attr_accessor :host
+
+        ##
+        # SSH or RSync port
+        #
+        # For `:ssh` or `:ssh_daemon` mode, this specifies the SSH port to use
+        # and defaults to 22.
+        #
+        # For `:rsync_daemon` mode, this specifies the TCP port to use
+        # and defaults to 873.
+        attr_accessor :port
+
+        ##
+        # SSH User
+        #
+        # If the user running the backup is not the same user that needs to
+        # authenticate with the remote server, specify the user here.
+        #
+        # The user must have SSH keys setup for passphrase-less access to the
+        # remote. If the SSH User does not have passphrase-less keys, or no
+        # default keys in their `~/.ssh` directory, you will need to use the
+        # `-i` option in `:additional_ssh_options` to specify the
+        # passphrase-less key to use.
+        #
+        # Used only for `:ssh` and `:ssh_daemon` modes.
+        attr_accessor :ssh_user
+
+        ##
+        # Additional SSH Options
+        #
+        # Used to supply a String or Array of options to be passed to the SSH
+        # command in `:ssh` and `:ssh_daemon` modes.
+        #
+        # For example, if you need to supply a specific SSH key for the `ssh_user`,
+        # you would set this to: "-i '/path/to/id_rsa'". Which would produce:
+        #
+        #   rsync -e "ssh -p 22 -i '/path/to/id_rsa'"
+        #
+        # Arguments may be single-quoted, but should not contain any double-quotes.
+        #
+        # Used only for `:ssh` and `:ssh_daemon` modes.
+        attr_accessor :additional_ssh_options
+
+        ##
+        # RSync User
+        #
+        # If the user running the backup is not the same user that needs to
+        # authenticate with the rsync daemon, specify the user here.
+        #
+        # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+        attr_accessor :rsync_user
+
+        ##
+        # RSync Password
+        #
+        # If specified, Backup will write the password to a temporary file and
+        # use it with rsync's `--password-file` option for daemon authentication.
+        #
+        # Note that setting this will override `rsync_password_file`.
+        #
+        # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+        attr_accessor :rsync_password
+
+        ##
+        # RSync Password File
+        #
+        # If specified, this path will be passed to rsync's `--password-file`
+        # option for daemon authentication.
+        #
+        # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+        attr_accessor :rsync_password_file
+
+        ##
+        # Flag for compressing (only compresses for the transfer)
+        attr_accessor :compress
+
+        def initialize(syncer_id = nil)
+          super
+
+          @mode ||= :ssh
+          @port ||= mode == :rsync_daemon ? 873 : 22
+          @compress ||= false
+        end
+
+        def perform!
+          log!(:started)
+          write_password_file!
+
+          create_dest_path!
+          run("#{rsync_command} #{paths_to_push} " \
+              "#{host_options}'#{dest_path}'")
+
+          log!(:finished)
+        ensure
+          remove_password_file!
+        end
+
+        private
+
+        ##
+        # Remove any preceeding '~/', since this is on the remote,
+        # and remove any trailing `/`.
+        def dest_path
+          @dest_path ||= path.sub(/^~\//, "").sub(/\/$/, "")
+        end
+
+        ##
+        # Runs a 'mkdir -p' command on the remote to ensure the dest_path exists.
+        # This used because rsync will attempt to create the path, but will only
+        # call 'mkdir' without the '-p' option. This is only applicable in :ssh
+        # mode, and only used if the path would require this.
+        def create_dest_path!
+          return unless mode == :ssh && dest_path.index("/").to_i > 0
+
+          run "#{utility(:ssh)} #{ssh_transport_args} #{host} " +
+            %("mkdir -p '#{dest_path}'")
+        end
+
+        ##
+        # For Push, this will prepend the #dest_path.
+        # For Pull, this will prepend the first path in #paths_to_pull.
+        def host_options
+          if mode == :ssh
+            "#{host}:"
+          else
+            user = "#{rsync_user}@" if rsync_user
+            "#{user}#{host}::"
+          end
+        end
+
+        ##
+        # Common base command, plus options for Push/Pull
+        def rsync_command
+          super << compress_option << password_option << transport_options
+        end
+
+        def compress_option
+          compress ? " --compress" : ""
+        end
+
+        def password_option
+          return "" if mode == :ssh
+
+          path = @password_file ? @password_file.path : rsync_password_file
+          path ? " --password-file='#{File.expand_path(path)}'" : ""
+        end
+
+        def transport_options
+          if mode == :rsync_daemon
+            " --port #{port}"
+          else
+            %( -e "#{utility(:ssh)} #{ssh_transport_args}")
+          end
+        end
+
+        def ssh_transport_args
+          args = "-p #{port} "
+          args << "-l #{ssh_user} " if ssh_user
+          args << Array(additional_ssh_options).join(" ")
+          args.rstrip
+        end
+
+        def write_password_file!
+          return unless rsync_password && mode != :ssh
+
+          @password_file = Tempfile.new("backup-rsync-password")
+          @password_file.write(rsync_password)
+          @password_file.close
+        end
+
+        def remove_password_file!
+          @password_file.delete if @password_file
+        end
+      end
+    end
+  end
+end