backedup 5.0.0.beta.3

data/lib/backup/storage/qiniu.rb

@@ -0,0 +1,65 @@
+require "qiniu"
+
+module Backup
+  module Storage
+    class Qiniu < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Qiniu API credentials
+      attr_accessor :access_key, :secret_key
+
+      ##
+      # Qiniu bucket name
+      attr_accessor :bucket
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @path ||= "backups"
+
+        check_configuration
+        config_credentials
+      end
+
+      private
+
+      def transfer!
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{dest}'..."
+
+          ::Qiniu.upload_file(uptoken: ::Qiniu.generate_upload_token,
+                              bucket: bucket,
+                              file: src,
+                              key: dest)
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{package.time}..."
+        remote_path = remote_path_for(package)
+        package.filenames.each do |filename|
+          ::Qiniu.delete(bucket, File.join(remote_path, filename))
+        end
+      end
+
+      def check_configuration
+        required = %w[access_key secret_key bucket]
+
+        raise Error, <<-EOS if required.map { |name| send(name) }.any?(&:nil?)
+          Configuration Error
+          #{required.map { |name| "##{name}" }.join(", ")} are all required
+        EOS
+      end
+
+      def config_credentials
+        ::Qiniu.establish_connection!(access_key: access_key, secret_key: secret_key)
+      end
+    end
+  end
+end

data/lib/backup/storage/rsync.rb

@@ -0,0 +1,246 @@
+module Backup
+  module Storage
+    class RSync < Base
+      include Utilities::Helpers
+
+      ##
+      # Mode of operation
+      #
+      # [:ssh (default)]
+      #   Connects to the remote via SSH.
+      #   Does not use an rsync daemon on the remote.
+      #
+      # [:ssh_daemon]
+      #   Connects to the remote via SSH.
+      #   Spawns a single-use daemon on the remote, which allows certain
+      #   daemon features (like modules) to be used.
+      #
+      # [:rsync_daemon]
+      #   Connects directly to an rsync daemon via TCP.
+      #   Data transferred is not encrypted.
+      #
+      attr_accessor :mode
+
+      ##
+      # Server Address
+      #
+      # If not specified, the storage operation will be local.
+      attr_accessor :host
+
+      ##
+      # SSH or RSync port
+      #
+      # For `:ssh` or `:ssh_daemon` mode, this specifies the SSH port to use
+      # and defaults to 22.
+      #
+      # For `:rsync_daemon` mode, this specifies the TCP port to use
+      # and defaults to 873.
+      attr_accessor :port
+
+      ##
+      # SSH User
+      #
+      # If the user running the backup is not the same user that needs to
+      # authenticate with the remote server, specify the user here.
+      #
+      # The user must have SSH keys setup for passphrase-less access to the
+      # remote. If the SSH User does not have passphrase-less keys, or no
+      # default keys in their `~/.ssh` directory, you will need to use the
+      # `-i` option in `:additional_ssh_options` to specify the
+      # passphrase-less key to use.
+      #
+      # Used only for `:ssh` and `:ssh_daemon` modes.
+      attr_accessor :ssh_user
+
+      ##
+      # Additional SSH Options
+      #
+      # Used to supply a String or Array of options to be passed to the SSH
+      # command in `:ssh` and `:ssh_daemon` modes.
+      #
+      # For example, if you need to supply a specific SSH key for the `ssh_user`,
+      # you would set this to: "-i '/path/to/id_rsa'". Which would produce:
+      #
+      #   rsync -e "ssh -p 22 -i '/path/to/id_rsa'"
+      #
+      # Arguments may be single-quoted, but should not contain any double-quotes.
+      #
+      # Used only for `:ssh` and `:ssh_daemon` modes.
+      attr_accessor :additional_ssh_options
+
+      ##
+      # RSync User
+      #
+      # If the user running the backup is not the same user that needs to
+      # authenticate with the rsync daemon, specify the user here.
+      #
+      # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+      attr_accessor :rsync_user
+
+      ##
+      # RSync Password
+      #
+      # If specified, Backup will write the password to a temporary file and
+      # use it with rsync's `--password-file` option for daemon authentication.
+      #
+      # Note that setting this will override `rsync_password_file`.
+      #
+      # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+      attr_accessor :rsync_password
+
+      ##
+      # RSync Password File
+      #
+      # If specified, this path will be passed to rsync's `--password-file`
+      # option for daemon authentication.
+      #
+      # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+      attr_accessor :rsync_password_file
+
+      ##
+      # Additional String or Array of options for the rsync cli
+      attr_accessor :additional_rsync_options
+
+      ##
+      # Flag for compressing (only compresses for the transfer)
+      attr_accessor :compress
+
+      ##
+      # Path to store the synced backup package file(s) to.
+      #
+      # If no +host+ is specified, then +path+ will be local, and the only
+      # other used option would be +additional_rsync_options+.
+      # +path+ will be expanded, so '~/my_path' will expand to '$HOME/my_path'.
+      #
+      # If a +host+ is specified, this will be a path on the host.
+      # If +mode+ is `:ssh` (default), then any relative path, or path starting
+      # with '~/' will be relative to the directory the ssh_user is logged
+      # into. For `:ssh_daemon` or `:rsync_daemon` modes, this would reference
+      # an rsync module/path.
+      #
+      # In :ssh_daemon and :rsync_daemon modes, +path+ (or path defined by
+      # your rsync module) must already exist.
+      #
+      # In :ssh mode or local operation (no +host+ specified), +path+ will
+      # be created if needed - either locally, or on the remote for :ssh mode.
+      attr_accessor :path
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @mode ||= :ssh
+        @port ||= mode == :rsync_daemon ? 873 : 22
+        @compress ||= false
+        @path ||= "~/backups"
+      end
+
+      private
+
+      def transfer!
+        write_password_file
+        create_remote_path
+
+        package.filenames.each do |filename|
+          src = "'#{File.join(Config.tmp_path, filename)}'"
+          dest = "#{host_options}'#{File.join(remote_path, filename)}'"
+          Logger.info "Syncing to #{dest}..."
+          run("#{rsync_command} #{src} #{dest}")
+        end
+      ensure
+        remove_password_file
+      end
+
+      ##
+      # Other storages add an additional timestamp directory to this path.
+      # This is not desired here, since we need to transfer the package files
+      # to the same location each time.
+      def remote_path
+        @remote_path ||= begin
+          if host
+            path.sub(/^~\//, "").sub(/\/$/, "")
+          else
+            File.expand_path(path)
+          end
+        end
+      end
+
+      ##
+      # Runs a 'mkdir -p' command on the host (or locally) to ensure the
+      # dest_path exists. This is used because we're transferring a single
+      # file, and rsync won't attempt to create the intermediate directories.
+      #
+      # This is only applicable locally and in :ssh mode.
+      # In :ssh_daemon and :rsync_daemon modes the `path` would include a
+      # module name that must define a path on the remote that already exists.
+      def create_remote_path
+        if host
+          return unless mode == :ssh
+          run "#{utility(:ssh)} #{ssh_transport_args} #{host} " +
+            %("mkdir -p '#{remote_path}'")
+        else
+          FileUtils.mkdir_p(remote_path)
+        end
+      end
+
+      def host_options
+        @host_options ||= begin
+          if !host
+            ""
+          elsif mode == :ssh
+            "#{host}:"
+          else
+            user = "#{rsync_user}@" if rsync_user
+            "#{user}#{host}::"
+          end
+        end
+      end
+
+      def rsync_command
+        @rsync_command ||= begin
+          cmd = utility(:rsync) << " --archive" <<
+            " #{Array(additional_rsync_options).join(" ")}".rstrip
+          cmd << compress_option << password_option << transport_options if host
+          cmd
+        end
+      end
+
+      def compress_option
+        compress ? " --compress" : ""
+      end
+
+      def password_option
+        return "" if mode == :ssh
+
+        path = @password_file ? @password_file.path : rsync_password_file
+        path ? " --password-file='#{File.expand_path(path)}'" : ""
+      end
+
+      def transport_options
+        if mode == :rsync_daemon
+          " --port #{port}"
+        else
+          %( -e "#{utility(:ssh)} #{ssh_transport_args}")
+        end
+      end
+
+      def ssh_transport_args
+        args = "-p #{port} "
+        args << "-l #{ssh_user} " if ssh_user
+        args << Array(additional_ssh_options).join(" ")
+        args.rstrip
+      end
+
+      def write_password_file
+        return unless host && rsync_password && mode != :ssh
+
+        @password_file = Tempfile.new("backup-rsync-password")
+        @password_file.write(rsync_password)
+        @password_file.close
+      end
+
+      def remove_password_file
+        @password_file.delete if @password_file
+      end
+    end
+  end
+end

data/lib/backup/storage/s3.rb

@@ -0,0 +1,155 @@
+require "backup/cloud_io/s3"
+
+module Backup
+  module Storage
+    class S3 < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Amazon Simple Storage Service (S3) Credentials
+      attr_accessor :access_key_id, :secret_access_key, :use_iam_profile
+
+      ##
+      # Amazon S3 bucket name
+      attr_accessor :bucket
+
+      ##
+      # Region of the specified S3 bucket
+      attr_accessor :region
+
+      ##
+      # Multipart chunk size, specified in MiB.
+      #
+      # Each package file larger than +chunk_size+
+      # will be uploaded using S3 Multipart Upload.
+      #
+      # Minimum: 5 (but may be disabled with 0)
+      # Maximum: 5120
+      # Default: 5
+      attr_accessor :chunk_size
+
+      ##
+      # Number of times to retry failed operations.
+      #
+      # Default: 10
+      attr_accessor :max_retries
+
+      ##
+      # Time in seconds to pause before each retry.
+      #
+      # Default: 30
+      attr_accessor :retry_waitsec
+
+      ##
+      # Encryption algorithm to use for Amazon Server-Side Encryption
+      #
+      # Supported values:
+      #
+      # - :aes256
+      #
+      # Default: nil
+      attr_accessor :encryption
+
+      ##
+      # Storage class to use for the S3 objects uploaded
+      #
+      # Supported values:
+      #
+      # - :standard (default)
+      # - :standard_ia
+      # - :reduced_redundancy
+      #
+      # Default: :standard
+      attr_accessor :storage_class
+
+      ##
+      # Additional options to pass along to fog.
+      # e.g. Fog::Storage.new({ :provider => 'AWS' }.merge(fog_options))
+      attr_accessor :fog_options
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @chunk_size     ||= 5 # MiB
+        @max_retries    ||= 10
+        @retry_waitsec  ||= 30
+        @path           ||= "backups"
+        @storage_class  ||= :standard
+
+        @path = @path.sub(/^\//, "")
+
+        check_configuration
+      end
+
+      private
+
+      def cloud_io
+        @cloud_io ||= CloudIO::S3.new(
+          access_key_id: access_key_id,
+          secret_access_key: secret_access_key,
+          use_iam_profile: use_iam_profile,
+          region: region,
+          bucket: bucket,
+          encryption: encryption,
+          storage_class: storage_class,
+          max_retries: max_retries,
+          retry_waitsec: retry_waitsec,
+          chunk_size: chunk_size,
+          fog_options: fog_options
+        )
+      end
+
+      def transfer!
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{bucket}/#{dest}'..."
+          cloud_io.upload(src, dest)
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{package.time}..."
+
+        remote_path = remote_path_for(package)
+        objects = cloud_io.objects(remote_path)
+
+        raise Error, "Package at '#{remote_path}' not found" if objects.empty?
+
+        cloud_io.delete(objects)
+      end
+
+      def check_configuration
+        required =
+          if use_iam_profile
+            %w[bucket]
+          else
+            %w[access_key_id secret_access_key bucket]
+          end
+        raise Error, <<-EOS if required.map { |name| send(name) }.any?(&:nil?)
+          Configuration Error
+          #{required.map { |name| "##{name}" }.join(", ")} are all required
+        EOS
+
+        raise Error, <<-EOS if chunk_size > 0 && !chunk_size.between?(5, 5120)
+          Configuration Error
+          #chunk_size must be between 5 and 5120 (or 0 to disable multipart)
+        EOS
+
+        raise Error, <<-EOS if encryption && encryption.to_s.upcase != "AES256"
+          Configuration Error
+          #encryption must be :aes256 or nil
+        EOS
+
+        classes = ["STANDARD", "STANDARD_IA", "REDUCED_REDUNDANCY"]
+        raise Error, <<-EOS unless classes.include?(storage_class.to_s.upcase)
+          Configuration Error
+          #storage_class must be :standard or :standard_ia or :reduced_redundancy
+        EOS
+      end
+    end
+  end
+end

data/lib/backup/storage/scp.rb

@@ -0,0 +1,65 @@
+require "net/scp"
+
+module Backup
+  module Storage
+    class SCP < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Server credentials
+      attr_accessor :username, :password, :ssh_options
+
+      ##
+      # Server IP Address and SCP port
+      attr_accessor :ip, :port
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @port ||= 22
+        @path ||= "backups"
+        @ssh_options ||= {}
+        path.sub!(/^~\//, "")
+      end
+
+      private
+
+      def connection
+        Net::SSH.start(
+          ip, username, { password: password, port: port }.merge(ssh_options)
+        ) { |ssh| yield ssh }
+      end
+
+      def transfer!
+        connection do |ssh|
+          ssh.exec!("mkdir -p '#{remote_path}'")
+
+          package.filenames.each do |filename|
+            src = File.join(Config.tmp_path, filename)
+            dest = File.join(remote_path, filename)
+            Logger.info "Storing '#{ip}:#{dest}'..."
+            ssh.scp.upload!(src, dest)
+          end
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{package.time}..."
+
+        errors = []
+        connection do |ssh|
+          ssh.exec!("rm -r '#{remote_path_for(package)}'") do |_, stream, data|
+            errors << data if stream == :stderr
+          end
+        end
+        unless errors.empty?
+          raise Error, "Net::SSH reported the following errors:\n" +
+            errors.join("\n")
+        end
+      end
+    end
+  end
+end

data/lib/backup/storage/sftp.rb

@@ -0,0 +1,80 @@
+require "net/sftp"
+
+module Backup
+  module Storage
+    class SFTP < Base
+      include Storage::Cycler
+
+      ##
+      # Server credentials
+      attr_accessor :username, :password, :ssh_options
+
+      ##
+      # Server IP Address and SFTP port
+      attr_accessor :ip, :port
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @ssh_options ||= {}
+        @port        ||= 22
+        @path        ||= "backups"
+        path.sub!(/^~\//, "")
+      end
+
+      private
+
+      def connection
+        Net::SFTP.start(
+          ip, username, { password: password, port: port }.merge(ssh_options)
+        ) { |sftp| yield sftp }
+      end
+
+      def transfer!
+        connection do |sftp|
+          create_remote_path(sftp)
+
+          package.filenames.each do |filename|
+            src = File.join(Config.tmp_path, filename)
+            dest = File.join(remote_path, filename)
+            Logger.info "Storing '#{ip}:#{dest}'..."
+            sftp.upload!(src, dest)
+          end
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{package.time}..."
+
+        remote_path = remote_path_for(package)
+        connection do |sftp|
+          package.filenames.each do |filename|
+            sftp.remove!(File.join(remote_path, filename))
+          end
+
+          sftp.rmdir!(remote_path)
+        end
+      end
+
+      ##
+      # Creates (if they don't exist yet) all the directories on the remote
+      # server in order to upload the backup file. Net::SFTP does not support
+      # paths to directories that don't yet exist when creating new
+      # directories. Instead, we split the parts up in to an array (for each
+      # '/') and loop through that to create the directories one by one.
+      # Net::SFTP raises an exception when the directory it's trying to create
+      # already exists, so we have rescue it
+      def create_remote_path(sftp)
+        path_parts = []
+        remote_path.split("/").each do |path_part|
+          path_parts << path_part
+          begin
+            sftp.mkdir!(path_parts.join("/"))
+          rescue Net::SFTP::StatusException; end
+        end
+      end
+    end
+  end
+end