nfm-backup 4.0.1a

data/lib/backup/storage/ninefold.rb

@@ -0,0 +1,74 @@
+# encoding: utf-8
+require 'fog'
+
+module Backup
+  module Storage
+    class Ninefold < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Ninefold Credentials
+      attr_accessor :storage_token, :storage_secret
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @path ||= 'backups'
+        path.sub!(/^\//, '')
+      end
+
+      private
+
+      def connection
+        @connection ||= Fog::Storage.new(
+          :provider                => 'Ninefold',
+          :ninefold_storage_token  => storage_token,
+          :ninefold_storage_secret => storage_secret
+        )
+      end
+
+      def transfer!
+        directory = directory_for(remote_path, true)
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{ dest }'..."
+          File.open(src, 'r') do |file|
+            directory.files.create(:key => filename, :body => file)
+          end
+        end
+      end
+
+      ##
+      # Queries the connection for the directory for the given +remote_path+
+      # Returns nil if not found, or creates the directory if +create+ is true.
+      def directory_for(remote_path, create = false)
+        directory = connection.directories.get(remote_path)
+        if directory.nil? && create
+          directory = connection.directories.create(:key => remote_path)
+        end
+        directory
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{ package.time }..."
+
+        remote_path = remote_path_for(package)
+        directory = directory_for(remote_path)
+
+        raise Error, "Directory at '#{ remote_path }' not found" unless directory
+
+        package.filenames.each do |filename|
+          file = directory.files.get(filename)
+          file.destroy if file
+        end
+
+        directory.destroy
+      end
+
+    end
+  end
+end

data/lib/backup/storage/rsync.rb

@@ -0,0 +1,248 @@
+# encoding: utf-8
+
+module Backup
+  module Storage
+    class RSync < Base
+      include Utilities::Helpers
+
+      ##
+      # Mode of operation
+      #
+      # [:ssh (default)]
+      #   Connects to the remote via SSH.
+      #   Does not use an rsync daemon on the remote.
+      #
+      # [:ssh_daemon]
+      #   Connects to the remote via SSH.
+      #   Spawns a single-use daemon on the remote, which allows certain
+      #   daemon features (like modules) to be used.
+      #
+      # [:rsync_daemon]
+      #   Connects directly to an rsync daemon via TCP.
+      #   Data transferred is not encrypted.
+      #
+      attr_accessor :mode
+
+      ##
+      # Server Address
+      #
+      # If not specified, the storage operation will be local.
+      attr_accessor :host
+
+      ##
+      # SSH or RSync port
+      #
+      # For `:ssh` or `:ssh_daemon` mode, this specifies the SSH port to use
+      # and defaults to 22.
+      #
+      # For `:rsync_daemon` mode, this specifies the TCP port to use
+      # and defaults to 873.
+      attr_accessor :port
+
+      ##
+      # SSH User
+      #
+      # If the user running the backup is not the same user that needs to
+      # authenticate with the remote server, specify the user here.
+      #
+      # The user must have SSH keys setup for passphrase-less access to the
+      # remote. If the SSH User does not have passphrase-less keys, or no
+      # default keys in their `~/.ssh` directory, you will need to use the
+      # `-i` option in `:additional_ssh_options` to specify the
+      # passphrase-less key to use.
+      #
+      # Used only for `:ssh` and `:ssh_daemon` modes.
+      attr_accessor :ssh_user
+
+      ##
+      # Additional SSH Options
+      #
+      # Used to supply a String or Array of options to be passed to the SSH
+      # command in `:ssh` and `:ssh_daemon` modes.
+      #
+      # For example, if you need to supply a specific SSH key for the `ssh_user`,
+      # you would set this to: "-i '/path/to/id_rsa'". Which would produce:
+      #
+      #   rsync -e "ssh -p 22 -i '/path/to/id_rsa'"
+      #
+      # Arguments may be single-quoted, but should not contain any double-quotes.
+      #
+      # Used only for `:ssh` and `:ssh_daemon` modes.
+      attr_accessor :additional_ssh_options
+
+      ##
+      # RSync User
+      #
+      # If the user running the backup is not the same user that needs to
+      # authenticate with the rsync daemon, specify the user here.
+      #
+      # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+      attr_accessor :rsync_user
+
+      ##
+      # RSync Password
+      #
+      # If specified, Backup will write the password to a temporary file and
+      # use it with rsync's `--password-file` option for daemon authentication.
+      #
+      # Note that setting this will override `rsync_password_file`.
+      #
+      # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+      attr_accessor :rsync_password
+
+      ##
+      # RSync Password File
+      #
+      # If specified, this path will be passed to rsync's `--password-file`
+      # option for daemon authentication.
+      #
+      # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+      attr_accessor :rsync_password_file
+
+      ##
+      # Additional String or Array of options for the rsync cli
+      attr_accessor :additional_rsync_options
+
+      ##
+      # Flag for compressing (only compresses for the transfer)
+      attr_accessor :compress
+
+      ##
+      # Path to store the synced backup package file(s) to.
+      #
+      # If no +host+ is specified, then +path+ will be local, and the only
+      # other used option would be +additional_rsync_options+.
+      # +path+ will be expanded, so '~/my_path' will expand to '$HOME/my_path'.
+      #
+      # If a +host+ is specified, this will be a path on the host.
+      # If +mode+ is `:ssh` (default), then any relative path, or path starting
+      # with '~/' will be relative to the directory the ssh_user is logged
+      # into. For `:ssh_daemon` or `:rsync_daemon` modes, this would reference
+      # an rsync module/path.
+      #
+      # In :ssh_daemon and :rsync_daemon modes, +path+ (or path defined by
+      # your rsync module) must already exist.
+      #
+      # In :ssh mode or local operation (no +host+ specified), +path+ will
+      # be created if needed - either locally, or on the remote for :ssh mode.
+      attr_accessor :path
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @mode ||= :ssh
+        @port ||= mode == :rsync_daemon ? 873 : 22
+        @compress ||= false
+        @path ||= '~/backups'
+      end
+
+      private
+
+      def transfer!
+        write_password_file
+        create_remote_path
+
+        package.filenames.each do |filename|
+          src = "'#{ File.join(Config.tmp_path, filename) }'"
+          dest = "#{ host_options }'#{ File.join(remote_path, filename) }'"
+          Logger.info "Syncing to #{ dest }..."
+          run("#{ rsync_command } #{ src } #{ dest }")
+        end
+      ensure
+        remove_password_file
+      end
+
+      ##
+      # Other storages add an additional timestamp directory to this path.
+      # This is not desired here, since we need to transfer the package files
+      # to the same location each time.
+      def remote_path
+        @remote_path ||= begin
+          if host
+            path.sub(/^~\//, '').sub(/\/$/, '')
+          else
+            File.expand_path(path)
+          end
+        end
+      end
+
+      ##
+      # Runs a 'mkdir -p' command on the host (or locally) to ensure the
+      # dest_path exists. This is used because we're transferring a single
+      # file, and rsync won't attempt to create the intermediate directories.
+      #
+      # This is only applicable locally and in :ssh mode.
+      # In :ssh_daemon and :rsync_daemon modes the `path` would include a
+      # module name that must define a path on the remote that already exists.
+      def create_remote_path
+        if host
+          run("#{ utility(:ssh) } #{ ssh_transport_args } #{ host } " +
+                  %Q["mkdir -p '#{ remote_path }'"]) if mode == :ssh
+        else
+          FileUtils.mkdir_p(remote_path)
+        end
+      end
+
+      def host_options
+        @host_options ||= begin
+          if !host
+            ''
+          elsif mode == :ssh
+            "#{ host }:"
+          else
+            user = "#{ rsync_user }@" if rsync_user
+            "#{ user }#{ host }::"
+          end
+        end
+      end
+
+      def rsync_command
+        @rsync_command ||= begin
+          cmd = utility(:rsync) << ' --archive' <<
+              " #{ Array(additional_rsync_options).join(' ') }".rstrip
+          cmd << compress_option << password_option << transport_options if host
+          cmd
+        end
+      end
+
+      def compress_option
+        compress ? ' --compress' : ''
+      end
+
+      def password_option
+        return '' if mode == :ssh
+
+        path = @password_file ? @password_file.path : rsync_password_file
+        path ? " --password-file='#{ File.expand_path(path) }'" : ''
+      end
+
+      def transport_options
+        if mode == :rsync_daemon
+          " --port #{ port }"
+        else
+          %Q[ -e "#{ utility(:ssh) } #{ ssh_transport_args }"]
+        end
+      end
+
+      def ssh_transport_args
+        args = "-p #{ port } "
+        args << "-l #{ ssh_user } " if ssh_user
+        args << Array(additional_ssh_options).join(' ')
+        args.rstrip
+      end
+
+      def write_password_file
+        return unless host && rsync_password && mode != :ssh
+
+        @password_file = Tempfile.new('backup-rsync-password')
+        @password_file.write(rsync_password)
+        @password_file.close
+      end
+
+      def remove_password_file
+        @password_file.delete if @password_file
+      end
+
+    end
+  end
+end

data/lib/backup/storage/s3.rb

@@ -0,0 +1,154 @@
+# encoding: utf-8
+require 'backup/cloud_io/s3'
+
+module Backup
+  module Storage
+    class S3 < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Amazon Simple Storage Service (S3) Credentials
+      attr_accessor :access_key_id, :secret_access_key, :use_iam_profile
+
+      ##
+      # Amazon S3 bucket name
+      attr_accessor :bucket
+
+      ##
+      # Region of the specified S3 bucket
+      attr_accessor :region
+
+      ##
+      # Multipart chunk size, specified in MiB.
+      #
+      # Each package file larger than +chunk_size+
+      # will be uploaded using S3 Multipart Upload.
+      #
+      # Minimum: 5 (but may be disabled with 0)
+      # Maximum: 5120
+      # Default: 5
+      attr_accessor :chunk_size
+
+      ##
+      # Number of times to retry failed operations.
+      #
+      # Default: 10
+      attr_accessor :max_retries
+
+      ##
+      # Time in seconds to pause before each retry.
+      #
+      # Default: 30
+      attr_accessor :retry_waitsec
+
+      ##
+      # Encryption algorithm to use for Amazon Server-Side Encryption
+      #
+      # Supported values:
+      #
+      # - :aes256
+      #
+      # Default: nil
+      attr_accessor :encryption
+
+      ##
+      # Storage class to use for the S3 objects uploaded
+      #
+      # Supported values:
+      #
+      # - :standard (default)
+      # - :reduced_redundancy
+      #
+      # Default: :standard
+      attr_accessor :storage_class
+
+      ##
+      # Additional options to pass along to fog.
+      # e.g. Fog::Storage.new({ :provider => 'AWS' }.merge(fog_options))
+      attr_accessor :fog_options
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @chunk_size     ||= 5 # MiB
+        @max_retries    ||= 10
+        @retry_waitsec  ||= 30
+        @path           ||= 'backups'
+        @storage_class  ||= :standard
+        path.sub!(/^\//, '')
+
+        check_configuration
+      end
+
+      private
+
+      def cloud_io
+        @cloud_io ||= CloudIO::S3.new(
+          :access_key_id      => access_key_id,
+          :secret_access_key  => secret_access_key,
+          :use_iam_profile    => use_iam_profile,
+          :region             => region,
+          :bucket             => bucket,
+          :encryption         => encryption,
+          :storage_class      => storage_class,
+          :max_retries        => max_retries,
+          :retry_waitsec      => retry_waitsec,
+          :chunk_size         => chunk_size,
+          :fog_options        => fog_options
+        )
+      end
+
+      def transfer!
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{ bucket }/#{ dest }'..."
+          cloud_io.upload(src, dest)
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{ package.time }..."
+
+        remote_path = remote_path_for(package)
+        objects = cloud_io.objects(remote_path)
+
+        raise Error, "Package at '#{ remote_path }' not found" if objects.empty?
+
+        cloud_io.delete(objects)
+      end
+
+      def check_configuration
+        if use_iam_profile
+          required = %w{ bucket }
+        else
+          required = %w{ access_key_id secret_access_key bucket }
+        end
+        raise Error, <<-EOS if required.map {|name| send(name) }.any?(&:nil?)
+          Configuration Error
+          #{ required.map {|name| "##{ name }"}.join(', ') } are all required
+        EOS
+
+        raise Error, <<-EOS if chunk_size > 0 && !chunk_size.between?(5, 5120)
+          Configuration Error
+          #chunk_size must be between 5 and 5120 (or 0 to disable multipart)
+        EOS
+
+        raise Error, <<-EOS if encryption && encryption.to_s.upcase != 'AES256'
+          Configuration Error
+          #encryption must be :aes256 or nil
+        EOS
+
+        classes = ['STANDARD', 'REDUCED_REDUNDANCY']
+        raise Error, <<-EOS unless classes.include?(storage_class.to_s.upcase)
+          Configuration Error
+          #storage_class must be :standard or :reduced_redundancy
+        EOS
+      end
+
+    end
+  end
+end