backedup 5.0.0.beta.3

data/lib/backup/storage/cloud_files.rb

@@ -0,0 +1,156 @@
+require "backup/cloud_io/cloud_files"
+
+module Backup
+  module Storage
+    class CloudFiles < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Rackspace CloudFiles Credentials
+      attr_accessor :username, :api_key
+
+      ##
+      # Rackspace Auth URL (optional)
+      attr_accessor :auth_url
+
+      ##
+      # Rackspace Service Net
+      # (LAN-based transfers to avoid charges and improve performance)
+      attr_accessor :servicenet
+
+      ##
+      # Rackspace Region (optional)
+      attr_accessor :region
+
+      ##
+      # Rackspace Container Name
+      attr_accessor :container
+
+      ##
+      # Rackspace Container Name for SLO Segments
+      # Required if #segment_size is set. Must be different from #container.
+      attr_accessor :segments_container
+
+      ##
+      # SLO Segment size, specified in MiB.
+      #
+      # Each package file larger than +segment_size+
+      # will be uploaded as a Static Large Objects (SLO).
+      #
+      # Defaults to 0 for backward compatibility (pre v.3.7.0),
+      # since #segments_container would be required.
+      #
+      # Minimum: 1 (0 disables SLO support)
+      # Maximum: 5120 (5 GiB)
+      attr_accessor :segment_size
+
+      ##
+      # If set, all backup package files (including SLO segments) will be
+      # scheduled for automatic removal by the server.
+      #
+      # The `keep` option should not be used if this is set,
+      # unless you're transitioning from the `keep` option.
+      attr_accessor :days_to_keep
+
+      ##
+      # Number of times to retry failed operations.
+      #
+      # Default: 10
+      attr_accessor :max_retries
+
+      ##
+      # Time in seconds to pause before each retry.
+      #
+      # Default: 30
+      attr_accessor :retry_waitsec
+
+      ##
+      # Additional options to pass along to fog.
+      # e.g. Fog::Storage.new({ :provider => 'Rackspace' }.merge(fog_options))
+      attr_accessor :fog_options
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @servicenet         ||= false
+        @segment_size       ||= 0
+        @max_retries        ||= 10
+        @retry_waitsec      ||= 30
+
+        @path ||= "backups"
+        path.sub!(/^\//, "")
+
+        check_configuration
+      end
+
+      private
+
+      def cloud_io
+        @cloud_io ||= CloudIO::CloudFiles.new(
+          username: username,
+          api_key: api_key,
+          auth_url: auth_url,
+          region: region,
+          servicenet: servicenet,
+          container: container,
+          segments_container: segments_container,
+          segment_size: segment_size,
+          days_to_keep: days_to_keep,
+          max_retries: max_retries,
+          retry_waitsec: retry_waitsec,
+          fog_options: fog_options
+        )
+      end
+
+      def transfer!
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{container}/#{dest}'..."
+          cloud_io.upload(src, dest)
+        end
+
+        package.no_cycle = true if days_to_keep
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{package.time}..."
+
+        remote_path = remote_path_for(package)
+        objects = cloud_io.objects(remote_path)
+
+        raise Error, "Package at '#{remote_path}' not found" if objects.empty?
+
+        slo_objects, objects = objects.partition(&:slo?)
+        cloud_io.delete_slo(slo_objects)
+        cloud_io.delete(objects)
+      end
+
+      def check_configuration
+        required = %w[username api_key container]
+        raise Error, <<-EOS if required.map { |name| send(name) }.any?(&:nil?)
+          Configuration Error
+          #{required.map { |name| "##{name}" }.join(", ")} are all required
+        EOS
+
+        raise Error, <<-EOS if segment_size > 0 && segments_container.to_s.empty?
+          Configuration Error
+          #segments_container is required if #segment_size is > 0
+        EOS
+
+        raise Error, <<-EOS if container == segments_container
+          Configuration Error
+          #container and #segments_container must not be the same container.
+        EOS
+
+        raise Error, <<-EOS if segment_size > 5120
+          Configuration Error
+          #segment_size is too large (max 5120)
+        EOS
+      end
+    end
+  end
+end

data/lib/backup/storage/cycler.rb

@@ -0,0 +1,70 @@
+module Backup
+  module Storage
+    module Cycler
+      class Error < Backup::Error; end
+
+      private
+
+      # Adds the current package being stored to the YAML cycle data file
+      # and will remove any old package file(s) when the storage limit
+      # set by #keep is exceeded.
+      def cycle!
+        Logger.info "Cycling Started..."
+
+        packages = yaml_load.unshift(package)
+        cycled_packages = []
+
+        if keep.is_a?(Date) || keep.is_a?(Time)
+          cycled_packages = packages.select do |p|
+            p.time_as_object < keep.to_time
+          end
+        else
+          excess = packages.count - keep.to_i
+          cycled_packages = packages.last(excess) if excess > 0
+        end
+
+        saved_packages = packages - cycled_packages
+        cycled_packages.each { |package| delete_package package }
+
+        yaml_save(saved_packages)
+      end
+
+      def delete_package(package)
+        remove!(package) unless package.no_cycle
+      rescue => err
+        Logger.warn Error.wrap(err, <<-EOS)
+            There was a problem removing the following package:
+            Trigger: #{package.trigger} :: Dated: #{package.time}
+            Package included the following #{package.filenames.count} file(s):
+            #{package.filenames.join("\n")}
+          EOS
+      end
+
+      # Returns path to the YAML data file.
+      def yaml_file
+        @yaml_file ||= begin
+          filename = self.class.to_s.split("::").last
+          filename << "-#{storage_id}" if storage_id
+          File.join(Config.data_path, package.trigger, "#{filename}.yml")
+        end
+      end
+
+      # Returns stored Package objects, sorted by #time descending (oldest last).
+      def yaml_load
+        if File.exist?(yaml_file) && !File.zero?(yaml_file)
+          YAML.load_file(yaml_file).sort_by!(&:time).reverse!
+        else
+          []
+        end
+      end
+
+      # Stores the given package objects to the YAML data file.
+      def yaml_save(packages)
+        FileUtils.mkdir_p(File.dirname(yaml_file))
+        File.open(yaml_file, "w") do |file|
+          file.write(packages.to_yaml)
+        end
+      end
+    end
+  end
+end

data/lib/backup/storage/dropbox.rb

@@ -0,0 +1,206 @@
+require "dropbox_sdk"
+
+module Backup
+  module Storage
+    class Dropbox < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Dropbox API credentials
+      attr_accessor :api_key, :api_secret
+
+      ##
+      # Path to store cached authorized session.
+      #
+      # Relative paths will be expanded using Config.root_path,
+      # which by default is ~/Backup unless --root-path was used
+      # on the command line or set in config.rb.
+      #
+      # By default, +cache_path+ is '.cache', which would be
+      # '~/Backup/.cache/' if using the default root_path.
+      attr_accessor :cache_path
+
+      ##
+      # Dropbox Access Type
+      # Valid values are:
+      #   :app_folder (default)
+      #   :dropbox (full access)
+      attr_accessor :access_type
+
+      ##
+      # Chunk size, specified in MiB, for the ChunkedUploader.
+      attr_accessor :chunk_size
+
+      ##
+      # Number of times to retry failed operations.
+      #
+      # Default: 10
+      attr_accessor :max_retries
+
+      ##
+      # Time in seconds to pause before each retry.
+      #
+      # Default: 30
+      attr_accessor :retry_waitsec
+
+      ##
+      # Creates a new instance of the storage object
+      def initialize(model, storage_id = nil)
+        super
+
+        @path           ||= "backups"
+        @cache_path     ||= ".cache"
+        @access_type    ||= :app_folder
+        @chunk_size     ||= 4 # MiB
+        @max_retries    ||= 10
+        @retry_waitsec  ||= 30
+        path.sub!(/^\//, "")
+      end
+
+      private
+
+      ##
+      # The initial connection to Dropbox will provide the user with an
+      # authorization url. The user must open this URL and confirm that the
+      # authorization successfully took place. If this is the case, then the
+      # user hits 'enter' and the session will be properly established.
+      # Immediately after establishing the session, the session will be
+      # serialized and written to a cache file in +cache_path+.
+      # The cached file will be used from that point on to re-establish a
+      # connection with Dropbox at a later time. This allows the user to avoid
+      # having to go to a new Dropbox URL to authorize over and over again.
+      def connection
+        return @connection if @connection
+
+        unless session = cached_session
+          Logger.info "Creating a new session!"
+          session = create_write_and_return_new_session!
+        end
+
+        # will raise an error if session not authorized
+        @connection = DropboxClient.new(session, access_type)
+      rescue => err
+        raise Error.wrap(err, "Authorization Failed")
+      end
+
+      ##
+      # Attempt to load a cached session
+      def cached_session
+        session = false
+        if File.exist?(cached_file)
+          begin
+            session = DropboxSession.deserialize(File.read(cached_file))
+            Logger.info "Session data loaded from cache!"
+          rescue => err
+            Logger.warn Error.wrap(err, <<-EOS)
+              Could not read session data from cache.
+              Cache data might be corrupt.
+            EOS
+          end
+        end
+        session
+      end
+
+      ##
+      # Transfer each of the package files to Dropbox in chunks of +chunk_size+.
+      # Each chunk will be retried +chunk_retries+ times, pausing +retry_waitsec+
+      # between retries, if errors occur.
+      def transfer!
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{dest}'..."
+
+          uploader = nil
+          File.open(src, "r") do |file|
+            uploader = connection.get_chunked_uploader(file, file.stat.size)
+            while uploader.offset < uploader.total_size
+              with_retries do
+                uploader.upload(1024**2 * chunk_size)
+              end
+            end
+          end
+
+          with_retries do
+            uploader.finish(dest)
+          end
+        end
+      rescue => err
+        raise Error.wrap(err, "Upload Failed!")
+      end
+
+      def with_retries
+        retries = 0
+        begin
+          yield
+        rescue StandardError => err
+          retries += 1
+          raise if retries > max_retries
+
+          Logger.info Error.wrap(err, "Retry ##{retries} of #{max_retries}.")
+          sleep(retry_waitsec)
+          retry
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{package.time}..."
+
+        connection.file_delete(remote_path_for(package))
+      end
+
+      def cached_file
+        path = cache_path.start_with?("/") ?
+               cache_path : File.join(Config.root_path, cache_path)
+        File.join(path, api_key + api_secret)
+      end
+
+      ##
+      # Serializes and writes the Dropbox session to a cache file
+      def write_cache!(session)
+        FileUtils.mkdir_p File.dirname(cached_file)
+        File.open(cached_file, "w") do |cache_file|
+          cache_file.write(session.serialize)
+        end
+      end
+
+      ##
+      # Create a new session, write a serialized version of it to the
+      # .cache directory, and return the session object
+      def create_write_and_return_new_session!
+        require "timeout"
+
+        session = DropboxSession.new(api_key, api_secret)
+
+        # grab the request token for session
+        session.get_request_token
+
+        template = Backup::Template.new(
+          session: session, cached_file: cached_file
+        )
+        template.render("storage/dropbox/authorization_url.erb")
+
+        # wait for user to hit 'return' to continue
+        Timeout.timeout(180) { STDIN.gets }
+
+        # this will raise an error if the user did not
+        # visit the authorization_url and grant access
+        #
+        # get the access token from the server
+        # this will be stored with the session in the cache file
+        session.get_access_token
+
+        template.render("storage/dropbox/authorized.erb")
+        write_cache!(session)
+        template.render("storage/dropbox/cache_file_written.erb")
+
+        session
+      rescue => err
+        raise Error.wrap(err, "Could not create or authenticate a new session")
+      end
+    end
+  end
+end

data/lib/backup/storage/ftp.rb

@@ -0,0 +1,116 @@
+require "net/ftp"
+
+module Backup
+  module Storage
+    class FTP < Base
+      include Storage::Cycler
+
+      ##
+      # Server credentials
+      attr_accessor :username, :password
+
+      ##
+      # Server IP Address and FTP port
+      attr_accessor :ip, :port
+
+      ##
+      # Use passive mode?
+      attr_accessor :passive_mode
+
+      ##
+      # Configure connection open and read timeouts.
+      # Net::FTP's open_timeout and read_timeout will both be configured using
+      # this setting.
+      # @!attribute [rw] timeout
+      #   @param [Integer|Float]
+      #   @return [Integer|Float]
+      attr_accessor :timeout
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @port         ||= 21
+        @path         ||= "backups"
+        @passive_mode ||= false
+        @timeout      ||= nil
+        path.sub!(/^~\//, "")
+      end
+
+      private
+
+      ##
+      # Establishes a connection to the remote server
+      #
+      # Note:
+      # Since the FTP port is defined as a constant in the Net::FTP class, and
+      # might be required to change by the user, we dynamically remove and
+      # re-add the constant with the provided port value
+      def connection
+        if Net::FTP.const_defined?(:FTP_PORT)
+          Net::FTP.send(:remove_const, :FTP_PORT)
+        end; Net::FTP.send(:const_set, :FTP_PORT, port)
+
+        # Ensure default passive mode to false.
+        # Note: The default passive setting changed between Ruby 2.2 and 2.3
+        if Net::FTP.respond_to?(:default_passive=)
+          Net::FTP.default_passive = false
+        end
+
+        Net::FTP.open(ip, username, password) do |ftp|
+          if timeout
+            ftp.open_timeout = timeout
+            ftp.read_timeout = timeout
+          end
+          ftp.passive = true if passive_mode
+          yield ftp
+        end
+      end
+
+      def transfer!
+        connection do |ftp|
+          create_remote_path(ftp)
+
+          package.filenames.each do |filename|
+            src = File.join(Config.tmp_path, filename)
+            dest = File.join(remote_path, filename)
+            Logger.info "Storing '#{ip}:#{dest}'..."
+            ftp.put(src, dest)
+          end
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{package.time}..."
+
+        remote_path = remote_path_for(package)
+        connection do |ftp|
+          package.filenames.each do |filename|
+            ftp.delete(File.join(remote_path, filename))
+          end
+
+          ftp.rmdir(remote_path)
+        end
+      end
+
+      ##
+      # Creates (if they don't exist yet) all the directories on the remote
+      # server in order to upload the backup file. Net::FTP does not support
+      # paths to directories that don't yet exist when creating new
+      # directories. Instead, we split the parts up in to an array (for each
+      # '/') and loop through that to create the directories one by one.
+      # Net::FTP raises an exception when the directory it's trying to create
+      # already exists, so we have rescue it
+      def create_remote_path(ftp)
+        path_parts = []
+        remote_path.split("/").each do |path_part|
+          path_parts << path_part
+          begin
+            ftp.mkdir(path_parts.join("/"))
+          rescue Net::FTPPermError; end
+        end
+      end
+    end
+  end
+end

data/lib/backup/storage/gcs.rb

@@ -0,0 +1,93 @@
+# encoding: utf-8
+require "backup/cloud_io/gcs"
+
+module Backup
+  module Storage
+    class GCS < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Google XML API credentials
+      attr_accessor :google_storage_secret_access_key
+      attr_accessor :google_storage_access_key_id
+
+      ##
+      # Amazon GCS bucket name
+      attr_accessor :bucket
+
+      ##
+      # Number of times to retry failed operations.
+      #
+      # Default: 10
+      attr_accessor :max_retries
+
+      ##
+      # Time in seconds to pause before each retry.
+      #
+      # Default: 30
+      attr_accessor :retry_waitsec
+
+      ##
+      # Additional options to pass along to fog.
+      # e.g. Fog::Storage.new({ :provider => 'AWS' }.merge(fog_options))
+      attr_accessor :fog_options
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @max_retries    ||= 10
+        @retry_waitsec  ||= 30
+        @path           ||= "backups"
+        @path = @path.sub(%r{^/}, "")
+
+        check_configuration
+      end
+
+      private
+
+      def cloud_io
+        @cloud_io ||= CloudIO::GCS.new(
+          google_storage_secret_access_key: google_storage_secret_access_key,
+          google_storage_access_key_id: google_storage_access_key_id,
+          bucket: bucket,
+          max_retries: max_retries,
+          retry_waitsec: retry_waitsec,
+          fog_options: fog_options
+        )
+      end
+
+      def transfer!
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{bucket}/#{dest}'..."
+          cloud_io.upload(src, dest)
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{package.time}..."
+
+        remote_path = remote_path_for(package)
+        objects = cloud_io.objects(remote_path)
+
+        raise Error, "Package at '#{remote_path}' not found" if objects.empty?
+
+        cloud_io.delete(objects)
+      end
+
+      def check_configuration
+        required =
+          %w(google_storage_secret_access_key google_storage_access_key_id bucket)
+
+        raise Error, <<-EOS if required.map { |name| send(name) }.any?(&:nil?)
+          Configuration Error
+          #{required.map { |name| "##{name}" }.join(", ")} are all required
+        EOS
+      end
+    end
+  end
+end

data/lib/backup/storage/local.rb

@@ -0,0 +1,61 @@
+module Backup
+  module Storage
+    class Local < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @path ||= "~/backups"
+      end
+
+      private
+
+      def transfer!
+        FileUtils.mkdir_p(remote_path)
+
+        transfer_method = package_movable? ? :mv : :cp
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{dest}'..."
+
+          FileUtils.send(transfer_method, src, dest)
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{package.time}..."
+
+        FileUtils.rm_r(remote_path_for(package))
+      end
+
+      # expanded since this is a local path
+      def remote_path(pkg = package)
+        File.expand_path(super)
+      end
+      alias :remote_path_for :remote_path
+
+      ##
+      # If this Local Storage is not the last Storage for the Model,
+      # force the transfer to use a *copy* operation and issue a warning.
+      def package_movable?
+        if self == model.storages.last
+          true
+        else
+          Logger.warn Error.new(<<-EOS)
+            Local File Copy Warning!
+            The final backup file(s) for '#{model.label}' (#{model.trigger})
+            will be *copied* to '#{remote_path}'
+            To avoid this, when using more than one Storage, the 'Local' Storage
+            should be added *last* so the files may be *moved* to their destination.
+          EOS
+          false
+        end
+      end
+    end
+  end
+end