venet-backup 4.1.3

data/lib/backup/storage/cycler.rb

@@ -0,0 +1,65 @@
+# encoding: utf-8
+
+module Backup
+  module Storage
+    module Cycler
+      class Error < Backup::Error; end
+
+      private
+
+      # Adds the current package being stored to the YAML cycle data file
+      # and will remove any old package file(s) when the storage limit
+      # set by #keep is exceeded.
+      def cycle!
+        Logger.info 'Cycling Started...'
+
+        packages = yaml_load.unshift(package)
+        excess = packages.count - keep.to_i
+
+        if excess > 0
+          packages.pop(excess).each do |pkg|
+            begin
+              remove!(pkg) unless pkg.no_cycle
+            rescue => err
+              Logger.warn Error.wrap(err, <<-EOS)
+                There was a problem removing the following package:
+                Trigger: #{pkg.trigger} :: Dated: #{pkg.time}
+                Package included the following #{ pkg.filenames.count } file(s):
+                #{ pkg.filenames.join("\n") }
+              EOS
+            end
+          end
+        end
+
+        yaml_save(packages)
+      end
+
+      # Returns path to the YAML data file.
+      def yaml_file
+        @yaml_file ||= begin
+          filename = self.class.to_s.split('::').last
+          filename << "-#{ storage_id }" if storage_id
+          File.join(Config.data_path, package.trigger, "#{ filename }.yml")
+        end
+      end
+
+      # Returns stored Package objects, sorted by #time descending (oldest last).
+      def yaml_load
+        if File.exist?(yaml_file) && !File.zero?(yaml_file)
+          YAML.load_file(yaml_file).sort_by!(&:time).reverse!
+        else
+          []
+        end
+      end
+
+      # Stores the given package objects to the YAML data file.
+      def yaml_save(packages)
+        FileUtils.mkdir_p(File.dirname(yaml_file))
+        File.open(yaml_file, 'w') do |file|
+          file.write(packages.to_yaml)
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/storage/dropbox.rb

@@ -0,0 +1,236 @@
+# encoding: utf-8
+require 'dropbox_sdk'
+
+module Backup
+  module Storage
+    class Dropbox < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Dropbox API credentials
+      attr_accessor :api_key, :api_secret
+
+      ##
+      # Path to store cached authorized session.
+      #
+      # Relative paths will be expanded using Config.root_path,
+      # which by default is ~/Backup unless --root-path was used
+      # on the command line or set in config.rb.
+      #
+      # By default, +cache_path+ is '.cache', which would be
+      # '~/Backup/.cache/' if using the default root_path.
+      attr_accessor :cache_path
+
+      ##
+      # Dropbox Access Type
+      # Valid values are:
+      #   :app_folder (default)
+      #   :dropbox (full access)
+      attr_accessor :access_type
+
+      ##
+      # Chunk size, specified in MiB, for the ChunkedUploader.
+      attr_accessor :chunk_size
+
+      ##
+      # Number of times to retry failed operations.
+      #
+      # Default: 10
+      attr_accessor :max_retries
+
+      ##
+      # Time in seconds to pause before each retry.
+      #
+      # Default: 30
+      attr_accessor :retry_waitsec
+
+      ##
+      # Creates a new instance of the storage object
+      def initialize(model, storage_id = nil)
+        super
+
+        @path           ||= 'backups'
+        @cache_path     ||= '.cache'
+        @access_type    ||= :app_folder
+        @chunk_size     ||= 4 # MiB
+        @max_retries    ||= 10
+        @retry_waitsec  ||= 30
+        path.sub!(/^\//, '')
+      end
+
+      private
+
+      ##
+      # The initial connection to Dropbox will provide the user with an
+      # authorization url. The user must open this URL and confirm that the
+      # authorization successfully took place. If this is the case, then the
+      # user hits 'enter' and the session will be properly established.
+      # Immediately after establishing the session, the session will be
+      # serialized and written to a cache file in +cache_path+.
+      # The cached file will be used from that point on to re-establish a
+      # connection with Dropbox at a later time. This allows the user to avoid
+      # having to go to a new Dropbox URL to authorize over and over again.
+      def connection
+        return @connection if @connection
+
+        unless session = cached_session
+          Logger.info "Creating a new session!"
+          session = create_write_and_return_new_session!
+        end
+
+        # will raise an error if session not authorized
+        @connection = DropboxClient.new(session, access_type)
+
+      rescue => err
+        raise Error.wrap(err, 'Authorization Failed')
+      end
+
+      ##
+      # Attempt to load a cached session
+      def cached_session
+        session = false
+        if File.exist?(cached_file)
+          begin
+            session = DropboxSession.deserialize(File.read(cached_file))
+            Logger.info "Session data loaded from cache!"
+
+          rescue => err
+            Logger.warn Error.wrap(err, <<-EOS)
+              Could not read session data from cache.
+              Cache data might be corrupt.
+            EOS
+          end
+        end
+        session
+      end
+
+      ##
+      # Transfer each of the package files to Dropbox in chunks of +chunk_size+.
+      # Each chunk will be retried +chunk_retries+ times, pausing +retry_waitsec+
+      # between retries, if errors occur.
+      def transfer!
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{ dest }'..."
+
+          uploader = nil
+          File.open(src, 'r') do |file|
+            uploader = connection.get_chunked_uploader(file, file.stat.size)
+            while uploader.offset < uploader.total_size
+              with_retries do
+                uploader.upload(1024**2 * chunk_size)
+              end
+            end
+          end
+
+          with_retries do
+            uploader.finish(dest)
+          end
+        end
+
+      rescue => err
+        raise Error.wrap(err, 'Upload Failed!')
+      end
+
+      def with_retries
+        retries = 0
+        begin
+          yield
+        rescue StandardError => err
+          retries += 1
+          raise if retries > max_retries
+
+          Logger.info Error.wrap(err, "Retry ##{ retries } of #{ max_retries }.")
+          sleep(retry_waitsec)
+          retry
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{ package.time }..."
+
+        connection.file_delete(remote_path_for(package))
+      end
+
+      def cached_file
+        path = cache_path.start_with?('/') ?
+               cache_path : File.join(Config.root_path, cache_path)
+        File.join(path, api_key + api_secret)
+      end
+
+      ##
+      # Serializes and writes the Dropbox session to a cache file
+      def write_cache!(session)
+        FileUtils.mkdir_p File.dirname(cached_file)
+        File.open(cached_file, "w") do |cache_file|
+          cache_file.write(session.serialize)
+        end
+      end
+
+      ##
+      # Create a new session, write a serialized version of it to the
+      # .cache directory, and return the session object
+      def create_write_and_return_new_session!
+        require 'timeout'
+
+        session = DropboxSession.new(api_key, api_secret)
+
+        # grab the request token for session
+        session.get_request_token
+
+        template = Backup::Template.new(
+          {:session => session, :cached_file => cached_file}
+        )
+        template.render("storage/dropbox/authorization_url.erb")
+
+        # wait for user to hit 'return' to continue
+        Timeout::timeout(180) { STDIN.gets }
+
+        # this will raise an error if the user did not
+        # visit the authorization_url and grant access
+        #
+        # get the access token from the server
+        # this will be stored with the session in the cache file
+        session.get_access_token
+
+        template.render("storage/dropbox/authorized.erb")
+        write_cache!(session)
+        template.render("storage/dropbox/cache_file_written.erb")
+
+        session
+
+      rescue => err
+        raise Error.wrap(err, 'Could not create or authenticate a new session')
+      end
+
+    end
+  end
+end
+
+# Patch for dropbox-ruby-sdk-1.5.1
+class DropboxClient
+  class ChunkedUploader
+    def upload(chunk_size = 1024**2 * 4)
+      while @offset < @total_size
+        @file_obj.seek(@offset) unless @file_obj.pos == @offset
+        data = @file_obj.read(chunk_size)
+
+        begin
+          resp = @client.parse_response(
+            @client.partial_chunked_upload(data, @upload_id, @offset)
+          )
+        rescue DropboxError => err
+          resp = JSON.parse(err.http_response.body) rescue {}
+          raise err unless resp['offset']
+        end
+
+        @offset = resp['offset']
+        @upload_id ||= resp['upload_id']
+      end
+    end
+  end
+end

data/lib/backup/storage/ftp.rb

@@ -0,0 +1,98 @@
+# encoding: utf-8
+require 'net/ftp'
+
+module Backup
+  module Storage
+    class FTP < Base
+      include Storage::Cycler
+
+      ##
+      # Server credentials
+      attr_accessor :username, :password
+
+      ##
+      # Server IP Address and FTP port
+      attr_accessor :ip, :port
+
+      ##
+      # use passive mode?
+      attr_accessor :passive_mode
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @port         ||= 21
+        @path         ||= 'backups'
+        @passive_mode ||= false
+        path.sub!(/^~\//, '')
+      end
+
+      private
+
+      ##
+      # Establishes a connection to the remote server
+      #
+      # Note:
+      # Since the FTP port is defined as a constant in the Net::FTP class, and
+      # might be required to change by the user, we dynamically remove and
+      # re-add the constant with the provided port value
+      def connection
+        if Net::FTP.const_defined?(:FTP_PORT)
+          Net::FTP.send(:remove_const, :FTP_PORT)
+        end; Net::FTP.send(:const_set, :FTP_PORT, port)
+
+        Net::FTP.open(ip, username, password) do |ftp|
+          ftp.passive = true if passive_mode
+          yield ftp
+        end
+      end
+
+      def transfer!
+        connection do |ftp|
+          create_remote_path(ftp)
+
+          package.filenames.each do |filename|
+            src = File.join(Config.tmp_path, filename)
+            dest = File.join(remote_path, filename)
+            Logger.info "Storing '#{ ip }:#{ dest }'..."
+            ftp.put(src, dest)
+          end
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{ package.time }..."
+
+        remote_path = remote_path_for(package)
+        connection do |ftp|
+          package.filenames.each do |filename|
+            ftp.delete(File.join(remote_path, filename))
+          end
+
+          ftp.rmdir(remote_path)
+        end
+      end
+
+      ##
+      # Creates (if they don't exist yet) all the directories on the remote
+      # server in order to upload the backup file. Net::FTP does not support
+      # paths to directories that don't yet exist when creating new
+      # directories. Instead, we split the parts up in to an array (for each
+      # '/') and loop through that to create the directories one by one.
+      # Net::FTP raises an exception when the directory it's trying to create
+      # already exists, so we have rescue it
+      def create_remote_path(ftp)
+        path_parts = Array.new
+        remote_path.split('/').each do |path_part|
+          path_parts << path_part
+          begin
+            ftp.mkdir(path_parts.join('/'))
+          rescue Net::FTPPermError; end
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/storage/google/google_drive_auth.rb

@@ -0,0 +1,96 @@
+module Backup
+  class GoogleDriveAuth
+    include Backup::Config::Helpers
+
+    attr_accessor :client, :drive
+    
+    def initialize(options = {})
+      defaults = {
+        client_id: "",
+        client_secret: "", 
+        api_version: "v2",
+        cache_path: ".cache",
+        scope: "https://www.googleapis.com/auth/drive",
+        authorization_uri: "https://accounts.google.com/o/oauth2/auth",
+        token_credential_uri: "https://accounts.google.com/o/oauth2/token",
+        redirect_uri: "urn:ietf:wg:oauth:2.0:oob"
+      }
+
+      @options = defaults.merge(options)
+
+      connect
+    end
+
+    def connect
+      @client = (auth.nil? ? authorize_and_cache : authorized_client)
+      @drive = @client.discovered_api("drive", @options[:api_version])
+    rescue => err
+      raise Error.wrap(err, "Authorization Failed")
+    end
+
+    def file_storage
+      Google::APIClient::FileStorage.new(credential_store_file)
+    end
+
+    def auth
+      file_storage.authorization
+    end
+
+    def authorize_and_cache
+      require "timeout"
+
+      Logger.info "Creating a new authorization!"
+
+      client = Google::APIClient.new(
+        application_name: "Ruby backup to google drive",
+        application_version: "0.1.0"
+      )
+
+      authorization = Signet::OAuth2::Client.new(@options)
+
+      template = Backup::Template.new(
+        auth: authorization,
+        credential_store_file: credential_store_file
+      )
+      template.render("storage/google_drive/authorization_url.erb")
+
+      Timeout::timeout(180) {
+        authorization.code = STDIN.gets
+      }
+
+      authorization.fetch_access_token!
+
+      template.render("storage/google_drive/authorized.erb")
+      write_cache!(authorization)
+      template.render("storage/google_drive/cache_file_written.erb")
+
+      client.authorization = authorization
+
+      client
+
+    rescue => err
+      raise Error.wrap(err, "Could not authorize")
+    end
+
+    def authorized_client
+      client = Google::APIClient.new(
+        application_name: "Ruby backup to google drive",
+        application_version: "0.1.0"
+      )
+
+      client.authorization = auth
+
+      client
+    end
+
+    def credential_store_file
+      path = @options[:cache_path].start_with?("/") ? @options[:cache_path] : File.join(Config.root_path, @options[:cache_path])
+      File.join(path, @options[:client_id] + @options[:client_secret])
+    end
+
+    def write_cache!(authorization)
+      FileUtils.mkdir_p File.dirname(credential_store_file)
+      file_storage.write_credentials(authorization)
+    end
+  end
+end