backup-googledrive 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6909a4f268118492f668382203b3c332e19a80c8
-  data.tar.gz: eac80a0ce6a50a9314f253a86f93c9c2b9b083ae
+  metadata.gz: 45c9de7b9020a457a977dc6b804b3bed1598c661
+  data.tar.gz: deb947444c13d07f39a145529f4c275527d90f6a
 SHA512:
-  metadata.gz: c0b37e31e8bf83fde792b284d30b564426425b3603596b73c89ef71426574b5e99b25d5bb058a73b1a9b865d20d5688d8e0a93bacf795675d34abda55c3fb116
-  data.tar.gz: 9a3693d57cb08c8d594b0903a075e919db6184f00196a56ab26118a16cd1e73d9abbbeda94739827af671b5239d3d3047e7c301b719702b79bb3f450c39fb078
+  metadata.gz: 775d57b43a92e45393fdb893fe809e9186909575718d651ff13d003c8a0724457c96496f85e73b283d54f8f19fd70fce1e480b41de13285a03c8918dd180ed14
+  data.tar.gz: 0def6331e624ae9d26add60519e035b571e11a78370217652c667d3074ee4ca81279d64eb65951be726da1b6d0320211b6bbf952afaebdf0a4205fcecafa8fcd

data/lib/backup-googledrive.rb

@@ -1,6 +1,216 @@
+require 'backup'
+
 module Backup
   module Storage
-    autoload :GoogleDrive, File.join(File.dirname(__FILE__), 'storage', 'googledrive')
-    Backup::Config::DSL.const_set('GoogleDrive', Module.new)
+    class GoogleDrive < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Path to the rclone executable
+      attr_accessor :rclone_exec
+
+      ##
+      # Path to the rclone config file
+      attr_accessor :rclone_config
+
+      ##
+      # Name of the rclone remote drive
+      attr_accessor :rclone_drive
+
+      ##
+      # Chunk size, specified in bytes (power of two, i.e. 8M = 8 * 1024 * 1024)
+      attr_accessor :chunk_size
+
+      ##
+      # Number of times to retry failed operations.
+      #
+      # Default: 10
+      attr_accessor :max_retries
+
+      ##
+      # Creates a new instance of the storage object
+      def initialize(model, storage_id = nil)
+        super
+        @rclone_exec    ||= "/usr/bin/rclone"
+        @rclone_config  ||= "/root/.config/rclone/rclone.conf"
+        @rclone_drive   ||= "remote"
+        @path           ||= ""
+        @chunk_size     ||= 8388608
+        @max_retries    ||= 3
+        @cmd_base         = "#{rclone_exec} --config #{rclone_config} --retries #{max_retries} --drive-chunk-size=#{chunk_size} --drive-use-trash=false"
+        path.sub!(/^\//, "")
+      end
+
+      private
+
+
+      def transfer!
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{dest}'..."
+          puts `#{@cmd_base} copy #{src} #{rclone_drive}:#{dest}`
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{package.time}..."
+
+        remote_path = remote_path_for(package)
+        package.filenames.each do |filename|
+          puts `#{@cmd_base} purge #{rclone_drive}:#{remote_path}`
+        end
+      end
+
+
+
+      ##
+      # The initial connection to Dropbox will provide the user with an
+      # authorization url. The user must open this URL and confirm that the
+      # authorization successfully took place. If this is the case, then the
+      # user hits 'enter' and the session will be properly established.
+      # Immediately after establishing the session, the session will be
+      # serialized and written to a cache file in +cache_path+.
+      # The cached file will be used from that point on to re-establish a
+      # connection with Dropbox at a later time. This allows the user to avoid
+      # having to go to a new Dropbox URL to authorize over and over again.
+    #   def connection
+    #     return @connection if @connection
+    #
+    #     unless session = cached_session
+    #       Logger.info "Creating a new session!"
+    #       session = create_write_and_return_new_session!
+    #     end
+    #
+    #     # will raise an error if session not authorized
+    #     @connection = DropboxClient.new(session, access_type)
+    #   rescue => err
+    #     raise Error.wrap(err, "Authorization Failed")
+    #   end
+    #
+    #   ##
+    #   # Attempt to load a cached session
+    #   def cached_session
+    #     session = false
+    #     if File.exist?(cached_file)
+    #       begin
+    #         session = DropboxSession.deserialize(File.read(cached_file))
+    #         Logger.info "Session data loaded from cache!"
+    #       rescue => err
+    #         Logger.warn Error.wrap(err, <<-EOS)
+    #           Could not read session data from cache.
+    #           Cache data might be corrupt.
+    #         EOS
+    #       end
+    #     end
+    #     session
+    #   end
+    #
+    #   ##
+    #   # Transfer each of the package files to Dropbox in chunks of +chunk_size+.
+    #   # Each chunk will be retried +chunk_retries+ times, pausing +retry_waitsec+
+    #   # between retries, if errors occur.
+    #   def transfer!
+    #     package.filenames.each do |filename|
+    #       src = File.join(Config.tmp_path, filename)
+    #       dest = File.join(remote_path, filename)
+    #       Logger.info "Storing '#{dest}'..."
+    #
+    #       uploader = nil
+    #       File.open(src, "r") do |file|
+    #         uploader = connection.get_chunked_uploader(file, file.stat.size)
+    #         while uploader.offset < uploader.total_size
+    #           with_retries do
+    #             uploader.upload(1024**2 * chunk_size)
+    #           end
+    #         end
+    #       end
+    #
+    #       with_retries do
+    #         uploader.finish(dest)
+    #       end
+    #     end
+    #   rescue => err
+    #     raise Error.wrap(err, "Upload Failed!")
+    #   end
+    #
+    #   def with_retries
+    #     retries = 0
+    #     begin
+    #       yield
+    #     rescue StandardError => err
+    #       retries += 1
+    #       raise if retries > max_retries
+    #
+    #       Logger.info Error.wrap(err, "Retry ##{retries} of #{max_retries}.")
+    #       sleep(retry_waitsec)
+    #       retry
+    #     end
+    #   end
+    #
+    #   # Called by the Cycler.
+    #   # Any error raised will be logged as a warning.
+    #   def remove!(package)
+    #     Logger.info "Removing backup package dated #{package.time}..."
+    #
+    #     connection.file_delete(remote_path_for(package))
+    #   end
+    #
+    #   def cached_file
+    #     path = cache_path.start_with?("/") ?
+    #            cache_path : File.join(Config.root_path, cache_path)
+    #     File.join(path, api_key + api_secret)
+    #   end
+    #
+    #   ##
+    #   # Serializes and writes the Dropbox session to a cache file
+    #   def write_cache!(session)
+    #     FileUtils.mkdir_p File.dirname(cached_file)
+    #     File.open(cached_file, "w") do |cache_file|
+    #       cache_file.write(session.serialize)
+    #     end
+    #   end
+    #
+    #   ##
+    #   # Create a new session, write a serialized version of it to the
+    #   # .cache directory, and return the session object
+    #   def create_write_and_return_new_session!
+    #     require "timeout"
+    #
+    #     session = DropboxSession.new(api_key, api_secret)
+    #
+    #     # grab the request token for session
+    #     session.get_request_token
+    #
+    #     template = Backup::Template.new(
+    #       session: session, cached_file: cached_file
+    #     )
+    #     template.render("storage/dropbox/authorization_url.erb")
+    #
+    #     # wait for user to hit 'return' to continue
+    #     Timeout.timeout(180) { STDIN.gets }
+    #
+    #     # this will raise an error if the user did not
+    #     # visit the authorization_url and grant access
+    #     #
+    #     # get the access token from the server
+    #     # this will be stored with the session in the cache file
+    #     session.get_access_token
+    #
+    #     template.render("storage/dropbox/authorized.erb")
+    #     write_cache!(session)
+    #     template.render("storage/dropbox/cache_file_written.erb")
+    #
+    #     session
+    #   rescue => err
+    #     raise Error.wrap(err, "Could not create or authenticate a new session")
+    #   end
+    end
   end
 end
+
+# Add our module name to the Backup DSL
+Backup::Config::DSL.const_set("GoogleDrive", Module.new)

metadata

@@ -1,24 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: backup-googledrive
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - Joseph Rafferty
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-10 00:00:00.000000000 Z
-dependencies: []
-description: Adds the GoogleDrive storage engine to the Backup ruby gem.
-email: oraff@gmail.comj
+date: 2018-04-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: backup
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.4.1
+description: 
+email: joraff@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - lib/backup-googledrive.rb
-- lib/storage/googledrive.rb
-homepage: http://github.com/joraff/backup-googledrive
+homepage: https://github.com/joraff/backup-googledrive
 licenses:
 - MIT
 metadata: {}
@@ -28,19 +47,18 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
-summary: GoogleDrive storage engine to Backup
+summary: Adds Google Drive as a storage option for Backup
 test_files: []
-has_rdoc: 

data/lib/storage/googledrive.rb

@@ -1,149 +0,0 @@
-# encoding: utf-8
-
-module Backup
-  module Storage
-    class GoogleDrive < Base
-
-      include Storage::Cycler
-      class Error < Backup::Error; end
-
-      # This adapter uses gdrive (https://github.com/prasmussen/gdrive) to handle all API interactions.
-      # Fortunately for us gdrive handles things like timeouts, error retries and large file chunking, too!
-      # I found gdrive's defaults to be acceptable, but should be easy to add accessors to customize if needed
-
-      # Path to gdrive executable
-      attr_accessor :gdrive_exe
-
-      # Use the gdrive executable to obtain a refresh token. Add that token to your backup model.
-      # The gdrive exe will handle refresing the access tokens
-      attr_accessor :refresh_token
-
-      ##
-      # Creates a new instance of the storage object
-      def initialize(model, storage_id = nil)
-        super
-
-        @path           ||= 'backups'
-        path.sub!(/^\//, '')
-
-        required = %w{ refresh_token }
-        raise Error, "Configuration Error: a refresh_token is required" if refresh_token.nil?
-
-        raise Error, "Configuration Error: gdrive executable is required." if gdrive_exe.nil?
-      end
-
-      # private
-
-
-      ##
-      # Transfer each of the package files to Dropbox in chunks of +chunk_size+.
-      # Each chunk will be retried +chunk_retries+ times, pausing +retry_waitsec+
-      # between retries, if errors occur.
-      def transfer!
-        package.filenames.each do |filename|
-          src = File.join(Config.tmp_path, filename)
-          dest = File.join(remote_path, filename)
-          Logger.info "Storing '#{ dest }'..."
-
-          parent_id = find_id_from_path(remote_path)
-          gdrive_upload(src, parent_id)
-        end
-      end
-
-      # # Called by the Cycler.
-      # # Any error raised will be logged as a warning.
-      def remove!(package)
-        Logger.info "Removing backup package dated #{ package.time }..."
-        id = find_id_from_path(remote_path_for(package), false)
-        if id.to_s.empty?
-          raise Error, "Backup packge #{ package.time } not found in Google Drive"
-        else
-          gdrive_delete(id)
-        end
-      end
-
-      def find_id_from_path(path = remote_path, create = true)
-        parent = nil
-        path.split('/').each do |path_part|
-          id = get_folder_id(path_part, parent)
-          if id.to_s.empty? && create
-            id = gdrive_mkdir(path_part, parent)
-          end
-          parent = id
-        end
-        return parent
-      end
-
-
-      def get_folder_id(name, parent = nil)
-        parent = parent ? parent : 'root'
-        gdrive_list("name = '#{name}' and '#{parent}' in parents")
-      end
-
-      def gdrive_list(query)
-        unless query.empty?
-          cmd = "gdrive --refresh-token '#{refresh_token}' list --no-header -q \"#{query}\""
-          output = `#{cmd}`
-          if output.downcase.include? "error"
-            raise Error, "Could not list or find the object with query string '#{query}'. gdrive output: #{output}"
-          elsif output.empty?
-            return nil
-          else
-            begin
-              return /^([^ ]*).*/.match(output)[1] # will return an empty string on no match
-            rescue => err
-              return nil
-            end
-          end
-        else
-          raise Error, "A search query is required to list/find a file or folder"
-        end
-      end
-
-      def gdrive_mkdir(name, parent = nil)
-        unless name.empty?
-          parent = parent ? parent : 'root'
-          cmd = "gdrive --refresh-token '#{refresh_token}' mkdir -p '#{parent}' '#{name}'"
-          output = `#{cmd}`
-          if output.downcase.include? "error"
-            raise Error, "Could not create the directory '#{name}' with parent '#{parent}'. gdrive output: #{output}"
-          else
-            id = /^Directory (.*?) created/.match(output)[1]
-            raise Error, "Could not determine ID of newly created folder. See gdrive output: #{output}" if id.to_s.empty?
-            Logger.info "Created folder #{name} successfully with id '#{id}'"
-            return id
-          end
-        else
-          raise Error, "Name parameter is required to make a directory"
-        end
-      end
-
-      def gdrive_upload(src, parent = nil)
-        parent = parent ? parent : 'root'
-        cmd = "gdrive --refresh-token '#{refresh_token}' upload -p '#{parent}' '#{src}'"
-        output = `#{cmd}`
-        if ( ["error", "failed"].any? {|s| output.downcase.include? s } )
-          raise Error, "Could not upload file. See gdrive output: #{output}"
-        else
-          begin
-            id = /.*Uploaded (.*?) .*/.match(output)[1]
-            raise Error, "empty id" if id.to_s.empty?
-            Logger.info "Uploaded #{src} into parent folder '#{parent}' successfully. Google Drive file_id: #{ id }"
-          rescue => err
-            raise Error.wrap(err, "Could not determine ID of newly created folder. See gdrive output: #{output}")
-          end
-        end
-      end
-
-      def gdrive_delete(id, recursive = true)
-        cmd = "gdrive --refresh-token '#{refresh_token}' delete #{'-r' if recursive} '#{id}'"
-        output = `#{cmd}`
-        if output.downcase.include? "error"
-          raise Error, "Could not delete object with id: #{id}. See gdrive output: #{output}"
-        else
-          Logger.info output
-        end
-      end
-    end
-  end
-end