backup 3.2.0 → 3.3.0

data/lib/backup/dependency.rb

@@ -24,7 +24,7 @@ module Backup
 
       'dropbox-sdk' => {
         :require => 'dropbox_sdk',
-        :version => '~> 1.5.0',
+        :version => '= 1.5.1', # patched in lib/backup/storage/dropbox.rb
         :for     => 'Dropbox Web Service (Dropbox Storage)'
       },
 

data/lib/backup/model.rb

@@ -85,6 +85,11 @@ module Backup
       end
 
       instance_eval(&block) if block_given?
+
+      # trigger all defined databases to generate their #dump_filename
+      # so warnings may be logged if `backup perform --check` is used
+      databases.each {|db| db.send(:dump_filename) }
+
       Model.all << self
     end
 
@@ -98,15 +103,17 @@ module Backup
     ##
     # Adds a database to the array of databases
     # to dump during the backup process
-    def database(name, &block)
-      @databases << get_class_from_scope(Database, name).new(self, &block)
+    def database(name, database_id = nil, &block)
+      @databases << get_class_from_scope(Database, name).
+          new(self, database_id, &block)
     end
 
     ##
     # Adds a storage method to the array of storage
     # methods to use during the backup process
     def store_with(name, storage_id = nil, &block)
-      @storages << get_class_from_scope(Storage, name).new(self, storage_id, &block)
+      @storages << get_class_from_scope(Storage, name).
+          new(self, storage_id, &block)
     end
 
     ##
@@ -247,14 +254,10 @@ module Backup
     private
 
     ##
-    # Ensure DATA_PATH and DATA_PATH/TRIGGER are created
-    # if they do not yet exist
-    #
     # Clean any temporary files and/or package files left over
     # from the last time this model/trigger was performed.
     # Logs warnings if files exist and are cleaned.
     def prepare!
-      FileUtils.mkdir_p(File.join(Config.data_path, trigger))
       Cleaner.prepare(self)
     end
 

data/lib/backup/storage/cloudfiles.rb

@@ -57,6 +57,8 @@ module Backup
       def transfer!
         remote_path = remote_path_for(@package)
 
+        connection.put_container(container)
+
         files_to_transfer_for(@package) do |local_file, remote_file|
           Logger.info "#{storage_name} started transferring '#{ local_file }'."
 

data/lib/backup/storage/cycler.rb

@@ -53,7 +53,7 @@ module Backup
         # based on the current values of @storage and @package
         def storage_file
           type     = @storage.class.to_s.split('::').last
-          suffix   = @storage.storage_id.to_s.strip.gsub(/[\W\s]/, '_')
+          suffix   = @storage.storage_id.to_s.strip.gsub(/[\W]/, '_')
           filename = suffix.empty? ? type : "#{type}-#{suffix}"
           File.join(Config.data_path, @package.trigger, "#{filename}.yml")
         end
@@ -77,6 +77,7 @@ module Backup
         ##
         # Store the given package objects to the YAML data file.
         def yaml_save(packages)
+          FileUtils.mkdir_p(File.dirname(@storage_file))
           File.open(@storage_file, 'w') do |file|
             file.write(packages.to_yaml)
           end

data/lib/backup/storage/dropbox.rb

@@ -23,6 +23,18 @@ module Backup
       # Path to where the backups will be stored
       attr_accessor :path
 
+      ##
+      # Chunk size, specified in MiB, for the ChunkedUploader.
+      attr_accessor :chunk_size
+
+      ##
+      # Number of times to retry a failed chunk.
+      attr_accessor :chunk_retries
+
+      ##
+      # Seconds to wait between chunk retries.
+      attr_accessor :retry_waitsec
+
       attr_deprecate :email,    :version => '3.0.17'
       attr_deprecate :password, :version => '3.0.17'
 
@@ -33,8 +45,11 @@ module Backup
       def initialize(model, storage_id = nil, &block)
         super(model, storage_id)
 
-        @path ||= 'backups'
-        @access_type ||= :app_folder
+        @path           ||= 'backups'
+        @access_type    ||= :app_folder
+        @chunk_size     ||= 4 # MiB
+        @chunk_retries  ||= 10
+        @retry_waitsec  ||= 30
 
         instance_eval(&block) if block_given?
       end
@@ -86,15 +101,36 @@ module Backup
       end
 
       ##
-      # Transfers the archived file to the specified Dropbox folder
+      # Transfer each of the package files to Dropbox in chunks of +chunk_size+.
+      # Each chunk will be retried +chunk_retries+ times, pausing +retry_waitsec+
+      # between retries, if errors occur.
       def transfer!
         remote_path = remote_path_for(@package)
 
         files_to_transfer_for(@package) do |local_file, remote_file|
-          Logger.info "#{storage_name} started transferring '#{ local_file }'."
+          Logger.info "#{ storage_name } started transferring '#{ local_file }'."
+
+          uploader, retries = nil, 0
           File.open(File.join(local_path, local_file), 'r') do |file|
-            connection.put_file(File.join(remote_path, remote_file), file)
+            uploader = connection.get_chunked_uploader(file, file.size)
+            while uploader.offset < uploader.total_size
+              begin
+                uploader.upload(1024**2 * chunk_size)
+                retries = 0
+              rescue => err
+                retries += 1
+                if retries <= chunk_retries
+                  Logger.info "Chunk retry #{ retries } of #{ chunk_retries }."
+                  sleep(retry_waitsec)
+                  retry
+                end
+                raise Errors::Storage::Dropbox::TransferError.
+                    wrap(err, 'Dropbox upload failed!')
+              end
+            end
           end
+
+          uploader.finish(File.join(remote_path, remote_file))
         end
       end
 
@@ -130,6 +166,7 @@ module Backup
       ##
       # Serializes and writes the Dropbox session to a cache file
       def write_cache!(session)
+        FileUtils.mkdir_p(Config.cache_path)
         File.open(cached_file, "w") do |cache_file|
           cache_file.write(session.serialize)
         end
@@ -176,3 +213,27 @@ module Backup
     end
   end
 end
+
+# Patch for dropbox-ruby-sdk-1.5.1
+class DropboxClient
+  class ChunkedUploader
+    def upload(chunk_size = 1024**2 * 4)
+      while @offset < @total_size
+        @file_obj.seek(@offset) unless @file_obj.pos == @offset
+        data = @file_obj.read(chunk_size)
+
+        begin
+          resp = @client.parse_response(
+            @client.partial_chunked_upload(data, @upload_id, @offset)
+          )
+        rescue DropboxError => err
+          resp = JSON.parse(err.http_response.body) rescue {}
+          raise err unless resp['offset']
+        end
+
+        @offset = resp['offset']
+        @upload_id ||= resp['upload_id']
+      end
+    end
+  end
+end

data/lib/backup/utilities.rb

@@ -4,7 +4,7 @@ module Backup
   module Utilities
     UTILITY = {}
     NAMES = %w{
-      tar cat split find xargs
+      tar cat split find xargs sudo chown
       gzip bzip2 lzma pbzip2
       mongo mongodump mysqldump pg_dump redis-cli riak-admin
       gpg openssl
@@ -63,6 +63,8 @@ module Backup
       #     split '/path/to/split'
       #     find  '/path/to/find'
       #     xargs '/path/to/xargs'
+      #     sudo  '/path/to/sudo'
+      #     chown '/path/to/chown'
       #
       #     # Compressors
       #     gzip    '/path/to/gzip'
@@ -124,23 +126,36 @@ module Backup
         raise Errors::Utilities::NotFoundError,
             'Utility Name Empty' if name.empty?
 
-        path = UTILITY[name] || %x[which '#{ name }' 2>/dev/null].chomp
-        if path.empty?
-          raise Errors::Utilities::NotFoundError, <<-EOS
-            Could not locate '#{ name }'.
-            Make sure the specified utility is installed
-            and available in your system's $PATH.
-          EOS
-        end
-        UTILITY[name] = path
+        UTILITY[name] ||= %x[which '#{ name }' 2>/dev/null].chomp
+        raise(Errors::Utilities::NotFoundError, <<-EOS) if UTILITY[name].empty?
+          Could not locate '#{ name }'.
+          Make sure the specified utility is installed
+          and available in your system's $PATH, or specify it's location
+          in your 'config.rb' file using Backup::Utilities.configure
+        EOS
+
+        UTILITY[name].dup
       end
 
       ##
-      # Returns the name of the command name from the given command line
+      # Returns the name of the command name from the given command line.
+      # This is only used to simplify log messages.
       def command_name(command)
-        i = command =~ /\s/
-        command = command.slice(0, i) if i
-        command.split('/')[-1]
+        parts = []
+        command = command.split(' ')
+        parts << command.shift.split('/')[-1]
+        if parts[0] == 'sudo'
+          until command.empty?
+            part = command.shift
+            if part.include?('/')
+              parts << part.split('/')[-1]
+              break
+            else
+              parts << part
+            end
+          end
+        end
+        parts.join(' ')
       end
 
       ##

data/lib/backup/version.rb

@@ -13,7 +13,7 @@ module Backup
     #  Defines the minor version
     # PATCH:
     #  Defines the patch version
-    MAJOR, MINOR, PATCH = 3, 2, 0
+    MAJOR, MINOR, PATCH = 3, 3, 0
 
     ##
     # Returns the major version ( big release based off of multiple minor releases )

data/templates/cli/archive

@@ -22,6 +22,8 @@
   # https://github.com/meskyanichi/backup/wiki/Archives
   #
   archive :my_archive do |archive|
+    # Run the `tar` command using `sudo`
+    # archive.use_sudo
     archive.add "/path/to/a/file.rb"
     archive.add "/path/to/a/folder/"
     archive.exclude "/path/to/a/excluded_file.rb"

data/templates/cli/database/mongodb

@@ -11,8 +11,5 @@
     db.only_collections   = ["only", "these", "collections"]
     db.additional_options = []
     db.lock               = false
-    # Optional: Use to set the location of these utilities
-    #   if they cannot be found by their name in your $PATH
-    # db.mongodump_utility = "/opt/local/bin/mongodump"
-    # db.mongo_utility     = "/opt/local/bin/mongo"
+    db.oplog              = false
   end

data/templates/cli/database/mysql

@@ -15,7 +15,4 @@
     db.skip_tables        = ["skip", "these", "tables"]
     db.only_tables        = ["only", "these", "tables"]
     db.additional_options = ["--quick", "--single-transaction"]
-    # Optional: Use to set the location of this utility
-    #   if it cannot be found by name in your $PATH
-    # db.mysqldump_utility = "/opt/local/bin/mysqldump"
   end

data/templates/cli/database/postgresql

@@ -11,7 +11,4 @@
     db.skip_tables        = ["skip", "these", "tables"]
     db.only_tables        = ["only", "these", "tables"]
     db.additional_options = ["-xc", "-E=utf8"]
-    # Optional: Use to set the location of this utility
-    #   if it cannot be found by name in your $PATH
-    # db.pg_dump_utility = "/opt/local/bin/pg_dump"
   end

data/templates/cli/database/redis

@@ -2,15 +2,17 @@
   # Redis [Database]
   #
   database Redis do |db|
-    db.name               = "my_database_name"
-    db.path               = "/usr/local/var/db/redis"
-    db.password           = "my_password"
-    db.host               = "localhost"
-    db.port               = 5432
-    db.socket             = "/tmp/redis.sock"
+    ##
+    # From `dbfilename` in your `redis.conf` under SNAPSHOTTING.
+    # Do not include the '.rdb' extension. Defaults to 'dump'
+    db.name               = 'dump'
+    ##
+    # From `dir` in your `redis.conf` under SNAPSHOTTING.
+    db.path               = '/var/lib/redis'
+    db.password           = 'my_password'
+    db.host               = 'localhost'
+    db.port               = 6379
+    db.socket             = '/tmp/redis.sock'
     db.additional_options = []
     db.invoke_save        = true
-    # Optional: Use to set the location of this utility
-    #   if it cannot be found by name in your $PATH
-    # db.redis_cli_utility = "/opt/local/bin/redis-cli"
   end

data/templates/cli/database/riak

@@ -2,19 +2,16 @@
   # Riak [Database]
   #
   database Riak do |db|
-    db.name   = "hostname"
-    db.node   = "riak@hostname"
-    db.cookie = "cookie"
     ##
-    # Optional: (defaults shown)
-    #
-    # Use to set the location of this utility
-    # if it cannot be found by name in your $PATH
-    # db.riak_admin_utility = '/opt/local/bin/riak-admin'
-    #
-    # Set username for the riak-admin utility
-    # db.user = 'riak'
-    #
-    # Set usergroup for the riak-admin utility
-    # db.group = 'riak'
+    # The node from which to perform the backup.
+    # default: 'riak@127.0.0.1'
+    db.node = 'riak@hostname'
+    ##
+    # The Erlang cookie/shared secret used to connect to the node.
+    # default: 'riak'
+    db.cookie = 'cookie'
+    ##
+    # The user for the Riak instance.
+    # default: 'riak'
+    db.user = 'riak'
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: backup
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 3.3.0
 platform: ruby
 authors:
 - Michael van Rooijen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-05 00:00:00.000000000 Z
+date: 2013-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor