backup 3.2.0 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -24,7 +24,7 @@ module Backup
24
24
 
25
25
  'dropbox-sdk' => {
26
26
  :require => 'dropbox_sdk',
27
- :version => '~> 1.5.0',
27
+ :version => '= 1.5.1', # patched in lib/backup/storage/dropbox.rb
28
28
  :for => 'Dropbox Web Service (Dropbox Storage)'
29
29
  },
30
30
 
data/lib/backup/model.rb CHANGED
@@ -85,6 +85,11 @@ module Backup
85
85
  end
86
86
 
87
87
  instance_eval(&block) if block_given?
88
+
89
+ # trigger all defined databases to generate their #dump_filename
90
+ # so warnings may be logged if `backup perform --check` is used
91
+ databases.each {|db| db.send(:dump_filename) }
92
+
88
93
  Model.all << self
89
94
  end
90
95
 
@@ -98,15 +103,17 @@ module Backup
98
103
  ##
99
104
  # Adds a database to the array of databases
100
105
  # to dump during the backup process
101
- def database(name, &block)
102
- @databases << get_class_from_scope(Database, name).new(self, &block)
106
+ def database(name, database_id = nil, &block)
107
+ @databases << get_class_from_scope(Database, name).
108
+ new(self, database_id, &block)
103
109
  end
104
110
 
105
111
  ##
106
112
  # Adds a storage method to the array of storage
107
113
  # methods to use during the backup process
108
114
  def store_with(name, storage_id = nil, &block)
109
- @storages << get_class_from_scope(Storage, name).new(self, storage_id, &block)
115
+ @storages << get_class_from_scope(Storage, name).
116
+ new(self, storage_id, &block)
110
117
  end
111
118
 
112
119
  ##
@@ -247,14 +254,10 @@ module Backup
247
254
  private
248
255
 
249
256
  ##
250
- # Ensure DATA_PATH and DATA_PATH/TRIGGER are created
251
- # if they do not yet exist
252
- #
253
257
  # Clean any temporary files and/or package files left over
254
258
  # from the last time this model/trigger was performed.
255
259
  # Logs warnings if files exist and are cleaned.
256
260
  def prepare!
257
- FileUtils.mkdir_p(File.join(Config.data_path, trigger))
258
261
  Cleaner.prepare(self)
259
262
  end
260
263
 
@@ -57,6 +57,8 @@ module Backup
57
57
  def transfer!
58
58
  remote_path = remote_path_for(@package)
59
59
 
60
+ connection.put_container(container)
61
+
60
62
  files_to_transfer_for(@package) do |local_file, remote_file|
61
63
  Logger.info "#{storage_name} started transferring '#{ local_file }'."
62
64
 
@@ -53,7 +53,7 @@ module Backup
53
53
  # based on the current values of @storage and @package
54
54
  def storage_file
55
55
  type = @storage.class.to_s.split('::').last
56
- suffix = @storage.storage_id.to_s.strip.gsub(/[\W\s]/, '_')
56
+ suffix = @storage.storage_id.to_s.strip.gsub(/[\W]/, '_')
57
57
  filename = suffix.empty? ? type : "#{type}-#{suffix}"
58
58
  File.join(Config.data_path, @package.trigger, "#{filename}.yml")
59
59
  end
@@ -77,6 +77,7 @@ module Backup
77
77
  ##
78
78
  # Store the given package objects to the YAML data file.
79
79
  def yaml_save(packages)
80
+ FileUtils.mkdir_p(File.dirname(@storage_file))
80
81
  File.open(@storage_file, 'w') do |file|
81
82
  file.write(packages.to_yaml)
82
83
  end
@@ -23,6 +23,18 @@ module Backup
23
23
  # Path to where the backups will be stored
24
24
  attr_accessor :path
25
25
 
26
+ ##
27
+ # Chunk size, specified in MiB, for the ChunkedUploader.
28
+ attr_accessor :chunk_size
29
+
30
+ ##
31
+ # Number of times to retry a failed chunk.
32
+ attr_accessor :chunk_retries
33
+
34
+ ##
35
+ # Seconds to wait between chunk retries.
36
+ attr_accessor :retry_waitsec
37
+
26
38
  attr_deprecate :email, :version => '3.0.17'
27
39
  attr_deprecate :password, :version => '3.0.17'
28
40
 
@@ -33,8 +45,11 @@ module Backup
33
45
  def initialize(model, storage_id = nil, &block)
34
46
  super(model, storage_id)
35
47
 
36
- @path ||= 'backups'
37
- @access_type ||= :app_folder
48
+ @path ||= 'backups'
49
+ @access_type ||= :app_folder
50
+ @chunk_size ||= 4 # MiB
51
+ @chunk_retries ||= 10
52
+ @retry_waitsec ||= 30
38
53
 
39
54
  instance_eval(&block) if block_given?
40
55
  end
@@ -86,15 +101,36 @@ module Backup
86
101
  end
87
102
 
88
103
  ##
89
- # Transfers the archived file to the specified Dropbox folder
104
+ # Transfer each of the package files to Dropbox in chunks of +chunk_size+.
105
+ # Each chunk will be retried +chunk_retries+ times, pausing +retry_waitsec+
106
+ # between retries, if errors occur.
90
107
  def transfer!
91
108
  remote_path = remote_path_for(@package)
92
109
 
93
110
  files_to_transfer_for(@package) do |local_file, remote_file|
94
- Logger.info "#{storage_name} started transferring '#{ local_file }'."
111
+ Logger.info "#{ storage_name } started transferring '#{ local_file }'."
112
+
113
+ uploader, retries = nil, 0
95
114
  File.open(File.join(local_path, local_file), 'r') do |file|
96
- connection.put_file(File.join(remote_path, remote_file), file)
115
+ uploader = connection.get_chunked_uploader(file, file.size)
116
+ while uploader.offset < uploader.total_size
117
+ begin
118
+ uploader.upload(1024**2 * chunk_size)
119
+ retries = 0
120
+ rescue => err
121
+ retries += 1
122
+ if retries <= chunk_retries
123
+ Logger.info "Chunk retry #{ retries } of #{ chunk_retries }."
124
+ sleep(retry_waitsec)
125
+ retry
126
+ end
127
+ raise Errors::Storage::Dropbox::TransferError.
128
+ wrap(err, 'Dropbox upload failed!')
129
+ end
130
+ end
97
131
  end
132
+
133
+ uploader.finish(File.join(remote_path, remote_file))
98
134
  end
99
135
  end
100
136
 
@@ -130,6 +166,7 @@ module Backup
130
166
  ##
131
167
  # Serializes and writes the Dropbox session to a cache file
132
168
  def write_cache!(session)
169
+ FileUtils.mkdir_p(Config.cache_path)
133
170
  File.open(cached_file, "w") do |cache_file|
134
171
  cache_file.write(session.serialize)
135
172
  end
@@ -176,3 +213,27 @@ module Backup
176
213
  end
177
214
  end
178
215
  end
216
+
217
+ # Patch for dropbox-ruby-sdk-1.5.1
218
+ class DropboxClient
219
+ class ChunkedUploader
220
+ def upload(chunk_size = 1024**2 * 4)
221
+ while @offset < @total_size
222
+ @file_obj.seek(@offset) unless @file_obj.pos == @offset
223
+ data = @file_obj.read(chunk_size)
224
+
225
+ begin
226
+ resp = @client.parse_response(
227
+ @client.partial_chunked_upload(data, @upload_id, @offset)
228
+ )
229
+ rescue DropboxError => err
230
+ resp = JSON.parse(err.http_response.body) rescue {}
231
+ raise err unless resp['offset']
232
+ end
233
+
234
+ @offset = resp['offset']
235
+ @upload_id ||= resp['upload_id']
236
+ end
237
+ end
238
+ end
239
+ end
@@ -4,7 +4,7 @@ module Backup
4
4
  module Utilities
5
5
  UTILITY = {}
6
6
  NAMES = %w{
7
- tar cat split find xargs
7
+ tar cat split find xargs sudo chown
8
8
  gzip bzip2 lzma pbzip2
9
9
  mongo mongodump mysqldump pg_dump redis-cli riak-admin
10
10
  gpg openssl
@@ -63,6 +63,8 @@ module Backup
63
63
  # split '/path/to/split'
64
64
  # find '/path/to/find'
65
65
  # xargs '/path/to/xargs'
66
+ # sudo '/path/to/sudo'
67
+ # chown '/path/to/chown'
66
68
  #
67
69
  # # Compressors
68
70
  # gzip '/path/to/gzip'
@@ -124,23 +126,36 @@ module Backup
124
126
  raise Errors::Utilities::NotFoundError,
125
127
  'Utility Name Empty' if name.empty?
126
128
 
127
- path = UTILITY[name] || %x[which '#{ name }' 2>/dev/null].chomp
128
- if path.empty?
129
- raise Errors::Utilities::NotFoundError, <<-EOS
130
- Could not locate '#{ name }'.
131
- Make sure the specified utility is installed
132
- and available in your system's $PATH.
133
- EOS
134
- end
135
- UTILITY[name] = path
129
+ UTILITY[name] ||= %x[which '#{ name }' 2>/dev/null].chomp
130
+ raise(Errors::Utilities::NotFoundError, <<-EOS) if UTILITY[name].empty?
131
+ Could not locate '#{ name }'.
132
+ Make sure the specified utility is installed
133
+ and available in your system's $PATH, or specify it's location
134
+ in your 'config.rb' file using Backup::Utilities.configure
135
+ EOS
136
+
137
+ UTILITY[name].dup
136
138
  end
137
139
 
138
140
  ##
139
- # Returns the name of the command name from the given command line
141
+ # Returns the name of the command name from the given command line.
142
+ # This is only used to simplify log messages.
140
143
  def command_name(command)
141
- i = command =~ /\s/
142
- command = command.slice(0, i) if i
143
- command.split('/')[-1]
144
+ parts = []
145
+ command = command.split(' ')
146
+ parts << command.shift.split('/')[-1]
147
+ if parts[0] == 'sudo'
148
+ until command.empty?
149
+ part = command.shift
150
+ if part.include?('/')
151
+ parts << part.split('/')[-1]
152
+ break
153
+ else
154
+ parts << part
155
+ end
156
+ end
157
+ end
158
+ parts.join(' ')
144
159
  end
145
160
 
146
161
  ##
@@ -13,7 +13,7 @@ module Backup
13
13
  # Defines the minor version
14
14
  # PATCH:
15
15
  # Defines the patch version
16
- MAJOR, MINOR, PATCH = 3, 2, 0
16
+ MAJOR, MINOR, PATCH = 3, 3, 0
17
17
 
18
18
  ##
19
19
  # Returns the major version ( big release based off of multiple minor releases )
@@ -22,6 +22,8 @@
22
22
  # https://github.com/meskyanichi/backup/wiki/Archives
23
23
  #
24
24
  archive :my_archive do |archive|
25
+ # Run the `tar` command using `sudo`
26
+ # archive.use_sudo
25
27
  archive.add "/path/to/a/file.rb"
26
28
  archive.add "/path/to/a/folder/"
27
29
  archive.exclude "/path/to/a/excluded_file.rb"
@@ -11,8 +11,5 @@
11
11
  db.only_collections = ["only", "these", "collections"]
12
12
  db.additional_options = []
13
13
  db.lock = false
14
- # Optional: Use to set the location of these utilities
15
- # if they cannot be found by their name in your $PATH
16
- # db.mongodump_utility = "/opt/local/bin/mongodump"
17
- # db.mongo_utility = "/opt/local/bin/mongo"
14
+ db.oplog = false
18
15
  end
@@ -15,7 +15,4 @@
15
15
  db.skip_tables = ["skip", "these", "tables"]
16
16
  db.only_tables = ["only", "these", "tables"]
17
17
  db.additional_options = ["--quick", "--single-transaction"]
18
- # Optional: Use to set the location of this utility
19
- # if it cannot be found by name in your $PATH
20
- # db.mysqldump_utility = "/opt/local/bin/mysqldump"
21
18
  end
@@ -11,7 +11,4 @@
11
11
  db.skip_tables = ["skip", "these", "tables"]
12
12
  db.only_tables = ["only", "these", "tables"]
13
13
  db.additional_options = ["-xc", "-E=utf8"]
14
- # Optional: Use to set the location of this utility
15
- # if it cannot be found by name in your $PATH
16
- # db.pg_dump_utility = "/opt/local/bin/pg_dump"
17
14
  end
@@ -2,15 +2,17 @@
2
2
  # Redis [Database]
3
3
  #
4
4
  database Redis do |db|
5
- db.name = "my_database_name"
6
- db.path = "/usr/local/var/db/redis"
7
- db.password = "my_password"
8
- db.host = "localhost"
9
- db.port = 5432
10
- db.socket = "/tmp/redis.sock"
5
+ ##
6
+ # From `dbfilename` in your `redis.conf` under SNAPSHOTTING.
7
+ # Do not include the '.rdb' extension. Defaults to 'dump'
8
+ db.name = 'dump'
9
+ ##
10
+ # From `dir` in your `redis.conf` under SNAPSHOTTING.
11
+ db.path = '/var/lib/redis'
12
+ db.password = 'my_password'
13
+ db.host = 'localhost'
14
+ db.port = 6379
15
+ db.socket = '/tmp/redis.sock'
11
16
  db.additional_options = []
12
17
  db.invoke_save = true
13
- # Optional: Use to set the location of this utility
14
- # if it cannot be found by name in your $PATH
15
- # db.redis_cli_utility = "/opt/local/bin/redis-cli"
16
18
  end
@@ -2,19 +2,16 @@
2
2
  # Riak [Database]
3
3
  #
4
4
  database Riak do |db|
5
- db.name = "hostname"
6
- db.node = "riak@hostname"
7
- db.cookie = "cookie"
8
5
  ##
9
- # Optional: (defaults shown)
10
- #
11
- # Use to set the location of this utility
12
- # if it cannot be found by name in your $PATH
13
- # db.riak_admin_utility = '/opt/local/bin/riak-admin'
14
- #
15
- # Set username for the riak-admin utility
16
- # db.user = 'riak'
17
- #
18
- # Set usergroup for the riak-admin utility
19
- # db.group = 'riak'
6
+ # The node from which to perform the backup.
7
+ # default: 'riak@127.0.0.1'
8
+ db.node = 'riak@hostname'
9
+ ##
10
+ # The Erlang cookie/shared secret used to connect to the node.
11
+ # default: 'riak'
12
+ db.cookie = 'cookie'
13
+ ##
14
+ # The user for the Riak instance.
15
+ # default: 'riak'
16
+ db.user = 'riak'
20
17
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: backup
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.2.0
4
+ version: 3.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Michael van Rooijen
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2013-04-05 00:00:00.000000000 Z
11
+ date: 2013-04-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: thor