backup 3.2.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -5
- data/lib/backup/archive.rb +11 -1
- data/lib/backup/cli.rb +252 -141
- data/lib/backup/database/base.rb +50 -24
- data/lib/backup/database/mongodb.rb +101 -127
- data/lib/backup/database/mysql.rb +50 -76
- data/lib/backup/database/postgresql.rb +44 -70
- data/lib/backup/database/redis.rb +61 -66
- data/lib/backup/database/riak.rb +64 -43
- data/lib/backup/dependency.rb +1 -1
- data/lib/backup/model.rb +10 -7
- data/lib/backup/storage/cloudfiles.rb +2 -0
- data/lib/backup/storage/cycler.rb +2 -1
- data/lib/backup/storage/dropbox.rb +66 -5
- data/lib/backup/utilities.rb +29 -14
- data/lib/backup/version.rb +1 -1
- data/templates/cli/archive +2 -0
- data/templates/cli/database/mongodb +1 -4
- data/templates/cli/database/mysql +0 -3
- data/templates/cli/database/postgresql +0 -3
- data/templates/cli/database/redis +11 -9
- data/templates/cli/database/riak +11 -14
- metadata +2 -2
data/lib/backup/dependency.rb
CHANGED
data/lib/backup/model.rb
CHANGED
@@ -85,6 +85,11 @@ module Backup
|
|
85
85
|
end
|
86
86
|
|
87
87
|
instance_eval(&block) if block_given?
|
88
|
+
|
89
|
+
# trigger all defined databases to generate their #dump_filename
|
90
|
+
# so warnings may be logged if `backup perform --check` is used
|
91
|
+
databases.each {|db| db.send(:dump_filename) }
|
92
|
+
|
88
93
|
Model.all << self
|
89
94
|
end
|
90
95
|
|
@@ -98,15 +103,17 @@ module Backup
|
|
98
103
|
##
|
99
104
|
# Adds a database to the array of databases
|
100
105
|
# to dump during the backup process
|
101
|
-
def database(name, &block)
|
102
|
-
@databases << get_class_from_scope(Database, name).
|
106
|
+
def database(name, database_id = nil, &block)
|
107
|
+
@databases << get_class_from_scope(Database, name).
|
108
|
+
new(self, database_id, &block)
|
103
109
|
end
|
104
110
|
|
105
111
|
##
|
106
112
|
# Adds a storage method to the array of storage
|
107
113
|
# methods to use during the backup process
|
108
114
|
def store_with(name, storage_id = nil, &block)
|
109
|
-
@storages << get_class_from_scope(Storage, name).
|
115
|
+
@storages << get_class_from_scope(Storage, name).
|
116
|
+
new(self, storage_id, &block)
|
110
117
|
end
|
111
118
|
|
112
119
|
##
|
@@ -247,14 +254,10 @@ module Backup
|
|
247
254
|
private
|
248
255
|
|
249
256
|
##
|
250
|
-
# Ensure DATA_PATH and DATA_PATH/TRIGGER are created
|
251
|
-
# if they do not yet exist
|
252
|
-
#
|
253
257
|
# Clean any temporary files and/or package files left over
|
254
258
|
# from the last time this model/trigger was performed.
|
255
259
|
# Logs warnings if files exist and are cleaned.
|
256
260
|
def prepare!
|
257
|
-
FileUtils.mkdir_p(File.join(Config.data_path, trigger))
|
258
261
|
Cleaner.prepare(self)
|
259
262
|
end
|
260
263
|
|
@@ -53,7 +53,7 @@ module Backup
|
|
53
53
|
# based on the current values of @storage and @package
|
54
54
|
def storage_file
|
55
55
|
type = @storage.class.to_s.split('::').last
|
56
|
-
suffix = @storage.storage_id.to_s.strip.gsub(/[\W
|
56
|
+
suffix = @storage.storage_id.to_s.strip.gsub(/[\W]/, '_')
|
57
57
|
filename = suffix.empty? ? type : "#{type}-#{suffix}"
|
58
58
|
File.join(Config.data_path, @package.trigger, "#{filename}.yml")
|
59
59
|
end
|
@@ -77,6 +77,7 @@ module Backup
|
|
77
77
|
##
|
78
78
|
# Store the given package objects to the YAML data file.
|
79
79
|
def yaml_save(packages)
|
80
|
+
FileUtils.mkdir_p(File.dirname(@storage_file))
|
80
81
|
File.open(@storage_file, 'w') do |file|
|
81
82
|
file.write(packages.to_yaml)
|
82
83
|
end
|
@@ -23,6 +23,18 @@ module Backup
|
|
23
23
|
# Path to where the backups will be stored
|
24
24
|
attr_accessor :path
|
25
25
|
|
26
|
+
##
|
27
|
+
# Chunk size, specified in MiB, for the ChunkedUploader.
|
28
|
+
attr_accessor :chunk_size
|
29
|
+
|
30
|
+
##
|
31
|
+
# Number of times to retry a failed chunk.
|
32
|
+
attr_accessor :chunk_retries
|
33
|
+
|
34
|
+
##
|
35
|
+
# Seconds to wait between chunk retries.
|
36
|
+
attr_accessor :retry_waitsec
|
37
|
+
|
26
38
|
attr_deprecate :email, :version => '3.0.17'
|
27
39
|
attr_deprecate :password, :version => '3.0.17'
|
28
40
|
|
@@ -33,8 +45,11 @@ module Backup
|
|
33
45
|
def initialize(model, storage_id = nil, &block)
|
34
46
|
super(model, storage_id)
|
35
47
|
|
36
|
-
@path
|
37
|
-
@access_type
|
48
|
+
@path ||= 'backups'
|
49
|
+
@access_type ||= :app_folder
|
50
|
+
@chunk_size ||= 4 # MiB
|
51
|
+
@chunk_retries ||= 10
|
52
|
+
@retry_waitsec ||= 30
|
38
53
|
|
39
54
|
instance_eval(&block) if block_given?
|
40
55
|
end
|
@@ -86,15 +101,36 @@ module Backup
|
|
86
101
|
end
|
87
102
|
|
88
103
|
##
|
89
|
-
#
|
104
|
+
# Transfer each of the package files to Dropbox in chunks of +chunk_size+.
|
105
|
+
# Each chunk will be retried +chunk_retries+ times, pausing +retry_waitsec+
|
106
|
+
# between retries, if errors occur.
|
90
107
|
def transfer!
|
91
108
|
remote_path = remote_path_for(@package)
|
92
109
|
|
93
110
|
files_to_transfer_for(@package) do |local_file, remote_file|
|
94
|
-
Logger.info "#{storage_name} started transferring '#{ local_file }'."
|
111
|
+
Logger.info "#{ storage_name } started transferring '#{ local_file }'."
|
112
|
+
|
113
|
+
uploader, retries = nil, 0
|
95
114
|
File.open(File.join(local_path, local_file), 'r') do |file|
|
96
|
-
connection.
|
115
|
+
uploader = connection.get_chunked_uploader(file, file.size)
|
116
|
+
while uploader.offset < uploader.total_size
|
117
|
+
begin
|
118
|
+
uploader.upload(1024**2 * chunk_size)
|
119
|
+
retries = 0
|
120
|
+
rescue => err
|
121
|
+
retries += 1
|
122
|
+
if retries <= chunk_retries
|
123
|
+
Logger.info "Chunk retry #{ retries } of #{ chunk_retries }."
|
124
|
+
sleep(retry_waitsec)
|
125
|
+
retry
|
126
|
+
end
|
127
|
+
raise Errors::Storage::Dropbox::TransferError.
|
128
|
+
wrap(err, 'Dropbox upload failed!')
|
129
|
+
end
|
130
|
+
end
|
97
131
|
end
|
132
|
+
|
133
|
+
uploader.finish(File.join(remote_path, remote_file))
|
98
134
|
end
|
99
135
|
end
|
100
136
|
|
@@ -130,6 +166,7 @@ module Backup
|
|
130
166
|
##
|
131
167
|
# Serializes and writes the Dropbox session to a cache file
|
132
168
|
def write_cache!(session)
|
169
|
+
FileUtils.mkdir_p(Config.cache_path)
|
133
170
|
File.open(cached_file, "w") do |cache_file|
|
134
171
|
cache_file.write(session.serialize)
|
135
172
|
end
|
@@ -176,3 +213,27 @@ module Backup
|
|
176
213
|
end
|
177
214
|
end
|
178
215
|
end
|
216
|
+
|
217
|
+
# Patch for dropbox-ruby-sdk-1.5.1
|
218
|
+
class DropboxClient
|
219
|
+
class ChunkedUploader
|
220
|
+
def upload(chunk_size = 1024**2 * 4)
|
221
|
+
while @offset < @total_size
|
222
|
+
@file_obj.seek(@offset) unless @file_obj.pos == @offset
|
223
|
+
data = @file_obj.read(chunk_size)
|
224
|
+
|
225
|
+
begin
|
226
|
+
resp = @client.parse_response(
|
227
|
+
@client.partial_chunked_upload(data, @upload_id, @offset)
|
228
|
+
)
|
229
|
+
rescue DropboxError => err
|
230
|
+
resp = JSON.parse(err.http_response.body) rescue {}
|
231
|
+
raise err unless resp['offset']
|
232
|
+
end
|
233
|
+
|
234
|
+
@offset = resp['offset']
|
235
|
+
@upload_id ||= resp['upload_id']
|
236
|
+
end
|
237
|
+
end
|
238
|
+
end
|
239
|
+
end
|
data/lib/backup/utilities.rb
CHANGED
@@ -4,7 +4,7 @@ module Backup
|
|
4
4
|
module Utilities
|
5
5
|
UTILITY = {}
|
6
6
|
NAMES = %w{
|
7
|
-
tar cat split find xargs
|
7
|
+
tar cat split find xargs sudo chown
|
8
8
|
gzip bzip2 lzma pbzip2
|
9
9
|
mongo mongodump mysqldump pg_dump redis-cli riak-admin
|
10
10
|
gpg openssl
|
@@ -63,6 +63,8 @@ module Backup
|
|
63
63
|
# split '/path/to/split'
|
64
64
|
# find '/path/to/find'
|
65
65
|
# xargs '/path/to/xargs'
|
66
|
+
# sudo '/path/to/sudo'
|
67
|
+
# chown '/path/to/chown'
|
66
68
|
#
|
67
69
|
# # Compressors
|
68
70
|
# gzip '/path/to/gzip'
|
@@ -124,23 +126,36 @@ module Backup
|
|
124
126
|
raise Errors::Utilities::NotFoundError,
|
125
127
|
'Utility Name Empty' if name.empty?
|
126
128
|
|
127
|
-
|
128
|
-
if
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
UTILITY[name]
|
129
|
+
UTILITY[name] ||= %x[which '#{ name }' 2>/dev/null].chomp
|
130
|
+
raise(Errors::Utilities::NotFoundError, <<-EOS) if UTILITY[name].empty?
|
131
|
+
Could not locate '#{ name }'.
|
132
|
+
Make sure the specified utility is installed
|
133
|
+
and available in your system's $PATH, or specify it's location
|
134
|
+
in your 'config.rb' file using Backup::Utilities.configure
|
135
|
+
EOS
|
136
|
+
|
137
|
+
UTILITY[name].dup
|
136
138
|
end
|
137
139
|
|
138
140
|
##
|
139
|
-
# Returns the name of the command name from the given command line
|
141
|
+
# Returns the name of the command name from the given command line.
|
142
|
+
# This is only used to simplify log messages.
|
140
143
|
def command_name(command)
|
141
|
-
|
142
|
-
command = command.
|
143
|
-
command.split('/')[-1]
|
144
|
+
parts = []
|
145
|
+
command = command.split(' ')
|
146
|
+
parts << command.shift.split('/')[-1]
|
147
|
+
if parts[0] == 'sudo'
|
148
|
+
until command.empty?
|
149
|
+
part = command.shift
|
150
|
+
if part.include?('/')
|
151
|
+
parts << part.split('/')[-1]
|
152
|
+
break
|
153
|
+
else
|
154
|
+
parts << part
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
158
|
+
parts.join(' ')
|
144
159
|
end
|
145
160
|
|
146
161
|
##
|
data/lib/backup/version.rb
CHANGED
data/templates/cli/archive
CHANGED
@@ -22,6 +22,8 @@
|
|
22
22
|
# https://github.com/meskyanichi/backup/wiki/Archives
|
23
23
|
#
|
24
24
|
archive :my_archive do |archive|
|
25
|
+
# Run the `tar` command using `sudo`
|
26
|
+
# archive.use_sudo
|
25
27
|
archive.add "/path/to/a/file.rb"
|
26
28
|
archive.add "/path/to/a/folder/"
|
27
29
|
archive.exclude "/path/to/a/excluded_file.rb"
|
@@ -11,8 +11,5 @@
|
|
11
11
|
db.only_collections = ["only", "these", "collections"]
|
12
12
|
db.additional_options = []
|
13
13
|
db.lock = false
|
14
|
-
|
15
|
-
# if they cannot be found by their name in your $PATH
|
16
|
-
# db.mongodump_utility = "/opt/local/bin/mongodump"
|
17
|
-
# db.mongo_utility = "/opt/local/bin/mongo"
|
14
|
+
db.oplog = false
|
18
15
|
end
|
@@ -15,7 +15,4 @@
|
|
15
15
|
db.skip_tables = ["skip", "these", "tables"]
|
16
16
|
db.only_tables = ["only", "these", "tables"]
|
17
17
|
db.additional_options = ["--quick", "--single-transaction"]
|
18
|
-
# Optional: Use to set the location of this utility
|
19
|
-
# if it cannot be found by name in your $PATH
|
20
|
-
# db.mysqldump_utility = "/opt/local/bin/mysqldump"
|
21
18
|
end
|
@@ -11,7 +11,4 @@
|
|
11
11
|
db.skip_tables = ["skip", "these", "tables"]
|
12
12
|
db.only_tables = ["only", "these", "tables"]
|
13
13
|
db.additional_options = ["-xc", "-E=utf8"]
|
14
|
-
# Optional: Use to set the location of this utility
|
15
|
-
# if it cannot be found by name in your $PATH
|
16
|
-
# db.pg_dump_utility = "/opt/local/bin/pg_dump"
|
17
14
|
end
|
@@ -2,15 +2,17 @@
|
|
2
2
|
# Redis [Database]
|
3
3
|
#
|
4
4
|
database Redis do |db|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
db.
|
9
|
-
|
10
|
-
|
5
|
+
##
|
6
|
+
# From `dbfilename` in your `redis.conf` under SNAPSHOTTING.
|
7
|
+
# Do not include the '.rdb' extension. Defaults to 'dump'
|
8
|
+
db.name = 'dump'
|
9
|
+
##
|
10
|
+
# From `dir` in your `redis.conf` under SNAPSHOTTING.
|
11
|
+
db.path = '/var/lib/redis'
|
12
|
+
db.password = 'my_password'
|
13
|
+
db.host = 'localhost'
|
14
|
+
db.port = 6379
|
15
|
+
db.socket = '/tmp/redis.sock'
|
11
16
|
db.additional_options = []
|
12
17
|
db.invoke_save = true
|
13
|
-
# Optional: Use to set the location of this utility
|
14
|
-
# if it cannot be found by name in your $PATH
|
15
|
-
# db.redis_cli_utility = "/opt/local/bin/redis-cli"
|
16
18
|
end
|
data/templates/cli/database/riak
CHANGED
@@ -2,19 +2,16 @@
|
|
2
2
|
# Riak [Database]
|
3
3
|
#
|
4
4
|
database Riak do |db|
|
5
|
-
db.name = "hostname"
|
6
|
-
db.node = "riak@hostname"
|
7
|
-
db.cookie = "cookie"
|
8
5
|
##
|
9
|
-
#
|
10
|
-
#
|
11
|
-
|
12
|
-
|
13
|
-
#
|
14
|
-
#
|
15
|
-
|
16
|
-
|
17
|
-
#
|
18
|
-
#
|
19
|
-
|
6
|
+
# The node from which to perform the backup.
|
7
|
+
# default: 'riak@127.0.0.1'
|
8
|
+
db.node = 'riak@hostname'
|
9
|
+
##
|
10
|
+
# The Erlang cookie/shared secret used to connect to the node.
|
11
|
+
# default: 'riak'
|
12
|
+
db.cookie = 'cookie'
|
13
|
+
##
|
14
|
+
# The user for the Riak instance.
|
15
|
+
# default: 'riak'
|
16
|
+
db.user = 'riak'
|
20
17
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: backup
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael van Rooijen
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-04-
|
11
|
+
date: 2013-04-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|