backup 4.4.1 → 5.0.0.beta.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/LICENSE +19 -0
- data/README.md +13 -9
- data/bin/docker_test +24 -0
- data/lib/backup/archive.rb +31 -32
- data/lib/backup/binder.rb +2 -6
- data/lib/backup/cleaner.rb +14 -18
- data/lib/backup/cli.rb +104 -108
- data/lib/backup/cloud_io/base.rb +4 -7
- data/lib/backup/cloud_io/cloud_files.rb +60 -62
- data/lib/backup/cloud_io/s3.rb +69 -76
- data/lib/backup/compressor/base.rb +4 -7
- data/lib/backup/compressor/bzip2.rb +3 -7
- data/lib/backup/compressor/custom.rb +2 -6
- data/lib/backup/compressor/gzip.rb +16 -17
- data/lib/backup/config/dsl.rb +16 -17
- data/lib/backup/config/helpers.rb +10 -16
- data/lib/backup/config.rb +17 -18
- data/lib/backup/database/base.rb +22 -21
- data/lib/backup/database/mongodb.rb +36 -37
- data/lib/backup/database/mysql.rb +40 -41
- data/lib/backup/database/openldap.rb +8 -10
- data/lib/backup/database/postgresql.rb +29 -30
- data/lib/backup/database/redis.rb +27 -30
- data/lib/backup/database/riak.rb +15 -18
- data/lib/backup/database/sqlite.rb +4 -6
- data/lib/backup/encryptor/base.rb +2 -4
- data/lib/backup/encryptor/gpg.rb +49 -59
- data/lib/backup/encryptor/open_ssl.rb +11 -14
- data/lib/backup/errors.rb +7 -12
- data/lib/backup/logger/console.rb +5 -8
- data/lib/backup/logger/fog_adapter.rb +2 -6
- data/lib/backup/logger/logfile.rb +10 -12
- data/lib/backup/logger/syslog.rb +2 -4
- data/lib/backup/logger.rb +16 -18
- data/lib/backup/model.rb +33 -40
- data/lib/backup/notifier/base.rb +24 -26
- data/lib/backup/notifier/campfire.rb +9 -11
- data/lib/backup/notifier/command.rb +0 -3
- data/lib/backup/notifier/datadog.rb +9 -12
- data/lib/backup/notifier/flowdock.rb +13 -17
- data/lib/backup/notifier/hipchat.rb +11 -13
- data/lib/backup/notifier/http_post.rb +11 -14
- data/lib/backup/notifier/mail.rb +42 -59
- data/lib/backup/notifier/nagios.rb +5 -9
- data/lib/backup/notifier/pagerduty.rb +10 -12
- data/lib/backup/notifier/prowl.rb +15 -15
- data/lib/backup/notifier/pushover.rb +7 -10
- data/lib/backup/notifier/ses.rb +34 -16
- data/lib/backup/notifier/slack.rb +39 -40
- data/lib/backup/notifier/twitter.rb +2 -5
- data/lib/backup/notifier/zabbix.rb +11 -14
- data/lib/backup/package.rb +5 -9
- data/lib/backup/packager.rb +16 -17
- data/lib/backup/pipeline.rb +17 -21
- data/lib/backup/splitter.rb +8 -11
- data/lib/backup/storage/base.rb +5 -8
- data/lib/backup/storage/cloud_files.rb +21 -23
- data/lib/backup/storage/cycler.rb +10 -15
- data/lib/backup/storage/dropbox.rb +15 -21
- data/lib/backup/storage/ftp.rb +14 -10
- data/lib/backup/storage/local.rb +5 -8
- data/lib/backup/storage/qiniu.rb +8 -8
- data/lib/backup/storage/rsync.rb +24 -26
- data/lib/backup/storage/s3.rb +27 -28
- data/lib/backup/storage/scp.rb +10 -12
- data/lib/backup/storage/sftp.rb +10 -12
- data/lib/backup/syncer/base.rb +5 -8
- data/lib/backup/syncer/cloud/base.rb +27 -30
- data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
- data/lib/backup/syncer/cloud/local_file.rb +5 -8
- data/lib/backup/syncer/cloud/s3.rb +23 -24
- data/lib/backup/syncer/rsync/base.rb +6 -10
- data/lib/backup/syncer/rsync/local.rb +1 -5
- data/lib/backup/syncer/rsync/pull.rb +6 -10
- data/lib/backup/syncer/rsync/push.rb +18 -22
- data/lib/backup/template.rb +9 -14
- data/lib/backup/utilities.rb +78 -69
- data/lib/backup/version.rb +1 -3
- data/lib/backup.rb +74 -78
- metadata +107 -676
data/lib/backup/cli.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
##
|
4
2
|
# Build the Backup Command Line Interface using Thor
|
5
3
|
module Backup
|
@@ -27,9 +25,9 @@ module Backup
|
|
27
25
|
#
|
28
26
|
# If the --check option is given, `backup check` will be run
|
29
27
|
# and no triggers will be performed.
|
30
|
-
desc
|
28
|
+
desc "perform", "Performs the backup for the specified trigger(s)."
|
31
29
|
|
32
|
-
long_desc <<-EOS.gsub(/^ +/,
|
30
|
+
long_desc <<-EOS.gsub(/^ +/, "")
|
33
31
|
Performs the backup for the specified trigger(s).
|
34
32
|
|
35
33
|
You may perform multiple backups by providing multiple triggers,
|
@@ -51,65 +49,65 @@ module Backup
|
|
51
49
|
EOS
|
52
50
|
|
53
51
|
method_option :trigger,
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
52
|
+
aliases: ["-t", "--triggers"],
|
53
|
+
required: true,
|
54
|
+
type: :string,
|
55
|
+
desc: "Triggers to perform. e.g. 'trigger_a,trigger_b'"
|
58
56
|
|
59
57
|
method_option :config_file,
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
58
|
+
aliases: "-c",
|
59
|
+
type: :string,
|
60
|
+
default: "",
|
61
|
+
desc: "Path to your config.rb file."
|
64
62
|
|
65
63
|
method_option :root_path,
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
64
|
+
aliases: "-r",
|
65
|
+
type: :string,
|
66
|
+
default: "",
|
67
|
+
desc: "Root path to base all relative path on."
|
70
68
|
|
71
69
|
method_option :data_path,
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
70
|
+
aliases: "-d",
|
71
|
+
type: :string,
|
72
|
+
default: "",
|
73
|
+
desc: "Path to store storage cycling data."
|
76
74
|
|
77
75
|
method_option :log_path,
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
76
|
+
aliases: "-l",
|
77
|
+
type: :string,
|
78
|
+
default: "",
|
79
|
+
desc: "Path to store Backup's log file."
|
82
80
|
|
83
81
|
method_option :tmp_path,
|
84
|
-
|
85
|
-
|
86
|
-
|
82
|
+
type: :string,
|
83
|
+
default: "",
|
84
|
+
desc: "Path to store temporary data during the backup."
|
87
85
|
|
88
86
|
# Note that :quiet, :syslog and :logfile are specified as :string types,
|
89
87
|
# so the --no-<option> usage will set the value to nil instead of false.
|
90
88
|
method_option :quiet,
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
89
|
+
aliases: "-q",
|
90
|
+
type: :boolean,
|
91
|
+
default: false,
|
92
|
+
banner: "",
|
93
|
+
desc: "Disable console log output."
|
96
94
|
|
97
95
|
method_option :syslog,
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
96
|
+
type: :boolean,
|
97
|
+
default: false,
|
98
|
+
banner: "",
|
99
|
+
desc: "Enable logging to syslog."
|
102
100
|
|
103
101
|
method_option :logfile,
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
102
|
+
type: :boolean,
|
103
|
+
default: true,
|
104
|
+
banner: "",
|
105
|
+
desc: "Enable Backup's log file."
|
108
106
|
|
109
107
|
method_option :check,
|
110
|
-
|
111
|
-
|
112
|
-
|
108
|
+
type: :boolean,
|
109
|
+
default: false,
|
110
|
+
desc: "Check configuration for errors or warnings."
|
113
111
|
|
114
112
|
def perform
|
115
113
|
check if options[:check] # this will exit()
|
@@ -119,27 +117,28 @@ module Backup
|
|
119
117
|
# Set logger options
|
120
118
|
opts = options
|
121
119
|
Logger.configure do
|
122
|
-
console.quiet
|
123
|
-
logfile.enabled
|
124
|
-
logfile.log_path
|
125
|
-
syslog.enabled
|
120
|
+
console.quiet = opts[:quiet]
|
121
|
+
logfile.enabled = opts[:logfile]
|
122
|
+
logfile.log_path = opts[:log_path]
|
123
|
+
syslog.enabled = opts[:syslog]
|
126
124
|
end
|
127
125
|
|
128
126
|
# Load the user's +config.rb+ file and all their Models
|
129
127
|
Config.load(options)
|
130
128
|
|
131
129
|
# Identify all Models to be run for the given +triggers+.
|
132
|
-
triggers = options[:trigger].split(
|
133
|
-
models = triggers.
|
130
|
+
triggers = options[:trigger].split(",").map(&:strip)
|
131
|
+
models = triggers.uniq.flat_map do |trigger|
|
134
132
|
Model.find_by_trigger(trigger)
|
135
|
-
|
133
|
+
end
|
136
134
|
|
137
|
-
|
138
|
-
|
135
|
+
if models.empty?
|
136
|
+
raise Error, "No Models found for trigger(s) " \
|
137
|
+
"'#{triggers.join(",")}'."
|
138
|
+
end
|
139
139
|
|
140
140
|
# Finalize Logger and begin real-time logging.
|
141
141
|
Logger.start!
|
142
|
-
|
143
142
|
rescue Exception => err
|
144
143
|
Logger.error Error.wrap(err)
|
145
144
|
unless Helpers.is_backup_error? err
|
@@ -148,7 +147,7 @@ module Backup
|
|
148
147
|
# Logger configuration will be ignored
|
149
148
|
# and messages will be output to the console only.
|
150
149
|
Logger.abort!
|
151
|
-
exit
|
150
|
+
exit 3
|
152
151
|
end
|
153
152
|
|
154
153
|
until models.empty?
|
@@ -164,7 +163,7 @@ module Backup
|
|
164
163
|
Logger.info Error.new(<<-EOS)
|
165
164
|
Backup will now continue...
|
166
165
|
The following triggers will now be processed:
|
167
|
-
(#{
|
166
|
+
(#{models.map(&:trigger).join(", ")})
|
168
167
|
EOS
|
169
168
|
end
|
170
169
|
when 3
|
@@ -173,7 +172,7 @@ module Backup
|
|
173
172
|
Logger.error FatalError.new(<<-EOS)
|
174
173
|
Backup will now exit.
|
175
174
|
The following triggers will not be processed:
|
176
|
-
(#{
|
175
|
+
(#{models.map(&:trigger).join(", ")})
|
177
176
|
EOS
|
178
177
|
end
|
179
178
|
end
|
@@ -203,19 +202,19 @@ module Backup
|
|
203
202
|
#
|
204
203
|
# If successful, this method with exit(0).
|
205
204
|
# If there are Errors or Warnings, it will exit(1).
|
206
|
-
desc
|
205
|
+
desc "check", "Check for configuration errors or warnings"
|
207
206
|
|
208
|
-
long_desc <<-EOS.gsub(/^ +/,
|
207
|
+
long_desc <<-EOS.gsub(/^ +/, "")
|
209
208
|
Loads your 'config.rb' file and all models and reports any
|
210
209
|
errors or warnings with your configuration, including missing
|
211
210
|
dependencies and the use of any deprecated settings.
|
212
211
|
EOS
|
213
212
|
|
214
213
|
method_option :config_file,
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
214
|
+
aliases: "-c",
|
215
|
+
type: :string,
|
216
|
+
default: "",
|
217
|
+
desc: "Path to your config.rb file."
|
219
218
|
|
220
219
|
def check
|
221
220
|
begin
|
@@ -228,10 +227,10 @@ module Backup
|
|
228
227
|
end
|
229
228
|
|
230
229
|
if Logger.has_warnings? || Logger.has_errors?
|
231
|
-
Logger.error
|
230
|
+
Logger.error "Configuration Check Failed."
|
232
231
|
exit_code = 1
|
233
232
|
else
|
234
|
-
Logger.info
|
233
|
+
Logger.info "Configuration Check Succeeded."
|
235
234
|
exit_code = 0
|
236
235
|
end
|
237
236
|
|
@@ -245,130 +244,127 @@ module Backup
|
|
245
244
|
# For example:
|
246
245
|
# $ backup generate:model --trigger my_backup --databases='mongodb'
|
247
246
|
# will generate a pre-populated model with a base MongoDB setup
|
248
|
-
desc
|
247
|
+
desc "generate:model", "Generates a Backup model file."
|
249
248
|
|
250
|
-
long_desc <<-EOS.gsub(/^ +/,
|
249
|
+
long_desc <<-EOS.gsub(/^ +/, "")
|
251
250
|
Generates a Backup model file.
|
252
251
|
|
253
252
|
If your configuration file is not in the default location at
|
254
|
-
#{
|
253
|
+
#{Config.config_file}
|
255
254
|
you must specify it's location using '--config-file'.
|
256
255
|
If no configuration file exists at this location, one will be created.
|
257
256
|
|
258
257
|
The model file will be created as '<config_path>/models/<trigger>.rb'
|
259
258
|
Your model file will be created in a 'models/' sub-directory
|
260
259
|
where your config file is located. The default location would be:
|
261
|
-
#{
|
260
|
+
#{Config.root_path}/models/<trigger>.rb
|
262
261
|
EOS
|
263
262
|
|
264
263
|
method_option :trigger,
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
264
|
+
aliases: "-t",
|
265
|
+
required: true,
|
266
|
+
type: :string,
|
267
|
+
desc: "Trigger name for the Backup model"
|
269
268
|
|
270
269
|
method_option :config_file,
|
271
|
-
|
272
|
-
|
270
|
+
type: :string,
|
271
|
+
desc: "Path to your Backup configuration file"
|
273
272
|
|
274
273
|
# options with their available values
|
275
|
-
%w
|
276
|
-
path = File.join(Backup::TEMPLATE_PATH,
|
277
|
-
opts = Dir[path +
|
278
|
-
method_option name, :
|
274
|
+
%w[databases storages syncers encryptor compressor notifiers].each do |name|
|
275
|
+
path = File.join(Backup::TEMPLATE_PATH, "cli", name)
|
276
|
+
opts = Dir[path + "/*"].sort.map { |p| File.basename(p) }.join(", ")
|
277
|
+
method_option name, type: :string, desc: "(#{opts})"
|
279
278
|
end
|
280
279
|
|
281
280
|
method_option :archives,
|
282
|
-
|
283
|
-
|
281
|
+
type: :boolean,
|
282
|
+
desc: "Model will include tar archives."
|
284
283
|
|
285
284
|
method_option :splitter,
|
286
|
-
|
287
|
-
|
288
|
-
|
285
|
+
type: :boolean,
|
286
|
+
default: false,
|
287
|
+
desc: "Add Splitter to the model"
|
289
288
|
|
290
|
-
define_method
|
291
|
-
opts = options.merge(:
|
289
|
+
define_method "generate:model" do
|
290
|
+
opts = options.merge(trigger: options[:trigger].gsub(/\W/, "_"))
|
292
291
|
config_file = opts[:config_file] ?
|
293
292
|
File.expand_path(opts.delete(:config_file)) : Config.config_file
|
294
|
-
models_path = File.join(File.dirname(config_file),
|
295
|
-
model_file = File.join(models_path, "#{
|
293
|
+
models_path = File.join(File.dirname(config_file), "models")
|
294
|
+
model_file = File.join(models_path, "#{opts[:trigger]}.rb")
|
296
295
|
|
297
296
|
unless File.exist?(config_file)
|
298
|
-
invoke
|
297
|
+
invoke "generate:config", [], config_file: config_file
|
299
298
|
end
|
300
299
|
|
301
300
|
FileUtils.mkdir_p(models_path)
|
302
301
|
if Helpers.overwrite?(model_file)
|
303
|
-
File.open(model_file,
|
304
|
-
file.write(Backup::Template.new(
|
302
|
+
File.open(model_file, "w") do |file|
|
303
|
+
file.write(Backup::Template.new(options: opts).result("cli/model"))
|
305
304
|
end
|
306
|
-
puts "Generated model file: '#{
|
305
|
+
puts "Generated model file: '#{model_file}'."
|
307
306
|
end
|
308
307
|
end
|
309
308
|
|
310
309
|
##
|
311
310
|
# [Generate:Config]
|
312
311
|
# Generates the main configuration file
|
313
|
-
desc
|
312
|
+
desc "generate:config", "Generates the main Backup configuration file"
|
314
313
|
|
315
|
-
long_desc <<-EOS.gsub(/^ +/,
|
314
|
+
long_desc <<-EOS.gsub(/^ +/, "")
|
316
315
|
Path to the Backup configuration file to generate.
|
317
316
|
|
318
317
|
Defaults to:
|
319
318
|
|
320
|
-
#{
|
319
|
+
#{Config.config_file}
|
321
320
|
EOS
|
322
321
|
|
323
322
|
method_option :config_file,
|
324
|
-
|
325
|
-
|
323
|
+
type: :string,
|
324
|
+
desc: "Path to the Backup configuration file to generate."
|
326
325
|
|
327
|
-
define_method
|
326
|
+
define_method "generate:config" do
|
328
327
|
config_file = options[:config_file] ?
|
329
328
|
File.expand_path(options[:config_file]) : Config.config_file
|
330
329
|
|
331
330
|
FileUtils.mkdir_p(File.dirname(config_file))
|
332
331
|
if Helpers.overwrite?(config_file)
|
333
|
-
File.open(config_file,
|
334
|
-
file.write(Backup::Template.new.result(
|
332
|
+
File.open(config_file, "w") do |file|
|
333
|
+
file.write(Backup::Template.new.result("cli/config"))
|
335
334
|
end
|
336
|
-
puts "Generated configuration file: '#{
|
335
|
+
puts "Generated configuration file: '#{config_file}'."
|
337
336
|
end
|
338
337
|
end
|
339
338
|
|
340
339
|
##
|
341
340
|
# [Version]
|
342
341
|
# Returns the current version of the Backup gem
|
343
|
-
map
|
344
|
-
desc
|
342
|
+
map "-v" => :version
|
343
|
+
desc "version", "Display installed Backup version"
|
345
344
|
def version
|
346
|
-
puts "Backup #{
|
345
|
+
puts "Backup #{Backup::VERSION}"
|
347
346
|
end
|
348
347
|
|
349
348
|
# This is to avoid Thor's warnings when stubbing methods on the Thor class.
|
350
349
|
module Helpers
|
351
350
|
class << self
|
352
|
-
|
353
351
|
def overwrite?(path)
|
354
352
|
return true unless File.exist?(path)
|
355
353
|
|
356
|
-
$stderr.print "A file already exists at '#{
|
354
|
+
$stderr.print "A file already exists at '#{path}'.\n" \
|
357
355
|
"Do you want to overwrite? [y/n] "
|
358
356
|
/^[Yy]/ =~ $stdin.gets
|
359
357
|
end
|
360
358
|
|
361
359
|
def exec!(cmd)
|
362
|
-
puts "Launching: #{
|
360
|
+
puts "Launching: #{cmd}"
|
363
361
|
exec(cmd)
|
364
362
|
end
|
365
363
|
|
366
364
|
def is_backup_error?(error)
|
367
365
|
error.class.ancestors.include? Backup::Error
|
368
366
|
end
|
369
|
-
|
370
367
|
end
|
371
368
|
end
|
372
|
-
|
373
369
|
end
|
374
370
|
end
|
data/lib/backup/cloud_io/base.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module CloudIO
|
5
3
|
class Error < Backup::Error; end
|
@@ -22,20 +20,19 @@ module Backup
|
|
22
20
|
rescue => err
|
23
21
|
retries += 1
|
24
22
|
raise Error.wrap(err, <<-EOS) if retries > max_retries
|
25
|
-
Max Retries (#{
|
26
|
-
Operation: #{
|
23
|
+
Max Retries (#{max_retries}) Exceeded!
|
24
|
+
Operation: #{operation}
|
27
25
|
Be sure to check the log messages for each retry attempt.
|
28
26
|
EOS
|
29
27
|
|
30
28
|
Logger.info Error.wrap(err, <<-EOS)
|
31
|
-
Retry ##{
|
32
|
-
Operation: #{
|
29
|
+
Retry ##{retries} of #{max_retries}
|
30
|
+
Operation: #{operation}
|
33
31
|
EOS
|
34
32
|
sleep(retry_waitsec)
|
35
33
|
retry
|
36
34
|
end
|
37
35
|
end
|
38
|
-
|
39
36
|
end
|
40
37
|
end
|
41
38
|
end
|