backup 4.4.1 → 5.0.0.beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE +19 -0
- data/README.md +13 -9
- data/bin/docker_test +24 -0
- data/lib/backup/archive.rb +31 -32
- data/lib/backup/binder.rb +2 -6
- data/lib/backup/cleaner.rb +14 -18
- data/lib/backup/cli.rb +104 -108
- data/lib/backup/cloud_io/base.rb +4 -7
- data/lib/backup/cloud_io/cloud_files.rb +60 -62
- data/lib/backup/cloud_io/s3.rb +69 -76
- data/lib/backup/compressor/base.rb +4 -7
- data/lib/backup/compressor/bzip2.rb +3 -7
- data/lib/backup/compressor/custom.rb +2 -6
- data/lib/backup/compressor/gzip.rb +16 -17
- data/lib/backup/config/dsl.rb +16 -17
- data/lib/backup/config/helpers.rb +10 -16
- data/lib/backup/config.rb +17 -18
- data/lib/backup/database/base.rb +22 -21
- data/lib/backup/database/mongodb.rb +36 -37
- data/lib/backup/database/mysql.rb +40 -41
- data/lib/backup/database/openldap.rb +8 -10
- data/lib/backup/database/postgresql.rb +29 -30
- data/lib/backup/database/redis.rb +27 -30
- data/lib/backup/database/riak.rb +15 -18
- data/lib/backup/database/sqlite.rb +4 -6
- data/lib/backup/encryptor/base.rb +2 -4
- data/lib/backup/encryptor/gpg.rb +49 -59
- data/lib/backup/encryptor/open_ssl.rb +11 -14
- data/lib/backup/errors.rb +7 -12
- data/lib/backup/logger/console.rb +5 -8
- data/lib/backup/logger/fog_adapter.rb +2 -6
- data/lib/backup/logger/logfile.rb +10 -12
- data/lib/backup/logger/syslog.rb +2 -4
- data/lib/backup/logger.rb +16 -18
- data/lib/backup/model.rb +33 -40
- data/lib/backup/notifier/base.rb +24 -26
- data/lib/backup/notifier/campfire.rb +9 -11
- data/lib/backup/notifier/command.rb +0 -3
- data/lib/backup/notifier/datadog.rb +9 -12
- data/lib/backup/notifier/flowdock.rb +13 -17
- data/lib/backup/notifier/hipchat.rb +11 -13
- data/lib/backup/notifier/http_post.rb +11 -14
- data/lib/backup/notifier/mail.rb +42 -59
- data/lib/backup/notifier/nagios.rb +5 -9
- data/lib/backup/notifier/pagerduty.rb +10 -12
- data/lib/backup/notifier/prowl.rb +15 -15
- data/lib/backup/notifier/pushover.rb +7 -10
- data/lib/backup/notifier/ses.rb +34 -16
- data/lib/backup/notifier/slack.rb +39 -40
- data/lib/backup/notifier/twitter.rb +2 -5
- data/lib/backup/notifier/zabbix.rb +11 -14
- data/lib/backup/package.rb +5 -9
- data/lib/backup/packager.rb +16 -17
- data/lib/backup/pipeline.rb +17 -21
- data/lib/backup/splitter.rb +8 -11
- data/lib/backup/storage/base.rb +5 -8
- data/lib/backup/storage/cloud_files.rb +21 -23
- data/lib/backup/storage/cycler.rb +10 -15
- data/lib/backup/storage/dropbox.rb +15 -21
- data/lib/backup/storage/ftp.rb +14 -10
- data/lib/backup/storage/local.rb +5 -8
- data/lib/backup/storage/qiniu.rb +8 -8
- data/lib/backup/storage/rsync.rb +24 -26
- data/lib/backup/storage/s3.rb +27 -28
- data/lib/backup/storage/scp.rb +10 -12
- data/lib/backup/storage/sftp.rb +10 -12
- data/lib/backup/syncer/base.rb +5 -8
- data/lib/backup/syncer/cloud/base.rb +27 -30
- data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
- data/lib/backup/syncer/cloud/local_file.rb +5 -8
- data/lib/backup/syncer/cloud/s3.rb +23 -24
- data/lib/backup/syncer/rsync/base.rb +6 -10
- data/lib/backup/syncer/rsync/local.rb +1 -5
- data/lib/backup/syncer/rsync/pull.rb +6 -10
- data/lib/backup/syncer/rsync/push.rb +18 -22
- data/lib/backup/template.rb +9 -14
- data/lib/backup/utilities.rb +78 -69
- data/lib/backup/version.rb +1 -3
- data/lib/backup.rb +74 -78
- metadata +107 -676
data/lib/backup/cli.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
##
|
4
2
|
# Build the Backup Command Line Interface using Thor
|
5
3
|
module Backup
|
@@ -27,9 +25,9 @@ module Backup
|
|
27
25
|
#
|
28
26
|
# If the --check option is given, `backup check` will be run
|
29
27
|
# and no triggers will be performed.
|
30
|
-
desc
|
28
|
+
desc "perform", "Performs the backup for the specified trigger(s)."
|
31
29
|
|
32
|
-
long_desc <<-EOS.gsub(/^ +/,
|
30
|
+
long_desc <<-EOS.gsub(/^ +/, "")
|
33
31
|
Performs the backup for the specified trigger(s).
|
34
32
|
|
35
33
|
You may perform multiple backups by providing multiple triggers,
|
@@ -51,65 +49,65 @@ module Backup
|
|
51
49
|
EOS
|
52
50
|
|
53
51
|
method_option :trigger,
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
52
|
+
aliases: ["-t", "--triggers"],
|
53
|
+
required: true,
|
54
|
+
type: :string,
|
55
|
+
desc: "Triggers to perform. e.g. 'trigger_a,trigger_b'"
|
58
56
|
|
59
57
|
method_option :config_file,
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
58
|
+
aliases: "-c",
|
59
|
+
type: :string,
|
60
|
+
default: "",
|
61
|
+
desc: "Path to your config.rb file."
|
64
62
|
|
65
63
|
method_option :root_path,
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
64
|
+
aliases: "-r",
|
65
|
+
type: :string,
|
66
|
+
default: "",
|
67
|
+
desc: "Root path to base all relative path on."
|
70
68
|
|
71
69
|
method_option :data_path,
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
70
|
+
aliases: "-d",
|
71
|
+
type: :string,
|
72
|
+
default: "",
|
73
|
+
desc: "Path to store storage cycling data."
|
76
74
|
|
77
75
|
method_option :log_path,
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
76
|
+
aliases: "-l",
|
77
|
+
type: :string,
|
78
|
+
default: "",
|
79
|
+
desc: "Path to store Backup's log file."
|
82
80
|
|
83
81
|
method_option :tmp_path,
|
84
|
-
|
85
|
-
|
86
|
-
|
82
|
+
type: :string,
|
83
|
+
default: "",
|
84
|
+
desc: "Path to store temporary data during the backup."
|
87
85
|
|
88
86
|
# Note that :quiet, :syslog and :logfile are specified as :string types,
|
89
87
|
# so the --no-<option> usage will set the value to nil instead of false.
|
90
88
|
method_option :quiet,
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
89
|
+
aliases: "-q",
|
90
|
+
type: :boolean,
|
91
|
+
default: false,
|
92
|
+
banner: "",
|
93
|
+
desc: "Disable console log output."
|
96
94
|
|
97
95
|
method_option :syslog,
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
96
|
+
type: :boolean,
|
97
|
+
default: false,
|
98
|
+
banner: "",
|
99
|
+
desc: "Enable logging to syslog."
|
102
100
|
|
103
101
|
method_option :logfile,
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
102
|
+
type: :boolean,
|
103
|
+
default: true,
|
104
|
+
banner: "",
|
105
|
+
desc: "Enable Backup's log file."
|
108
106
|
|
109
107
|
method_option :check,
|
110
|
-
|
111
|
-
|
112
|
-
|
108
|
+
type: :boolean,
|
109
|
+
default: false,
|
110
|
+
desc: "Check configuration for errors or warnings."
|
113
111
|
|
114
112
|
def perform
|
115
113
|
check if options[:check] # this will exit()
|
@@ -119,27 +117,28 @@ module Backup
|
|
119
117
|
# Set logger options
|
120
118
|
opts = options
|
121
119
|
Logger.configure do
|
122
|
-
console.quiet
|
123
|
-
logfile.enabled
|
124
|
-
logfile.log_path
|
125
|
-
syslog.enabled
|
120
|
+
console.quiet = opts[:quiet]
|
121
|
+
logfile.enabled = opts[:logfile]
|
122
|
+
logfile.log_path = opts[:log_path]
|
123
|
+
syslog.enabled = opts[:syslog]
|
126
124
|
end
|
127
125
|
|
128
126
|
# Load the user's +config.rb+ file and all their Models
|
129
127
|
Config.load(options)
|
130
128
|
|
131
129
|
# Identify all Models to be run for the given +triggers+.
|
132
|
-
triggers = options[:trigger].split(
|
133
|
-
models = triggers.
|
130
|
+
triggers = options[:trigger].split(",").map(&:strip)
|
131
|
+
models = triggers.uniq.flat_map do |trigger|
|
134
132
|
Model.find_by_trigger(trigger)
|
135
|
-
|
133
|
+
end
|
136
134
|
|
137
|
-
|
138
|
-
|
135
|
+
if models.empty?
|
136
|
+
raise Error, "No Models found for trigger(s) " \
|
137
|
+
"'#{triggers.join(",")}'."
|
138
|
+
end
|
139
139
|
|
140
140
|
# Finalize Logger and begin real-time logging.
|
141
141
|
Logger.start!
|
142
|
-
|
143
142
|
rescue Exception => err
|
144
143
|
Logger.error Error.wrap(err)
|
145
144
|
unless Helpers.is_backup_error? err
|
@@ -148,7 +147,7 @@ module Backup
|
|
148
147
|
# Logger configuration will be ignored
|
149
148
|
# and messages will be output to the console only.
|
150
149
|
Logger.abort!
|
151
|
-
exit
|
150
|
+
exit 3
|
152
151
|
end
|
153
152
|
|
154
153
|
until models.empty?
|
@@ -164,7 +163,7 @@ module Backup
|
|
164
163
|
Logger.info Error.new(<<-EOS)
|
165
164
|
Backup will now continue...
|
166
165
|
The following triggers will now be processed:
|
167
|
-
(#{
|
166
|
+
(#{models.map(&:trigger).join(", ")})
|
168
167
|
EOS
|
169
168
|
end
|
170
169
|
when 3
|
@@ -173,7 +172,7 @@ module Backup
|
|
173
172
|
Logger.error FatalError.new(<<-EOS)
|
174
173
|
Backup will now exit.
|
175
174
|
The following triggers will not be processed:
|
176
|
-
(#{
|
175
|
+
(#{models.map(&:trigger).join(", ")})
|
177
176
|
EOS
|
178
177
|
end
|
179
178
|
end
|
@@ -203,19 +202,19 @@ module Backup
|
|
203
202
|
#
|
204
203
|
# If successful, this method with exit(0).
|
205
204
|
# If there are Errors or Warnings, it will exit(1).
|
206
|
-
desc
|
205
|
+
desc "check", "Check for configuration errors or warnings"
|
207
206
|
|
208
|
-
long_desc <<-EOS.gsub(/^ +/,
|
207
|
+
long_desc <<-EOS.gsub(/^ +/, "")
|
209
208
|
Loads your 'config.rb' file and all models and reports any
|
210
209
|
errors or warnings with your configuration, including missing
|
211
210
|
dependencies and the use of any deprecated settings.
|
212
211
|
EOS
|
213
212
|
|
214
213
|
method_option :config_file,
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
214
|
+
aliases: "-c",
|
215
|
+
type: :string,
|
216
|
+
default: "",
|
217
|
+
desc: "Path to your config.rb file."
|
219
218
|
|
220
219
|
def check
|
221
220
|
begin
|
@@ -228,10 +227,10 @@ module Backup
|
|
228
227
|
end
|
229
228
|
|
230
229
|
if Logger.has_warnings? || Logger.has_errors?
|
231
|
-
Logger.error
|
230
|
+
Logger.error "Configuration Check Failed."
|
232
231
|
exit_code = 1
|
233
232
|
else
|
234
|
-
Logger.info
|
233
|
+
Logger.info "Configuration Check Succeeded."
|
235
234
|
exit_code = 0
|
236
235
|
end
|
237
236
|
|
@@ -245,130 +244,127 @@ module Backup
|
|
245
244
|
# For example:
|
246
245
|
# $ backup generate:model --trigger my_backup --databases='mongodb'
|
247
246
|
# will generate a pre-populated model with a base MongoDB setup
|
248
|
-
desc
|
247
|
+
desc "generate:model", "Generates a Backup model file."
|
249
248
|
|
250
|
-
long_desc <<-EOS.gsub(/^ +/,
|
249
|
+
long_desc <<-EOS.gsub(/^ +/, "")
|
251
250
|
Generates a Backup model file.
|
252
251
|
|
253
252
|
If your configuration file is not in the default location at
|
254
|
-
#{
|
253
|
+
#{Config.config_file}
|
255
254
|
you must specify it's location using '--config-file'.
|
256
255
|
If no configuration file exists at this location, one will be created.
|
257
256
|
|
258
257
|
The model file will be created as '<config_path>/models/<trigger>.rb'
|
259
258
|
Your model file will be created in a 'models/' sub-directory
|
260
259
|
where your config file is located. The default location would be:
|
261
|
-
#{
|
260
|
+
#{Config.root_path}/models/<trigger>.rb
|
262
261
|
EOS
|
263
262
|
|
264
263
|
method_option :trigger,
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
264
|
+
aliases: "-t",
|
265
|
+
required: true,
|
266
|
+
type: :string,
|
267
|
+
desc: "Trigger name for the Backup model"
|
269
268
|
|
270
269
|
method_option :config_file,
|
271
|
-
|
272
|
-
|
270
|
+
type: :string,
|
271
|
+
desc: "Path to your Backup configuration file"
|
273
272
|
|
274
273
|
# options with their available values
|
275
|
-
%w
|
276
|
-
path = File.join(Backup::TEMPLATE_PATH,
|
277
|
-
opts = Dir[path +
|
278
|
-
method_option name, :
|
274
|
+
%w[databases storages syncers encryptor compressor notifiers].each do |name|
|
275
|
+
path = File.join(Backup::TEMPLATE_PATH, "cli", name)
|
276
|
+
opts = Dir[path + "/*"].sort.map { |p| File.basename(p) }.join(", ")
|
277
|
+
method_option name, type: :string, desc: "(#{opts})"
|
279
278
|
end
|
280
279
|
|
281
280
|
method_option :archives,
|
282
|
-
|
283
|
-
|
281
|
+
type: :boolean,
|
282
|
+
desc: "Model will include tar archives."
|
284
283
|
|
285
284
|
method_option :splitter,
|
286
|
-
|
287
|
-
|
288
|
-
|
285
|
+
type: :boolean,
|
286
|
+
default: false,
|
287
|
+
desc: "Add Splitter to the model"
|
289
288
|
|
290
|
-
define_method
|
291
|
-
opts = options.merge(:
|
289
|
+
define_method "generate:model" do
|
290
|
+
opts = options.merge(trigger: options[:trigger].gsub(/\W/, "_"))
|
292
291
|
config_file = opts[:config_file] ?
|
293
292
|
File.expand_path(opts.delete(:config_file)) : Config.config_file
|
294
|
-
models_path = File.join(File.dirname(config_file),
|
295
|
-
model_file = File.join(models_path, "#{
|
293
|
+
models_path = File.join(File.dirname(config_file), "models")
|
294
|
+
model_file = File.join(models_path, "#{opts[:trigger]}.rb")
|
296
295
|
|
297
296
|
unless File.exist?(config_file)
|
298
|
-
invoke
|
297
|
+
invoke "generate:config", [], config_file: config_file
|
299
298
|
end
|
300
299
|
|
301
300
|
FileUtils.mkdir_p(models_path)
|
302
301
|
if Helpers.overwrite?(model_file)
|
303
|
-
File.open(model_file,
|
304
|
-
file.write(Backup::Template.new(
|
302
|
+
File.open(model_file, "w") do |file|
|
303
|
+
file.write(Backup::Template.new(options: opts).result("cli/model"))
|
305
304
|
end
|
306
|
-
puts "Generated model file: '#{
|
305
|
+
puts "Generated model file: '#{model_file}'."
|
307
306
|
end
|
308
307
|
end
|
309
308
|
|
310
309
|
##
|
311
310
|
# [Generate:Config]
|
312
311
|
# Generates the main configuration file
|
313
|
-
desc
|
312
|
+
desc "generate:config", "Generates the main Backup configuration file"
|
314
313
|
|
315
|
-
long_desc <<-EOS.gsub(/^ +/,
|
314
|
+
long_desc <<-EOS.gsub(/^ +/, "")
|
316
315
|
Path to the Backup configuration file to generate.
|
317
316
|
|
318
317
|
Defaults to:
|
319
318
|
|
320
|
-
#{
|
319
|
+
#{Config.config_file}
|
321
320
|
EOS
|
322
321
|
|
323
322
|
method_option :config_file,
|
324
|
-
|
325
|
-
|
323
|
+
type: :string,
|
324
|
+
desc: "Path to the Backup configuration file to generate."
|
326
325
|
|
327
|
-
define_method
|
326
|
+
define_method "generate:config" do
|
328
327
|
config_file = options[:config_file] ?
|
329
328
|
File.expand_path(options[:config_file]) : Config.config_file
|
330
329
|
|
331
330
|
FileUtils.mkdir_p(File.dirname(config_file))
|
332
331
|
if Helpers.overwrite?(config_file)
|
333
|
-
File.open(config_file,
|
334
|
-
file.write(Backup::Template.new.result(
|
332
|
+
File.open(config_file, "w") do |file|
|
333
|
+
file.write(Backup::Template.new.result("cli/config"))
|
335
334
|
end
|
336
|
-
puts "Generated configuration file: '#{
|
335
|
+
puts "Generated configuration file: '#{config_file}'."
|
337
336
|
end
|
338
337
|
end
|
339
338
|
|
340
339
|
##
|
341
340
|
# [Version]
|
342
341
|
# Returns the current version of the Backup gem
|
343
|
-
map
|
344
|
-
desc
|
342
|
+
map "-v" => :version
|
343
|
+
desc "version", "Display installed Backup version"
|
345
344
|
def version
|
346
|
-
puts "Backup #{
|
345
|
+
puts "Backup #{Backup::VERSION}"
|
347
346
|
end
|
348
347
|
|
349
348
|
# This is to avoid Thor's warnings when stubbing methods on the Thor class.
|
350
349
|
module Helpers
|
351
350
|
class << self
|
352
|
-
|
353
351
|
def overwrite?(path)
|
354
352
|
return true unless File.exist?(path)
|
355
353
|
|
356
|
-
$stderr.print "A file already exists at '#{
|
354
|
+
$stderr.print "A file already exists at '#{path}'.\n" \
|
357
355
|
"Do you want to overwrite? [y/n] "
|
358
356
|
/^[Yy]/ =~ $stdin.gets
|
359
357
|
end
|
360
358
|
|
361
359
|
def exec!(cmd)
|
362
|
-
puts "Launching: #{
|
360
|
+
puts "Launching: #{cmd}"
|
363
361
|
exec(cmd)
|
364
362
|
end
|
365
363
|
|
366
364
|
def is_backup_error?(error)
|
367
365
|
error.class.ancestors.include? Backup::Error
|
368
366
|
end
|
369
|
-
|
370
367
|
end
|
371
368
|
end
|
372
|
-
|
373
369
|
end
|
374
370
|
end
|
data/lib/backup/cloud_io/base.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module CloudIO
|
5
3
|
class Error < Backup::Error; end
|
@@ -22,20 +20,19 @@ module Backup
|
|
22
20
|
rescue => err
|
23
21
|
retries += 1
|
24
22
|
raise Error.wrap(err, <<-EOS) if retries > max_retries
|
25
|
-
Max Retries (#{
|
26
|
-
Operation: #{
|
23
|
+
Max Retries (#{max_retries}) Exceeded!
|
24
|
+
Operation: #{operation}
|
27
25
|
Be sure to check the log messages for each retry attempt.
|
28
26
|
EOS
|
29
27
|
|
30
28
|
Logger.info Error.wrap(err, <<-EOS)
|
31
|
-
Retry ##{
|
32
|
-
Operation: #{
|
29
|
+
Retry ##{retries} of #{max_retries}
|
30
|
+
Operation: #{operation}
|
33
31
|
EOS
|
34
32
|
sleep(retry_waitsec)
|
35
33
|
retry
|
36
34
|
end
|
37
35
|
end
|
38
|
-
|
39
36
|
end
|
40
37
|
end
|
41
38
|
end
|