backup 3.6.0 → 3.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -0
- data/lib/backup.rb +14 -4
- data/lib/backup/archive.rb +3 -2
- data/lib/backup/cleaner.rb +4 -2
- data/lib/backup/cli.rb +7 -5
- data/lib/backup/cloud_io/base.rb +41 -0
- data/lib/backup/cloud_io/cloud_files.rb +296 -0
- data/lib/backup/cloud_io/s3.rb +252 -0
- data/lib/backup/compressor/gzip.rb +2 -1
- data/lib/backup/config.rb +13 -5
- data/lib/backup/configuration.rb +1 -1
- data/lib/backup/configuration/helpers.rb +3 -1
- data/lib/backup/database/base.rb +3 -1
- data/lib/backup/database/mongodb.rb +2 -2
- data/lib/backup/database/mysql.rb +2 -2
- data/lib/backup/database/postgresql.rb +12 -2
- data/lib/backup/database/redis.rb +3 -2
- data/lib/backup/encryptor/gpg.rb +8 -10
- data/lib/backup/errors.rb +39 -70
- data/lib/backup/logger.rb +7 -2
- data/lib/backup/logger/fog_adapter.rb +30 -0
- data/lib/backup/model.rb +32 -14
- data/lib/backup/notifier/base.rb +4 -3
- data/lib/backup/notifier/campfire.rb +0 -1
- data/lib/backup/notifier/http_post.rb +122 -0
- data/lib/backup/notifier/mail.rb +38 -0
- data/lib/backup/notifier/nagios.rb +69 -0
- data/lib/backup/notifier/prowl.rb +0 -1
- data/lib/backup/notifier/pushover.rb +0 -1
- data/lib/backup/package.rb +5 -0
- data/lib/backup/packager.rb +3 -2
- data/lib/backup/pipeline.rb +4 -2
- data/lib/backup/storage/base.rb +2 -1
- data/lib/backup/storage/cloud_files.rb +151 -0
- data/lib/backup/storage/cycler.rb +4 -2
- data/lib/backup/storage/dropbox.rb +20 -16
- data/lib/backup/storage/ftp.rb +1 -2
- data/lib/backup/storage/local.rb +3 -3
- data/lib/backup/storage/ninefold.rb +3 -4
- data/lib/backup/storage/rsync.rb +1 -2
- data/lib/backup/storage/s3.rb +49 -158
- data/lib/backup/storage/scp.rb +3 -4
- data/lib/backup/storage/sftp.rb +1 -2
- data/lib/backup/syncer/base.rb +0 -1
- data/lib/backup/syncer/cloud/base.rb +129 -208
- data/lib/backup/syncer/cloud/cloud_files.rb +56 -41
- data/lib/backup/syncer/cloud/local_file.rb +93 -0
- data/lib/backup/syncer/cloud/s3.rb +78 -31
- data/lib/backup/syncer/rsync/base.rb +7 -0
- data/lib/backup/syncer/rsync/local.rb +0 -5
- data/lib/backup/syncer/rsync/push.rb +1 -2
- data/lib/backup/utilities.rb +18 -15
- data/lib/backup/version.rb +1 -1
- data/templates/cli/notifier/http_post +35 -0
- data/templates/cli/notifier/nagios +13 -0
- data/templates/cli/storage/cloud_files +8 -17
- data/templates/cli/storage/s3 +3 -10
- data/templates/cli/syncer/cloud_files +3 -31
- data/templates/cli/syncer/s3 +3 -27
- data/templates/notifier/mail/failure.erb +6 -1
- data/templates/notifier/mail/success.erb +6 -1
- data/templates/notifier/mail/warning.erb +6 -1
- metadata +37 -42
- data/lib/backup/storage/cloudfiles.rb +0 -68
@@ -3,6 +3,7 @@
|
|
3
3
|
module Backup
|
4
4
|
module Database
|
5
5
|
class Redis < Base
|
6
|
+
class Error < Backup::Error; end
|
6
7
|
|
7
8
|
##
|
8
9
|
# Name of the redis dump file.
|
@@ -65,7 +66,7 @@ module Backup
|
|
65
66
|
def invoke_save!
|
66
67
|
resp = run(redis_save_cmd)
|
67
68
|
unless resp =~ /OK$/
|
68
|
-
raise
|
69
|
+
raise Error, <<-EOS
|
69
70
|
Could not invoke the Redis SAVE command.
|
70
71
|
Command was: #{ redis_save_cmd }
|
71
72
|
Response was: #{ resp }
|
@@ -76,7 +77,7 @@ module Backup
|
|
76
77
|
def copy!
|
77
78
|
src_path = File.join(path, name + '.rdb')
|
78
79
|
unless File.exist?(src_path)
|
79
|
-
raise
|
80
|
+
raise Error, <<-EOS
|
80
81
|
Redis database dump not found
|
81
82
|
File path was #{ src_path }
|
82
83
|
EOS
|
data/lib/backup/encryptor/gpg.rb
CHANGED
@@ -69,6 +69,8 @@ module Backup
|
|
69
69
|
# end
|
70
70
|
#
|
71
71
|
class GPG < Base
|
72
|
+
class Error < Backup::Error; end
|
73
|
+
|
72
74
|
MODES = [:asymmetric, :symmetric, :both]
|
73
75
|
|
74
76
|
##
|
@@ -102,8 +104,7 @@ module Backup
|
|
102
104
|
attr_reader :mode
|
103
105
|
def mode=(mode)
|
104
106
|
@mode = mode.to_sym
|
105
|
-
raise
|
106
|
-
"'#{ @mode }' is not a valid mode." unless MODES.include?(@mode)
|
107
|
+
raise Error, "'#{ @mode }' is not a valid mode." unless MODES.include?(@mode)
|
107
108
|
end
|
108
109
|
|
109
110
|
##
|
@@ -421,8 +422,7 @@ module Backup
|
|
421
422
|
prepare
|
422
423
|
|
423
424
|
if mode_options.empty?
|
424
|
-
raise
|
425
|
-
"Encryption could not be performed for mode '#{ mode }'"
|
425
|
+
raise Error, "Encryption could not be performed for mode '#{ mode }'"
|
426
426
|
end
|
427
427
|
|
428
428
|
yield "#{ utility(:gpg) } #{ base_options } #{ mode_options }", '.gpg'
|
@@ -491,7 +491,7 @@ module Backup
|
|
491
491
|
path
|
492
492
|
|
493
493
|
rescue => err
|
494
|
-
raise
|
494
|
+
raise Error.wrap(
|
495
495
|
err, "Failed to create or set permissions for #gpg_homedir")
|
496
496
|
end
|
497
497
|
|
@@ -524,8 +524,7 @@ module Backup
|
|
524
524
|
|
525
525
|
rescue => err
|
526
526
|
cleanup
|
527
|
-
raise
|
528
|
-
err, "Error creating temporary file for #gpg_config.")
|
527
|
+
raise Error.wrap(err, "Error creating temporary file for #gpg_config.")
|
529
528
|
end
|
530
529
|
|
531
530
|
##
|
@@ -591,8 +590,7 @@ module Backup
|
|
591
590
|
file.path
|
592
591
|
|
593
592
|
rescue => err
|
594
|
-
Logger.warn
|
595
|
-
err, "Error creating temporary passphrase file.")
|
593
|
+
Logger.warn Error.wrap(err, "Error creating temporary passphrase file.")
|
596
594
|
false
|
597
595
|
end
|
598
596
|
|
@@ -691,7 +689,7 @@ module Backup
|
|
691
689
|
keyid
|
692
690
|
|
693
691
|
rescue => err
|
694
|
-
Logger.warn
|
692
|
+
Logger.warn Error.wrap(
|
695
693
|
err, "Public key import failed for '#{ identifier }'")
|
696
694
|
nil
|
697
695
|
end
|
data/lib/backup/errors.rb
CHANGED
@@ -1,89 +1,58 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
3
|
module Backup
|
4
|
-
##
|
5
|
-
# - automatically defines module namespaces referenced under Backup::Errors
|
6
|
-
# - any constant name referenced that ends with 'Error' will be created
|
7
|
-
# as a subclass of Backup::Errors::Error
|
8
|
-
# - any constant name referenced that ends with 'FatalError' will be created
|
9
|
-
# as a subclass of Backup::Errors::FatalError
|
10
|
-
#
|
11
|
-
# e.g.
|
12
|
-
# err = Backup::Errors::Foo::Bar::FooError.new('error message')
|
13
|
-
# err.message => "Foo::Bar::FooError: error message"
|
14
|
-
#
|
15
|
-
# err = Backup::Errors::Foo::Bar::FooFatalError.new('error message')
|
16
|
-
# err.message => "Foo::Bar::FooFatalError: error message"
|
17
|
-
#
|
18
|
-
module ErrorsHelper
|
19
|
-
def const_missing(const)
|
20
|
-
if const.to_s.end_with?('FatalError')
|
21
|
-
module_eval("class #{const} < Backup::Errors::FatalError; end")
|
22
|
-
elsif const.to_s.end_with?('Error')
|
23
|
-
module_eval("class #{const} < Backup::Errors::Error; end")
|
24
|
-
else
|
25
|
-
module_eval("module #{const}; extend Backup::ErrorsHelper; end")
|
26
|
-
end
|
27
|
-
const_get(const)
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
module Errors
|
32
|
-
extend ErrorsHelper
|
33
4
|
|
34
|
-
|
35
|
-
|
36
|
-
|
5
|
+
# Provides cascading errors with formatted messages.
|
6
|
+
# See the specs for details.
|
7
|
+
module NestedExceptions
|
37
8
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
end
|
43
|
-
}
|
44
|
-
end
|
45
|
-
|
46
|
-
def initialize(obj = nil, wrapped_exception = nil)
|
47
|
-
@wrapped_exception = wrapped_exception
|
48
|
-
msg = (obj.respond_to?(:to_str) ? obj.to_str : obj.to_s).
|
49
|
-
gsub(/^ */, ' ').strip
|
50
|
-
msg = clean_name(self.class.name) + (msg.empty? ? '' : ": #{ msg }")
|
51
|
-
|
52
|
-
if wrapped_exception
|
53
|
-
msg << "\n--- Wrapped Exception ---\n"
|
54
|
-
class_name = clean_name(wrapped_exception.class.name)
|
55
|
-
msg << class_name + ': ' unless
|
56
|
-
wrapped_exception.message.start_with? class_name
|
57
|
-
msg << wrapped_exception.message
|
9
|
+
def self.included(klass)
|
10
|
+
klass.extend Module.new {
|
11
|
+
def wrap(wrapped_exception, msg = nil)
|
12
|
+
new(msg, wrapped_exception)
|
58
13
|
end
|
14
|
+
}
|
15
|
+
end
|
59
16
|
|
60
|
-
|
61
|
-
|
17
|
+
def initialize(obj = nil, wrapped_exception = nil)
|
18
|
+
@wrapped_exception = wrapped_exception
|
19
|
+
msg = (obj.respond_to?(:to_str) ? obj.to_str : obj.to_s).
|
20
|
+
gsub(/^ */, ' ').strip
|
21
|
+
msg = clean_name(self.class.name) + (msg.empty? ? '' : ": #{ msg }")
|
22
|
+
|
23
|
+
if wrapped_exception
|
24
|
+
msg << "\n--- Wrapped Exception ---\n"
|
25
|
+
class_name = clean_name(wrapped_exception.class.name)
|
26
|
+
msg << class_name + ': ' unless
|
27
|
+
wrapped_exception.message.start_with? class_name
|
28
|
+
msg << wrapped_exception.message
|
62
29
|
end
|
63
30
|
|
64
|
-
|
65
|
-
|
31
|
+
super(msg)
|
32
|
+
set_backtrace(wrapped_exception.backtrace) if wrapped_exception
|
33
|
+
end
|
66
34
|
|
67
|
-
|
68
|
-
|
69
|
-
ex
|
70
|
-
end
|
35
|
+
def exception(obj = nil)
|
36
|
+
return self if obj.nil? || equal?(obj)
|
71
37
|
|
72
|
-
|
38
|
+
ex = self.class.new(obj, @wrapped_exception)
|
39
|
+
ex.set_backtrace(backtrace) unless ex.backtrace
|
40
|
+
ex
|
41
|
+
end
|
73
42
|
|
74
|
-
|
75
|
-
name.sub(/^Backup::Errors::/, '')
|
76
|
-
end
|
43
|
+
private
|
77
44
|
|
45
|
+
def clean_name(name)
|
46
|
+
name.sub(/^Backup::/, '')
|
78
47
|
end
|
79
48
|
|
80
|
-
|
81
|
-
include NestedExceptions
|
82
|
-
end
|
49
|
+
end
|
83
50
|
|
84
|
-
|
85
|
-
|
86
|
-
|
51
|
+
class Error < StandardError
|
52
|
+
include NestedExceptions
|
53
|
+
end
|
87
54
|
|
55
|
+
class FatalError < Exception
|
56
|
+
include NestedExceptions
|
88
57
|
end
|
89
58
|
end
|
data/lib/backup/logger.rb
CHANGED
@@ -3,6 +3,7 @@
|
|
3
3
|
require 'backup/logger/console'
|
4
4
|
require 'backup/logger/logfile'
|
5
5
|
require 'backup/logger/syslog'
|
6
|
+
require 'backup/logger/fog_adapter'
|
6
7
|
|
7
8
|
module Backup
|
8
9
|
class Logger
|
@@ -118,6 +119,8 @@ module Backup
|
|
118
119
|
end
|
119
120
|
end
|
120
121
|
|
122
|
+
MUTEX = Mutex.new
|
123
|
+
|
121
124
|
##
|
122
125
|
# Returns an Array of Message objects for all logged messages received.
|
123
126
|
# These are used to attach log files to Mail notifications.
|
@@ -134,7 +137,9 @@ module Backup
|
|
134
137
|
# Sends a message to the Logger using the specified log level.
|
135
138
|
# +obj+ may be any Object that responds to #to_s (i.e. an Exception)
|
136
139
|
[:info, :warn, :error].each do |level|
|
137
|
-
define_method level, lambda {|obj|
|
140
|
+
define_method level, lambda {|obj|
|
141
|
+
MUTEX.synchronize { log(obj, level) }
|
142
|
+
}
|
138
143
|
end
|
139
144
|
|
140
145
|
##
|
@@ -180,7 +185,7 @@ module Backup
|
|
180
185
|
private
|
181
186
|
|
182
187
|
def log(obj, level)
|
183
|
-
message = Message.new(Time.now, level, obj.to_s.split("\n"))
|
188
|
+
message = Message.new(Time.now.utc, level, obj.to_s.split("\n"))
|
184
189
|
|
185
190
|
message.level = :info if message.level == :warn &&
|
186
191
|
message.matches?(@config.ignores)
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
# require only the logger
|
4
|
+
require 'formatador'
|
5
|
+
require 'fog/core/logger'
|
6
|
+
|
7
|
+
module Backup
|
8
|
+
class Logger
|
9
|
+
module FogAdapter
|
10
|
+
class << self
|
11
|
+
|
12
|
+
# Logged as :info so these won't generate warnings.
|
13
|
+
# This is mostly to keep STDOUT clean and to provide
|
14
|
+
# supplemental messages for our own warnings.
|
15
|
+
# These will generally occur during retry attempts.
|
16
|
+
def write(message)
|
17
|
+
Logger.info message.split("\n").
|
18
|
+
map {|line| "[fog] #{ line }" }.join("\n")
|
19
|
+
end
|
20
|
+
|
21
|
+
def tty?
|
22
|
+
false
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
Fog::Logger[:warning] = Backup::Logger::FogAdapter
|
data/lib/backup/model.rb
CHANGED
@@ -2,6 +2,9 @@
|
|
2
2
|
|
3
3
|
module Backup
|
4
4
|
class Model
|
5
|
+
class Error < Backup::Error; end
|
6
|
+
class FatalError < Backup::FatalError; end
|
7
|
+
|
5
8
|
class << self
|
6
9
|
##
|
7
10
|
# The Backup::Model.all class method keeps track of all the models
|
@@ -73,6 +76,14 @@ module Backup
|
|
73
76
|
# The time when the backup initiated (in format: 2011.02.20.03.29.59)
|
74
77
|
attr_reader :time
|
75
78
|
|
79
|
+
##
|
80
|
+
# The time when the backup initiated (as a Time object)
|
81
|
+
attr_reader :started_at
|
82
|
+
|
83
|
+
##
|
84
|
+
# The time when the backup finished (as a Time object)
|
85
|
+
attr_reader :finished_at
|
86
|
+
|
76
87
|
##
|
77
88
|
# Result of this model's backup process.
|
78
89
|
#
|
@@ -136,7 +147,7 @@ module Backup
|
|
136
147
|
# Warn user of DSL changes
|
137
148
|
case name.to_s
|
138
149
|
when 'Backup::Config::RSync'
|
139
|
-
Logger.warn
|
150
|
+
Logger.warn Error.new(<<-EOS)
|
140
151
|
Configuration Update Needed for Syncer::RSync
|
141
152
|
The RSync Syncer has been split into three separate modules:
|
142
153
|
RSync::Local, RSync::Push and RSync::Pull
|
@@ -146,7 +157,7 @@ module Backup
|
|
146
157
|
name = 'RSync::Push'
|
147
158
|
when /(Backup::Config::S3|Backup::Config::CloudFiles)/
|
148
159
|
syncer = $1.split('::')[2]
|
149
|
-
Logger.warn
|
160
|
+
Logger.warn Error.new(<<-EOS)
|
150
161
|
Configuration Update Needed for '#{ syncer }' Syncer.
|
151
162
|
This Syncer is now referenced as Cloud::#{ syncer }
|
152
163
|
i.e. 'sync_with #{ syncer }' is now 'sync_with Cloud::#{ syncer }'
|
@@ -184,7 +195,7 @@ module Backup
|
|
184
195
|
if chunk_size.is_a?(Integer)
|
185
196
|
@splitter = Splitter.new(self, chunk_size)
|
186
197
|
else
|
187
|
-
raise
|
198
|
+
raise Error, <<-EOS
|
188
199
|
Invalid Chunk Size for Splitter
|
189
200
|
Argument to #split_into_chunks_of() must be an Integer
|
190
201
|
EOS
|
@@ -250,8 +261,8 @@ module Backup
|
|
250
261
|
# those files *** will be removed *** before the next scheduled backup for
|
251
262
|
# the same trigger.
|
252
263
|
def perform!
|
253
|
-
@started_at = Time.now
|
254
|
-
@time = package.time =
|
264
|
+
@started_at = Time.now.utc
|
265
|
+
@time = package.time = started_at.strftime("%Y.%m.%d.%H.%M.%S")
|
255
266
|
|
256
267
|
log!(:started)
|
257
268
|
before_hook
|
@@ -267,10 +278,18 @@ module Backup
|
|
267
278
|
|
268
279
|
ensure
|
269
280
|
set_exit_status
|
281
|
+
@finished_at = Time.now.utc
|
270
282
|
log!(:finished)
|
271
283
|
after_hook
|
272
284
|
end
|
273
285
|
|
286
|
+
##
|
287
|
+
# The duration of the backup process (in format: HH:MM:SS)
|
288
|
+
def duration
|
289
|
+
return unless finished_at
|
290
|
+
elapsed_time(started_at, finished_at)
|
291
|
+
end
|
292
|
+
|
274
293
|
private
|
275
294
|
|
276
295
|
##
|
@@ -359,8 +378,7 @@ module Backup
|
|
359
378
|
|
360
379
|
rescue Exception => err
|
361
380
|
@before_hook_failed = true
|
362
|
-
ex = err.is_a?(StandardError) ?
|
363
|
-
Errors::Model::HookError : Errors::Model::HookFatalError
|
381
|
+
ex = err.is_a?(StandardError) ? Error : FatalError
|
364
382
|
raise ex.wrap(err, 'Before Hook Failed!')
|
365
383
|
end
|
366
384
|
|
@@ -379,7 +397,7 @@ module Backup
|
|
379
397
|
|
380
398
|
rescue Exception => err
|
381
399
|
fatal = !err.is_a?(StandardError)
|
382
|
-
ex = fatal ?
|
400
|
+
ex = fatal ? FatalError : Error
|
383
401
|
Logger.error ex.wrap(err, 'After Hook Failed!')
|
384
402
|
# upgrade exit_status if needed
|
385
403
|
(@exit_status = fatal ? 3 : 2) unless exit_status == 3
|
@@ -398,7 +416,7 @@ module Backup
|
|
398
416
|
|
399
417
|
when :finished
|
400
418
|
if exit_status > 1
|
401
|
-
ex = exit_status == 2 ?
|
419
|
+
ex = exit_status == 2 ? Error : FatalError
|
402
420
|
err = ex.wrap(exception, "Backup for #{ label } (#{ trigger }) Failed!")
|
403
421
|
Logger.error err
|
404
422
|
Logger.error "\nBacktrace:\n\s\s" + err.backtrace.join("\n\s\s") + "\n\n"
|
@@ -407,10 +425,10 @@ module Backup
|
|
407
425
|
else
|
408
426
|
msg = "Backup for '#{ label } (#{ trigger })' "
|
409
427
|
if exit_status == 1
|
410
|
-
msg << "Completed Successfully (with Warnings) in #{
|
428
|
+
msg << "Completed Successfully (with Warnings) in #{ duration }"
|
411
429
|
Logger.warn msg
|
412
430
|
else
|
413
|
-
msg << "Completed Successfully in #{
|
431
|
+
msg << "Completed Successfully in #{ duration }"
|
414
432
|
Logger.info msg
|
415
433
|
end
|
416
434
|
end
|
@@ -418,9 +436,9 @@ module Backup
|
|
418
436
|
end
|
419
437
|
|
420
438
|
##
|
421
|
-
# Returns a string representing the elapsed time
|
422
|
-
def elapsed_time
|
423
|
-
duration =
|
439
|
+
# Returns a string representing the elapsed time in HH:MM:SS.
|
440
|
+
def elapsed_time(start_time, finish_time)
|
441
|
+
duration = finish_time.to_i - start_time.to_i
|
424
442
|
hours = duration / 3600
|
425
443
|
remainder = duration - (hours * 3600)
|
426
444
|
minutes = remainder / 60
|
data/lib/backup/notifier/base.rb
CHANGED
@@ -2,6 +2,8 @@
|
|
2
2
|
|
3
3
|
module Backup
|
4
4
|
module Notifier
|
5
|
+
class Error < Backup::Error; end
|
6
|
+
|
5
7
|
class Base
|
6
8
|
include Backup::Utilities::Helpers
|
7
9
|
include Backup::Configuration::Helpers
|
@@ -66,7 +68,7 @@ module Backup
|
|
66
68
|
end
|
67
69
|
|
68
70
|
rescue Exception => err
|
69
|
-
Logger.error
|
71
|
+
Logger.error Error.wrap(err, "#{ notifier_name } Failed!")
|
70
72
|
end
|
71
73
|
|
72
74
|
private
|
@@ -79,8 +81,7 @@ module Backup
|
|
79
81
|
retries += 1
|
80
82
|
raise if retries > max_retries
|
81
83
|
|
82
|
-
Logger.info
|
83
|
-
wrap(err, "Retry ##{ retries } of #{ max_retries }.")
|
84
|
+
Logger.info Error.wrap(err, "Retry ##{ retries } of #{ max_retries }.")
|
84
85
|
sleep(retry_waitsec)
|
85
86
|
retry
|
86
87
|
end
|