backup 3.6.0 → 3.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2 -0
- data/lib/backup.rb +14 -4
- data/lib/backup/archive.rb +3 -2
- data/lib/backup/cleaner.rb +4 -2
- data/lib/backup/cli.rb +7 -5
- data/lib/backup/cloud_io/base.rb +41 -0
- data/lib/backup/cloud_io/cloud_files.rb +296 -0
- data/lib/backup/cloud_io/s3.rb +252 -0
- data/lib/backup/compressor/gzip.rb +2 -1
- data/lib/backup/config.rb +13 -5
- data/lib/backup/configuration.rb +1 -1
- data/lib/backup/configuration/helpers.rb +3 -1
- data/lib/backup/database/base.rb +3 -1
- data/lib/backup/database/mongodb.rb +2 -2
- data/lib/backup/database/mysql.rb +2 -2
- data/lib/backup/database/postgresql.rb +12 -2
- data/lib/backup/database/redis.rb +3 -2
- data/lib/backup/encryptor/gpg.rb +8 -10
- data/lib/backup/errors.rb +39 -70
- data/lib/backup/logger.rb +7 -2
- data/lib/backup/logger/fog_adapter.rb +30 -0
- data/lib/backup/model.rb +32 -14
- data/lib/backup/notifier/base.rb +4 -3
- data/lib/backup/notifier/campfire.rb +0 -1
- data/lib/backup/notifier/http_post.rb +122 -0
- data/lib/backup/notifier/mail.rb +38 -0
- data/lib/backup/notifier/nagios.rb +69 -0
- data/lib/backup/notifier/prowl.rb +0 -1
- data/lib/backup/notifier/pushover.rb +0 -1
- data/lib/backup/package.rb +5 -0
- data/lib/backup/packager.rb +3 -2
- data/lib/backup/pipeline.rb +4 -2
- data/lib/backup/storage/base.rb +2 -1
- data/lib/backup/storage/cloud_files.rb +151 -0
- data/lib/backup/storage/cycler.rb +4 -2
- data/lib/backup/storage/dropbox.rb +20 -16
- data/lib/backup/storage/ftp.rb +1 -2
- data/lib/backup/storage/local.rb +3 -3
- data/lib/backup/storage/ninefold.rb +3 -4
- data/lib/backup/storage/rsync.rb +1 -2
- data/lib/backup/storage/s3.rb +49 -158
- data/lib/backup/storage/scp.rb +3 -4
- data/lib/backup/storage/sftp.rb +1 -2
- data/lib/backup/syncer/base.rb +0 -1
- data/lib/backup/syncer/cloud/base.rb +129 -208
- data/lib/backup/syncer/cloud/cloud_files.rb +56 -41
- data/lib/backup/syncer/cloud/local_file.rb +93 -0
- data/lib/backup/syncer/cloud/s3.rb +78 -31
- data/lib/backup/syncer/rsync/base.rb +7 -0
- data/lib/backup/syncer/rsync/local.rb +0 -5
- data/lib/backup/syncer/rsync/push.rb +1 -2
- data/lib/backup/utilities.rb +18 -15
- data/lib/backup/version.rb +1 -1
- data/templates/cli/notifier/http_post +35 -0
- data/templates/cli/notifier/nagios +13 -0
- data/templates/cli/storage/cloud_files +8 -17
- data/templates/cli/storage/s3 +3 -10
- data/templates/cli/syncer/cloud_files +3 -31
- data/templates/cli/syncer/s3 +3 -27
- data/templates/notifier/mail/failure.erb +6 -1
- data/templates/notifier/mail/success.erb +6 -1
- data/templates/notifier/mail/warning.erb +6 -1
- metadata +37 -42
- data/lib/backup/storage/cloudfiles.rb +0 -68
@@ -3,6 +3,7 @@
|
|
3
3
|
module Backup
|
4
4
|
module Database
|
5
5
|
class Redis < Base
|
6
|
+
class Error < Backup::Error; end
|
6
7
|
|
7
8
|
##
|
8
9
|
# Name of the redis dump file.
|
@@ -65,7 +66,7 @@ module Backup
|
|
65
66
|
def invoke_save!
|
66
67
|
resp = run(redis_save_cmd)
|
67
68
|
unless resp =~ /OK$/
|
68
|
-
raise
|
69
|
+
raise Error, <<-EOS
|
69
70
|
Could not invoke the Redis SAVE command.
|
70
71
|
Command was: #{ redis_save_cmd }
|
71
72
|
Response was: #{ resp }
|
@@ -76,7 +77,7 @@ module Backup
|
|
76
77
|
def copy!
|
77
78
|
src_path = File.join(path, name + '.rdb')
|
78
79
|
unless File.exist?(src_path)
|
79
|
-
raise
|
80
|
+
raise Error, <<-EOS
|
80
81
|
Redis database dump not found
|
81
82
|
File path was #{ src_path }
|
82
83
|
EOS
|
data/lib/backup/encryptor/gpg.rb
CHANGED
@@ -69,6 +69,8 @@ module Backup
|
|
69
69
|
# end
|
70
70
|
#
|
71
71
|
class GPG < Base
|
72
|
+
class Error < Backup::Error; end
|
73
|
+
|
72
74
|
MODES = [:asymmetric, :symmetric, :both]
|
73
75
|
|
74
76
|
##
|
@@ -102,8 +104,7 @@ module Backup
|
|
102
104
|
attr_reader :mode
|
103
105
|
def mode=(mode)
|
104
106
|
@mode = mode.to_sym
|
105
|
-
raise
|
106
|
-
"'#{ @mode }' is not a valid mode." unless MODES.include?(@mode)
|
107
|
+
raise Error, "'#{ @mode }' is not a valid mode." unless MODES.include?(@mode)
|
107
108
|
end
|
108
109
|
|
109
110
|
##
|
@@ -421,8 +422,7 @@ module Backup
|
|
421
422
|
prepare
|
422
423
|
|
423
424
|
if mode_options.empty?
|
424
|
-
raise
|
425
|
-
"Encryption could not be performed for mode '#{ mode }'"
|
425
|
+
raise Error, "Encryption could not be performed for mode '#{ mode }'"
|
426
426
|
end
|
427
427
|
|
428
428
|
yield "#{ utility(:gpg) } #{ base_options } #{ mode_options }", '.gpg'
|
@@ -491,7 +491,7 @@ module Backup
|
|
491
491
|
path
|
492
492
|
|
493
493
|
rescue => err
|
494
|
-
raise
|
494
|
+
raise Error.wrap(
|
495
495
|
err, "Failed to create or set permissions for #gpg_homedir")
|
496
496
|
end
|
497
497
|
|
@@ -524,8 +524,7 @@ module Backup
|
|
524
524
|
|
525
525
|
rescue => err
|
526
526
|
cleanup
|
527
|
-
raise
|
528
|
-
err, "Error creating temporary file for #gpg_config.")
|
527
|
+
raise Error.wrap(err, "Error creating temporary file for #gpg_config.")
|
529
528
|
end
|
530
529
|
|
531
530
|
##
|
@@ -591,8 +590,7 @@ module Backup
|
|
591
590
|
file.path
|
592
591
|
|
593
592
|
rescue => err
|
594
|
-
Logger.warn
|
595
|
-
err, "Error creating temporary passphrase file.")
|
593
|
+
Logger.warn Error.wrap(err, "Error creating temporary passphrase file.")
|
596
594
|
false
|
597
595
|
end
|
598
596
|
|
@@ -691,7 +689,7 @@ module Backup
|
|
691
689
|
keyid
|
692
690
|
|
693
691
|
rescue => err
|
694
|
-
Logger.warn
|
692
|
+
Logger.warn Error.wrap(
|
695
693
|
err, "Public key import failed for '#{ identifier }'")
|
696
694
|
nil
|
697
695
|
end
|
data/lib/backup/errors.rb
CHANGED
@@ -1,89 +1,58 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
3
|
module Backup
|
4
|
-
##
|
5
|
-
# - automatically defines module namespaces referenced under Backup::Errors
|
6
|
-
# - any constant name referenced that ends with 'Error' will be created
|
7
|
-
# as a subclass of Backup::Errors::Error
|
8
|
-
# - any constant name referenced that ends with 'FatalError' will be created
|
9
|
-
# as a subclass of Backup::Errors::FatalError
|
10
|
-
#
|
11
|
-
# e.g.
|
12
|
-
# err = Backup::Errors::Foo::Bar::FooError.new('error message')
|
13
|
-
# err.message => "Foo::Bar::FooError: error message"
|
14
|
-
#
|
15
|
-
# err = Backup::Errors::Foo::Bar::FooFatalError.new('error message')
|
16
|
-
# err.message => "Foo::Bar::FooFatalError: error message"
|
17
|
-
#
|
18
|
-
module ErrorsHelper
|
19
|
-
def const_missing(const)
|
20
|
-
if const.to_s.end_with?('FatalError')
|
21
|
-
module_eval("class #{const} < Backup::Errors::FatalError; end")
|
22
|
-
elsif const.to_s.end_with?('Error')
|
23
|
-
module_eval("class #{const} < Backup::Errors::Error; end")
|
24
|
-
else
|
25
|
-
module_eval("module #{const}; extend Backup::ErrorsHelper; end")
|
26
|
-
end
|
27
|
-
const_get(const)
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
module Errors
|
32
|
-
extend ErrorsHelper
|
33
4
|
|
34
|
-
|
35
|
-
|
36
|
-
|
5
|
+
# Provides cascading errors with formatted messages.
|
6
|
+
# See the specs for details.
|
7
|
+
module NestedExceptions
|
37
8
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
end
|
43
|
-
}
|
44
|
-
end
|
45
|
-
|
46
|
-
def initialize(obj = nil, wrapped_exception = nil)
|
47
|
-
@wrapped_exception = wrapped_exception
|
48
|
-
msg = (obj.respond_to?(:to_str) ? obj.to_str : obj.to_s).
|
49
|
-
gsub(/^ */, ' ').strip
|
50
|
-
msg = clean_name(self.class.name) + (msg.empty? ? '' : ": #{ msg }")
|
51
|
-
|
52
|
-
if wrapped_exception
|
53
|
-
msg << "\n--- Wrapped Exception ---\n"
|
54
|
-
class_name = clean_name(wrapped_exception.class.name)
|
55
|
-
msg << class_name + ': ' unless
|
56
|
-
wrapped_exception.message.start_with? class_name
|
57
|
-
msg << wrapped_exception.message
|
9
|
+
def self.included(klass)
|
10
|
+
klass.extend Module.new {
|
11
|
+
def wrap(wrapped_exception, msg = nil)
|
12
|
+
new(msg, wrapped_exception)
|
58
13
|
end
|
14
|
+
}
|
15
|
+
end
|
59
16
|
|
60
|
-
|
61
|
-
|
17
|
+
def initialize(obj = nil, wrapped_exception = nil)
|
18
|
+
@wrapped_exception = wrapped_exception
|
19
|
+
msg = (obj.respond_to?(:to_str) ? obj.to_str : obj.to_s).
|
20
|
+
gsub(/^ */, ' ').strip
|
21
|
+
msg = clean_name(self.class.name) + (msg.empty? ? '' : ": #{ msg }")
|
22
|
+
|
23
|
+
if wrapped_exception
|
24
|
+
msg << "\n--- Wrapped Exception ---\n"
|
25
|
+
class_name = clean_name(wrapped_exception.class.name)
|
26
|
+
msg << class_name + ': ' unless
|
27
|
+
wrapped_exception.message.start_with? class_name
|
28
|
+
msg << wrapped_exception.message
|
62
29
|
end
|
63
30
|
|
64
|
-
|
65
|
-
|
31
|
+
super(msg)
|
32
|
+
set_backtrace(wrapped_exception.backtrace) if wrapped_exception
|
33
|
+
end
|
66
34
|
|
67
|
-
|
68
|
-
|
69
|
-
ex
|
70
|
-
end
|
35
|
+
def exception(obj = nil)
|
36
|
+
return self if obj.nil? || equal?(obj)
|
71
37
|
|
72
|
-
|
38
|
+
ex = self.class.new(obj, @wrapped_exception)
|
39
|
+
ex.set_backtrace(backtrace) unless ex.backtrace
|
40
|
+
ex
|
41
|
+
end
|
73
42
|
|
74
|
-
|
75
|
-
name.sub(/^Backup::Errors::/, '')
|
76
|
-
end
|
43
|
+
private
|
77
44
|
|
45
|
+
def clean_name(name)
|
46
|
+
name.sub(/^Backup::/, '')
|
78
47
|
end
|
79
48
|
|
80
|
-
|
81
|
-
include NestedExceptions
|
82
|
-
end
|
49
|
+
end
|
83
50
|
|
84
|
-
|
85
|
-
|
86
|
-
|
51
|
+
class Error < StandardError
|
52
|
+
include NestedExceptions
|
53
|
+
end
|
87
54
|
|
55
|
+
class FatalError < Exception
|
56
|
+
include NestedExceptions
|
88
57
|
end
|
89
58
|
end
|
data/lib/backup/logger.rb
CHANGED
@@ -3,6 +3,7 @@
|
|
3
3
|
require 'backup/logger/console'
|
4
4
|
require 'backup/logger/logfile'
|
5
5
|
require 'backup/logger/syslog'
|
6
|
+
require 'backup/logger/fog_adapter'
|
6
7
|
|
7
8
|
module Backup
|
8
9
|
class Logger
|
@@ -118,6 +119,8 @@ module Backup
|
|
118
119
|
end
|
119
120
|
end
|
120
121
|
|
122
|
+
MUTEX = Mutex.new
|
123
|
+
|
121
124
|
##
|
122
125
|
# Returns an Array of Message objects for all logged messages received.
|
123
126
|
# These are used to attach log files to Mail notifications.
|
@@ -134,7 +137,9 @@ module Backup
|
|
134
137
|
# Sends a message to the Logger using the specified log level.
|
135
138
|
# +obj+ may be any Object that responds to #to_s (i.e. an Exception)
|
136
139
|
[:info, :warn, :error].each do |level|
|
137
|
-
define_method level, lambda {|obj|
|
140
|
+
define_method level, lambda {|obj|
|
141
|
+
MUTEX.synchronize { log(obj, level) }
|
142
|
+
}
|
138
143
|
end
|
139
144
|
|
140
145
|
##
|
@@ -180,7 +185,7 @@ module Backup
|
|
180
185
|
private
|
181
186
|
|
182
187
|
def log(obj, level)
|
183
|
-
message = Message.new(Time.now, level, obj.to_s.split("\n"))
|
188
|
+
message = Message.new(Time.now.utc, level, obj.to_s.split("\n"))
|
184
189
|
|
185
190
|
message.level = :info if message.level == :warn &&
|
186
191
|
message.matches?(@config.ignores)
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
# require only the logger
|
4
|
+
require 'formatador'
|
5
|
+
require 'fog/core/logger'
|
6
|
+
|
7
|
+
module Backup
|
8
|
+
class Logger
|
9
|
+
module FogAdapter
|
10
|
+
class << self
|
11
|
+
|
12
|
+
# Logged as :info so these won't generate warnings.
|
13
|
+
# This is mostly to keep STDOUT clean and to provide
|
14
|
+
# supplemental messages for our own warnings.
|
15
|
+
# These will generally occur during retry attempts.
|
16
|
+
def write(message)
|
17
|
+
Logger.info message.split("\n").
|
18
|
+
map {|line| "[fog] #{ line }" }.join("\n")
|
19
|
+
end
|
20
|
+
|
21
|
+
def tty?
|
22
|
+
false
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
Fog::Logger[:warning] = Backup::Logger::FogAdapter
|
data/lib/backup/model.rb
CHANGED
@@ -2,6 +2,9 @@
|
|
2
2
|
|
3
3
|
module Backup
|
4
4
|
class Model
|
5
|
+
class Error < Backup::Error; end
|
6
|
+
class FatalError < Backup::FatalError; end
|
7
|
+
|
5
8
|
class << self
|
6
9
|
##
|
7
10
|
# The Backup::Model.all class method keeps track of all the models
|
@@ -73,6 +76,14 @@ module Backup
|
|
73
76
|
# The time when the backup initiated (in format: 2011.02.20.03.29.59)
|
74
77
|
attr_reader :time
|
75
78
|
|
79
|
+
##
|
80
|
+
# The time when the backup initiated (as a Time object)
|
81
|
+
attr_reader :started_at
|
82
|
+
|
83
|
+
##
|
84
|
+
# The time when the backup finished (as a Time object)
|
85
|
+
attr_reader :finished_at
|
86
|
+
|
76
87
|
##
|
77
88
|
# Result of this model's backup process.
|
78
89
|
#
|
@@ -136,7 +147,7 @@ module Backup
|
|
136
147
|
# Warn user of DSL changes
|
137
148
|
case name.to_s
|
138
149
|
when 'Backup::Config::RSync'
|
139
|
-
Logger.warn
|
150
|
+
Logger.warn Error.new(<<-EOS)
|
140
151
|
Configuration Update Needed for Syncer::RSync
|
141
152
|
The RSync Syncer has been split into three separate modules:
|
142
153
|
RSync::Local, RSync::Push and RSync::Pull
|
@@ -146,7 +157,7 @@ module Backup
|
|
146
157
|
name = 'RSync::Push'
|
147
158
|
when /(Backup::Config::S3|Backup::Config::CloudFiles)/
|
148
159
|
syncer = $1.split('::')[2]
|
149
|
-
Logger.warn
|
160
|
+
Logger.warn Error.new(<<-EOS)
|
150
161
|
Configuration Update Needed for '#{ syncer }' Syncer.
|
151
162
|
This Syncer is now referenced as Cloud::#{ syncer }
|
152
163
|
i.e. 'sync_with #{ syncer }' is now 'sync_with Cloud::#{ syncer }'
|
@@ -184,7 +195,7 @@ module Backup
|
|
184
195
|
if chunk_size.is_a?(Integer)
|
185
196
|
@splitter = Splitter.new(self, chunk_size)
|
186
197
|
else
|
187
|
-
raise
|
198
|
+
raise Error, <<-EOS
|
188
199
|
Invalid Chunk Size for Splitter
|
189
200
|
Argument to #split_into_chunks_of() must be an Integer
|
190
201
|
EOS
|
@@ -250,8 +261,8 @@ module Backup
|
|
250
261
|
# those files *** will be removed *** before the next scheduled backup for
|
251
262
|
# the same trigger.
|
252
263
|
def perform!
|
253
|
-
@started_at = Time.now
|
254
|
-
@time = package.time =
|
264
|
+
@started_at = Time.now.utc
|
265
|
+
@time = package.time = started_at.strftime("%Y.%m.%d.%H.%M.%S")
|
255
266
|
|
256
267
|
log!(:started)
|
257
268
|
before_hook
|
@@ -267,10 +278,18 @@ module Backup
|
|
267
278
|
|
268
279
|
ensure
|
269
280
|
set_exit_status
|
281
|
+
@finished_at = Time.now.utc
|
270
282
|
log!(:finished)
|
271
283
|
after_hook
|
272
284
|
end
|
273
285
|
|
286
|
+
##
|
287
|
+
# The duration of the backup process (in format: HH:MM:SS)
|
288
|
+
def duration
|
289
|
+
return unless finished_at
|
290
|
+
elapsed_time(started_at, finished_at)
|
291
|
+
end
|
292
|
+
|
274
293
|
private
|
275
294
|
|
276
295
|
##
|
@@ -359,8 +378,7 @@ module Backup
|
|
359
378
|
|
360
379
|
rescue Exception => err
|
361
380
|
@before_hook_failed = true
|
362
|
-
ex = err.is_a?(StandardError) ?
|
363
|
-
Errors::Model::HookError : Errors::Model::HookFatalError
|
381
|
+
ex = err.is_a?(StandardError) ? Error : FatalError
|
364
382
|
raise ex.wrap(err, 'Before Hook Failed!')
|
365
383
|
end
|
366
384
|
|
@@ -379,7 +397,7 @@ module Backup
|
|
379
397
|
|
380
398
|
rescue Exception => err
|
381
399
|
fatal = !err.is_a?(StandardError)
|
382
|
-
ex = fatal ?
|
400
|
+
ex = fatal ? FatalError : Error
|
383
401
|
Logger.error ex.wrap(err, 'After Hook Failed!')
|
384
402
|
# upgrade exit_status if needed
|
385
403
|
(@exit_status = fatal ? 3 : 2) unless exit_status == 3
|
@@ -398,7 +416,7 @@ module Backup
|
|
398
416
|
|
399
417
|
when :finished
|
400
418
|
if exit_status > 1
|
401
|
-
ex = exit_status == 2 ?
|
419
|
+
ex = exit_status == 2 ? Error : FatalError
|
402
420
|
err = ex.wrap(exception, "Backup for #{ label } (#{ trigger }) Failed!")
|
403
421
|
Logger.error err
|
404
422
|
Logger.error "\nBacktrace:\n\s\s" + err.backtrace.join("\n\s\s") + "\n\n"
|
@@ -407,10 +425,10 @@ module Backup
|
|
407
425
|
else
|
408
426
|
msg = "Backup for '#{ label } (#{ trigger })' "
|
409
427
|
if exit_status == 1
|
410
|
-
msg << "Completed Successfully (with Warnings) in #{
|
428
|
+
msg << "Completed Successfully (with Warnings) in #{ duration }"
|
411
429
|
Logger.warn msg
|
412
430
|
else
|
413
|
-
msg << "Completed Successfully in #{
|
431
|
+
msg << "Completed Successfully in #{ duration }"
|
414
432
|
Logger.info msg
|
415
433
|
end
|
416
434
|
end
|
@@ -418,9 +436,9 @@ module Backup
|
|
418
436
|
end
|
419
437
|
|
420
438
|
##
|
421
|
-
# Returns a string representing the elapsed time
|
422
|
-
def elapsed_time
|
423
|
-
duration =
|
439
|
+
# Returns a string representing the elapsed time in HH:MM:SS.
|
440
|
+
def elapsed_time(start_time, finish_time)
|
441
|
+
duration = finish_time.to_i - start_time.to_i
|
424
442
|
hours = duration / 3600
|
425
443
|
remainder = duration - (hours * 3600)
|
426
444
|
minutes = remainder / 60
|
data/lib/backup/notifier/base.rb
CHANGED
@@ -2,6 +2,8 @@
|
|
2
2
|
|
3
3
|
module Backup
|
4
4
|
module Notifier
|
5
|
+
class Error < Backup::Error; end
|
6
|
+
|
5
7
|
class Base
|
6
8
|
include Backup::Utilities::Helpers
|
7
9
|
include Backup::Configuration::Helpers
|
@@ -66,7 +68,7 @@ module Backup
|
|
66
68
|
end
|
67
69
|
|
68
70
|
rescue Exception => err
|
69
|
-
Logger.error
|
71
|
+
Logger.error Error.wrap(err, "#{ notifier_name } Failed!")
|
70
72
|
end
|
71
73
|
|
72
74
|
private
|
@@ -79,8 +81,7 @@ module Backup
|
|
79
81
|
retries += 1
|
80
82
|
raise if retries > max_retries
|
81
83
|
|
82
|
-
Logger.info
|
83
|
-
wrap(err, "Retry ##{ retries } of #{ max_retries }.")
|
84
|
+
Logger.info Error.wrap(err, "Retry ##{ retries } of #{ max_retries }.")
|
84
85
|
sleep(retry_waitsec)
|
85
86
|
retry
|
86
87
|
end
|