backup 3.6.0 → 3.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -0
  3. data/lib/backup.rb +14 -4
  4. data/lib/backup/archive.rb +3 -2
  5. data/lib/backup/cleaner.rb +4 -2
  6. data/lib/backup/cli.rb +7 -5
  7. data/lib/backup/cloud_io/base.rb +41 -0
  8. data/lib/backup/cloud_io/cloud_files.rb +296 -0
  9. data/lib/backup/cloud_io/s3.rb +252 -0
  10. data/lib/backup/compressor/gzip.rb +2 -1
  11. data/lib/backup/config.rb +13 -5
  12. data/lib/backup/configuration.rb +1 -1
  13. data/lib/backup/configuration/helpers.rb +3 -1
  14. data/lib/backup/database/base.rb +3 -1
  15. data/lib/backup/database/mongodb.rb +2 -2
  16. data/lib/backup/database/mysql.rb +2 -2
  17. data/lib/backup/database/postgresql.rb +12 -2
  18. data/lib/backup/database/redis.rb +3 -2
  19. data/lib/backup/encryptor/gpg.rb +8 -10
  20. data/lib/backup/errors.rb +39 -70
  21. data/lib/backup/logger.rb +7 -2
  22. data/lib/backup/logger/fog_adapter.rb +30 -0
  23. data/lib/backup/model.rb +32 -14
  24. data/lib/backup/notifier/base.rb +4 -3
  25. data/lib/backup/notifier/campfire.rb +0 -1
  26. data/lib/backup/notifier/http_post.rb +122 -0
  27. data/lib/backup/notifier/mail.rb +38 -0
  28. data/lib/backup/notifier/nagios.rb +69 -0
  29. data/lib/backup/notifier/prowl.rb +0 -1
  30. data/lib/backup/notifier/pushover.rb +0 -1
  31. data/lib/backup/package.rb +5 -0
  32. data/lib/backup/packager.rb +3 -2
  33. data/lib/backup/pipeline.rb +4 -2
  34. data/lib/backup/storage/base.rb +2 -1
  35. data/lib/backup/storage/cloud_files.rb +151 -0
  36. data/lib/backup/storage/cycler.rb +4 -2
  37. data/lib/backup/storage/dropbox.rb +20 -16
  38. data/lib/backup/storage/ftp.rb +1 -2
  39. data/lib/backup/storage/local.rb +3 -3
  40. data/lib/backup/storage/ninefold.rb +3 -4
  41. data/lib/backup/storage/rsync.rb +1 -2
  42. data/lib/backup/storage/s3.rb +49 -158
  43. data/lib/backup/storage/scp.rb +3 -4
  44. data/lib/backup/storage/sftp.rb +1 -2
  45. data/lib/backup/syncer/base.rb +0 -1
  46. data/lib/backup/syncer/cloud/base.rb +129 -208
  47. data/lib/backup/syncer/cloud/cloud_files.rb +56 -41
  48. data/lib/backup/syncer/cloud/local_file.rb +93 -0
  49. data/lib/backup/syncer/cloud/s3.rb +78 -31
  50. data/lib/backup/syncer/rsync/base.rb +7 -0
  51. data/lib/backup/syncer/rsync/local.rb +0 -5
  52. data/lib/backup/syncer/rsync/push.rb +1 -2
  53. data/lib/backup/utilities.rb +18 -15
  54. data/lib/backup/version.rb +1 -1
  55. data/templates/cli/notifier/http_post +35 -0
  56. data/templates/cli/notifier/nagios +13 -0
  57. data/templates/cli/storage/cloud_files +8 -17
  58. data/templates/cli/storage/s3 +3 -10
  59. data/templates/cli/syncer/cloud_files +3 -31
  60. data/templates/cli/syncer/s3 +3 -27
  61. data/templates/notifier/mail/failure.erb +6 -1
  62. data/templates/notifier/mail/success.erb +6 -1
  63. data/templates/notifier/mail/warning.erb +6 -1
  64. metadata +37 -42
  65. data/lib/backup/storage/cloudfiles.rb +0 -68
@@ -3,6 +3,7 @@
3
3
  module Backup
4
4
  module Database
5
5
  class Redis < Base
6
+ class Error < Backup::Error; end
6
7
 
7
8
  ##
8
9
  # Name of the redis dump file.
@@ -65,7 +66,7 @@ module Backup
65
66
  def invoke_save!
66
67
  resp = run(redis_save_cmd)
67
68
  unless resp =~ /OK$/
68
- raise Errors::Database::Redis::CommandError, <<-EOS
69
+ raise Error, <<-EOS
69
70
  Could not invoke the Redis SAVE command.
70
71
  Command was: #{ redis_save_cmd }
71
72
  Response was: #{ resp }
@@ -76,7 +77,7 @@ module Backup
76
77
  def copy!
77
78
  src_path = File.join(path, name + '.rdb')
78
79
  unless File.exist?(src_path)
79
- raise Errors::Database::Redis::NotFoundError, <<-EOS
80
+ raise Error, <<-EOS
80
81
  Redis database dump not found
81
82
  File path was #{ src_path }
82
83
  EOS
@@ -69,6 +69,8 @@ module Backup
69
69
  # end
70
70
  #
71
71
  class GPG < Base
72
+ class Error < Backup::Error; end
73
+
72
74
  MODES = [:asymmetric, :symmetric, :both]
73
75
 
74
76
  ##
@@ -102,8 +104,7 @@ module Backup
102
104
  attr_reader :mode
103
105
  def mode=(mode)
104
106
  @mode = mode.to_sym
105
- raise Errors::Encryptor::GPG::InvalidModeError,
106
- "'#{ @mode }' is not a valid mode." unless MODES.include?(@mode)
107
+ raise Error, "'#{ @mode }' is not a valid mode." unless MODES.include?(@mode)
107
108
  end
108
109
 
109
110
  ##
@@ -421,8 +422,7 @@ module Backup
421
422
  prepare
422
423
 
423
424
  if mode_options.empty?
424
- raise Errors::Encryptor::GPG::EncryptionError,
425
- "Encryption could not be performed for mode '#{ mode }'"
425
+ raise Error, "Encryption could not be performed for mode '#{ mode }'"
426
426
  end
427
427
 
428
428
  yield "#{ utility(:gpg) } #{ base_options } #{ mode_options }", '.gpg'
@@ -491,7 +491,7 @@ module Backup
491
491
  path
492
492
 
493
493
  rescue => err
494
- raise Errors::Encryptor::GPG::HomedirError.wrap(
494
+ raise Error.wrap(
495
495
  err, "Failed to create or set permissions for #gpg_homedir")
496
496
  end
497
497
 
@@ -524,8 +524,7 @@ module Backup
524
524
 
525
525
  rescue => err
526
526
  cleanup
527
- raise Errors::Encryptor::GPG::GPGConfigError.wrap(
528
- err, "Error creating temporary file for #gpg_config.")
527
+ raise Error.wrap(err, "Error creating temporary file for #gpg_config.")
529
528
  end
530
529
 
531
530
  ##
@@ -591,8 +590,7 @@ module Backup
591
590
  file.path
592
591
 
593
592
  rescue => err
594
- Logger.warn Errors::Encryptor::GPG::PassphraseError.wrap(
595
- err, "Error creating temporary passphrase file.")
593
+ Logger.warn Error.wrap(err, "Error creating temporary passphrase file.")
596
594
  false
597
595
  end
598
596
 
@@ -691,7 +689,7 @@ module Backup
691
689
  keyid
692
690
 
693
691
  rescue => err
694
- Logger.warn Errors::Encryptor::GPG::KeyImportError.wrap(
692
+ Logger.warn Error.wrap(
695
693
  err, "Public key import failed for '#{ identifier }'")
696
694
  nil
697
695
  end
@@ -1,89 +1,58 @@
1
1
  # encoding: utf-8
2
2
 
3
3
  module Backup
4
- ##
5
- # - automatically defines module namespaces referenced under Backup::Errors
6
- # - any constant name referenced that ends with 'Error' will be created
7
- # as a subclass of Backup::Errors::Error
8
- # - any constant name referenced that ends with 'FatalError' will be created
9
- # as a subclass of Backup::Errors::FatalError
10
- #
11
- # e.g.
12
- # err = Backup::Errors::Foo::Bar::FooError.new('error message')
13
- # err.message => "Foo::Bar::FooError: error message"
14
- #
15
- # err = Backup::Errors::Foo::Bar::FooFatalError.new('error message')
16
- # err.message => "Foo::Bar::FooFatalError: error message"
17
- #
18
- module ErrorsHelper
19
- def const_missing(const)
20
- if const.to_s.end_with?('FatalError')
21
- module_eval("class #{const} < Backup::Errors::FatalError; end")
22
- elsif const.to_s.end_with?('Error')
23
- module_eval("class #{const} < Backup::Errors::Error; end")
24
- else
25
- module_eval("module #{const}; extend Backup::ErrorsHelper; end")
26
- end
27
- const_get(const)
28
- end
29
- end
30
-
31
- module Errors
32
- extend ErrorsHelper
33
4
 
34
- # Provides cascading errors with formatted messages.
35
- # See the specs for details.
36
- module NestedExceptions
5
+ # Provides cascading errors with formatted messages.
6
+ # See the specs for details.
7
+ module NestedExceptions
37
8
 
38
- def self.included(klass)
39
- klass.extend Module.new {
40
- def wrap(wrapped_exception, msg = nil)
41
- new(msg, wrapped_exception)
42
- end
43
- }
44
- end
45
-
46
- def initialize(obj = nil, wrapped_exception = nil)
47
- @wrapped_exception = wrapped_exception
48
- msg = (obj.respond_to?(:to_str) ? obj.to_str : obj.to_s).
49
- gsub(/^ */, ' ').strip
50
- msg = clean_name(self.class.name) + (msg.empty? ? '' : ": #{ msg }")
51
-
52
- if wrapped_exception
53
- msg << "\n--- Wrapped Exception ---\n"
54
- class_name = clean_name(wrapped_exception.class.name)
55
- msg << class_name + ': ' unless
56
- wrapped_exception.message.start_with? class_name
57
- msg << wrapped_exception.message
9
+ def self.included(klass)
10
+ klass.extend Module.new {
11
+ def wrap(wrapped_exception, msg = nil)
12
+ new(msg, wrapped_exception)
58
13
  end
14
+ }
15
+ end
59
16
 
60
- super(msg)
61
- set_backtrace(wrapped_exception.backtrace) if wrapped_exception
17
+ def initialize(obj = nil, wrapped_exception = nil)
18
+ @wrapped_exception = wrapped_exception
19
+ msg = (obj.respond_to?(:to_str) ? obj.to_str : obj.to_s).
20
+ gsub(/^ */, ' ').strip
21
+ msg = clean_name(self.class.name) + (msg.empty? ? '' : ": #{ msg }")
22
+
23
+ if wrapped_exception
24
+ msg << "\n--- Wrapped Exception ---\n"
25
+ class_name = clean_name(wrapped_exception.class.name)
26
+ msg << class_name + ': ' unless
27
+ wrapped_exception.message.start_with? class_name
28
+ msg << wrapped_exception.message
62
29
  end
63
30
 
64
- def exception(obj = nil)
65
- return self if obj.nil? || equal?(obj)
31
+ super(msg)
32
+ set_backtrace(wrapped_exception.backtrace) if wrapped_exception
33
+ end
66
34
 
67
- ex = self.class.new(obj, @wrapped_exception)
68
- ex.set_backtrace(backtrace) unless ex.backtrace
69
- ex
70
- end
35
+ def exception(obj = nil)
36
+ return self if obj.nil? || equal?(obj)
71
37
 
72
- private
38
+ ex = self.class.new(obj, @wrapped_exception)
39
+ ex.set_backtrace(backtrace) unless ex.backtrace
40
+ ex
41
+ end
73
42
 
74
- def clean_name(name)
75
- name.sub(/^Backup::Errors::/, '')
76
- end
43
+ private
77
44
 
45
+ def clean_name(name)
46
+ name.sub(/^Backup::/, '')
78
47
  end
79
48
 
80
- class Error < StandardError
81
- include NestedExceptions
82
- end
49
+ end
83
50
 
84
- class FatalError < Exception
85
- include NestedExceptions
86
- end
51
+ class Error < StandardError
52
+ include NestedExceptions
53
+ end
87
54
 
55
+ class FatalError < Exception
56
+ include NestedExceptions
88
57
  end
89
58
  end
@@ -3,6 +3,7 @@
3
3
  require 'backup/logger/console'
4
4
  require 'backup/logger/logfile'
5
5
  require 'backup/logger/syslog'
6
+ require 'backup/logger/fog_adapter'
6
7
 
7
8
  module Backup
8
9
  class Logger
@@ -118,6 +119,8 @@ module Backup
118
119
  end
119
120
  end
120
121
 
122
+ MUTEX = Mutex.new
123
+
121
124
  ##
122
125
  # Returns an Array of Message objects for all logged messages received.
123
126
  # These are used to attach log files to Mail notifications.
@@ -134,7 +137,9 @@ module Backup
134
137
  # Sends a message to the Logger using the specified log level.
135
138
  # +obj+ may be any Object that responds to #to_s (i.e. an Exception)
136
139
  [:info, :warn, :error].each do |level|
137
- define_method level, lambda {|obj| log(obj, level) }
140
+ define_method level, lambda {|obj|
141
+ MUTEX.synchronize { log(obj, level) }
142
+ }
138
143
  end
139
144
 
140
145
  ##
@@ -180,7 +185,7 @@ module Backup
180
185
  private
181
186
 
182
187
  def log(obj, level)
183
- message = Message.new(Time.now, level, obj.to_s.split("\n"))
188
+ message = Message.new(Time.now.utc, level, obj.to_s.split("\n"))
184
189
 
185
190
  message.level = :info if message.level == :warn &&
186
191
  message.matches?(@config.ignores)
@@ -0,0 +1,30 @@
1
+ # encoding: utf-8
2
+
3
+ # require only the logger
4
+ require 'formatador'
5
+ require 'fog/core/logger'
6
+
7
+ module Backup
8
+ class Logger
9
+ module FogAdapter
10
+ class << self
11
+
12
+ # Logged as :info so these won't generate warnings.
13
+ # This is mostly to keep STDOUT clean and to provide
14
+ # supplemental messages for our own warnings.
15
+ # These will generally occur during retry attempts.
16
+ def write(message)
17
+ Logger.info message.split("\n").
18
+ map {|line| "[fog] #{ line }" }.join("\n")
19
+ end
20
+
21
+ def tty?
22
+ false
23
+ end
24
+
25
+ end
26
+ end
27
+ end
28
+ end
29
+
30
+ Fog::Logger[:warning] = Backup::Logger::FogAdapter
@@ -2,6 +2,9 @@
2
2
 
3
3
  module Backup
4
4
  class Model
5
+ class Error < Backup::Error; end
6
+ class FatalError < Backup::FatalError; end
7
+
5
8
  class << self
6
9
  ##
7
10
  # The Backup::Model.all class method keeps track of all the models
@@ -73,6 +76,14 @@ module Backup
73
76
  # The time when the backup initiated (in format: 2011.02.20.03.29.59)
74
77
  attr_reader :time
75
78
 
79
+ ##
80
+ # The time when the backup initiated (as a Time object)
81
+ attr_reader :started_at
82
+
83
+ ##
84
+ # The time when the backup finished (as a Time object)
85
+ attr_reader :finished_at
86
+
76
87
  ##
77
88
  # Result of this model's backup process.
78
89
  #
@@ -136,7 +147,7 @@ module Backup
136
147
  # Warn user of DSL changes
137
148
  case name.to_s
138
149
  when 'Backup::Config::RSync'
139
- Logger.warn Errors::ConfigError.new(<<-EOS)
150
+ Logger.warn Error.new(<<-EOS)
140
151
  Configuration Update Needed for Syncer::RSync
141
152
  The RSync Syncer has been split into three separate modules:
142
153
  RSync::Local, RSync::Push and RSync::Pull
@@ -146,7 +157,7 @@ module Backup
146
157
  name = 'RSync::Push'
147
158
  when /(Backup::Config::S3|Backup::Config::CloudFiles)/
148
159
  syncer = $1.split('::')[2]
149
- Logger.warn Errors::ConfigError.new(<<-EOS)
160
+ Logger.warn Error.new(<<-EOS)
150
161
  Configuration Update Needed for '#{ syncer }' Syncer.
151
162
  This Syncer is now referenced as Cloud::#{ syncer }
152
163
  i.e. 'sync_with #{ syncer }' is now 'sync_with Cloud::#{ syncer }'
@@ -184,7 +195,7 @@ module Backup
184
195
  if chunk_size.is_a?(Integer)
185
196
  @splitter = Splitter.new(self, chunk_size)
186
197
  else
187
- raise Errors::Model::ConfigurationError, <<-EOS
198
+ raise Error, <<-EOS
188
199
  Invalid Chunk Size for Splitter
189
200
  Argument to #split_into_chunks_of() must be an Integer
190
201
  EOS
@@ -250,8 +261,8 @@ module Backup
250
261
  # those files *** will be removed *** before the next scheduled backup for
251
262
  # the same trigger.
252
263
  def perform!
253
- @started_at = Time.now
254
- @time = package.time = @started_at.strftime("%Y.%m.%d.%H.%M.%S")
264
+ @started_at = Time.now.utc
265
+ @time = package.time = started_at.strftime("%Y.%m.%d.%H.%M.%S")
255
266
 
256
267
  log!(:started)
257
268
  before_hook
@@ -267,10 +278,18 @@ module Backup
267
278
 
268
279
  ensure
269
280
  set_exit_status
281
+ @finished_at = Time.now.utc
270
282
  log!(:finished)
271
283
  after_hook
272
284
  end
273
285
 
286
+ ##
287
+ # The duration of the backup process (in format: HH:MM:SS)
288
+ def duration
289
+ return unless finished_at
290
+ elapsed_time(started_at, finished_at)
291
+ end
292
+
274
293
  private
275
294
 
276
295
  ##
@@ -359,8 +378,7 @@ module Backup
359
378
 
360
379
  rescue Exception => err
361
380
  @before_hook_failed = true
362
- ex = err.is_a?(StandardError) ?
363
- Errors::Model::HookError : Errors::Model::HookFatalError
381
+ ex = err.is_a?(StandardError) ? Error : FatalError
364
382
  raise ex.wrap(err, 'Before Hook Failed!')
365
383
  end
366
384
 
@@ -379,7 +397,7 @@ module Backup
379
397
 
380
398
  rescue Exception => err
381
399
  fatal = !err.is_a?(StandardError)
382
- ex = fatal ? Errors::Model::HookFatalError : Errors::Model::HookError
400
+ ex = fatal ? FatalError : Error
383
401
  Logger.error ex.wrap(err, 'After Hook Failed!')
384
402
  # upgrade exit_status if needed
385
403
  (@exit_status = fatal ? 3 : 2) unless exit_status == 3
@@ -398,7 +416,7 @@ module Backup
398
416
 
399
417
  when :finished
400
418
  if exit_status > 1
401
- ex = exit_status == 2 ? Errors::ModelError : Errors::ModelFatalError
419
+ ex = exit_status == 2 ? Error : FatalError
402
420
  err = ex.wrap(exception, "Backup for #{ label } (#{ trigger }) Failed!")
403
421
  Logger.error err
404
422
  Logger.error "\nBacktrace:\n\s\s" + err.backtrace.join("\n\s\s") + "\n\n"
@@ -407,10 +425,10 @@ module Backup
407
425
  else
408
426
  msg = "Backup for '#{ label } (#{ trigger })' "
409
427
  if exit_status == 1
410
- msg << "Completed Successfully (with Warnings) in #{ elapsed_time }"
428
+ msg << "Completed Successfully (with Warnings) in #{ duration }"
411
429
  Logger.warn msg
412
430
  else
413
- msg << "Completed Successfully in #{ elapsed_time }"
431
+ msg << "Completed Successfully in #{ duration }"
414
432
  Logger.info msg
415
433
  end
416
434
  end
@@ -418,9 +436,9 @@ module Backup
418
436
  end
419
437
 
420
438
  ##
421
- # Returns a string representing the elapsed time since the backup started.
422
- def elapsed_time
423
- duration = Time.now.to_i - @started_at.to_i
439
+ # Returns a string representing the elapsed time in HH:MM:SS.
440
+ def elapsed_time(start_time, finish_time)
441
+ duration = finish_time.to_i - start_time.to_i
424
442
  hours = duration / 3600
425
443
  remainder = duration - (hours * 3600)
426
444
  minutes = remainder / 60
@@ -2,6 +2,8 @@
2
2
 
3
3
  module Backup
4
4
  module Notifier
5
+ class Error < Backup::Error; end
6
+
5
7
  class Base
6
8
  include Backup::Utilities::Helpers
7
9
  include Backup::Configuration::Helpers
@@ -66,7 +68,7 @@ module Backup
66
68
  end
67
69
 
68
70
  rescue Exception => err
69
- Logger.error Errors::NotifierError.wrap(err, "#{ notifier_name } Failed!")
71
+ Logger.error Error.wrap(err, "#{ notifier_name } Failed!")
70
72
  end
71
73
 
72
74
  private
@@ -79,8 +81,7 @@ module Backup
79
81
  retries += 1
80
82
  raise if retries > max_retries
81
83
 
82
- Logger.info Errors::NotifierError.
83
- wrap(err, "Retry ##{ retries } of #{ max_retries }.")
84
+ Logger.info Error.wrap(err, "Retry ##{ retries } of #{ max_retries }.")
84
85
  sleep(retry_waitsec)
85
86
  retry
86
87
  end
@@ -1,5 +1,4 @@
1
1
  # encoding: utf-8
2
- require 'excon'
3
2
  require 'json'
4
3
 
5
4
  module Backup