backup 3.6.0 → 3.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -0
  3. data/lib/backup.rb +14 -4
  4. data/lib/backup/archive.rb +3 -2
  5. data/lib/backup/cleaner.rb +4 -2
  6. data/lib/backup/cli.rb +7 -5
  7. data/lib/backup/cloud_io/base.rb +41 -0
  8. data/lib/backup/cloud_io/cloud_files.rb +296 -0
  9. data/lib/backup/cloud_io/s3.rb +252 -0
  10. data/lib/backup/compressor/gzip.rb +2 -1
  11. data/lib/backup/config.rb +13 -5
  12. data/lib/backup/configuration.rb +1 -1
  13. data/lib/backup/configuration/helpers.rb +3 -1
  14. data/lib/backup/database/base.rb +3 -1
  15. data/lib/backup/database/mongodb.rb +2 -2
  16. data/lib/backup/database/mysql.rb +2 -2
  17. data/lib/backup/database/postgresql.rb +12 -2
  18. data/lib/backup/database/redis.rb +3 -2
  19. data/lib/backup/encryptor/gpg.rb +8 -10
  20. data/lib/backup/errors.rb +39 -70
  21. data/lib/backup/logger.rb +7 -2
  22. data/lib/backup/logger/fog_adapter.rb +30 -0
  23. data/lib/backup/model.rb +32 -14
  24. data/lib/backup/notifier/base.rb +4 -3
  25. data/lib/backup/notifier/campfire.rb +0 -1
  26. data/lib/backup/notifier/http_post.rb +122 -0
  27. data/lib/backup/notifier/mail.rb +38 -0
  28. data/lib/backup/notifier/nagios.rb +69 -0
  29. data/lib/backup/notifier/prowl.rb +0 -1
  30. data/lib/backup/notifier/pushover.rb +0 -1
  31. data/lib/backup/package.rb +5 -0
  32. data/lib/backup/packager.rb +3 -2
  33. data/lib/backup/pipeline.rb +4 -2
  34. data/lib/backup/storage/base.rb +2 -1
  35. data/lib/backup/storage/cloud_files.rb +151 -0
  36. data/lib/backup/storage/cycler.rb +4 -2
  37. data/lib/backup/storage/dropbox.rb +20 -16
  38. data/lib/backup/storage/ftp.rb +1 -2
  39. data/lib/backup/storage/local.rb +3 -3
  40. data/lib/backup/storage/ninefold.rb +3 -4
  41. data/lib/backup/storage/rsync.rb +1 -2
  42. data/lib/backup/storage/s3.rb +49 -158
  43. data/lib/backup/storage/scp.rb +3 -4
  44. data/lib/backup/storage/sftp.rb +1 -2
  45. data/lib/backup/syncer/base.rb +0 -1
  46. data/lib/backup/syncer/cloud/base.rb +129 -208
  47. data/lib/backup/syncer/cloud/cloud_files.rb +56 -41
  48. data/lib/backup/syncer/cloud/local_file.rb +93 -0
  49. data/lib/backup/syncer/cloud/s3.rb +78 -31
  50. data/lib/backup/syncer/rsync/base.rb +7 -0
  51. data/lib/backup/syncer/rsync/local.rb +0 -5
  52. data/lib/backup/syncer/rsync/push.rb +1 -2
  53. data/lib/backup/utilities.rb +18 -15
  54. data/lib/backup/version.rb +1 -1
  55. data/templates/cli/notifier/http_post +35 -0
  56. data/templates/cli/notifier/nagios +13 -0
  57. data/templates/cli/storage/cloud_files +8 -17
  58. data/templates/cli/storage/s3 +3 -10
  59. data/templates/cli/syncer/cloud_files +3 -31
  60. data/templates/cli/syncer/s3 +3 -27
  61. data/templates/notifier/mail/failure.erb +6 -1
  62. data/templates/notifier/mail/success.erb +6 -1
  63. data/templates/notifier/mail/warning.erb +6 -1
  64. metadata +37 -42
  65. data/lib/backup/storage/cloudfiles.rb +0 -68
@@ -3,6 +3,7 @@
3
3
  module Backup
4
4
  module Database
5
5
  class Redis < Base
6
+ class Error < Backup::Error; end
6
7
 
7
8
  ##
8
9
  # Name of the redis dump file.
@@ -65,7 +66,7 @@ module Backup
65
66
  def invoke_save!
66
67
  resp = run(redis_save_cmd)
67
68
  unless resp =~ /OK$/
68
- raise Errors::Database::Redis::CommandError, <<-EOS
69
+ raise Error, <<-EOS
69
70
  Could not invoke the Redis SAVE command.
70
71
  Command was: #{ redis_save_cmd }
71
72
  Response was: #{ resp }
@@ -76,7 +77,7 @@ module Backup
76
77
  def copy!
77
78
  src_path = File.join(path, name + '.rdb')
78
79
  unless File.exist?(src_path)
79
- raise Errors::Database::Redis::NotFoundError, <<-EOS
80
+ raise Error, <<-EOS
80
81
  Redis database dump not found
81
82
  File path was #{ src_path }
82
83
  EOS
@@ -69,6 +69,8 @@ module Backup
69
69
  # end
70
70
  #
71
71
  class GPG < Base
72
+ class Error < Backup::Error; end
73
+
72
74
  MODES = [:asymmetric, :symmetric, :both]
73
75
 
74
76
  ##
@@ -102,8 +104,7 @@ module Backup
102
104
  attr_reader :mode
103
105
  def mode=(mode)
104
106
  @mode = mode.to_sym
105
- raise Errors::Encryptor::GPG::InvalidModeError,
106
- "'#{ @mode }' is not a valid mode." unless MODES.include?(@mode)
107
+ raise Error, "'#{ @mode }' is not a valid mode." unless MODES.include?(@mode)
107
108
  end
108
109
 
109
110
  ##
@@ -421,8 +422,7 @@ module Backup
421
422
  prepare
422
423
 
423
424
  if mode_options.empty?
424
- raise Errors::Encryptor::GPG::EncryptionError,
425
- "Encryption could not be performed for mode '#{ mode }'"
425
+ raise Error, "Encryption could not be performed for mode '#{ mode }'"
426
426
  end
427
427
 
428
428
  yield "#{ utility(:gpg) } #{ base_options } #{ mode_options }", '.gpg'
@@ -491,7 +491,7 @@ module Backup
491
491
  path
492
492
 
493
493
  rescue => err
494
- raise Errors::Encryptor::GPG::HomedirError.wrap(
494
+ raise Error.wrap(
495
495
  err, "Failed to create or set permissions for #gpg_homedir")
496
496
  end
497
497
 
@@ -524,8 +524,7 @@ module Backup
524
524
 
525
525
  rescue => err
526
526
  cleanup
527
- raise Errors::Encryptor::GPG::GPGConfigError.wrap(
528
- err, "Error creating temporary file for #gpg_config.")
527
+ raise Error.wrap(err, "Error creating temporary file for #gpg_config.")
529
528
  end
530
529
 
531
530
  ##
@@ -591,8 +590,7 @@ module Backup
591
590
  file.path
592
591
 
593
592
  rescue => err
594
- Logger.warn Errors::Encryptor::GPG::PassphraseError.wrap(
595
- err, "Error creating temporary passphrase file.")
593
+ Logger.warn Error.wrap(err, "Error creating temporary passphrase file.")
596
594
  false
597
595
  end
598
596
 
@@ -691,7 +689,7 @@ module Backup
691
689
  keyid
692
690
 
693
691
  rescue => err
694
- Logger.warn Errors::Encryptor::GPG::KeyImportError.wrap(
692
+ Logger.warn Error.wrap(
695
693
  err, "Public key import failed for '#{ identifier }'")
696
694
  nil
697
695
  end
@@ -1,89 +1,58 @@
1
1
  # encoding: utf-8
2
2
 
3
3
  module Backup
4
- ##
5
- # - automatically defines module namespaces referenced under Backup::Errors
6
- # - any constant name referenced that ends with 'Error' will be created
7
- # as a subclass of Backup::Errors::Error
8
- # - any constant name referenced that ends with 'FatalError' will be created
9
- # as a subclass of Backup::Errors::FatalError
10
- #
11
- # e.g.
12
- # err = Backup::Errors::Foo::Bar::FooError.new('error message')
13
- # err.message => "Foo::Bar::FooError: error message"
14
- #
15
- # err = Backup::Errors::Foo::Bar::FooFatalError.new('error message')
16
- # err.message => "Foo::Bar::FooFatalError: error message"
17
- #
18
- module ErrorsHelper
19
- def const_missing(const)
20
- if const.to_s.end_with?('FatalError')
21
- module_eval("class #{const} < Backup::Errors::FatalError; end")
22
- elsif const.to_s.end_with?('Error')
23
- module_eval("class #{const} < Backup::Errors::Error; end")
24
- else
25
- module_eval("module #{const}; extend Backup::ErrorsHelper; end")
26
- end
27
- const_get(const)
28
- end
29
- end
30
-
31
- module Errors
32
- extend ErrorsHelper
33
4
 
34
- # Provides cascading errors with formatted messages.
35
- # See the specs for details.
36
- module NestedExceptions
5
+ # Provides cascading errors with formatted messages.
6
+ # See the specs for details.
7
+ module NestedExceptions
37
8
 
38
- def self.included(klass)
39
- klass.extend Module.new {
40
- def wrap(wrapped_exception, msg = nil)
41
- new(msg, wrapped_exception)
42
- end
43
- }
44
- end
45
-
46
- def initialize(obj = nil, wrapped_exception = nil)
47
- @wrapped_exception = wrapped_exception
48
- msg = (obj.respond_to?(:to_str) ? obj.to_str : obj.to_s).
49
- gsub(/^ */, ' ').strip
50
- msg = clean_name(self.class.name) + (msg.empty? ? '' : ": #{ msg }")
51
-
52
- if wrapped_exception
53
- msg << "\n--- Wrapped Exception ---\n"
54
- class_name = clean_name(wrapped_exception.class.name)
55
- msg << class_name + ': ' unless
56
- wrapped_exception.message.start_with? class_name
57
- msg << wrapped_exception.message
9
+ def self.included(klass)
10
+ klass.extend Module.new {
11
+ def wrap(wrapped_exception, msg = nil)
12
+ new(msg, wrapped_exception)
58
13
  end
14
+ }
15
+ end
59
16
 
60
- super(msg)
61
- set_backtrace(wrapped_exception.backtrace) if wrapped_exception
17
+ def initialize(obj = nil, wrapped_exception = nil)
18
+ @wrapped_exception = wrapped_exception
19
+ msg = (obj.respond_to?(:to_str) ? obj.to_str : obj.to_s).
20
+ gsub(/^ */, ' ').strip
21
+ msg = clean_name(self.class.name) + (msg.empty? ? '' : ": #{ msg }")
22
+
23
+ if wrapped_exception
24
+ msg << "\n--- Wrapped Exception ---\n"
25
+ class_name = clean_name(wrapped_exception.class.name)
26
+ msg << class_name + ': ' unless
27
+ wrapped_exception.message.start_with? class_name
28
+ msg << wrapped_exception.message
62
29
  end
63
30
 
64
- def exception(obj = nil)
65
- return self if obj.nil? || equal?(obj)
31
+ super(msg)
32
+ set_backtrace(wrapped_exception.backtrace) if wrapped_exception
33
+ end
66
34
 
67
- ex = self.class.new(obj, @wrapped_exception)
68
- ex.set_backtrace(backtrace) unless ex.backtrace
69
- ex
70
- end
35
+ def exception(obj = nil)
36
+ return self if obj.nil? || equal?(obj)
71
37
 
72
- private
38
+ ex = self.class.new(obj, @wrapped_exception)
39
+ ex.set_backtrace(backtrace) unless ex.backtrace
40
+ ex
41
+ end
73
42
 
74
- def clean_name(name)
75
- name.sub(/^Backup::Errors::/, '')
76
- end
43
+ private
77
44
 
45
+ def clean_name(name)
46
+ name.sub(/^Backup::/, '')
78
47
  end
79
48
 
80
- class Error < StandardError
81
- include NestedExceptions
82
- end
49
+ end
83
50
 
84
- class FatalError < Exception
85
- include NestedExceptions
86
- end
51
+ class Error < StandardError
52
+ include NestedExceptions
53
+ end
87
54
 
55
+ class FatalError < Exception
56
+ include NestedExceptions
88
57
  end
89
58
  end
@@ -3,6 +3,7 @@
3
3
  require 'backup/logger/console'
4
4
  require 'backup/logger/logfile'
5
5
  require 'backup/logger/syslog'
6
+ require 'backup/logger/fog_adapter'
6
7
 
7
8
  module Backup
8
9
  class Logger
@@ -118,6 +119,8 @@ module Backup
118
119
  end
119
120
  end
120
121
 
122
+ MUTEX = Mutex.new
123
+
121
124
  ##
122
125
  # Returns an Array of Message objects for all logged messages received.
123
126
  # These are used to attach log files to Mail notifications.
@@ -134,7 +137,9 @@ module Backup
134
137
  # Sends a message to the Logger using the specified log level.
135
138
  # +obj+ may be any Object that responds to #to_s (i.e. an Exception)
136
139
  [:info, :warn, :error].each do |level|
137
- define_method level, lambda {|obj| log(obj, level) }
140
+ define_method level, lambda {|obj|
141
+ MUTEX.synchronize { log(obj, level) }
142
+ }
138
143
  end
139
144
 
140
145
  ##
@@ -180,7 +185,7 @@ module Backup
180
185
  private
181
186
 
182
187
  def log(obj, level)
183
- message = Message.new(Time.now, level, obj.to_s.split("\n"))
188
+ message = Message.new(Time.now.utc, level, obj.to_s.split("\n"))
184
189
 
185
190
  message.level = :info if message.level == :warn &&
186
191
  message.matches?(@config.ignores)
@@ -0,0 +1,30 @@
1
+ # encoding: utf-8
2
+
3
+ # require only the logger
4
+ require 'formatador'
5
+ require 'fog/core/logger'
6
+
7
+ module Backup
8
+ class Logger
9
+ module FogAdapter
10
+ class << self
11
+
12
+ # Logged as :info so these won't generate warnings.
13
+ # This is mostly to keep STDOUT clean and to provide
14
+ # supplemental messages for our own warnings.
15
+ # These will generally occur during retry attempts.
16
+ def write(message)
17
+ Logger.info message.split("\n").
18
+ map {|line| "[fog] #{ line }" }.join("\n")
19
+ end
20
+
21
+ def tty?
22
+ false
23
+ end
24
+
25
+ end
26
+ end
27
+ end
28
+ end
29
+
30
+ Fog::Logger[:warning] = Backup::Logger::FogAdapter
@@ -2,6 +2,9 @@
2
2
 
3
3
  module Backup
4
4
  class Model
5
+ class Error < Backup::Error; end
6
+ class FatalError < Backup::FatalError; end
7
+
5
8
  class << self
6
9
  ##
7
10
  # The Backup::Model.all class method keeps track of all the models
@@ -73,6 +76,14 @@ module Backup
73
76
  # The time when the backup initiated (in format: 2011.02.20.03.29.59)
74
77
  attr_reader :time
75
78
 
79
+ ##
80
+ # The time when the backup initiated (as a Time object)
81
+ attr_reader :started_at
82
+
83
+ ##
84
+ # The time when the backup finished (as a Time object)
85
+ attr_reader :finished_at
86
+
76
87
  ##
77
88
  # Result of this model's backup process.
78
89
  #
@@ -136,7 +147,7 @@ module Backup
136
147
  # Warn user of DSL changes
137
148
  case name.to_s
138
149
  when 'Backup::Config::RSync'
139
- Logger.warn Errors::ConfigError.new(<<-EOS)
150
+ Logger.warn Error.new(<<-EOS)
140
151
  Configuration Update Needed for Syncer::RSync
141
152
  The RSync Syncer has been split into three separate modules:
142
153
  RSync::Local, RSync::Push and RSync::Pull
@@ -146,7 +157,7 @@ module Backup
146
157
  name = 'RSync::Push'
147
158
  when /(Backup::Config::S3|Backup::Config::CloudFiles)/
148
159
  syncer = $1.split('::')[2]
149
- Logger.warn Errors::ConfigError.new(<<-EOS)
160
+ Logger.warn Error.new(<<-EOS)
150
161
  Configuration Update Needed for '#{ syncer }' Syncer.
151
162
  This Syncer is now referenced as Cloud::#{ syncer }
152
163
  i.e. 'sync_with #{ syncer }' is now 'sync_with Cloud::#{ syncer }'
@@ -184,7 +195,7 @@ module Backup
184
195
  if chunk_size.is_a?(Integer)
185
196
  @splitter = Splitter.new(self, chunk_size)
186
197
  else
187
- raise Errors::Model::ConfigurationError, <<-EOS
198
+ raise Error, <<-EOS
188
199
  Invalid Chunk Size for Splitter
189
200
  Argument to #split_into_chunks_of() must be an Integer
190
201
  EOS
@@ -250,8 +261,8 @@ module Backup
250
261
  # those files *** will be removed *** before the next scheduled backup for
251
262
  # the same trigger.
252
263
  def perform!
253
- @started_at = Time.now
254
- @time = package.time = @started_at.strftime("%Y.%m.%d.%H.%M.%S")
264
+ @started_at = Time.now.utc
265
+ @time = package.time = started_at.strftime("%Y.%m.%d.%H.%M.%S")
255
266
 
256
267
  log!(:started)
257
268
  before_hook
@@ -267,10 +278,18 @@ module Backup
267
278
 
268
279
  ensure
269
280
  set_exit_status
281
+ @finished_at = Time.now.utc
270
282
  log!(:finished)
271
283
  after_hook
272
284
  end
273
285
 
286
+ ##
287
+ # The duration of the backup process (in format: HH:MM:SS)
288
+ def duration
289
+ return unless finished_at
290
+ elapsed_time(started_at, finished_at)
291
+ end
292
+
274
293
  private
275
294
 
276
295
  ##
@@ -359,8 +378,7 @@ module Backup
359
378
 
360
379
  rescue Exception => err
361
380
  @before_hook_failed = true
362
- ex = err.is_a?(StandardError) ?
363
- Errors::Model::HookError : Errors::Model::HookFatalError
381
+ ex = err.is_a?(StandardError) ? Error : FatalError
364
382
  raise ex.wrap(err, 'Before Hook Failed!')
365
383
  end
366
384
 
@@ -379,7 +397,7 @@ module Backup
379
397
 
380
398
  rescue Exception => err
381
399
  fatal = !err.is_a?(StandardError)
382
- ex = fatal ? Errors::Model::HookFatalError : Errors::Model::HookError
400
+ ex = fatal ? FatalError : Error
383
401
  Logger.error ex.wrap(err, 'After Hook Failed!')
384
402
  # upgrade exit_status if needed
385
403
  (@exit_status = fatal ? 3 : 2) unless exit_status == 3
@@ -398,7 +416,7 @@ module Backup
398
416
 
399
417
  when :finished
400
418
  if exit_status > 1
401
- ex = exit_status == 2 ? Errors::ModelError : Errors::ModelFatalError
419
+ ex = exit_status == 2 ? Error : FatalError
402
420
  err = ex.wrap(exception, "Backup for #{ label } (#{ trigger }) Failed!")
403
421
  Logger.error err
404
422
  Logger.error "\nBacktrace:\n\s\s" + err.backtrace.join("\n\s\s") + "\n\n"
@@ -407,10 +425,10 @@ module Backup
407
425
  else
408
426
  msg = "Backup for '#{ label } (#{ trigger })' "
409
427
  if exit_status == 1
410
- msg << "Completed Successfully (with Warnings) in #{ elapsed_time }"
428
+ msg << "Completed Successfully (with Warnings) in #{ duration }"
411
429
  Logger.warn msg
412
430
  else
413
- msg << "Completed Successfully in #{ elapsed_time }"
431
+ msg << "Completed Successfully in #{ duration }"
414
432
  Logger.info msg
415
433
  end
416
434
  end
@@ -418,9 +436,9 @@ module Backup
418
436
  end
419
437
 
420
438
  ##
421
- # Returns a string representing the elapsed time since the backup started.
422
- def elapsed_time
423
- duration = Time.now.to_i - @started_at.to_i
439
+ # Returns a string representing the elapsed time in HH:MM:SS.
440
+ def elapsed_time(start_time, finish_time)
441
+ duration = finish_time.to_i - start_time.to_i
424
442
  hours = duration / 3600
425
443
  remainder = duration - (hours * 3600)
426
444
  minutes = remainder / 60
@@ -2,6 +2,8 @@
2
2
 
3
3
  module Backup
4
4
  module Notifier
5
+ class Error < Backup::Error; end
6
+
5
7
  class Base
6
8
  include Backup::Utilities::Helpers
7
9
  include Backup::Configuration::Helpers
@@ -66,7 +68,7 @@ module Backup
66
68
  end
67
69
 
68
70
  rescue Exception => err
69
- Logger.error Errors::NotifierError.wrap(err, "#{ notifier_name } Failed!")
71
+ Logger.error Error.wrap(err, "#{ notifier_name } Failed!")
70
72
  end
71
73
 
72
74
  private
@@ -79,8 +81,7 @@ module Backup
79
81
  retries += 1
80
82
  raise if retries > max_retries
81
83
 
82
- Logger.info Errors::NotifierError.
83
- wrap(err, "Retry ##{ retries } of #{ max_retries }.")
84
+ Logger.info Error.wrap(err, "Retry ##{ retries } of #{ max_retries }.")
84
85
  sleep(retry_waitsec)
85
86
  retry
86
87
  end
@@ -1,5 +1,4 @@
1
1
  # encoding: utf-8
2
- require 'excon'
3
2
  require 'json'
4
3
 
5
4
  module Backup