backup 4.4.1 → 5.0.0.beta.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (81) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +19 -0
  3. data/README.md +13 -9
  4. data/bin/docker_test +24 -0
  5. data/lib/backup/archive.rb +31 -32
  6. data/lib/backup/binder.rb +2 -6
  7. data/lib/backup/cleaner.rb +14 -18
  8. data/lib/backup/cli.rb +104 -108
  9. data/lib/backup/cloud_io/base.rb +4 -7
  10. data/lib/backup/cloud_io/cloud_files.rb +60 -62
  11. data/lib/backup/cloud_io/s3.rb +69 -76
  12. data/lib/backup/compressor/base.rb +4 -7
  13. data/lib/backup/compressor/bzip2.rb +3 -7
  14. data/lib/backup/compressor/custom.rb +2 -6
  15. data/lib/backup/compressor/gzip.rb +16 -17
  16. data/lib/backup/config/dsl.rb +16 -17
  17. data/lib/backup/config/helpers.rb +10 -16
  18. data/lib/backup/config.rb +17 -18
  19. data/lib/backup/database/base.rb +22 -21
  20. data/lib/backup/database/mongodb.rb +36 -37
  21. data/lib/backup/database/mysql.rb +40 -41
  22. data/lib/backup/database/openldap.rb +8 -10
  23. data/lib/backup/database/postgresql.rb +29 -30
  24. data/lib/backup/database/redis.rb +27 -30
  25. data/lib/backup/database/riak.rb +15 -18
  26. data/lib/backup/database/sqlite.rb +4 -6
  27. data/lib/backup/encryptor/base.rb +2 -4
  28. data/lib/backup/encryptor/gpg.rb +49 -59
  29. data/lib/backup/encryptor/open_ssl.rb +11 -14
  30. data/lib/backup/errors.rb +7 -12
  31. data/lib/backup/logger/console.rb +5 -8
  32. data/lib/backup/logger/fog_adapter.rb +2 -6
  33. data/lib/backup/logger/logfile.rb +10 -12
  34. data/lib/backup/logger/syslog.rb +2 -4
  35. data/lib/backup/logger.rb +16 -18
  36. data/lib/backup/model.rb +33 -40
  37. data/lib/backup/notifier/base.rb +24 -26
  38. data/lib/backup/notifier/campfire.rb +9 -11
  39. data/lib/backup/notifier/command.rb +0 -3
  40. data/lib/backup/notifier/datadog.rb +9 -12
  41. data/lib/backup/notifier/flowdock.rb +13 -17
  42. data/lib/backup/notifier/hipchat.rb +11 -13
  43. data/lib/backup/notifier/http_post.rb +11 -14
  44. data/lib/backup/notifier/mail.rb +42 -59
  45. data/lib/backup/notifier/nagios.rb +5 -9
  46. data/lib/backup/notifier/pagerduty.rb +10 -12
  47. data/lib/backup/notifier/prowl.rb +15 -15
  48. data/lib/backup/notifier/pushover.rb +7 -10
  49. data/lib/backup/notifier/ses.rb +34 -16
  50. data/lib/backup/notifier/slack.rb +39 -40
  51. data/lib/backup/notifier/twitter.rb +2 -5
  52. data/lib/backup/notifier/zabbix.rb +11 -14
  53. data/lib/backup/package.rb +5 -9
  54. data/lib/backup/packager.rb +16 -17
  55. data/lib/backup/pipeline.rb +17 -21
  56. data/lib/backup/splitter.rb +8 -11
  57. data/lib/backup/storage/base.rb +5 -8
  58. data/lib/backup/storage/cloud_files.rb +21 -23
  59. data/lib/backup/storage/cycler.rb +10 -15
  60. data/lib/backup/storage/dropbox.rb +15 -21
  61. data/lib/backup/storage/ftp.rb +14 -10
  62. data/lib/backup/storage/local.rb +5 -8
  63. data/lib/backup/storage/qiniu.rb +8 -8
  64. data/lib/backup/storage/rsync.rb +24 -26
  65. data/lib/backup/storage/s3.rb +27 -28
  66. data/lib/backup/storage/scp.rb +10 -12
  67. data/lib/backup/storage/sftp.rb +10 -12
  68. data/lib/backup/syncer/base.rb +5 -8
  69. data/lib/backup/syncer/cloud/base.rb +27 -30
  70. data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
  71. data/lib/backup/syncer/cloud/local_file.rb +5 -8
  72. data/lib/backup/syncer/cloud/s3.rb +23 -24
  73. data/lib/backup/syncer/rsync/base.rb +6 -10
  74. data/lib/backup/syncer/rsync/local.rb +1 -5
  75. data/lib/backup/syncer/rsync/pull.rb +6 -10
  76. data/lib/backup/syncer/rsync/push.rb +18 -22
  77. data/lib/backup/template.rb +9 -14
  78. data/lib/backup/utilities.rb +78 -69
  79. data/lib/backup/version.rb +1 -3
  80. data/lib/backup.rb +74 -78
  81. metadata +107 -676
data/lib/backup/cli.rb CHANGED
@@ -1,5 +1,3 @@
1
- # encoding: utf-8
2
-
3
1
  ##
4
2
  # Build the Backup Command Line Interface using Thor
5
3
  module Backup
@@ -27,9 +25,9 @@ module Backup
27
25
  #
28
26
  # If the --check option is given, `backup check` will be run
29
27
  # and no triggers will be performed.
30
- desc 'perform', "Performs the backup for the specified trigger(s)."
28
+ desc "perform", "Performs the backup for the specified trigger(s)."
31
29
 
32
- long_desc <<-EOS.gsub(/^ +/, '')
30
+ long_desc <<-EOS.gsub(/^ +/, "")
33
31
  Performs the backup for the specified trigger(s).
34
32
 
35
33
  You may perform multiple backups by providing multiple triggers,
@@ -51,65 +49,65 @@ module Backup
51
49
  EOS
52
50
 
53
51
  method_option :trigger,
54
- :aliases => ['-t', '--triggers'],
55
- :required => true,
56
- :type => :string,
57
- :desc => "Triggers to perform. e.g. 'trigger_a,trigger_b'"
52
+ aliases: ["-t", "--triggers"],
53
+ required: true,
54
+ type: :string,
55
+ desc: "Triggers to perform. e.g. 'trigger_a,trigger_b'"
58
56
 
59
57
  method_option :config_file,
60
- :aliases => '-c',
61
- :type => :string,
62
- :default => '',
63
- :desc => 'Path to your config.rb file.'
58
+ aliases: "-c",
59
+ type: :string,
60
+ default: "",
61
+ desc: "Path to your config.rb file."
64
62
 
65
63
  method_option :root_path,
66
- :aliases => '-r',
67
- :type => :string,
68
- :default => '',
69
- :desc => 'Root path to base all relative path on.'
64
+ aliases: "-r",
65
+ type: :string,
66
+ default: "",
67
+ desc: "Root path to base all relative path on."
70
68
 
71
69
  method_option :data_path,
72
- :aliases => '-d',
73
- :type => :string,
74
- :default => '',
75
- :desc => 'Path to store storage cycling data.'
70
+ aliases: "-d",
71
+ type: :string,
72
+ default: "",
73
+ desc: "Path to store storage cycling data."
76
74
 
77
75
  method_option :log_path,
78
- :aliases => '-l',
79
- :type => :string,
80
- :default => '',
81
- :desc => "Path to store Backup's log file."
76
+ aliases: "-l",
77
+ type: :string,
78
+ default: "",
79
+ desc: "Path to store Backup's log file."
82
80
 
83
81
  method_option :tmp_path,
84
- :type => :string,
85
- :default => '',
86
- :desc => 'Path to store temporary data during the backup.'
82
+ type: :string,
83
+ default: "",
84
+ desc: "Path to store temporary data during the backup."
87
85
 
88
86
  # Note that :quiet, :syslog and :logfile are specified as :string types,
89
87
  # so the --no-<option> usage will set the value to nil instead of false.
90
88
  method_option :quiet,
91
- :aliases => '-q',
92
- :type => :string,
93
- :default => false,
94
- :banner => '',
95
- :desc => 'Disable console log output.'
89
+ aliases: "-q",
90
+ type: :boolean,
91
+ default: false,
92
+ banner: "",
93
+ desc: "Disable console log output."
96
94
 
97
95
  method_option :syslog,
98
- :type => :string,
99
- :default => false,
100
- :banner => '',
101
- :desc => 'Enable logging to syslog.'
96
+ type: :boolean,
97
+ default: false,
98
+ banner: "",
99
+ desc: "Enable logging to syslog."
102
100
 
103
101
  method_option :logfile,
104
- :type => :string,
105
- :default => true,
106
- :banner => '',
107
- :desc => "Enable Backup's log file."
102
+ type: :boolean,
103
+ default: true,
104
+ banner: "",
105
+ desc: "Enable Backup's log file."
108
106
 
109
107
  method_option :check,
110
- :type => :boolean,
111
- :default => false,
112
- :desc => 'Check configuration for errors or warnings.'
108
+ type: :boolean,
109
+ default: false,
110
+ desc: "Check configuration for errors or warnings."
113
111
 
114
112
  def perform
115
113
  check if options[:check] # this will exit()
@@ -119,27 +117,28 @@ module Backup
119
117
  # Set logger options
120
118
  opts = options
121
119
  Logger.configure do
122
- console.quiet = opts[:quiet]
123
- logfile.enabled = opts[:logfile]
124
- logfile.log_path = opts[:log_path]
125
- syslog.enabled = opts[:syslog]
120
+ console.quiet = opts[:quiet]
121
+ logfile.enabled = opts[:logfile]
122
+ logfile.log_path = opts[:log_path]
123
+ syslog.enabled = opts[:syslog]
126
124
  end
127
125
 
128
126
  # Load the user's +config.rb+ file and all their Models
129
127
  Config.load(options)
130
128
 
131
129
  # Identify all Models to be run for the given +triggers+.
132
- triggers = options[:trigger].split(',').map(&:strip)
133
- models = triggers.map {|trigger|
130
+ triggers = options[:trigger].split(",").map(&:strip)
131
+ models = triggers.uniq.flat_map do |trigger|
134
132
  Model.find_by_trigger(trigger)
135
- }.flatten.uniq
133
+ end
136
134
 
137
- raise Error, "No Models found for trigger(s) " +
138
- "'#{ triggers.join(',') }'." if models.empty?
135
+ if models.empty?
136
+ raise Error, "No Models found for trigger(s) " \
137
+ "'#{triggers.join(",")}'."
138
+ end
139
139
 
140
140
  # Finalize Logger and begin real-time logging.
141
141
  Logger.start!
142
-
143
142
  rescue Exception => err
144
143
  Logger.error Error.wrap(err)
145
144
  unless Helpers.is_backup_error? err
@@ -148,7 +147,7 @@ module Backup
148
147
  # Logger configuration will be ignored
149
148
  # and messages will be output to the console only.
150
149
  Logger.abort!
151
- exit(3)
150
+ exit 3
152
151
  end
153
152
 
154
153
  until models.empty?
@@ -164,7 +163,7 @@ module Backup
164
163
  Logger.info Error.new(<<-EOS)
165
164
  Backup will now continue...
166
165
  The following triggers will now be processed:
167
- (#{ models.map {|m| m.trigger }.join(', ') })
166
+ (#{models.map(&:trigger).join(", ")})
168
167
  EOS
169
168
  end
170
169
  when 3
@@ -173,7 +172,7 @@ module Backup
173
172
  Logger.error FatalError.new(<<-EOS)
174
173
  Backup will now exit.
175
174
  The following triggers will not be processed:
176
- (#{ models.map {|m| m.trigger }.join(', ') })
175
+ (#{models.map(&:trigger).join(", ")})
177
176
  EOS
178
177
  end
179
178
  end
@@ -203,19 +202,19 @@ module Backup
203
202
  #
204
203
  # If successful, this method with exit(0).
205
204
  # If there are Errors or Warnings, it will exit(1).
206
- desc 'check', 'Check for configuration errors or warnings'
205
+ desc "check", "Check for configuration errors or warnings"
207
206
 
208
- long_desc <<-EOS.gsub(/^ +/, '')
207
+ long_desc <<-EOS.gsub(/^ +/, "")
209
208
  Loads your 'config.rb' file and all models and reports any
210
209
  errors or warnings with your configuration, including missing
211
210
  dependencies and the use of any deprecated settings.
212
211
  EOS
213
212
 
214
213
  method_option :config_file,
215
- :aliases => '-c',
216
- :type => :string,
217
- :default => '',
218
- :desc => "Path to your config.rb file."
214
+ aliases: "-c",
215
+ type: :string,
216
+ default: "",
217
+ desc: "Path to your config.rb file."
219
218
 
220
219
  def check
221
220
  begin
@@ -228,10 +227,10 @@ module Backup
228
227
  end
229
228
 
230
229
  if Logger.has_warnings? || Logger.has_errors?
231
- Logger.error 'Configuration Check Failed.'
230
+ Logger.error "Configuration Check Failed."
232
231
  exit_code = 1
233
232
  else
234
- Logger.info 'Configuration Check Succeeded.'
233
+ Logger.info "Configuration Check Succeeded."
235
234
  exit_code = 0
236
235
  end
237
236
 
@@ -245,130 +244,127 @@ module Backup
245
244
  # For example:
246
245
  # $ backup generate:model --trigger my_backup --databases='mongodb'
247
246
  # will generate a pre-populated model with a base MongoDB setup
248
- desc 'generate:model', "Generates a Backup model file."
247
+ desc "generate:model", "Generates a Backup model file."
249
248
 
250
- long_desc <<-EOS.gsub(/^ +/, '')
249
+ long_desc <<-EOS.gsub(/^ +/, "")
251
250
  Generates a Backup model file.
252
251
 
253
252
  If your configuration file is not in the default location at
254
- #{ Config.config_file }
253
+ #{Config.config_file}
255
254
  you must specify it's location using '--config-file'.
256
255
  If no configuration file exists at this location, one will be created.
257
256
 
258
257
  The model file will be created as '<config_path>/models/<trigger>.rb'
259
258
  Your model file will be created in a 'models/' sub-directory
260
259
  where your config file is located. The default location would be:
261
- #{ Config.root_path }/models/<trigger>.rb
260
+ #{Config.root_path}/models/<trigger>.rb
262
261
  EOS
263
262
 
264
263
  method_option :trigger,
265
- :aliases => '-t',
266
- :required => true,
267
- :type => :string,
268
- :desc => 'Trigger name for the Backup model'
264
+ aliases: "-t",
265
+ required: true,
266
+ type: :string,
267
+ desc: "Trigger name for the Backup model"
269
268
 
270
269
  method_option :config_file,
271
- :type => :string,
272
- :desc => 'Path to your Backup configuration file'
270
+ type: :string,
271
+ desc: "Path to your Backup configuration file"
273
272
 
274
273
  # options with their available values
275
- %w{ databases storages syncers encryptor compressor notifiers }.each do |name|
276
- path = File.join(Backup::TEMPLATE_PATH, 'cli', name)
277
- opts = Dir[path + '/*'].sort.map {|p| File.basename(p) }.join(', ')
278
- method_option name, :type => :string, :desc => "(#{ opts })"
274
+ %w[databases storages syncers encryptor compressor notifiers].each do |name|
275
+ path = File.join(Backup::TEMPLATE_PATH, "cli", name)
276
+ opts = Dir[path + "/*"].sort.map { |p| File.basename(p) }.join(", ")
277
+ method_option name, type: :string, desc: "(#{opts})"
279
278
  end
280
279
 
281
280
  method_option :archives,
282
- :type => :boolean,
283
- :desc => 'Model will include tar archives.'
281
+ type: :boolean,
282
+ desc: "Model will include tar archives."
284
283
 
285
284
  method_option :splitter,
286
- :type => :boolean,
287
- :default => false,
288
- :desc => 'Add Splitter to the model'
285
+ type: :boolean,
286
+ default: false,
287
+ desc: "Add Splitter to the model"
289
288
 
290
- define_method 'generate:model' do
291
- opts = options.merge(:trigger => options[:trigger].gsub(/\W/, '_'))
289
+ define_method "generate:model" do
290
+ opts = options.merge(trigger: options[:trigger].gsub(/\W/, "_"))
292
291
  config_file = opts[:config_file] ?
293
292
  File.expand_path(opts.delete(:config_file)) : Config.config_file
294
- models_path = File.join(File.dirname(config_file), 'models')
295
- model_file = File.join(models_path, "#{ opts[:trigger] }.rb")
293
+ models_path = File.join(File.dirname(config_file), "models")
294
+ model_file = File.join(models_path, "#{opts[:trigger]}.rb")
296
295
 
297
296
  unless File.exist?(config_file)
298
- invoke 'generate:config', [], :config_file => config_file
297
+ invoke "generate:config", [], config_file: config_file
299
298
  end
300
299
 
301
300
  FileUtils.mkdir_p(models_path)
302
301
  if Helpers.overwrite?(model_file)
303
- File.open(model_file, 'w') do |file|
304
- file.write(Backup::Template.new({:options => opts}).result('cli/model'))
302
+ File.open(model_file, "w") do |file|
303
+ file.write(Backup::Template.new(options: opts).result("cli/model"))
305
304
  end
306
- puts "Generated model file: '#{ model_file }'."
305
+ puts "Generated model file: '#{model_file}'."
307
306
  end
308
307
  end
309
308
 
310
309
  ##
311
310
  # [Generate:Config]
312
311
  # Generates the main configuration file
313
- desc 'generate:config', 'Generates the main Backup configuration file'
312
+ desc "generate:config", "Generates the main Backup configuration file"
314
313
 
315
- long_desc <<-EOS.gsub(/^ +/, '')
314
+ long_desc <<-EOS.gsub(/^ +/, "")
316
315
  Path to the Backup configuration file to generate.
317
316
 
318
317
  Defaults to:
319
318
 
320
- #{ Config.config_file }
319
+ #{Config.config_file}
321
320
  EOS
322
321
 
323
322
  method_option :config_file,
324
- :type => :string,
325
- :desc => 'Path to the Backup configuration file to generate.'
323
+ type: :string,
324
+ desc: "Path to the Backup configuration file to generate."
326
325
 
327
- define_method 'generate:config' do
326
+ define_method "generate:config" do
328
327
  config_file = options[:config_file] ?
329
328
  File.expand_path(options[:config_file]) : Config.config_file
330
329
 
331
330
  FileUtils.mkdir_p(File.dirname(config_file))
332
331
  if Helpers.overwrite?(config_file)
333
- File.open(config_file, 'w') do |file|
334
- file.write(Backup::Template.new.result('cli/config'))
332
+ File.open(config_file, "w") do |file|
333
+ file.write(Backup::Template.new.result("cli/config"))
335
334
  end
336
- puts "Generated configuration file: '#{ config_file }'."
335
+ puts "Generated configuration file: '#{config_file}'."
337
336
  end
338
337
  end
339
338
 
340
339
  ##
341
340
  # [Version]
342
341
  # Returns the current version of the Backup gem
343
- map '-v' => :version
344
- desc 'version', 'Display installed Backup version'
342
+ map "-v" => :version
343
+ desc "version", "Display installed Backup version"
345
344
  def version
346
- puts "Backup #{ Backup::VERSION }"
345
+ puts "Backup #{Backup::VERSION}"
347
346
  end
348
347
 
349
348
  # This is to avoid Thor's warnings when stubbing methods on the Thor class.
350
349
  module Helpers
351
350
  class << self
352
-
353
351
  def overwrite?(path)
354
352
  return true unless File.exist?(path)
355
353
 
356
- $stderr.print "A file already exists at '#{ path }'.\n" +
354
+ $stderr.print "A file already exists at '#{path}'.\n" \
357
355
  "Do you want to overwrite? [y/n] "
358
356
  /^[Yy]/ =~ $stdin.gets
359
357
  end
360
358
 
361
359
  def exec!(cmd)
362
- puts "Launching: #{ cmd }"
360
+ puts "Launching: #{cmd}"
363
361
  exec(cmd)
364
362
  end
365
363
 
366
364
  def is_backup_error?(error)
367
365
  error.class.ancestors.include? Backup::Error
368
366
  end
369
-
370
367
  end
371
368
  end
372
-
373
369
  end
374
370
  end
@@ -1,5 +1,3 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module CloudIO
5
3
  class Error < Backup::Error; end
@@ -22,20 +20,19 @@ module Backup
22
20
  rescue => err
23
21
  retries += 1
24
22
  raise Error.wrap(err, <<-EOS) if retries > max_retries
25
- Max Retries (#{ max_retries }) Exceeded!
26
- Operation: #{ operation }
23
+ Max Retries (#{max_retries}) Exceeded!
24
+ Operation: #{operation}
27
25
  Be sure to check the log messages for each retry attempt.
28
26
  EOS
29
27
 
30
28
  Logger.info Error.wrap(err, <<-EOS)
31
- Retry ##{ retries } of #{ max_retries }
32
- Operation: #{ operation }
29
+ Retry ##{retries} of #{max_retries}
30
+ Operation: #{operation}
33
31
  EOS
34
32
  sleep(retry_waitsec)
35
33
  retry
36
34
  end
37
35
  end
38
-
39
36
  end
40
37
  end
41
38
  end