backup 3.11.0 → 4.0.0rc1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +4 -4
  3. data/lib/backup.rb +1 -4
  4. data/lib/backup/archive.rb +1 -1
  5. data/lib/backup/cli.rb +51 -107
  6. data/lib/backup/compressor/base.rb +2 -2
  7. data/lib/backup/compressor/bzip2.rb +0 -11
  8. data/lib/backup/compressor/gzip.rb +0 -11
  9. data/lib/backup/config.rb +45 -123
  10. data/lib/backup/config/dsl.rb +102 -0
  11. data/lib/backup/{configuration → config}/helpers.rb +23 -14
  12. data/lib/backup/database/base.rb +2 -2
  13. data/lib/backup/database/mongodb.rb +0 -18
  14. data/lib/backup/database/mysql.rb +6 -75
  15. data/lib/backup/database/postgresql.rb +0 -12
  16. data/lib/backup/database/redis.rb +85 -47
  17. data/lib/backup/database/riak.rb +0 -19
  18. data/lib/backup/encryptor/base.rb +2 -2
  19. data/lib/backup/encryptor/gpg.rb +1 -12
  20. data/lib/backup/logger/fog_adapter.rb +1 -2
  21. data/lib/backup/model.rb +3 -24
  22. data/lib/backup/notifier/base.rb +2 -17
  23. data/lib/backup/notifier/http_post.rb +1 -1
  24. data/lib/backup/notifier/mail.rb +5 -47
  25. data/lib/backup/notifier/prowl.rb +1 -1
  26. data/lib/backup/notifier/pushover.rb +1 -1
  27. data/lib/backup/packager.rb +1 -1
  28. data/lib/backup/pipeline.rb +1 -1
  29. data/lib/backup/splitter.rb +1 -1
  30. data/lib/backup/storage/base.rb +2 -14
  31. data/lib/backup/storage/cloud_files.rb +1 -0
  32. data/lib/backup/storage/cycler.rb +33 -88
  33. data/lib/backup/storage/dropbox.rb +19 -12
  34. data/lib/backup/storage/ftp.rb +1 -0
  35. data/lib/backup/storage/local.rb +1 -0
  36. data/lib/backup/storage/ninefold.rb +1 -0
  37. data/lib/backup/storage/rsync.rb +7 -41
  38. data/lib/backup/storage/s3.rb +1 -0
  39. data/lib/backup/storage/scp.rb +1 -0
  40. data/lib/backup/storage/sftp.rb +1 -0
  41. data/lib/backup/syncer/base.rb +2 -2
  42. data/lib/backup/syncer/cloud/cloud_files.rb +0 -16
  43. data/lib/backup/syncer/cloud/s3.rb +0 -16
  44. data/lib/backup/syncer/rsync/local.rb +0 -5
  45. data/lib/backup/syncer/rsync/pull.rb +0 -21
  46. data/lib/backup/syncer/rsync/push.rb +0 -21
  47. data/lib/backup/utilities.rb +2 -22
  48. data/lib/backup/version.rb +1 -1
  49. data/templates/cli/archive +0 -3
  50. data/templates/cli/compressor/custom +0 -4
  51. data/templates/cli/config +39 -17
  52. data/templates/cli/{database → databases}/mongodb +0 -0
  53. data/templates/cli/{database → databases}/mysql +0 -0
  54. data/templates/cli/{database → databases}/postgresql +0 -0
  55. data/templates/cli/databases/redis +16 -0
  56. data/templates/cli/{database → databases}/riak +0 -0
  57. data/templates/cli/{model.erb → model} +8 -5
  58. data/templates/cli/{notifier → notifiers}/campfire +0 -0
  59. data/templates/cli/{notifier → notifiers}/hipchat +0 -0
  60. data/templates/cli/{notifier → notifiers}/http_post +0 -3
  61. data/templates/cli/{notifier → notifiers}/mail +1 -2
  62. data/templates/cli/notifiers/nagios +13 -0
  63. data/templates/cli/{notifier → notifiers}/prowl +0 -0
  64. data/templates/cli/{notifier → notifiers}/pushover +0 -0
  65. data/templates/cli/{notifier → notifiers}/twitter +0 -0
  66. data/templates/cli/{storage → storages}/cloud_files +0 -2
  67. data/templates/cli/storages/dropbox +19 -0
  68. data/templates/cli/{storage → storages}/ftp +0 -0
  69. data/templates/cli/{storage → storages}/local +0 -0
  70. data/templates/cli/{storage → storages}/ninefold +0 -0
  71. data/templates/cli/{storage → storages}/rsync +0 -2
  72. data/templates/cli/{storage → storages}/s3 +0 -2
  73. data/templates/cli/{storage → storages}/scp +0 -0
  74. data/templates/cli/{storage → storages}/sftp +0 -0
  75. data/templates/cli/{syncer → syncers}/cloud_files +0 -2
  76. data/templates/cli/{syncer → syncers}/rsync_local +0 -0
  77. data/templates/cli/{syncer → syncers}/rsync_pull +0 -2
  78. data/templates/cli/{syncer → syncers}/rsync_push +0 -2
  79. data/templates/cli/{syncer → syncers}/s3 +0 -2
  80. data/templates/general/links +1 -1
  81. metadata +241 -69
  82. data/lib/backup/compressor/lzma.rb +0 -52
  83. data/lib/backup/compressor/pbzip2.rb +0 -59
  84. data/lib/backup/configuration.rb +0 -33
  85. data/lib/backup/configuration/store.rb +0 -24
  86. data/templates/cli/compressor/lzma +0 -10
  87. data/templates/cli/compressor/pbzip2 +0 -10
  88. data/templates/cli/database/redis +0 -18
  89. data/templates/cli/notifier/nagios +0 -13
  90. data/templates/cli/storage/dropbox +0 -20
@@ -77,25 +77,6 @@ module Backup
77
77
  "#{ utility(:sudo) } -n -u #{ user } #{ utility('riak-admin') }"
78
78
  end
79
79
 
80
- attr_deprecate :utility_path, :version => '3.0.21',
81
- :message => 'Use Backup::Utilities.configure instead.',
82
- :action => lambda {|klass, val|
83
- Utilities.configure { riak_admin val }
84
- }
85
-
86
- attr_deprecate :riak_admin_utility, :version => '3.3.0',
87
- :message => 'Use Backup::Utilities.configure instead.',
88
- :action => lambda {|klass, val|
89
- Utilities.configure { riak_admin val }
90
- }
91
-
92
- attr_deprecate :name, :version => '3.3.0',
93
- :message => "If you wish to add an identifier to the dump filename,\n" +
94
- "use a +database_id+ when defining the database in your Model.\n" +
95
- "e.g. database Riak, :my_id do |db|"
96
-
97
- attr_deprecate :group, :version => '3.3.0'
98
-
99
80
  end
100
81
  end
101
82
  end
@@ -3,8 +3,8 @@
3
3
  module Backup
4
4
  module Encryptor
5
5
  class Base
6
- include Backup::Utilities::Helpers
7
- include Backup::Configuration::Helpers
6
+ include Utilities::Helpers
7
+ include Config::Helpers
8
8
 
9
9
  def initialize
10
10
  load_defaults!
@@ -217,17 +217,6 @@ module Backup
217
217
  # @return [Hash]
218
218
  attr_accessor :keys
219
219
 
220
- ##
221
- # @deprecated Use {#keys} and {#recipients}.
222
- # @!attribute key
223
- attr_deprecate :key,
224
- :version => '3.0.26',
225
- :message => "This has been replaced with #keys and #recipients",
226
- :action => lambda {|klass, val|
227
- identifier = klass.send(:import_key, 'deprecated :key', val)
228
- klass.recipients = identifier
229
- }
230
-
231
220
  ##
232
221
  # Specifies the recipients to use when encrypting the backup archive.
233
222
  #
@@ -370,7 +359,7 @@ module Backup
370
359
  # end
371
360
  #
372
361
  # @!scope class
373
- # @see Configuration::Helpers::ClassMethods#defaults
362
+ # @see Config::Helpers::ClassMethods#defaults
374
363
  # @yield [config] OpenStruct object
375
364
  # @!method defaults
376
365
 
@@ -14,8 +14,7 @@ module Backup
14
14
  # supplemental messages for our own warnings.
15
15
  # These will generally occur during retry attempts.
16
16
  def write(message)
17
- Logger.info message.split("\n").
18
- map {|line| "[fog] #{ line }" }.join("\n")
17
+ Logger.info message.chomp
19
18
  end
20
19
 
21
20
  def tty?
data/lib/backup/model.rb CHANGED
@@ -156,27 +156,6 @@ module Backup
156
156
  ##
157
157
  # Adds an Syncer. Multiple Syncers may be added to the model.
158
158
  def sync_with(name, syncer_id = nil, &block)
159
- ##
160
- # Warn user of DSL changes
161
- case name.to_s
162
- when 'Backup::Config::RSync'
163
- Logger.warn Error.new(<<-EOS)
164
- Configuration Update Needed for Syncer::RSync
165
- The RSync Syncer has been split into three separate modules:
166
- RSync::Local, RSync::Push and RSync::Pull
167
- Please update your configuration.
168
- i.e. 'sync_with RSync' is now 'sync_with RSync::Push'
169
- EOS
170
- name = 'RSync::Push'
171
- when /(Backup::Config::S3|Backup::Config::CloudFiles)/
172
- syncer = $1.split('::')[2]
173
- Logger.warn Error.new(<<-EOS)
174
- Configuration Update Needed for '#{ syncer }' Syncer.
175
- This Syncer is now referenced as Cloud::#{ syncer }
176
- i.e. 'sync_with #{ syncer }' is now 'sync_with Cloud::#{ syncer }'
177
- EOS
178
- name = "Cloud::#{ syncer }"
179
- end
180
159
  @syncers << get_class_from_scope(Syncer, name).new(syncer_id, &block)
181
160
  end
182
161
 
@@ -208,7 +187,7 @@ module Backup
208
187
  # +suffix_length+ controls the number of characters used in the suffix
209
188
  # (and the maximum number of chunks possible).
210
189
  # ie. 1 (-a, -b), 2 (-aa, -ab), 3 (-aaa, -aab)
211
- def split_into_chunks_of(chunk_size, suffix_length = 2)
190
+ def split_into_chunks_of(chunk_size, suffix_length = 3)
212
191
  if chunk_size.is_a?(Integer) && suffix_length.is_a?(Integer)
213
192
  @splitter = Splitter.new(self, chunk_size, suffix_length)
214
193
  else
@@ -358,7 +337,7 @@ module Backup
358
337
  # +name+ may be Class/Module or String representation
359
338
  # of any namespace which exists under +scope+.
360
339
  #
361
- # The 'Backup::Config::' namespace is stripped from +name+,
340
+ # The 'Backup::Config::DSL' namespace is stripped from +name+,
362
341
  # since this is the namespace where we define module namespaces
363
342
  # for use with Model's DSL methods.
364
343
  #
@@ -371,7 +350,7 @@ module Backup
371
350
  #
372
351
  def get_class_from_scope(scope, name)
373
352
  klass = scope
374
- name = name.to_s.sub(/^Backup::Config::/, '')
353
+ name = name.to_s.sub(/^Backup::Config::DSL::/, '')
375
354
  name.split('::').each do |chunk|
376
355
  klass = klass.const_get(chunk)
377
356
  end
@@ -5,8 +5,8 @@ module Backup
5
5
  class Error < Backup::Error; end
6
6
 
7
7
  class Base
8
- include Backup::Utilities::Helpers
9
- include Backup::Configuration::Helpers
8
+ include Utilities::Helpers
9
+ include Config::Helpers
10
10
 
11
11
  ##
12
12
  # When set to true, the user will be notified by email
@@ -93,21 +93,6 @@ module Backup
93
93
  self.class.to_s.sub('Backup::', '')
94
94
  end
95
95
 
96
- # For ruby-1.8.7. Both sorted so specs will match.
97
- def encode_www_form(enum)
98
- if RUBY_VERSION < '1.9'
99
- require 'cgi'
100
- str = ''
101
- enum.to_a.map {|k,v| [k.to_s, v] }.sort.each do |k,v|
102
- str << '&' unless str.empty?
103
- str << CGI.escape(k) << '=' << CGI.escape(v)
104
- end
105
- str
106
- else
107
- URI.encode_www_form(enum.sort)
108
- end
109
- end
110
-
111
96
  end
112
97
  end
113
98
  end
@@ -106,7 +106,7 @@ module Backup
106
106
  :headers => { 'User-Agent' => "Backup/#{ VERSION }" }.
107
107
  merge(headers).reject {|k,v| v.nil? }.
108
108
  merge('Content-Type' => 'application/x-www-form-urlencoded'),
109
- :body => encode_www_form({ 'message' => message }.
109
+ :body => URI.encode_www_form({ 'message' => message }.
110
110
  merge(params).reject {|k,v| v.nil? }.
111
111
  merge('status' => status.to_s)),
112
112
  :expects => success_codes # raise error if unsuccessful
@@ -66,14 +66,14 @@ module Backup
66
66
  ##
67
67
  # Set the method of encryption to be used for the +SMTP+ connection.
68
68
  #
69
- # [:none (default)]
70
- # No encryption will be used.
71
- #
72
- # [:starttls]
69
+ # [:starttls (default)]
73
70
  # Use +STARTTLS+ to upgrade the connection to a +SSL/TLS+ connection.
74
71
  #
75
72
  # [:tls or :ssl]
76
73
  # Use a +SSL/TLS+ connection.
74
+ #
75
+ # [:none]
76
+ # No encryption will be used.
77
77
  attr_accessor :encryption
78
78
 
79
79
  ##
@@ -123,6 +123,7 @@ module Backup
123
123
  instance_eval(&block) if block_given?
124
124
 
125
125
  @send_log_on ||= [:warning, :failure]
126
+ @encryption ||= :starttls
126
127
  end
127
128
 
128
129
  private
@@ -218,25 +219,6 @@ module Backup
218
219
  email
219
220
  end
220
221
 
221
- attr_deprecate :enable_starttls_auto, :version => '3.2.0',
222
- :message => "Use #encryption instead.\n" +
223
- 'e.g. mail.encryption = :starttls',
224
- :action => lambda {|klass, val|
225
- klass.encryption = val ? :starttls : :none
226
- }
227
-
228
- attr_deprecate :sendmail, :version => '3.6.0',
229
- :message => 'Use Backup::Utilities.configure instead.',
230
- :action => lambda {|klass, val|
231
- Utilities.configure { sendmail val }
232
- }
233
-
234
- attr_deprecate :exim, :version => '3.6.0',
235
- :message => 'Use Backup::Utilities.configure instead.',
236
- :action => lambda {|klass, val|
237
- Utilities.configure { exim val }
238
- }
239
-
240
222
  end
241
223
  end
242
224
  end
@@ -254,27 +236,3 @@ module Mail
254
236
  end
255
237
  end
256
238
  end
257
-
258
- # Patch mail v2.5.4 for ruby-1.8.7
259
- # https://github.com/mikel/mail/issues/548
260
- # https://github.com/mikel/mail/commit/c7318a6c03c1ecb3f574ccd2e3f06778687d1d15
261
- if RUBY_VERSION < '1.9'
262
- module Mail
263
- class SMTP
264
- private
265
- def ssl_context
266
- openssl_verify_mode = settings[:openssl_verify_mode]
267
-
268
- if openssl_verify_mode.kind_of?(String)
269
- openssl_verify_mode = "OpenSSL::SSL::VERIFY_#{openssl_verify_mode.upcase}".constantize
270
- end
271
-
272
- context = Net::SMTP.default_ssl_context
273
- context.verify_mode = openssl_verify_mode
274
- context.ca_path = settings[:ca_path] if settings[:ca_path]
275
- context.ca_file = settings[:ca_file] if settings[:ca_file]
276
- context
277
- end
278
- end
279
- end
280
- end
@@ -58,7 +58,7 @@ module Backup
58
58
  }
59
59
  options = {
60
60
  :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
61
- :body => encode_www_form(data)
61
+ :body => URI.encode_www_form(data)
62
62
  }
63
63
  options.merge!(:expects => 200) # raise error if unsuccessful
64
64
  Excon.post(uri, options)
@@ -69,7 +69,7 @@ module Backup
69
69
  end
70
70
  options = {
71
71
  :headers => { 'Content-Type' => 'application/x-www-form-urlencoded' },
72
- :body => encode_www_form(data)
72
+ :body => URI.encode_www_form(data)
73
73
  }
74
74
  options.merge!(:expects => 200) # raise error if unsuccessful
75
75
  Excon.post(uri, options)
@@ -5,7 +5,7 @@ module Backup
5
5
  class Error < Backup::Error; end
6
6
 
7
7
  class << self
8
- include Backup::Utilities::Helpers
8
+ include Utilities::Helpers
9
9
 
10
10
  ##
11
11
  # Build the final package for the backup model.
@@ -4,7 +4,7 @@ module Backup
4
4
  class Pipeline
5
5
  class Error < Backup::Error; end
6
6
 
7
- include Backup::Utilities::Helpers
7
+ include Utilities::Helpers
8
8
 
9
9
  attr_reader :stderr, :errors
10
10
 
@@ -2,7 +2,7 @@
2
2
 
3
3
  module Backup
4
4
  class Splitter
5
- include Backup::Utilities::Helpers
5
+ include Utilities::Helpers
6
6
 
7
7
  attr_reader :package, :chunk_size, :suffix_length
8
8
 
@@ -3,7 +3,7 @@
3
3
  module Backup
4
4
  module Storage
5
5
  class Base
6
- include Backup::Configuration::Helpers
6
+ include Config::Helpers
7
7
 
8
8
  ##
9
9
  # Base path on the remote where backup package files will be stored.
@@ -33,7 +33,7 @@ module Backup
33
33
  def perform!
34
34
  Logger.info "#{ storage_name } Started..."
35
35
  transfer!
36
- cycle!
36
+ cycle! if respond_to?(:cycle!, true) && keep.to_i > 0
37
37
  Logger.info "#{ storage_name } Finished!"
38
38
  end
39
39
 
@@ -46,18 +46,6 @@ module Backup
46
46
  end
47
47
  alias :remote_path_for :remote_path
48
48
 
49
- ##
50
- # Adds the current package being stored to the YAML cycle data file
51
- # and will remove any old Package file(s) when the storage limit
52
- # set by #keep is exceeded. Any errors raised while attempting to
53
- # remove older packages will be rescued and a warning will be logged
54
- # containing the original error message.
55
- def cycle!
56
- return unless keep.to_i > 0
57
- Logger.info "Cycling Started..."
58
- Cycler.cycle!(self)
59
- end
60
-
61
49
  def storage_name
62
50
  @storage_name ||= self.class.to_s.sub('Backup::', '') +
63
51
  (storage_id ? " (#{ storage_id })" : '')
@@ -4,6 +4,7 @@ require 'backup/cloud_io/cloud_files'
4
4
  module Backup
5
5
  module Storage
6
6
  class CloudFiles < Base
7
+ include Storage::Cycler
7
8
  class Error < Backup::Error; end
8
9
 
9
10
  ##
@@ -5,41 +5,21 @@ module Backup
5
5
  module Cycler
6
6
  class Error < Backup::Error; end
7
7
 
8
- class << self
8
+ private
9
9
 
10
- ##
11
- # Adds the given +package+ to the YAML storage file corresponding
12
- # to the given +storage+ and Package#trigger (Model#trigger).
13
- # Then, calls the +storage+ to remove the files for any older
14
- # packages that were removed from the YAML storage file.
15
- def cycle!(storage)
16
- @storage = storage
17
- @package = storage.package
18
- @storage_file = storage_file
10
+ # Adds the current package being stored to the YAML cycle data file
11
+ # and will remove any old package file(s) when the storage limit
12
+ # set by #keep is exceeded.
13
+ def cycle!
14
+ Logger.info 'Cycling Started...'
19
15
 
20
- update_storage_file!
21
- remove_packages!
22
- end
23
-
24
- private
25
-
26
- ##
27
- # Updates the YAML data file according to the #keep setting
28
- # for the storage and sets the @packages_to_remove
29
- def update_storage_file!
30
- packages = yaml_load.unshift(@package)
31
- excess = packages.count - @storage.keep.to_i
32
- @packages_to_remove = (excess > 0) ? packages.pop(excess) : []
33
- yaml_save(packages)
34
- end
16
+ packages = yaml_load.unshift(package)
17
+ excess = packages.count - keep.to_i
35
18
 
36
- ##
37
- # Calls the @storage to remove any old packages
38
- # which were cycled out of the storage file.
39
- def remove_packages!
40
- @packages_to_remove.each do |pkg|
19
+ if excess > 0
20
+ packages.pop(excess).each do |pkg|
41
21
  begin
42
- @storage.send(:remove!, pkg) unless pkg.no_cycle
22
+ remove!(pkg) unless pkg.no_cycle
43
23
  rescue => err
44
24
  Logger.warn Error.wrap(err, <<-EOS)
45
25
  There was a problem removing the following package:
@@ -51,70 +31,35 @@ module Backup
51
31
  end
52
32
  end
53
33
 
54
- ##
55
- # Return full path to the YAML data file,
56
- # based on the current values of @storage and @package
57
- def storage_file
58
- filename = @storage.class.to_s.split('::').last
59
- filename << "-#{ @storage.storage_id }" if @storage.storage_id
60
- File.join(Config.data_path, @package.trigger, "#{ filename }.yml")
61
- end
34
+ yaml_save(packages)
35
+ end
62
36
 
63
- ##
64
- # Load Package objects from YAML file.
65
- # Returns an Array, sorted by @time descending.
66
- # i.e. most recent is objects[0]
67
- def yaml_load
68
- packages = []
69
- if File.exist?(@storage_file) && !File.zero?(@storage_file)
70
- packages = check_upgrade(
71
- YAML.load_file(@storage_file).sort do |a, b|
72
- b.instance_variable_get(:@time) <=> a.instance_variable_get(:@time)
73
- end
74
- )
75
- end
76
- packages
37
+ # Returns path to the YAML data file.
38
+ def yaml_file
39
+ @yaml_file ||= begin
40
+ filename = self.class.to_s.split('::').last
41
+ filename << "-#{ storage_id }" if storage_id
42
+ File.join(Config.data_path, package.trigger, "#{ filename }.yml")
77
43
  end
44
+ end
78
45
 
79
- ##
80
- # Store the given package objects to the YAML data file.
81
- def yaml_save(packages)
82
- FileUtils.mkdir_p(File.dirname(@storage_file))
83
- File.open(@storage_file, 'w') do |file|
84
- file.write(packages.to_yaml)
85
- end
46
+ # Returns stored Package objects, sorted by #time descending (oldest last).
47
+ def yaml_load
48
+ if File.exist?(yaml_file) && !File.zero?(yaml_file)
49
+ YAML.load_file(yaml_file).sort_by!(&:time).reverse!
50
+ else
51
+ []
86
52
  end
53
+ end
87
54
 
88
- ##
89
- # Upgrade the objects loaded from the YAML file, if needed.
90
- def check_upgrade(objects)
91
- if objects.any? {|obj| obj.class.to_s =~ /Backup::Storage/ }
92
- # Version <= 3.0.20
93
- model = @storage.instance_variable_get(:@model)
94
- v3_0_20 = objects.any? {|obj| obj.instance_variable_defined?(:@version) }
95
- objects.map! do |obj|
96
- if v3_0_20 # Version == 3.0.20
97
- filename = obj.instance_variable_get(:@filename)[20..-1]
98
- chunk_suffixes = obj.instance_variable_get(:@chunk_suffixes)
99
- else # Version <= 3.0.19
100
- filename = obj.instance_variable_get(:@remote_file)[20..-1]
101
- chunk_suffixes = []
102
- end
103
- time = obj.instance_variable_get(:@time)
104
- extension = filename.match(/\.(tar.*)$/)[1]
105
-
106
- package = Backup::Package.new(model)
107
- package.instance_variable_set(:@time, time)
108
- package.extension = extension
109
- package.chunk_suffixes = chunk_suffixes
110
-
111
- package
112
- end
113
- end
114
- objects
55
+ # Stores the given package objects to the YAML data file.
56
+ def yaml_save(packages)
57
+ FileUtils.mkdir_p(File.dirname(yaml_file))
58
+ File.open(yaml_file, 'w') do |file|
59
+ file.write(packages.to_yaml)
115
60
  end
116
-
117
61
  end
62
+
118
63
  end
119
64
  end
120
65
  end