backup 3.11.0 → 4.0.0rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +4 -4
  3. data/lib/backup.rb +1 -4
  4. data/lib/backup/archive.rb +1 -1
  5. data/lib/backup/cli.rb +51 -107
  6. data/lib/backup/compressor/base.rb +2 -2
  7. data/lib/backup/compressor/bzip2.rb +0 -11
  8. data/lib/backup/compressor/gzip.rb +0 -11
  9. data/lib/backup/config.rb +45 -123
  10. data/lib/backup/config/dsl.rb +102 -0
  11. data/lib/backup/{configuration → config}/helpers.rb +23 -14
  12. data/lib/backup/database/base.rb +2 -2
  13. data/lib/backup/database/mongodb.rb +0 -18
  14. data/lib/backup/database/mysql.rb +6 -75
  15. data/lib/backup/database/postgresql.rb +0 -12
  16. data/lib/backup/database/redis.rb +85 -47
  17. data/lib/backup/database/riak.rb +0 -19
  18. data/lib/backup/encryptor/base.rb +2 -2
  19. data/lib/backup/encryptor/gpg.rb +1 -12
  20. data/lib/backup/logger/fog_adapter.rb +1 -2
  21. data/lib/backup/model.rb +3 -24
  22. data/lib/backup/notifier/base.rb +2 -17
  23. data/lib/backup/notifier/http_post.rb +1 -1
  24. data/lib/backup/notifier/mail.rb +5 -47
  25. data/lib/backup/notifier/prowl.rb +1 -1
  26. data/lib/backup/notifier/pushover.rb +1 -1
  27. data/lib/backup/packager.rb +1 -1
  28. data/lib/backup/pipeline.rb +1 -1
  29. data/lib/backup/splitter.rb +1 -1
  30. data/lib/backup/storage/base.rb +2 -14
  31. data/lib/backup/storage/cloud_files.rb +1 -0
  32. data/lib/backup/storage/cycler.rb +33 -88
  33. data/lib/backup/storage/dropbox.rb +19 -12
  34. data/lib/backup/storage/ftp.rb +1 -0
  35. data/lib/backup/storage/local.rb +1 -0
  36. data/lib/backup/storage/ninefold.rb +1 -0
  37. data/lib/backup/storage/rsync.rb +7 -41
  38. data/lib/backup/storage/s3.rb +1 -0
  39. data/lib/backup/storage/scp.rb +1 -0
  40. data/lib/backup/storage/sftp.rb +1 -0
  41. data/lib/backup/syncer/base.rb +2 -2
  42. data/lib/backup/syncer/cloud/cloud_files.rb +0 -16
  43. data/lib/backup/syncer/cloud/s3.rb +0 -16
  44. data/lib/backup/syncer/rsync/local.rb +0 -5
  45. data/lib/backup/syncer/rsync/pull.rb +0 -21
  46. data/lib/backup/syncer/rsync/push.rb +0 -21
  47. data/lib/backup/utilities.rb +2 -22
  48. data/lib/backup/version.rb +1 -1
  49. data/templates/cli/archive +0 -3
  50. data/templates/cli/compressor/custom +0 -4
  51. data/templates/cli/config +39 -17
  52. data/templates/cli/{database → databases}/mongodb +0 -0
  53. data/templates/cli/{database → databases}/mysql +0 -0
  54. data/templates/cli/{database → databases}/postgresql +0 -0
  55. data/templates/cli/databases/redis +16 -0
  56. data/templates/cli/{database → databases}/riak +0 -0
  57. data/templates/cli/{model.erb → model} +8 -5
  58. data/templates/cli/{notifier → notifiers}/campfire +0 -0
  59. data/templates/cli/{notifier → notifiers}/hipchat +0 -0
  60. data/templates/cli/{notifier → notifiers}/http_post +0 -3
  61. data/templates/cli/{notifier → notifiers}/mail +1 -2
  62. data/templates/cli/notifiers/nagios +13 -0
  63. data/templates/cli/{notifier → notifiers}/prowl +0 -0
  64. data/templates/cli/{notifier → notifiers}/pushover +0 -0
  65. data/templates/cli/{notifier → notifiers}/twitter +0 -0
  66. data/templates/cli/{storage → storages}/cloud_files +0 -2
  67. data/templates/cli/storages/dropbox +19 -0
  68. data/templates/cli/{storage → storages}/ftp +0 -0
  69. data/templates/cli/{storage → storages}/local +0 -0
  70. data/templates/cli/{storage → storages}/ninefold +0 -0
  71. data/templates/cli/{storage → storages}/rsync +0 -2
  72. data/templates/cli/{storage → storages}/s3 +0 -2
  73. data/templates/cli/{storage → storages}/scp +0 -0
  74. data/templates/cli/{storage → storages}/sftp +0 -0
  75. data/templates/cli/{syncer → syncers}/cloud_files +0 -2
  76. data/templates/cli/{syncer → syncers}/rsync_local +0 -0
  77. data/templates/cli/{syncer → syncers}/rsync_pull +0 -2
  78. data/templates/cli/{syncer → syncers}/rsync_push +0 -2
  79. data/templates/cli/{syncer → syncers}/s3 +0 -2
  80. data/templates/general/links +1 -1
  81. metadata +241 -69
  82. data/lib/backup/compressor/lzma.rb +0 -52
  83. data/lib/backup/compressor/pbzip2.rb +0 -59
  84. data/lib/backup/configuration.rb +0 -33
  85. data/lib/backup/configuration/store.rb +0 -24
  86. data/templates/cli/compressor/lzma +0 -10
  87. data/templates/cli/compressor/pbzip2 +0 -10
  88. data/templates/cli/database/redis +0 -18
  89. data/templates/cli/notifier/nagios +0 -13
  90. data/templates/cli/storage/dropbox +0 -20
@@ -4,12 +4,24 @@ require 'dropbox_sdk'
4
4
  module Backup
5
5
  module Storage
6
6
  class Dropbox < Base
7
+ include Storage::Cycler
7
8
  class Error < Backup::Error; end
8
9
 
9
10
  ##
10
11
  # Dropbox API credentials
11
12
  attr_accessor :api_key, :api_secret
12
13
 
14
+ ##
15
+ # Path to store cached authorized session.
16
+ #
17
+ # Relative paths will be expanded using Config.root_path,
18
+ # which by default is ~/Backup unless --root-path was used
19
+ # on the command line or set in config.rb.
20
+ #
21
+ # By default, +cache_path+ is '.cache', which would be
22
+ # '~/Backup/.cache/' if using the default root_path.
23
+ attr_accessor :cache_path
24
+
13
25
  ##
14
26
  # Dropbox Access Type
15
27
  # Valid values are:
@@ -39,6 +51,7 @@ module Backup
39
51
  super
40
52
 
41
53
  @path ||= 'backups'
54
+ @cache_path ||= '.cache'
42
55
  @access_type ||= :app_folder
43
56
  @chunk_size ||= 4 # MiB
44
57
  @max_retries ||= 10
@@ -54,7 +67,7 @@ module Backup
54
67
  # authorization successfully took place. If this is the case, then the
55
68
  # user hits 'enter' and the session will be properly established.
56
69
  # Immediately after establishing the session, the session will be
57
- # serialized and written to a cache file in Backup::Config.cache_path.
70
+ # serialized and written to a cache file in +cache_path+.
58
71
  # The cached file will be used from that point on to re-establish a
59
72
  # connection with Dropbox at a later time. This allows the user to avoid
60
73
  # having to go to a new Dropbox URL to authorize over and over again.
@@ -121,12 +134,11 @@ module Backup
121
134
  raise Error.wrap(err, 'Upload Failed!')
122
135
  end
123
136
 
124
- # Timeout::Error is not a StandardError under ruby-1.8.7
125
137
  def with_retries
126
138
  retries = 0
127
139
  begin
128
140
  yield
129
- rescue StandardError, Timeout::Error => err
141
+ rescue StandardError => err
130
142
  retries += 1
131
143
  raise if retries > max_retries
132
144
 
@@ -145,13 +157,15 @@ module Backup
145
157
  end
146
158
 
147
159
  def cached_file
148
- File.join(Config.cache_path, api_key + api_secret)
160
+ path = cache_path.start_with?('/') ?
161
+ cache_path : File.join(Config.root_path, cache_path)
162
+ File.join(path, api_key + api_secret)
149
163
  end
150
164
 
151
165
  ##
152
166
  # Serializes and writes the Dropbox session to a cache file
153
167
  def write_cache!(session)
154
- FileUtils.mkdir_p(Config.cache_path)
168
+ FileUtils.mkdir_p File.dirname(cached_file)
155
169
  File.open(cached_file, "w") do |cache_file|
156
170
  cache_file.write(session.serialize)
157
171
  end
@@ -193,13 +207,6 @@ module Backup
193
207
  raise Error.wrap(err, 'Could not create or authenticate a new session')
194
208
  end
195
209
 
196
- attr_deprecate :email, :version => '3.0.17'
197
- attr_deprecate :password, :version => '3.0.17'
198
- attr_deprecate :timeout, :version => '3.0.21'
199
-
200
- attr_deprecate :chunk_retries, :version => '3.7.0',
201
- :message => 'Use #max_retries instead.',
202
- :action => lambda {|klass, val| klass.max_retries = val }
203
210
  end
204
211
  end
205
212
  end
@@ -4,6 +4,7 @@ require 'net/ftp'
4
4
  module Backup
5
5
  module Storage
6
6
  class FTP < Base
7
+ include Storage::Cycler
7
8
 
8
9
  ##
9
10
  # Server credentials
@@ -3,6 +3,7 @@
3
3
  module Backup
4
4
  module Storage
5
5
  class Local < Base
6
+ include Storage::Cycler
6
7
  class Error < Backup::Error; end
7
8
 
8
9
  def initialize(model, storage_id = nil)
@@ -4,6 +4,7 @@ require 'fog'
4
4
  module Backup
5
5
  module Storage
6
6
  class Ninefold < Base
7
+ include Storage::Cycler
7
8
  class Error < Backup::Error; end
8
9
 
9
10
  ##
@@ -3,7 +3,7 @@
3
3
  module Backup
4
4
  module Storage
5
5
  class RSync < Base
6
- include Backup::Utilities::Helpers
6
+ include Utilities::Helpers
7
7
 
8
8
  ##
9
9
  # Mode of operation
@@ -120,16 +120,11 @@ module Backup
120
120
  # into. For `:ssh_daemon` or `:rsync_daemon` modes, this would reference
121
121
  # an rsync module/path.
122
122
  #
123
- # In :ssh_daemon and :rsync_daemon modes, the files will be stored
124
- # directly to the +path+ given. The path (or path defined by your rsync
125
- # module) must already exist.
126
- # Note that no additional `<trigger>` directory will be added to this path.
123
+ # In :ssh_daemon and :rsync_daemon modes, +path+ (or path defined by
124
+ # your rsync module) must already exist.
127
125
  #
128
- # In :ssh mode or local operation (no +host+ specified), the actual
129
- # destination path will be `<path>/<trigger>/`. This path will be created
130
- # if needed - either locally, or on the remote for :ssh mode.
131
- # This behavior will change in v4.0, when :ssh mode and local operations
132
- # will also store the files directly in the +path+ given.
126
+ # In :ssh mode or local operation (no +host+ specified), +path+ will
127
+ # be created if needed - either locally, or on the remote for :ssh mode.
133
128
  attr_accessor :path
134
129
 
135
130
  def initialize(model, storage_id = nil)
@@ -157,27 +152,16 @@ module Backup
157
152
  remove_password_file
158
153
  end
159
154
 
160
- # Storage::RSync doesn't cycle
161
- def cycle!; end
162
-
163
155
  ##
164
156
  # Other storages add an additional timestamp directory to this path.
165
157
  # This is not desired here, since we need to transfer the package files
166
158
  # to the same location each time.
167
- #
168
- # Note: In v4.0, the additional trigger directory will to be dropped
169
- # from remote_path for both local and :ssh mode, so the package files
170
- # will be stored directly in #path.
171
159
  def remote_path
172
160
  @remote_path ||= begin
173
161
  if host
174
- if mode == :ssh
175
- File.join(path.sub(/^~\//, ''), package.trigger)
176
- else
177
- path.sub(/^~\//, '').sub(/\/$/, '')
178
- end
162
+ path.sub(/^~\//, '').sub(/\/$/, '')
179
163
  else
180
- File.join(File.expand_path(path), package.trigger)
164
+ File.expand_path(path)
181
165
  end
182
166
  end
183
167
  end
@@ -259,24 +243,6 @@ module Backup
259
243
  @password_file.delete if @password_file
260
244
  end
261
245
 
262
- attr_deprecate :local, :version => '3.2.0',
263
- :message => "If 'host' is not set, the operation will be local."
264
-
265
- attr_deprecate :username, :version => '3.2.0',
266
- :message => 'Use #ssh_user instead.',
267
- :action => lambda {|klass, val|
268
- klass.ssh_user = val
269
- }
270
- attr_deprecate :password, :version => '3.2.0',
271
- :message => 'Use #rsync_password instead.',
272
- :action => lambda {|klass, val|
273
- klass.rsync_password = val
274
- }
275
- attr_deprecate :ip, :version => '3.2.0',
276
- :message => 'Use #host instead.',
277
- :action => lambda {|klass, val|
278
- klass.host = val
279
- }
280
246
  end
281
247
  end
282
248
  end
@@ -4,6 +4,7 @@ require 'backup/cloud_io/s3'
4
4
  module Backup
5
5
  module Storage
6
6
  class S3 < Base
7
+ include Storage::Cycler
7
8
  class Error < Backup::Error; end
8
9
 
9
10
  ##
@@ -4,6 +4,7 @@ require 'net/scp'
4
4
  module Backup
5
5
  module Storage
6
6
  class SCP < Base
7
+ include Storage::Cycler
7
8
  class Error < Backup::Error; end
8
9
 
9
10
  ##
@@ -4,6 +4,7 @@ require 'net/sftp'
4
4
  module Backup
5
5
  module Storage
6
6
  class SFTP < Base
7
+ include Storage::Cycler
7
8
 
8
9
  ##
9
10
  # Server credentials
@@ -3,8 +3,8 @@
3
3
  module Backup
4
4
  module Syncer
5
5
  class Base
6
- include Backup::Utilities::Helpers
7
- include Backup::Configuration::Helpers
6
+ include Utilities::Helpers
7
+ include Config::Helpers
8
8
 
9
9
  ##
10
10
  # Path to store the synced files/directories to
@@ -77,22 +77,6 @@ module Backup
77
77
  EOS
78
78
  end
79
79
 
80
- attr_deprecate :concurrency_type, :version => '3.7.0',
81
- :message => 'Use #thread_count instead.',
82
- :action => lambda {|klass, val|
83
- if val == :threads
84
- klass.thread_count = 2 unless klass.thread_count
85
- else
86
- klass.thread_count = 0
87
- end
88
- }
89
-
90
- attr_deprecate :concurrency_level, :version => '3.7.0',
91
- :message => 'Use #thread_count instead.',
92
- :action => lambda {|klass, val|
93
- klass.thread_count = val unless klass.thread_count == 0
94
- }
95
-
96
80
  end # class Cloudfiles < Base
97
81
  end # module Cloud
98
82
  end
@@ -104,22 +104,6 @@ module Backup
104
104
  EOS
105
105
  end
106
106
 
107
- attr_deprecate :concurrency_type, :version => '3.7.0',
108
- :message => 'Use #thread_count instead.',
109
- :action => lambda {|klass, val|
110
- if val == :threads
111
- klass.thread_count = 2 unless klass.thread_count
112
- else
113
- klass.thread_count = 0
114
- end
115
- }
116
-
117
- attr_deprecate :concurrency_level, :version => '3.7.0',
118
- :message => 'Use #thread_count instead.',
119
- :action => lambda {|klass, val|
120
- klass.thread_count = val unless klass.thread_count == 0
121
- }
122
-
123
107
  end # Class S3 < Base
124
108
  end # module Cloud
125
109
  end
@@ -25,11 +25,6 @@ module Backup
25
25
  FileUtils.mkdir_p dest_path
26
26
  end
27
27
 
28
- attr_deprecate :additional_options, :version => '3.2.0',
29
- :message => 'Use #additional_rsync_options instead.',
30
- :action => lambda {|klass, val|
31
- klass.additional_rsync_options = val
32
- }
33
28
  end
34
29
  end
35
30
  end
@@ -45,27 +45,6 @@ module Backup
45
45
  FileUtils.mkdir_p dest_path
46
46
  end
47
47
 
48
- attr_deprecate :additional_options, :version => '3.2.0',
49
- :message => 'Use #additional_rsync_options instead.',
50
- :action => lambda {|klass, val|
51
- klass.additional_rsync_options = val
52
- }
53
-
54
- attr_deprecate :username, :version => '3.2.0',
55
- :message => 'Use #ssh_user instead.',
56
- :action => lambda {|klass, val|
57
- klass.ssh_user = val
58
- }
59
- attr_deprecate :password, :version => '3.2.0',
60
- :message => 'Use #rsync_password instead.',
61
- :action => lambda {|klass, val|
62
- klass.rsync_password = val
63
- }
64
- attr_deprecate :ip, :version => '3.2.0',
65
- :message => 'Use #host instead.',
66
- :action => lambda {|klass, val|
67
- klass.host = val
68
- }
69
48
  end
70
49
  end
71
50
  end
@@ -199,27 +199,6 @@ module Backup
199
199
  @password_file.delete if @password_file
200
200
  end
201
201
 
202
- attr_deprecate :additional_options, :version => '3.2.0',
203
- :message => 'Use #additional_rsync_options instead.',
204
- :action => lambda {|klass, val|
205
- klass.additional_rsync_options = val
206
- }
207
-
208
- attr_deprecate :username, :version => '3.2.0',
209
- :message => 'Use #ssh_user instead.',
210
- :action => lambda {|klass, val|
211
- klass.ssh_user = val
212
- }
213
- attr_deprecate :password, :version => '3.2.0',
214
- :message => 'Use #rsync_password instead.',
215
- :action => lambda {|klass, val|
216
- klass.rsync_password = val
217
- }
218
- attr_deprecate :ip, :version => '3.2.0',
219
- :message => 'Use #host instead.',
220
- :action => lambda {|klass, val|
221
- klass.host = val
222
- }
223
202
  end
224
203
  end
225
204
  end
@@ -7,9 +7,8 @@ module Backup
7
7
  UTILITY = {}
8
8
  NAMES = %w{
9
9
  tar cat split sudo chown hostname
10
- gzip bzip2 lzma pbzip2
11
- mongo mongodump mysqldump innobackupex
12
- pg_dump pg_dumpall redis-cli riak-admin
10
+ gzip bzip2
11
+ mongo mongodump mysqldump pg_dump pg_dumpall redis-cli riak-admin
13
12
  gpg openssl
14
13
  rsync ssh
15
14
  sendmail exim
@@ -73,8 +72,6 @@ module Backup
73
72
  # # Compressors
74
73
  # gzip '/path/to/gzip'
75
74
  # bzip2 '/path/to/bzip2'
76
- # lzma '/path/to/lzma' # deprecated. use a Custom Compressor
77
- # pbzip2 '/path/to/pbzip2' # deprecated. use a Custom Compressor
78
75
  #
79
76
  # # Database Utilities
80
77
  # mongo '/path/to/mongo'
@@ -101,17 +98,6 @@ module Backup
101
98
  #
102
99
  # These paths may be set using absolute paths, or relative to the
103
100
  # working directory when Backup is run.
104
- #
105
- # Note that many of Backup's components currently have their own
106
- # configuration settings for utility paths. For instance, when configuring
107
- # a +MySQL+ database backup, +mysqldump_utility+ may be used:
108
- #
109
- # database MySQL do |db|
110
- # db.mysqldump_utility = '/path/to/mysqldump'
111
- # end
112
- #
113
- # Use of these configuration settings will override the path set here.
114
- # (The use of these may be deprecated in the future)
115
101
  def configure(&block)
116
102
  DSL.instance_eval(&block)
117
103
  end
@@ -184,18 +170,12 @@ module Backup
184
170
 
185
171
  begin
186
172
  out, err = '', ''
187
- # popen4 doesn't work in 1.8.7 with stock versions of ruby shipped
188
- # with major OSs. Hack to make it stop segfaulting.
189
- # See: https://github.com/engineyard/engineyard/issues/115
190
- GC.disable if RUBY_VERSION < '1.9'
191
173
  ps = Open4.popen4(command) do |pid, stdin, stdout, stderr|
192
174
  stdin.close
193
175
  out, err = stdout.read.strip, stderr.read.strip
194
176
  end
195
177
  rescue Exception => e
196
178
  raise Error.wrap(e, "Failed to execute '#{ name }'")
197
- ensure
198
- GC.enable if RUBY_VERSION < '1.9'
199
179
  end
200
180
 
201
181
  if ps.success?
@@ -1,5 +1,5 @@
1
1
  # encoding: utf-8
2
2
 
3
3
  module Backup
4
- VERSION = '3.11.0'
4
+ VERSION = '4.0.0rc1'
5
5
  end
@@ -18,9 +18,6 @@
18
18
  #
19
19
  # archive.root '/path/to/archive/root'
20
20
  #
21
- # For more details, please see:
22
- # https://github.com/meskyanichi/backup/wiki/Archives
23
- #
24
21
  archive :my_archive do |archive|
25
22
  # Run the `tar` command using `sudo`
26
23
  # archive.use_sudo
@@ -1,10 +1,6 @@
1
1
  ##
2
2
  # Custom [Compressor]
3
3
  #
4
- # For information on using a Custom Compressor,
5
- # please see the following Wiki page:
6
- # https://github.com/meskyanichi/backup/wiki/Compressors
7
- #
8
4
  compress_with Custom do |compressor|
9
5
  compressor.command = 'gzip'
10
6
  compressor.extension = '.gz'
data/templates/cli/config CHANGED
@@ -1,14 +1,47 @@
1
1
  # encoding: utf-8
2
2
 
3
3
  ##
4
- # Backup
5
- # Generated Main Config Template
4
+ # Backup v<%= Backup::VERSION.split('.').first %>.x Configuration
6
5
  #
7
- # For more information:
6
+ # Documentation: http://meskyanichi.github.io/backup
7
+ # Issue Tracker: https://github.com/meskyanichi/backup/issues
8
+
9
+ ##
10
+ # Config Options
11
+ #
12
+ # The options here may be overridden on the command line, but the result
13
+ # will depend on the use of --root-path on the command line.
14
+ #
15
+ # If --root-path is used on the command line, then all paths set here
16
+ # will be overridden. If a path (like --tmp-path) is not given along with
17
+ # --root-path, that path will use it's default location _relative to --root-path_.
18
+ #
19
+ # If --root-path is not used on the command line, a path option (like --tmp-path)
20
+ # given on the command line will override the tmp_path set here, but all other
21
+ # paths set here will be used.
22
+ #
23
+ # Note that relative paths given on the command line without --root-path
24
+ # are relative to the current directory. The root_path set here only applies
25
+ # to relative paths set here.
26
+ #
27
+ # ---
28
+ #
29
+ # Sets the root path for all relative paths, including default paths.
30
+ # May be an absolute path, or relative to the current working directory.
8
31
  #
9
- # View the Git repository at https://github.com/meskyanichi/backup
10
- # View the Wiki/Documentation at https://github.com/meskyanichi/backup/wiki
11
- # View the issue log at https://github.com/meskyanichi/backup/issues
32
+ # root_path 'my/root'
33
+ #
34
+ # Sets the path where backups are processed until they're stored.
35
+ # This must have enough free space to hold apx. 2 backups.
36
+ # May be an absolute path, or relative to the current directory or +root_path+.
37
+ #
38
+ # tmp_path 'my/tmp'
39
+ #
40
+ # Sets the path where backup stores persistent information.
41
+ # When Backup's Cycler is used, small YAML files are stored here.
42
+ # May be an absolute path, or relative to the current directory or +root_path+.
43
+ #
44
+ # data_path 'my/data'
12
45
 
13
46
  ##
14
47
  # Utilities
@@ -88,14 +121,3 @@
88
121
  # mail.to = 'john.smith@email.com'
89
122
  # end
90
123
  # end
91
-
92
-
93
- # * * * * * * * * * * * * * * * * * * * *
94
- # Do Not Edit Below Here.
95
- # All Configuration Should Be Made Above.
96
-
97
- ##
98
- # Load all models from the models directory.
99
- Dir[File.join(File.dirname(Config.config_file), "models", "*.rb")].each do |model|
100
- instance_eval(File.read(model))
101
- end