backup 4.4.1 → 5.0.0.beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +5 -5
  2. data/LICENSE +19 -0
  3. data/README.md +1 -1
  4. data/lib/backup.rb +74 -78
  5. data/lib/backup/archive.rb +31 -32
  6. data/lib/backup/binder.rb +2 -6
  7. data/lib/backup/cleaner.rb +14 -18
  8. data/lib/backup/cli.rb +104 -108
  9. data/lib/backup/cloud_io/base.rb +4 -7
  10. data/lib/backup/cloud_io/cloud_files.rb +60 -62
  11. data/lib/backup/cloud_io/s3.rb +69 -76
  12. data/lib/backup/compressor/base.rb +4 -7
  13. data/lib/backup/compressor/bzip2.rb +3 -7
  14. data/lib/backup/compressor/custom.rb +2 -6
  15. data/lib/backup/compressor/gzip.rb +16 -17
  16. data/lib/backup/config.rb +17 -18
  17. data/lib/backup/config/dsl.rb +16 -17
  18. data/lib/backup/config/helpers.rb +10 -16
  19. data/lib/backup/database/base.rb +22 -21
  20. data/lib/backup/database/mongodb.rb +36 -37
  21. data/lib/backup/database/mysql.rb +40 -41
  22. data/lib/backup/database/openldap.rb +8 -10
  23. data/lib/backup/database/postgresql.rb +29 -30
  24. data/lib/backup/database/redis.rb +27 -30
  25. data/lib/backup/database/riak.rb +15 -18
  26. data/lib/backup/database/sqlite.rb +4 -6
  27. data/lib/backup/encryptor/base.rb +2 -4
  28. data/lib/backup/encryptor/gpg.rb +49 -59
  29. data/lib/backup/encryptor/open_ssl.rb +11 -14
  30. data/lib/backup/errors.rb +7 -12
  31. data/lib/backup/logger.rb +16 -18
  32. data/lib/backup/logger/console.rb +5 -8
  33. data/lib/backup/logger/fog_adapter.rb +2 -6
  34. data/lib/backup/logger/logfile.rb +10 -12
  35. data/lib/backup/logger/syslog.rb +2 -4
  36. data/lib/backup/model.rb +75 -40
  37. data/lib/backup/notifier/base.rb +24 -26
  38. data/lib/backup/notifier/campfire.rb +9 -11
  39. data/lib/backup/notifier/command.rb +0 -3
  40. data/lib/backup/notifier/datadog.rb +9 -12
  41. data/lib/backup/notifier/flowdock.rb +13 -17
  42. data/lib/backup/notifier/hipchat.rb +11 -13
  43. data/lib/backup/notifier/http_post.rb +11 -14
  44. data/lib/backup/notifier/mail.rb +44 -47
  45. data/lib/backup/notifier/nagios.rb +5 -9
  46. data/lib/backup/notifier/pagerduty.rb +10 -12
  47. data/lib/backup/notifier/prowl.rb +15 -15
  48. data/lib/backup/notifier/pushover.rb +7 -10
  49. data/lib/backup/notifier/ses.rb +34 -16
  50. data/lib/backup/notifier/slack.rb +39 -40
  51. data/lib/backup/notifier/twitter.rb +2 -5
  52. data/lib/backup/notifier/zabbix.rb +11 -14
  53. data/lib/backup/package.rb +5 -9
  54. data/lib/backup/packager.rb +16 -17
  55. data/lib/backup/pipeline.rb +17 -21
  56. data/lib/backup/splitter.rb +8 -11
  57. data/lib/backup/storage/base.rb +5 -8
  58. data/lib/backup/storage/cloud_files.rb +21 -23
  59. data/lib/backup/storage/cycler.rb +10 -15
  60. data/lib/backup/storage/dropbox.rb +15 -21
  61. data/lib/backup/storage/ftp.rb +8 -10
  62. data/lib/backup/storage/local.rb +5 -8
  63. data/lib/backup/storage/qiniu.rb +8 -8
  64. data/lib/backup/storage/rsync.rb +24 -26
  65. data/lib/backup/storage/s3.rb +27 -28
  66. data/lib/backup/storage/scp.rb +10 -12
  67. data/lib/backup/storage/sftp.rb +10 -12
  68. data/lib/backup/syncer/base.rb +5 -8
  69. data/lib/backup/syncer/cloud/base.rb +27 -30
  70. data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
  71. data/lib/backup/syncer/cloud/local_file.rb +5 -8
  72. data/lib/backup/syncer/cloud/s3.rb +23 -24
  73. data/lib/backup/syncer/rsync/base.rb +6 -10
  74. data/lib/backup/syncer/rsync/local.rb +1 -5
  75. data/lib/backup/syncer/rsync/pull.rb +6 -10
  76. data/lib/backup/syncer/rsync/push.rb +18 -22
  77. data/lib/backup/template.rb +9 -14
  78. data/lib/backup/utilities.rb +82 -69
  79. data/lib/backup/version.rb +1 -3
  80. metadata +100 -660
@@ -1,5 +1,4 @@
1
- # encoding: utf-8
2
- require 'net/ftp'
1
+ require "net/ftp"
3
2
 
4
3
  module Backup
5
4
  module Storage
@@ -31,10 +30,10 @@ module Backup
31
30
  super
32
31
 
33
32
  @port ||= 21
34
- @path ||= 'backups'
33
+ @path ||= "backups"
35
34
  @passive_mode ||= false
36
35
  @timeout ||= nil
37
- path.sub!(/^~\//, '')
36
+ path.sub!(/^~\//, "")
38
37
  end
39
38
 
40
39
  private
@@ -68,7 +67,7 @@ module Backup
68
67
  package.filenames.each do |filename|
69
68
  src = File.join(Config.tmp_path, filename)
70
69
  dest = File.join(remote_path, filename)
71
- Logger.info "Storing '#{ ip }:#{ dest }'..."
70
+ Logger.info "Storing '#{ip}:#{dest}'..."
72
71
  ftp.put(src, dest)
73
72
  end
74
73
  end
@@ -77,7 +76,7 @@ module Backup
77
76
  # Called by the Cycler.
78
77
  # Any error raised will be logged as a warning.
79
78
  def remove!(package)
80
- Logger.info "Removing backup package dated #{ package.time }..."
79
+ Logger.info "Removing backup package dated #{package.time}..."
81
80
 
82
81
  remote_path = remote_path_for(package)
83
82
  connection do |ftp|
@@ -98,15 +97,14 @@ module Backup
98
97
  # Net::FTP raises an exception when the directory it's trying to create
99
98
  # already exists, so we have rescue it
100
99
  def create_remote_path(ftp)
101
- path_parts = Array.new
102
- remote_path.split('/').each do |path_part|
100
+ path_parts = []
101
+ remote_path.split("/").each do |path_part|
103
102
  path_parts << path_part
104
103
  begin
105
- ftp.mkdir(path_parts.join('/'))
104
+ ftp.mkdir(path_parts.join("/"))
106
105
  rescue Net::FTPPermError; end
107
106
  end
108
107
  end
109
-
110
108
  end
111
109
  end
112
110
  end
@@ -1,5 +1,3 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Storage
5
3
  class Local < Base
@@ -9,7 +7,7 @@ module Backup
9
7
  def initialize(model, storage_id = nil)
10
8
  super
11
9
 
12
- @path ||= '~/backups'
10
+ @path ||= "~/backups"
13
11
  end
14
12
 
15
13
  private
@@ -21,7 +19,7 @@ module Backup
21
19
  package.filenames.each do |filename|
22
20
  src = File.join(Config.tmp_path, filename)
23
21
  dest = File.join(remote_path, filename)
24
- Logger.info "Storing '#{ dest }'..."
22
+ Logger.info "Storing '#{dest}'..."
25
23
 
26
24
  FileUtils.send(transfer_method, src, dest)
27
25
  end
@@ -30,7 +28,7 @@ module Backup
30
28
  # Called by the Cycler.
31
29
  # Any error raised will be logged as a warning.
32
30
  def remove!(package)
33
- Logger.info "Removing backup package dated #{ package.time }..."
31
+ Logger.info "Removing backup package dated #{package.time}..."
34
32
 
35
33
  FileUtils.rm_r(remote_path_for(package))
36
34
  end
@@ -50,15 +48,14 @@ module Backup
50
48
  else
51
49
  Logger.warn Error.new(<<-EOS)
52
50
  Local File Copy Warning!
53
- The final backup file(s) for '#{ model.label }' (#{ model.trigger })
54
- will be *copied* to '#{ remote_path }'
51
+ The final backup file(s) for '#{model.label}' (#{model.trigger})
52
+ will be *copied* to '#{remote_path}'
55
53
  To avoid this, when using more than one Storage, the 'Local' Storage
56
54
  should be added *last* so the files may be *moved* to their destination.
57
55
  EOS
58
56
  false
59
57
  end
60
58
  end
61
-
62
59
  end
63
60
  end
64
61
  end
@@ -1,5 +1,4 @@
1
- # encoding: utf-8
2
- require 'qiniu'
1
+ require "qiniu"
3
2
 
4
3
  module Backup
5
4
  module Storage
@@ -18,18 +17,19 @@ module Backup
18
17
  def initialize(model, storage_id = nil)
19
18
  super
20
19
 
21
- @path ||= 'backups'
20
+ @path ||= "backups"
22
21
 
23
22
  check_configuration
24
23
  config_credentials
25
24
  end
26
25
 
27
26
  private
27
+
28
28
  def transfer!
29
29
  package.filenames.each do |filename|
30
30
  src = File.join(Config.tmp_path, filename)
31
31
  dest = File.join(remote_path, filename)
32
- Logger.info "Storing '#{ dest }'..."
32
+ Logger.info "Storing '#{dest}'..."
33
33
 
34
34
  ::Qiniu.upload_file(uptoken: ::Qiniu.generate_upload_token,
35
35
  bucket: bucket,
@@ -41,7 +41,7 @@ module Backup
41
41
  # Called by the Cycler.
42
42
  # Any error raised will be logged as a warning.
43
43
  def remove!(package)
44
- Logger.info "Removing backup package dated #{ package.time }..."
44
+ Logger.info "Removing backup package dated #{package.time}..."
45
45
  remote_path = remote_path_for(package)
46
46
  package.filenames.each do |filename|
47
47
  ::Qiniu.delete(bucket, File.join(remote_path, filename))
@@ -49,11 +49,11 @@ module Backup
49
49
  end
50
50
 
51
51
  def check_configuration
52
- required = %w{ access_key secret_key bucket }
52
+ required = %w[access_key secret_key bucket]
53
53
 
54
- raise Error, <<-EOS if required.map {|name| send(name) }.any?(&:nil?)
54
+ raise Error, <<-EOS if required.map { |name| send(name) }.any?(&:nil?)
55
55
  Configuration Error
56
- #{ required.map {|name| "##{ name }"}.join(', ') } are all required
56
+ #{required.map { |name| "##{name}" }.join(", ")} are all required
57
57
  EOS
58
58
  end
59
59
 
@@ -1,5 +1,3 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Storage
5
3
  class RSync < Base
@@ -133,7 +131,7 @@ module Backup
133
131
  @mode ||= :ssh
134
132
  @port ||= mode == :rsync_daemon ? 873 : 22
135
133
  @compress ||= false
136
- @path ||= '~/backups'
134
+ @path ||= "~/backups"
137
135
  end
138
136
 
139
137
  private
@@ -143,10 +141,10 @@ module Backup
143
141
  create_remote_path
144
142
 
145
143
  package.filenames.each do |filename|
146
- src = "'#{ File.join(Config.tmp_path, filename) }'"
147
- dest = "#{ host_options }'#{ File.join(remote_path, filename) }'"
148
- Logger.info "Syncing to #{ dest }..."
149
- run("#{ rsync_command } #{ src } #{ dest }")
144
+ src = "'#{File.join(Config.tmp_path, filename)}'"
145
+ dest = "#{host_options}'#{File.join(remote_path, filename)}'"
146
+ Logger.info "Syncing to #{dest}..."
147
+ run("#{rsync_command} #{src} #{dest}")
150
148
  end
151
149
  ensure
152
150
  remove_password_file
@@ -159,7 +157,7 @@ module Backup
159
157
  def remote_path
160
158
  @remote_path ||= begin
161
159
  if host
162
- path.sub(/^~\//, '').sub(/\/$/, '')
160
+ path.sub(/^~\//, "").sub(/\/$/, "")
163
161
  else
164
162
  File.expand_path(path)
165
163
  end
@@ -176,8 +174,9 @@ module Backup
176
174
  # module name that must define a path on the remote that already exists.
177
175
  def create_remote_path
178
176
  if host
179
- run("#{ utility(:ssh) } #{ ssh_transport_args } #{ host } " +
180
- %Q["mkdir -p '#{ remote_path }'"]) if mode == :ssh
177
+ return unless mode == :ssh
178
+ run "#{utility(:ssh)} #{ssh_transport_args} #{host} " +
179
+ %("mkdir -p '#{remote_path}'")
181
180
  else
182
181
  FileUtils.mkdir_p(remote_path)
183
182
  end
@@ -186,55 +185,55 @@ module Backup
186
185
  def host_options
187
186
  @host_options ||= begin
188
187
  if !host
189
- ''
188
+ ""
190
189
  elsif mode == :ssh
191
- "#{ host }:"
190
+ "#{host}:"
192
191
  else
193
- user = "#{ rsync_user }@" if rsync_user
194
- "#{ user }#{ host }::"
192
+ user = "#{rsync_user}@" if rsync_user
193
+ "#{user}#{host}::"
195
194
  end
196
195
  end
197
196
  end
198
197
 
199
198
  def rsync_command
200
199
  @rsync_command ||= begin
201
- cmd = utility(:rsync) << ' --archive' <<
202
- " #{ Array(additional_rsync_options).join(' ') }".rstrip
200
+ cmd = utility(:rsync) << " --archive" <<
201
+ " #{Array(additional_rsync_options).join(" ")}".rstrip
203
202
  cmd << compress_option << password_option << transport_options if host
204
203
  cmd
205
204
  end
206
205
  end
207
206
 
208
207
  def compress_option
209
- compress ? ' --compress' : ''
208
+ compress ? " --compress" : ""
210
209
  end
211
210
 
212
211
  def password_option
213
- return '' if mode == :ssh
212
+ return "" if mode == :ssh
214
213
 
215
214
  path = @password_file ? @password_file.path : rsync_password_file
216
- path ? " --password-file='#{ File.expand_path(path) }'" : ''
215
+ path ? " --password-file='#{File.expand_path(path)}'" : ""
217
216
  end
218
217
 
219
218
  def transport_options
220
219
  if mode == :rsync_daemon
221
- " --port #{ port }"
220
+ " --port #{port}"
222
221
  else
223
- %Q[ -e "#{ utility(:ssh) } #{ ssh_transport_args }"]
222
+ %( -e "#{utility(:ssh)} #{ssh_transport_args}")
224
223
  end
225
224
  end
226
225
 
227
226
  def ssh_transport_args
228
- args = "-p #{ port } "
229
- args << "-l #{ ssh_user } " if ssh_user
230
- args << Array(additional_ssh_options).join(' ')
227
+ args = "-p #{port} "
228
+ args << "-l #{ssh_user} " if ssh_user
229
+ args << Array(additional_ssh_options).join(" ")
231
230
  args.rstrip
232
231
  end
233
232
 
234
233
  def write_password_file
235
234
  return unless host && rsync_password && mode != :ssh
236
235
 
237
- @password_file = Tempfile.new('backup-rsync-password')
236
+ @password_file = Tempfile.new("backup-rsync-password")
238
237
  @password_file.write(rsync_password)
239
238
  @password_file.close
240
239
  end
@@ -242,7 +241,6 @@ module Backup
242
241
  def remove_password_file
243
242
  @password_file.delete if @password_file
244
243
  end
245
-
246
244
  end
247
245
  end
248
246
  end
@@ -1,5 +1,4 @@
1
- # encoding: utf-8
2
- require 'backup/cloud_io/s3'
1
+ require "backup/cloud_io/s3"
3
2
 
4
3
  module Backup
5
4
  module Storage
@@ -75,10 +74,10 @@ module Backup
75
74
  @chunk_size ||= 5 # MiB
76
75
  @max_retries ||= 10
77
76
  @retry_waitsec ||= 30
78
- @path ||= 'backups'
77
+ @path ||= "backups"
79
78
  @storage_class ||= :standard
80
79
 
81
- @path = @path.sub(/^\//, '')
80
+ @path = @path.sub(/^\//, "")
82
81
 
83
82
  check_configuration
84
83
  end
@@ -87,17 +86,17 @@ module Backup
87
86
 
88
87
  def cloud_io
89
88
  @cloud_io ||= CloudIO::S3.new(
90
- :access_key_id => access_key_id,
91
- :secret_access_key => secret_access_key,
92
- :use_iam_profile => use_iam_profile,
93
- :region => region,
94
- :bucket => bucket,
95
- :encryption => encryption,
96
- :storage_class => storage_class,
97
- :max_retries => max_retries,
98
- :retry_waitsec => retry_waitsec,
99
- :chunk_size => chunk_size,
100
- :fog_options => fog_options
89
+ access_key_id: access_key_id,
90
+ secret_access_key: secret_access_key,
91
+ use_iam_profile: use_iam_profile,
92
+ region: region,
93
+ bucket: bucket,
94
+ encryption: encryption,
95
+ storage_class: storage_class,
96
+ max_retries: max_retries,
97
+ retry_waitsec: retry_waitsec,
98
+ chunk_size: chunk_size,
99
+ fog_options: fog_options
101
100
  )
102
101
  end
103
102
 
@@ -105,7 +104,7 @@ module Backup
105
104
  package.filenames.each do |filename|
106
105
  src = File.join(Config.tmp_path, filename)
107
106
  dest = File.join(remote_path, filename)
108
- Logger.info "Storing '#{ bucket }/#{ dest }'..."
107
+ Logger.info "Storing '#{bucket}/#{dest}'..."
109
108
  cloud_io.upload(src, dest)
110
109
  end
111
110
  end
@@ -113,25 +112,26 @@ module Backup
113
112
  # Called by the Cycler.
114
113
  # Any error raised will be logged as a warning.
115
114
  def remove!(package)
116
- Logger.info "Removing backup package dated #{ package.time }..."
115
+ Logger.info "Removing backup package dated #{package.time}..."
117
116
 
118
117
  remote_path = remote_path_for(package)
119
118
  objects = cloud_io.objects(remote_path)
120
119
 
121
- raise Error, "Package at '#{ remote_path }' not found" if objects.empty?
120
+ raise Error, "Package at '#{remote_path}' not found" if objects.empty?
122
121
 
123
122
  cloud_io.delete(objects)
124
123
  end
125
124
 
126
125
  def check_configuration
127
- if use_iam_profile
128
- required = %w{ bucket }
129
- else
130
- required = %w{ access_key_id secret_access_key bucket }
131
- end
132
- raise Error, <<-EOS if required.map {|name| send(name) }.any?(&:nil?)
126
+ required =
127
+ if use_iam_profile
128
+ %w[bucket]
129
+ else
130
+ %w[access_key_id secret_access_key bucket]
131
+ end
132
+ raise Error, <<-EOS if required.map { |name| send(name) }.any?(&:nil?)
133
133
  Configuration Error
134
- #{ required.map {|name| "##{ name }"}.join(', ') } are all required
134
+ #{required.map { |name| "##{name}" }.join(", ")} are all required
135
135
  EOS
136
136
 
137
137
  raise Error, <<-EOS if chunk_size > 0 && !chunk_size.between?(5, 5120)
@@ -139,18 +139,17 @@ module Backup
139
139
  #chunk_size must be between 5 and 5120 (or 0 to disable multipart)
140
140
  EOS
141
141
 
142
- raise Error, <<-EOS if encryption && encryption.to_s.upcase != 'AES256'
142
+ raise Error, <<-EOS if encryption && encryption.to_s.upcase != "AES256"
143
143
  Configuration Error
144
144
  #encryption must be :aes256 or nil
145
145
  EOS
146
146
 
147
- classes = ['STANDARD', 'STANDARD_IA', 'REDUCED_REDUNDANCY']
147
+ classes = ["STANDARD", "STANDARD_IA", "REDUCED_REDUNDANCY"]
148
148
  raise Error, <<-EOS unless classes.include?(storage_class.to_s.upcase)
149
149
  Configuration Error
150
150
  #storage_class must be :standard or :standard_ia or :reduced_redundancy
151
151
  EOS
152
152
  end
153
-
154
153
  end
155
154
  end
156
155
  end
@@ -1,5 +1,4 @@
1
- # encoding: utf-8
2
- require 'net/scp'
1
+ require "net/scp"
3
2
 
4
3
  module Backup
5
4
  module Storage
@@ -19,27 +18,27 @@ module Backup
19
18
  super
20
19
 
21
20
  @port ||= 22
22
- @path ||= 'backups'
21
+ @path ||= "backups"
23
22
  @ssh_options ||= {}
24
- path.sub!(/^~\//, '')
23
+ path.sub!(/^~\//, "")
25
24
  end
26
25
 
27
26
  private
28
27
 
29
28
  def connection
30
29
  Net::SSH.start(
31
- ip, username, { :password => password, :port => port }.merge(ssh_options)
32
- ) {|ssh| yield ssh }
30
+ ip, username, { password: password, port: port }.merge(ssh_options)
31
+ ) { |ssh| yield ssh }
33
32
  end
34
33
 
35
34
  def transfer!
36
35
  connection do |ssh|
37
- ssh.exec!("mkdir -p '#{ remote_path }'")
36
+ ssh.exec!("mkdir -p '#{remote_path}'")
38
37
 
39
38
  package.filenames.each do |filename|
40
39
  src = File.join(Config.tmp_path, filename)
41
40
  dest = File.join(remote_path, filename)
42
- Logger.info "Storing '#{ ip }:#{ dest }'..."
41
+ Logger.info "Storing '#{ip}:#{dest}'..."
43
42
  ssh.scp.upload!(src, dest)
44
43
  end
45
44
  end
@@ -48,20 +47,19 @@ module Backup
48
47
  # Called by the Cycler.
49
48
  # Any error raised will be logged as a warning.
50
49
  def remove!(package)
51
- Logger.info "Removing backup package dated #{ package.time }..."
50
+ Logger.info "Removing backup package dated #{package.time}..."
52
51
 
53
52
  errors = []
54
53
  connection do |ssh|
55
- ssh.exec!("rm -r '#{ remote_path_for(package) }'") do |ch, stream, data|
54
+ ssh.exec!("rm -r '#{remote_path_for(package)}'") do |_, stream, data|
56
55
  errors << data if stream == :stderr
57
56
  end
58
57
  end
59
58
  unless errors.empty?
60
59
  raise Error, "Net::SSH reported the following errors:\n" +
61
- errors.join("\n")
60
+ errors.join("\n")
62
61
  end
63
62
  end
64
-
65
63
  end
66
64
  end
67
65
  end