backup 4.4.1 → 5.0.0.beta.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (80) hide show
  1. checksums.yaml +5 -5
  2. data/LICENSE +19 -0
  3. data/README.md +1 -1
  4. data/lib/backup.rb +74 -78
  5. data/lib/backup/archive.rb +31 -32
  6. data/lib/backup/binder.rb +2 -6
  7. data/lib/backup/cleaner.rb +14 -18
  8. data/lib/backup/cli.rb +104 -108
  9. data/lib/backup/cloud_io/base.rb +4 -7
  10. data/lib/backup/cloud_io/cloud_files.rb +60 -62
  11. data/lib/backup/cloud_io/s3.rb +69 -76
  12. data/lib/backup/compressor/base.rb +4 -7
  13. data/lib/backup/compressor/bzip2.rb +3 -7
  14. data/lib/backup/compressor/custom.rb +2 -6
  15. data/lib/backup/compressor/gzip.rb +16 -17
  16. data/lib/backup/config.rb +17 -18
  17. data/lib/backup/config/dsl.rb +16 -17
  18. data/lib/backup/config/helpers.rb +10 -16
  19. data/lib/backup/database/base.rb +22 -21
  20. data/lib/backup/database/mongodb.rb +36 -37
  21. data/lib/backup/database/mysql.rb +40 -41
  22. data/lib/backup/database/openldap.rb +8 -10
  23. data/lib/backup/database/postgresql.rb +29 -30
  24. data/lib/backup/database/redis.rb +27 -30
  25. data/lib/backup/database/riak.rb +15 -18
  26. data/lib/backup/database/sqlite.rb +4 -6
  27. data/lib/backup/encryptor/base.rb +2 -4
  28. data/lib/backup/encryptor/gpg.rb +49 -59
  29. data/lib/backup/encryptor/open_ssl.rb +11 -14
  30. data/lib/backup/errors.rb +7 -12
  31. data/lib/backup/logger.rb +16 -18
  32. data/lib/backup/logger/console.rb +5 -8
  33. data/lib/backup/logger/fog_adapter.rb +2 -6
  34. data/lib/backup/logger/logfile.rb +10 -12
  35. data/lib/backup/logger/syslog.rb +2 -4
  36. data/lib/backup/model.rb +75 -40
  37. data/lib/backup/notifier/base.rb +24 -26
  38. data/lib/backup/notifier/campfire.rb +9 -11
  39. data/lib/backup/notifier/command.rb +0 -3
  40. data/lib/backup/notifier/datadog.rb +9 -12
  41. data/lib/backup/notifier/flowdock.rb +13 -17
  42. data/lib/backup/notifier/hipchat.rb +11 -13
  43. data/lib/backup/notifier/http_post.rb +11 -14
  44. data/lib/backup/notifier/mail.rb +44 -47
  45. data/lib/backup/notifier/nagios.rb +5 -9
  46. data/lib/backup/notifier/pagerduty.rb +10 -12
  47. data/lib/backup/notifier/prowl.rb +15 -15
  48. data/lib/backup/notifier/pushover.rb +7 -10
  49. data/lib/backup/notifier/ses.rb +34 -16
  50. data/lib/backup/notifier/slack.rb +39 -40
  51. data/lib/backup/notifier/twitter.rb +2 -5
  52. data/lib/backup/notifier/zabbix.rb +11 -14
  53. data/lib/backup/package.rb +5 -9
  54. data/lib/backup/packager.rb +16 -17
  55. data/lib/backup/pipeline.rb +17 -21
  56. data/lib/backup/splitter.rb +8 -11
  57. data/lib/backup/storage/base.rb +5 -8
  58. data/lib/backup/storage/cloud_files.rb +21 -23
  59. data/lib/backup/storage/cycler.rb +10 -15
  60. data/lib/backup/storage/dropbox.rb +15 -21
  61. data/lib/backup/storage/ftp.rb +8 -10
  62. data/lib/backup/storage/local.rb +5 -8
  63. data/lib/backup/storage/qiniu.rb +8 -8
  64. data/lib/backup/storage/rsync.rb +24 -26
  65. data/lib/backup/storage/s3.rb +27 -28
  66. data/lib/backup/storage/scp.rb +10 -12
  67. data/lib/backup/storage/sftp.rb +10 -12
  68. data/lib/backup/syncer/base.rb +5 -8
  69. data/lib/backup/syncer/cloud/base.rb +27 -30
  70. data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
  71. data/lib/backup/syncer/cloud/local_file.rb +5 -8
  72. data/lib/backup/syncer/cloud/s3.rb +23 -24
  73. data/lib/backup/syncer/rsync/base.rb +6 -10
  74. data/lib/backup/syncer/rsync/local.rb +1 -5
  75. data/lib/backup/syncer/rsync/pull.rb +6 -10
  76. data/lib/backup/syncer/rsync/push.rb +18 -22
  77. data/lib/backup/template.rb +9 -14
  78. data/lib/backup/utilities.rb +82 -69
  79. data/lib/backup/version.rb +1 -3
  80. metadata +100 -660
@@ -1,5 +1,4 @@
1
- # encoding: utf-8
2
- require 'net/ftp'
1
+ require "net/ftp"
3
2
 
4
3
  module Backup
5
4
  module Storage
@@ -31,10 +30,10 @@ module Backup
31
30
  super
32
31
 
33
32
  @port ||= 21
34
- @path ||= 'backups'
33
+ @path ||= "backups"
35
34
  @passive_mode ||= false
36
35
  @timeout ||= nil
37
- path.sub!(/^~\//, '')
36
+ path.sub!(/^~\//, "")
38
37
  end
39
38
 
40
39
  private
@@ -68,7 +67,7 @@ module Backup
68
67
  package.filenames.each do |filename|
69
68
  src = File.join(Config.tmp_path, filename)
70
69
  dest = File.join(remote_path, filename)
71
- Logger.info "Storing '#{ ip }:#{ dest }'..."
70
+ Logger.info "Storing '#{ip}:#{dest}'..."
72
71
  ftp.put(src, dest)
73
72
  end
74
73
  end
@@ -77,7 +76,7 @@ module Backup
77
76
  # Called by the Cycler.
78
77
  # Any error raised will be logged as a warning.
79
78
  def remove!(package)
80
- Logger.info "Removing backup package dated #{ package.time }..."
79
+ Logger.info "Removing backup package dated #{package.time}..."
81
80
 
82
81
  remote_path = remote_path_for(package)
83
82
  connection do |ftp|
@@ -98,15 +97,14 @@ module Backup
98
97
  # Net::FTP raises an exception when the directory it's trying to create
99
98
  # already exists, so we have rescue it
100
99
  def create_remote_path(ftp)
101
- path_parts = Array.new
102
- remote_path.split('/').each do |path_part|
100
+ path_parts = []
101
+ remote_path.split("/").each do |path_part|
103
102
  path_parts << path_part
104
103
  begin
105
- ftp.mkdir(path_parts.join('/'))
104
+ ftp.mkdir(path_parts.join("/"))
106
105
  rescue Net::FTPPermError; end
107
106
  end
108
107
  end
109
-
110
108
  end
111
109
  end
112
110
  end
@@ -1,5 +1,3 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Storage
5
3
  class Local < Base
@@ -9,7 +7,7 @@ module Backup
9
7
  def initialize(model, storage_id = nil)
10
8
  super
11
9
 
12
- @path ||= '~/backups'
10
+ @path ||= "~/backups"
13
11
  end
14
12
 
15
13
  private
@@ -21,7 +19,7 @@ module Backup
21
19
  package.filenames.each do |filename|
22
20
  src = File.join(Config.tmp_path, filename)
23
21
  dest = File.join(remote_path, filename)
24
- Logger.info "Storing '#{ dest }'..."
22
+ Logger.info "Storing '#{dest}'..."
25
23
 
26
24
  FileUtils.send(transfer_method, src, dest)
27
25
  end
@@ -30,7 +28,7 @@ module Backup
30
28
  # Called by the Cycler.
31
29
  # Any error raised will be logged as a warning.
32
30
  def remove!(package)
33
- Logger.info "Removing backup package dated #{ package.time }..."
31
+ Logger.info "Removing backup package dated #{package.time}..."
34
32
 
35
33
  FileUtils.rm_r(remote_path_for(package))
36
34
  end
@@ -50,15 +48,14 @@ module Backup
50
48
  else
51
49
  Logger.warn Error.new(<<-EOS)
52
50
  Local File Copy Warning!
53
- The final backup file(s) for '#{ model.label }' (#{ model.trigger })
54
- will be *copied* to '#{ remote_path }'
51
+ The final backup file(s) for '#{model.label}' (#{model.trigger})
52
+ will be *copied* to '#{remote_path}'
55
53
  To avoid this, when using more than one Storage, the 'Local' Storage
56
54
  should be added *last* so the files may be *moved* to their destination.
57
55
  EOS
58
56
  false
59
57
  end
60
58
  end
61
-
62
59
  end
63
60
  end
64
61
  end
@@ -1,5 +1,4 @@
1
- # encoding: utf-8
2
- require 'qiniu'
1
+ require "qiniu"
3
2
 
4
3
  module Backup
5
4
  module Storage
@@ -18,18 +17,19 @@ module Backup
18
17
  def initialize(model, storage_id = nil)
19
18
  super
20
19
 
21
- @path ||= 'backups'
20
+ @path ||= "backups"
22
21
 
23
22
  check_configuration
24
23
  config_credentials
25
24
  end
26
25
 
27
26
  private
27
+
28
28
  def transfer!
29
29
  package.filenames.each do |filename|
30
30
  src = File.join(Config.tmp_path, filename)
31
31
  dest = File.join(remote_path, filename)
32
- Logger.info "Storing '#{ dest }'..."
32
+ Logger.info "Storing '#{dest}'..."
33
33
 
34
34
  ::Qiniu.upload_file(uptoken: ::Qiniu.generate_upload_token,
35
35
  bucket: bucket,
@@ -41,7 +41,7 @@ module Backup
41
41
  # Called by the Cycler.
42
42
  # Any error raised will be logged as a warning.
43
43
  def remove!(package)
44
- Logger.info "Removing backup package dated #{ package.time }..."
44
+ Logger.info "Removing backup package dated #{package.time}..."
45
45
  remote_path = remote_path_for(package)
46
46
  package.filenames.each do |filename|
47
47
  ::Qiniu.delete(bucket, File.join(remote_path, filename))
@@ -49,11 +49,11 @@ module Backup
49
49
  end
50
50
 
51
51
  def check_configuration
52
- required = %w{ access_key secret_key bucket }
52
+ required = %w[access_key secret_key bucket]
53
53
 
54
- raise Error, <<-EOS if required.map {|name| send(name) }.any?(&:nil?)
54
+ raise Error, <<-EOS if required.map { |name| send(name) }.any?(&:nil?)
55
55
  Configuration Error
56
- #{ required.map {|name| "##{ name }"}.join(', ') } are all required
56
+ #{required.map { |name| "##{name}" }.join(", ")} are all required
57
57
  EOS
58
58
  end
59
59
 
@@ -1,5 +1,3 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Storage
5
3
  class RSync < Base
@@ -133,7 +131,7 @@ module Backup
133
131
  @mode ||= :ssh
134
132
  @port ||= mode == :rsync_daemon ? 873 : 22
135
133
  @compress ||= false
136
- @path ||= '~/backups'
134
+ @path ||= "~/backups"
137
135
  end
138
136
 
139
137
  private
@@ -143,10 +141,10 @@ module Backup
143
141
  create_remote_path
144
142
 
145
143
  package.filenames.each do |filename|
146
- src = "'#{ File.join(Config.tmp_path, filename) }'"
147
- dest = "#{ host_options }'#{ File.join(remote_path, filename) }'"
148
- Logger.info "Syncing to #{ dest }..."
149
- run("#{ rsync_command } #{ src } #{ dest }")
144
+ src = "'#{File.join(Config.tmp_path, filename)}'"
145
+ dest = "#{host_options}'#{File.join(remote_path, filename)}'"
146
+ Logger.info "Syncing to #{dest}..."
147
+ run("#{rsync_command} #{src} #{dest}")
150
148
  end
151
149
  ensure
152
150
  remove_password_file
@@ -159,7 +157,7 @@ module Backup
159
157
  def remote_path
160
158
  @remote_path ||= begin
161
159
  if host
162
- path.sub(/^~\//, '').sub(/\/$/, '')
160
+ path.sub(/^~\//, "").sub(/\/$/, "")
163
161
  else
164
162
  File.expand_path(path)
165
163
  end
@@ -176,8 +174,9 @@ module Backup
176
174
  # module name that must define a path on the remote that already exists.
177
175
  def create_remote_path
178
176
  if host
179
- run("#{ utility(:ssh) } #{ ssh_transport_args } #{ host } " +
180
- %Q["mkdir -p '#{ remote_path }'"]) if mode == :ssh
177
+ return unless mode == :ssh
178
+ run "#{utility(:ssh)} #{ssh_transport_args} #{host} " +
179
+ %("mkdir -p '#{remote_path}'")
181
180
  else
182
181
  FileUtils.mkdir_p(remote_path)
183
182
  end
@@ -186,55 +185,55 @@ module Backup
186
185
  def host_options
187
186
  @host_options ||= begin
188
187
  if !host
189
- ''
188
+ ""
190
189
  elsif mode == :ssh
191
- "#{ host }:"
190
+ "#{host}:"
192
191
  else
193
- user = "#{ rsync_user }@" if rsync_user
194
- "#{ user }#{ host }::"
192
+ user = "#{rsync_user}@" if rsync_user
193
+ "#{user}#{host}::"
195
194
  end
196
195
  end
197
196
  end
198
197
 
199
198
  def rsync_command
200
199
  @rsync_command ||= begin
201
- cmd = utility(:rsync) << ' --archive' <<
202
- " #{ Array(additional_rsync_options).join(' ') }".rstrip
200
+ cmd = utility(:rsync) << " --archive" <<
201
+ " #{Array(additional_rsync_options).join(" ")}".rstrip
203
202
  cmd << compress_option << password_option << transport_options if host
204
203
  cmd
205
204
  end
206
205
  end
207
206
 
208
207
  def compress_option
209
- compress ? ' --compress' : ''
208
+ compress ? " --compress" : ""
210
209
  end
211
210
 
212
211
  def password_option
213
- return '' if mode == :ssh
212
+ return "" if mode == :ssh
214
213
 
215
214
  path = @password_file ? @password_file.path : rsync_password_file
216
- path ? " --password-file='#{ File.expand_path(path) }'" : ''
215
+ path ? " --password-file='#{File.expand_path(path)}'" : ""
217
216
  end
218
217
 
219
218
  def transport_options
220
219
  if mode == :rsync_daemon
221
- " --port #{ port }"
220
+ " --port #{port}"
222
221
  else
223
- %Q[ -e "#{ utility(:ssh) } #{ ssh_transport_args }"]
222
+ %( -e "#{utility(:ssh)} #{ssh_transport_args}")
224
223
  end
225
224
  end
226
225
 
227
226
  def ssh_transport_args
228
- args = "-p #{ port } "
229
- args << "-l #{ ssh_user } " if ssh_user
230
- args << Array(additional_ssh_options).join(' ')
227
+ args = "-p #{port} "
228
+ args << "-l #{ssh_user} " if ssh_user
229
+ args << Array(additional_ssh_options).join(" ")
231
230
  args.rstrip
232
231
  end
233
232
 
234
233
  def write_password_file
235
234
  return unless host && rsync_password && mode != :ssh
236
235
 
237
- @password_file = Tempfile.new('backup-rsync-password')
236
+ @password_file = Tempfile.new("backup-rsync-password")
238
237
  @password_file.write(rsync_password)
239
238
  @password_file.close
240
239
  end
@@ -242,7 +241,6 @@ module Backup
242
241
  def remove_password_file
243
242
  @password_file.delete if @password_file
244
243
  end
245
-
246
244
  end
247
245
  end
248
246
  end
@@ -1,5 +1,4 @@
1
- # encoding: utf-8
2
- require 'backup/cloud_io/s3'
1
+ require "backup/cloud_io/s3"
3
2
 
4
3
  module Backup
5
4
  module Storage
@@ -75,10 +74,10 @@ module Backup
75
74
  @chunk_size ||= 5 # MiB
76
75
  @max_retries ||= 10
77
76
  @retry_waitsec ||= 30
78
- @path ||= 'backups'
77
+ @path ||= "backups"
79
78
  @storage_class ||= :standard
80
79
 
81
- @path = @path.sub(/^\//, '')
80
+ @path = @path.sub(/^\//, "")
82
81
 
83
82
  check_configuration
84
83
  end
@@ -87,17 +86,17 @@ module Backup
87
86
 
88
87
  def cloud_io
89
88
  @cloud_io ||= CloudIO::S3.new(
90
- :access_key_id => access_key_id,
91
- :secret_access_key => secret_access_key,
92
- :use_iam_profile => use_iam_profile,
93
- :region => region,
94
- :bucket => bucket,
95
- :encryption => encryption,
96
- :storage_class => storage_class,
97
- :max_retries => max_retries,
98
- :retry_waitsec => retry_waitsec,
99
- :chunk_size => chunk_size,
100
- :fog_options => fog_options
89
+ access_key_id: access_key_id,
90
+ secret_access_key: secret_access_key,
91
+ use_iam_profile: use_iam_profile,
92
+ region: region,
93
+ bucket: bucket,
94
+ encryption: encryption,
95
+ storage_class: storage_class,
96
+ max_retries: max_retries,
97
+ retry_waitsec: retry_waitsec,
98
+ chunk_size: chunk_size,
99
+ fog_options: fog_options
101
100
  )
102
101
  end
103
102
 
@@ -105,7 +104,7 @@ module Backup
105
104
  package.filenames.each do |filename|
106
105
  src = File.join(Config.tmp_path, filename)
107
106
  dest = File.join(remote_path, filename)
108
- Logger.info "Storing '#{ bucket }/#{ dest }'..."
107
+ Logger.info "Storing '#{bucket}/#{dest}'..."
109
108
  cloud_io.upload(src, dest)
110
109
  end
111
110
  end
@@ -113,25 +112,26 @@ module Backup
113
112
  # Called by the Cycler.
114
113
  # Any error raised will be logged as a warning.
115
114
  def remove!(package)
116
- Logger.info "Removing backup package dated #{ package.time }..."
115
+ Logger.info "Removing backup package dated #{package.time}..."
117
116
 
118
117
  remote_path = remote_path_for(package)
119
118
  objects = cloud_io.objects(remote_path)
120
119
 
121
- raise Error, "Package at '#{ remote_path }' not found" if objects.empty?
120
+ raise Error, "Package at '#{remote_path}' not found" if objects.empty?
122
121
 
123
122
  cloud_io.delete(objects)
124
123
  end
125
124
 
126
125
  def check_configuration
127
- if use_iam_profile
128
- required = %w{ bucket }
129
- else
130
- required = %w{ access_key_id secret_access_key bucket }
131
- end
132
- raise Error, <<-EOS if required.map {|name| send(name) }.any?(&:nil?)
126
+ required =
127
+ if use_iam_profile
128
+ %w[bucket]
129
+ else
130
+ %w[access_key_id secret_access_key bucket]
131
+ end
132
+ raise Error, <<-EOS if required.map { |name| send(name) }.any?(&:nil?)
133
133
  Configuration Error
134
- #{ required.map {|name| "##{ name }"}.join(', ') } are all required
134
+ #{required.map { |name| "##{name}" }.join(", ")} are all required
135
135
  EOS
136
136
 
137
137
  raise Error, <<-EOS if chunk_size > 0 && !chunk_size.between?(5, 5120)
@@ -139,18 +139,17 @@ module Backup
139
139
  #chunk_size must be between 5 and 5120 (or 0 to disable multipart)
140
140
  EOS
141
141
 
142
- raise Error, <<-EOS if encryption && encryption.to_s.upcase != 'AES256'
142
+ raise Error, <<-EOS if encryption && encryption.to_s.upcase != "AES256"
143
143
  Configuration Error
144
144
  #encryption must be :aes256 or nil
145
145
  EOS
146
146
 
147
- classes = ['STANDARD', 'STANDARD_IA', 'REDUCED_REDUNDANCY']
147
+ classes = ["STANDARD", "STANDARD_IA", "REDUCED_REDUNDANCY"]
148
148
  raise Error, <<-EOS unless classes.include?(storage_class.to_s.upcase)
149
149
  Configuration Error
150
150
  #storage_class must be :standard or :standard_ia or :reduced_redundancy
151
151
  EOS
152
152
  end
153
-
154
153
  end
155
154
  end
156
155
  end
@@ -1,5 +1,4 @@
1
- # encoding: utf-8
2
- require 'net/scp'
1
+ require "net/scp"
3
2
 
4
3
  module Backup
5
4
  module Storage
@@ -19,27 +18,27 @@ module Backup
19
18
  super
20
19
 
21
20
  @port ||= 22
22
- @path ||= 'backups'
21
+ @path ||= "backups"
23
22
  @ssh_options ||= {}
24
- path.sub!(/^~\//, '')
23
+ path.sub!(/^~\//, "")
25
24
  end
26
25
 
27
26
  private
28
27
 
29
28
  def connection
30
29
  Net::SSH.start(
31
- ip, username, { :password => password, :port => port }.merge(ssh_options)
32
- ) {|ssh| yield ssh }
30
+ ip, username, { password: password, port: port }.merge(ssh_options)
31
+ ) { |ssh| yield ssh }
33
32
  end
34
33
 
35
34
  def transfer!
36
35
  connection do |ssh|
37
- ssh.exec!("mkdir -p '#{ remote_path }'")
36
+ ssh.exec!("mkdir -p '#{remote_path}'")
38
37
 
39
38
  package.filenames.each do |filename|
40
39
  src = File.join(Config.tmp_path, filename)
41
40
  dest = File.join(remote_path, filename)
42
- Logger.info "Storing '#{ ip }:#{ dest }'..."
41
+ Logger.info "Storing '#{ip}:#{dest}'..."
43
42
  ssh.scp.upload!(src, dest)
44
43
  end
45
44
  end
@@ -48,20 +47,19 @@ module Backup
48
47
  # Called by the Cycler.
49
48
  # Any error raised will be logged as a warning.
50
49
  def remove!(package)
51
- Logger.info "Removing backup package dated #{ package.time }..."
50
+ Logger.info "Removing backup package dated #{package.time}..."
52
51
 
53
52
  errors = []
54
53
  connection do |ssh|
55
- ssh.exec!("rm -r '#{ remote_path_for(package) }'") do |ch, stream, data|
54
+ ssh.exec!("rm -r '#{remote_path_for(package)}'") do |_, stream, data|
56
55
  errors << data if stream == :stderr
57
56
  end
58
57
  end
59
58
  unless errors.empty?
60
59
  raise Error, "Net::SSH reported the following errors:\n" +
61
- errors.join("\n")
60
+ errors.join("\n")
62
61
  end
63
62
  end
64
-
65
63
  end
66
64
  end
67
65
  end