backup 4.3.0 → 5.0.0.beta.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (81) hide show
  1. checksums.yaml +5 -5
  2. data/LICENSE +19 -0
  3. data/README.md +13 -9
  4. data/bin/docker_test +24 -0
  5. data/lib/backup.rb +74 -78
  6. data/lib/backup/archive.rb +31 -32
  7. data/lib/backup/binder.rb +2 -6
  8. data/lib/backup/cleaner.rb +14 -18
  9. data/lib/backup/cli.rb +104 -108
  10. data/lib/backup/cloud_io/base.rb +4 -7
  11. data/lib/backup/cloud_io/cloud_files.rb +60 -62
  12. data/lib/backup/cloud_io/s3.rb +69 -76
  13. data/lib/backup/compressor/base.rb +4 -7
  14. data/lib/backup/compressor/bzip2.rb +3 -7
  15. data/lib/backup/compressor/custom.rb +2 -6
  16. data/lib/backup/compressor/gzip.rb +16 -17
  17. data/lib/backup/config.rb +17 -18
  18. data/lib/backup/config/dsl.rb +16 -17
  19. data/lib/backup/config/helpers.rb +10 -16
  20. data/lib/backup/database/base.rb +22 -21
  21. data/lib/backup/database/mongodb.rb +36 -37
  22. data/lib/backup/database/mysql.rb +40 -41
  23. data/lib/backup/database/openldap.rb +8 -10
  24. data/lib/backup/database/postgresql.rb +29 -30
  25. data/lib/backup/database/redis.rb +27 -30
  26. data/lib/backup/database/riak.rb +15 -18
  27. data/lib/backup/database/sqlite.rb +4 -6
  28. data/lib/backup/encryptor/base.rb +2 -4
  29. data/lib/backup/encryptor/gpg.rb +49 -59
  30. data/lib/backup/encryptor/open_ssl.rb +11 -14
  31. data/lib/backup/errors.rb +7 -12
  32. data/lib/backup/logger.rb +16 -18
  33. data/lib/backup/logger/console.rb +5 -8
  34. data/lib/backup/logger/fog_adapter.rb +2 -6
  35. data/lib/backup/logger/logfile.rb +10 -12
  36. data/lib/backup/logger/syslog.rb +2 -4
  37. data/lib/backup/model.rb +33 -40
  38. data/lib/backup/notifier/base.rb +24 -26
  39. data/lib/backup/notifier/campfire.rb +9 -11
  40. data/lib/backup/notifier/command.rb +3 -3
  41. data/lib/backup/notifier/datadog.rb +9 -12
  42. data/lib/backup/notifier/flowdock.rb +13 -17
  43. data/lib/backup/notifier/hipchat.rb +18 -14
  44. data/lib/backup/notifier/http_post.rb +11 -14
  45. data/lib/backup/notifier/mail.rb +42 -54
  46. data/lib/backup/notifier/nagios.rb +5 -9
  47. data/lib/backup/notifier/pagerduty.rb +10 -12
  48. data/lib/backup/notifier/prowl.rb +15 -15
  49. data/lib/backup/notifier/pushover.rb +7 -10
  50. data/lib/backup/notifier/ses.rb +52 -17
  51. data/lib/backup/notifier/slack.rb +39 -40
  52. data/lib/backup/notifier/twitter.rb +2 -5
  53. data/lib/backup/notifier/zabbix.rb +11 -14
  54. data/lib/backup/package.rb +5 -9
  55. data/lib/backup/packager.rb +16 -17
  56. data/lib/backup/pipeline.rb +17 -21
  57. data/lib/backup/splitter.rb +8 -11
  58. data/lib/backup/storage/base.rb +5 -8
  59. data/lib/backup/storage/cloud_files.rb +21 -23
  60. data/lib/backup/storage/cycler.rb +10 -15
  61. data/lib/backup/storage/dropbox.rb +15 -21
  62. data/lib/backup/storage/ftp.rb +14 -10
  63. data/lib/backup/storage/local.rb +5 -8
  64. data/lib/backup/storage/qiniu.rb +8 -8
  65. data/lib/backup/storage/rsync.rb +24 -26
  66. data/lib/backup/storage/s3.rb +27 -28
  67. data/lib/backup/storage/scp.rb +10 -12
  68. data/lib/backup/storage/sftp.rb +10 -12
  69. data/lib/backup/syncer/base.rb +5 -8
  70. data/lib/backup/syncer/cloud/base.rb +27 -30
  71. data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
  72. data/lib/backup/syncer/cloud/local_file.rb +5 -8
  73. data/lib/backup/syncer/cloud/s3.rb +23 -24
  74. data/lib/backup/syncer/rsync/base.rb +6 -10
  75. data/lib/backup/syncer/rsync/local.rb +1 -5
  76. data/lib/backup/syncer/rsync/pull.rb +6 -10
  77. data/lib/backup/syncer/rsync/push.rb +18 -22
  78. data/lib/backup/template.rb +9 -14
  79. data/lib/backup/utilities.rb +78 -69
  80. data/lib/backup/version.rb +1 -3
  81. metadata +107 -677
@@ -1,10 +1,7 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Syncer
5
3
  module RSync
6
4
  class Base < Syncer::Base
7
-
8
5
  ##
9
6
  # Additional String or Array of options for the rsync cli
10
7
  attr_accessor :additional_rsync_options
@@ -14,7 +11,7 @@ module Backup
14
11
  super
15
12
  instance_eval(&block) if block_given?
16
13
 
17
- @path ||= '~/backups'
14
+ @path ||= "~/backups"
18
15
  @archive = @archive.nil? ? true : @archive
19
16
  end
20
17
 
@@ -24,19 +21,19 @@ module Backup
24
21
  # Common base command for Local/Push/Pull
25
22
  def rsync_command
26
23
  utility(:rsync) << archive_option << mirror_option << exclude_option <<
27
- " #{ Array(additional_rsync_options).join(' ') }".rstrip
24
+ " #{Array(additional_rsync_options).join(" ")}".rstrip
28
25
  end
29
26
 
30
27
  def mirror_option
31
- mirror ? ' --delete' : ''
28
+ mirror ? " --delete" : ""
32
29
  end
33
30
 
34
31
  def archive_option
35
- archive ? ' --archive' : ''
32
+ archive ? " --archive" : ""
36
33
  end
37
34
 
38
35
  def exclude_option
39
- excludes.map {|pattern| " --exclude='#{ pattern }'" }.join
36
+ excludes.map { |pattern| " --exclude='#{pattern}'" }.join
40
37
  end
41
38
 
42
39
  ##
@@ -45,9 +42,8 @@ module Backup
45
42
  # each path, as we don't want rsync's "trailing / on source directories"
46
43
  # behavior. This method is used by RSync::Local and RSync::Push.
47
44
  def paths_to_push
48
- directories.map {|dir| "'#{ File.expand_path(dir) }'" }.join(' ')
45
+ directories.map { |dir| "'#{File.expand_path(dir)}'" }.join(" ")
49
46
  end
50
-
51
47
  end
52
48
  end
53
49
  end
@@ -1,15 +1,12 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Syncer
5
3
  module RSync
6
4
  class Local < Base
7
-
8
5
  def perform!
9
6
  log!(:started)
10
7
 
11
8
  create_dest_path!
12
- run("#{ rsync_command } #{ paths_to_push } '#{ dest_path }'")
9
+ run("#{rsync_command} #{paths_to_push} '#{dest_path}'")
13
10
 
14
11
  log!(:finished)
15
12
  end
@@ -24,7 +21,6 @@ module Backup
24
21
  def create_dest_path!
25
22
  FileUtils.mkdir_p dest_path
26
23
  end
27
-
28
24
  end
29
25
  end
30
26
  end
@@ -1,17 +1,14 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Syncer
5
3
  module RSync
6
4
  class Pull < Push
7
-
8
5
  def perform!
9
6
  log!(:started)
10
7
  write_password_file!
11
8
 
12
9
  create_dest_path!
13
- run("#{ rsync_command } #{ host_options }#{ paths_to_pull } " +
14
- "'#{ dest_path }'")
10
+ run("#{rsync_command} #{host_options}#{paths_to_pull} " \
11
+ "'#{dest_path}'")
15
12
 
16
13
  log!(:finished)
17
14
  ensure
@@ -30,10 +27,10 @@ module Backup
30
27
  # Also remove any trailing `/`, since we don't want rsync's
31
28
  # "trailing / on source directories" behavior.
32
29
  def paths_to_pull
33
- sep = mode == :ssh ? ':' : '::'
34
- directories.map {|dir|
35
- "#{ sep }'#{ dir.sub(/^~\//, '').sub(/\/$/, '') }'"
36
- }.join(' ').sub(/^#{ sep }/, '')
30
+ sep = mode == :ssh ? ":" : "::"
31
+ directories.map do |dir|
32
+ "#{sep}'#{dir.sub(/^~\//, "").sub(/\/$/, "")}'"
33
+ end.join(" ").sub(/^#{ sep }/, "")
37
34
  end
38
35
 
39
36
  # Expand path, since this is local and shell-quoted.
@@ -44,7 +41,6 @@ module Backup
44
41
  def create_dest_path!
45
42
  FileUtils.mkdir_p dest_path
46
43
  end
47
-
48
44
  end
49
45
  end
50
46
  end
@@ -1,10 +1,7 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Syncer
5
3
  module RSync
6
4
  class Push < Base
7
-
8
5
  ##
9
6
  # Mode of operation
10
7
  #
@@ -114,8 +111,8 @@ module Backup
114
111
  write_password_file!
115
112
 
116
113
  create_dest_path!
117
- run("#{ rsync_command } #{ paths_to_push } " +
118
- "#{ host_options }'#{ dest_path }'")
114
+ run("#{rsync_command} #{paths_to_push} " \
115
+ "#{host_options}'#{dest_path}'")
119
116
 
120
117
  log!(:finished)
121
118
  ensure
@@ -128,7 +125,7 @@ module Backup
128
125
  # Remove any preceeding '~/', since this is on the remote,
129
126
  # and remove any trailing `/`.
130
127
  def dest_path
131
- @dest_path ||= path.sub(/^~\//, '').sub(/\/$/, '')
128
+ @dest_path ||= path.sub(/^~\//, "").sub(/\/$/, "")
132
129
  end
133
130
 
134
131
  ##
@@ -137,10 +134,10 @@ module Backup
137
134
  # call 'mkdir' without the '-p' option. This is only applicable in :ssh
138
135
  # mode, and only used if the path would require this.
139
136
  def create_dest_path!
140
- return unless mode == :ssh && dest_path.index('/').to_i > 0
137
+ return unless mode == :ssh && dest_path.index("/").to_i > 0
141
138
 
142
- run "#{ utility(:ssh) } #{ ssh_transport_args } #{ host } " +
143
- %Q["mkdir -p '#{ dest_path }'"]
139
+ run "#{utility(:ssh)} #{ssh_transport_args} #{host} " +
140
+ %("mkdir -p '#{dest_path}'")
144
141
  end
145
142
 
146
143
  ##
@@ -148,10 +145,10 @@ module Backup
148
145
  # For Pull, this will prepend the first path in #paths_to_pull.
149
146
  def host_options
150
147
  if mode == :ssh
151
- "#{ host }:"
148
+ "#{host}:"
152
149
  else
153
- user = "#{ rsync_user }@" if rsync_user
154
- "#{ user }#{ host }::"
150
+ user = "#{rsync_user}@" if rsync_user
151
+ "#{user}#{host}::"
155
152
  end
156
153
  end
157
154
 
@@ -162,35 +159,35 @@ module Backup
162
159
  end
163
160
 
164
161
  def compress_option
165
- compress ? ' --compress' : ''
162
+ compress ? " --compress" : ""
166
163
  end
167
164
 
168
165
  def password_option
169
- return '' if mode == :ssh
166
+ return "" if mode == :ssh
170
167
 
171
168
  path = @password_file ? @password_file.path : rsync_password_file
172
- path ? " --password-file='#{ File.expand_path(path) }'" : ''
169
+ path ? " --password-file='#{File.expand_path(path)}'" : ""
173
170
  end
174
171
 
175
172
  def transport_options
176
173
  if mode == :rsync_daemon
177
- " --port #{ port }"
174
+ " --port #{port}"
178
175
  else
179
- %Q[ -e "#{ utility(:ssh) } #{ ssh_transport_args }"]
176
+ %( -e "#{utility(:ssh)} #{ssh_transport_args}")
180
177
  end
181
178
  end
182
179
 
183
180
  def ssh_transport_args
184
- args = "-p #{ port } "
185
- args << "-l #{ ssh_user } " if ssh_user
186
- args << Array(additional_ssh_options).join(' ')
181
+ args = "-p #{port} "
182
+ args << "-l #{ssh_user} " if ssh_user
183
+ args << Array(additional_ssh_options).join(" ")
187
184
  args.rstrip
188
185
  end
189
186
 
190
187
  def write_password_file!
191
188
  return unless rsync_password && mode != :ssh
192
189
 
193
- @password_file = Tempfile.new('backup-rsync-password')
190
+ @password_file = Tempfile.new("backup-rsync-password")
194
191
  @password_file.write(rsync_password)
195
192
  @password_file.close
196
193
  end
@@ -198,7 +195,6 @@ module Backup
198
195
  def remove_password_file!
199
196
  @password_file.delete if @password_file
200
197
  end
201
-
202
198
  end
203
199
  end
204
200
  end
@@ -1,10 +1,7 @@
1
- # encoding: utf-8
2
-
3
- require 'erb'
1
+ require "erb"
4
2
 
5
3
  module Backup
6
4
  class Template
7
-
8
5
  # Holds a binding object. Nil if not provided.
9
6
  attr_accessor :binding
10
7
 
@@ -12,13 +9,12 @@ module Backup
12
9
  # Creates a new instance of the Backup::Template class
13
10
  # and optionally takes an argument that can be either a binding object, a Hash or nil
14
11
  def initialize(object = nil)
15
- if object.is_a?(Binding)
16
- @binding = object
17
- elsif object.is_a?(Hash)
18
- @binding = Backup::Binder.new(object).get_binding
19
- else
20
- @binding = nil
21
- end
12
+ @binding =
13
+ if object.is_a?(Binding)
14
+ object
15
+ elsif object.is_a?(Hash)
16
+ Backup::Binder.new(object).get_binding
17
+ end
22
18
  end
23
19
 
24
20
  ##
@@ -30,10 +26,10 @@ module Backup
30
26
  ##
31
27
  # Returns a String object containing the contents of the file (in the context of the binding if any)
32
28
  def result(file)
33
- ERB.new(file_contents(file), nil, '<>').result(binding)
29
+ ERB.new(file_contents(file), nil, "<>").result(binding)
34
30
  end
35
31
 
36
- private
32
+ private
37
33
 
38
34
  ##
39
35
  # Reads and returns the contents of the provided file path,
@@ -41,6 +37,5 @@ module Backup
41
37
  def file_contents(file)
42
38
  File.read(File.join(Backup::TEMPLATE_PATH, file))
43
39
  end
44
-
45
40
  end
46
41
  end
@@ -1,11 +1,8 @@
1
- # encoding: utf-8
2
-
3
1
  module Backup
4
2
  module Utilities
5
3
  class Error < Backup::Error; end
6
4
 
7
- UTILITY = {}
8
- NAMES = %w{
5
+ UTILITIES_NAMES = %w[
9
6
  tar cat split sudo chown hostname
10
7
  gzip bzip2
11
8
  mongo mongodump mysqldump innobackupex
@@ -15,34 +12,38 @@ module Backup
15
12
  sendmail exim
16
13
  send_nsca
17
14
  zabbix_sender
18
- }
19
-
20
- module DSL
21
- class << self
22
- ##
23
- # Allow users to set the path for all utilities in the .configure block.
24
- #
25
- # Utility names with dashes ('redis-cli') will be set using method calls
26
- # with an underscore ('redis_cli').
27
- NAMES.each do |name|
28
- define_method name.gsub('-', '_'), lambda {|val|
29
- path = File.expand_path(val)
30
- unless File.executable?(path)
31
- raise Utilities::Error, <<-EOS
32
- The path given for '#{ name }' was not found or not executable.
33
- Path was: #{ path }
34
- EOS
35
- end
36
- UTILITY[name] = path
37
- }
38
- end
15
+ ].freeze
16
+
17
+ # @api private
18
+ class DSL
19
+ def initialize(utils)
20
+ @utilities = utils
21
+ end
22
+
23
+ # Helper methods to allow users to set the path for all utilities in the
24
+ # .configure block.
25
+ #
26
+ # Utility names with dashes (`redis-cli`) will be set using method calls
27
+ # with an underscore (`redis_cli`).
28
+ UTILITIES_NAMES.each do |util_name|
29
+ define_method util_name.tr("-", "_") do |raw_path|
30
+ path = File.expand_path(raw_path)
31
+
32
+ unless File.executable?(path)
33
+ raise Utilities::Error, <<-EOS
34
+ The path given for '#{util_name}' was not found or not executable.
35
+ Path was: #{path}
36
+ EOS
37
+ end
39
38
 
40
- ##
41
- # Allow users to set the +tar+ distribution if needed. (:gnu or :bsd)
42
- def tar_dist(val)
43
- Utilities.tar_dist(val)
39
+ @utilities.utilities[util_name] = path
44
40
  end
45
41
  end
42
+
43
+ # Allow users to set the +tar+ distribution if needed. (:gnu or :bsd)
44
+ def tar_dist(val)
45
+ Utilities.tar_dist(val)
46
+ end
46
47
  end
47
48
 
48
49
  class << self
@@ -102,7 +103,7 @@ module Backup
102
103
  # These paths may be set using absolute paths, or relative to the
103
104
  # working directory when Backup is run.
104
105
  def configure(&block)
105
- DSL.instance_eval(&block)
106
+ DSL.new(self).instance_eval(&block)
106
107
  end
107
108
 
108
109
  def tar_dist(val)
@@ -112,7 +113,11 @@ module Backup
112
113
 
113
114
  def gnu_tar?
114
115
  return @gnu_tar unless @gnu_tar.nil?
115
- @gnu_tar = !!run("#{ utility(:tar) } --version").match(/GNU/)
116
+ @gnu_tar = !!run("#{utility(:tar)} --version").match(/GNU/)
117
+ end
118
+
119
+ def utilities
120
+ @utilities ||= {}
116
121
  end
117
122
 
118
123
  private
@@ -122,17 +127,17 @@ module Backup
122
127
  # Raises an error if utility can not be found in the system's $PATH
123
128
  def utility(name)
124
129
  name = name.to_s.strip
125
- raise Error, 'Utility Name Empty' if name.empty?
130
+ raise Error, "Utility Name Empty" if name.empty?
126
131
 
127
- UTILITY[name] ||= %x[which '#{ name }' 2>/dev/null].chomp
128
- raise Error, <<-EOS if UTILITY[name].empty?
129
- Could not locate '#{ name }'.
132
+ utilities[name] ||= `which '#{name}' 2>/dev/null`.chomp
133
+ raise Error, <<-EOS if utilities[name].empty?
134
+ Could not locate '#{name}'.
130
135
  Make sure the specified utility is installed
131
136
  and available in your system's $PATH, or specify it's location
132
137
  in your 'config.rb' file using Backup::Utilities.configure
133
138
  EOS
134
139
 
135
- UTILITY[name].dup
140
+ utilities[name].dup
136
141
  end
137
142
 
138
143
  ##
@@ -140,21 +145,21 @@ module Backup
140
145
  # This is only used to simplify log messages.
141
146
  def command_name(command)
142
147
  parts = []
143
- command = command.split(' ')
144
- command.shift while command[0].to_s.include?('=')
145
- parts << command.shift.split('/')[-1]
146
- if parts[0] == 'sudo'
148
+ command = command.split(" ")
149
+ command.shift while command[0].to_s.include?("=")
150
+ parts << command.shift.split("/")[-1]
151
+ if parts[0] == "sudo"
147
152
  until command.empty?
148
153
  part = command.shift
149
- if part.include?('/')
150
- parts << part.split('/')[-1]
154
+ if part.include?("/")
155
+ parts << part.split("/")[-1]
151
156
  break
152
157
  else
153
158
  parts << part
154
159
  end
155
160
  end
156
161
  end
157
- parts.join(' ')
162
+ parts.join(" ")
158
163
  end
159
164
 
160
165
  ##
@@ -169,43 +174,41 @@ module Backup
169
174
  # Returns STDOUT
170
175
  def run(command)
171
176
  name = command_name(command)
172
- Logger.info "Running system utility '#{ name }'..."
177
+ Logger.info "Running system utility '#{name}'..."
173
178
 
174
179
  begin
175
- out, err = '', ''
176
- ps = Open4.popen4(command) do |pid, stdin, stdout, stderr|
180
+ out = ""
181
+ err = ""
182
+ ps = Open4.popen4(command) do |_pid, stdin, stdout, stderr|
177
183
  stdin.close
178
- out, err = stdout.read.strip, stderr.read.strip
184
+ out = stdout.read.strip
185
+ err = stderr.read.strip
179
186
  end
180
187
  rescue Exception => e
181
- raise Error.wrap(e, "Failed to execute '#{ name }'")
188
+ raise Error.wrap(e, "Failed to execute '#{name}'")
182
189
  end
183
190
 
184
- if ps.success?
185
- unless out.empty?
186
- Logger.info(
187
- out.lines.map {|line| "#{ name }:STDOUT: #{ line }" }.join
188
- )
189
- end
190
-
191
- unless err.empty?
192
- Logger.warn(
193
- err.lines.map {|line| "#{ name }:STDERR: #{ line }" }.join
194
- )
195
- end
196
-
197
- return out
198
- else
191
+ unless ps.success?
199
192
  raise Error, <<-EOS
200
- '#{ name }' failed with exit status: #{ ps.exitstatus }
201
- STDOUT Messages: #{ out.empty? ? 'None' : "\n#{ out }" }
202
- STDERR Messages: #{ err.empty? ? 'None' : "\n#{ err }" }
193
+ '#{name}' failed with exit status: #{ps.exitstatus}
194
+ STDOUT Messages: #{out.empty? ? "None" : "\n#{out}"}
195
+ STDERR Messages: #{err.empty? ? "None" : "\n#{err}"}
203
196
  EOS
204
197
  end
198
+
199
+ unless out.empty?
200
+ Logger.info(out.lines.map { |line| "#{name}:STDOUT: #{line}" }.join)
201
+ end
202
+
203
+ unless err.empty?
204
+ Logger.warn(err.lines.map { |line| "#{name}:STDERR: #{line}" }.join)
205
+ end
206
+
207
+ out
205
208
  end
206
209
 
207
210
  def reset!
208
- UTILITY.clear
211
+ utilities.clear
209
212
  @gnu_tar = nil
210
213
  end
211
214
  end
@@ -214,11 +217,17 @@ module Backup
214
217
  # while allowing them to be stubbed in spec_helper for all specs.
215
218
  module Helpers
216
219
  [:utility, :command_name, :run].each do |name|
217
- define_method name, lambda {|arg| Utilities.send(name, arg) }
220
+ define_method name do |arg|
221
+ Utilities.send(name, arg)
222
+ end
218
223
  private name
219
224
  end
225
+
220
226
  private
221
- def gnu_tar?; Utilities.gnu_tar?; end
227
+
228
+ def gnu_tar?
229
+ Utilities.gnu_tar?
230
+ end
222
231
  end
223
232
  end
224
233
  end