backup 4.4.1 → 5.0.0.beta.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/LICENSE +19 -0
- data/README.md +1 -1
- data/lib/backup.rb +74 -78
- data/lib/backup/archive.rb +31 -32
- data/lib/backup/binder.rb +2 -6
- data/lib/backup/cleaner.rb +14 -18
- data/lib/backup/cli.rb +104 -108
- data/lib/backup/cloud_io/base.rb +4 -7
- data/lib/backup/cloud_io/cloud_files.rb +60 -62
- data/lib/backup/cloud_io/s3.rb +69 -76
- data/lib/backup/compressor/base.rb +4 -7
- data/lib/backup/compressor/bzip2.rb +3 -7
- data/lib/backup/compressor/custom.rb +2 -6
- data/lib/backup/compressor/gzip.rb +16 -17
- data/lib/backup/config.rb +17 -18
- data/lib/backup/config/dsl.rb +16 -17
- data/lib/backup/config/helpers.rb +10 -16
- data/lib/backup/database/base.rb +22 -21
- data/lib/backup/database/mongodb.rb +36 -37
- data/lib/backup/database/mysql.rb +40 -41
- data/lib/backup/database/openldap.rb +8 -10
- data/lib/backup/database/postgresql.rb +29 -30
- data/lib/backup/database/redis.rb +27 -30
- data/lib/backup/database/riak.rb +15 -18
- data/lib/backup/database/sqlite.rb +4 -6
- data/lib/backup/encryptor/base.rb +2 -4
- data/lib/backup/encryptor/gpg.rb +49 -59
- data/lib/backup/encryptor/open_ssl.rb +11 -14
- data/lib/backup/errors.rb +7 -12
- data/lib/backup/logger.rb +16 -18
- data/lib/backup/logger/console.rb +5 -8
- data/lib/backup/logger/fog_adapter.rb +2 -6
- data/lib/backup/logger/logfile.rb +10 -12
- data/lib/backup/logger/syslog.rb +2 -4
- data/lib/backup/model.rb +75 -40
- data/lib/backup/notifier/base.rb +24 -26
- data/lib/backup/notifier/campfire.rb +9 -11
- data/lib/backup/notifier/command.rb +0 -3
- data/lib/backup/notifier/datadog.rb +9 -12
- data/lib/backup/notifier/flowdock.rb +13 -17
- data/lib/backup/notifier/hipchat.rb +11 -13
- data/lib/backup/notifier/http_post.rb +11 -14
- data/lib/backup/notifier/mail.rb +44 -47
- data/lib/backup/notifier/nagios.rb +5 -9
- data/lib/backup/notifier/pagerduty.rb +10 -12
- data/lib/backup/notifier/prowl.rb +15 -15
- data/lib/backup/notifier/pushover.rb +7 -10
- data/lib/backup/notifier/ses.rb +34 -16
- data/lib/backup/notifier/slack.rb +39 -40
- data/lib/backup/notifier/twitter.rb +2 -5
- data/lib/backup/notifier/zabbix.rb +11 -14
- data/lib/backup/package.rb +5 -9
- data/lib/backup/packager.rb +16 -17
- data/lib/backup/pipeline.rb +17 -21
- data/lib/backup/splitter.rb +8 -11
- data/lib/backup/storage/base.rb +5 -8
- data/lib/backup/storage/cloud_files.rb +21 -23
- data/lib/backup/storage/cycler.rb +10 -15
- data/lib/backup/storage/dropbox.rb +15 -21
- data/lib/backup/storage/ftp.rb +8 -10
- data/lib/backup/storage/local.rb +5 -8
- data/lib/backup/storage/qiniu.rb +8 -8
- data/lib/backup/storage/rsync.rb +24 -26
- data/lib/backup/storage/s3.rb +27 -28
- data/lib/backup/storage/scp.rb +10 -12
- data/lib/backup/storage/sftp.rb +10 -12
- data/lib/backup/syncer/base.rb +5 -8
- data/lib/backup/syncer/cloud/base.rb +27 -30
- data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
- data/lib/backup/syncer/cloud/local_file.rb +5 -8
- data/lib/backup/syncer/cloud/s3.rb +23 -24
- data/lib/backup/syncer/rsync/base.rb +6 -10
- data/lib/backup/syncer/rsync/local.rb +1 -5
- data/lib/backup/syncer/rsync/pull.rb +6 -10
- data/lib/backup/syncer/rsync/push.rb +18 -22
- data/lib/backup/template.rb +9 -14
- data/lib/backup/utilities.rb +82 -69
- data/lib/backup/version.rb +1 -3
- metadata +100 -660
@@ -1,15 +1,12 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Syncer
|
5
3
|
module RSync
|
6
4
|
class Local < Base
|
7
|
-
|
8
5
|
def perform!
|
9
6
|
log!(:started)
|
10
7
|
|
11
8
|
create_dest_path!
|
12
|
-
run("#{
|
9
|
+
run("#{rsync_command} #{paths_to_push} '#{dest_path}'")
|
13
10
|
|
14
11
|
log!(:finished)
|
15
12
|
end
|
@@ -24,7 +21,6 @@ module Backup
|
|
24
21
|
def create_dest_path!
|
25
22
|
FileUtils.mkdir_p dest_path
|
26
23
|
end
|
27
|
-
|
28
24
|
end
|
29
25
|
end
|
30
26
|
end
|
@@ -1,17 +1,14 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Syncer
|
5
3
|
module RSync
|
6
4
|
class Pull < Push
|
7
|
-
|
8
5
|
def perform!
|
9
6
|
log!(:started)
|
10
7
|
write_password_file!
|
11
8
|
|
12
9
|
create_dest_path!
|
13
|
-
run("#{
|
14
|
-
"'#{
|
10
|
+
run("#{rsync_command} #{host_options}#{paths_to_pull} " \
|
11
|
+
"'#{dest_path}'")
|
15
12
|
|
16
13
|
log!(:finished)
|
17
14
|
ensure
|
@@ -30,10 +27,10 @@ module Backup
|
|
30
27
|
# Also remove any trailing `/`, since we don't want rsync's
|
31
28
|
# "trailing / on source directories" behavior.
|
32
29
|
def paths_to_pull
|
33
|
-
sep = mode == :ssh ?
|
34
|
-
directories.map
|
35
|
-
"#{
|
36
|
-
|
30
|
+
sep = mode == :ssh ? ":" : "::"
|
31
|
+
directories.map do |dir|
|
32
|
+
"#{sep}'#{dir.sub(/^~\//, "").sub(/\/$/, "")}'"
|
33
|
+
end.join(" ").sub(/^#{ sep }/, "")
|
37
34
|
end
|
38
35
|
|
39
36
|
# Expand path, since this is local and shell-quoted.
|
@@ -44,7 +41,6 @@ module Backup
|
|
44
41
|
def create_dest_path!
|
45
42
|
FileUtils.mkdir_p dest_path
|
46
43
|
end
|
47
|
-
|
48
44
|
end
|
49
45
|
end
|
50
46
|
end
|
@@ -1,10 +1,7 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Syncer
|
5
3
|
module RSync
|
6
4
|
class Push < Base
|
7
|
-
|
8
5
|
##
|
9
6
|
# Mode of operation
|
10
7
|
#
|
@@ -114,8 +111,8 @@ module Backup
|
|
114
111
|
write_password_file!
|
115
112
|
|
116
113
|
create_dest_path!
|
117
|
-
run("#{
|
118
|
-
"#{
|
114
|
+
run("#{rsync_command} #{paths_to_push} " \
|
115
|
+
"#{host_options}'#{dest_path}'")
|
119
116
|
|
120
117
|
log!(:finished)
|
121
118
|
ensure
|
@@ -128,7 +125,7 @@ module Backup
|
|
128
125
|
# Remove any preceeding '~/', since this is on the remote,
|
129
126
|
# and remove any trailing `/`.
|
130
127
|
def dest_path
|
131
|
-
@dest_path ||= path.sub(/^~\//,
|
128
|
+
@dest_path ||= path.sub(/^~\//, "").sub(/\/$/, "")
|
132
129
|
end
|
133
130
|
|
134
131
|
##
|
@@ -137,10 +134,10 @@ module Backup
|
|
137
134
|
# call 'mkdir' without the '-p' option. This is only applicable in :ssh
|
138
135
|
# mode, and only used if the path would require this.
|
139
136
|
def create_dest_path!
|
140
|
-
return unless mode == :ssh && dest_path.index(
|
137
|
+
return unless mode == :ssh && dest_path.index("/").to_i > 0
|
141
138
|
|
142
|
-
run "#{
|
143
|
-
|
139
|
+
run "#{utility(:ssh)} #{ssh_transport_args} #{host} " +
|
140
|
+
%("mkdir -p '#{dest_path}'")
|
144
141
|
end
|
145
142
|
|
146
143
|
##
|
@@ -148,10 +145,10 @@ module Backup
|
|
148
145
|
# For Pull, this will prepend the first path in #paths_to_pull.
|
149
146
|
def host_options
|
150
147
|
if mode == :ssh
|
151
|
-
"#{
|
148
|
+
"#{host}:"
|
152
149
|
else
|
153
|
-
user = "#{
|
154
|
-
"#{
|
150
|
+
user = "#{rsync_user}@" if rsync_user
|
151
|
+
"#{user}#{host}::"
|
155
152
|
end
|
156
153
|
end
|
157
154
|
|
@@ -162,35 +159,35 @@ module Backup
|
|
162
159
|
end
|
163
160
|
|
164
161
|
def compress_option
|
165
|
-
compress ?
|
162
|
+
compress ? " --compress" : ""
|
166
163
|
end
|
167
164
|
|
168
165
|
def password_option
|
169
|
-
return
|
166
|
+
return "" if mode == :ssh
|
170
167
|
|
171
168
|
path = @password_file ? @password_file.path : rsync_password_file
|
172
|
-
path ? " --password-file='#{
|
169
|
+
path ? " --password-file='#{File.expand_path(path)}'" : ""
|
173
170
|
end
|
174
171
|
|
175
172
|
def transport_options
|
176
173
|
if mode == :rsync_daemon
|
177
|
-
" --port #{
|
174
|
+
" --port #{port}"
|
178
175
|
else
|
179
|
-
%
|
176
|
+
%( -e "#{utility(:ssh)} #{ssh_transport_args}")
|
180
177
|
end
|
181
178
|
end
|
182
179
|
|
183
180
|
def ssh_transport_args
|
184
|
-
args = "-p #{
|
185
|
-
args << "-l #{
|
186
|
-
args << Array(additional_ssh_options).join(
|
181
|
+
args = "-p #{port} "
|
182
|
+
args << "-l #{ssh_user} " if ssh_user
|
183
|
+
args << Array(additional_ssh_options).join(" ")
|
187
184
|
args.rstrip
|
188
185
|
end
|
189
186
|
|
190
187
|
def write_password_file!
|
191
188
|
return unless rsync_password && mode != :ssh
|
192
189
|
|
193
|
-
@password_file = Tempfile.new(
|
190
|
+
@password_file = Tempfile.new("backup-rsync-password")
|
194
191
|
@password_file.write(rsync_password)
|
195
192
|
@password_file.close
|
196
193
|
end
|
@@ -198,7 +195,6 @@ module Backup
|
|
198
195
|
def remove_password_file!
|
199
196
|
@password_file.delete if @password_file
|
200
197
|
end
|
201
|
-
|
202
198
|
end
|
203
199
|
end
|
204
200
|
end
|
data/lib/backup/template.rb
CHANGED
@@ -1,10 +1,7 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require 'erb'
|
1
|
+
require "erb"
|
4
2
|
|
5
3
|
module Backup
|
6
4
|
class Template
|
7
|
-
|
8
5
|
# Holds a binding object. Nil if not provided.
|
9
6
|
attr_accessor :binding
|
10
7
|
|
@@ -12,13 +9,12 @@ module Backup
|
|
12
9
|
# Creates a new instance of the Backup::Template class
|
13
10
|
# and optionally takes an argument that can be either a binding object, a Hash or nil
|
14
11
|
def initialize(object = nil)
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
end
|
12
|
+
@binding =
|
13
|
+
if object.is_a?(Binding)
|
14
|
+
object
|
15
|
+
elsif object.is_a?(Hash)
|
16
|
+
Backup::Binder.new(object).get_binding
|
17
|
+
end
|
22
18
|
end
|
23
19
|
|
24
20
|
##
|
@@ -30,10 +26,10 @@ module Backup
|
|
30
26
|
##
|
31
27
|
# Returns a String object containing the contents of the file (in the context of the binding if any)
|
32
28
|
def result(file)
|
33
|
-
ERB.new(file_contents(file), nil,
|
29
|
+
ERB.new(file_contents(file), nil, "<>").result(binding)
|
34
30
|
end
|
35
31
|
|
36
|
-
|
32
|
+
private
|
37
33
|
|
38
34
|
##
|
39
35
|
# Reads and returns the contents of the provided file path,
|
@@ -41,6 +37,5 @@ module Backup
|
|
41
37
|
def file_contents(file)
|
42
38
|
File.read(File.join(Backup::TEMPLATE_PATH, file))
|
43
39
|
end
|
44
|
-
|
45
40
|
end
|
46
41
|
end
|
data/lib/backup/utilities.rb
CHANGED
@@ -1,11 +1,8 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Utilities
|
5
3
|
class Error < Backup::Error; end
|
6
4
|
|
7
|
-
|
8
|
-
NAMES = %w{
|
5
|
+
UTILITIES_NAMES = %w[
|
9
6
|
tar cat split sudo chown hostname
|
10
7
|
gzip bzip2
|
11
8
|
mongo mongodump mysqldump innobackupex
|
@@ -15,34 +12,38 @@ module Backup
|
|
15
12
|
sendmail exim
|
16
13
|
send_nsca
|
17
14
|
zabbix_sender
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
15
|
+
].freeze
|
16
|
+
|
17
|
+
# @api private
|
18
|
+
class DSL
|
19
|
+
def initialize(utils)
|
20
|
+
@utilities = utils
|
21
|
+
end
|
22
|
+
|
23
|
+
# Helper methods to allow users to set the path for all utilities in the
|
24
|
+
# .configure block.
|
25
|
+
#
|
26
|
+
# Utility names with dashes (`redis-cli`) will be set using method calls
|
27
|
+
# with an underscore (`redis_cli`).
|
28
|
+
UTILITIES_NAMES.each do |util_name|
|
29
|
+
define_method util_name.tr("-", "_") do |raw_path|
|
30
|
+
path = File.expand_path(raw_path)
|
31
|
+
|
32
|
+
unless File.executable?(path)
|
33
|
+
raise Utilities::Error, <<-EOS
|
34
|
+
The path given for '#{util_name}' was not found or not executable.
|
35
|
+
Path was: #{path}
|
36
|
+
EOS
|
37
|
+
end
|
39
38
|
|
40
|
-
|
41
|
-
# Allow users to set the +tar+ distribution if needed. (:gnu or :bsd)
|
42
|
-
def tar_dist(val)
|
43
|
-
Utilities.tar_dist(val)
|
39
|
+
@utilities.utilities[util_name] = path
|
44
40
|
end
|
45
41
|
end
|
42
|
+
|
43
|
+
# Allow users to set the +tar+ distribution if needed. (:gnu or :bsd)
|
44
|
+
def tar_dist(val)
|
45
|
+
Utilities.tar_dist(val)
|
46
|
+
end
|
46
47
|
end
|
47
48
|
|
48
49
|
class << self
|
@@ -102,7 +103,7 @@ module Backup
|
|
102
103
|
# These paths may be set using absolute paths, or relative to the
|
103
104
|
# working directory when Backup is run.
|
104
105
|
def configure(&block)
|
105
|
-
DSL.instance_eval(&block)
|
106
|
+
DSL.new(self).instance_eval(&block)
|
106
107
|
end
|
107
108
|
|
108
109
|
def tar_dist(val)
|
@@ -112,7 +113,11 @@ module Backup
|
|
112
113
|
|
113
114
|
def gnu_tar?
|
114
115
|
return @gnu_tar unless @gnu_tar.nil?
|
115
|
-
@gnu_tar = !!run("#{
|
116
|
+
@gnu_tar = !!run("#{utility(:tar)} --version").match(/GNU/)
|
117
|
+
end
|
118
|
+
|
119
|
+
def utilities
|
120
|
+
@utilities ||= {}
|
116
121
|
end
|
117
122
|
|
118
123
|
private
|
@@ -122,17 +127,17 @@ module Backup
|
|
122
127
|
# Raises an error if utility can not be found in the system's $PATH
|
123
128
|
def utility(name)
|
124
129
|
name = name.to_s.strip
|
125
|
-
raise Error,
|
130
|
+
raise Error, "Utility Name Empty" if name.empty?
|
126
131
|
|
127
|
-
|
128
|
-
raise Error, <<-EOS if
|
129
|
-
Could not locate '#{
|
132
|
+
utilities[name] ||= `which '#{name}' 2>/dev/null`.chomp
|
133
|
+
raise Error, <<-EOS if utilities[name].empty?
|
134
|
+
Could not locate '#{name}'.
|
130
135
|
Make sure the specified utility is installed
|
131
136
|
and available in your system's $PATH, or specify it's location
|
132
137
|
in your 'config.rb' file using Backup::Utilities.configure
|
133
138
|
EOS
|
134
139
|
|
135
|
-
|
140
|
+
utilities[name].dup
|
136
141
|
end
|
137
142
|
|
138
143
|
##
|
@@ -140,21 +145,25 @@ module Backup
|
|
140
145
|
# This is only used to simplify log messages.
|
141
146
|
def command_name(command)
|
142
147
|
parts = []
|
143
|
-
command = command.split(
|
144
|
-
command.shift while command[0].to_s.include?(
|
145
|
-
parts << command.shift.split(
|
146
|
-
if parts[0] ==
|
148
|
+
command = command.split(" ")
|
149
|
+
command.shift while command[0].to_s.include?("=")
|
150
|
+
parts << command.shift.split("/")[-1]
|
151
|
+
if parts[0] == "sudo"
|
152
|
+
Logger.warn "The Backup option \"sudo\" is " \
|
153
|
+
"scheduled to be deprecated. If you're using this feature, " \
|
154
|
+
"please see: https://github.com/backup/backup/issues/851"
|
155
|
+
|
147
156
|
until command.empty?
|
148
157
|
part = command.shift
|
149
|
-
if part.include?(
|
150
|
-
parts << part.split(
|
158
|
+
if part.include?("/")
|
159
|
+
parts << part.split("/")[-1]
|
151
160
|
break
|
152
161
|
else
|
153
162
|
parts << part
|
154
163
|
end
|
155
164
|
end
|
156
165
|
end
|
157
|
-
parts.join(
|
166
|
+
parts.join(" ")
|
158
167
|
end
|
159
168
|
|
160
169
|
##
|
@@ -169,43 +178,41 @@ module Backup
|
|
169
178
|
# Returns STDOUT
|
170
179
|
def run(command)
|
171
180
|
name = command_name(command)
|
172
|
-
Logger.info "Running system utility '#{
|
181
|
+
Logger.info "Running system utility '#{name}'..."
|
173
182
|
|
174
183
|
begin
|
175
|
-
out
|
176
|
-
|
184
|
+
out = ""
|
185
|
+
err = ""
|
186
|
+
ps = Open4.popen4(command) do |_pid, stdin, stdout, stderr|
|
177
187
|
stdin.close
|
178
|
-
out
|
188
|
+
out = stdout.read.strip
|
189
|
+
err = stderr.read.strip
|
179
190
|
end
|
180
191
|
rescue Exception => e
|
181
|
-
raise Error.wrap(e, "Failed to execute '#{
|
192
|
+
raise Error.wrap(e, "Failed to execute '#{name}'")
|
182
193
|
end
|
183
194
|
|
184
|
-
|
185
|
-
unless out.empty?
|
186
|
-
Logger.info(
|
187
|
-
out.lines.map {|line| "#{ name }:STDOUT: #{ line }" }.join
|
188
|
-
)
|
189
|
-
end
|
190
|
-
|
191
|
-
unless err.empty?
|
192
|
-
Logger.warn(
|
193
|
-
err.lines.map {|line| "#{ name }:STDERR: #{ line }" }.join
|
194
|
-
)
|
195
|
-
end
|
196
|
-
|
197
|
-
return out
|
198
|
-
else
|
195
|
+
unless ps.success?
|
199
196
|
raise Error, <<-EOS
|
200
|
-
'#{
|
201
|
-
STDOUT Messages: #{
|
202
|
-
STDERR Messages: #{
|
197
|
+
'#{name}' failed with exit status: #{ps.exitstatus}
|
198
|
+
STDOUT Messages: #{out.empty? ? "None" : "\n#{out}"}
|
199
|
+
STDERR Messages: #{err.empty? ? "None" : "\n#{err}"}
|
203
200
|
EOS
|
204
201
|
end
|
202
|
+
|
203
|
+
unless out.empty?
|
204
|
+
Logger.info(out.lines.map { |line| "#{name}:STDOUT: #{line}" }.join)
|
205
|
+
end
|
206
|
+
|
207
|
+
unless err.empty?
|
208
|
+
Logger.warn(err.lines.map { |line| "#{name}:STDERR: #{line}" }.join)
|
209
|
+
end
|
210
|
+
|
211
|
+
out
|
205
212
|
end
|
206
213
|
|
207
214
|
def reset!
|
208
|
-
|
215
|
+
utilities.clear
|
209
216
|
@gnu_tar = nil
|
210
217
|
end
|
211
218
|
end
|
@@ -214,11 +221,17 @@ module Backup
|
|
214
221
|
# while allowing them to be stubbed in spec_helper for all specs.
|
215
222
|
module Helpers
|
216
223
|
[:utility, :command_name, :run].each do |name|
|
217
|
-
define_method name
|
224
|
+
define_method name do |arg|
|
225
|
+
Utilities.send(name, arg)
|
226
|
+
end
|
218
227
|
private name
|
219
228
|
end
|
229
|
+
|
220
230
|
private
|
221
|
-
|
231
|
+
|
232
|
+
def gnu_tar?
|
233
|
+
Utilities.gnu_tar?
|
234
|
+
end
|
222
235
|
end
|
223
236
|
end
|
224
237
|
end
|