backup 4.4.1 → 5.0.0.beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/LICENSE +19 -0
- data/README.md +1 -1
- data/lib/backup.rb +74 -78
- data/lib/backup/archive.rb +31 -32
- data/lib/backup/binder.rb +2 -6
- data/lib/backup/cleaner.rb +14 -18
- data/lib/backup/cli.rb +104 -108
- data/lib/backup/cloud_io/base.rb +4 -7
- data/lib/backup/cloud_io/cloud_files.rb +60 -62
- data/lib/backup/cloud_io/s3.rb +69 -76
- data/lib/backup/compressor/base.rb +4 -7
- data/lib/backup/compressor/bzip2.rb +3 -7
- data/lib/backup/compressor/custom.rb +2 -6
- data/lib/backup/compressor/gzip.rb +16 -17
- data/lib/backup/config.rb +17 -18
- data/lib/backup/config/dsl.rb +16 -17
- data/lib/backup/config/helpers.rb +10 -16
- data/lib/backup/database/base.rb +22 -21
- data/lib/backup/database/mongodb.rb +36 -37
- data/lib/backup/database/mysql.rb +40 -41
- data/lib/backup/database/openldap.rb +8 -10
- data/lib/backup/database/postgresql.rb +29 -30
- data/lib/backup/database/redis.rb +27 -30
- data/lib/backup/database/riak.rb +15 -18
- data/lib/backup/database/sqlite.rb +4 -6
- data/lib/backup/encryptor/base.rb +2 -4
- data/lib/backup/encryptor/gpg.rb +49 -59
- data/lib/backup/encryptor/open_ssl.rb +11 -14
- data/lib/backup/errors.rb +7 -12
- data/lib/backup/logger.rb +16 -18
- data/lib/backup/logger/console.rb +5 -8
- data/lib/backup/logger/fog_adapter.rb +2 -6
- data/lib/backup/logger/logfile.rb +10 -12
- data/lib/backup/logger/syslog.rb +2 -4
- data/lib/backup/model.rb +75 -40
- data/lib/backup/notifier/base.rb +24 -26
- data/lib/backup/notifier/campfire.rb +9 -11
- data/lib/backup/notifier/command.rb +0 -3
- data/lib/backup/notifier/datadog.rb +9 -12
- data/lib/backup/notifier/flowdock.rb +13 -17
- data/lib/backup/notifier/hipchat.rb +11 -13
- data/lib/backup/notifier/http_post.rb +11 -14
- data/lib/backup/notifier/mail.rb +44 -47
- data/lib/backup/notifier/nagios.rb +5 -9
- data/lib/backup/notifier/pagerduty.rb +10 -12
- data/lib/backup/notifier/prowl.rb +15 -15
- data/lib/backup/notifier/pushover.rb +7 -10
- data/lib/backup/notifier/ses.rb +34 -16
- data/lib/backup/notifier/slack.rb +39 -40
- data/lib/backup/notifier/twitter.rb +2 -5
- data/lib/backup/notifier/zabbix.rb +11 -14
- data/lib/backup/package.rb +5 -9
- data/lib/backup/packager.rb +16 -17
- data/lib/backup/pipeline.rb +17 -21
- data/lib/backup/splitter.rb +8 -11
- data/lib/backup/storage/base.rb +5 -8
- data/lib/backup/storage/cloud_files.rb +21 -23
- data/lib/backup/storage/cycler.rb +10 -15
- data/lib/backup/storage/dropbox.rb +15 -21
- data/lib/backup/storage/ftp.rb +8 -10
- data/lib/backup/storage/local.rb +5 -8
- data/lib/backup/storage/qiniu.rb +8 -8
- data/lib/backup/storage/rsync.rb +24 -26
- data/lib/backup/storage/s3.rb +27 -28
- data/lib/backup/storage/scp.rb +10 -12
- data/lib/backup/storage/sftp.rb +10 -12
- data/lib/backup/syncer/base.rb +5 -8
- data/lib/backup/syncer/cloud/base.rb +27 -30
- data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
- data/lib/backup/syncer/cloud/local_file.rb +5 -8
- data/lib/backup/syncer/cloud/s3.rb +23 -24
- data/lib/backup/syncer/rsync/base.rb +6 -10
- data/lib/backup/syncer/rsync/local.rb +1 -5
- data/lib/backup/syncer/rsync/pull.rb +6 -10
- data/lib/backup/syncer/rsync/push.rb +18 -22
- data/lib/backup/template.rb +9 -14
- data/lib/backup/utilities.rb +82 -69
- data/lib/backup/version.rb +1 -3
- metadata +100 -660
@@ -1,15 +1,12 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Syncer
|
5
3
|
module RSync
|
6
4
|
class Local < Base
|
7
|
-
|
8
5
|
def perform!
|
9
6
|
log!(:started)
|
10
7
|
|
11
8
|
create_dest_path!
|
12
|
-
run("#{
|
9
|
+
run("#{rsync_command} #{paths_to_push} '#{dest_path}'")
|
13
10
|
|
14
11
|
log!(:finished)
|
15
12
|
end
|
@@ -24,7 +21,6 @@ module Backup
|
|
24
21
|
def create_dest_path!
|
25
22
|
FileUtils.mkdir_p dest_path
|
26
23
|
end
|
27
|
-
|
28
24
|
end
|
29
25
|
end
|
30
26
|
end
|
@@ -1,17 +1,14 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Syncer
|
5
3
|
module RSync
|
6
4
|
class Pull < Push
|
7
|
-
|
8
5
|
def perform!
|
9
6
|
log!(:started)
|
10
7
|
write_password_file!
|
11
8
|
|
12
9
|
create_dest_path!
|
13
|
-
run("#{
|
14
|
-
"'#{
|
10
|
+
run("#{rsync_command} #{host_options}#{paths_to_pull} " \
|
11
|
+
"'#{dest_path}'")
|
15
12
|
|
16
13
|
log!(:finished)
|
17
14
|
ensure
|
@@ -30,10 +27,10 @@ module Backup
|
|
30
27
|
# Also remove any trailing `/`, since we don't want rsync's
|
31
28
|
# "trailing / on source directories" behavior.
|
32
29
|
def paths_to_pull
|
33
|
-
sep = mode == :ssh ?
|
34
|
-
directories.map
|
35
|
-
"#{
|
36
|
-
|
30
|
+
sep = mode == :ssh ? ":" : "::"
|
31
|
+
directories.map do |dir|
|
32
|
+
"#{sep}'#{dir.sub(/^~\//, "").sub(/\/$/, "")}'"
|
33
|
+
end.join(" ").sub(/^#{ sep }/, "")
|
37
34
|
end
|
38
35
|
|
39
36
|
# Expand path, since this is local and shell-quoted.
|
@@ -44,7 +41,6 @@ module Backup
|
|
44
41
|
def create_dest_path!
|
45
42
|
FileUtils.mkdir_p dest_path
|
46
43
|
end
|
47
|
-
|
48
44
|
end
|
49
45
|
end
|
50
46
|
end
|
@@ -1,10 +1,7 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Syncer
|
5
3
|
module RSync
|
6
4
|
class Push < Base
|
7
|
-
|
8
5
|
##
|
9
6
|
# Mode of operation
|
10
7
|
#
|
@@ -114,8 +111,8 @@ module Backup
|
|
114
111
|
write_password_file!
|
115
112
|
|
116
113
|
create_dest_path!
|
117
|
-
run("#{
|
118
|
-
"#{
|
114
|
+
run("#{rsync_command} #{paths_to_push} " \
|
115
|
+
"#{host_options}'#{dest_path}'")
|
119
116
|
|
120
117
|
log!(:finished)
|
121
118
|
ensure
|
@@ -128,7 +125,7 @@ module Backup
|
|
128
125
|
# Remove any preceeding '~/', since this is on the remote,
|
129
126
|
# and remove any trailing `/`.
|
130
127
|
def dest_path
|
131
|
-
@dest_path ||= path.sub(/^~\//,
|
128
|
+
@dest_path ||= path.sub(/^~\//, "").sub(/\/$/, "")
|
132
129
|
end
|
133
130
|
|
134
131
|
##
|
@@ -137,10 +134,10 @@ module Backup
|
|
137
134
|
# call 'mkdir' without the '-p' option. This is only applicable in :ssh
|
138
135
|
# mode, and only used if the path would require this.
|
139
136
|
def create_dest_path!
|
140
|
-
return unless mode == :ssh && dest_path.index(
|
137
|
+
return unless mode == :ssh && dest_path.index("/").to_i > 0
|
141
138
|
|
142
|
-
run "#{
|
143
|
-
|
139
|
+
run "#{utility(:ssh)} #{ssh_transport_args} #{host} " +
|
140
|
+
%("mkdir -p '#{dest_path}'")
|
144
141
|
end
|
145
142
|
|
146
143
|
##
|
@@ -148,10 +145,10 @@ module Backup
|
|
148
145
|
# For Pull, this will prepend the first path in #paths_to_pull.
|
149
146
|
def host_options
|
150
147
|
if mode == :ssh
|
151
|
-
"#{
|
148
|
+
"#{host}:"
|
152
149
|
else
|
153
|
-
user = "#{
|
154
|
-
"#{
|
150
|
+
user = "#{rsync_user}@" if rsync_user
|
151
|
+
"#{user}#{host}::"
|
155
152
|
end
|
156
153
|
end
|
157
154
|
|
@@ -162,35 +159,35 @@ module Backup
|
|
162
159
|
end
|
163
160
|
|
164
161
|
def compress_option
|
165
|
-
compress ?
|
162
|
+
compress ? " --compress" : ""
|
166
163
|
end
|
167
164
|
|
168
165
|
def password_option
|
169
|
-
return
|
166
|
+
return "" if mode == :ssh
|
170
167
|
|
171
168
|
path = @password_file ? @password_file.path : rsync_password_file
|
172
|
-
path ? " --password-file='#{
|
169
|
+
path ? " --password-file='#{File.expand_path(path)}'" : ""
|
173
170
|
end
|
174
171
|
|
175
172
|
def transport_options
|
176
173
|
if mode == :rsync_daemon
|
177
|
-
" --port #{
|
174
|
+
" --port #{port}"
|
178
175
|
else
|
179
|
-
%
|
176
|
+
%( -e "#{utility(:ssh)} #{ssh_transport_args}")
|
180
177
|
end
|
181
178
|
end
|
182
179
|
|
183
180
|
def ssh_transport_args
|
184
|
-
args = "-p #{
|
185
|
-
args << "-l #{
|
186
|
-
args << Array(additional_ssh_options).join(
|
181
|
+
args = "-p #{port} "
|
182
|
+
args << "-l #{ssh_user} " if ssh_user
|
183
|
+
args << Array(additional_ssh_options).join(" ")
|
187
184
|
args.rstrip
|
188
185
|
end
|
189
186
|
|
190
187
|
def write_password_file!
|
191
188
|
return unless rsync_password && mode != :ssh
|
192
189
|
|
193
|
-
@password_file = Tempfile.new(
|
190
|
+
@password_file = Tempfile.new("backup-rsync-password")
|
194
191
|
@password_file.write(rsync_password)
|
195
192
|
@password_file.close
|
196
193
|
end
|
@@ -198,7 +195,6 @@ module Backup
|
|
198
195
|
def remove_password_file!
|
199
196
|
@password_file.delete if @password_file
|
200
197
|
end
|
201
|
-
|
202
198
|
end
|
203
199
|
end
|
204
200
|
end
|
data/lib/backup/template.rb
CHANGED
@@ -1,10 +1,7 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require 'erb'
|
1
|
+
require "erb"
|
4
2
|
|
5
3
|
module Backup
|
6
4
|
class Template
|
7
|
-
|
8
5
|
# Holds a binding object. Nil if not provided.
|
9
6
|
attr_accessor :binding
|
10
7
|
|
@@ -12,13 +9,12 @@ module Backup
|
|
12
9
|
# Creates a new instance of the Backup::Template class
|
13
10
|
# and optionally takes an argument that can be either a binding object, a Hash or nil
|
14
11
|
def initialize(object = nil)
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
end
|
12
|
+
@binding =
|
13
|
+
if object.is_a?(Binding)
|
14
|
+
object
|
15
|
+
elsif object.is_a?(Hash)
|
16
|
+
Backup::Binder.new(object).get_binding
|
17
|
+
end
|
22
18
|
end
|
23
19
|
|
24
20
|
##
|
@@ -30,10 +26,10 @@ module Backup
|
|
30
26
|
##
|
31
27
|
# Returns a String object containing the contents of the file (in the context of the binding if any)
|
32
28
|
def result(file)
|
33
|
-
ERB.new(file_contents(file), nil,
|
29
|
+
ERB.new(file_contents(file), nil, "<>").result(binding)
|
34
30
|
end
|
35
31
|
|
36
|
-
|
32
|
+
private
|
37
33
|
|
38
34
|
##
|
39
35
|
# Reads and returns the contents of the provided file path,
|
@@ -41,6 +37,5 @@ module Backup
|
|
41
37
|
def file_contents(file)
|
42
38
|
File.read(File.join(Backup::TEMPLATE_PATH, file))
|
43
39
|
end
|
44
|
-
|
45
40
|
end
|
46
41
|
end
|
data/lib/backup/utilities.rb
CHANGED
@@ -1,11 +1,8 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Utilities
|
5
3
|
class Error < Backup::Error; end
|
6
4
|
|
7
|
-
|
8
|
-
NAMES = %w{
|
5
|
+
UTILITIES_NAMES = %w[
|
9
6
|
tar cat split sudo chown hostname
|
10
7
|
gzip bzip2
|
11
8
|
mongo mongodump mysqldump innobackupex
|
@@ -15,34 +12,38 @@ module Backup
|
|
15
12
|
sendmail exim
|
16
13
|
send_nsca
|
17
14
|
zabbix_sender
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
15
|
+
].freeze
|
16
|
+
|
17
|
+
# @api private
|
18
|
+
class DSL
|
19
|
+
def initialize(utils)
|
20
|
+
@utilities = utils
|
21
|
+
end
|
22
|
+
|
23
|
+
# Helper methods to allow users to set the path for all utilities in the
|
24
|
+
# .configure block.
|
25
|
+
#
|
26
|
+
# Utility names with dashes (`redis-cli`) will be set using method calls
|
27
|
+
# with an underscore (`redis_cli`).
|
28
|
+
UTILITIES_NAMES.each do |util_name|
|
29
|
+
define_method util_name.tr("-", "_") do |raw_path|
|
30
|
+
path = File.expand_path(raw_path)
|
31
|
+
|
32
|
+
unless File.executable?(path)
|
33
|
+
raise Utilities::Error, <<-EOS
|
34
|
+
The path given for '#{util_name}' was not found or not executable.
|
35
|
+
Path was: #{path}
|
36
|
+
EOS
|
37
|
+
end
|
39
38
|
|
40
|
-
|
41
|
-
# Allow users to set the +tar+ distribution if needed. (:gnu or :bsd)
|
42
|
-
def tar_dist(val)
|
43
|
-
Utilities.tar_dist(val)
|
39
|
+
@utilities.utilities[util_name] = path
|
44
40
|
end
|
45
41
|
end
|
42
|
+
|
43
|
+
# Allow users to set the +tar+ distribution if needed. (:gnu or :bsd)
|
44
|
+
def tar_dist(val)
|
45
|
+
Utilities.tar_dist(val)
|
46
|
+
end
|
46
47
|
end
|
47
48
|
|
48
49
|
class << self
|
@@ -102,7 +103,7 @@ module Backup
|
|
102
103
|
# These paths may be set using absolute paths, or relative to the
|
103
104
|
# working directory when Backup is run.
|
104
105
|
def configure(&block)
|
105
|
-
DSL.instance_eval(&block)
|
106
|
+
DSL.new(self).instance_eval(&block)
|
106
107
|
end
|
107
108
|
|
108
109
|
def tar_dist(val)
|
@@ -112,7 +113,11 @@ module Backup
|
|
112
113
|
|
113
114
|
def gnu_tar?
|
114
115
|
return @gnu_tar unless @gnu_tar.nil?
|
115
|
-
@gnu_tar = !!run("#{
|
116
|
+
@gnu_tar = !!run("#{utility(:tar)} --version").match(/GNU/)
|
117
|
+
end
|
118
|
+
|
119
|
+
def utilities
|
120
|
+
@utilities ||= {}
|
116
121
|
end
|
117
122
|
|
118
123
|
private
|
@@ -122,17 +127,17 @@ module Backup
|
|
122
127
|
# Raises an error if utility can not be found in the system's $PATH
|
123
128
|
def utility(name)
|
124
129
|
name = name.to_s.strip
|
125
|
-
raise Error,
|
130
|
+
raise Error, "Utility Name Empty" if name.empty?
|
126
131
|
|
127
|
-
|
128
|
-
raise Error, <<-EOS if
|
129
|
-
Could not locate '#{
|
132
|
+
utilities[name] ||= `which '#{name}' 2>/dev/null`.chomp
|
133
|
+
raise Error, <<-EOS if utilities[name].empty?
|
134
|
+
Could not locate '#{name}'.
|
130
135
|
Make sure the specified utility is installed
|
131
136
|
and available in your system's $PATH, or specify it's location
|
132
137
|
in your 'config.rb' file using Backup::Utilities.configure
|
133
138
|
EOS
|
134
139
|
|
135
|
-
|
140
|
+
utilities[name].dup
|
136
141
|
end
|
137
142
|
|
138
143
|
##
|
@@ -140,21 +145,25 @@ module Backup
|
|
140
145
|
# This is only used to simplify log messages.
|
141
146
|
def command_name(command)
|
142
147
|
parts = []
|
143
|
-
command = command.split(
|
144
|
-
command.shift while command[0].to_s.include?(
|
145
|
-
parts << command.shift.split(
|
146
|
-
if parts[0] ==
|
148
|
+
command = command.split(" ")
|
149
|
+
command.shift while command[0].to_s.include?("=")
|
150
|
+
parts << command.shift.split("/")[-1]
|
151
|
+
if parts[0] == "sudo"
|
152
|
+
Logger.warn "The Backup option \"sudo\" is " \
|
153
|
+
"scheduled to be deprecated. If you're using this feature, " \
|
154
|
+
"please see: https://github.com/backup/backup/issues/851"
|
155
|
+
|
147
156
|
until command.empty?
|
148
157
|
part = command.shift
|
149
|
-
if part.include?(
|
150
|
-
parts << part.split(
|
158
|
+
if part.include?("/")
|
159
|
+
parts << part.split("/")[-1]
|
151
160
|
break
|
152
161
|
else
|
153
162
|
parts << part
|
154
163
|
end
|
155
164
|
end
|
156
165
|
end
|
157
|
-
parts.join(
|
166
|
+
parts.join(" ")
|
158
167
|
end
|
159
168
|
|
160
169
|
##
|
@@ -169,43 +178,41 @@ module Backup
|
|
169
178
|
# Returns STDOUT
|
170
179
|
def run(command)
|
171
180
|
name = command_name(command)
|
172
|
-
Logger.info "Running system utility '#{
|
181
|
+
Logger.info "Running system utility '#{name}'..."
|
173
182
|
|
174
183
|
begin
|
175
|
-
out
|
176
|
-
|
184
|
+
out = ""
|
185
|
+
err = ""
|
186
|
+
ps = Open4.popen4(command) do |_pid, stdin, stdout, stderr|
|
177
187
|
stdin.close
|
178
|
-
out
|
188
|
+
out = stdout.read.strip
|
189
|
+
err = stderr.read.strip
|
179
190
|
end
|
180
191
|
rescue Exception => e
|
181
|
-
raise Error.wrap(e, "Failed to execute '#{
|
192
|
+
raise Error.wrap(e, "Failed to execute '#{name}'")
|
182
193
|
end
|
183
194
|
|
184
|
-
|
185
|
-
unless out.empty?
|
186
|
-
Logger.info(
|
187
|
-
out.lines.map {|line| "#{ name }:STDOUT: #{ line }" }.join
|
188
|
-
)
|
189
|
-
end
|
190
|
-
|
191
|
-
unless err.empty?
|
192
|
-
Logger.warn(
|
193
|
-
err.lines.map {|line| "#{ name }:STDERR: #{ line }" }.join
|
194
|
-
)
|
195
|
-
end
|
196
|
-
|
197
|
-
return out
|
198
|
-
else
|
195
|
+
unless ps.success?
|
199
196
|
raise Error, <<-EOS
|
200
|
-
'#{
|
201
|
-
STDOUT Messages: #{
|
202
|
-
STDERR Messages: #{
|
197
|
+
'#{name}' failed with exit status: #{ps.exitstatus}
|
198
|
+
STDOUT Messages: #{out.empty? ? "None" : "\n#{out}"}
|
199
|
+
STDERR Messages: #{err.empty? ? "None" : "\n#{err}"}
|
203
200
|
EOS
|
204
201
|
end
|
202
|
+
|
203
|
+
unless out.empty?
|
204
|
+
Logger.info(out.lines.map { |line| "#{name}:STDOUT: #{line}" }.join)
|
205
|
+
end
|
206
|
+
|
207
|
+
unless err.empty?
|
208
|
+
Logger.warn(err.lines.map { |line| "#{name}:STDERR: #{line}" }.join)
|
209
|
+
end
|
210
|
+
|
211
|
+
out
|
205
212
|
end
|
206
213
|
|
207
214
|
def reset!
|
208
|
-
|
215
|
+
utilities.clear
|
209
216
|
@gnu_tar = nil
|
210
217
|
end
|
211
218
|
end
|
@@ -214,11 +221,17 @@ module Backup
|
|
214
221
|
# while allowing them to be stubbed in spec_helper for all specs.
|
215
222
|
module Helpers
|
216
223
|
[:utility, :command_name, :run].each do |name|
|
217
|
-
define_method name
|
224
|
+
define_method name do |arg|
|
225
|
+
Utilities.send(name, arg)
|
226
|
+
end
|
218
227
|
private name
|
219
228
|
end
|
229
|
+
|
220
230
|
private
|
221
|
-
|
231
|
+
|
232
|
+
def gnu_tar?
|
233
|
+
Utilities.gnu_tar?
|
234
|
+
end
|
222
235
|
end
|
223
236
|
end
|
224
237
|
end
|