backup 4.3.0 → 5.0.0.beta.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/LICENSE +19 -0
- data/README.md +13 -9
- data/bin/docker_test +24 -0
- data/lib/backup.rb +74 -78
- data/lib/backup/archive.rb +31 -32
- data/lib/backup/binder.rb +2 -6
- data/lib/backup/cleaner.rb +14 -18
- data/lib/backup/cli.rb +104 -108
- data/lib/backup/cloud_io/base.rb +4 -7
- data/lib/backup/cloud_io/cloud_files.rb +60 -62
- data/lib/backup/cloud_io/s3.rb +69 -76
- data/lib/backup/compressor/base.rb +4 -7
- data/lib/backup/compressor/bzip2.rb +3 -7
- data/lib/backup/compressor/custom.rb +2 -6
- data/lib/backup/compressor/gzip.rb +16 -17
- data/lib/backup/config.rb +17 -18
- data/lib/backup/config/dsl.rb +16 -17
- data/lib/backup/config/helpers.rb +10 -16
- data/lib/backup/database/base.rb +22 -21
- data/lib/backup/database/mongodb.rb +36 -37
- data/lib/backup/database/mysql.rb +40 -41
- data/lib/backup/database/openldap.rb +8 -10
- data/lib/backup/database/postgresql.rb +29 -30
- data/lib/backup/database/redis.rb +27 -30
- data/lib/backup/database/riak.rb +15 -18
- data/lib/backup/database/sqlite.rb +4 -6
- data/lib/backup/encryptor/base.rb +2 -4
- data/lib/backup/encryptor/gpg.rb +49 -59
- data/lib/backup/encryptor/open_ssl.rb +11 -14
- data/lib/backup/errors.rb +7 -12
- data/lib/backup/logger.rb +16 -18
- data/lib/backup/logger/console.rb +5 -8
- data/lib/backup/logger/fog_adapter.rb +2 -6
- data/lib/backup/logger/logfile.rb +10 -12
- data/lib/backup/logger/syslog.rb +2 -4
- data/lib/backup/model.rb +33 -40
- data/lib/backup/notifier/base.rb +24 -26
- data/lib/backup/notifier/campfire.rb +9 -11
- data/lib/backup/notifier/command.rb +3 -3
- data/lib/backup/notifier/datadog.rb +9 -12
- data/lib/backup/notifier/flowdock.rb +13 -17
- data/lib/backup/notifier/hipchat.rb +18 -14
- data/lib/backup/notifier/http_post.rb +11 -14
- data/lib/backup/notifier/mail.rb +42 -54
- data/lib/backup/notifier/nagios.rb +5 -9
- data/lib/backup/notifier/pagerduty.rb +10 -12
- data/lib/backup/notifier/prowl.rb +15 -15
- data/lib/backup/notifier/pushover.rb +7 -10
- data/lib/backup/notifier/ses.rb +52 -17
- data/lib/backup/notifier/slack.rb +39 -40
- data/lib/backup/notifier/twitter.rb +2 -5
- data/lib/backup/notifier/zabbix.rb +11 -14
- data/lib/backup/package.rb +5 -9
- data/lib/backup/packager.rb +16 -17
- data/lib/backup/pipeline.rb +17 -21
- data/lib/backup/splitter.rb +8 -11
- data/lib/backup/storage/base.rb +5 -8
- data/lib/backup/storage/cloud_files.rb +21 -23
- data/lib/backup/storage/cycler.rb +10 -15
- data/lib/backup/storage/dropbox.rb +15 -21
- data/lib/backup/storage/ftp.rb +14 -10
- data/lib/backup/storage/local.rb +5 -8
- data/lib/backup/storage/qiniu.rb +8 -8
- data/lib/backup/storage/rsync.rb +24 -26
- data/lib/backup/storage/s3.rb +27 -28
- data/lib/backup/storage/scp.rb +10 -12
- data/lib/backup/storage/sftp.rb +10 -12
- data/lib/backup/syncer/base.rb +5 -8
- data/lib/backup/syncer/cloud/base.rb +27 -30
- data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
- data/lib/backup/syncer/cloud/local_file.rb +5 -8
- data/lib/backup/syncer/cloud/s3.rb +23 -24
- data/lib/backup/syncer/rsync/base.rb +6 -10
- data/lib/backup/syncer/rsync/local.rb +1 -5
- data/lib/backup/syncer/rsync/pull.rb +6 -10
- data/lib/backup/syncer/rsync/push.rb +18 -22
- data/lib/backup/template.rb +9 -14
- data/lib/backup/utilities.rb +78 -69
- data/lib/backup/version.rb +1 -3
- metadata +107 -677
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Database
|
5
3
|
class MongoDB < Base
|
@@ -60,7 +58,6 @@ module Backup
|
|
60
58
|
lock_database if @lock
|
61
59
|
dump!
|
62
60
|
package!
|
63
|
-
|
64
61
|
ensure
|
65
62
|
unlock_database if @lock
|
66
63
|
end
|
@@ -78,7 +75,7 @@ module Backup
|
|
78
75
|
run(mongodump)
|
79
76
|
else
|
80
77
|
collections.each do |collection|
|
81
|
-
run("#{
|
78
|
+
run("#{mongodump} --collection='#{collection}'")
|
82
79
|
end
|
83
80
|
end
|
84
81
|
end
|
@@ -92,25 +89,27 @@ module Backup
|
|
92
89
|
# If successful, +dump_packaging_path+ is removed.
|
93
90
|
def package!
|
94
91
|
pipeline = Pipeline.new
|
95
|
-
dump_ext =
|
92
|
+
dump_ext = "tar"
|
96
93
|
|
97
|
-
pipeline << "#{
|
98
|
-
"-C '#{
|
94
|
+
pipeline << "#{utility(:tar)} -cf - " \
|
95
|
+
"-C '#{dump_path}' '#{dump_filename}'"
|
99
96
|
|
100
|
-
model.compressor
|
101
|
-
|
102
|
-
|
103
|
-
|
97
|
+
if model.compressor
|
98
|
+
model.compressor.compress_with do |command, ext|
|
99
|
+
pipeline << command
|
100
|
+
dump_ext << ext
|
101
|
+
end
|
102
|
+
end
|
104
103
|
|
105
|
-
pipeline << "#{
|
106
|
-
|
104
|
+
pipeline << "#{utility(:cat)} > " \
|
105
|
+
"'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
|
107
106
|
|
108
107
|
pipeline.run
|
109
108
|
if pipeline.success?
|
110
109
|
FileUtils.rm_rf dump_packaging_path
|
111
110
|
log!(:finished)
|
112
111
|
else
|
113
|
-
raise Error, "Dump Failed!\n
|
112
|
+
raise Error, "Dump Failed!\n#{pipeline.error_messages}"
|
114
113
|
end
|
115
114
|
end
|
116
115
|
|
@@ -119,69 +118,69 @@ module Backup
|
|
119
118
|
end
|
120
119
|
|
121
120
|
def mongodump
|
122
|
-
"#{
|
123
|
-
|
124
|
-
|
121
|
+
"#{utility(:mongodump)} #{name_option} #{credential_options} " \
|
122
|
+
"#{connectivity_options} #{ipv6_option} #{oplog_option} " \
|
123
|
+
"#{user_options} --out='#{dump_packaging_path}'"
|
125
124
|
end
|
126
125
|
|
127
126
|
def name_option
|
128
|
-
|
127
|
+
return unless name
|
128
|
+
"--db='#{name}'"
|
129
129
|
end
|
130
130
|
|
131
131
|
def credential_options
|
132
132
|
opts = []
|
133
|
-
opts << "--username='#{
|
134
|
-
opts << "--password='#{
|
135
|
-
opts << "--authenticationDatabase='#{
|
136
|
-
opts.join(
|
133
|
+
opts << "--username='#{username}'" if username
|
134
|
+
opts << "--password='#{password}'" if password
|
135
|
+
opts << "--authenticationDatabase='#{authdb}'" if authdb
|
136
|
+
opts.join(" ")
|
137
137
|
end
|
138
138
|
|
139
139
|
def connectivity_options
|
140
140
|
opts = []
|
141
|
-
opts << "--host='#{
|
142
|
-
opts << "--port='#{
|
143
|
-
opts.join(
|
141
|
+
opts << "--host='#{host}'" if host
|
142
|
+
opts << "--port='#{port}'" if port
|
143
|
+
opts.join(" ")
|
144
144
|
end
|
145
145
|
|
146
146
|
def ipv6_option
|
147
|
-
|
147
|
+
"--ipv6" if ipv6
|
148
148
|
end
|
149
149
|
|
150
150
|
def oplog_option
|
151
|
-
|
151
|
+
"--oplog" if oplog
|
152
152
|
end
|
153
153
|
|
154
154
|
def user_options
|
155
|
-
Array(additional_options).join(
|
155
|
+
Array(additional_options).join(" ")
|
156
156
|
end
|
157
157
|
|
158
158
|
def lock_database
|
159
|
-
lock_command = <<-EOS.gsub(/^ +/,
|
159
|
+
lock_command = <<-EOS.gsub(/^ +/, "")
|
160
160
|
echo 'use admin
|
161
161
|
db.setProfilingLevel(0)
|
162
|
-
db.fsyncLock()' | #{
|
162
|
+
db.fsyncLock()' | #{mongo_shell}
|
163
163
|
EOS
|
164
164
|
|
165
165
|
run(lock_command)
|
166
166
|
end
|
167
167
|
|
168
168
|
def unlock_database
|
169
|
-
unlock_command = <<-EOS.gsub(/^ +/,
|
169
|
+
unlock_command = <<-EOS.gsub(/^ +/, "")
|
170
170
|
echo 'use admin
|
171
|
-
db.fsyncUnlock()' | #{
|
171
|
+
db.fsyncUnlock()' | #{mongo_shell}
|
172
172
|
EOS
|
173
173
|
|
174
174
|
run(unlock_command)
|
175
175
|
end
|
176
176
|
|
177
177
|
def mongo_shell
|
178
|
-
cmd = "#{
|
179
|
-
cmd << " #{
|
180
|
-
cmd << " #{
|
181
|
-
cmd << " '#{
|
178
|
+
cmd = "#{utility(:mongo)} #{connectivity_options}".rstrip
|
179
|
+
cmd << " #{credential_options}".rstrip
|
180
|
+
cmd << " #{ipv6_option}".rstrip
|
181
|
+
cmd << " '#{name}'" if name
|
182
182
|
cmd
|
183
183
|
end
|
184
|
-
|
185
184
|
end
|
186
185
|
end
|
187
186
|
end
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Database
|
5
3
|
class MySQL < Base
|
@@ -78,71 +76,73 @@ module Backup
|
|
78
76
|
super
|
79
77
|
|
80
78
|
pipeline = Pipeline.new
|
81
|
-
dump_ext = sql_backup? ?
|
79
|
+
dump_ext = sql_backup? ? "sql" : "tar"
|
82
80
|
|
83
81
|
pipeline << sudo_option(sql_backup? ? mysqldump : innobackupex)
|
84
82
|
|
85
|
-
model.compressor
|
86
|
-
|
87
|
-
|
88
|
-
|
83
|
+
if model.compressor
|
84
|
+
model.compressor.compress_with do |command, ext|
|
85
|
+
pipeline << command
|
86
|
+
dump_ext << ext
|
87
|
+
end
|
88
|
+
end
|
89
89
|
|
90
|
-
pipeline << "#{
|
91
|
-
|
90
|
+
pipeline << "#{utility(:cat)} > " \
|
91
|
+
"'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
|
92
92
|
|
93
93
|
pipeline.run
|
94
94
|
if pipeline.success?
|
95
95
|
log!(:finished)
|
96
96
|
else
|
97
|
-
raise Error, "Dump Failed!\n
|
97
|
+
raise Error, "Dump Failed!\n#{pipeline.error_messages}"
|
98
98
|
end
|
99
99
|
end
|
100
100
|
|
101
101
|
private
|
102
102
|
|
103
103
|
def mysqldump
|
104
|
-
"#{
|
105
|
-
|
106
|
-
|
104
|
+
"#{utility(:mysqldump)} #{user_options} #{credential_options} " \
|
105
|
+
"#{connectivity_options} #{name_option} " \
|
106
|
+
"#{tables_to_dump} #{tables_to_skip}"
|
107
107
|
end
|
108
108
|
|
109
109
|
def credential_options
|
110
110
|
opts = []
|
111
|
-
opts << "--user=#{
|
112
|
-
opts << "--password=#{
|
113
|
-
opts.join(
|
111
|
+
opts << "--user=#{Shellwords.escape(username)}" if username
|
112
|
+
opts << "--password=#{Shellwords.escape(password)}" if password
|
113
|
+
opts.join(" ")
|
114
114
|
end
|
115
115
|
|
116
116
|
def connectivity_options
|
117
|
-
return "--socket='#{
|
117
|
+
return "--socket='#{socket}'" if socket
|
118
118
|
|
119
119
|
opts = []
|
120
|
-
opts << "--host='#{
|
121
|
-
opts << "--port='#{
|
122
|
-
opts.join(
|
120
|
+
opts << "--host='#{host}'" if host
|
121
|
+
opts << "--port='#{port}'" if port
|
122
|
+
opts.join(" ")
|
123
123
|
end
|
124
124
|
|
125
125
|
def user_options
|
126
|
-
Array(additional_options).join(
|
126
|
+
Array(additional_options).join(" ")
|
127
127
|
end
|
128
128
|
|
129
129
|
def user_prepare_options
|
130
|
-
Array(prepare_options).join(
|
130
|
+
Array(prepare_options).join(" ")
|
131
131
|
end
|
132
132
|
|
133
133
|
def name_option
|
134
|
-
dump_all? ?
|
134
|
+
dump_all? ? "--all-databases" : name
|
135
135
|
end
|
136
136
|
|
137
137
|
def tables_to_dump
|
138
|
-
Array(only_tables).join(
|
138
|
+
Array(only_tables).join(" ") unless dump_all?
|
139
139
|
end
|
140
140
|
|
141
141
|
def tables_to_skip
|
142
142
|
Array(skip_tables).map do |table|
|
143
|
-
table =
|
144
|
-
"--ignore-table='#{
|
145
|
-
end.join(
|
143
|
+
table = dump_all? || table["."] ? table : "#{name}.#{table}"
|
144
|
+
"--ignore-table='#{table}'"
|
145
|
+
end.join(" ")
|
146
146
|
end
|
147
147
|
|
148
148
|
def dump_all?
|
@@ -155,28 +155,28 @@ module Backup
|
|
155
155
|
|
156
156
|
def innobackupex
|
157
157
|
# Creation phase
|
158
|
-
"#{
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
158
|
+
"#{utility(:innobackupex)} #{credential_options} " \
|
159
|
+
"#{connectivity_options} #{user_options} " \
|
160
|
+
"--no-timestamp #{temp_dir} #{quiet_option} && " +
|
161
|
+
innobackupex_prepare +
|
162
|
+
# Move files to tar-ed stream on stdout
|
163
|
+
"#{utility(:tar)} --remove-files -cf - " \
|
164
|
+
"-C #{File.dirname(temp_dir)} #{File.basename(temp_dir)}"
|
165
165
|
end
|
166
166
|
|
167
167
|
def innobackupex_prepare
|
168
168
|
return "" unless @prepare_backup
|
169
169
|
# Log applying phase (prepare for restore)
|
170
|
-
"#{
|
171
|
-
|
170
|
+
"#{utility(:innobackupex)} --apply-log #{temp_dir} " \
|
171
|
+
"#{user_prepare_options} #{quiet_option} && "
|
172
172
|
end
|
173
173
|
|
174
174
|
def sudo_option(command_block)
|
175
175
|
return command_block unless sudo_user
|
176
176
|
|
177
|
-
"sudo -s -u #{
|
178
|
-
|
179
|
-
|
177
|
+
"sudo -s -u #{sudo_user} -- <<END_OF_SUDO\n" \
|
178
|
+
"#{command_block}\n" \
|
179
|
+
"END_OF_SUDO\n"
|
180
180
|
end
|
181
181
|
|
182
182
|
def quiet_option
|
@@ -184,9 +184,8 @@ module Backup
|
|
184
184
|
end
|
185
185
|
|
186
186
|
def temp_dir
|
187
|
-
File.join(dump_path, dump_filename
|
187
|
+
File.join(dump_path, "#{dump_filename}.bkpdir")
|
188
188
|
end
|
189
|
-
|
190
189
|
end
|
191
190
|
end
|
192
191
|
end
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Database
|
5
3
|
class OpenLDAP < Base
|
@@ -33,11 +31,11 @@ module Backup
|
|
33
31
|
super
|
34
32
|
instance_eval(&block) if block_given?
|
35
33
|
|
36
|
-
@name ||=
|
34
|
+
@name ||= "ldap_backup"
|
37
35
|
@use_sudo ||= false
|
38
|
-
@slapcat_args ||=
|
36
|
+
@slapcat_args ||= []
|
39
37
|
@slapcat_utility ||= utility(:slapcat)
|
40
|
-
@slapcat_conf ||=
|
38
|
+
@slapcat_conf ||= "/etc/ldap/slapd.d"
|
41
39
|
end
|
42
40
|
|
43
41
|
##
|
@@ -47,7 +45,7 @@ module Backup
|
|
47
45
|
super
|
48
46
|
|
49
47
|
pipeline = Pipeline.new
|
50
|
-
dump_ext =
|
48
|
+
dump_ext = "ldif"
|
51
49
|
|
52
50
|
pipeline << slapcat
|
53
51
|
if @model.compressor
|
@@ -57,8 +55,8 @@ module Backup
|
|
57
55
|
end
|
58
56
|
end
|
59
57
|
|
60
|
-
pipeline << "#{
|
61
|
-
"'#{
|
58
|
+
pipeline << "#{utility(:cat)} > " \
|
59
|
+
"'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
|
62
60
|
|
63
61
|
pipeline.run
|
64
62
|
if pipeline.success?
|
@@ -73,7 +71,7 @@ module Backup
|
|
73
71
|
##
|
74
72
|
# Builds the full slapcat string based on all attributes
|
75
73
|
def slapcat
|
76
|
-
command = "#{
|
74
|
+
command = "#{slapcat_utility} #{slapcat_conf_option} #{slapcat_conf} #{user_options}"
|
77
75
|
command.prepend("sudo ") if use_sudo
|
78
76
|
command
|
79
77
|
end
|
@@ -88,7 +86,7 @@ module Backup
|
|
88
86
|
# Builds a compatible string for the additional options
|
89
87
|
# specified by the user
|
90
88
|
def user_options
|
91
|
-
slapcat_args.join(
|
89
|
+
slapcat_args.join(" ")
|
92
90
|
end
|
93
91
|
end
|
94
92
|
end
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Database
|
5
3
|
class PostgreSQL < Base
|
@@ -53,17 +51,19 @@ module Backup
|
|
53
51
|
super
|
54
52
|
|
55
53
|
pipeline = Pipeline.new
|
56
|
-
dump_ext =
|
54
|
+
dump_ext = "sql"
|
57
55
|
|
58
56
|
pipeline << (dump_all? ? pgdumpall : pgdump)
|
59
57
|
|
60
|
-
model.compressor
|
61
|
-
|
62
|
-
|
63
|
-
|
58
|
+
if model.compressor
|
59
|
+
model.compressor.compress_with do |command, ext|
|
60
|
+
pipeline << command
|
61
|
+
dump_ext << ext
|
62
|
+
end
|
63
|
+
end
|
64
64
|
|
65
|
-
pipeline << "#{
|
66
|
-
|
65
|
+
pipeline << "#{utility(:cat)} > " \
|
66
|
+
"'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
|
67
67
|
|
68
68
|
pipeline.run
|
69
69
|
if pipeline.success?
|
@@ -74,60 +74,59 @@ module Backup
|
|
74
74
|
end
|
75
75
|
|
76
76
|
def pgdump
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
77
|
+
password_option.to_s +
|
78
|
+
sudo_option.to_s +
|
79
|
+
"#{utility(:pg_dump)} #{username_option} #{connectivity_options} " \
|
80
|
+
"#{user_options} #{tables_to_dump} #{tables_to_skip} #{name}"
|
81
81
|
end
|
82
82
|
|
83
83
|
def pgdumpall
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
84
|
+
password_option.to_s +
|
85
|
+
sudo_option.to_s +
|
86
|
+
"#{utility(:pg_dumpall)} #{username_option} " \
|
87
|
+
"#{connectivity_options} #{user_options}"
|
88
88
|
end
|
89
89
|
|
90
90
|
def password_option
|
91
|
-
"PGPASSWORD=#{
|
91
|
+
"PGPASSWORD=#{Shellwords.escape(password)} " if password
|
92
92
|
end
|
93
93
|
|
94
94
|
def sudo_option
|
95
|
-
"#{
|
95
|
+
"#{utility(:sudo)} -n -H -u #{sudo_user} " if sudo_user
|
96
96
|
end
|
97
97
|
|
98
98
|
def username_option
|
99
|
-
"--username=#{
|
99
|
+
"--username=#{Shellwords.escape(username)}" if username
|
100
100
|
end
|
101
101
|
|
102
102
|
def connectivity_options
|
103
|
-
return "--host='#{
|
103
|
+
return "--host='#{socket}'" if socket
|
104
104
|
|
105
105
|
opts = []
|
106
|
-
opts << "--host='#{
|
107
|
-
opts << "--port='#{
|
108
|
-
opts.join(
|
106
|
+
opts << "--host='#{host}'" if host
|
107
|
+
opts << "--port='#{port}'" if port
|
108
|
+
opts.join(" ")
|
109
109
|
end
|
110
110
|
|
111
111
|
def user_options
|
112
|
-
Array(additional_options).join(
|
112
|
+
Array(additional_options).join(" ")
|
113
113
|
end
|
114
114
|
|
115
115
|
def tables_to_dump
|
116
116
|
Array(only_tables).map do |table|
|
117
|
-
"--table='#{
|
118
|
-
end.join(
|
117
|
+
"--table='#{table}'"
|
118
|
+
end.join(" ")
|
119
119
|
end
|
120
120
|
|
121
121
|
def tables_to_skip
|
122
122
|
Array(skip_tables).map do |table|
|
123
|
-
"--exclude-table='#{
|
124
|
-
end.join(
|
123
|
+
"--exclude-table='#{table}'"
|
124
|
+
end.join(" ")
|
125
125
|
end
|
126
126
|
|
127
127
|
def dump_all?
|
128
128
|
name == :all
|
129
129
|
end
|
130
|
-
|
131
130
|
end
|
132
131
|
end
|
133
132
|
end
|