backup 4.3.0 → 5.0.0.beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/LICENSE +19 -0
- data/README.md +13 -9
- data/bin/docker_test +24 -0
- data/lib/backup.rb +74 -78
- data/lib/backup/archive.rb +31 -32
- data/lib/backup/binder.rb +2 -6
- data/lib/backup/cleaner.rb +14 -18
- data/lib/backup/cli.rb +104 -108
- data/lib/backup/cloud_io/base.rb +4 -7
- data/lib/backup/cloud_io/cloud_files.rb +60 -62
- data/lib/backup/cloud_io/s3.rb +69 -76
- data/lib/backup/compressor/base.rb +4 -7
- data/lib/backup/compressor/bzip2.rb +3 -7
- data/lib/backup/compressor/custom.rb +2 -6
- data/lib/backup/compressor/gzip.rb +16 -17
- data/lib/backup/config.rb +17 -18
- data/lib/backup/config/dsl.rb +16 -17
- data/lib/backup/config/helpers.rb +10 -16
- data/lib/backup/database/base.rb +22 -21
- data/lib/backup/database/mongodb.rb +36 -37
- data/lib/backup/database/mysql.rb +40 -41
- data/lib/backup/database/openldap.rb +8 -10
- data/lib/backup/database/postgresql.rb +29 -30
- data/lib/backup/database/redis.rb +27 -30
- data/lib/backup/database/riak.rb +15 -18
- data/lib/backup/database/sqlite.rb +4 -6
- data/lib/backup/encryptor/base.rb +2 -4
- data/lib/backup/encryptor/gpg.rb +49 -59
- data/lib/backup/encryptor/open_ssl.rb +11 -14
- data/lib/backup/errors.rb +7 -12
- data/lib/backup/logger.rb +16 -18
- data/lib/backup/logger/console.rb +5 -8
- data/lib/backup/logger/fog_adapter.rb +2 -6
- data/lib/backup/logger/logfile.rb +10 -12
- data/lib/backup/logger/syslog.rb +2 -4
- data/lib/backup/model.rb +33 -40
- data/lib/backup/notifier/base.rb +24 -26
- data/lib/backup/notifier/campfire.rb +9 -11
- data/lib/backup/notifier/command.rb +3 -3
- data/lib/backup/notifier/datadog.rb +9 -12
- data/lib/backup/notifier/flowdock.rb +13 -17
- data/lib/backup/notifier/hipchat.rb +18 -14
- data/lib/backup/notifier/http_post.rb +11 -14
- data/lib/backup/notifier/mail.rb +42 -54
- data/lib/backup/notifier/nagios.rb +5 -9
- data/lib/backup/notifier/pagerduty.rb +10 -12
- data/lib/backup/notifier/prowl.rb +15 -15
- data/lib/backup/notifier/pushover.rb +7 -10
- data/lib/backup/notifier/ses.rb +52 -17
- data/lib/backup/notifier/slack.rb +39 -40
- data/lib/backup/notifier/twitter.rb +2 -5
- data/lib/backup/notifier/zabbix.rb +11 -14
- data/lib/backup/package.rb +5 -9
- data/lib/backup/packager.rb +16 -17
- data/lib/backup/pipeline.rb +17 -21
- data/lib/backup/splitter.rb +8 -11
- data/lib/backup/storage/base.rb +5 -8
- data/lib/backup/storage/cloud_files.rb +21 -23
- data/lib/backup/storage/cycler.rb +10 -15
- data/lib/backup/storage/dropbox.rb +15 -21
- data/lib/backup/storage/ftp.rb +14 -10
- data/lib/backup/storage/local.rb +5 -8
- data/lib/backup/storage/qiniu.rb +8 -8
- data/lib/backup/storage/rsync.rb +24 -26
- data/lib/backup/storage/s3.rb +27 -28
- data/lib/backup/storage/scp.rb +10 -12
- data/lib/backup/storage/sftp.rb +10 -12
- data/lib/backup/syncer/base.rb +5 -8
- data/lib/backup/syncer/cloud/base.rb +27 -30
- data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
- data/lib/backup/syncer/cloud/local_file.rb +5 -8
- data/lib/backup/syncer/cloud/s3.rb +23 -24
- data/lib/backup/syncer/rsync/base.rb +6 -10
- data/lib/backup/syncer/rsync/local.rb +1 -5
- data/lib/backup/syncer/rsync/pull.rb +6 -10
- data/lib/backup/syncer/rsync/push.rb +18 -22
- data/lib/backup/template.rb +9 -14
- data/lib/backup/utilities.rb +78 -69
- data/lib/backup/version.rb +1 -3
- metadata +107 -677
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Database
|
5
3
|
class MongoDB < Base
|
@@ -60,7 +58,6 @@ module Backup
|
|
60
58
|
lock_database if @lock
|
61
59
|
dump!
|
62
60
|
package!
|
63
|
-
|
64
61
|
ensure
|
65
62
|
unlock_database if @lock
|
66
63
|
end
|
@@ -78,7 +75,7 @@ module Backup
|
|
78
75
|
run(mongodump)
|
79
76
|
else
|
80
77
|
collections.each do |collection|
|
81
|
-
run("#{
|
78
|
+
run("#{mongodump} --collection='#{collection}'")
|
82
79
|
end
|
83
80
|
end
|
84
81
|
end
|
@@ -92,25 +89,27 @@ module Backup
|
|
92
89
|
# If successful, +dump_packaging_path+ is removed.
|
93
90
|
def package!
|
94
91
|
pipeline = Pipeline.new
|
95
|
-
dump_ext =
|
92
|
+
dump_ext = "tar"
|
96
93
|
|
97
|
-
pipeline << "#{
|
98
|
-
"-C '#{
|
94
|
+
pipeline << "#{utility(:tar)} -cf - " \
|
95
|
+
"-C '#{dump_path}' '#{dump_filename}'"
|
99
96
|
|
100
|
-
model.compressor
|
101
|
-
|
102
|
-
|
103
|
-
|
97
|
+
if model.compressor
|
98
|
+
model.compressor.compress_with do |command, ext|
|
99
|
+
pipeline << command
|
100
|
+
dump_ext << ext
|
101
|
+
end
|
102
|
+
end
|
104
103
|
|
105
|
-
pipeline << "#{
|
106
|
-
|
104
|
+
pipeline << "#{utility(:cat)} > " \
|
105
|
+
"'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
|
107
106
|
|
108
107
|
pipeline.run
|
109
108
|
if pipeline.success?
|
110
109
|
FileUtils.rm_rf dump_packaging_path
|
111
110
|
log!(:finished)
|
112
111
|
else
|
113
|
-
raise Error, "Dump Failed!\n
|
112
|
+
raise Error, "Dump Failed!\n#{pipeline.error_messages}"
|
114
113
|
end
|
115
114
|
end
|
116
115
|
|
@@ -119,69 +118,69 @@ module Backup
|
|
119
118
|
end
|
120
119
|
|
121
120
|
def mongodump
|
122
|
-
"#{
|
123
|
-
|
124
|
-
|
121
|
+
"#{utility(:mongodump)} #{name_option} #{credential_options} " \
|
122
|
+
"#{connectivity_options} #{ipv6_option} #{oplog_option} " \
|
123
|
+
"#{user_options} --out='#{dump_packaging_path}'"
|
125
124
|
end
|
126
125
|
|
127
126
|
def name_option
|
128
|
-
|
127
|
+
return unless name
|
128
|
+
"--db='#{name}'"
|
129
129
|
end
|
130
130
|
|
131
131
|
def credential_options
|
132
132
|
opts = []
|
133
|
-
opts << "--username='#{
|
134
|
-
opts << "--password='#{
|
135
|
-
opts << "--authenticationDatabase='#{
|
136
|
-
opts.join(
|
133
|
+
opts << "--username='#{username}'" if username
|
134
|
+
opts << "--password='#{password}'" if password
|
135
|
+
opts << "--authenticationDatabase='#{authdb}'" if authdb
|
136
|
+
opts.join(" ")
|
137
137
|
end
|
138
138
|
|
139
139
|
def connectivity_options
|
140
140
|
opts = []
|
141
|
-
opts << "--host='#{
|
142
|
-
opts << "--port='#{
|
143
|
-
opts.join(
|
141
|
+
opts << "--host='#{host}'" if host
|
142
|
+
opts << "--port='#{port}'" if port
|
143
|
+
opts.join(" ")
|
144
144
|
end
|
145
145
|
|
146
146
|
def ipv6_option
|
147
|
-
|
147
|
+
"--ipv6" if ipv6
|
148
148
|
end
|
149
149
|
|
150
150
|
def oplog_option
|
151
|
-
|
151
|
+
"--oplog" if oplog
|
152
152
|
end
|
153
153
|
|
154
154
|
def user_options
|
155
|
-
Array(additional_options).join(
|
155
|
+
Array(additional_options).join(" ")
|
156
156
|
end
|
157
157
|
|
158
158
|
def lock_database
|
159
|
-
lock_command = <<-EOS.gsub(/^ +/,
|
159
|
+
lock_command = <<-EOS.gsub(/^ +/, "")
|
160
160
|
echo 'use admin
|
161
161
|
db.setProfilingLevel(0)
|
162
|
-
db.fsyncLock()' | #{
|
162
|
+
db.fsyncLock()' | #{mongo_shell}
|
163
163
|
EOS
|
164
164
|
|
165
165
|
run(lock_command)
|
166
166
|
end
|
167
167
|
|
168
168
|
def unlock_database
|
169
|
-
unlock_command = <<-EOS.gsub(/^ +/,
|
169
|
+
unlock_command = <<-EOS.gsub(/^ +/, "")
|
170
170
|
echo 'use admin
|
171
|
-
db.fsyncUnlock()' | #{
|
171
|
+
db.fsyncUnlock()' | #{mongo_shell}
|
172
172
|
EOS
|
173
173
|
|
174
174
|
run(unlock_command)
|
175
175
|
end
|
176
176
|
|
177
177
|
def mongo_shell
|
178
|
-
cmd = "#{
|
179
|
-
cmd << " #{
|
180
|
-
cmd << " #{
|
181
|
-
cmd << " '#{
|
178
|
+
cmd = "#{utility(:mongo)} #{connectivity_options}".rstrip
|
179
|
+
cmd << " #{credential_options}".rstrip
|
180
|
+
cmd << " #{ipv6_option}".rstrip
|
181
|
+
cmd << " '#{name}'" if name
|
182
182
|
cmd
|
183
183
|
end
|
184
|
-
|
185
184
|
end
|
186
185
|
end
|
187
186
|
end
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Database
|
5
3
|
class MySQL < Base
|
@@ -78,71 +76,73 @@ module Backup
|
|
78
76
|
super
|
79
77
|
|
80
78
|
pipeline = Pipeline.new
|
81
|
-
dump_ext = sql_backup? ?
|
79
|
+
dump_ext = sql_backup? ? "sql" : "tar"
|
82
80
|
|
83
81
|
pipeline << sudo_option(sql_backup? ? mysqldump : innobackupex)
|
84
82
|
|
85
|
-
model.compressor
|
86
|
-
|
87
|
-
|
88
|
-
|
83
|
+
if model.compressor
|
84
|
+
model.compressor.compress_with do |command, ext|
|
85
|
+
pipeline << command
|
86
|
+
dump_ext << ext
|
87
|
+
end
|
88
|
+
end
|
89
89
|
|
90
|
-
pipeline << "#{
|
91
|
-
|
90
|
+
pipeline << "#{utility(:cat)} > " \
|
91
|
+
"'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
|
92
92
|
|
93
93
|
pipeline.run
|
94
94
|
if pipeline.success?
|
95
95
|
log!(:finished)
|
96
96
|
else
|
97
|
-
raise Error, "Dump Failed!\n
|
97
|
+
raise Error, "Dump Failed!\n#{pipeline.error_messages}"
|
98
98
|
end
|
99
99
|
end
|
100
100
|
|
101
101
|
private
|
102
102
|
|
103
103
|
def mysqldump
|
104
|
-
"#{
|
105
|
-
|
106
|
-
|
104
|
+
"#{utility(:mysqldump)} #{user_options} #{credential_options} " \
|
105
|
+
"#{connectivity_options} #{name_option} " \
|
106
|
+
"#{tables_to_dump} #{tables_to_skip}"
|
107
107
|
end
|
108
108
|
|
109
109
|
def credential_options
|
110
110
|
opts = []
|
111
|
-
opts << "--user=#{
|
112
|
-
opts << "--password=#{
|
113
|
-
opts.join(
|
111
|
+
opts << "--user=#{Shellwords.escape(username)}" if username
|
112
|
+
opts << "--password=#{Shellwords.escape(password)}" if password
|
113
|
+
opts.join(" ")
|
114
114
|
end
|
115
115
|
|
116
116
|
def connectivity_options
|
117
|
-
return "--socket='#{
|
117
|
+
return "--socket='#{socket}'" if socket
|
118
118
|
|
119
119
|
opts = []
|
120
|
-
opts << "--host='#{
|
121
|
-
opts << "--port='#{
|
122
|
-
opts.join(
|
120
|
+
opts << "--host='#{host}'" if host
|
121
|
+
opts << "--port='#{port}'" if port
|
122
|
+
opts.join(" ")
|
123
123
|
end
|
124
124
|
|
125
125
|
def user_options
|
126
|
-
Array(additional_options).join(
|
126
|
+
Array(additional_options).join(" ")
|
127
127
|
end
|
128
128
|
|
129
129
|
def user_prepare_options
|
130
|
-
Array(prepare_options).join(
|
130
|
+
Array(prepare_options).join(" ")
|
131
131
|
end
|
132
132
|
|
133
133
|
def name_option
|
134
|
-
dump_all? ?
|
134
|
+
dump_all? ? "--all-databases" : name
|
135
135
|
end
|
136
136
|
|
137
137
|
def tables_to_dump
|
138
|
-
Array(only_tables).join(
|
138
|
+
Array(only_tables).join(" ") unless dump_all?
|
139
139
|
end
|
140
140
|
|
141
141
|
def tables_to_skip
|
142
142
|
Array(skip_tables).map do |table|
|
143
|
-
table =
|
144
|
-
"--ignore-table='#{
|
145
|
-
end.join(
|
143
|
+
table = dump_all? || table["."] ? table : "#{name}.#{table}"
|
144
|
+
"--ignore-table='#{table}'"
|
145
|
+
end.join(" ")
|
146
146
|
end
|
147
147
|
|
148
148
|
def dump_all?
|
@@ -155,28 +155,28 @@ module Backup
|
|
155
155
|
|
156
156
|
def innobackupex
|
157
157
|
# Creation phase
|
158
|
-
"#{
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
158
|
+
"#{utility(:innobackupex)} #{credential_options} " \
|
159
|
+
"#{connectivity_options} #{user_options} " \
|
160
|
+
"--no-timestamp #{temp_dir} #{quiet_option} && " +
|
161
|
+
innobackupex_prepare +
|
162
|
+
# Move files to tar-ed stream on stdout
|
163
|
+
"#{utility(:tar)} --remove-files -cf - " \
|
164
|
+
"-C #{File.dirname(temp_dir)} #{File.basename(temp_dir)}"
|
165
165
|
end
|
166
166
|
|
167
167
|
def innobackupex_prepare
|
168
168
|
return "" unless @prepare_backup
|
169
169
|
# Log applying phase (prepare for restore)
|
170
|
-
"#{
|
171
|
-
|
170
|
+
"#{utility(:innobackupex)} --apply-log #{temp_dir} " \
|
171
|
+
"#{user_prepare_options} #{quiet_option} && "
|
172
172
|
end
|
173
173
|
|
174
174
|
def sudo_option(command_block)
|
175
175
|
return command_block unless sudo_user
|
176
176
|
|
177
|
-
"sudo -s -u #{
|
178
|
-
|
179
|
-
|
177
|
+
"sudo -s -u #{sudo_user} -- <<END_OF_SUDO\n" \
|
178
|
+
"#{command_block}\n" \
|
179
|
+
"END_OF_SUDO\n"
|
180
180
|
end
|
181
181
|
|
182
182
|
def quiet_option
|
@@ -184,9 +184,8 @@ module Backup
|
|
184
184
|
end
|
185
185
|
|
186
186
|
def temp_dir
|
187
|
-
File.join(dump_path, dump_filename
|
187
|
+
File.join(dump_path, "#{dump_filename}.bkpdir")
|
188
188
|
end
|
189
|
-
|
190
189
|
end
|
191
190
|
end
|
192
191
|
end
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Database
|
5
3
|
class OpenLDAP < Base
|
@@ -33,11 +31,11 @@ module Backup
|
|
33
31
|
super
|
34
32
|
instance_eval(&block) if block_given?
|
35
33
|
|
36
|
-
@name ||=
|
34
|
+
@name ||= "ldap_backup"
|
37
35
|
@use_sudo ||= false
|
38
|
-
@slapcat_args ||=
|
36
|
+
@slapcat_args ||= []
|
39
37
|
@slapcat_utility ||= utility(:slapcat)
|
40
|
-
@slapcat_conf ||=
|
38
|
+
@slapcat_conf ||= "/etc/ldap/slapd.d"
|
41
39
|
end
|
42
40
|
|
43
41
|
##
|
@@ -47,7 +45,7 @@ module Backup
|
|
47
45
|
super
|
48
46
|
|
49
47
|
pipeline = Pipeline.new
|
50
|
-
dump_ext =
|
48
|
+
dump_ext = "ldif"
|
51
49
|
|
52
50
|
pipeline << slapcat
|
53
51
|
if @model.compressor
|
@@ -57,8 +55,8 @@ module Backup
|
|
57
55
|
end
|
58
56
|
end
|
59
57
|
|
60
|
-
pipeline << "#{
|
61
|
-
"'#{
|
58
|
+
pipeline << "#{utility(:cat)} > " \
|
59
|
+
"'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
|
62
60
|
|
63
61
|
pipeline.run
|
64
62
|
if pipeline.success?
|
@@ -73,7 +71,7 @@ module Backup
|
|
73
71
|
##
|
74
72
|
# Builds the full slapcat string based on all attributes
|
75
73
|
def slapcat
|
76
|
-
command = "#{
|
74
|
+
command = "#{slapcat_utility} #{slapcat_conf_option} #{slapcat_conf} #{user_options}"
|
77
75
|
command.prepend("sudo ") if use_sudo
|
78
76
|
command
|
79
77
|
end
|
@@ -88,7 +86,7 @@ module Backup
|
|
88
86
|
# Builds a compatible string for the additional options
|
89
87
|
# specified by the user
|
90
88
|
def user_options
|
91
|
-
slapcat_args.join(
|
89
|
+
slapcat_args.join(" ")
|
92
90
|
end
|
93
91
|
end
|
94
92
|
end
|
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Database
|
5
3
|
class PostgreSQL < Base
|
@@ -53,17 +51,19 @@ module Backup
|
|
53
51
|
super
|
54
52
|
|
55
53
|
pipeline = Pipeline.new
|
56
|
-
dump_ext =
|
54
|
+
dump_ext = "sql"
|
57
55
|
|
58
56
|
pipeline << (dump_all? ? pgdumpall : pgdump)
|
59
57
|
|
60
|
-
model.compressor
|
61
|
-
|
62
|
-
|
63
|
-
|
58
|
+
if model.compressor
|
59
|
+
model.compressor.compress_with do |command, ext|
|
60
|
+
pipeline << command
|
61
|
+
dump_ext << ext
|
62
|
+
end
|
63
|
+
end
|
64
64
|
|
65
|
-
pipeline << "#{
|
66
|
-
|
65
|
+
pipeline << "#{utility(:cat)} > " \
|
66
|
+
"'#{File.join(dump_path, dump_filename)}.#{dump_ext}'"
|
67
67
|
|
68
68
|
pipeline.run
|
69
69
|
if pipeline.success?
|
@@ -74,60 +74,59 @@ module Backup
|
|
74
74
|
end
|
75
75
|
|
76
76
|
def pgdump
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
77
|
+
password_option.to_s +
|
78
|
+
sudo_option.to_s +
|
79
|
+
"#{utility(:pg_dump)} #{username_option} #{connectivity_options} " \
|
80
|
+
"#{user_options} #{tables_to_dump} #{tables_to_skip} #{name}"
|
81
81
|
end
|
82
82
|
|
83
83
|
def pgdumpall
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
84
|
+
password_option.to_s +
|
85
|
+
sudo_option.to_s +
|
86
|
+
"#{utility(:pg_dumpall)} #{username_option} " \
|
87
|
+
"#{connectivity_options} #{user_options}"
|
88
88
|
end
|
89
89
|
|
90
90
|
def password_option
|
91
|
-
"PGPASSWORD=#{
|
91
|
+
"PGPASSWORD=#{Shellwords.escape(password)} " if password
|
92
92
|
end
|
93
93
|
|
94
94
|
def sudo_option
|
95
|
-
"#{
|
95
|
+
"#{utility(:sudo)} -n -H -u #{sudo_user} " if sudo_user
|
96
96
|
end
|
97
97
|
|
98
98
|
def username_option
|
99
|
-
"--username=#{
|
99
|
+
"--username=#{Shellwords.escape(username)}" if username
|
100
100
|
end
|
101
101
|
|
102
102
|
def connectivity_options
|
103
|
-
return "--host='#{
|
103
|
+
return "--host='#{socket}'" if socket
|
104
104
|
|
105
105
|
opts = []
|
106
|
-
opts << "--host='#{
|
107
|
-
opts << "--port='#{
|
108
|
-
opts.join(
|
106
|
+
opts << "--host='#{host}'" if host
|
107
|
+
opts << "--port='#{port}'" if port
|
108
|
+
opts.join(" ")
|
109
109
|
end
|
110
110
|
|
111
111
|
def user_options
|
112
|
-
Array(additional_options).join(
|
112
|
+
Array(additional_options).join(" ")
|
113
113
|
end
|
114
114
|
|
115
115
|
def tables_to_dump
|
116
116
|
Array(only_tables).map do |table|
|
117
|
-
"--table='#{
|
118
|
-
end.join(
|
117
|
+
"--table='#{table}'"
|
118
|
+
end.join(" ")
|
119
119
|
end
|
120
120
|
|
121
121
|
def tables_to_skip
|
122
122
|
Array(skip_tables).map do |table|
|
123
|
-
"--exclude-table='#{
|
124
|
-
end.join(
|
123
|
+
"--exclude-table='#{table}'"
|
124
|
+
end.join(" ")
|
125
125
|
end
|
126
126
|
|
127
127
|
def dump_all?
|
128
128
|
name == :all
|
129
129
|
end
|
130
|
-
|
131
130
|
end
|
132
131
|
end
|
133
132
|
end
|