backup 4.4.1 → 5.0.0.beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/LICENSE +19 -0
- data/README.md +1 -1
- data/lib/backup.rb +74 -78
- data/lib/backup/archive.rb +31 -32
- data/lib/backup/binder.rb +2 -6
- data/lib/backup/cleaner.rb +14 -18
- data/lib/backup/cli.rb +104 -108
- data/lib/backup/cloud_io/base.rb +4 -7
- data/lib/backup/cloud_io/cloud_files.rb +60 -62
- data/lib/backup/cloud_io/s3.rb +69 -76
- data/lib/backup/compressor/base.rb +4 -7
- data/lib/backup/compressor/bzip2.rb +3 -7
- data/lib/backup/compressor/custom.rb +2 -6
- data/lib/backup/compressor/gzip.rb +16 -17
- data/lib/backup/config.rb +17 -18
- data/lib/backup/config/dsl.rb +16 -17
- data/lib/backup/config/helpers.rb +10 -16
- data/lib/backup/database/base.rb +22 -21
- data/lib/backup/database/mongodb.rb +36 -37
- data/lib/backup/database/mysql.rb +40 -41
- data/lib/backup/database/openldap.rb +8 -10
- data/lib/backup/database/postgresql.rb +29 -30
- data/lib/backup/database/redis.rb +27 -30
- data/lib/backup/database/riak.rb +15 -18
- data/lib/backup/database/sqlite.rb +4 -6
- data/lib/backup/encryptor/base.rb +2 -4
- data/lib/backup/encryptor/gpg.rb +49 -59
- data/lib/backup/encryptor/open_ssl.rb +11 -14
- data/lib/backup/errors.rb +7 -12
- data/lib/backup/logger.rb +16 -18
- data/lib/backup/logger/console.rb +5 -8
- data/lib/backup/logger/fog_adapter.rb +2 -6
- data/lib/backup/logger/logfile.rb +10 -12
- data/lib/backup/logger/syslog.rb +2 -4
- data/lib/backup/model.rb +75 -40
- data/lib/backup/notifier/base.rb +24 -26
- data/lib/backup/notifier/campfire.rb +9 -11
- data/lib/backup/notifier/command.rb +0 -3
- data/lib/backup/notifier/datadog.rb +9 -12
- data/lib/backup/notifier/flowdock.rb +13 -17
- data/lib/backup/notifier/hipchat.rb +11 -13
- data/lib/backup/notifier/http_post.rb +11 -14
- data/lib/backup/notifier/mail.rb +44 -47
- data/lib/backup/notifier/nagios.rb +5 -9
- data/lib/backup/notifier/pagerduty.rb +10 -12
- data/lib/backup/notifier/prowl.rb +15 -15
- data/lib/backup/notifier/pushover.rb +7 -10
- data/lib/backup/notifier/ses.rb +34 -16
- data/lib/backup/notifier/slack.rb +39 -40
- data/lib/backup/notifier/twitter.rb +2 -5
- data/lib/backup/notifier/zabbix.rb +11 -14
- data/lib/backup/package.rb +5 -9
- data/lib/backup/packager.rb +16 -17
- data/lib/backup/pipeline.rb +17 -21
- data/lib/backup/splitter.rb +8 -11
- data/lib/backup/storage/base.rb +5 -8
- data/lib/backup/storage/cloud_files.rb +21 -23
- data/lib/backup/storage/cycler.rb +10 -15
- data/lib/backup/storage/dropbox.rb +15 -21
- data/lib/backup/storage/ftp.rb +8 -10
- data/lib/backup/storage/local.rb +5 -8
- data/lib/backup/storage/qiniu.rb +8 -8
- data/lib/backup/storage/rsync.rb +24 -26
- data/lib/backup/storage/s3.rb +27 -28
- data/lib/backup/storage/scp.rb +10 -12
- data/lib/backup/storage/sftp.rb +10 -12
- data/lib/backup/syncer/base.rb +5 -8
- data/lib/backup/syncer/cloud/base.rb +27 -30
- data/lib/backup/syncer/cloud/cloud_files.rb +16 -18
- data/lib/backup/syncer/cloud/local_file.rb +5 -8
- data/lib/backup/syncer/cloud/s3.rb +23 -24
- data/lib/backup/syncer/rsync/base.rb +6 -10
- data/lib/backup/syncer/rsync/local.rb +1 -5
- data/lib/backup/syncer/rsync/pull.rb +6 -10
- data/lib/backup/syncer/rsync/push.rb +18 -22
- data/lib/backup/template.rb +9 -14
- data/lib/backup/utilities.rb +82 -69
- data/lib/backup/version.rb +1 -3
- metadata +100 -660
data/lib/backup/storage/ftp.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
|
-
|
2
|
-
require 'net/ftp'
|
1
|
+
require "net/ftp"
|
3
2
|
|
4
3
|
module Backup
|
5
4
|
module Storage
|
@@ -31,10 +30,10 @@ module Backup
|
|
31
30
|
super
|
32
31
|
|
33
32
|
@port ||= 21
|
34
|
-
@path ||=
|
33
|
+
@path ||= "backups"
|
35
34
|
@passive_mode ||= false
|
36
35
|
@timeout ||= nil
|
37
|
-
path.sub!(/^~\//,
|
36
|
+
path.sub!(/^~\//, "")
|
38
37
|
end
|
39
38
|
|
40
39
|
private
|
@@ -68,7 +67,7 @@ module Backup
|
|
68
67
|
package.filenames.each do |filename|
|
69
68
|
src = File.join(Config.tmp_path, filename)
|
70
69
|
dest = File.join(remote_path, filename)
|
71
|
-
Logger.info "Storing '#{
|
70
|
+
Logger.info "Storing '#{ip}:#{dest}'..."
|
72
71
|
ftp.put(src, dest)
|
73
72
|
end
|
74
73
|
end
|
@@ -77,7 +76,7 @@ module Backup
|
|
77
76
|
# Called by the Cycler.
|
78
77
|
# Any error raised will be logged as a warning.
|
79
78
|
def remove!(package)
|
80
|
-
Logger.info "Removing backup package dated #{
|
79
|
+
Logger.info "Removing backup package dated #{package.time}..."
|
81
80
|
|
82
81
|
remote_path = remote_path_for(package)
|
83
82
|
connection do |ftp|
|
@@ -98,15 +97,14 @@ module Backup
|
|
98
97
|
# Net::FTP raises an exception when the directory it's trying to create
|
99
98
|
# already exists, so we have rescue it
|
100
99
|
def create_remote_path(ftp)
|
101
|
-
path_parts =
|
102
|
-
remote_path.split(
|
100
|
+
path_parts = []
|
101
|
+
remote_path.split("/").each do |path_part|
|
103
102
|
path_parts << path_part
|
104
103
|
begin
|
105
|
-
ftp.mkdir(path_parts.join(
|
104
|
+
ftp.mkdir(path_parts.join("/"))
|
106
105
|
rescue Net::FTPPermError; end
|
107
106
|
end
|
108
107
|
end
|
109
|
-
|
110
108
|
end
|
111
109
|
end
|
112
110
|
end
|
data/lib/backup/storage/local.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Storage
|
5
3
|
class Local < Base
|
@@ -9,7 +7,7 @@ module Backup
|
|
9
7
|
def initialize(model, storage_id = nil)
|
10
8
|
super
|
11
9
|
|
12
|
-
@path ||=
|
10
|
+
@path ||= "~/backups"
|
13
11
|
end
|
14
12
|
|
15
13
|
private
|
@@ -21,7 +19,7 @@ module Backup
|
|
21
19
|
package.filenames.each do |filename|
|
22
20
|
src = File.join(Config.tmp_path, filename)
|
23
21
|
dest = File.join(remote_path, filename)
|
24
|
-
Logger.info "Storing '#{
|
22
|
+
Logger.info "Storing '#{dest}'..."
|
25
23
|
|
26
24
|
FileUtils.send(transfer_method, src, dest)
|
27
25
|
end
|
@@ -30,7 +28,7 @@ module Backup
|
|
30
28
|
# Called by the Cycler.
|
31
29
|
# Any error raised will be logged as a warning.
|
32
30
|
def remove!(package)
|
33
|
-
Logger.info "Removing backup package dated #{
|
31
|
+
Logger.info "Removing backup package dated #{package.time}..."
|
34
32
|
|
35
33
|
FileUtils.rm_r(remote_path_for(package))
|
36
34
|
end
|
@@ -50,15 +48,14 @@ module Backup
|
|
50
48
|
else
|
51
49
|
Logger.warn Error.new(<<-EOS)
|
52
50
|
Local File Copy Warning!
|
53
|
-
The final backup file(s) for '#{
|
54
|
-
will be *copied* to '#{
|
51
|
+
The final backup file(s) for '#{model.label}' (#{model.trigger})
|
52
|
+
will be *copied* to '#{remote_path}'
|
55
53
|
To avoid this, when using more than one Storage, the 'Local' Storage
|
56
54
|
should be added *last* so the files may be *moved* to their destination.
|
57
55
|
EOS
|
58
56
|
false
|
59
57
|
end
|
60
58
|
end
|
61
|
-
|
62
59
|
end
|
63
60
|
end
|
64
61
|
end
|
data/lib/backup/storage/qiniu.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
|
-
|
2
|
-
require 'qiniu'
|
1
|
+
require "qiniu"
|
3
2
|
|
4
3
|
module Backup
|
5
4
|
module Storage
|
@@ -18,18 +17,19 @@ module Backup
|
|
18
17
|
def initialize(model, storage_id = nil)
|
19
18
|
super
|
20
19
|
|
21
|
-
@path ||=
|
20
|
+
@path ||= "backups"
|
22
21
|
|
23
22
|
check_configuration
|
24
23
|
config_credentials
|
25
24
|
end
|
26
25
|
|
27
26
|
private
|
27
|
+
|
28
28
|
def transfer!
|
29
29
|
package.filenames.each do |filename|
|
30
30
|
src = File.join(Config.tmp_path, filename)
|
31
31
|
dest = File.join(remote_path, filename)
|
32
|
-
Logger.info "Storing '#{
|
32
|
+
Logger.info "Storing '#{dest}'..."
|
33
33
|
|
34
34
|
::Qiniu.upload_file(uptoken: ::Qiniu.generate_upload_token,
|
35
35
|
bucket: bucket,
|
@@ -41,7 +41,7 @@ module Backup
|
|
41
41
|
# Called by the Cycler.
|
42
42
|
# Any error raised will be logged as a warning.
|
43
43
|
def remove!(package)
|
44
|
-
Logger.info "Removing backup package dated #{
|
44
|
+
Logger.info "Removing backup package dated #{package.time}..."
|
45
45
|
remote_path = remote_path_for(package)
|
46
46
|
package.filenames.each do |filename|
|
47
47
|
::Qiniu.delete(bucket, File.join(remote_path, filename))
|
@@ -49,11 +49,11 @@ module Backup
|
|
49
49
|
end
|
50
50
|
|
51
51
|
def check_configuration
|
52
|
-
|
52
|
+
required = %w[access_key secret_key bucket]
|
53
53
|
|
54
|
-
raise Error, <<-EOS if required.map {|name| send(name) }.any?(&:nil?)
|
54
|
+
raise Error, <<-EOS if required.map { |name| send(name) }.any?(&:nil?)
|
55
55
|
Configuration Error
|
56
|
-
#{
|
56
|
+
#{required.map { |name| "##{name}" }.join(", ")} are all required
|
57
57
|
EOS
|
58
58
|
end
|
59
59
|
|
data/lib/backup/storage/rsync.rb
CHANGED
@@ -1,5 +1,3 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
module Backup
|
4
2
|
module Storage
|
5
3
|
class RSync < Base
|
@@ -133,7 +131,7 @@ module Backup
|
|
133
131
|
@mode ||= :ssh
|
134
132
|
@port ||= mode == :rsync_daemon ? 873 : 22
|
135
133
|
@compress ||= false
|
136
|
-
@path ||=
|
134
|
+
@path ||= "~/backups"
|
137
135
|
end
|
138
136
|
|
139
137
|
private
|
@@ -143,10 +141,10 @@ module Backup
|
|
143
141
|
create_remote_path
|
144
142
|
|
145
143
|
package.filenames.each do |filename|
|
146
|
-
src = "'#{
|
147
|
-
dest = "#{
|
148
|
-
Logger.info "Syncing to #{
|
149
|
-
run("#{
|
144
|
+
src = "'#{File.join(Config.tmp_path, filename)}'"
|
145
|
+
dest = "#{host_options}'#{File.join(remote_path, filename)}'"
|
146
|
+
Logger.info "Syncing to #{dest}..."
|
147
|
+
run("#{rsync_command} #{src} #{dest}")
|
150
148
|
end
|
151
149
|
ensure
|
152
150
|
remove_password_file
|
@@ -159,7 +157,7 @@ module Backup
|
|
159
157
|
def remote_path
|
160
158
|
@remote_path ||= begin
|
161
159
|
if host
|
162
|
-
path.sub(/^~\//,
|
160
|
+
path.sub(/^~\//, "").sub(/\/$/, "")
|
163
161
|
else
|
164
162
|
File.expand_path(path)
|
165
163
|
end
|
@@ -176,8 +174,9 @@ module Backup
|
|
176
174
|
# module name that must define a path on the remote that already exists.
|
177
175
|
def create_remote_path
|
178
176
|
if host
|
179
|
-
|
180
|
-
|
177
|
+
return unless mode == :ssh
|
178
|
+
run "#{utility(:ssh)} #{ssh_transport_args} #{host} " +
|
179
|
+
%("mkdir -p '#{remote_path}'")
|
181
180
|
else
|
182
181
|
FileUtils.mkdir_p(remote_path)
|
183
182
|
end
|
@@ -186,55 +185,55 @@ module Backup
|
|
186
185
|
def host_options
|
187
186
|
@host_options ||= begin
|
188
187
|
if !host
|
189
|
-
|
188
|
+
""
|
190
189
|
elsif mode == :ssh
|
191
|
-
"#{
|
190
|
+
"#{host}:"
|
192
191
|
else
|
193
|
-
user = "#{
|
194
|
-
"#{
|
192
|
+
user = "#{rsync_user}@" if rsync_user
|
193
|
+
"#{user}#{host}::"
|
195
194
|
end
|
196
195
|
end
|
197
196
|
end
|
198
197
|
|
199
198
|
def rsync_command
|
200
199
|
@rsync_command ||= begin
|
201
|
-
cmd = utility(:rsync) <<
|
202
|
-
|
200
|
+
cmd = utility(:rsync) << " --archive" <<
|
201
|
+
" #{Array(additional_rsync_options).join(" ")}".rstrip
|
203
202
|
cmd << compress_option << password_option << transport_options if host
|
204
203
|
cmd
|
205
204
|
end
|
206
205
|
end
|
207
206
|
|
208
207
|
def compress_option
|
209
|
-
compress ?
|
208
|
+
compress ? " --compress" : ""
|
210
209
|
end
|
211
210
|
|
212
211
|
def password_option
|
213
|
-
return
|
212
|
+
return "" if mode == :ssh
|
214
213
|
|
215
214
|
path = @password_file ? @password_file.path : rsync_password_file
|
216
|
-
path ? " --password-file='#{
|
215
|
+
path ? " --password-file='#{File.expand_path(path)}'" : ""
|
217
216
|
end
|
218
217
|
|
219
218
|
def transport_options
|
220
219
|
if mode == :rsync_daemon
|
221
|
-
" --port #{
|
220
|
+
" --port #{port}"
|
222
221
|
else
|
223
|
-
%
|
222
|
+
%( -e "#{utility(:ssh)} #{ssh_transport_args}")
|
224
223
|
end
|
225
224
|
end
|
226
225
|
|
227
226
|
def ssh_transport_args
|
228
|
-
args = "-p #{
|
229
|
-
args << "-l #{
|
230
|
-
args << Array(additional_ssh_options).join(
|
227
|
+
args = "-p #{port} "
|
228
|
+
args << "-l #{ssh_user} " if ssh_user
|
229
|
+
args << Array(additional_ssh_options).join(" ")
|
231
230
|
args.rstrip
|
232
231
|
end
|
233
232
|
|
234
233
|
def write_password_file
|
235
234
|
return unless host && rsync_password && mode != :ssh
|
236
235
|
|
237
|
-
@password_file = Tempfile.new(
|
236
|
+
@password_file = Tempfile.new("backup-rsync-password")
|
238
237
|
@password_file.write(rsync_password)
|
239
238
|
@password_file.close
|
240
239
|
end
|
@@ -242,7 +241,6 @@ module Backup
|
|
242
241
|
def remove_password_file
|
243
242
|
@password_file.delete if @password_file
|
244
243
|
end
|
245
|
-
|
246
244
|
end
|
247
245
|
end
|
248
246
|
end
|
data/lib/backup/storage/s3.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
|
-
|
2
|
-
require 'backup/cloud_io/s3'
|
1
|
+
require "backup/cloud_io/s3"
|
3
2
|
|
4
3
|
module Backup
|
5
4
|
module Storage
|
@@ -75,10 +74,10 @@ module Backup
|
|
75
74
|
@chunk_size ||= 5 # MiB
|
76
75
|
@max_retries ||= 10
|
77
76
|
@retry_waitsec ||= 30
|
78
|
-
@path ||=
|
77
|
+
@path ||= "backups"
|
79
78
|
@storage_class ||= :standard
|
80
79
|
|
81
|
-
@path = @path.sub(/^\//,
|
80
|
+
@path = @path.sub(/^\//, "")
|
82
81
|
|
83
82
|
check_configuration
|
84
83
|
end
|
@@ -87,17 +86,17 @@ module Backup
|
|
87
86
|
|
88
87
|
def cloud_io
|
89
88
|
@cloud_io ||= CloudIO::S3.new(
|
90
|
-
:
|
91
|
-
:
|
92
|
-
:
|
93
|
-
:
|
94
|
-
:
|
95
|
-
:
|
96
|
-
:
|
97
|
-
:
|
98
|
-
:
|
99
|
-
:
|
100
|
-
:
|
89
|
+
access_key_id: access_key_id,
|
90
|
+
secret_access_key: secret_access_key,
|
91
|
+
use_iam_profile: use_iam_profile,
|
92
|
+
region: region,
|
93
|
+
bucket: bucket,
|
94
|
+
encryption: encryption,
|
95
|
+
storage_class: storage_class,
|
96
|
+
max_retries: max_retries,
|
97
|
+
retry_waitsec: retry_waitsec,
|
98
|
+
chunk_size: chunk_size,
|
99
|
+
fog_options: fog_options
|
101
100
|
)
|
102
101
|
end
|
103
102
|
|
@@ -105,7 +104,7 @@ module Backup
|
|
105
104
|
package.filenames.each do |filename|
|
106
105
|
src = File.join(Config.tmp_path, filename)
|
107
106
|
dest = File.join(remote_path, filename)
|
108
|
-
Logger.info "Storing '#{
|
107
|
+
Logger.info "Storing '#{bucket}/#{dest}'..."
|
109
108
|
cloud_io.upload(src, dest)
|
110
109
|
end
|
111
110
|
end
|
@@ -113,25 +112,26 @@ module Backup
|
|
113
112
|
# Called by the Cycler.
|
114
113
|
# Any error raised will be logged as a warning.
|
115
114
|
def remove!(package)
|
116
|
-
Logger.info "Removing backup package dated #{
|
115
|
+
Logger.info "Removing backup package dated #{package.time}..."
|
117
116
|
|
118
117
|
remote_path = remote_path_for(package)
|
119
118
|
objects = cloud_io.objects(remote_path)
|
120
119
|
|
121
|
-
raise Error, "Package at '#{
|
120
|
+
raise Error, "Package at '#{remote_path}' not found" if objects.empty?
|
122
121
|
|
123
122
|
cloud_io.delete(objects)
|
124
123
|
end
|
125
124
|
|
126
125
|
def check_configuration
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
126
|
+
required =
|
127
|
+
if use_iam_profile
|
128
|
+
%w[bucket]
|
129
|
+
else
|
130
|
+
%w[access_key_id secret_access_key bucket]
|
131
|
+
end
|
132
|
+
raise Error, <<-EOS if required.map { |name| send(name) }.any?(&:nil?)
|
133
133
|
Configuration Error
|
134
|
-
#{
|
134
|
+
#{required.map { |name| "##{name}" }.join(", ")} are all required
|
135
135
|
EOS
|
136
136
|
|
137
137
|
raise Error, <<-EOS if chunk_size > 0 && !chunk_size.between?(5, 5120)
|
@@ -139,18 +139,17 @@ module Backup
|
|
139
139
|
#chunk_size must be between 5 and 5120 (or 0 to disable multipart)
|
140
140
|
EOS
|
141
141
|
|
142
|
-
raise Error, <<-EOS if encryption && encryption.to_s.upcase !=
|
142
|
+
raise Error, <<-EOS if encryption && encryption.to_s.upcase != "AES256"
|
143
143
|
Configuration Error
|
144
144
|
#encryption must be :aes256 or nil
|
145
145
|
EOS
|
146
146
|
|
147
|
-
classes = [
|
147
|
+
classes = ["STANDARD", "STANDARD_IA", "REDUCED_REDUNDANCY"]
|
148
148
|
raise Error, <<-EOS unless classes.include?(storage_class.to_s.upcase)
|
149
149
|
Configuration Error
|
150
150
|
#storage_class must be :standard or :standard_ia or :reduced_redundancy
|
151
151
|
EOS
|
152
152
|
end
|
153
|
-
|
154
153
|
end
|
155
154
|
end
|
156
155
|
end
|
data/lib/backup/storage/scp.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
|
-
|
2
|
-
require 'net/scp'
|
1
|
+
require "net/scp"
|
3
2
|
|
4
3
|
module Backup
|
5
4
|
module Storage
|
@@ -19,27 +18,27 @@ module Backup
|
|
19
18
|
super
|
20
19
|
|
21
20
|
@port ||= 22
|
22
|
-
@path ||=
|
21
|
+
@path ||= "backups"
|
23
22
|
@ssh_options ||= {}
|
24
|
-
path.sub!(/^~\//,
|
23
|
+
path.sub!(/^~\//, "")
|
25
24
|
end
|
26
25
|
|
27
26
|
private
|
28
27
|
|
29
28
|
def connection
|
30
29
|
Net::SSH.start(
|
31
|
-
ip, username, { :
|
32
|
-
) {|ssh| yield ssh }
|
30
|
+
ip, username, { password: password, port: port }.merge(ssh_options)
|
31
|
+
) { |ssh| yield ssh }
|
33
32
|
end
|
34
33
|
|
35
34
|
def transfer!
|
36
35
|
connection do |ssh|
|
37
|
-
ssh.exec!("mkdir -p '#{
|
36
|
+
ssh.exec!("mkdir -p '#{remote_path}'")
|
38
37
|
|
39
38
|
package.filenames.each do |filename|
|
40
39
|
src = File.join(Config.tmp_path, filename)
|
41
40
|
dest = File.join(remote_path, filename)
|
42
|
-
Logger.info "Storing '#{
|
41
|
+
Logger.info "Storing '#{ip}:#{dest}'..."
|
43
42
|
ssh.scp.upload!(src, dest)
|
44
43
|
end
|
45
44
|
end
|
@@ -48,20 +47,19 @@ module Backup
|
|
48
47
|
# Called by the Cycler.
|
49
48
|
# Any error raised will be logged as a warning.
|
50
49
|
def remove!(package)
|
51
|
-
Logger.info "Removing backup package dated #{
|
50
|
+
Logger.info "Removing backup package dated #{package.time}..."
|
52
51
|
|
53
52
|
errors = []
|
54
53
|
connection do |ssh|
|
55
|
-
ssh.exec!("rm -r '#{
|
54
|
+
ssh.exec!("rm -r '#{remote_path_for(package)}'") do |_, stream, data|
|
56
55
|
errors << data if stream == :stderr
|
57
56
|
end
|
58
57
|
end
|
59
58
|
unless errors.empty?
|
60
59
|
raise Error, "Net::SSH reported the following errors:\n" +
|
61
|
-
|
60
|
+
errors.join("\n")
|
62
61
|
end
|
63
62
|
end
|
64
|
-
|
65
63
|
end
|
66
64
|
end
|
67
65
|
end
|