kamal-backup 0.3.0.beta21 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -0
- data/exe/kamal-backup +7 -6
- data/lib/kamal_backup/app.rb +330 -393
- data/lib/kamal_backup/cli/helpers.rb +298 -0
- data/lib/kamal_backup/cli.rb +73 -367
- data/lib/kamal_backup/command.rb +77 -258
- data/lib/kamal_backup/command_output.rb +189 -0
- data/lib/kamal_backup/config.rb +242 -624
- data/lib/kamal_backup/config_file.rb +376 -0
- data/lib/kamal_backup/databases/base.rb +28 -14
- data/lib/kamal_backup/databases/mysql.rb +68 -67
- data/lib/kamal_backup/databases/postgres.rb +59 -58
- data/lib/kamal_backup/databases/sqlite.rb +21 -20
- data/lib/kamal_backup/errors.rb +3 -1
- data/lib/kamal_backup/evidence.rb +61 -63
- data/lib/kamal_backup/kamal_bridge.rb +270 -254
- data/lib/kamal_backup/rails_app.rb +94 -104
- data/lib/kamal_backup/redactor.rb +18 -13
- data/lib/kamal_backup/restic.rb +207 -183
- data/lib/kamal_backup/scheduler.rb +17 -14
- data/lib/kamal_backup/schema.rb +2 -0
- data/lib/kamal_backup/version.rb +3 -1
- data/lib/kamal_backup/yaml_access.rb +13 -0
- data/lib/kamal_backup.rb +22 -17
- metadata +76 -2
data/lib/kamal_backup/restic.rb
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
require
|
|
4
|
-
require
|
|
5
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'English'
|
|
4
|
+
require 'fileutils'
|
|
5
|
+
require 'json'
|
|
6
|
+
require 'open3'
|
|
7
|
+
require 'time'
|
|
8
|
+
require_relative 'command'
|
|
6
9
|
|
|
7
10
|
module KamalBackup
|
|
8
11
|
class Restic
|
|
@@ -18,36 +21,42 @@ module KamalBackup
|
|
|
18
21
|
def ensure_repository
|
|
19
22
|
run(%w[snapshots --json], log_output: false)
|
|
20
23
|
rescue CommandError => e
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
raise e
|
|
26
|
-
end
|
|
24
|
+
raise e unless config.restic_init_if_missing?
|
|
25
|
+
|
|
26
|
+
log('restic repository not ready, running restic init')
|
|
27
|
+
run(%w[init])
|
|
27
28
|
end
|
|
28
29
|
|
|
29
30
|
def backup_stream(command, filename:, tags:)
|
|
30
31
|
restic_command = CommandSpec.new(
|
|
31
|
-
argv: [
|
|
32
|
+
argv: %w[restic
|
|
33
|
+
backup] + host_args + ['--stdin', '--stdin-filename', filename] + tag_args(common_tags + tags),
|
|
32
34
|
env: restic_env
|
|
33
35
|
)
|
|
34
36
|
log("backing up stream as #{filename}")
|
|
35
|
-
pipe_commands(command, restic_command, producer_label:
|
|
37
|
+
pipe_commands(command, restic_command, producer_label: 'dump', consumer_label: 'restic backup')
|
|
36
38
|
end
|
|
37
39
|
|
|
38
40
|
def backup_file(path, filename:, tags:)
|
|
39
41
|
command = CommandSpec.new(
|
|
40
|
-
argv: [
|
|
42
|
+
argv: %w[restic
|
|
43
|
+
backup] + host_args + ['--stdin', '--stdin-filename', filename] + tag_args(common_tags + tags),
|
|
41
44
|
env: restic_env
|
|
42
45
|
)
|
|
43
46
|
log("backing up file content as #{filename}")
|
|
44
47
|
|
|
45
|
-
File.open(path,
|
|
48
|
+
File.open(path, 'rb') do |file|
|
|
46
49
|
output = Command.output
|
|
47
50
|
context = output&.command_start(command, redactor: redactor)
|
|
48
51
|
Open3.popen3(command.env, *command.argv) do |stdin, stdout, stderr, wait_thread|
|
|
49
|
-
stdout_reader = Thread.new
|
|
50
|
-
|
|
52
|
+
stdout_reader = Thread.new do
|
|
53
|
+
Command.collect_stream(stdout, command_output: output, context: context, stream: :stdout,
|
|
54
|
+
redactor: redactor)
|
|
55
|
+
end
|
|
56
|
+
stderr_reader = Thread.new do
|
|
57
|
+
Command.collect_stream(stderr, command_output: output, context: context, stream: :stderr,
|
|
58
|
+
redactor: redactor)
|
|
59
|
+
end
|
|
51
60
|
copy_error = nil
|
|
52
61
|
begin
|
|
53
62
|
IO.copy_stream(file, stdin)
|
|
@@ -67,31 +76,24 @@ module KamalBackup
|
|
|
67
76
|
end
|
|
68
77
|
end
|
|
69
78
|
rescue Errno::ENOENT => e
|
|
70
|
-
raise CommandError.new("command not found: #{command.argv.first}", command: command, status: 127,
|
|
79
|
+
raise CommandError.new("command not found: #{command.argv.first}", command: command, status: 127,
|
|
80
|
+
stderr: e.message)
|
|
71
81
|
end
|
|
72
82
|
|
|
73
83
|
def backup_paths(paths, tags:)
|
|
74
84
|
paths = Array(paths).compact.map(&:to_s).reject(&:empty?)
|
|
75
85
|
|
|
76
|
-
|
|
77
|
-
path_tags = paths.map { |path| "path:#{config.backup_path_label(path)}" }
|
|
78
|
-
excludes = config.backup_path_excludes(paths)
|
|
79
|
-
log("backing up #{paths.size} file path(s): #{paths.join(", ")}")
|
|
80
|
-
run(["backup"] + host_args + exclude_args(excludes) + paths + tag_args(common_tags + tags + path_tags))
|
|
81
|
-
end
|
|
82
|
-
end
|
|
83
|
-
|
|
84
|
-
def backup_path(path, tags:)
|
|
85
|
-
backup_paths([path], tags: tags)
|
|
86
|
-
end
|
|
86
|
+
return unless paths.any?
|
|
87
87
|
|
|
88
|
-
|
|
89
|
-
|
|
88
|
+
path_tags = paths.map { |path| "path:#{config.backup_path_label(path)}" }
|
|
89
|
+
excludes = config.backup_path_excludes(paths)
|
|
90
|
+
log("backing up #{paths.size} file path(s): #{paths.join(', ')}")
|
|
91
|
+
run(['backup'] + host_args + exclude_args(excludes) + paths + tag_args(common_tags + tags + path_tags))
|
|
90
92
|
end
|
|
91
93
|
|
|
92
94
|
def prune
|
|
93
95
|
retention_tag_sets.map do |tags|
|
|
94
|
-
args = [
|
|
96
|
+
args = ['forget', '--prune', '--group-by', 'host'] + config.retention_args + filter_tag_args(tags)
|
|
95
97
|
log("running restic forget/prune with retention policy for #{retention_scope(tags)}")
|
|
96
98
|
run(args)
|
|
97
99
|
end
|
|
@@ -99,39 +101,40 @@ module KamalBackup
|
|
|
99
101
|
|
|
100
102
|
def check
|
|
101
103
|
args = %w[check]
|
|
102
|
-
args.concat([
|
|
104
|
+
args.concat(['--read-data-subset', config.check_read_data_subset]) if config.check_read_data_subset
|
|
103
105
|
started_at = Time.now.utc
|
|
104
106
|
result = run(args)
|
|
105
|
-
write_last_check(status:
|
|
107
|
+
write_last_check(status: 'ok', started_at: started_at, finished_at: Time.now.utc, output: result.stdout)
|
|
106
108
|
result
|
|
107
109
|
rescue CommandError => e
|
|
108
|
-
write_last_check(status:
|
|
110
|
+
write_last_check(status: 'failed', started_at: started_at || Time.now.utc, finished_at: Time.now.utc,
|
|
111
|
+
error: e.message)
|
|
109
112
|
raise
|
|
110
113
|
end
|
|
111
114
|
|
|
112
115
|
def snapshots(tags: common_tags)
|
|
113
|
-
run([
|
|
116
|
+
run(['snapshots'] + filter_tag_args(tags))
|
|
114
117
|
end
|
|
115
118
|
|
|
116
119
|
def snapshots_json(tags: common_tags)
|
|
117
|
-
output = run([
|
|
120
|
+
output = run(['snapshots', '--json'] + filter_tag_args(tags), log_output: false).stdout
|
|
118
121
|
snapshots = JSON.parse(output)
|
|
119
122
|
required_tags = tags.compact
|
|
120
123
|
snapshots.select do |snapshot|
|
|
121
|
-
snapshot_tags = Array(snapshot[
|
|
124
|
+
snapshot_tags = Array(snapshot['tags'])
|
|
122
125
|
required_tags.all? { |tag| snapshot_tags.include?(tag) }
|
|
123
126
|
end
|
|
124
127
|
end
|
|
125
128
|
|
|
126
129
|
def latest_snapshot(tags:)
|
|
127
130
|
snapshots = snapshots_json(tags: common_tags + tags)
|
|
128
|
-
snapshots.max_by { |snapshot| Time.parse(snapshot.fetch(
|
|
131
|
+
snapshots.max_by { |snapshot| Time.parse(snapshot.fetch('time')) }
|
|
129
132
|
rescue JSON::ParserError
|
|
130
133
|
nil
|
|
131
134
|
end
|
|
132
135
|
|
|
133
136
|
def ls_json(snapshot)
|
|
134
|
-
output = run([
|
|
137
|
+
output = run(['ls', '--json', snapshot], log_output: false).stdout
|
|
135
138
|
output.lines.filter_map do |line|
|
|
136
139
|
JSON.parse(line)
|
|
137
140
|
rescue JSON::ParserError
|
|
@@ -139,51 +142,58 @@ module KamalBackup
|
|
|
139
142
|
end
|
|
140
143
|
end
|
|
141
144
|
|
|
142
|
-
|
|
145
|
+
# Database dumps from 0.2 and earlier used different layouts. Keep matching
|
|
146
|
+
# them so snapshots taken before 0.3 stay restorable:
|
|
147
|
+
# databases/<app>/<database>/<adapter>.<ext> is current,
|
|
148
|
+
# databases/<app>/<adapter>/... is 0.2 (no database name), and the
|
|
149
|
+
# databases-<app>-... basenames are the earliest flat filenames.
|
|
150
|
+
def database_file(snapshot, adapter, database_name:)
|
|
143
151
|
legacy_prefix = "databases/#{config.app_name}/#{adapter}/"
|
|
144
|
-
app = config.app_name.gsub(/[^A-Za-z0-9_.-]+/,
|
|
145
|
-
database = database_name.to_s.gsub(/[^A-Za-z0-9_.-]+/,
|
|
146
|
-
|
|
152
|
+
app = config.app_name.gsub(/[^A-Za-z0-9_.-]+/, '-')
|
|
153
|
+
database = database_name.to_s.gsub(/[^A-Za-z0-9_.-]+/, '-')
|
|
154
|
+
prefix = "databases/#{app}/#{database}/#{adapter}."
|
|
147
155
|
flat_prefix = "databases-#{app}-#{adapter}-"
|
|
148
|
-
named_flat_prefix =
|
|
156
|
+
named_flat_prefix = "databases-#{app}-#{database}-#{adapter}-"
|
|
149
157
|
ls_json(snapshot).find do |entry|
|
|
150
|
-
next false unless entry[
|
|
158
|
+
next false unless entry['type'] == 'file'
|
|
151
159
|
|
|
152
|
-
normalized = entry[
|
|
153
|
-
|
|
160
|
+
normalized = entry['path'].to_s.sub(%r{\A/+}, '')
|
|
161
|
+
normalized.start_with?(prefix) ||
|
|
154
162
|
normalized.start_with?(legacy_prefix) ||
|
|
155
|
-
File.basename(normalized).start_with?(flat_prefix)
|
|
156
|
-
|
|
157
|
-
end&.fetch("path")
|
|
163
|
+
File.basename(normalized).start_with?(flat_prefix, named_flat_prefix)
|
|
164
|
+
end&.fetch('path')
|
|
158
165
|
end
|
|
159
166
|
|
|
160
167
|
def pipe_dump_to_command(snapshot, filename, command)
|
|
161
|
-
restic_command = CommandSpec.new(argv: [
|
|
162
|
-
pipe_commands(restic_command, command, producer_label:
|
|
168
|
+
restic_command = CommandSpec.new(argv: ['restic', 'dump', snapshot, filename], env: restic_env)
|
|
169
|
+
pipe_commands(restic_command, command, producer_label: 'restic dump', consumer_label: command.argv.first)
|
|
163
170
|
end
|
|
164
171
|
|
|
165
172
|
def write_dump_to_path(snapshot, filename, target_path)
|
|
166
|
-
command = CommandSpec.new(argv: [
|
|
173
|
+
command = CommandSpec.new(argv: ['restic', 'dump', snapshot, filename], env: restic_env)
|
|
167
174
|
target_path = File.expand_path(target_path)
|
|
168
175
|
FileUtils.mkdir_p(File.dirname(target_path))
|
|
169
|
-
temp_path = "#{target_path}.kamal-backup-#{
|
|
176
|
+
temp_path = "#{target_path}.kamal-backup-#{$PROCESS_ID}.tmp"
|
|
170
177
|
|
|
171
178
|
output = Command.output
|
|
172
179
|
context = output&.command_start(command, redactor: redactor)
|
|
173
180
|
Open3.popen3(command.env, *command.argv) do |stdin, stdout, stderr, wait_thread|
|
|
174
181
|
stdin.close
|
|
175
|
-
stderr_reader = Thread.new
|
|
176
|
-
|
|
182
|
+
stderr_reader = Thread.new do
|
|
183
|
+
Command.collect_stream(stderr, command_output: output, context: context, stream: :stderr, redactor: redactor)
|
|
184
|
+
end
|
|
185
|
+
File.open(temp_path, 'wb') { |file| IO.copy_stream(stdout, file) }
|
|
177
186
|
err = stderr_reader.value
|
|
178
187
|
status = wait_thread.value
|
|
179
188
|
output&.command_exit(context, status.exitstatus)
|
|
180
|
-
raise_command_error(command, status,
|
|
189
|
+
raise_command_error(command, status, '', err) unless status.success?
|
|
181
190
|
end
|
|
182
191
|
File.rename(temp_path, target_path)
|
|
183
192
|
target_path
|
|
184
193
|
rescue Errno::ENOENT => e
|
|
185
194
|
FileUtils.rm_f(temp_path) if temp_path
|
|
186
|
-
raise CommandError.new("command not found: #{command.argv.first}", command: command, status: 127,
|
|
195
|
+
raise CommandError.new("command not found: #{command.argv.first}", command: command, status: 127,
|
|
196
|
+
stderr: e.message)
|
|
187
197
|
rescue StandardError
|
|
188
198
|
FileUtils.rm_f(temp_path) if temp_path
|
|
189
199
|
raise
|
|
@@ -191,162 +201,176 @@ module KamalBackup
|
|
|
191
201
|
|
|
192
202
|
def restore_snapshot(snapshot, target)
|
|
193
203
|
log("restoring file snapshot #{snapshot} to #{target}")
|
|
194
|
-
run([
|
|
204
|
+
run(['restore', snapshot, '--target', target])
|
|
195
205
|
end
|
|
196
206
|
|
|
207
|
+
private
|
|
208
|
+
|
|
197
209
|
def run(args, log_output: true)
|
|
198
210
|
Command.capture(
|
|
199
|
-
CommandSpec.new(argv: [
|
|
211
|
+
CommandSpec.new(argv: ['restic'] + args, env: restic_env),
|
|
200
212
|
redactor: redactor,
|
|
201
213
|
log_output: log_output
|
|
202
214
|
)
|
|
203
215
|
end
|
|
204
216
|
|
|
205
217
|
def common_tags
|
|
206
|
-
[
|
|
218
|
+
['kamal-backup', "app:#{config.app_name}"]
|
|
207
219
|
end
|
|
208
220
|
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
221
|
+
def host_args
|
|
222
|
+
['--host', restic_host]
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def restic_host
|
|
226
|
+
normalize_restic_host([config.app_name, config.accessory_name || 'backup'].compact.join('-'))
|
|
227
|
+
end
|
|
213
228
|
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
229
|
+
def normalize_restic_host(value)
|
|
230
|
+
normalized = value.to_s.gsub(/[^A-Za-z0-9_.-]+/, '-').gsub(/\A-+|-+\z/, '')
|
|
231
|
+
normalized.empty? ? 'kamal-backup' : normalized
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
def tag_args(tags)
|
|
235
|
+
tags.compact.each_with_object([]) { |tag, args| args.concat(['--tag', tag]) }
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
def exclude_args(patterns)
|
|
239
|
+
patterns.compact.each_with_object([]) { |pattern, args| args.concat(['--exclude', pattern]) }
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
def retention_tag_sets
|
|
243
|
+
database_retention_tag_sets + file_retention_tag_sets
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
def database_retention_tag_sets
|
|
247
|
+
config.databases.group_by(&:database_adapter).flat_map do |adapter, databases|
|
|
248
|
+
if databases.one?
|
|
249
|
+
# Snapshots from 0.2 carry no database:<name> tag, so keep the
|
|
250
|
+
# single-database filter broad enough for retention to prune them too.
|
|
251
|
+
[common_tags + ['type:database', "adapter:#{adapter}"]]
|
|
252
|
+
else
|
|
253
|
+
databases.map do |database|
|
|
254
|
+
common_tags + ['type:database', "database:#{database.database_name}", "adapter:#{adapter}"]
|
|
224
255
|
end
|
|
225
256
|
end
|
|
226
257
|
end
|
|
258
|
+
end
|
|
227
259
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
260
|
+
def file_retention_tag_sets
|
|
261
|
+
config.backup_paths.any? ? [common_tags + ['type:files']] : []
|
|
262
|
+
end
|
|
231
263
|
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
264
|
+
def retention_scope(tags)
|
|
265
|
+
tags.reject { |tag| tag == 'kamal-backup' || tag.start_with?('app:') }.join(', ')
|
|
266
|
+
end
|
|
235
267
|
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
268
|
+
def write_last_check(payload)
|
|
269
|
+
FileUtils.mkdir_p(config.state_dir)
|
|
270
|
+
File.write(config.last_check_path, JSON.pretty_generate(payload.transform_values do |value|
|
|
271
|
+
value.respond_to?(:iso8601) ? value.iso8601 : redactor.redact_string(value.to_s)
|
|
272
|
+
end))
|
|
273
|
+
rescue SystemCallError
|
|
274
|
+
nil
|
|
275
|
+
end
|
|
239
276
|
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
277
|
+
def filter_tag_args(tags)
|
|
278
|
+
tags = tags.compact
|
|
279
|
+
tags.empty? ? [] : ['--tag', tags.join(',')]
|
|
280
|
+
end
|
|
243
281
|
|
|
244
|
-
|
|
245
|
-
|
|
282
|
+
def restic_env
|
|
283
|
+
config.env.each_with_object({}) do |(key, value), env|
|
|
284
|
+
env[key] = value if key.to_s.match?(RESTIC_ENV_PATTERN)
|
|
246
285
|
end
|
|
286
|
+
end
|
|
247
287
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
288
|
+
def pipe_commands(producer, consumer, producer_label:, consumer_label:)
|
|
289
|
+
output = Command.output
|
|
290
|
+
producer_context = output&.command_start(producer, redactor: redactor)
|
|
291
|
+
Open3.popen3(producer.env, *producer.argv) do |producer_stdin, producer_stdout, producer_stderr, producer_wait|
|
|
292
|
+
producer_stdin.close
|
|
293
|
+
|
|
294
|
+
consumer_context = output&.command_start(consumer, redactor: redactor)
|
|
295
|
+
Open3.popen3(consumer.env,
|
|
296
|
+
*consumer.argv) do |consumer_stdin, consumer_stdout, consumer_stderr, consumer_wait|
|
|
297
|
+
producer_err_reader = Thread.new do
|
|
298
|
+
Command.collect_stream(producer_stderr, command_output: output, context: producer_context, stream: :stderr,
|
|
299
|
+
redactor: redactor)
|
|
300
|
+
end
|
|
301
|
+
consumer_out_reader = Thread.new do
|
|
302
|
+
Command.collect_stream(consumer_stdout, command_output: output, context: consumer_context, stream: :stdout,
|
|
303
|
+
redactor: redactor)
|
|
304
|
+
end
|
|
305
|
+
consumer_err_reader = Thread.new do
|
|
306
|
+
Command.collect_stream(consumer_stderr, command_output: output, context: consumer_context, stream: :stderr,
|
|
307
|
+
redactor: redactor)
|
|
308
|
+
end
|
|
251
309
|
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
310
|
+
copy_error = nil
|
|
311
|
+
copy_thread = Thread.new do
|
|
312
|
+
IO.copy_stream(producer_stdout, consumer_stdin)
|
|
313
|
+
rescue StandardError => e
|
|
314
|
+
copy_error = e
|
|
315
|
+
ensure
|
|
316
|
+
consumer_stdin.close unless consumer_stdin.closed?
|
|
317
|
+
end
|
|
256
318
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
319
|
+
copy_thread.join
|
|
320
|
+
producer_status = producer_wait.value
|
|
321
|
+
consumer_status = consumer_wait.value
|
|
322
|
+
output&.command_exit(producer_context, producer_status.exitstatus)
|
|
323
|
+
output&.command_exit(consumer_context, consumer_status.exitstatus)
|
|
324
|
+
|
|
325
|
+
producer_err = producer_err_reader.value
|
|
326
|
+
consumer_out = consumer_out_reader.value
|
|
327
|
+
consumer_err = consumer_err_reader.value
|
|
328
|
+
|
|
329
|
+
if copy_error
|
|
330
|
+
raise CommandError.new(
|
|
331
|
+
"failed to pipe #{producer_label} to #{consumer_label}: #{copy_error.message}",
|
|
332
|
+
command: consumer,
|
|
333
|
+
stderr: copy_error.message
|
|
334
|
+
)
|
|
335
|
+
end
|
|
261
336
|
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
env[key] = value if key.to_s.match?(RESTIC_ENV_PATTERN)
|
|
265
|
-
end
|
|
266
|
-
end
|
|
337
|
+
raise_command_error(producer, producer_status, '', producer_err) unless producer_status.success?
|
|
338
|
+
raise_command_error(consumer, consumer_status, consumer_out, consumer_err) unless consumer_status.success?
|
|
267
339
|
|
|
268
|
-
|
|
269
|
-
output = Command.output
|
|
270
|
-
producer_context = output&.command_start(producer, redactor: redactor)
|
|
271
|
-
Open3.popen3(producer.env, *producer.argv) do |producer_stdin, producer_stdout, producer_stderr, producer_wait|
|
|
272
|
-
producer_stdin.close
|
|
273
|
-
|
|
274
|
-
consumer_context = output&.command_start(consumer, redactor: redactor)
|
|
275
|
-
Open3.popen3(consumer.env, *consumer.argv) do |consumer_stdin, consumer_stdout, consumer_stderr, consumer_wait|
|
|
276
|
-
producer_err_reader = Thread.new { Command.collect_stream(producer_stderr, command_output: output, context: producer_context, stream: :stderr, redactor: redactor) }
|
|
277
|
-
consumer_out_reader = Thread.new { Command.collect_stream(consumer_stdout, command_output: output, context: consumer_context, stream: :stdout, redactor: redactor) }
|
|
278
|
-
consumer_err_reader = Thread.new { Command.collect_stream(consumer_stderr, command_output: output, context: consumer_context, stream: :stderr, redactor: redactor) }
|
|
279
|
-
|
|
280
|
-
copy_error = nil
|
|
281
|
-
copy_thread = Thread.new do
|
|
282
|
-
IO.copy_stream(producer_stdout, consumer_stdin)
|
|
283
|
-
rescue StandardError => e
|
|
284
|
-
copy_error = e
|
|
285
|
-
ensure
|
|
286
|
-
consumer_stdin.close unless consumer_stdin.closed?
|
|
287
|
-
end
|
|
288
|
-
|
|
289
|
-
copy_thread.join
|
|
290
|
-
producer_status = producer_wait.value
|
|
291
|
-
consumer_status = consumer_wait.value
|
|
292
|
-
output&.command_exit(producer_context, producer_status.exitstatus)
|
|
293
|
-
output&.command_exit(consumer_context, consumer_status.exitstatus)
|
|
294
|
-
|
|
295
|
-
producer_err = producer_err_reader.value
|
|
296
|
-
consumer_out = consumer_out_reader.value
|
|
297
|
-
consumer_err = consumer_err_reader.value
|
|
298
|
-
|
|
299
|
-
if copy_error
|
|
300
|
-
raise CommandError.new(
|
|
301
|
-
"failed to pipe #{producer_label} to #{consumer_label}: #{copy_error.message}",
|
|
302
|
-
command: consumer,
|
|
303
|
-
stderr: copy_error.message
|
|
304
|
-
)
|
|
305
|
-
end
|
|
306
|
-
|
|
307
|
-
raise_command_error(producer, producer_status, "", producer_err) unless producer_status.success?
|
|
308
|
-
raise_command_error(consumer, consumer_status, consumer_out, consumer_err) unless consumer_status.success?
|
|
309
|
-
|
|
310
|
-
CommandResult.new(stdout: consumer_out, stderr: consumer_err, status: consumer_status.exitstatus)
|
|
311
|
-
end
|
|
340
|
+
CommandResult.new(stdout: consumer_out, stderr: consumer_err, status: consumer_status.exitstatus)
|
|
312
341
|
end
|
|
313
|
-
rescue Errno::ENOENT => e
|
|
314
|
-
command = e.message.include?(producer.argv.first) ? producer : consumer
|
|
315
|
-
raise CommandError.new("command not found: #{command.argv.first}", command: command, status: 127, stderr: e.message)
|
|
316
|
-
end
|
|
317
|
-
|
|
318
|
-
def raise_command_error(command, status, stdout, stderr)
|
|
319
|
-
raise CommandError.new(
|
|
320
|
-
"command failed (#{status.exitstatus}): #{command.display(redactor)}\n#{redactor.redact_string(stderr)}",
|
|
321
|
-
command: command,
|
|
322
|
-
status: status.exitstatus,
|
|
323
|
-
stdout: redactor.redact_string(stdout),
|
|
324
|
-
stderr: redactor.redact_string(stderr)
|
|
325
|
-
)
|
|
326
342
|
end
|
|
343
|
+
rescue Errno::ENOENT => e
|
|
344
|
+
command = e.message.include?(producer.argv.first) ? producer : consumer
|
|
345
|
+
raise CommandError.new("command not found: #{command.argv.first}", command: command, status: 127,
|
|
346
|
+
stderr: e.message)
|
|
347
|
+
end
|
|
327
348
|
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
)
|
|
335
|
-
|
|
349
|
+
def raise_command_error(command, status, stdout, stderr)
|
|
350
|
+
raise CommandError.new(
|
|
351
|
+
"command failed (#{status.exitstatus}): #{command.display(redactor)}\n#{redactor.redact_string(stderr)}",
|
|
352
|
+
command: command,
|
|
353
|
+
status: status.exitstatus,
|
|
354
|
+
stdout: redactor.redact_string(stdout),
|
|
355
|
+
stderr: redactor.redact_string(stderr)
|
|
356
|
+
)
|
|
357
|
+
end
|
|
336
358
|
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
359
|
+
def raise_stream_error(command, error, stdout, stderr)
|
|
360
|
+
raise CommandError.new(
|
|
361
|
+
"failed to stream file to #{command.display(redactor)}: #{error.message}\n#{redactor.redact_string(stderr)}",
|
|
362
|
+
command: command,
|
|
363
|
+
stdout: redactor.redact_string(stdout),
|
|
364
|
+
stderr: redactor.redact_string(stderr)
|
|
365
|
+
)
|
|
366
|
+
end
|
|
343
367
|
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
end
|
|
368
|
+
def log(message)
|
|
369
|
+
if Command.output
|
|
370
|
+
Command.output.info(message, redactor: redactor)
|
|
371
|
+
else
|
|
372
|
+
$stdout.puts("[kamal-backup] #{redactor.redact_string(message)}")
|
|
350
373
|
end
|
|
374
|
+
end
|
|
351
375
|
end
|
|
352
376
|
end
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'time'
|
|
2
4
|
|
|
3
5
|
module KamalBackup
|
|
4
6
|
class Scheduler
|
|
@@ -32,21 +34,22 @@ module KamalBackup
|
|
|
32
34
|
end
|
|
33
35
|
|
|
34
36
|
private
|
|
35
|
-
def install_signal_handlers
|
|
36
|
-
%w[TERM INT].each do |signal|
|
|
37
|
-
Signal.trap(signal) { @stop = true }
|
|
38
|
-
rescue ArgumentError
|
|
39
|
-
nil
|
|
40
|
-
end
|
|
41
|
-
end
|
|
42
37
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
38
|
+
def install_signal_handlers
|
|
39
|
+
%w[TERM INT].each do |signal|
|
|
40
|
+
Signal.trap(signal) { @stop = true }
|
|
41
|
+
rescue ArgumentError
|
|
42
|
+
nil
|
|
46
43
|
end
|
|
44
|
+
end
|
|
47
45
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
46
|
+
def sleep_interruptibly(seconds)
|
|
47
|
+
deadline = Time.now + seconds
|
|
48
|
+
sleep([deadline - Time.now, 1].min) while !@stop && Time.now < deadline
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def log(message)
|
|
52
|
+
$stdout.puts("[kamal-backup] #{message}")
|
|
53
|
+
end
|
|
51
54
|
end
|
|
52
55
|
end
|
data/lib/kamal_backup/schema.rb
CHANGED
data/lib/kamal_backup/version.rb
CHANGED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module KamalBackup
|
|
4
|
+
# Kamal and Rails YAML configs mix string and symbol keys depending on how
|
|
5
|
+
# they were rendered, so look up both.
|
|
6
|
+
module YamlAccess
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def fetch(hash, key)
|
|
10
|
+
hash[key] || hash[key.to_s] || hash[key.to_sym]
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
data/lib/kamal_backup.rb
CHANGED
|
@@ -1,17 +1,22 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
require_relative
|
|
4
|
-
require_relative
|
|
5
|
-
require_relative
|
|
6
|
-
require_relative
|
|
7
|
-
require_relative
|
|
8
|
-
require_relative
|
|
9
|
-
require_relative
|
|
10
|
-
require_relative
|
|
11
|
-
require_relative
|
|
12
|
-
require_relative
|
|
13
|
-
require_relative
|
|
14
|
-
require_relative
|
|
15
|
-
require_relative
|
|
16
|
-
require_relative
|
|
17
|
-
require_relative
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative 'kamal_backup/version'
|
|
4
|
+
require_relative 'kamal_backup/schema'
|
|
5
|
+
require_relative 'kamal_backup/errors'
|
|
6
|
+
require_relative 'kamal_backup/yaml_access'
|
|
7
|
+
require_relative 'kamal_backup/command'
|
|
8
|
+
require_relative 'kamal_backup/command_output'
|
|
9
|
+
require_relative 'kamal_backup/redactor'
|
|
10
|
+
require_relative 'kamal_backup/config_file'
|
|
11
|
+
require_relative 'kamal_backup/config'
|
|
12
|
+
require_relative 'kamal_backup/rails_app'
|
|
13
|
+
require_relative 'kamal_backup/kamal_bridge'
|
|
14
|
+
require_relative 'kamal_backup/restic'
|
|
15
|
+
require_relative 'kamal_backup/evidence'
|
|
16
|
+
require_relative 'kamal_backup/scheduler'
|
|
17
|
+
require_relative 'kamal_backup/databases/base'
|
|
18
|
+
require_relative 'kamal_backup/databases/postgres'
|
|
19
|
+
require_relative 'kamal_backup/databases/mysql'
|
|
20
|
+
require_relative 'kamal_backup/databases/sqlite'
|
|
21
|
+
require_relative 'kamal_backup/app'
|
|
22
|
+
require_relative 'kamal_backup/cli'
|