kamal-backup 0.3.0.beta21 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/exe/kamal-backup +7 -6
- data/lib/kamal_backup/app.rb +350 -356
- data/lib/kamal_backup/cli.rb +107 -111
- data/lib/kamal_backup/command.rb +165 -161
- data/lib/kamal_backup/config.rb +533 -511
- data/lib/kamal_backup/databases/base.rb +17 -14
- data/lib/kamal_backup/databases/mysql.rb +68 -67
- data/lib/kamal_backup/databases/postgres.rb +59 -58
- data/lib/kamal_backup/databases/sqlite.rb +21 -20
- data/lib/kamal_backup/errors.rb +3 -1
- data/lib/kamal_backup/evidence.rb +62 -61
- data/lib/kamal_backup/kamal_bridge.rb +254 -250
- data/lib/kamal_backup/rails_app.rb +102 -101
- data/lib/kamal_backup/redactor.rb +18 -13
- data/lib/kamal_backup/restic.rb +195 -167
- data/lib/kamal_backup/scheduler.rb +17 -14
- data/lib/kamal_backup/schema.rb +2 -0
- data/lib/kamal_backup/version.rb +3 -1
- data/lib/kamal_backup.rb +19 -17
- metadata +30 -2
data/lib/kamal_backup/command.rb
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
require
|
|
4
|
-
require
|
|
5
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'open3'
|
|
4
|
+
require 'pty'
|
|
5
|
+
require 'securerandom'
|
|
6
|
+
require 'shellwords'
|
|
7
|
+
require_relative 'errors'
|
|
6
8
|
|
|
7
9
|
module KamalBackup
|
|
8
10
|
class CommandSpec
|
|
@@ -17,12 +19,12 @@ module KamalBackup
|
|
|
17
19
|
result[key.to_s] = value.to_s
|
|
18
20
|
end
|
|
19
21
|
|
|
20
|
-
raise ArgumentError,
|
|
22
|
+
raise ArgumentError, 'command argv cannot be empty' if @argv.empty?
|
|
21
23
|
end
|
|
22
24
|
|
|
23
25
|
def display(redactor)
|
|
24
26
|
env_prefix = env.keys.sort.map { |key| "#{key}=#{redactor.redact_value(key, env[key])}" }
|
|
25
|
-
redactor.redact_string((env_prefix + [argv.shelljoin]).join(
|
|
27
|
+
redactor.redact_string((env_prefix + [argv.shelljoin]).join(' '))
|
|
26
28
|
end
|
|
27
29
|
end
|
|
28
30
|
|
|
@@ -30,18 +32,18 @@ module KamalBackup
|
|
|
30
32
|
|
|
31
33
|
class CommandOutput
|
|
32
34
|
LEVELS = {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
35
|
+
'DEBUG' => 0,
|
|
36
|
+
'INFO' => 1,
|
|
37
|
+
'WARN' => 2,
|
|
38
|
+
'ERROR' => 3,
|
|
39
|
+
'FATAL' => 4
|
|
38
40
|
}.freeze
|
|
39
41
|
LEVEL_COLORS = {
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
42
|
+
'DEBUG' => :black,
|
|
43
|
+
'INFO' => :blue,
|
|
44
|
+
'WARN' => :yellow,
|
|
45
|
+
'ERROR' => :red,
|
|
46
|
+
'FATAL' => :red
|
|
45
47
|
}.freeze
|
|
46
48
|
COLOR_CODES = {
|
|
47
49
|
black: 30,
|
|
@@ -71,11 +73,11 @@ module KamalBackup
|
|
|
71
73
|
end
|
|
72
74
|
|
|
73
75
|
def info(message, redactor:)
|
|
74
|
-
write_message(
|
|
76
|
+
write_message('INFO', redactor.redact_string(message))
|
|
75
77
|
end
|
|
76
78
|
|
|
77
79
|
def error(message, redactor:)
|
|
78
|
-
write_message(
|
|
80
|
+
write_message('ERROR', colorize(redactor.redact_string(message), :red, :bold))
|
|
79
81
|
end
|
|
80
82
|
|
|
81
83
|
def decorate(value, color, mode = nil)
|
|
@@ -87,19 +89,19 @@ module KamalBackup
|
|
|
87
89
|
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
88
90
|
display = spec.display(redactor)
|
|
89
91
|
|
|
90
|
-
write_message(
|
|
91
|
-
write_message(
|
|
92
|
+
write_message('INFO', "Running #{colorize(display, :yellow, :bold)} #{target_for(spec)}", id)
|
|
93
|
+
write_message('DEBUG', "Command: #{colorize(display, :blue)}", id)
|
|
92
94
|
|
|
93
95
|
{ id: id, started_at: started_at, redactor: redactor }
|
|
94
96
|
end
|
|
95
97
|
|
|
96
|
-
def command_output(context,
|
|
98
|
+
def command_output(context, stream, data, redactor:)
|
|
97
99
|
raw = data.to_s
|
|
98
100
|
return if raw.empty?
|
|
99
|
-
return unless log_level?(
|
|
101
|
+
return unless log_level?('DEBUG')
|
|
100
102
|
|
|
101
103
|
synchronize do
|
|
102
|
-
key = [context.fetch(:id),
|
|
104
|
+
key = [context.fetch(:id), stream]
|
|
103
105
|
@buffers[key] = "#{@buffers[key]}#{raw}"
|
|
104
106
|
flush_complete_output_lines(context, key, redactor: redactor)
|
|
105
107
|
end
|
|
@@ -107,106 +109,109 @@ module KamalBackup
|
|
|
107
109
|
|
|
108
110
|
def command_exit(context, status)
|
|
109
111
|
runtime = Process.clock_gettime(Process::CLOCK_MONOTONIC) - context.fetch(:started_at)
|
|
110
|
-
result = status.to_i.zero? ?
|
|
112
|
+
result = status.to_i.zero? ? 'successful' : 'failed'
|
|
111
113
|
result_color = status.to_i.zero? ? :green : :red
|
|
112
114
|
|
|
113
115
|
synchronize do
|
|
114
116
|
flush_output_buffers(context)
|
|
115
|
-
|
|
117
|
+
message = "Finished in #{format('%.3f seconds', runtime)} with exit status #{status} " \
|
|
118
|
+
"(#{colorize(result, result_color, :bold)})."
|
|
119
|
+
write_message_unlocked('INFO', message, context.fetch(:id))
|
|
116
120
|
end
|
|
117
121
|
end
|
|
118
122
|
|
|
119
123
|
private
|
|
120
|
-
def write_message(level, message, id = nil)
|
|
121
|
-
return unless log_level?(level)
|
|
122
124
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
+
def write_message(level, message, id = nil)
|
|
126
|
+
return unless log_level?(level)
|
|
125
127
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
end
|
|
128
|
+
synchronize { write_message_unlocked(level, message, id) }
|
|
129
|
+
end
|
|
129
130
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
131
|
+
def write_message_unlocked(level, message, id = nil)
|
|
132
|
+
@io.puts(format_message(level, message, id)) if log_level?(level)
|
|
133
|
+
end
|
|
133
134
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
135
|
+
def synchronize(&block)
|
|
136
|
+
@mutex.synchronize(&block)
|
|
137
|
+
end
|
|
137
138
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
139
|
+
def flush_complete_output_lines(context, key, redactor:)
|
|
140
|
+
buffer = @buffers.fetch(key)
|
|
141
|
+
output = +''
|
|
141
142
|
|
|
142
|
-
|
|
143
|
-
|
|
143
|
+
while (index = buffer.index("\n"))
|
|
144
|
+
output << buffer.slice!(0..index)
|
|
144
145
|
end
|
|
145
146
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
147
|
+
@buffers[key] = buffer
|
|
148
|
+
write_output(context, output, redactor: redactor, stream: key.last) unless output.empty?
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def flush_output_buffers(context)
|
|
152
|
+
id = context.fetch(:id)
|
|
153
|
+
keys = @buffers.keys.select { |key_id, _stream| key_id == id }
|
|
149
154
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
155
|
+
keys.each do |key|
|
|
156
|
+
output = @buffers.delete(key)
|
|
157
|
+
next if output.to_s.empty?
|
|
153
158
|
|
|
154
|
-
|
|
155
|
-
end
|
|
159
|
+
write_output(context, output, redactor: context.fetch(:redactor), stream: key.last)
|
|
156
160
|
end
|
|
161
|
+
end
|
|
157
162
|
|
|
158
|
-
|
|
159
|
-
|
|
163
|
+
def write_output(context, output, redactor:, stream: nil)
|
|
164
|
+
color = stream == :stderr ? :red : :green
|
|
160
165
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
end
|
|
164
|
-
@io.flush if @io.respond_to?(:flush)
|
|
166
|
+
redactor.redact_string(output).each_line do |line|
|
|
167
|
+
write_message_unlocked('DEBUG', colorize("\t#{line}".chomp, color), context.fetch(:id))
|
|
165
168
|
end
|
|
169
|
+
@io.flush if @io.respond_to?(:flush)
|
|
170
|
+
end
|
|
166
171
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
172
|
+
def format_message(level, message, id = nil)
|
|
173
|
+
message = "[#{colorize(id, :green)}] #{message}" if id
|
|
174
|
+
"#{colorize(level.rjust(6), LEVEL_COLORS.fetch(level))} #{message}"
|
|
175
|
+
end
|
|
171
176
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
177
|
+
def local_target
|
|
178
|
+
if (remote_host = @env['KAMAL_HOST'].to_s) && !remote_host.empty?
|
|
179
|
+
return "on #{colorize(remote_host, :blue)}"
|
|
180
|
+
end
|
|
176
181
|
|
|
177
|
-
|
|
182
|
+
user = @env['USER'].to_s.empty? ? @env['USERNAME'].to_s : @env['USER'].to_s
|
|
178
183
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
end
|
|
184
|
+
if user.empty?
|
|
185
|
+
"on #{colorize('localhost', :blue)}"
|
|
186
|
+
else
|
|
187
|
+
"as #{colorize(user, :blue)}@#{colorize('localhost', :blue)}"
|
|
184
188
|
end
|
|
189
|
+
end
|
|
185
190
|
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
end
|
|
191
|
+
def target_for(spec)
|
|
192
|
+
if spec.host
|
|
193
|
+
"on #{colorize(spec.host, :blue)}"
|
|
194
|
+
else
|
|
195
|
+
local_target
|
|
192
196
|
end
|
|
197
|
+
end
|
|
193
198
|
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
199
|
+
def log_level?(level)
|
|
200
|
+
LEVELS.fetch(level) >= @verbosity
|
|
201
|
+
end
|
|
197
202
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
203
|
+
def colorize(value, color, mode = nil)
|
|
204
|
+
string = value.to_s
|
|
205
|
+
return string unless colorize?
|
|
206
|
+
return string unless COLOR_CODES.key?(color)
|
|
202
207
|
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
208
|
+
prefix = mode == :bold ? "\e[1;" : "\e[0;"
|
|
209
|
+
"#{prefix}#{COLOR_CODES.fetch(color)};49m#{string}\e[0m"
|
|
210
|
+
end
|
|
206
211
|
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
212
|
+
def colorize?
|
|
213
|
+
@env['SSHKIT_COLOR'] || (@io.respond_to?(:tty?) && @io.tty?)
|
|
214
|
+
end
|
|
210
215
|
end
|
|
211
216
|
|
|
212
217
|
class Command
|
|
@@ -222,13 +227,13 @@ module KamalBackup
|
|
|
222
227
|
end
|
|
223
228
|
|
|
224
229
|
def available?(name)
|
|
225
|
-
ENV.fetch(
|
|
230
|
+
ENV.fetch('PATH', '').split(File::PATH_SEPARATOR).any? do |dir|
|
|
226
231
|
path = File.join(dir, name)
|
|
227
232
|
File.executable?(path) && !File.directory?(path)
|
|
228
233
|
end
|
|
229
234
|
end
|
|
230
235
|
|
|
231
|
-
def capture(spec, input: nil,
|
|
236
|
+
def capture(spec, redactor:, input: nil, log: true, log_output: true, tee_stdout: nil, tee_stderr: nil)
|
|
232
237
|
output = log ? self.output : nil
|
|
233
238
|
return capture_quietly(spec, input: input, redactor: redactor) unless output || tee_stdout || tee_stderr
|
|
234
239
|
|
|
@@ -244,20 +249,19 @@ module KamalBackup
|
|
|
244
249
|
tee_stderr: tee_stderr
|
|
245
250
|
)
|
|
246
251
|
output&.command_exit(context, status.exitstatus)
|
|
247
|
-
result = CommandResult.new(stdout: stdout, stderr: stderr, status: status.exitstatus,
|
|
252
|
+
result = CommandResult.new(stdout: stdout, stderr: stderr, status: status.exitstatus,
|
|
253
|
+
streamed: !(tee_stdout || tee_stderr).nil?)
|
|
248
254
|
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
raise command_failure(spec, status.exitstatus, stdout, stderr, redactor)
|
|
253
|
-
end
|
|
255
|
+
raise command_failure(spec, status.exitstatus, stdout, stderr, redactor) unless status.success?
|
|
256
|
+
|
|
257
|
+
result
|
|
254
258
|
rescue Errno::ENOENT => e
|
|
255
259
|
raise command_not_found(spec, e)
|
|
256
260
|
end
|
|
257
261
|
|
|
258
262
|
def capture_pty(spec, redactor:, tee_stdout: nil)
|
|
259
|
-
output = +
|
|
260
|
-
tee_buffer = +
|
|
263
|
+
output = +''
|
|
264
|
+
tee_buffer = +''
|
|
261
265
|
result = nil
|
|
262
266
|
|
|
263
267
|
PTY.spawn(spec.env, *spec.argv) do |reader, writer, pid|
|
|
@@ -276,11 +280,9 @@ module KamalBackup
|
|
|
276
280
|
end
|
|
277
281
|
|
|
278
282
|
_, status = Process.wait2(pid)
|
|
279
|
-
result = CommandResult.new(stdout: output, stderr:
|
|
283
|
+
result = CommandResult.new(stdout: output, stderr: '', status: status.exitstatus, streamed: !tee_stdout.nil?)
|
|
280
284
|
|
|
281
|
-
unless status.success?
|
|
282
|
-
raise command_failure(spec, status.exitstatus, output, output, redactor)
|
|
283
|
-
end
|
|
285
|
+
raise command_failure(spec, status.exitstatus, output, output, redactor) unless status.success?
|
|
284
286
|
end
|
|
285
287
|
|
|
286
288
|
result
|
|
@@ -288,9 +290,10 @@ module KamalBackup
|
|
|
288
290
|
raise command_not_found(spec, e)
|
|
289
291
|
end
|
|
290
292
|
|
|
291
|
-
def collect_stream(io, command_output:
|
|
292
|
-
|
|
293
|
-
|
|
293
|
+
def collect_stream(io, command_output: output, context: nil, stream: :stdout, log_output: true, tee_io: nil,
|
|
294
|
+
redactor: nil)
|
|
295
|
+
captured_output = +''
|
|
296
|
+
tee_buffer = +''
|
|
294
297
|
|
|
295
298
|
loop do
|
|
296
299
|
chunk = io.readpartial(16 * 1024)
|
|
@@ -306,72 +309,73 @@ module KamalBackup
|
|
|
306
309
|
end
|
|
307
310
|
|
|
308
311
|
private
|
|
309
|
-
def tee_stream(io, redactor, buffer, chunk)
|
|
310
|
-
buffer << chunk
|
|
311
312
|
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
end
|
|
313
|
+
def tee_stream(io, redactor, buffer, chunk)
|
|
314
|
+
buffer << chunk
|
|
315
315
|
|
|
316
|
-
|
|
317
|
-
buffer
|
|
316
|
+
while (index = buffer.index("\n"))
|
|
317
|
+
io.print(redactor.redact_string(buffer.slice!(0..index)))
|
|
318
318
|
end
|
|
319
319
|
|
|
320
|
-
|
|
321
|
-
|
|
320
|
+
io.flush if io.respond_to?(:flush)
|
|
321
|
+
buffer
|
|
322
|
+
end
|
|
322
323
|
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
end
|
|
324
|
+
def flush_tee_stream(io, redactor, buffer)
|
|
325
|
+
return if buffer.empty?
|
|
326
326
|
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
327
|
+
io.print(redactor.redact_string(buffer))
|
|
328
|
+
io.flush if io.respond_to?(:flush)
|
|
329
|
+
end
|
|
330
330
|
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
raise command_failure(spec, status.exitstatus, stdout, stderr, redactor)
|
|
335
|
-
end
|
|
336
|
-
end
|
|
331
|
+
def capture_quietly(spec, input:, redactor:)
|
|
332
|
+
stdout, stderr, status = Open3.capture3(spec.env, *spec.argv, stdin_data: input)
|
|
333
|
+
result = CommandResult.new(stdout: stdout, stderr: stderr, status: status.exitstatus, streamed: false)
|
|
337
334
|
|
|
338
|
-
|
|
339
|
-
Open3.popen3(spec.env, *spec.argv) do |stdin, stdout, stderr, wait_thread|
|
|
340
|
-
stdin_writer = Thread.new do
|
|
341
|
-
stdin.write(input) if input
|
|
342
|
-
ensure
|
|
343
|
-
stdin.close unless stdin.closed?
|
|
344
|
-
end
|
|
345
|
-
stdout_reader = Thread.new do
|
|
346
|
-
collect_stream(stdout, command_output: output, context: context, stream: :stdout, log_output: log_output, tee_io: tee_stdout, redactor: redactor)
|
|
347
|
-
end
|
|
348
|
-
stderr_reader = Thread.new do
|
|
349
|
-
collect_stream(stderr, command_output: output, context: context, stream: :stderr, log_output: log_output, tee_io: tee_stderr, redactor: redactor)
|
|
350
|
-
end
|
|
335
|
+
raise command_failure(spec, status.exitstatus, stdout, stderr, redactor) unless status.success?
|
|
351
336
|
|
|
352
|
-
|
|
353
|
-
|
|
337
|
+
result
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
def popen_capture(spec, input:, redactor:, output:, context:, log_output:, tee_stdout:, tee_stderr:)
|
|
341
|
+
Open3.popen3(spec.env, *spec.argv) do |stdin, stdout, stderr, wait_thread|
|
|
342
|
+
stdin_writer = Thread.new do
|
|
343
|
+
stdin.write(input) if input
|
|
344
|
+
ensure
|
|
345
|
+
stdin.close unless stdin.closed?
|
|
346
|
+
end
|
|
347
|
+
stdout_reader = Thread.new do
|
|
348
|
+
collect_stream(stdout, command_output: output, context: context, stream: :stdout, log_output: log_output,
|
|
349
|
+
tee_io: tee_stdout, redactor: redactor)
|
|
350
|
+
end
|
|
351
|
+
stderr_reader = Thread.new do
|
|
352
|
+
collect_stream(stderr, command_output: output, context: context, stream: :stderr, log_output: log_output,
|
|
353
|
+
tee_io: tee_stderr, redactor: redactor)
|
|
354
354
|
end
|
|
355
|
-
end
|
|
356
355
|
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
"command failed (#{status}): #{spec.display(redactor)}\n#{redactor.redact_string(stderr)}",
|
|
360
|
-
command: spec,
|
|
361
|
-
status: status,
|
|
362
|
-
stdout: redactor.redact_string(stdout),
|
|
363
|
-
stderr: redactor.redact_string(stderr)
|
|
364
|
-
)
|
|
356
|
+
stdin_writer.join
|
|
357
|
+
[stdout_reader.value, stderr_reader.value, wait_thread.value]
|
|
365
358
|
end
|
|
359
|
+
end
|
|
366
360
|
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
)
|
|
374
|
-
|
|
361
|
+
def command_failure(spec, status, stdout, stderr, redactor)
|
|
362
|
+
CommandError.new(
|
|
363
|
+
"command failed (#{status}): #{spec.display(redactor)}\n#{redactor.redact_string(stderr)}",
|
|
364
|
+
command: spec,
|
|
365
|
+
status: status,
|
|
366
|
+
stdout: redactor.redact_string(stdout),
|
|
367
|
+
stderr: redactor.redact_string(stderr)
|
|
368
|
+
)
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
def command_not_found(spec, error)
|
|
372
|
+
CommandError.new(
|
|
373
|
+
"command not found: #{spec.argv.first}",
|
|
374
|
+
command: spec,
|
|
375
|
+
status: 127,
|
|
376
|
+
stderr: error.message
|
|
377
|
+
)
|
|
378
|
+
end
|
|
375
379
|
end
|
|
376
380
|
end
|
|
377
381
|
end
|