kamal-backup 0.3.0.beta21 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -0
- data/exe/kamal-backup +7 -6
- data/lib/kamal_backup/app.rb +330 -393
- data/lib/kamal_backup/cli/helpers.rb +298 -0
- data/lib/kamal_backup/cli.rb +73 -367
- data/lib/kamal_backup/command.rb +77 -258
- data/lib/kamal_backup/command_output.rb +189 -0
- data/lib/kamal_backup/config.rb +242 -624
- data/lib/kamal_backup/config_file.rb +376 -0
- data/lib/kamal_backup/databases/base.rb +28 -14
- data/lib/kamal_backup/databases/mysql.rb +68 -67
- data/lib/kamal_backup/databases/postgres.rb +59 -58
- data/lib/kamal_backup/databases/sqlite.rb +21 -20
- data/lib/kamal_backup/errors.rb +3 -1
- data/lib/kamal_backup/evidence.rb +61 -63
- data/lib/kamal_backup/kamal_bridge.rb +270 -254
- data/lib/kamal_backup/rails_app.rb +94 -104
- data/lib/kamal_backup/redactor.rb +18 -13
- data/lib/kamal_backup/restic.rb +207 -183
- data/lib/kamal_backup/scheduler.rb +17 -14
- data/lib/kamal_backup/schema.rb +2 -0
- data/lib/kamal_backup/version.rb +3 -1
- data/lib/kamal_backup/yaml_access.rb +13 -0
- data/lib/kamal_backup.rb +22 -17
- metadata +76 -2
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'uri'
|
|
4
|
+
require_relative 'base'
|
|
3
5
|
|
|
4
6
|
module KamalBackup
|
|
5
7
|
module Databases
|
|
@@ -20,11 +22,11 @@ module KamalBackup
|
|
|
20
22
|
].freeze
|
|
21
23
|
|
|
22
24
|
def adapter_name
|
|
23
|
-
|
|
25
|
+
'postgres'
|
|
24
26
|
end
|
|
25
27
|
|
|
26
28
|
def dump_extension
|
|
27
|
-
|
|
29
|
+
'pgdump'
|
|
28
30
|
end
|
|
29
31
|
|
|
30
32
|
def dump_command
|
|
@@ -34,7 +36,7 @@ module KamalBackup
|
|
|
34
36
|
|
|
35
37
|
def current_restore_command
|
|
36
38
|
connection = current_connection
|
|
37
|
-
database = connection.fetch(
|
|
39
|
+
database = connection.fetch('PGDATABASE')
|
|
38
40
|
|
|
39
41
|
argv = %w[pg_restore --clean --if-exists --no-owner --no-privileges --dbname]
|
|
40
42
|
argv << database
|
|
@@ -42,7 +44,7 @@ module KamalBackup
|
|
|
42
44
|
end
|
|
43
45
|
|
|
44
46
|
def scratch_restore_command(target)
|
|
45
|
-
connection = current_connection.merge(
|
|
47
|
+
connection = current_connection.merge('PGDATABASE' => target)
|
|
46
48
|
|
|
47
49
|
argv = %w[pg_restore --clean --if-exists --no-owner --no-privileges --dbname]
|
|
48
50
|
argv << target
|
|
@@ -50,74 +52,73 @@ module KamalBackup
|
|
|
50
52
|
end
|
|
51
53
|
|
|
52
54
|
def current_target_identifier
|
|
53
|
-
value(
|
|
55
|
+
value('DATABASE_URL') || value('PGDATABASE')
|
|
54
56
|
end
|
|
55
57
|
|
|
56
58
|
def scratch_target_identifier(target)
|
|
57
|
-
[current_connection[
|
|
59
|
+
[current_connection['PGHOST'], target].compact.join('/')
|
|
58
60
|
end
|
|
59
61
|
|
|
60
62
|
private
|
|
61
|
-
def validate_scratch_restore_target(target)
|
|
62
|
-
if current_connection.fetch("PGDATABASE") == target
|
|
63
|
-
raise ConfigurationError, "scratch database must differ from the current PostgreSQL database"
|
|
64
|
-
end
|
|
65
63
|
|
|
66
|
-
|
|
67
|
-
|
|
64
|
+
def validate_scratch_restore_target(target)
|
|
65
|
+
raise ConfigurationError, 'scratch database must differ from the current PostgreSQL database' if current_connection.fetch('PGDATABASE') == target
|
|
68
66
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
connection_from_url(value("DATABASE_URL"), "DATABASE_URL").tap do |connection|
|
|
72
|
-
connection["PGPASSWORD"] ||= value("PGPASSWORD") if value("PGPASSWORD")
|
|
73
|
-
end
|
|
74
|
-
else
|
|
75
|
-
connection = prefixed_env("", SOURCE_ENV_KEYS)
|
|
76
|
-
raise ConfigurationError, "DATABASE_URL or PGDATABASE is required for PostgreSQL restore" unless connection["PGDATABASE"]
|
|
67
|
+
super
|
|
68
|
+
end
|
|
77
69
|
|
|
78
|
-
|
|
70
|
+
def current_connection
|
|
71
|
+
if value('DATABASE_URL')
|
|
72
|
+
connection = connection_from_url(value('DATABASE_URL'), 'DATABASE_URL')
|
|
73
|
+
connection['PGPASSWORD'] ||= value('PGPASSWORD')
|
|
74
|
+
connection.compact
|
|
75
|
+
else
|
|
76
|
+
connection = prefixed_env('', SOURCE_ENV_KEYS)
|
|
77
|
+
unless connection['PGDATABASE']
|
|
78
|
+
raise ConfigurationError,
|
|
79
|
+
'DATABASE_URL or PGDATABASE is required for PostgreSQL restore'
|
|
79
80
|
end
|
|
80
|
-
end
|
|
81
81
|
|
|
82
|
-
|
|
83
|
-
keys.each_with_object({}) do |key, env|
|
|
84
|
-
env[key] = value("#{prefix}#{key}") if value("#{prefix}#{key}")
|
|
85
|
-
end
|
|
82
|
+
connection
|
|
86
83
|
end
|
|
84
|
+
end
|
|
87
85
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
database = URI.decode_www_form_component(uri.path.to_s.sub(%r{\A/}, ""))
|
|
95
|
-
raise ConfigurationError, "database name is missing in #{name}" if database.empty?
|
|
96
|
-
|
|
97
|
-
env = {
|
|
98
|
-
"PGHOST" => uri.host,
|
|
99
|
-
"PGPORT" => uri.port&.to_s,
|
|
100
|
-
"PGUSER" => uri.user ? URI.decode_www_form_component(uri.user) : nil,
|
|
101
|
-
"PGPASSWORD" => uri.password ? URI.decode_www_form_component(uri.password) : nil,
|
|
102
|
-
"PGDATABASE" => database
|
|
103
|
-
}.compact
|
|
104
|
-
|
|
105
|
-
query = URI.decode_www_form(uri.query.to_s).to_h
|
|
106
|
-
{
|
|
107
|
-
"sslmode" => "PGSSLMODE",
|
|
108
|
-
"sslrootcert" => "PGSSLROOTCERT",
|
|
109
|
-
"sslcert" => "PGSSLCERT",
|
|
110
|
-
"sslkey" => "PGSSLKEY",
|
|
111
|
-
"connect_timeout" => "PGCONNECT_TIMEOUT"
|
|
112
|
-
}.each do |source, target|
|
|
113
|
-
env[target] = query[source] if query[source]
|
|
114
|
-
end
|
|
86
|
+
def prefixed_env(prefix, keys)
|
|
87
|
+
keys.each_with_object({}) do |key, env|
|
|
88
|
+
env[key] = value("#{prefix}#{key}") if value("#{prefix}#{key}")
|
|
89
|
+
end
|
|
90
|
+
end
|
|
115
91
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
92
|
+
def connection_from_url(url, name)
|
|
93
|
+
uri = URI.parse(url)
|
|
94
|
+
raise ConfigurationError, "#{name} must use postgres:// or postgresql://" unless %w[postgres postgresql].include?(uri.scheme)
|
|
95
|
+
|
|
96
|
+
database = URI.decode_www_form_component(uri.path.to_s.sub(%r{\A/}, ''))
|
|
97
|
+
raise ConfigurationError, "database name is missing in #{name}" if database.empty?
|
|
98
|
+
|
|
99
|
+
env = {
|
|
100
|
+
'PGHOST' => uri.host,
|
|
101
|
+
'PGPORT' => uri.port&.to_s,
|
|
102
|
+
'PGUSER' => uri.user ? URI.decode_www_form_component(uri.user) : nil,
|
|
103
|
+
'PGPASSWORD' => uri.password ? URI.decode_www_form_component(uri.password) : nil,
|
|
104
|
+
'PGDATABASE' => database
|
|
105
|
+
}.compact
|
|
106
|
+
|
|
107
|
+
query = URI.decode_www_form(uri.query.to_s).to_h
|
|
108
|
+
{
|
|
109
|
+
'sslmode' => 'PGSSLMODE',
|
|
110
|
+
'sslrootcert' => 'PGSSLROOTCERT',
|
|
111
|
+
'sslcert' => 'PGSSLCERT',
|
|
112
|
+
'sslkey' => 'PGSSLKEY',
|
|
113
|
+
'connect_timeout' => 'PGCONNECT_TIMEOUT'
|
|
114
|
+
}.each do |source, target|
|
|
115
|
+
env[target] = query[source] if query[source]
|
|
119
116
|
end
|
|
120
117
|
|
|
118
|
+
env
|
|
119
|
+
rescue URI::InvalidURIError => e
|
|
120
|
+
raise ConfigurationError, "invalid #{name}: #{e.message}"
|
|
121
|
+
end
|
|
121
122
|
end
|
|
122
123
|
end
|
|
123
124
|
end
|
|
@@ -1,24 +1,26 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'fileutils'
|
|
4
|
+
require 'tempfile'
|
|
5
|
+
require_relative 'base'
|
|
4
6
|
|
|
5
7
|
module KamalBackup
|
|
6
8
|
module Databases
|
|
7
9
|
class Sqlite < Base
|
|
8
10
|
def adapter_name
|
|
9
|
-
|
|
11
|
+
'sqlite'
|
|
10
12
|
end
|
|
11
13
|
|
|
12
14
|
def dump_extension
|
|
13
|
-
|
|
15
|
+
'sqlite3'
|
|
14
16
|
end
|
|
15
17
|
|
|
16
18
|
def backup(restic)
|
|
17
19
|
source = sqlite_source
|
|
18
|
-
Tempfile.create([
|
|
20
|
+
Tempfile.create(['kamal-backup-', '.sqlite3']) do |tempfile|
|
|
19
21
|
tempfile.close
|
|
20
22
|
Command.capture(
|
|
21
|
-
CommandSpec.new(argv: [
|
|
23
|
+
CommandSpec.new(argv: ['sqlite3', source, ".backup #{sqlite_literal(tempfile.path)}"]),
|
|
22
24
|
redactor: redactor
|
|
23
25
|
)
|
|
24
26
|
restic.backup_file(
|
|
@@ -39,7 +41,7 @@ module KamalBackup
|
|
|
39
41
|
end
|
|
40
42
|
|
|
41
43
|
def dump_command
|
|
42
|
-
raise NotImplementedError,
|
|
44
|
+
raise NotImplementedError, 'SQLite backup uses .backup into a temporary file'
|
|
43
45
|
end
|
|
44
46
|
|
|
45
47
|
def current_target_identifier
|
|
@@ -51,21 +53,20 @@ module KamalBackup
|
|
|
51
53
|
end
|
|
52
54
|
|
|
53
55
|
private
|
|
54
|
-
def sqlite_source
|
|
55
|
-
config.required_value("SQLITE_DATABASE_PATH")
|
|
56
|
-
end
|
|
57
56
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
end
|
|
57
|
+
def sqlite_source
|
|
58
|
+
config.required_value('SQLITE_DATABASE_PATH')
|
|
59
|
+
end
|
|
62
60
|
|
|
63
|
-
|
|
64
|
-
|
|
61
|
+
def validate_scratch_restore_target(target)
|
|
62
|
+
raise ConfigurationError, 'scratch SQLite path must differ from SQLITE_DATABASE_PATH' if File.expand_path(sqlite_source) == File.expand_path(target)
|
|
65
63
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
64
|
+
super
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def sqlite_literal(value)
|
|
68
|
+
"'#{value.to_s.gsub("'", "''")}'"
|
|
69
|
+
end
|
|
69
70
|
end
|
|
70
71
|
end
|
|
71
72
|
end
|
data/lib/kamal_backup/errors.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module KamalBackup
|
|
2
4
|
class Error < StandardError; end
|
|
3
5
|
class ConfigurationError < Error; end
|
|
@@ -5,7 +7,7 @@ module KamalBackup
|
|
|
5
7
|
class CommandError < Error
|
|
6
8
|
attr_reader :command, :status, :stdout, :stderr
|
|
7
9
|
|
|
8
|
-
def initialize(message, command:, status: nil, stdout:
|
|
10
|
+
def initialize(message, command:, status: nil, stdout: '', stderr: '')
|
|
9
11
|
super(message)
|
|
10
12
|
@command = command
|
|
11
13
|
@status = status
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
require_relative
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'time'
|
|
5
|
+
require_relative 'command'
|
|
6
|
+
require_relative 'schema'
|
|
7
|
+
require_relative 'version'
|
|
6
8
|
|
|
7
9
|
module KamalBackup
|
|
8
10
|
class Evidence
|
|
@@ -14,19 +16,18 @@ module KamalBackup
|
|
|
14
16
|
|
|
15
17
|
def to_h
|
|
16
18
|
Schema.record(
|
|
17
|
-
kind:
|
|
19
|
+
kind: 'evidence',
|
|
18
20
|
app_name: @config.app_name,
|
|
19
21
|
generated_at: Time.now.utc.iso8601,
|
|
20
|
-
|
|
21
|
-
|
|
22
|
+
databases: @config.databases.map do |database|
|
|
23
|
+
{ name: database.database_name, adapter: database.database_adapter }
|
|
24
|
+
end,
|
|
22
25
|
restic_repository: @redactor.redact_string(@config.restic_repository.to_s),
|
|
23
|
-
backup_paths: @config.backup_paths,
|
|
24
26
|
paths: @config.backup_paths,
|
|
25
27
|
forget_after_backup: @config.forget_after_backup?,
|
|
26
28
|
retention: @config.retention,
|
|
27
|
-
latest_database_backup: latest_snapshot_summary(["type:database"]),
|
|
28
29
|
latest_database_backups: latest_database_backups,
|
|
29
|
-
latest_file_backup: latest_snapshot_summary([
|
|
30
|
+
latest_file_backup: latest_snapshot_summary(['type:files']),
|
|
30
31
|
last_restic_check: last_check,
|
|
31
32
|
last_restore_drill: last_restore_drill,
|
|
32
33
|
image_version: VERSION,
|
|
@@ -39,67 +40,64 @@ module KamalBackup
|
|
|
39
40
|
end
|
|
40
41
|
|
|
41
42
|
private
|
|
42
|
-
def latest_snapshot_summary(tags)
|
|
43
|
-
snapshot = @restic.latest_snapshot(tags: tags)
|
|
44
43
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
id: snapshot["short_id"] || snapshot["id"],
|
|
48
|
-
time: snapshot["time"],
|
|
49
|
-
tags: snapshot["tags"]
|
|
50
|
-
}
|
|
51
|
-
end
|
|
52
|
-
rescue Error => e
|
|
53
|
-
{ error: @redactor.redact_string(e.message) }
|
|
54
|
-
end
|
|
44
|
+
def latest_snapshot_summary(tags)
|
|
45
|
+
snapshot = @restic.latest_snapshot(tags: tags)
|
|
55
46
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
])
|
|
63
|
-
end
|
|
47
|
+
if snapshot
|
|
48
|
+
{
|
|
49
|
+
id: snapshot['short_id'] || snapshot['id'],
|
|
50
|
+
time: snapshot['time'],
|
|
51
|
+
tags: snapshot['tags']
|
|
52
|
+
}
|
|
64
53
|
end
|
|
54
|
+
rescue Error => e
|
|
55
|
+
{ error: @redactor.redact_string(e.message) }
|
|
56
|
+
end
|
|
65
57
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
58
|
+
def latest_database_backups
|
|
59
|
+
@config.databases.each_with_object({}) do |database, backups|
|
|
60
|
+
backups[database.database_name] = latest_snapshot_summary([
|
|
61
|
+
'type:database',
|
|
62
|
+
"database:#{database.database_name}",
|
|
63
|
+
"adapter:#{database.database_adapter}"
|
|
64
|
+
])
|
|
72
65
|
end
|
|
66
|
+
end
|
|
73
67
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
{ error: @redactor.redact_string(e.message) }
|
|
80
|
-
end
|
|
68
|
+
def last_check
|
|
69
|
+
JSON.parse(File.read(@config.last_check_path)) if File.file?(@config.last_check_path)
|
|
70
|
+
rescue JSON::ParserError, SystemCallError => e
|
|
71
|
+
{ error: @redactor.redact_string(e.message) }
|
|
72
|
+
end
|
|
81
73
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
mysql_client: version_for(["mariadb", "--version"], ["mysql", "--version"]),
|
|
88
|
-
sqlite3: version_for(["sqlite3", "--version"]),
|
|
89
|
-
restic: version_for(["restic", "version"])
|
|
90
|
-
}
|
|
91
|
-
end
|
|
74
|
+
def last_restore_drill
|
|
75
|
+
JSON.parse(File.read(@config.last_restore_drill_path)) if File.file?(@config.last_restore_drill_path)
|
|
76
|
+
rescue JSON::ParserError, SystemCallError => e
|
|
77
|
+
{ error: @redactor.redact_string(e.message) }
|
|
78
|
+
end
|
|
92
79
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
80
|
+
def tool_versions
|
|
81
|
+
{
|
|
82
|
+
pg_dump: version_for(['pg_dump', '--version']),
|
|
83
|
+
pg_restore: version_for(['pg_restore', '--version']),
|
|
84
|
+
mysql_dump: version_for(['mariadb-dump', '--version'], ['mysqldump', '--version']),
|
|
85
|
+
mysql_client: version_for(['mariadb', '--version'], ['mysql', '--version']),
|
|
86
|
+
sqlite3: version_for(['sqlite3', '--version']),
|
|
87
|
+
restic: version_for(%w[restic version])
|
|
88
|
+
}
|
|
89
|
+
end
|
|
101
90
|
|
|
102
|
-
|
|
91
|
+
def version_for(*commands)
|
|
92
|
+
commands.each do |argv|
|
|
93
|
+
result = Command.capture(CommandSpec.new(argv: argv), redactor: @redactor)
|
|
94
|
+
output = result.stdout.empty? ? result.stderr : result.stdout
|
|
95
|
+
return @redactor.redact_string(output.strip)
|
|
96
|
+
rescue CommandError
|
|
97
|
+
next
|
|
103
98
|
end
|
|
99
|
+
|
|
100
|
+
'unavailable'
|
|
101
|
+
end
|
|
104
102
|
end
|
|
105
103
|
end
|