@softure-ai/backup 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/LICENSE +21 -0
- package/README.md +243 -2
- package/dist/cli/main.d.ts +3 -0
- package/dist/cli/main.d.ts.map +1 -0
- package/dist/cli/main.js +16 -0
- package/dist/cli/main.js.map +1 -0
- package/dist/cli/run.d.ts +15 -0
- package/dist/cli/run.d.ts.map +1 -0
- package/dist/cli/run.js +442 -0
- package/dist/cli/run.js.map +1 -0
- package/dist/contract.d.ts +41 -0
- package/dist/contract.d.ts.map +1 -0
- package/dist/contract.js +15 -0
- package/dist/contract.js.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +30 -0
- package/dist/index.js.map +1 -0
- package/dist/messages/en.d.ts +17 -0
- package/dist/messages/en.d.ts.map +1 -0
- package/dist/messages/en.js +17 -0
- package/dist/messages/en.js.map +1 -0
- package/dist/messages/index.d.ts +22 -0
- package/dist/messages/index.d.ts.map +1 -0
- package/dist/messages/index.js +4 -0
- package/dist/messages/index.js.map +1 -0
- package/dist/offsite/config.d.ts +45 -0
- package/dist/offsite/config.d.ts.map +1 -0
- package/dist/offsite/config.js +147 -0
- package/dist/offsite/config.js.map +1 -0
- package/dist/offsite/crypto.d.ts +22 -0
- package/dist/offsite/crypto.d.ts.map +1 -0
- package/dist/offsite/crypto.js +48 -0
- package/dist/offsite/crypto.js.map +1 -0
- package/dist/offsite/dump.d.ts +23 -0
- package/dist/offsite/dump.d.ts.map +1 -0
- package/dist/offsite/dump.js +85 -0
- package/dist/offsite/dump.js.map +1 -0
- package/dist/offsite/index.d.ts +13 -0
- package/dist/offsite/index.d.ts.map +1 -0
- package/dist/offsite/index.js +15 -0
- package/dist/offsite/index.js.map +1 -0
- package/dist/offsite/lifecycle.d.ts +17 -0
- package/dist/offsite/lifecycle.d.ts.map +1 -0
- package/dist/offsite/lifecycle.js +45 -0
- package/dist/offsite/lifecycle.js.map +1 -0
- package/dist/offsite/objects.d.ts +19 -0
- package/dist/offsite/objects.d.ts.map +1 -0
- package/dist/offsite/objects.js +44 -0
- package/dist/offsite/objects.js.map +1 -0
- package/dist/offsite/postgres.d.ts +32 -0
- package/dist/offsite/postgres.d.ts.map +1 -0
- package/dist/offsite/postgres.js +97 -0
- package/dist/offsite/postgres.js.map +1 -0
- package/dist/offsite/record.d.ts +3 -0
- package/dist/offsite/record.d.ts.map +1 -0
- package/dist/offsite/record.js +10 -0
- package/dist/offsite/record.js.map +1 -0
- package/dist/offsite/restore.d.ts +54 -0
- package/dist/offsite/restore.d.ts.map +1 -0
- package/dist/offsite/restore.js +67 -0
- package/dist/offsite/restore.js.map +1 -0
- package/dist/offsite/s3.d.ts +34 -0
- package/dist/offsite/s3.d.ts.map +1 -0
- package/dist/offsite/s3.js +119 -0
- package/dist/offsite/s3.js.map +1 -0
- package/dist/offsite/sigv4.d.ts +19 -0
- package/dist/offsite/sigv4.d.ts.map +1 -0
- package/dist/offsite/sigv4.js +69 -0
- package/dist/offsite/sigv4.js.map +1 -0
- package/dist/offsite/status-sql.d.ts +10 -0
- package/dist/offsite/status-sql.d.ts.map +1 -0
- package/dist/offsite/status-sql.js +37 -0
- package/dist/offsite/status-sql.js.map +1 -0
- package/dist/offsite/upload.d.ts +33 -0
- package/dist/offsite/upload.d.ts.map +1 -0
- package/dist/offsite/upload.js +98 -0
- package/dist/offsite/upload.js.map +1 -0
- package/dist/offsite/verify.d.ts +32 -0
- package/dist/offsite/verify.d.ts.map +1 -0
- package/dist/offsite/verify.js +75 -0
- package/dist/offsite/verify.js.map +1 -0
- package/dist/schema.d.ts +150 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/schema.js +15 -0
- package/dist/schema.js.map +1 -0
- package/dist/server/index.d.ts +28 -0
- package/dist/server/index.d.ts.map +1 -0
- package/dist/server/index.js +98 -0
- package/dist/server/index.js.map +1 -0
- package/examples/backup-verify.yml +84 -0
- package/migrations/0001_create_runs.sql +21 -0
- package/module.json +12 -0
- package/package.json +73 -4
- package/src/cli/main.ts +16 -0
- package/src/cli/run.ts +441 -0
- package/src/contract.ts +47 -0
- package/src/index.ts +41 -0
- package/src/messages/en.ts +16 -0
- package/src/messages/index.ts +6 -0
- package/src/offsite/config.ts +173 -0
- package/src/offsite/crypto.ts +53 -0
- package/src/offsite/dump.ts +97 -0
- package/src/offsite/index.ts +33 -0
- package/src/offsite/lifecycle.ts +47 -0
- package/src/offsite/objects.ts +54 -0
- package/src/offsite/postgres.ts +115 -0
- package/src/offsite/record.ts +11 -0
- package/src/offsite/restore.ts +108 -0
- package/src/offsite/s3.ts +144 -0
- package/src/offsite/sigv4.ts +84 -0
- package/src/offsite/status-sql.ts +45 -0
- package/src/offsite/upload.ts +116 -0
- package/src/offsite/verify.ts +97 -0
- package/src/schema.ts +17 -0
- package/src/server/index.ts +111 -0
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
// Restore and fetch from the off-site copy. Restore puts a database dump into a NEW database by default, never
|
|
2
|
+
// silently over the one in production: overwriting an existing database needs `overwrite` (pg_restore --clean
|
|
3
|
+
// --if-exists). Fetch decrypts any object (a data archive) to a local file.
|
|
4
|
+
import type { StoreConfig } from "./config.js";
|
|
5
|
+
import { decryptStream, writeStreamToFile } from "./crypto.js";
|
|
6
|
+
import { parseObjectKey, selectNewestKey } from "./objects.js";
|
|
7
|
+
import {
|
|
8
|
+
createDatabase,
|
|
9
|
+
createRestoreDatabaseName,
|
|
10
|
+
databaseExists,
|
|
11
|
+
getUrlDatabase,
|
|
12
|
+
isDatabaseName,
|
|
13
|
+
runPgRestore,
|
|
14
|
+
} from "./postgres.js";
|
|
15
|
+
import { getObjectStream, listObjects, type S3Options } from "./s3.js";
|
|
16
|
+
import type { BackupKind } from "../contract.js";
|
|
17
|
+
|
|
18
|
+
export type OffsiteResult<T> = { ok: true; value: T } | { ok: false; problem: string };
|
|
19
|
+
|
|
20
|
+
/** The key to read: the one named (checked to be a backup of `kind` under the prefix), else the newest of `kind`. */
|
|
21
|
+
export async function resolveKey(store: StoreConfig, kind: BackupKind, key: string | undefined, s3?: S3Options): Promise<OffsiteResult<{ key: string; takenAt: Date }>> {
|
|
22
|
+
if (key !== undefined) {
|
|
23
|
+
const parsed = parseObjectKey(store.prefix, key);
|
|
24
|
+
if (parsed?.kind !== kind) return { ok: false, problem: `${key} is not a ${kind} backup under ${store.prefix}/` };
|
|
25
|
+
return { ok: true, value: { key, takenAt: parsed.takenAt } };
|
|
26
|
+
}
|
|
27
|
+
const objects = await listObjects(store, `${store.prefix}/`, s3);
|
|
28
|
+
const newest = selectNewestKey(store.prefix, objects.map((object) => object.key), kind);
|
|
29
|
+
if (newest === null) return { ok: false, problem: `no ${kind} backup is stored under ${store.prefix}/` };
|
|
30
|
+
return { ok: true, value: newest };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface RestoreOptions {
|
|
34
|
+
readonly store: StoreConfig;
|
|
35
|
+
readonly identities: readonly string[];
|
|
36
|
+
/** A superuser's URL on the target server (any database of it). */
|
|
37
|
+
readonly adminUrl: string;
|
|
38
|
+
/** An object key; the newest database backup by default. */
|
|
39
|
+
readonly key?: string;
|
|
40
|
+
/** The database to restore into; a new `<url database>_restore_<time>` by default. */
|
|
41
|
+
readonly into?: string;
|
|
42
|
+
/** Restore over an existing `into` (pg_restore --clean --if-exists). */
|
|
43
|
+
readonly overwrite?: boolean;
|
|
44
|
+
/** Leave out ownership and grants, for a server without the app's roles. */
|
|
45
|
+
readonly noOwner?: boolean;
|
|
46
|
+
readonly pgRestore: string;
|
|
47
|
+
readonly env: Readonly<Record<string, string | undefined>>;
|
|
48
|
+
readonly now?: Date;
|
|
49
|
+
readonly s3?: S3Options;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface RestoreOutcome {
|
|
53
|
+
readonly key: string;
|
|
54
|
+
readonly database: string;
|
|
55
|
+
readonly created: boolean;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export async function restoreBackup(options: RestoreOptions): Promise<OffsiteResult<RestoreOutcome>> {
|
|
59
|
+
const resolved = await resolveKey(options.store, "database", options.key, options.s3);
|
|
60
|
+
if (!resolved.ok) return resolved;
|
|
61
|
+
const target = options.into ?? createRestoreDatabaseName(getUrlDatabase(options.adminUrl), options.now ?? new Date());
|
|
62
|
+
if (!isDatabaseName(target)) return { ok: false, problem: `${target} is not a database name (lower-case letters, digits and _)` };
|
|
63
|
+
const exists = await databaseExists(options.adminUrl, target);
|
|
64
|
+
if (exists && options.overwrite !== true) {
|
|
65
|
+
return { ok: false, problem: `the database ${target} exists; pass --overwrite to restore over it, or restore into a new one` };
|
|
66
|
+
}
|
|
67
|
+
if (!exists && options.overwrite === true) return { ok: false, problem: `--overwrite needs an existing database; ${target} does not exist` };
|
|
68
|
+
// Decrypt before creating anything: a wrong key fails here, at the header.
|
|
69
|
+
const encrypted = await getObjectStream(options.store, resolved.value.key, options.s3);
|
|
70
|
+
let dump: ReadableStream<Uint8Array>;
|
|
71
|
+
try {
|
|
72
|
+
dump = await decryptStream(encrypted, options.identities);
|
|
73
|
+
} catch (error) {
|
|
74
|
+
await encrypted.cancel().catch(() => undefined);
|
|
75
|
+
return { ok: false, problem: `cannot decrypt ${resolved.value.key} with this identity (${error instanceof Error ? error.message : String(error)})` };
|
|
76
|
+
}
|
|
77
|
+
if (!exists) await createDatabase(options.adminUrl, target);
|
|
78
|
+
const flags = [...(options.overwrite === true ? ["--clean", "--if-exists"] : []), ...(options.noOwner === true ? ["--no-owner", "--no-acl"] : [])];
|
|
79
|
+
const restored = await runPgRestore({ adminUrl: options.adminUrl, database: target, flags, dump, pgRestore: options.pgRestore, env: options.env });
|
|
80
|
+
if (!restored.ok) return { ok: false, problem: `${resolved.value.key} did not restore into ${target}: ${restored.problem}` };
|
|
81
|
+
return { ok: true, value: { key: resolved.value.key, database: target, created: !exists } };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export interface FetchOptions {
|
|
85
|
+
readonly store: StoreConfig;
|
|
86
|
+
readonly identities: readonly string[];
|
|
87
|
+
readonly kind: BackupKind;
|
|
88
|
+
readonly key?: string;
|
|
89
|
+
/** The decrypted file to write (0600). */
|
|
90
|
+
readonly out: string;
|
|
91
|
+
readonly s3?: S3Options;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Downloads and decrypts one object to a file, e.g. a data archive to unpack into the new volume. */
|
|
95
|
+
export async function fetchBackup(options: FetchOptions): Promise<OffsiteResult<{ key: string; out: string }>> {
|
|
96
|
+
const resolved = await resolveKey(options.store, options.kind, options.key, options.s3);
|
|
97
|
+
if (!resolved.ok) return resolved;
|
|
98
|
+
const encrypted = await getObjectStream(options.store, resolved.value.key, options.s3);
|
|
99
|
+
let plain: ReadableStream<Uint8Array>;
|
|
100
|
+
try {
|
|
101
|
+
plain = await decryptStream(encrypted, options.identities);
|
|
102
|
+
} catch (error) {
|
|
103
|
+
await encrypted.cancel().catch(() => undefined);
|
|
104
|
+
return { ok: false, problem: `cannot decrypt ${resolved.value.key} with this identity (${error instanceof Error ? error.message : String(error)})` };
|
|
105
|
+
}
|
|
106
|
+
await writeStreamToFile(plain, options.out);
|
|
107
|
+
return { ok: true, value: { key: resolved.value.key, out: options.out } };
|
|
108
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
// The three S3 requests: PUT of an encrypted file (the nightly run, a write-only token suffices), GET of an object
|
|
2
|
+
// and ListObjectsV2 under the prefix (restore and verify, a read token). Path style: <endpoint>/<bucket>/<key>.
|
|
3
|
+
import { createReadStream } from "node:fs";
|
|
4
|
+
import { Readable } from "node:stream";
|
|
5
|
+
import type { StoreConfig } from "./config.js";
|
|
6
|
+
import { EMPTY_SHA256, encodeRfc3986, signRequest } from "./sigv4.js";
|
|
7
|
+
|
|
8
|
+
/** A failed request, with what S3 said (its error code) but never a header or a signature. */
|
|
9
|
+
export class S3RequestError extends Error {
|
|
10
|
+
readonly status: number | null;
|
|
11
|
+
readonly s3Code: string | null;
|
|
12
|
+
|
|
13
|
+
constructor(message: string, status: number | null, s3Code: string | null) {
|
|
14
|
+
super(message);
|
|
15
|
+
this.name = "S3RequestError";
|
|
16
|
+
this.status = status;
|
|
17
|
+
this.s3Code = s3Code;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** 401 and 403: the token is wrong or lacks the permission. */
|
|
21
|
+
get isDenied(): boolean {
|
|
22
|
+
return this.status === 401 || this.status === 403;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface StoredObject {
|
|
27
|
+
readonly key: string;
|
|
28
|
+
readonly bytes: number;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface S3Options {
|
|
32
|
+
/** The clock of the signature. */
|
|
33
|
+
readonly now?: () => Date;
|
|
34
|
+
readonly fetch?: typeof fetch;
|
|
35
|
+
/** Per request; an upload of a large dump needs minutes. Default 30 minutes. */
|
|
36
|
+
readonly timeoutMs?: number;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const DEFAULT_TIMEOUT_MS = 30 * 60 * 1000;
|
|
40
|
+
const MAX_ERROR_TEXT = 200;
|
|
41
|
+
|
|
42
|
+
/** `/a/b//` → `/a/b`, by index: a regular expression here would backtrack on a path of many slashes. */
|
|
43
|
+
function trimTrailingSlashes(path: string): string {
|
|
44
|
+
let end = path.length;
|
|
45
|
+
while (end > 0 && path.charCodeAt(end - 1) === 47) end -= 1;
|
|
46
|
+
return path.slice(0, end);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The URL of a key (or of the bucket, for a list), path style under the endpoint's own path. */
|
|
50
|
+
export function getObjectUrl(store: StoreConfig, key: string | null): URL {
|
|
51
|
+
const base = trimTrailingSlashes(store.endpoint.pathname);
|
|
52
|
+
const path = key === null ? `${base}/${encodeRfc3986(store.bucket)}` : `${base}/${encodeRfc3986(store.bucket)}/${encodeRfc3986(key, true)}`;
|
|
53
|
+
return new URL(`${store.endpoint.origin}${path}`);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function readXmlTag(xml: string, tag: string): string | null {
|
|
57
|
+
const match = new RegExp(`<${tag}>([^<]*)</${tag}>`).exec(xml);
|
|
58
|
+
return match?.[1] === undefined ? null : decodeXml(match[1]);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function decodeXml(text: string): string {
|
|
62
|
+
return text
|
|
63
|
+
.replace(/</g, "<")
|
|
64
|
+
.replace(/>/g, ">")
|
|
65
|
+
.replace(/"/g, '"')
|
|
66
|
+
.replace(/'/g, "'")
|
|
67
|
+
.replace(/&/g, "&");
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async function toError(operation: string, response: Response): Promise<S3RequestError> {
|
|
71
|
+
const body = await response.text().catch(() => "");
|
|
72
|
+
const code = readXmlTag(body, "Code");
|
|
73
|
+
const detail = readXmlTag(body, "Message")?.slice(0, MAX_ERROR_TEXT);
|
|
74
|
+
const said = [code, detail].filter((part) => part !== null && part !== undefined && part !== "").join(": ");
|
|
75
|
+
return new S3RequestError(`${operation} answered HTTP ${String(response.status)}${said === "" ? "" : ` (${said})`}`, response.status, code);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async function send(operation: string, url: URL, init: RequestInit, options: S3Options): Promise<Response> {
|
|
79
|
+
const doFetch = options.fetch ?? fetch;
|
|
80
|
+
let response: Response;
|
|
81
|
+
try {
|
|
82
|
+
response = await doFetch(url, { ...init, signal: AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS) });
|
|
83
|
+
} catch (error) {
|
|
84
|
+
// The cause names the host or the errno, never a header.
|
|
85
|
+
const cause = error instanceof Error ? (error.cause instanceof Error ? error.cause.message : error.message) : String(error);
|
|
86
|
+
throw new S3RequestError(`${operation} could not reach ${url.host} (${cause})`, null, null);
|
|
87
|
+
}
|
|
88
|
+
if (!response.ok) throw await toError(operation, response);
|
|
89
|
+
return response;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function sign(store: StoreConfig, method: "GET" | "PUT", url: URL, payloadSha256: string, options: S3Options, headers?: Record<string, string>): Record<string, string> {
|
|
93
|
+
return signRequest({
|
|
94
|
+
method,
|
|
95
|
+
url,
|
|
96
|
+
payloadSha256,
|
|
97
|
+
region: store.region,
|
|
98
|
+
accessKeyId: store.accessKeyId,
|
|
99
|
+
secretAccessKey: store.secretAccessKey,
|
|
100
|
+
now: (options.now ?? (() => new Date()))(),
|
|
101
|
+
...(headers === undefined ? {} : { headers }),
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** PUTs the file at `path` (its sha256 and size known) as `key`. */
|
|
106
|
+
export async function putObjectFile(
|
|
107
|
+
store: StoreConfig,
|
|
108
|
+
input: { key: string; path: string; bytes: number; sha256: string },
|
|
109
|
+
options: S3Options = {},
|
|
110
|
+
): Promise<void> {
|
|
111
|
+
const url = getObjectUrl(store, input.key);
|
|
112
|
+
const headers = sign(store, "PUT", url, input.sha256, options, { "content-type": "application/octet-stream" });
|
|
113
|
+
const body = Readable.toWeb(createReadStream(input.path)) as ReadableStream<Uint8Array>;
|
|
114
|
+
await send(`PUT ${input.key}`, url, { method: "PUT", headers: { ...headers, "content-length": String(input.bytes) }, body, duplex: "half" } as RequestInit, options);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** The body of `key` as a stream. */
|
|
118
|
+
export async function getObjectStream(store: StoreConfig, key: string, options: S3Options = {}): Promise<ReadableStream<Uint8Array>> {
|
|
119
|
+
const url = getObjectUrl(store, key);
|
|
120
|
+
const response = await send(`GET ${key}`, url, { method: "GET", headers: sign(store, "GET", url, EMPTY_SHA256, options) }, options);
|
|
121
|
+
if (response.body === null) throw new S3RequestError(`GET ${key} answered without a body`, response.status, null);
|
|
122
|
+
return response.body;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Every object under `prefix/`, through ListObjectsV2 pages. */
|
|
126
|
+
export async function listObjects(store: StoreConfig, prefix: string, options: S3Options = {}): Promise<StoredObject[]> {
|
|
127
|
+
const objects: StoredObject[] = [];
|
|
128
|
+
let token: string | null = null;
|
|
129
|
+
do {
|
|
130
|
+
const url = getObjectUrl(store, null);
|
|
131
|
+
url.searchParams.set("list-type", "2");
|
|
132
|
+
url.searchParams.set("prefix", prefix);
|
|
133
|
+
if (token !== null) url.searchParams.set("continuation-token", token);
|
|
134
|
+
const response = await send(`LIST ${prefix}`, url, { method: "GET", headers: sign(store, "GET", url, EMPTY_SHA256, options) }, options);
|
|
135
|
+
const xml = await response.text();
|
|
136
|
+
for (const match of xml.matchAll(/<Contents>([\s\S]*?)<\/Contents>/g)) {
|
|
137
|
+
const entry = match[1] ?? "";
|
|
138
|
+
const key = readXmlTag(entry, "Key");
|
|
139
|
+
if (key !== null) objects.push({ key, bytes: Number(readXmlTag(entry, "Size") ?? 0) });
|
|
140
|
+
}
|
|
141
|
+
token = readXmlTag(xml, "IsTruncated") === "true" ? readXmlTag(xml, "NextContinuationToken") : null;
|
|
142
|
+
} while (token !== null);
|
|
143
|
+
return objects;
|
|
144
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
// AWS Signature Version 4 for the three S3 requests this package sends (PUT, GET, ListObjectsV2), path style, so it
|
|
2
|
+
// works the same with Cloudflare R2, Backblaze B2, Hetzner Object Storage, MinIO and AWS. The payload hash is the
|
|
3
|
+
// real sha256 of the body (the empty body's for GET and list), never UNSIGNED-PAYLOAD.
|
|
4
|
+
import { createHash, createHmac } from "node:crypto";
|
|
5
|
+
|
|
6
|
+
export const EMPTY_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
|
|
7
|
+
|
|
8
|
+
export interface SignInput {
|
|
9
|
+
readonly method: "GET" | "PUT" | "HEAD";
|
|
10
|
+
readonly url: URL;
|
|
11
|
+
/** Headers to send and sign besides host, x-amz-date and x-amz-content-sha256; names in any case. */
|
|
12
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
13
|
+
readonly payloadSha256: string;
|
|
14
|
+
readonly region: string;
|
|
15
|
+
readonly accessKeyId: string;
|
|
16
|
+
readonly secretAccessKey: string;
|
|
17
|
+
readonly now: Date;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** RFC 3986 encoding of every byte except the unreserved ones, as SigV4 wants; `/` kept when `keepSlash`. */
|
|
21
|
+
export function encodeRfc3986(text: string, keepSlash = false): string {
|
|
22
|
+
let encoded = "";
|
|
23
|
+
for (const byte of Buffer.from(text, "utf8")) {
|
|
24
|
+
const char = String.fromCharCode(byte);
|
|
25
|
+
if (/[A-Za-z0-9\-._~]/.test(char) || (keepSlash && char === "/")) encoded += char;
|
|
26
|
+
else encoded += `%${byte.toString(16).toUpperCase().padStart(2, "0")}`;
|
|
27
|
+
}
|
|
28
|
+
return encoded;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** `20130524T000000Z` */
|
|
32
|
+
export function formatAmzDate(now: Date): string {
|
|
33
|
+
return now.toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z");
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function sha256Hex(text: string): string {
|
|
37
|
+
return createHash("sha256").update(text, "utf8").digest("hex");
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function hmac(key: Buffer | string, text: string): Buffer {
|
|
41
|
+
return createHmac("sha256", key).update(text, "utf8").digest();
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function getCanonicalQuery(url: URL): string {
|
|
45
|
+
return [...url.searchParams.entries()]
|
|
46
|
+
.map(([key, value]) => [encodeRfc3986(key), encodeRfc3986(value)] as const)
|
|
47
|
+
.sort(([left, leftValue], [right, rightValue]) => (left === right ? (leftValue < rightValue ? -1 : 1) : left < right ? -1 : 1))
|
|
48
|
+
.map(([key, value]) => `${key}=${value}`)
|
|
49
|
+
.join("&");
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** The headers to send, the `Authorization` one included. */
|
|
53
|
+
export function signRequest(input: SignInput): Record<string, string> {
|
|
54
|
+
const amzDate = formatAmzDate(input.now);
|
|
55
|
+
const day = amzDate.slice(0, 8);
|
|
56
|
+
const headers: Record<string, string> = {
|
|
57
|
+
host: input.url.host,
|
|
58
|
+
"x-amz-content-sha256": input.payloadSha256,
|
|
59
|
+
"x-amz-date": amzDate,
|
|
60
|
+
};
|
|
61
|
+
for (const [name, value] of Object.entries(input.headers ?? {})) headers[name.toLowerCase()] = value.trim().replace(/\s+/g, " ");
|
|
62
|
+
const names = Object.keys(headers).sort();
|
|
63
|
+
const canonicalHeaders = names.map((name) => `${name}:${headers[name] ?? ""}\n`).join("");
|
|
64
|
+
const signedHeaders = names.join(";");
|
|
65
|
+
// The path as the URL holds it is already percent-encoded once; S3 signs it without a second encoding.
|
|
66
|
+
const canonicalRequest = [
|
|
67
|
+
input.method,
|
|
68
|
+
input.url.pathname,
|
|
69
|
+
getCanonicalQuery(input.url),
|
|
70
|
+
canonicalHeaders,
|
|
71
|
+
signedHeaders,
|
|
72
|
+
input.payloadSha256,
|
|
73
|
+
].join("\n");
|
|
74
|
+
const scope = `${day}/${input.region}/s3/aws4_request`;
|
|
75
|
+
const stringToSign = ["AWS4-HMAC-SHA256", amzDate, scope, sha256Hex(canonicalRequest)].join("\n");
|
|
76
|
+
const signingKey = hmac(hmac(hmac(hmac(`AWS4${input.secretAccessKey}`, day), input.region), "s3"), "aws4_request");
|
|
77
|
+
const signature = createHmac("sha256", signingKey).update(stringToSign, "utf8").digest("hex");
|
|
78
|
+
// fetch sets Host itself from the URL.
|
|
79
|
+
const sent = Object.fromEntries(Object.entries(headers).filter(([name]) => name !== "host"));
|
|
80
|
+
return {
|
|
81
|
+
...sent,
|
|
82
|
+
authorization: `AWS4-HMAC-SHA256 Credential=${input.accessKeyId}/${scope}, SignedHeaders=${signedHeaders}, Signature=${signature}`,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// The status of a run as one SQL file for psql: the host run cannot always connect to the database itself
|
|
2
|
+
// (database.access "compose-exec" publishes no port), but deploy.sh can always pipe a file into psql in the postgres
|
|
3
|
+
// service. Values are literals quoted here (no psql variables, no server-side parameters), each checked first.
|
|
4
|
+
import type { RunRecord } from "./upload.js";
|
|
5
|
+
|
|
6
|
+
/** Rows of `backup.runs` older than this before the newest run are removed with it: a year and a month of history. */
|
|
7
|
+
export const STATUS_RETENTION_DAYS = 400;
|
|
8
|
+
const MAX_ERROR_MESSAGE = 500;
|
|
9
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
10
|
+
|
|
11
|
+
/** A standard SQL string literal (`standard_conforming_strings` is on since Postgres 9.1); NUL is dropped. */
|
|
12
|
+
export function quoteLiteral(text: string): string {
|
|
13
|
+
return `'${text.replace(/\0/g, "").replace(/'/g, "''")}'`;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function quoteTime(date: Date): string {
|
|
17
|
+
return `${quoteLiteral(date.toISOString())}::timestamptz`;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** The time before which `backup.runs` rows are pruned when a run that started at `startedAt` is recorded. */
|
|
21
|
+
export function getPruneBefore(startedAt: Date): Date {
|
|
22
|
+
return new Date(startedAt.getTime() - STATUS_RETENTION_DAYS * DAY_MS);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** One transaction: the run's row, then the prune of rows older than the retention. */
|
|
26
|
+
export function buildStatusSql(record: RunRecord): string {
|
|
27
|
+
const objects = JSON.stringify(record.objects.map(({ kind, key, bytes, sha256 }) => ({ kind, key, bytes, sha256 })));
|
|
28
|
+
const error = record.error;
|
|
29
|
+
const values = [
|
|
30
|
+
quoteTime(record.startedAt),
|
|
31
|
+
quoteTime(record.finishedAt),
|
|
32
|
+
quoteLiteral(record.result),
|
|
33
|
+
`${quoteLiteral(objects)}::jsonb`,
|
|
34
|
+
String(Math.max(0, Math.trunc(record.sizeBytes))),
|
|
35
|
+
error === null ? "NULL" : quoteLiteral(error.code),
|
|
36
|
+
error === null ? "NULL" : quoteLiteral(error.message.slice(0, MAX_ERROR_MESSAGE)),
|
|
37
|
+
];
|
|
38
|
+
return [
|
|
39
|
+
"BEGIN;",
|
|
40
|
+
`INSERT INTO backup.runs (started_at, finished_at, result, objects, size_bytes, error_code, error_message) VALUES (${values.join(", ")});`,
|
|
41
|
+
`DELETE FROM backup.runs WHERE started_at < ${quoteTime(getPruneBefore(record.startedAt))};`,
|
|
42
|
+
"COMMIT;",
|
|
43
|
+
"",
|
|
44
|
+
].join("\n");
|
|
45
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
// The nightly off-site copy: each file is encrypted to the recipient into a temporary file next to it (0600),
|
|
2
|
+
// hashed, PUT under the prefix and removed again. Nothing remote is ever listed, read or deleted, so a write-only
|
|
3
|
+
// token is enough. The result is a run record with safe error codes; secrets never reach its message.
|
|
4
|
+
import { createHash } from "node:crypto";
|
|
5
|
+
import { createReadStream, existsSync, rmSync, statSync } from "node:fs";
|
|
6
|
+
import { basename, dirname, join } from "node:path";
|
|
7
|
+
import { pipeline } from "node:stream/promises";
|
|
8
|
+
import type { BackupErrorCode, BackupObject, BackupResult } from "../contract.js";
|
|
9
|
+
import type { UploadConfig } from "./config.js";
|
|
10
|
+
import { encryptFile } from "./crypto.js";
|
|
11
|
+
import { buildObjectKey, parseBackupFileName } from "./objects.js";
|
|
12
|
+
import { putObjectFile, S3RequestError, type S3Options } from "./s3.js";
|
|
13
|
+
|
|
14
|
+
export interface RunRecord {
|
|
15
|
+
readonly startedAt: Date;
|
|
16
|
+
readonly finishedAt: Date;
|
|
17
|
+
readonly result: BackupResult;
|
|
18
|
+
readonly objects: readonly BackupObject[];
|
|
19
|
+
readonly sizeBytes: number;
|
|
20
|
+
readonly error: { readonly code: BackupErrorCode; readonly message: string } | null;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface UploadOptions {
|
|
24
|
+
readonly config: UploadConfig;
|
|
25
|
+
/** Local backup files: the run's dump, and its data archive when there is one. */
|
|
26
|
+
readonly files: readonly string[];
|
|
27
|
+
readonly now?: () => Date;
|
|
28
|
+
readonly s3?: S3Options;
|
|
29
|
+
/** Called after each uploaded object (for a log line). */
|
|
30
|
+
readonly onUploaded?: (object: BackupObject) => void;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
class UploadFailure extends Error {
|
|
34
|
+
readonly code: BackupErrorCode;
|
|
35
|
+
|
|
36
|
+
constructor(code: BackupErrorCode, message: string) {
|
|
37
|
+
super(message);
|
|
38
|
+
this.code = code;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** sha256 (hex) and size of a file, streamed. */
|
|
43
|
+
export async function hashFile(path: string): Promise<{ sha256: string; bytes: number }> {
|
|
44
|
+
const hash = createHash("sha256");
|
|
45
|
+
await pipeline(createReadStream(path), hash);
|
|
46
|
+
return { sha256: hash.digest("hex"), bytes: statSync(path).size };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function describeS3Error(error: unknown, key: string): UploadFailure {
|
|
50
|
+
if (error instanceof S3RequestError) {
|
|
51
|
+
if (error.isDenied) return new UploadFailure("upload_denied", `the store refused ${key}: ${error.message}`);
|
|
52
|
+
if (error.status === null) return new UploadFailure("network_error", error.message);
|
|
53
|
+
return new UploadFailure("upload_failed", error.message);
|
|
54
|
+
}
|
|
55
|
+
return new UploadFailure("upload_failed", `uploading ${key} failed (${error instanceof Error ? error.message : String(error)})`);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async function uploadFile(config: UploadConfig, path: string, options: UploadOptions): Promise<BackupObject> {
|
|
59
|
+
const name = basename(path);
|
|
60
|
+
const parsed = parseBackupFileName(name);
|
|
61
|
+
if (parsed === null) throw new UploadFailure("file_missing", `${name} is not a backup file name (db-<time>.dump or data-<time>.tar.gz)`);
|
|
62
|
+
if (!existsSync(path) || !statSync(path).isFile()) throw new UploadFailure("file_missing", `${name} is not in ${dirname(path)}`);
|
|
63
|
+
const encrypted = join(dirname(path), `.offsite-${String(process.pid)}-${name}.age`);
|
|
64
|
+
try {
|
|
65
|
+
try {
|
|
66
|
+
await encryptFile(path, encrypted, config.recipient);
|
|
67
|
+
} catch (error) {
|
|
68
|
+
throw new UploadFailure("encrypt_failed", `encrypting ${name} failed (${error instanceof Error ? error.message : String(error)})`);
|
|
69
|
+
}
|
|
70
|
+
const { sha256, bytes } = await hashFile(encrypted);
|
|
71
|
+
const key = buildObjectKey(config.store.prefix, name);
|
|
72
|
+
try {
|
|
73
|
+
await putObjectFile(config.store, { key, path: encrypted, bytes, sha256 }, options.s3);
|
|
74
|
+
} catch (error) {
|
|
75
|
+
throw describeS3Error(error, key);
|
|
76
|
+
}
|
|
77
|
+
return { kind: parsed.kind, key, bytes, sha256 };
|
|
78
|
+
} finally {
|
|
79
|
+
rmSync(encrypted, { force: true });
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Uploads every file in order; the first failure ends the run (already uploaded objects stay listed). */
|
|
84
|
+
export async function uploadBackups(options: UploadOptions): Promise<RunRecord> {
|
|
85
|
+
const now = options.now ?? (() => new Date());
|
|
86
|
+
const startedAt = now();
|
|
87
|
+
const objects: BackupObject[] = [];
|
|
88
|
+
let error: RunRecord["error"] = null;
|
|
89
|
+
for (const file of options.files) {
|
|
90
|
+
try {
|
|
91
|
+
const object = await uploadFile(options.config, file, options);
|
|
92
|
+
objects.push(object);
|
|
93
|
+
options.onUploaded?.(object);
|
|
94
|
+
} catch (caught) {
|
|
95
|
+
error =
|
|
96
|
+
caught instanceof UploadFailure
|
|
97
|
+
? { code: caught.code, message: caught.message }
|
|
98
|
+
: { code: "upload_failed", message: caught instanceof Error ? caught.message : String(caught) };
|
|
99
|
+
break;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if (options.files.length === 0) error = { code: "file_missing", message: "no backup file was given" };
|
|
103
|
+
return {
|
|
104
|
+
startedAt,
|
|
105
|
+
finishedAt: now(),
|
|
106
|
+
result: error === null ? "ok" : "failed",
|
|
107
|
+
objects,
|
|
108
|
+
sizeBytes: objects.reduce((sum, object) => sum + object.bytes, 0),
|
|
109
|
+
error,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** A run that did nothing because the host has no off-site configuration. */
|
|
114
|
+
export function buildNotConfiguredRecord(now: Date): RunRecord {
|
|
115
|
+
return { startedAt: now, finishedAt: now, result: "not_configured", objects: [], sizeBytes: 0, error: null };
|
|
116
|
+
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
// Verify the newest off-site copy end to end: it is recent enough, it decrypts, pg_restore takes it whole into a
|
|
2
|
+
// throwaway database, the deploy's schema guard accepts it against the app's migrations, and every table of
|
|
3
|
+
// deploy.json's rowCountTables (less excludeTableData, whose rows the dump leaves out) is there; then the database is
|
|
4
|
+
// dropped, also after a failure. It needs the private key, so it runs on the operator's machine or in a CI job,
|
|
5
|
+
// never on the server.
|
|
6
|
+
import { countRows, guardSchema, withPgClient, type RowCounts } from "@softure-ai/deploy";
|
|
7
|
+
import type { StoreConfig } from "./config.js";
|
|
8
|
+
import { decryptStream } from "./crypto.js";
|
|
9
|
+
import { createDatabase, createVerifyDatabaseName, dropDatabase, runPgRestore, withDatabase } from "./postgres.js";
|
|
10
|
+
import { resolveKey, type OffsiteResult } from "./restore.js";
|
|
11
|
+
import { getObjectStream, type S3Options } from "./s3.js";
|
|
12
|
+
|
|
13
|
+
export const DEFAULT_MAX_AGE_HOURS = 26;
|
|
14
|
+
const HOUR_MS = 60 * 60 * 1000;
|
|
15
|
+
|
|
16
|
+
export interface VerifyOptions {
|
|
17
|
+
readonly store: StoreConfig;
|
|
18
|
+
readonly identities: readonly string[];
|
|
19
|
+
/** A superuser's URL on a scratch server (or the operator's own); a throwaway database is created there. */
|
|
20
|
+
readonly adminUrl: string;
|
|
21
|
+
/** Tables that must exist in the restored copy. */
|
|
22
|
+
readonly tables: readonly string[];
|
|
23
|
+
/** The app's exported migrations (the image's softure-migrations folder): the schema guard runs when set. */
|
|
24
|
+
readonly migrationsDir?: string;
|
|
25
|
+
readonly maxAgeHours?: number;
|
|
26
|
+
readonly pgRestore: string;
|
|
27
|
+
readonly env: Readonly<Record<string, string | undefined>>;
|
|
28
|
+
readonly now?: Date;
|
|
29
|
+
readonly s3?: S3Options;
|
|
30
|
+
/** A fixed name instead of a random one (tests). */
|
|
31
|
+
readonly database?: string;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface VerifyOutcome {
|
|
35
|
+
readonly key: string;
|
|
36
|
+
readonly ageHours: number;
|
|
37
|
+
readonly counts: RowCounts;
|
|
38
|
+
readonly schemaGuard: "passed" | "skipped";
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** The tables to check: rowCountTables less excludeTableData. */
|
|
42
|
+
export function selectVerifyTables(rowCountTables: readonly string[] | undefined, excludeTableData: readonly string[] | undefined): string[] {
|
|
43
|
+
const excluded = new Set(excludeTableData ?? []);
|
|
44
|
+
return (rowCountTables ?? []).filter((table) => !excluded.has(table));
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async function checkRestored(url: string, options: VerifyOptions): Promise<OffsiteResult<{ counts: RowCounts; schemaGuard: "passed" | "skipped" }>> {
|
|
48
|
+
return withPgClient(url, async (client) => {
|
|
49
|
+
let schemaGuard: "passed" | "skipped" = "skipped";
|
|
50
|
+
if (options.migrationsDir !== undefined) {
|
|
51
|
+
const guard = await guardSchema(client, options.migrationsDir);
|
|
52
|
+
if (!guard.ok) return { ok: false, problem: `the schema guard refused the restored copy: ${guard.problems.join("; ")}` };
|
|
53
|
+
schemaGuard = "passed";
|
|
54
|
+
}
|
|
55
|
+
const counts = await countRows(client, options.tables);
|
|
56
|
+
const missing = Object.entries(counts).filter(([, count]) => count === null).map(([table]) => table);
|
|
57
|
+
if (missing.length > 0) return { ok: false, problem: `the restored copy lacks ${missing.join(", ")}` };
|
|
58
|
+
return { ok: true, value: { counts, schemaGuard } };
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export async function verifyBackup(options: VerifyOptions): Promise<OffsiteResult<VerifyOutcome>> {
|
|
63
|
+
const now = options.now ?? new Date();
|
|
64
|
+
const resolved = await resolveKey(options.store, "database", undefined, options.s3);
|
|
65
|
+
if (!resolved.ok) return resolved;
|
|
66
|
+
const ageHours = (now.getTime() - resolved.value.takenAt.getTime()) / HOUR_MS;
|
|
67
|
+
const maxAgeHours = options.maxAgeHours ?? DEFAULT_MAX_AGE_HOURS;
|
|
68
|
+
if (ageHours > maxAgeHours) {
|
|
69
|
+
return { ok: false, problem: `the newest off-site copy ${resolved.value.key} is ${ageHours.toFixed(1)} h old, more than ${String(maxAgeHours)} h` };
|
|
70
|
+
}
|
|
71
|
+
const encrypted = await getObjectStream(options.store, resolved.value.key, options.s3);
|
|
72
|
+
let dump: ReadableStream<Uint8Array>;
|
|
73
|
+
try {
|
|
74
|
+
dump = await decryptStream(encrypted, options.identities);
|
|
75
|
+
} catch (error) {
|
|
76
|
+
await encrypted.cancel().catch(() => undefined);
|
|
77
|
+
return { ok: false, problem: `cannot decrypt ${resolved.value.key} with this identity (${error instanceof Error ? error.message : String(error)})` };
|
|
78
|
+
}
|
|
79
|
+
const database = options.database ?? createVerifyDatabaseName();
|
|
80
|
+
await createDatabase(options.adminUrl, database);
|
|
81
|
+
try {
|
|
82
|
+
const restored = await runPgRestore({
|
|
83
|
+
adminUrl: options.adminUrl,
|
|
84
|
+
database,
|
|
85
|
+
flags: ["--no-owner", "--no-acl"],
|
|
86
|
+
dump,
|
|
87
|
+
pgRestore: options.pgRestore,
|
|
88
|
+
env: options.env,
|
|
89
|
+
});
|
|
90
|
+
if (!restored.ok) return { ok: false, problem: `${resolved.value.key} does not restore: ${restored.problem}` };
|
|
91
|
+
const checked = await checkRestored(withDatabase(options.adminUrl, database), options);
|
|
92
|
+
if (!checked.ok) return checked;
|
|
93
|
+
return { ok: true, value: { key: resolved.value.key, ageHours, ...checked.value } };
|
|
94
|
+
} finally {
|
|
95
|
+
await dropDatabase(options.adminUrl, database);
|
|
96
|
+
}
|
|
97
|
+
}
|
package/src/schema.ts
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// Drizzle view of the module's table (migrations/0001_create_runs.sql). The migration is the source of truth; this
|
|
2
|
+
// file only types the queries.
|
|
3
|
+
import { bigint, jsonb, pgSchema, text, timestamp } from "drizzle-orm/pg-core";
|
|
4
|
+
import type { BackupObject, BackupResult } from "./contract.js";
|
|
5
|
+
|
|
6
|
+
export const backupSchema = pgSchema("backup");
|
|
7
|
+
|
|
8
|
+
export const runs = backupSchema.table("runs", {
|
|
9
|
+
id: bigint("id", { mode: "number" }).primaryKey(),
|
|
10
|
+
startedAt: timestamp("started_at", { withTimezone: true }).notNull(),
|
|
11
|
+
finishedAt: timestamp("finished_at", { withTimezone: true }).notNull(),
|
|
12
|
+
result: text("result").$type<BackupResult>().notNull(),
|
|
13
|
+
objects: jsonb("objects").$type<BackupObject[]>().notNull(),
|
|
14
|
+
sizeBytes: bigint("size_bytes", { mode: "number" }).notNull(),
|
|
15
|
+
errorCode: text("error_code"),
|
|
16
|
+
errorMessage: text("error_message"),
|
|
17
|
+
});
|