@softure-ai/backup 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/CHANGELOG.md +30 -0
  2. package/LICENSE +21 -0
  3. package/README.md +243 -2
  4. package/dist/cli/main.d.ts +3 -0
  5. package/dist/cli/main.d.ts.map +1 -0
  6. package/dist/cli/main.js +16 -0
  7. package/dist/cli/main.js.map +1 -0
  8. package/dist/cli/run.d.ts +15 -0
  9. package/dist/cli/run.d.ts.map +1 -0
  10. package/dist/cli/run.js +442 -0
  11. package/dist/cli/run.js.map +1 -0
  12. package/dist/contract.d.ts +41 -0
  13. package/dist/contract.d.ts.map +1 -0
  14. package/dist/contract.js +15 -0
  15. package/dist/contract.js.map +1 -0
  16. package/dist/index.d.ts +22 -0
  17. package/dist/index.d.ts.map +1 -0
  18. package/dist/index.js +30 -0
  19. package/dist/index.js.map +1 -0
  20. package/dist/messages/en.d.ts +17 -0
  21. package/dist/messages/en.d.ts.map +1 -0
  22. package/dist/messages/en.js +17 -0
  23. package/dist/messages/en.js.map +1 -0
  24. package/dist/messages/index.d.ts +22 -0
  25. package/dist/messages/index.d.ts.map +1 -0
  26. package/dist/messages/index.js +4 -0
  27. package/dist/messages/index.js.map +1 -0
  28. package/dist/offsite/config.d.ts +45 -0
  29. package/dist/offsite/config.d.ts.map +1 -0
  30. package/dist/offsite/config.js +147 -0
  31. package/dist/offsite/config.js.map +1 -0
  32. package/dist/offsite/crypto.d.ts +22 -0
  33. package/dist/offsite/crypto.d.ts.map +1 -0
  34. package/dist/offsite/crypto.js +48 -0
  35. package/dist/offsite/crypto.js.map +1 -0
  36. package/dist/offsite/dump.d.ts +23 -0
  37. package/dist/offsite/dump.d.ts.map +1 -0
  38. package/dist/offsite/dump.js +85 -0
  39. package/dist/offsite/dump.js.map +1 -0
  40. package/dist/offsite/index.d.ts +13 -0
  41. package/dist/offsite/index.d.ts.map +1 -0
  42. package/dist/offsite/index.js +15 -0
  43. package/dist/offsite/index.js.map +1 -0
  44. package/dist/offsite/lifecycle.d.ts +17 -0
  45. package/dist/offsite/lifecycle.d.ts.map +1 -0
  46. package/dist/offsite/lifecycle.js +45 -0
  47. package/dist/offsite/lifecycle.js.map +1 -0
  48. package/dist/offsite/objects.d.ts +19 -0
  49. package/dist/offsite/objects.d.ts.map +1 -0
  50. package/dist/offsite/objects.js +44 -0
  51. package/dist/offsite/objects.js.map +1 -0
  52. package/dist/offsite/postgres.d.ts +32 -0
  53. package/dist/offsite/postgres.d.ts.map +1 -0
  54. package/dist/offsite/postgres.js +97 -0
  55. package/dist/offsite/postgres.js.map +1 -0
  56. package/dist/offsite/record.d.ts +3 -0
  57. package/dist/offsite/record.d.ts.map +1 -0
  58. package/dist/offsite/record.js +10 -0
  59. package/dist/offsite/record.js.map +1 -0
  60. package/dist/offsite/restore.d.ts +54 -0
  61. package/dist/offsite/restore.d.ts.map +1 -0
  62. package/dist/offsite/restore.js +67 -0
  63. package/dist/offsite/restore.js.map +1 -0
  64. package/dist/offsite/s3.d.ts +34 -0
  65. package/dist/offsite/s3.d.ts.map +1 -0
  66. package/dist/offsite/s3.js +119 -0
  67. package/dist/offsite/s3.js.map +1 -0
  68. package/dist/offsite/sigv4.d.ts +19 -0
  69. package/dist/offsite/sigv4.d.ts.map +1 -0
  70. package/dist/offsite/sigv4.js +69 -0
  71. package/dist/offsite/sigv4.js.map +1 -0
  72. package/dist/offsite/status-sql.d.ts +10 -0
  73. package/dist/offsite/status-sql.d.ts.map +1 -0
  74. package/dist/offsite/status-sql.js +37 -0
  75. package/dist/offsite/status-sql.js.map +1 -0
  76. package/dist/offsite/upload.d.ts +33 -0
  77. package/dist/offsite/upload.d.ts.map +1 -0
  78. package/dist/offsite/upload.js +98 -0
  79. package/dist/offsite/upload.js.map +1 -0
  80. package/dist/offsite/verify.d.ts +32 -0
  81. package/dist/offsite/verify.d.ts.map +1 -0
  82. package/dist/offsite/verify.js +75 -0
  83. package/dist/offsite/verify.js.map +1 -0
  84. package/dist/schema.d.ts +150 -0
  85. package/dist/schema.d.ts.map +1 -0
  86. package/dist/schema.js +15 -0
  87. package/dist/schema.js.map +1 -0
  88. package/dist/server/index.d.ts +28 -0
  89. package/dist/server/index.d.ts.map +1 -0
  90. package/dist/server/index.js +98 -0
  91. package/dist/server/index.js.map +1 -0
  92. package/examples/backup-verify.yml +84 -0
  93. package/migrations/0001_create_runs.sql +21 -0
  94. package/module.json +12 -0
  95. package/package.json +73 -4
  96. package/src/cli/main.ts +16 -0
  97. package/src/cli/run.ts +441 -0
  98. package/src/contract.ts +47 -0
  99. package/src/index.ts +41 -0
  100. package/src/messages/en.ts +16 -0
  101. package/src/messages/index.ts +6 -0
  102. package/src/offsite/config.ts +173 -0
  103. package/src/offsite/crypto.ts +53 -0
  104. package/src/offsite/dump.ts +97 -0
  105. package/src/offsite/index.ts +33 -0
  106. package/src/offsite/lifecycle.ts +47 -0
  107. package/src/offsite/objects.ts +54 -0
  108. package/src/offsite/postgres.ts +115 -0
  109. package/src/offsite/record.ts +11 -0
  110. package/src/offsite/restore.ts +108 -0
  111. package/src/offsite/s3.ts +144 -0
  112. package/src/offsite/sigv4.ts +84 -0
  113. package/src/offsite/status-sql.ts +45 -0
  114. package/src/offsite/upload.ts +116 -0
  115. package/src/offsite/verify.ts +97 -0
  116. package/src/schema.ts +17 -0
  117. package/src/server/index.ts +111 -0
@@ -0,0 +1,173 @@
1
+ // The off-site settings, read only from the environment and validated with zod. A host that sets none of the names
2
+ // is "not configured" (the step is skipped, never failed); one that sets some gets a list of what is missing or
3
+ // wrong, never a value. Two tokens: the nightly upload needs only a write token, restore and verify a read one.
4
+ import { z } from "zod";
5
+
6
+ /** The names the nightly upload reads (`.env.prod` on the server). */
7
+ export const UPLOAD_ENV_NAMES = [
8
+ "BACKUP_S3_ENDPOINT",
9
+ "BACKUP_S3_REGION",
10
+ "BACKUP_S3_BUCKET",
11
+ "BACKUP_S3_PREFIX",
12
+ "BACKUP_S3_ACCESS_KEY_ID",
13
+ "BACKUP_S3_SECRET_ACCESS_KEY",
14
+ "BACKUP_AGE_RECIPIENT",
15
+ ] as const;
16
+
17
+ /** The names restore, fetch and verify read (the operator's machine or a CI job, never the server). */
18
+ export const READ_ENV_NAMES = [
19
+ "BACKUP_S3_ENDPOINT",
20
+ "BACKUP_S3_REGION",
21
+ "BACKUP_S3_BUCKET",
22
+ "BACKUP_S3_PREFIX",
23
+ "BACKUP_S3_READ_ACCESS_KEY_ID",
24
+ "BACKUP_S3_READ_SECRET_ACCESS_KEY",
25
+ ] as const;
26
+
27
+ /** The secret names: their values are redacted from every message. */
28
+ export const SECRET_ENV_NAMES = [
29
+ "BACKUP_S3_ACCESS_KEY_ID",
30
+ "BACKUP_S3_SECRET_ACCESS_KEY",
31
+ "BACKUP_S3_READ_ACCESS_KEY_ID",
32
+ "BACKUP_S3_READ_SECRET_ACCESS_KEY",
33
+ ] as const;
34
+
35
+ export const DEFAULT_REGION = "auto";
36
+ /** A prefix is path segments of lower-case letters, digits, `.`, `_` and `-`, so a key can never climb out of it. */
37
+ const PREFIX_PATTERN = /^[a-z0-9][a-z0-9._-]*(\/[a-z0-9][a-z0-9._-]*)*$/;
38
+ /** S3 bucket names: 3 to 63 lower-case letters, digits, `.` and `-`. */
39
+ const BUCKET_PATTERN = /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/;
40
+ const REGION_PATTERN = /^[a-z0-9-]{1,32}$/;
41
+ /** An age X25519 recipient (`age1` and 58 bech32 characters) or a hybrid post-quantum one (`age1pq1…`). */
42
+ const RECIPIENT_PATTERN = /^age1(pq1)?[02-9ac-hj-np-z]{40,}$/;
43
+
44
+ /** Where the objects go: the same for the write and the read side. */
45
+ export interface StoreConfig {
46
+ readonly endpoint: URL;
47
+ readonly region: string;
48
+ readonly bucket: string;
49
+ readonly prefix: string;
50
+ readonly accessKeyId: string;
51
+ readonly secretAccessKey: string;
52
+ }
53
+
54
+ export interface UploadConfig {
55
+ readonly store: StoreConfig;
56
+ readonly recipient: string;
57
+ }
58
+
59
+ export type ConfigResult<T> =
60
+ | { readonly status: "ok"; readonly config: T }
61
+ | { readonly status: "not_configured" }
62
+ | { readonly status: "invalid"; readonly problems: readonly string[] };
63
+
64
+ type Env = Readonly<Record<string, string | undefined>>;
65
+
66
+ const endpointSchema = z
67
+ .string()
68
+ .refine((value) => URL.canParse(value), "is not a URL")
69
+ .transform((value) => new URL(value))
70
+ .refine((url) => url.protocol === "https:" || url.protocol === "http:", "must start with https:// (or http:// for a local MinIO)")
71
+ .refine((url) => url.username === "" && url.password === "", "must not carry a user or password")
72
+ .refine((url) => url.search === "" && url.hash === "", "must not carry a query or fragment");
73
+
74
+ function storeSchema(keyName: string, secretName: string) {
75
+ return z.object({
76
+ BACKUP_S3_ENDPOINT: endpointSchema,
77
+ BACKUP_S3_REGION: z.string().regex(REGION_PATTERN, "is not a region name").default(DEFAULT_REGION),
78
+ BACKUP_S3_BUCKET: z.string().regex(BUCKET_PATTERN, "is not an S3 bucket name"),
79
+ BACKUP_S3_PREFIX: z.string().regex(PREFIX_PATTERN, "is not a key prefix (lower-case path segments)").optional(),
80
+ [keyName]: z.string().min(1, "is empty").max(256, "is too long"),
81
+ [secretName]: z.string().min(1, "is empty").max(256, "is too long"),
82
+ });
83
+ }
84
+
85
+ /** Unset and empty values count as unset; values are trimmed of surrounding blanks a secret store may add. */
86
+ function readNames(env: Env, names: readonly string[]): Record<string, string> {
87
+ const values: Record<string, string> = {};
88
+ for (const name of names) {
89
+ const value = env[name]?.trim();
90
+ if (value !== undefined && value !== "") values[name] = value;
91
+ }
92
+ return values;
93
+ }
94
+
95
+ function describeIssues(error: z.ZodError): string[] {
96
+ return error.issues.map((issue) => {
97
+ const name = String(issue.path[0] ?? "configuration");
98
+ return issue.code === "invalid_type" ? `${name} is not set` : `${name} ${issue.message}`;
99
+ });
100
+ }
101
+
102
+ function toStore(values: Record<string, unknown>, keyName: string, secretName: string, defaultPrefix: string): StoreConfig {
103
+ return {
104
+ endpoint: values.BACKUP_S3_ENDPOINT as URL,
105
+ region: values.BACKUP_S3_REGION as string,
106
+ bucket: values.BACKUP_S3_BUCKET as string,
107
+ prefix: (values.BACKUP_S3_PREFIX as string | undefined) ?? defaultPrefix,
108
+ accessKeyId: values[keyName] as string,
109
+ secretAccessKey: values[secretName] as string,
110
+ };
111
+ }
112
+
113
+ function checkPrefix(prefix: string): string | null {
114
+ return PREFIX_PATTERN.test(prefix) ? null : `the app name ${JSON.stringify(prefix)} is not a key prefix; set BACKUP_S3_PREFIX`;
115
+ }
116
+
117
+ /**
118
+ * The upload settings of the nightly run. `appName` is the default prefix (each app its own folder of a shared
119
+ * bucket); `BACKUP_S3_PREFIX` overrides it. Without `BACKUP_S3_BUCKET` and every other name, not configured.
120
+ */
121
+ export function readUploadConfig(env: Env, appName: string): ConfigResult<UploadConfig> {
122
+ const values = readNames(env, UPLOAD_ENV_NAMES);
123
+ if (Object.keys(values).length === 0) return { status: "not_configured" };
124
+ const schema = storeSchema("BACKUP_S3_ACCESS_KEY_ID", "BACKUP_S3_SECRET_ACCESS_KEY").extend({
125
+ BACKUP_AGE_RECIPIENT: z.string().regex(RECIPIENT_PATTERN, "is not an age public key (age1…)"),
126
+ });
127
+ const parsed = schema.safeParse(values);
128
+ if (!parsed.success) return { status: "invalid", problems: describeIssues(parsed.error) };
129
+ const store = toStore(parsed.data, "BACKUP_S3_ACCESS_KEY_ID", "BACKUP_S3_SECRET_ACCESS_KEY", appName);
130
+ const prefixProblem = checkPrefix(store.prefix);
131
+ if (prefixProblem !== null) return { status: "invalid", problems: [prefixProblem] };
132
+ return { status: "ok", config: { store, recipient: values.BACKUP_AGE_RECIPIENT ?? "" } };
133
+ }
134
+
135
+ /** The read settings of restore, fetch and verify: the same store with the read token. */
136
+ export function readReadConfig(env: Env, appName: string): ConfigResult<StoreConfig> {
137
+ const values = readNames(env, READ_ENV_NAMES);
138
+ if (Object.keys(values).length === 0) return { status: "not_configured" };
139
+ const parsed = storeSchema("BACKUP_S3_READ_ACCESS_KEY_ID", "BACKUP_S3_READ_SECRET_ACCESS_KEY").safeParse(values);
140
+ if (!parsed.success) return { status: "invalid", problems: describeIssues(parsed.error) };
141
+ const store = toStore(parsed.data, "BACKUP_S3_READ_ACCESS_KEY_ID", "BACKUP_S3_READ_SECRET_ACCESS_KEY", appName);
142
+ const prefixProblem = checkPrefix(store.prefix);
143
+ if (prefixProblem !== null) return { status: "invalid", problems: [prefixProblem] };
144
+ return { status: "ok", config: store };
145
+ }
146
+
147
+ /** The non-empty values of the secret names, for `redactSecrets`. */
148
+ export function collectSecrets(env: Env): string[] {
149
+ const values = SECRET_ENV_NAMES.map((name) => env[name]?.trim() ?? "");
150
+ for (const name of DATABASE_URL_ENV_NAMES) values.push(...readUrlSecrets(env[name]));
151
+ return values.filter((value) => value.length >= 4);
152
+ }
153
+
154
+ /** The database URLs the CLI reads: their whole text and their password are redacted from every message. */
155
+ export const DATABASE_URL_ENV_NAMES = ["BACKUP_DATABASE_URL", "BACKUP_RESTORE_DATABASE_URL", "BACKUP_VERIFY_DATABASE_URL"] as const;
156
+
157
+ function readUrlSecrets(value: string | undefined): string[] {
158
+ const url = value?.trim() ?? "";
159
+ if (url === "") return [];
160
+ try {
161
+ const password = decodeURIComponent(new URL(url).password);
162
+ return password === "" ? [url] : [url, password];
163
+ } catch {
164
+ return [url];
165
+ }
166
+ }
167
+
168
+ /** `text` with every secret value replaced by `***`. */
169
+ export function redactSecrets(text: string, secrets: readonly string[]): string {
170
+ let redacted = text;
171
+ for (const secret of secrets) redacted = redacted.split(secret).join("***");
172
+ return redacted;
173
+ }
@@ -0,0 +1,53 @@
1
+ // age encryption with the pure-JS implementation by the age author (interoperable with the `age` command): the
2
+ // server holds only the public key (`age1…`); the private key (`AGE-SECRET-KEY-1…`) is read at run time from a file
3
+ // or stdin, never from an argument.
4
+ import { createReadStream, createWriteStream } from "node:fs";
5
+ import { Readable, Writable } from "node:stream";
6
+ import { pipeline } from "node:stream/promises";
7
+ import { Decrypter, Encrypter } from "age-encryption";
8
+
9
+ const IDENTITY_PATTERN = /^AGE-SECRET-KEY-(PQ-)?1[0-9A-Z]+$/;
10
+
11
+ /** Encrypts the file at `source` to `recipient` into `target`. */
12
+ export async function encryptFile(source: string, target: string, recipient: string): Promise<void> {
13
+ const encrypter = new Encrypter();
14
+ encrypter.addRecipient(recipient);
15
+ const input = Readable.toWeb(createReadStream(source)) as ReadableStream<Uint8Array>;
16
+ const encrypted = await encrypter.encrypt(input);
17
+ await pipeline(Readable.fromWeb(encrypted as import("node:stream/web").ReadableStream<Uint8Array>), createWriteStream(target, { mode: 0o600 }));
18
+ }
19
+
20
+ /** A stream of the plaintext of an age `stream`, decrypted with `identities`. */
21
+ export async function decryptStream(stream: ReadableStream<Uint8Array>, identities: readonly string[]): Promise<ReadableStream<Uint8Array>> {
22
+ const decrypter = new Decrypter();
23
+ for (const identity of identities) decrypter.addIdentity(identity);
24
+ return decrypter.decrypt(stream);
25
+ }
26
+
27
+ /** Writes a web stream to a file, owner read and write only. */
28
+ export async function writeStreamToFile(stream: ReadableStream<Uint8Array>, path: string): Promise<void> {
29
+ await pipeline(Readable.fromWeb(stream as import("node:stream/web").ReadableStream<Uint8Array>), createWriteStream(path, { mode: 0o600 }));
30
+ }
31
+
32
+ /** Pipes a web stream into a Node writable (a child's stdin). */
33
+ export async function pipeStreamTo(stream: ReadableStream<Uint8Array>, target: Writable): Promise<void> {
34
+ await pipeline(Readable.fromWeb(stream as import("node:stream/web").ReadableStream<Uint8Array>), target);
35
+ }
36
+
37
+ export type IdentitiesResult = { ok: true; identities: string[] } | { ok: false; problem: string };
38
+
39
+ /**
40
+ * The identities of an age key file's text (the `age-keygen` output: comment lines and one or more keys). The
41
+ * problem never repeats the text.
42
+ */
43
+ export function parseIdentities(text: string): IdentitiesResult {
44
+ const identities = text
45
+ .split(/\r?\n/)
46
+ .map((line) => line.trim())
47
+ .filter((line) => line !== "" && !line.startsWith("#"));
48
+ if (identities.length === 0) return { ok: false, problem: "the identity holds no AGE-SECRET-KEY-1… line" };
49
+ if (!identities.every((line) => IDENTITY_PATTERN.test(line))) {
50
+ return { ok: false, problem: "the identity has a line that is not an age secret key (AGE-SECRET-KEY-1…)" };
51
+ }
52
+ return { ok: true, identities };
53
+ }
@@ -0,0 +1,97 @@
1
+ // The night's dump for a host without a server (a Cloudflare app's scheduled workflow): `pg_dump --format=custom` of
2
+ // the database at a URL, to stdout, into a hidden file that becomes the backup file only once pg_dump exited 0, so a
3
+ // dump cut short is never uploaded. The URL goes in as libpq variables, never as an argument. Writing to stdout lets
4
+ // `BACKUP_PG_DUMP` be a `docker run postgres:<major> pg_dump` wrapper of the server's major version.
5
+ import { spawn } from "node:child_process";
6
+ import { createWriteStream, renameSync, rmSync } from "node:fs";
7
+ import { basename, dirname, join } from "node:path";
8
+ import { toLibpqEnv } from "@softure-ai/deploy";
9
+
10
+ const MAX_STDERR = 4000;
11
+ const PASSED_ENV = ["PATH", "HOME", "LANG", "DOCKER_HOST", "DOCKER_CONFIG"] as const;
12
+
13
+ export interface PgDumpOptions {
14
+ readonly url: string;
15
+ /** The backup file to write, e.g. `<dir>/db-20261117T031700Z.dump`. */
16
+ readonly out: string;
17
+ /** Tables whose rows the dump leaves out (deploy.json `database.excludeTableData`). */
18
+ readonly excludeTableData: readonly string[];
19
+ /** The executable: a name on PATH or a path. */
20
+ readonly pgDump: string;
21
+ readonly env: Readonly<Record<string, string | undefined>>;
22
+ }
23
+
24
+ export type PgDumpResult = { ok: true; bytes: number } | { ok: false; problem: string };
25
+
26
+ /** `db-<YYYYMMDD>T<HHMMSS>Z.dump`, the deploy's own backup name, so the off-site keys sort with a VPS's. */
27
+ export function buildDumpFileName(at: Date): string {
28
+ const stamp = at.toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z");
29
+ return `db-${stamp}.dump`;
30
+ }
31
+
32
+ /** The arguments of the dump: custom format, no password prompt, the excluded tables' rows left out. */
33
+ export function buildPgDumpArgs(excludeTableData: readonly string[]): string[] {
34
+ return ["--format=custom", "--no-password", ...excludeTableData.map((table) => `--exclude-table-data=${table}`)];
35
+ }
36
+
37
+ export async function runPgDump(options: PgDumpOptions): Promise<PgDumpResult> {
38
+ const libpq = toLibpqEnv(options.url);
39
+ if (!libpq.ok) return { ok: false, problem: libpq.problem };
40
+ const env: Record<string, string> = { ...libpq.env };
41
+ for (const name of PASSED_ENV) {
42
+ const value = options.env[name];
43
+ if (value !== undefined) env[name] = value;
44
+ }
45
+ const partial = join(dirname(options.out), `.partial-${basename(options.out)}`);
46
+ const file = createWriteStream(partial, { mode: 0o600 });
47
+ const fileOpened = new Promise<void>((resolve, reject) => {
48
+ file.once("open", () => {
49
+ resolve();
50
+ });
51
+ file.once("error", reject);
52
+ });
53
+ try {
54
+ await fileOpened;
55
+ } catch (error) {
56
+ return { ok: false, problem: `cannot write ${partial} (${error instanceof Error ? error.message : String(error)})` };
57
+ }
58
+ const child = spawn(options.pgDump, buildPgDumpArgs(options.excludeTableData), { env, stdio: ["ignore", "pipe", "pipe"] });
59
+ let stderr = "";
60
+ let bytes = 0;
61
+ child.stderr.on("data", (chunk: Buffer) => {
62
+ stderr = (stderr + chunk.toString("utf8")).slice(-MAX_STDERR);
63
+ });
64
+ child.stdout.on("data", (chunk: Buffer) => {
65
+ bytes += chunk.length;
66
+ });
67
+ child.stdout.pipe(file);
68
+ const written = new Promise<void>((resolve, reject) => {
69
+ file.once("finish", resolve);
70
+ file.once("error", reject);
71
+ });
72
+ const exited = new Promise<number | null>((resolve, reject) => {
73
+ child.once("error", reject);
74
+ child.once("close", (code) => {
75
+ resolve(code);
76
+ });
77
+ });
78
+ let code: number | null;
79
+ try {
80
+ code = await exited;
81
+ await written;
82
+ } catch (error) {
83
+ file.destroy();
84
+ rmSync(partial, { force: true });
85
+ return { ok: false, problem: `cannot run ${options.pgDump} (${error instanceof Error ? error.message : String(error)})` };
86
+ }
87
+ if (code !== 0) {
88
+ rmSync(partial, { force: true });
89
+ return { ok: false, problem: `pg_dump exited with ${String(code)}: ${stderr.trim().split("\n").slice(-3).join(" / ")}` };
90
+ }
91
+ if (bytes === 0) {
92
+ rmSync(partial, { force: true });
93
+ return { ok: false, problem: "pg_dump wrote nothing" };
94
+ }
95
+ renameSync(partial, options.out);
96
+ return { ok: true, bytes };
97
+ }
@@ -0,0 +1,33 @@
1
+ // The host and operator side of @softure-ai/backup: configuration, the nightly upload, status SQL, restore, fetch and
2
+ // verify. The `softure-backup` command line wraps these; an app's own code needs only `@softure-ai/backup/server`.
3
+ export {
4
+ collectSecrets,
5
+ DEFAULT_REGION,
6
+ READ_ENV_NAMES,
7
+ readReadConfig,
8
+ readUploadConfig,
9
+ redactSecrets,
10
+ SECRET_ENV_NAMES,
11
+ UPLOAD_ENV_NAMES,
12
+ type ConfigResult,
13
+ type StoreConfig,
14
+ type UploadConfig,
15
+ } from "./config.js";
16
+ export { decryptStream, encryptFile, parseIdentities, type IdentitiesResult } from "./crypto.js";
17
+ export { buildDumpFileName, buildPgDumpArgs, runPgDump, type PgDumpOptions, type PgDumpResult } from "./dump.js";
18
+ export {
19
+ buildLifecycleJson,
20
+ buildLifecycleXml,
21
+ DEFAULT_LIFECYCLE_DAYS,
22
+ MAX_LIFECYCLE_DAYS,
23
+ MIN_LIFECYCLE_DAYS,
24
+ parseLifecycleDays,
25
+ } from "./lifecycle.js";
26
+ export { buildObjectKey, OBJECT_SUFFIX, parseBackupFileName, parseObjectKey, selectNewestKey, type BackupFileName } from "./objects.js";
27
+ export { fetchBackup, resolveKey, restoreBackup, type FetchOptions, type OffsiteResult, type RestoreOptions, type RestoreOutcome } from "./restore.js";
28
+ export { getObjectStream, getObjectUrl, listObjects, putObjectFile, S3RequestError, type S3Options, type StoredObject } from "./s3.js";
29
+ export { EMPTY_SHA256, encodeRfc3986, formatAmzDate, signRequest, type SignInput } from "./sigv4.js";
30
+ export { recordRun } from "./record.js";
31
+ export { buildStatusSql, getPruneBefore, quoteLiteral, STATUS_RETENTION_DAYS } from "./status-sql.js";
32
+ export { buildNotConfiguredRecord, hashFile, uploadBackups, type RunRecord, type UploadOptions } from "./upload.js";
33
+ export { DEFAULT_MAX_AGE_HOURS, selectVerifyTables, verifyBackup, type VerifyOptions, type VerifyOutcome } from "./verify.js";
@@ -0,0 +1,47 @@
1
+ // Remote retention is the bucket's: a lifecycle rule on the prefix expires objects after N days. The nightly run
2
+ // never deletes (its token cannot), so a compromised server cannot wipe the history either.
3
+ export const DEFAULT_LIFECYCLE_DAYS = 30;
4
+ export const MIN_LIFECYCLE_DAYS = 1;
5
+ export const MAX_LIFECYCLE_DAYS = 3650;
6
+
7
+ export type LifecycleDaysResult = { ok: true; days: number } | { ok: false; problem: string };
8
+
9
+ /** Whole days from 1 to 3650 (ten years). */
10
+ export function parseLifecycleDays(text: string | undefined): LifecycleDaysResult {
11
+ if (text === undefined) return { ok: true, days: DEFAULT_LIFECYCLE_DAYS };
12
+ if (!/^\d{1,4}$/.test(text)) return { ok: false, problem: `--days is a whole number of days, not ${JSON.stringify(text)}` };
13
+ const days = Number(text);
14
+ if (days < MIN_LIFECYCLE_DAYS || days > MAX_LIFECYCLE_DAYS) {
15
+ return { ok: false, problem: `--days is from ${String(MIN_LIFECYCLE_DAYS)} to ${String(MAX_LIFECYCLE_DAYS)}, not ${String(days)}` };
16
+ }
17
+ return { ok: true, days };
18
+ }
19
+
20
+ /** The S3 `LifecycleConfiguration` (XML, for PutBucketLifecycleConfiguration) expiring `<prefix>/` after `days`. */
21
+ export function buildLifecycleXml(prefix: string, days: number): string {
22
+ return [
23
+ '<?xml version="1.0" encoding="UTF-8"?>',
24
+ '<LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">',
25
+ " <Rule>",
26
+ ` <ID>softure-backup-${prefix.replace(/[^a-z0-9-]/g, "-")}</ID>`,
27
+ " <Status>Enabled</Status>",
28
+ ` <Filter><Prefix>${prefix}/</Prefix></Filter>`,
29
+ ` <Expiration><Days>${String(days)}</Days></Expiration>`,
30
+ " <AbortIncompleteMultipartUpload><DaysAfterInitiation>1</DaysAfterInitiation></AbortIncompleteMultipartUpload>",
31
+ " </Rule>",
32
+ "</LifecycleConfiguration>",
33
+ "",
34
+ ].join("\n");
35
+ }
36
+
37
+ /** The same rule as JSON for `aws s3api put-bucket-lifecycle-configuration --lifecycle-configuration file://…`. */
38
+ export function buildLifecycleJson(prefix: string, days: number): string {
39
+ const rule = {
40
+ ID: `softure-backup-${prefix.replace(/[^a-z0-9-]/g, "-")}`,
41
+ Status: "Enabled",
42
+ Filter: { Prefix: `${prefix}/` },
43
+ Expiration: { Days: days },
44
+ AbortIncompleteMultipartUpload: { DaysAfterInitiation: 1 },
45
+ };
46
+ return `${JSON.stringify({ Rules: [rule] }, null, 2)}\n`;
47
+ }
@@ -0,0 +1,54 @@
1
+ // Object names: `<prefix>/<the local file name>.age`. Only the deploy's own backup names are accepted, so a key
2
+ // sorts by time and nothing else can be uploaded by mistake. Retention is the bucket's lifecycle rule on the prefix;
3
+ // this package never deletes an object.
4
+ import type { BackupKind } from "../contract.js";
5
+
6
+ /** `db-20261109T031700Z.dump` (`@softure-ai/deploy` backup, any short prefix). */
7
+ const DATABASE_NAME = /^([a-z0-9][a-z0-9_-]*)-(\d{8})T(\d{6})Z\.dump$/;
8
+ /** `data-20261109-031700.tar.gz` (deploy.sh backup-data). */
9
+ const DATA_NAME = /^data-(\d{8})-(\d{6})\.tar\.gz$/;
10
+ export const OBJECT_SUFFIX = ".age";
11
+
12
+ export interface BackupFileName {
13
+ readonly kind: BackupKind;
14
+ readonly name: string;
15
+ /** The time in the name, UTC. */
16
+ readonly takenAt: Date;
17
+ }
18
+
19
+ function toDate(day: string, time: string): Date {
20
+ return new Date(`${day.slice(0, 4)}-${day.slice(4, 6)}-${day.slice(6, 8)}T${time.slice(0, 2)}:${time.slice(2, 4)}:${time.slice(4, 6)}Z`);
21
+ }
22
+
23
+ /** The kind and time of a backup file name, or null for any other name (or an impossible date). */
24
+ export function parseBackupFileName(name: string): BackupFileName | null {
25
+ const data = DATA_NAME.exec(name);
26
+ const database = data === null ? DATABASE_NAME.exec(name) : null;
27
+ if (data === null && database === null) return null;
28
+ const takenAt = data === null ? toDate(database?.[2] ?? "", database?.[3] ?? "") : toDate(data[1] ?? "", data[2] ?? "");
29
+ if (Number.isNaN(takenAt.getTime())) return null;
30
+ return { kind: data === null ? "database" : "data", name, takenAt };
31
+ }
32
+
33
+ export function buildObjectKey(prefix: string, fileName: string): string {
34
+ return `${prefix}/${fileName}${OBJECT_SUFFIX}`;
35
+ }
36
+
37
+ /** The backup file name of a key under `prefix`, or null for any other key. */
38
+ export function parseObjectKey(prefix: string, key: string): BackupFileName | null {
39
+ if (!key.startsWith(`${prefix}/`) || !key.endsWith(OBJECT_SUFFIX)) return null;
40
+ const name = key.slice(prefix.length + 1, -OBJECT_SUFFIX.length);
41
+ if (name.includes("/")) return null;
42
+ return parseBackupFileName(name);
43
+ }
44
+
45
+ /** The newest key of `kind` under `prefix`, by the time in its name; null when there is none. */
46
+ export function selectNewestKey(prefix: string, keys: readonly string[], kind: BackupKind): { key: string; takenAt: Date } | null {
47
+ let newest: { key: string; takenAt: Date } | null = null;
48
+ for (const key of keys) {
49
+ const parsed = parseObjectKey(prefix, key);
50
+ if (parsed?.kind !== kind) continue;
51
+ if (newest === null || parsed.takenAt.getTime() > newest.takenAt.getTime()) newest = { key, takenAt: parsed.takenAt };
52
+ }
53
+ return newest;
54
+ }
@@ -0,0 +1,115 @@
1
+ // The Postgres side of restore and verify: an admin connection from a URL in the environment, databases created and
2
+ // dropped through it, and pg_restore fed from a decrypted stream with the URL as libpq variables, so the password
3
+ // never shows in the process list.
4
+ import { spawn } from "node:child_process";
5
+ import { randomBytes } from "node:crypto";
6
+ import { toLibpqEnv, withPgClient } from "@softure-ai/deploy";
7
+ import { pipeStreamTo } from "./crypto.js";
8
+
9
+ /** Database names this package creates or accepts: lower-case identifiers within Postgres's 63 bytes. */
10
+ const DATABASE_NAME = /^[a-z_][a-z0-9_]{0,62}$/;
11
+ const MAX_STDERR = 4000;
12
+
13
+ export function isDatabaseName(name: string): boolean {
14
+ return DATABASE_NAME.test(name);
15
+ }
16
+
17
+ /** The URL with another database. */
18
+ export function withDatabase(url: string, database: string): string {
19
+ const parsed = new URL(url);
20
+ parsed.pathname = `/${database}`;
21
+ return parsed.toString();
22
+ }
23
+
24
+ /** The database of a URL (`postgres` when it names none). */
25
+ export function getUrlDatabase(url: string): string {
26
+ const name = decodeURIComponent(new URL(url).pathname.replace(/^\//, ""));
27
+ return name === "" ? "postgres" : name;
28
+ }
29
+
30
+ /** `softure_offsite_verify_<8 hex>`: a fresh throwaway name. */
31
+ export function createVerifyDatabaseName(): string {
32
+ return `softure_offsite_verify_${randomBytes(4).toString("hex")}`;
33
+ }
34
+
35
+ /** `<base>_restore_<YYYYMMDDHHMMSS>`, shortened so it stays a valid name. */
36
+ export function createRestoreDatabaseName(base: string, now: Date): string {
37
+ const stamp = now.toISOString().replace(/[-:T]/g, "").slice(0, 14);
38
+ const safeBase = base.replace(/[^a-z0-9_]/g, "_").replace(/^[^a-z_]/, "_").slice(0, 63 - "_restore_".length - stamp.length);
39
+ return `${safeBase}_restore_${stamp}`;
40
+ }
41
+
42
+ export async function databaseExists(adminUrl: string, name: string): Promise<boolean> {
43
+ return withPgClient(adminUrl, async (client) => {
44
+ const result = await client.query<{ present: boolean }>("SELECT EXISTS (SELECT 1 FROM pg_database WHERE datname = $1) AS present", [name]);
45
+ return result.rows[0]?.present === true;
46
+ });
47
+ }
48
+
49
+ export async function createDatabase(adminUrl: string, name: string): Promise<void> {
50
+ if (!isDatabaseName(name)) throw new Error(`${name} is not a database name this tool creates`);
51
+ await withPgClient(adminUrl, async (client) => {
52
+ await client.query(`CREATE DATABASE "${name}" TEMPLATE template0`);
53
+ });
54
+ }
55
+
56
+ /** Drops `name`, ending its connections first (Postgres 13+). */
57
+ export async function dropDatabase(adminUrl: string, name: string): Promise<void> {
58
+ if (!isDatabaseName(name)) throw new Error(`${name} is not a database name this tool drops`);
59
+ await withPgClient(adminUrl, async (client) => {
60
+ await client.query(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`);
61
+ });
62
+ }
63
+
64
+ export interface PgRestoreOptions {
65
+ readonly adminUrl: string;
66
+ readonly database: string;
67
+ /** Extra flags: `--no-owner`, `--no-acl`, `--clean --if-exists`. */
68
+ readonly flags: readonly string[];
69
+ readonly dump: ReadableStream<Uint8Array>;
70
+ /** The executable: a name on PATH or a path. */
71
+ readonly pgRestore: string;
72
+ readonly env: Readonly<Record<string, string | undefined>>;
73
+ }
74
+
75
+ export type PgRestoreResult = { ok: true } | { ok: false; problem: string };
76
+
77
+ /** `pg_restore --exit-on-error` of the dump on stdin into `database`. */
78
+ export async function runPgRestore(options: PgRestoreOptions): Promise<PgRestoreResult> {
79
+ const libpq = toLibpqEnv(options.adminUrl);
80
+ if (!libpq.ok) return { ok: false, problem: libpq.problem };
81
+ const env: Record<string, string> = { ...libpq.env, PGDATABASE: options.database };
82
+ for (const name of ["PATH", "HOME", "LANG"]) {
83
+ const value = options.env[name];
84
+ if (value !== undefined) env[name] = value;
85
+ }
86
+ const child = spawn(options.pgRestore, ["--exit-on-error", "--no-password", ...options.flags, "--dbname", options.database], {
87
+ env,
88
+ stdio: ["pipe", "ignore", "pipe"],
89
+ });
90
+ let stderr = "";
91
+ child.stderr.on("data", (chunk: Buffer) => {
92
+ stderr = (stderr + chunk.toString("utf8")).slice(-MAX_STDERR);
93
+ });
94
+ const exited = new Promise<number | null>((resolve, reject) => {
95
+ child.on("error", reject);
96
+ child.on("close", (code) => {
97
+ resolve(code);
98
+ });
99
+ });
100
+ // A pg_restore that stops early closes its stdin; the pipe error is then the exit status's to report.
101
+ const fed = pipeStreamTo(options.dump, child.stdin).then(
102
+ () => null,
103
+ (error: unknown) => error,
104
+ );
105
+ let code: number | null;
106
+ try {
107
+ code = await exited;
108
+ } catch (error) {
109
+ return { ok: false, problem: `cannot run ${options.pgRestore} (${error instanceof Error ? error.message : String(error)})` };
110
+ }
111
+ const feedError = await fed;
112
+ if (code !== 0) return { ok: false, problem: `pg_restore exited with ${String(code)}: ${stderr.trim().split("\n").slice(-3).join(" / ")}` };
113
+ if (feedError !== null) return { ok: false, problem: `the backup stream failed (${feedError instanceof Error ? feedError.message : "unknown error"})` };
114
+ return { ok: true };
115
+ }
@@ -0,0 +1,11 @@
1
+ // The run's row in `backup.runs`, written over a connection: the same SQL deploy.sh pipes into psql on a VPS
2
+ // (`buildStatusSql`), sent as one simple query so the insert and the prune share its transaction.
3
+ import { withPgClient } from "@softure-ai/deploy";
4
+ import { buildStatusSql } from "./status-sql.js";
5
+ import type { RunRecord } from "./upload.js";
6
+
7
+ export async function recordRun(url: string, record: RunRecord): Promise<void> {
8
+ await withPgClient(url, async (client) => {
9
+ await client.query(buildStatusSql(record));
10
+ });
11
+ }