@softure-ai/backup 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/LICENSE +21 -0
- package/README.md +223 -2
- package/dist/cli/main.d.ts +3 -0
- package/dist/cli/main.d.ts.map +1 -0
- package/dist/cli/main.js +16 -0
- package/dist/cli/main.js.map +1 -0
- package/dist/cli/run.d.ts +12 -0
- package/dist/cli/run.d.ts.map +1 -0
- package/dist/cli/run.js +335 -0
- package/dist/cli/run.js.map +1 -0
- package/dist/contract.d.ts +41 -0
- package/dist/contract.d.ts.map +1 -0
- package/dist/contract.js +14 -0
- package/dist/contract.js.map +1 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +30 -0
- package/dist/index.js.map +1 -0
- package/dist/messages/en.d.ts +16 -0
- package/dist/messages/en.d.ts.map +1 -0
- package/dist/messages/en.js +16 -0
- package/dist/messages/en.js.map +1 -0
- package/dist/messages/index.d.ts +21 -0
- package/dist/messages/index.d.ts.map +1 -0
- package/dist/messages/index.js +4 -0
- package/dist/messages/index.js.map +1 -0
- package/dist/offsite/config.d.ts +43 -0
- package/dist/offsite/config.d.ts.map +1 -0
- package/dist/offsite/config.js +130 -0
- package/dist/offsite/config.js.map +1 -0
- package/dist/offsite/crypto.d.ts +22 -0
- package/dist/offsite/crypto.d.ts.map +1 -0
- package/dist/offsite/crypto.js +48 -0
- package/dist/offsite/crypto.js.map +1 -0
- package/dist/offsite/index.d.ts +11 -0
- package/dist/offsite/index.d.ts.map +1 -0
- package/dist/offsite/index.js +13 -0
- package/dist/offsite/index.js.map +1 -0
- package/dist/offsite/lifecycle.d.ts +17 -0
- package/dist/offsite/lifecycle.d.ts.map +1 -0
- package/dist/offsite/lifecycle.js +45 -0
- package/dist/offsite/lifecycle.js.map +1 -0
- package/dist/offsite/objects.d.ts +19 -0
- package/dist/offsite/objects.d.ts.map +1 -0
- package/dist/offsite/objects.js +44 -0
- package/dist/offsite/objects.js.map +1 -0
- package/dist/offsite/postgres.d.ts +32 -0
- package/dist/offsite/postgres.d.ts.map +1 -0
- package/dist/offsite/postgres.js +97 -0
- package/dist/offsite/postgres.js.map +1 -0
- package/dist/offsite/restore.d.ts +54 -0
- package/dist/offsite/restore.d.ts.map +1 -0
- package/dist/offsite/restore.js +67 -0
- package/dist/offsite/restore.js.map +1 -0
- package/dist/offsite/s3.d.ts +34 -0
- package/dist/offsite/s3.d.ts.map +1 -0
- package/dist/offsite/s3.js +119 -0
- package/dist/offsite/s3.js.map +1 -0
- package/dist/offsite/sigv4.d.ts +19 -0
- package/dist/offsite/sigv4.d.ts.map +1 -0
- package/dist/offsite/sigv4.js +69 -0
- package/dist/offsite/sigv4.js.map +1 -0
- package/dist/offsite/status-sql.d.ts +10 -0
- package/dist/offsite/status-sql.d.ts.map +1 -0
- package/dist/offsite/status-sql.js +37 -0
- package/dist/offsite/status-sql.js.map +1 -0
- package/dist/offsite/upload.d.ts +33 -0
- package/dist/offsite/upload.d.ts.map +1 -0
- package/dist/offsite/upload.js +98 -0
- package/dist/offsite/upload.js.map +1 -0
- package/dist/offsite/verify.d.ts +32 -0
- package/dist/offsite/verify.d.ts.map +1 -0
- package/dist/offsite/verify.js +75 -0
- package/dist/offsite/verify.js.map +1 -0
- package/dist/schema.d.ts +150 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/schema.js +15 -0
- package/dist/schema.js.map +1 -0
- package/dist/server/index.d.ts +28 -0
- package/dist/server/index.d.ts.map +1 -0
- package/dist/server/index.js +98 -0
- package/dist/server/index.js.map +1 -0
- package/examples/backup-verify.yml +84 -0
- package/migrations/0001_create_runs.sql +21 -0
- package/module.json +12 -0
- package/package.json +67 -4
- package/src/cli/main.ts +16 -0
- package/src/cli/run.ts +338 -0
- package/src/contract.ts +46 -0
- package/src/index.ts +41 -0
- package/src/messages/en.ts +15 -0
- package/src/messages/index.ts +6 -0
- package/src/offsite/config.ts +157 -0
- package/src/offsite/crypto.ts +53 -0
- package/src/offsite/index.ts +31 -0
- package/src/offsite/lifecycle.ts +47 -0
- package/src/offsite/objects.ts +54 -0
- package/src/offsite/postgres.ts +115 -0
- package/src/offsite/restore.ts +108 -0
- package/src/offsite/s3.ts +144 -0
- package/src/offsite/sigv4.ts +84 -0
- package/src/offsite/status-sql.ts +45 -0
- package/src/offsite/upload.ts +116 -0
- package/src/offsite/verify.ts +97 -0
- package/src/schema.ts +17 -0
- package/src/server/index.ts +111 -0
package/src/contract.ts
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// Result types and codes of the backup module. No user-facing copy here: an app's settings screen translates the
|
|
2
|
+
// result and error codes through `messages` (docs/02-module-standard.md §6).
|
|
3
|
+
|
|
4
|
+
/** How a run ended: uploaded, failed, or skipped because the host has no off-site configuration. */
|
|
5
|
+
export const BACKUP_RESULTS = ["ok", "failed", "not_configured"] as const;
|
|
6
|
+
export type BackupResult = (typeof BACKUP_RESULTS)[number];
|
|
7
|
+
|
|
8
|
+
/** Safe error codes of a failed run; the message beside them never holds a secret. */
|
|
9
|
+
export const BACKUP_ERROR_CODES = [
|
|
10
|
+
"config_invalid",
|
|
11
|
+
"file_missing",
|
|
12
|
+
"encrypt_failed",
|
|
13
|
+
"upload_denied",
|
|
14
|
+
"upload_failed",
|
|
15
|
+
"network_error",
|
|
16
|
+
] as const;
|
|
17
|
+
export type BackupErrorCode = (typeof BACKUP_ERROR_CODES)[number];
|
|
18
|
+
|
|
19
|
+
/** What a backup file holds: the database dump or an archive of the data volume's folders. */
|
|
20
|
+
export type BackupKind = "database" | "data";
|
|
21
|
+
|
|
22
|
+
/** One encrypted object a run uploaded. */
|
|
23
|
+
export interface BackupObject {
|
|
24
|
+
readonly kind: BackupKind;
|
|
25
|
+
/** The object key in the bucket, `<prefix>/<file name>.age`. */
|
|
26
|
+
readonly key: string;
|
|
27
|
+
/** Bytes of the encrypted object. */
|
|
28
|
+
readonly bytes: number;
|
|
29
|
+
/** sha256 of the encrypted object, hex. */
|
|
30
|
+
readonly sha256: string;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** One recorded run of the off-site copy. */
|
|
34
|
+
export interface BackupRun {
|
|
35
|
+
readonly startedAt: Date;
|
|
36
|
+
readonly finishedAt: Date;
|
|
37
|
+
readonly result: BackupResult;
|
|
38
|
+
readonly objects: readonly BackupObject[];
|
|
39
|
+
readonly sizeBytes: number;
|
|
40
|
+
readonly error: { readonly code: string; readonly message: string } | null;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** What a settings screen shows: the newest run, the newest successful one, or nothing recorded yet. */
|
|
44
|
+
export type BackupStatus =
|
|
45
|
+
| { readonly result: "not_configured"; readonly lastRun: null; readonly lastSuccess: null }
|
|
46
|
+
| { readonly result: BackupResult; readonly lastRun: BackupRun; readonly lastSuccess: BackupRun | null };
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// Public API of @softure-ai/backup: the module factory for softure.config.ts (its migration creates backup.runs,
|
|
2
|
+
// the status the host run writes), the contract types and messages. The app reads the status through
|
|
3
|
+
// `@softure-ai/backup/server`; the host side (upload, restore, verify) is `@softure-ai/backup/offsite` and the
|
|
4
|
+
// `softure-backup` command.
|
|
5
|
+
import { defineModule, resolveMigrationsDir } from "@softure-ai/core";
|
|
6
|
+
import { backupMessages } from "./messages/index.js";
|
|
7
|
+
import { checkRunsTable } from "./server/index.js";
|
|
8
|
+
|
|
9
|
+
export const MODULE_ID = "backup";
|
|
10
|
+
|
|
11
|
+
/** Adds the status table of the off-site backup: `modules: [backup()]` in `softure.config.ts`. */
|
|
12
|
+
export const backup = defineModule({
|
|
13
|
+
manifest: {
|
|
14
|
+
id: MODULE_ID,
|
|
15
|
+
version: "0.1.0",
|
|
16
|
+
dependsOn: {},
|
|
17
|
+
dbSchema: "backup",
|
|
18
|
+
tables: ["runs"],
|
|
19
|
+
env: [],
|
|
20
|
+
switches: [],
|
|
21
|
+
routes: {},
|
|
22
|
+
mount: [],
|
|
23
|
+
privacy: { exports: false, deletes: false },
|
|
24
|
+
},
|
|
25
|
+
messages: backupMessages,
|
|
26
|
+
migrations: { dir: resolveMigrationsDir(import.meta.url, "../migrations/") },
|
|
27
|
+
health: checkRunsTable,
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
export {
|
|
31
|
+
BACKUP_ERROR_CODES,
|
|
32
|
+
BACKUP_RESULTS,
|
|
33
|
+
type BackupErrorCode,
|
|
34
|
+
type BackupKind,
|
|
35
|
+
type BackupObject,
|
|
36
|
+
type BackupResult,
|
|
37
|
+
type BackupRun,
|
|
38
|
+
type BackupStatus,
|
|
39
|
+
} from "./contract.js";
|
|
40
|
+
export { backupMessages, type BackupMessages } from "./messages/index.js";
|
|
41
|
+
export { backupSchema, runs } from "./schema.js";
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export const en = {
|
|
2
|
+
result: {
|
|
3
|
+
ok: "Off-site backup succeeded",
|
|
4
|
+
failed: "Off-site backup failed",
|
|
5
|
+
not_configured: "Off-site backup is not configured",
|
|
6
|
+
},
|
|
7
|
+
errors: {
|
|
8
|
+
config_invalid: "The off-site backup settings are incomplete or invalid.",
|
|
9
|
+
file_missing: "The local backup file was not found.",
|
|
10
|
+
encrypt_failed: "The backup could not be encrypted.",
|
|
11
|
+
upload_denied: "The storage refused the upload; check the token's permissions.",
|
|
12
|
+
upload_failed: "The upload to the storage failed.",
|
|
13
|
+
network_error: "The storage could not be reached.",
|
|
14
|
+
},
|
|
15
|
+
};
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
// The off-site settings, read only from the environment and validated with zod. A host that sets none of the names
|
|
2
|
+
// is "not configured" (the step is skipped, never failed); one that sets some gets a list of what is missing or
|
|
3
|
+
// wrong, never a value. Two tokens: the nightly upload needs only a write token, restore and verify a read one.
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
|
|
6
|
+
/** The names the nightly upload reads (`.env.prod` on the server). */
|
|
7
|
+
export const UPLOAD_ENV_NAMES = [
|
|
8
|
+
"BACKUP_S3_ENDPOINT",
|
|
9
|
+
"BACKUP_S3_REGION",
|
|
10
|
+
"BACKUP_S3_BUCKET",
|
|
11
|
+
"BACKUP_S3_PREFIX",
|
|
12
|
+
"BACKUP_S3_ACCESS_KEY_ID",
|
|
13
|
+
"BACKUP_S3_SECRET_ACCESS_KEY",
|
|
14
|
+
"BACKUP_AGE_RECIPIENT",
|
|
15
|
+
] as const;
|
|
16
|
+
|
|
17
|
+
/** The names restore, fetch and verify read (the operator's machine or a CI job, never the server). */
|
|
18
|
+
export const READ_ENV_NAMES = [
|
|
19
|
+
"BACKUP_S3_ENDPOINT",
|
|
20
|
+
"BACKUP_S3_REGION",
|
|
21
|
+
"BACKUP_S3_BUCKET",
|
|
22
|
+
"BACKUP_S3_PREFIX",
|
|
23
|
+
"BACKUP_S3_READ_ACCESS_KEY_ID",
|
|
24
|
+
"BACKUP_S3_READ_SECRET_ACCESS_KEY",
|
|
25
|
+
] as const;
|
|
26
|
+
|
|
27
|
+
/** The secret names: their values are redacted from every message. */
|
|
28
|
+
export const SECRET_ENV_NAMES = [
|
|
29
|
+
"BACKUP_S3_ACCESS_KEY_ID",
|
|
30
|
+
"BACKUP_S3_SECRET_ACCESS_KEY",
|
|
31
|
+
"BACKUP_S3_READ_ACCESS_KEY_ID",
|
|
32
|
+
"BACKUP_S3_READ_SECRET_ACCESS_KEY",
|
|
33
|
+
] as const;
|
|
34
|
+
|
|
35
|
+
export const DEFAULT_REGION = "auto";
|
|
36
|
+
/** A prefix is path segments of lower-case letters, digits, `.`, `_` and `-`, so a key can never climb out of it. */
|
|
37
|
+
const PREFIX_PATTERN = /^[a-z0-9][a-z0-9._-]*(\/[a-z0-9][a-z0-9._-]*)*$/;
|
|
38
|
+
/** S3 bucket names: 3 to 63 lower-case letters, digits, `.` and `-`. */
|
|
39
|
+
const BUCKET_PATTERN = /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/;
|
|
40
|
+
const REGION_PATTERN = /^[a-z0-9-]{1,32}$/;
|
|
41
|
+
/** An age X25519 recipient (`age1` and 58 bech32 characters) or a hybrid post-quantum one (`age1pq1…`). */
|
|
42
|
+
const RECIPIENT_PATTERN = /^age1(pq1)?[02-9ac-hj-np-z]{40,}$/;
|
|
43
|
+
|
|
44
|
+
/** Where the objects go: the same for the write and the read side. */
|
|
45
|
+
export interface StoreConfig {
|
|
46
|
+
readonly endpoint: URL;
|
|
47
|
+
readonly region: string;
|
|
48
|
+
readonly bucket: string;
|
|
49
|
+
readonly prefix: string;
|
|
50
|
+
readonly accessKeyId: string;
|
|
51
|
+
readonly secretAccessKey: string;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export interface UploadConfig {
|
|
55
|
+
readonly store: StoreConfig;
|
|
56
|
+
readonly recipient: string;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export type ConfigResult<T> =
|
|
60
|
+
| { readonly status: "ok"; readonly config: T }
|
|
61
|
+
| { readonly status: "not_configured" }
|
|
62
|
+
| { readonly status: "invalid"; readonly problems: readonly string[] };
|
|
63
|
+
|
|
64
|
+
type Env = Readonly<Record<string, string | undefined>>;
|
|
65
|
+
|
|
66
|
+
const endpointSchema = z
|
|
67
|
+
.string()
|
|
68
|
+
.refine((value) => URL.canParse(value), "is not a URL")
|
|
69
|
+
.transform((value) => new URL(value))
|
|
70
|
+
.refine((url) => url.protocol === "https:" || url.protocol === "http:", "must start with https:// (or http:// for a local MinIO)")
|
|
71
|
+
.refine((url) => url.username === "" && url.password === "", "must not carry a user or password")
|
|
72
|
+
.refine((url) => url.search === "" && url.hash === "", "must not carry a query or fragment");
|
|
73
|
+
|
|
74
|
+
function storeSchema(keyName: string, secretName: string) {
|
|
75
|
+
return z.object({
|
|
76
|
+
BACKUP_S3_ENDPOINT: endpointSchema,
|
|
77
|
+
BACKUP_S3_REGION: z.string().regex(REGION_PATTERN, "is not a region name").default(DEFAULT_REGION),
|
|
78
|
+
BACKUP_S3_BUCKET: z.string().regex(BUCKET_PATTERN, "is not an S3 bucket name"),
|
|
79
|
+
BACKUP_S3_PREFIX: z.string().regex(PREFIX_PATTERN, "is not a key prefix (lower-case path segments)").optional(),
|
|
80
|
+
[keyName]: z.string().min(1, "is empty").max(256, "is too long"),
|
|
81
|
+
[secretName]: z.string().min(1, "is empty").max(256, "is too long"),
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Unset and empty values count as unset; values are trimmed of surrounding blanks a secret store may add. */
|
|
86
|
+
function readNames(env: Env, names: readonly string[]): Record<string, string> {
|
|
87
|
+
const values: Record<string, string> = {};
|
|
88
|
+
for (const name of names) {
|
|
89
|
+
const value = env[name]?.trim();
|
|
90
|
+
if (value !== undefined && value !== "") values[name] = value;
|
|
91
|
+
}
|
|
92
|
+
return values;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function describeIssues(error: z.ZodError): string[] {
|
|
96
|
+
return error.issues.map((issue) => {
|
|
97
|
+
const name = String(issue.path[0] ?? "configuration");
|
|
98
|
+
return issue.code === "invalid_type" ? `${name} is not set` : `${name} ${issue.message}`;
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function toStore(values: Record<string, unknown>, keyName: string, secretName: string, defaultPrefix: string): StoreConfig {
|
|
103
|
+
return {
|
|
104
|
+
endpoint: values.BACKUP_S3_ENDPOINT as URL,
|
|
105
|
+
region: values.BACKUP_S3_REGION as string,
|
|
106
|
+
bucket: values.BACKUP_S3_BUCKET as string,
|
|
107
|
+
prefix: (values.BACKUP_S3_PREFIX as string | undefined) ?? defaultPrefix,
|
|
108
|
+
accessKeyId: values[keyName] as string,
|
|
109
|
+
secretAccessKey: values[secretName] as string,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function checkPrefix(prefix: string): string | null {
|
|
114
|
+
return PREFIX_PATTERN.test(prefix) ? null : `the app name ${JSON.stringify(prefix)} is not a key prefix; set BACKUP_S3_PREFIX`;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* The upload settings of the nightly run. `appName` is the default prefix (each app its own folder of a shared
|
|
119
|
+
* bucket); `BACKUP_S3_PREFIX` overrides it. Without `BACKUP_S3_BUCKET` and every other name, not configured.
|
|
120
|
+
*/
|
|
121
|
+
export function readUploadConfig(env: Env, appName: string): ConfigResult<UploadConfig> {
|
|
122
|
+
const values = readNames(env, UPLOAD_ENV_NAMES);
|
|
123
|
+
if (Object.keys(values).length === 0) return { status: "not_configured" };
|
|
124
|
+
const schema = storeSchema("BACKUP_S3_ACCESS_KEY_ID", "BACKUP_S3_SECRET_ACCESS_KEY").extend({
|
|
125
|
+
BACKUP_AGE_RECIPIENT: z.string().regex(RECIPIENT_PATTERN, "is not an age public key (age1…)"),
|
|
126
|
+
});
|
|
127
|
+
const parsed = schema.safeParse(values);
|
|
128
|
+
if (!parsed.success) return { status: "invalid", problems: describeIssues(parsed.error) };
|
|
129
|
+
const store = toStore(parsed.data, "BACKUP_S3_ACCESS_KEY_ID", "BACKUP_S3_SECRET_ACCESS_KEY", appName);
|
|
130
|
+
const prefixProblem = checkPrefix(store.prefix);
|
|
131
|
+
if (prefixProblem !== null) return { status: "invalid", problems: [prefixProblem] };
|
|
132
|
+
return { status: "ok", config: { store, recipient: values.BACKUP_AGE_RECIPIENT ?? "" } };
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** The read settings of restore, fetch and verify: the same store with the read token. */
|
|
136
|
+
export function readReadConfig(env: Env, appName: string): ConfigResult<StoreConfig> {
|
|
137
|
+
const values = readNames(env, READ_ENV_NAMES);
|
|
138
|
+
if (Object.keys(values).length === 0) return { status: "not_configured" };
|
|
139
|
+
const parsed = storeSchema("BACKUP_S3_READ_ACCESS_KEY_ID", "BACKUP_S3_READ_SECRET_ACCESS_KEY").safeParse(values);
|
|
140
|
+
if (!parsed.success) return { status: "invalid", problems: describeIssues(parsed.error) };
|
|
141
|
+
const store = toStore(parsed.data, "BACKUP_S3_READ_ACCESS_KEY_ID", "BACKUP_S3_READ_SECRET_ACCESS_KEY", appName);
|
|
142
|
+
const prefixProblem = checkPrefix(store.prefix);
|
|
143
|
+
if (prefixProblem !== null) return { status: "invalid", problems: [prefixProblem] };
|
|
144
|
+
return { status: "ok", config: store };
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** The non-empty values of the secret names, for `redactSecrets`. */
|
|
148
|
+
export function collectSecrets(env: Env): string[] {
|
|
149
|
+
return SECRET_ENV_NAMES.map((name) => env[name]?.trim() ?? "").filter((value) => value.length >= 4);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** `text` with every secret value replaced by `***`. */
|
|
153
|
+
export function redactSecrets(text: string, secrets: readonly string[]): string {
|
|
154
|
+
let redacted = text;
|
|
155
|
+
for (const secret of secrets) redacted = redacted.split(secret).join("***");
|
|
156
|
+
return redacted;
|
|
157
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// age encryption with the pure-JS implementation by the age author (interoperable with the `age` command): the
|
|
2
|
+
// server holds only the public key (`age1…`); the private key (`AGE-SECRET-KEY-1…`) is read at run time from a file
|
|
3
|
+
// or stdin, never from an argument.
|
|
4
|
+
import { createReadStream, createWriteStream } from "node:fs";
|
|
5
|
+
import { Readable, Writable } from "node:stream";
|
|
6
|
+
import { pipeline } from "node:stream/promises";
|
|
7
|
+
import { Decrypter, Encrypter } from "age-encryption";
|
|
8
|
+
|
|
9
|
+
const IDENTITY_PATTERN = /^AGE-SECRET-KEY-(PQ-)?1[0-9A-Z]+$/;
|
|
10
|
+
|
|
11
|
+
/** Encrypts the file at `source` to `recipient` into `target`. */
|
|
12
|
+
export async function encryptFile(source: string, target: string, recipient: string): Promise<void> {
|
|
13
|
+
const encrypter = new Encrypter();
|
|
14
|
+
encrypter.addRecipient(recipient);
|
|
15
|
+
const input = Readable.toWeb(createReadStream(source)) as ReadableStream<Uint8Array>;
|
|
16
|
+
const encrypted = await encrypter.encrypt(input);
|
|
17
|
+
await pipeline(Readable.fromWeb(encrypted as import("node:stream/web").ReadableStream<Uint8Array>), createWriteStream(target, { mode: 0o600 }));
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** A stream of the plaintext of an age `stream`, decrypted with `identities`. */
|
|
21
|
+
export async function decryptStream(stream: ReadableStream<Uint8Array>, identities: readonly string[]): Promise<ReadableStream<Uint8Array>> {
|
|
22
|
+
const decrypter = new Decrypter();
|
|
23
|
+
for (const identity of identities) decrypter.addIdentity(identity);
|
|
24
|
+
return decrypter.decrypt(stream);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Writes a web stream to a file, owner read and write only. */
|
|
28
|
+
export async function writeStreamToFile(stream: ReadableStream<Uint8Array>, path: string): Promise<void> {
|
|
29
|
+
await pipeline(Readable.fromWeb(stream as import("node:stream/web").ReadableStream<Uint8Array>), createWriteStream(path, { mode: 0o600 }));
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Pipes a web stream into a Node writable (a child's stdin). */
|
|
33
|
+
export async function pipeStreamTo(stream: ReadableStream<Uint8Array>, target: Writable): Promise<void> {
|
|
34
|
+
await pipeline(Readable.fromWeb(stream as import("node:stream/web").ReadableStream<Uint8Array>), target);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export type IdentitiesResult = { ok: true; identities: string[] } | { ok: false; problem: string };
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* The identities of an age key file's text (the `age-keygen` output: comment lines and one or more keys). The
|
|
41
|
+
* problem never repeats the text.
|
|
42
|
+
*/
|
|
43
|
+
export function parseIdentities(text: string): IdentitiesResult {
|
|
44
|
+
const identities = text
|
|
45
|
+
.split(/\r?\n/)
|
|
46
|
+
.map((line) => line.trim())
|
|
47
|
+
.filter((line) => line !== "" && !line.startsWith("#"));
|
|
48
|
+
if (identities.length === 0) return { ok: false, problem: "the identity holds no AGE-SECRET-KEY-1… line" };
|
|
49
|
+
if (!identities.every((line) => IDENTITY_PATTERN.test(line))) {
|
|
50
|
+
return { ok: false, problem: "the identity has a line that is not an age secret key (AGE-SECRET-KEY-1…)" };
|
|
51
|
+
}
|
|
52
|
+
return { ok: true, identities };
|
|
53
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// The host and operator side of @softure-ai/backup: configuration, the nightly upload, status SQL, restore, fetch and
|
|
2
|
+
// verify. The `softure-backup` command line wraps these; an app's own code needs only `@softure-ai/backup/server`.
|
|
3
|
+
export {
|
|
4
|
+
collectSecrets,
|
|
5
|
+
DEFAULT_REGION,
|
|
6
|
+
READ_ENV_NAMES,
|
|
7
|
+
readReadConfig,
|
|
8
|
+
readUploadConfig,
|
|
9
|
+
redactSecrets,
|
|
10
|
+
SECRET_ENV_NAMES,
|
|
11
|
+
UPLOAD_ENV_NAMES,
|
|
12
|
+
type ConfigResult,
|
|
13
|
+
type StoreConfig,
|
|
14
|
+
type UploadConfig,
|
|
15
|
+
} from "./config.js";
|
|
16
|
+
export { decryptStream, encryptFile, parseIdentities, type IdentitiesResult } from "./crypto.js";
|
|
17
|
+
export {
|
|
18
|
+
buildLifecycleJson,
|
|
19
|
+
buildLifecycleXml,
|
|
20
|
+
DEFAULT_LIFECYCLE_DAYS,
|
|
21
|
+
MAX_LIFECYCLE_DAYS,
|
|
22
|
+
MIN_LIFECYCLE_DAYS,
|
|
23
|
+
parseLifecycleDays,
|
|
24
|
+
} from "./lifecycle.js";
|
|
25
|
+
export { buildObjectKey, OBJECT_SUFFIX, parseBackupFileName, parseObjectKey, selectNewestKey, type BackupFileName } from "./objects.js";
|
|
26
|
+
export { fetchBackup, resolveKey, restoreBackup, type FetchOptions, type OffsiteResult, type RestoreOptions, type RestoreOutcome } from "./restore.js";
|
|
27
|
+
export { getObjectStream, getObjectUrl, listObjects, putObjectFile, S3RequestError, type S3Options, type StoredObject } from "./s3.js";
|
|
28
|
+
export { EMPTY_SHA256, encodeRfc3986, formatAmzDate, signRequest, type SignInput } from "./sigv4.js";
|
|
29
|
+
export { buildStatusSql, getPruneBefore, quoteLiteral, STATUS_RETENTION_DAYS } from "./status-sql.js";
|
|
30
|
+
export { buildNotConfiguredRecord, hashFile, uploadBackups, type RunRecord, type UploadOptions } from "./upload.js";
|
|
31
|
+
export { DEFAULT_MAX_AGE_HOURS, selectVerifyTables, verifyBackup, type VerifyOptions, type VerifyOutcome } from "./verify.js";
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// Remote retention is the bucket's: a lifecycle rule on the prefix expires objects after N days. The nightly run
|
|
2
|
+
// never deletes (its token cannot), so a compromised server cannot wipe the history either.
|
|
3
|
+
export const DEFAULT_LIFECYCLE_DAYS = 30;
|
|
4
|
+
export const MIN_LIFECYCLE_DAYS = 1;
|
|
5
|
+
export const MAX_LIFECYCLE_DAYS = 3650;
|
|
6
|
+
|
|
7
|
+
export type LifecycleDaysResult = { ok: true; days: number } | { ok: false; problem: string };
|
|
8
|
+
|
|
9
|
+
/** Whole days from 1 to 3650 (ten years). */
|
|
10
|
+
export function parseLifecycleDays(text: string | undefined): LifecycleDaysResult {
|
|
11
|
+
if (text === undefined) return { ok: true, days: DEFAULT_LIFECYCLE_DAYS };
|
|
12
|
+
if (!/^\d{1,4}$/.test(text)) return { ok: false, problem: `--days is a whole number of days, not ${JSON.stringify(text)}` };
|
|
13
|
+
const days = Number(text);
|
|
14
|
+
if (days < MIN_LIFECYCLE_DAYS || days > MAX_LIFECYCLE_DAYS) {
|
|
15
|
+
return { ok: false, problem: `--days is from ${String(MIN_LIFECYCLE_DAYS)} to ${String(MAX_LIFECYCLE_DAYS)}, not ${String(days)}` };
|
|
16
|
+
}
|
|
17
|
+
return { ok: true, days };
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** The S3 `LifecycleConfiguration` (XML, for PutBucketLifecycleConfiguration) expiring `<prefix>/` after `days`. */
|
|
21
|
+
export function buildLifecycleXml(prefix: string, days: number): string {
|
|
22
|
+
return [
|
|
23
|
+
'<?xml version="1.0" encoding="UTF-8"?>',
|
|
24
|
+
'<LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">',
|
|
25
|
+
" <Rule>",
|
|
26
|
+
` <ID>softure-backup-${prefix.replace(/[^a-z0-9-]/g, "-")}</ID>`,
|
|
27
|
+
" <Status>Enabled</Status>",
|
|
28
|
+
` <Filter><Prefix>${prefix}/</Prefix></Filter>`,
|
|
29
|
+
` <Expiration><Days>${String(days)}</Days></Expiration>`,
|
|
30
|
+
" <AbortIncompleteMultipartUpload><DaysAfterInitiation>1</DaysAfterInitiation></AbortIncompleteMultipartUpload>",
|
|
31
|
+
" </Rule>",
|
|
32
|
+
"</LifecycleConfiguration>",
|
|
33
|
+
"",
|
|
34
|
+
].join("\n");
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** The same rule as JSON for `aws s3api put-bucket-lifecycle-configuration --lifecycle-configuration file://…`. */
|
|
38
|
+
export function buildLifecycleJson(prefix: string, days: number): string {
|
|
39
|
+
const rule = {
|
|
40
|
+
ID: `softure-backup-${prefix.replace(/[^a-z0-9-]/g, "-")}`,
|
|
41
|
+
Status: "Enabled",
|
|
42
|
+
Filter: { Prefix: `${prefix}/` },
|
|
43
|
+
Expiration: { Days: days },
|
|
44
|
+
AbortIncompleteMultipartUpload: { DaysAfterInitiation: 1 },
|
|
45
|
+
};
|
|
46
|
+
return `${JSON.stringify({ Rules: [rule] }, null, 2)}\n`;
|
|
47
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// Object names: `<prefix>/<the local file name>.age`. Only the deploy's own backup names are accepted, so a key
|
|
2
|
+
// sorts by time and nothing else can be uploaded by mistake. Retention is the bucket's lifecycle rule on the prefix;
|
|
3
|
+
// this package never deletes an object.
|
|
4
|
+
import type { BackupKind } from "../contract.js";
|
|
5
|
+
|
|
6
|
+
/** `db-20261109T031700Z.dump` (`@softure-ai/deploy` backup, any short prefix). */
|
|
7
|
+
const DATABASE_NAME = /^([a-z0-9][a-z0-9_-]*)-(\d{8})T(\d{6})Z\.dump$/;
|
|
8
|
+
/** `data-20261109-031700.tar.gz` (deploy.sh backup-data). */
|
|
9
|
+
const DATA_NAME = /^data-(\d{8})-(\d{6})\.tar\.gz$/;
|
|
10
|
+
export const OBJECT_SUFFIX = ".age";
|
|
11
|
+
|
|
12
|
+
export interface BackupFileName {
|
|
13
|
+
readonly kind: BackupKind;
|
|
14
|
+
readonly name: string;
|
|
15
|
+
/** The time in the name, UTC. */
|
|
16
|
+
readonly takenAt: Date;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function toDate(day: string, time: string): Date {
|
|
20
|
+
return new Date(`${day.slice(0, 4)}-${day.slice(4, 6)}-${day.slice(6, 8)}T${time.slice(0, 2)}:${time.slice(2, 4)}:${time.slice(4, 6)}Z`);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** The kind and time of a backup file name, or null for any other name (or an impossible date). */
|
|
24
|
+
export function parseBackupFileName(name: string): BackupFileName | null {
|
|
25
|
+
const data = DATA_NAME.exec(name);
|
|
26
|
+
const database = data === null ? DATABASE_NAME.exec(name) : null;
|
|
27
|
+
if (data === null && database === null) return null;
|
|
28
|
+
const takenAt = data === null ? toDate(database?.[2] ?? "", database?.[3] ?? "") : toDate(data[1] ?? "", data[2] ?? "");
|
|
29
|
+
if (Number.isNaN(takenAt.getTime())) return null;
|
|
30
|
+
return { kind: data === null ? "database" : "data", name, takenAt };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function buildObjectKey(prefix: string, fileName: string): string {
|
|
34
|
+
return `${prefix}/${fileName}${OBJECT_SUFFIX}`;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** The backup file name of a key under `prefix`, or null for any other key. */
|
|
38
|
+
export function parseObjectKey(prefix: string, key: string): BackupFileName | null {
|
|
39
|
+
if (!key.startsWith(`${prefix}/`) || !key.endsWith(OBJECT_SUFFIX)) return null;
|
|
40
|
+
const name = key.slice(prefix.length + 1, -OBJECT_SUFFIX.length);
|
|
41
|
+
if (name.includes("/")) return null;
|
|
42
|
+
return parseBackupFileName(name);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** The newest key of `kind` under `prefix`, by the time in its name; null when there is none. */
|
|
46
|
+
export function selectNewestKey(prefix: string, keys: readonly string[], kind: BackupKind): { key: string; takenAt: Date } | null {
|
|
47
|
+
let newest: { key: string; takenAt: Date } | null = null;
|
|
48
|
+
for (const key of keys) {
|
|
49
|
+
const parsed = parseObjectKey(prefix, key);
|
|
50
|
+
if (parsed?.kind !== kind) continue;
|
|
51
|
+
if (newest === null || parsed.takenAt.getTime() > newest.takenAt.getTime()) newest = { key, takenAt: parsed.takenAt };
|
|
52
|
+
}
|
|
53
|
+
return newest;
|
|
54
|
+
}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
// The Postgres side of restore and verify: an admin connection from a URL in the environment, databases created and
|
|
2
|
+
// dropped through it, and pg_restore fed from a decrypted stream with the URL as libpq variables, so the password
|
|
3
|
+
// never shows in the process list.
|
|
4
|
+
import { spawn } from "node:child_process";
|
|
5
|
+
import { randomBytes } from "node:crypto";
|
|
6
|
+
import { toLibpqEnv, withPgClient } from "@softure-ai/deploy";
|
|
7
|
+
import { pipeStreamTo } from "./crypto.js";
|
|
8
|
+
|
|
9
|
+
/** Database names this package creates or accepts: lower-case identifiers within Postgres's 63 bytes. */
|
|
10
|
+
const DATABASE_NAME = /^[a-z_][a-z0-9_]{0,62}$/;
|
|
11
|
+
const MAX_STDERR = 4000;
|
|
12
|
+
|
|
13
|
+
export function isDatabaseName(name: string): boolean {
|
|
14
|
+
return DATABASE_NAME.test(name);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/** The URL with another database. */
|
|
18
|
+
export function withDatabase(url: string, database: string): string {
|
|
19
|
+
const parsed = new URL(url);
|
|
20
|
+
parsed.pathname = `/${database}`;
|
|
21
|
+
return parsed.toString();
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** The database of a URL (`postgres` when it names none). */
|
|
25
|
+
export function getUrlDatabase(url: string): string {
|
|
26
|
+
const name = decodeURIComponent(new URL(url).pathname.replace(/^\//, ""));
|
|
27
|
+
return name === "" ? "postgres" : name;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** `softure_offsite_verify_<8 hex>`: a fresh throwaway name. */
|
|
31
|
+
export function createVerifyDatabaseName(): string {
|
|
32
|
+
return `softure_offsite_verify_${randomBytes(4).toString("hex")}`;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** `<base>_restore_<YYYYMMDDHHMMSS>`, shortened so it stays a valid name. */
|
|
36
|
+
export function createRestoreDatabaseName(base: string, now: Date): string {
|
|
37
|
+
const stamp = now.toISOString().replace(/[-:T]/g, "").slice(0, 14);
|
|
38
|
+
const safeBase = base.replace(/[^a-z0-9_]/g, "_").replace(/^[^a-z_]/, "_").slice(0, 63 - "_restore_".length - stamp.length);
|
|
39
|
+
return `${safeBase}_restore_${stamp}`;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export async function databaseExists(adminUrl: string, name: string): Promise<boolean> {
|
|
43
|
+
return withPgClient(adminUrl, async (client) => {
|
|
44
|
+
const result = await client.query<{ present: boolean }>("SELECT EXISTS (SELECT 1 FROM pg_database WHERE datname = $1) AS present", [name]);
|
|
45
|
+
return result.rows[0]?.present === true;
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export async function createDatabase(adminUrl: string, name: string): Promise<void> {
|
|
50
|
+
if (!isDatabaseName(name)) throw new Error(`${name} is not a database name this tool creates`);
|
|
51
|
+
await withPgClient(adminUrl, async (client) => {
|
|
52
|
+
await client.query(`CREATE DATABASE "${name}" TEMPLATE template0`);
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Drops `name`, ending its connections first (Postgres 13+). */
|
|
57
|
+
export async function dropDatabase(adminUrl: string, name: string): Promise<void> {
|
|
58
|
+
if (!isDatabaseName(name)) throw new Error(`${name} is not a database name this tool drops`);
|
|
59
|
+
await withPgClient(adminUrl, async (client) => {
|
|
60
|
+
await client.query(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`);
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export interface PgRestoreOptions {
|
|
65
|
+
readonly adminUrl: string;
|
|
66
|
+
readonly database: string;
|
|
67
|
+
/** Extra flags: `--no-owner`, `--no-acl`, `--clean --if-exists`. */
|
|
68
|
+
readonly flags: readonly string[];
|
|
69
|
+
readonly dump: ReadableStream<Uint8Array>;
|
|
70
|
+
/** The executable: a name on PATH or a path. */
|
|
71
|
+
readonly pgRestore: string;
|
|
72
|
+
readonly env: Readonly<Record<string, string | undefined>>;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export type PgRestoreResult = { ok: true } | { ok: false; problem: string };
|
|
76
|
+
|
|
77
|
+
/** `pg_restore --exit-on-error` of the dump on stdin into `database`. */
|
|
78
|
+
export async function runPgRestore(options: PgRestoreOptions): Promise<PgRestoreResult> {
|
|
79
|
+
const libpq = toLibpqEnv(options.adminUrl);
|
|
80
|
+
if (!libpq.ok) return { ok: false, problem: libpq.problem };
|
|
81
|
+
const env: Record<string, string> = { ...libpq.env, PGDATABASE: options.database };
|
|
82
|
+
for (const name of ["PATH", "HOME", "LANG"]) {
|
|
83
|
+
const value = options.env[name];
|
|
84
|
+
if (value !== undefined) env[name] = value;
|
|
85
|
+
}
|
|
86
|
+
const child = spawn(options.pgRestore, ["--exit-on-error", "--no-password", ...options.flags, "--dbname", options.database], {
|
|
87
|
+
env,
|
|
88
|
+
stdio: ["pipe", "ignore", "pipe"],
|
|
89
|
+
});
|
|
90
|
+
let stderr = "";
|
|
91
|
+
child.stderr.on("data", (chunk: Buffer) => {
|
|
92
|
+
stderr = (stderr + chunk.toString("utf8")).slice(-MAX_STDERR);
|
|
93
|
+
});
|
|
94
|
+
const exited = new Promise<number | null>((resolve, reject) => {
|
|
95
|
+
child.on("error", reject);
|
|
96
|
+
child.on("close", (code) => {
|
|
97
|
+
resolve(code);
|
|
98
|
+
});
|
|
99
|
+
});
|
|
100
|
+
// A pg_restore that stops early closes its stdin; the pipe error is then the exit status's to report.
|
|
101
|
+
const fed = pipeStreamTo(options.dump, child.stdin).then(
|
|
102
|
+
() => null,
|
|
103
|
+
(error: unknown) => error,
|
|
104
|
+
);
|
|
105
|
+
let code: number | null;
|
|
106
|
+
try {
|
|
107
|
+
code = await exited;
|
|
108
|
+
} catch (error) {
|
|
109
|
+
return { ok: false, problem: `cannot run ${options.pgRestore} (${error instanceof Error ? error.message : String(error)})` };
|
|
110
|
+
}
|
|
111
|
+
const feedError = await fed;
|
|
112
|
+
if (code !== 0) return { ok: false, problem: `pg_restore exited with ${String(code)}: ${stderr.trim().split("\n").slice(-3).join(" / ")}` };
|
|
113
|
+
if (feedError !== null) return { ok: false, problem: `the backup stream failed (${feedError instanceof Error ? feedError.message : "unknown error"})` };
|
|
114
|
+
return { ok: true };
|
|
115
|
+
}
|