@softure-ai/backup 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/LICENSE +21 -0
- package/README.md +243 -2
- package/dist/cli/main.d.ts +3 -0
- package/dist/cli/main.d.ts.map +1 -0
- package/dist/cli/main.js +16 -0
- package/dist/cli/main.js.map +1 -0
- package/dist/cli/run.d.ts +15 -0
- package/dist/cli/run.d.ts.map +1 -0
- package/dist/cli/run.js +442 -0
- package/dist/cli/run.js.map +1 -0
- package/dist/contract.d.ts +41 -0
- package/dist/contract.d.ts.map +1 -0
- package/dist/contract.js +15 -0
- package/dist/contract.js.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +30 -0
- package/dist/index.js.map +1 -0
- package/dist/messages/en.d.ts +17 -0
- package/dist/messages/en.d.ts.map +1 -0
- package/dist/messages/en.js +17 -0
- package/dist/messages/en.js.map +1 -0
- package/dist/messages/index.d.ts +22 -0
- package/dist/messages/index.d.ts.map +1 -0
- package/dist/messages/index.js +4 -0
- package/dist/messages/index.js.map +1 -0
- package/dist/offsite/config.d.ts +45 -0
- package/dist/offsite/config.d.ts.map +1 -0
- package/dist/offsite/config.js +147 -0
- package/dist/offsite/config.js.map +1 -0
- package/dist/offsite/crypto.d.ts +22 -0
- package/dist/offsite/crypto.d.ts.map +1 -0
- package/dist/offsite/crypto.js +48 -0
- package/dist/offsite/crypto.js.map +1 -0
- package/dist/offsite/dump.d.ts +23 -0
- package/dist/offsite/dump.d.ts.map +1 -0
- package/dist/offsite/dump.js +85 -0
- package/dist/offsite/dump.js.map +1 -0
- package/dist/offsite/index.d.ts +13 -0
- package/dist/offsite/index.d.ts.map +1 -0
- package/dist/offsite/index.js +15 -0
- package/dist/offsite/index.js.map +1 -0
- package/dist/offsite/lifecycle.d.ts +17 -0
- package/dist/offsite/lifecycle.d.ts.map +1 -0
- package/dist/offsite/lifecycle.js +45 -0
- package/dist/offsite/lifecycle.js.map +1 -0
- package/dist/offsite/objects.d.ts +19 -0
- package/dist/offsite/objects.d.ts.map +1 -0
- package/dist/offsite/objects.js +44 -0
- package/dist/offsite/objects.js.map +1 -0
- package/dist/offsite/postgres.d.ts +32 -0
- package/dist/offsite/postgres.d.ts.map +1 -0
- package/dist/offsite/postgres.js +97 -0
- package/dist/offsite/postgres.js.map +1 -0
- package/dist/offsite/record.d.ts +3 -0
- package/dist/offsite/record.d.ts.map +1 -0
- package/dist/offsite/record.js +10 -0
- package/dist/offsite/record.js.map +1 -0
- package/dist/offsite/restore.d.ts +54 -0
- package/dist/offsite/restore.d.ts.map +1 -0
- package/dist/offsite/restore.js +67 -0
- package/dist/offsite/restore.js.map +1 -0
- package/dist/offsite/s3.d.ts +34 -0
- package/dist/offsite/s3.d.ts.map +1 -0
- package/dist/offsite/s3.js +119 -0
- package/dist/offsite/s3.js.map +1 -0
- package/dist/offsite/sigv4.d.ts +19 -0
- package/dist/offsite/sigv4.d.ts.map +1 -0
- package/dist/offsite/sigv4.js +69 -0
- package/dist/offsite/sigv4.js.map +1 -0
- package/dist/offsite/status-sql.d.ts +10 -0
- package/dist/offsite/status-sql.d.ts.map +1 -0
- package/dist/offsite/status-sql.js +37 -0
- package/dist/offsite/status-sql.js.map +1 -0
- package/dist/offsite/upload.d.ts +33 -0
- package/dist/offsite/upload.d.ts.map +1 -0
- package/dist/offsite/upload.js +98 -0
- package/dist/offsite/upload.js.map +1 -0
- package/dist/offsite/verify.d.ts +32 -0
- package/dist/offsite/verify.d.ts.map +1 -0
- package/dist/offsite/verify.js +75 -0
- package/dist/offsite/verify.js.map +1 -0
- package/dist/schema.d.ts +150 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/schema.js +15 -0
- package/dist/schema.js.map +1 -0
- package/dist/server/index.d.ts +28 -0
- package/dist/server/index.d.ts.map +1 -0
- package/dist/server/index.js +98 -0
- package/dist/server/index.js.map +1 -0
- package/examples/backup-verify.yml +84 -0
- package/migrations/0001_create_runs.sql +21 -0
- package/module.json +12 -0
- package/package.json +73 -4
- package/src/cli/main.ts +16 -0
- package/src/cli/run.ts +441 -0
- package/src/contract.ts +47 -0
- package/src/index.ts +41 -0
- package/src/messages/en.ts +16 -0
- package/src/messages/index.ts +6 -0
- package/src/offsite/config.ts +173 -0
- package/src/offsite/crypto.ts +53 -0
- package/src/offsite/dump.ts +97 -0
- package/src/offsite/index.ts +33 -0
- package/src/offsite/lifecycle.ts +47 -0
- package/src/offsite/objects.ts +54 -0
- package/src/offsite/postgres.ts +115 -0
- package/src/offsite/record.ts +11 -0
- package/src/offsite/restore.ts +108 -0
- package/src/offsite/s3.ts +144 -0
- package/src/offsite/sigv4.ts +84 -0
- package/src/offsite/status-sql.ts +45 -0
- package/src/offsite/upload.ts +116 -0
- package/src/offsite/verify.ts +97 -0
- package/src/schema.ts +17 -0
- package/src/server/index.ts +111 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/** `db-20261109T031700Z.dump` (`@softure-ai/deploy` backup, any short prefix). */
|
|
2
|
+
const DATABASE_NAME = /^([a-z0-9][a-z0-9_-]*)-(\d{8})T(\d{6})Z\.dump$/;
|
|
3
|
+
/** `data-20261109-031700.tar.gz` (deploy.sh backup-data). */
|
|
4
|
+
const DATA_NAME = /^data-(\d{8})-(\d{6})\.tar\.gz$/;
|
|
5
|
+
export const OBJECT_SUFFIX = ".age";
|
|
6
|
+
function toDate(day, time) {
|
|
7
|
+
return new Date(`${day.slice(0, 4)}-${day.slice(4, 6)}-${day.slice(6, 8)}T${time.slice(0, 2)}:${time.slice(2, 4)}:${time.slice(4, 6)}Z`);
|
|
8
|
+
}
|
|
9
|
+
/** The kind and time of a backup file name, or null for any other name (or an impossible date). */
|
|
10
|
+
export function parseBackupFileName(name) {
|
|
11
|
+
const data = DATA_NAME.exec(name);
|
|
12
|
+
const database = data === null ? DATABASE_NAME.exec(name) : null;
|
|
13
|
+
if (data === null && database === null)
|
|
14
|
+
return null;
|
|
15
|
+
const takenAt = data === null ? toDate(database?.[2] ?? "", database?.[3] ?? "") : toDate(data[1] ?? "", data[2] ?? "");
|
|
16
|
+
if (Number.isNaN(takenAt.getTime()))
|
|
17
|
+
return null;
|
|
18
|
+
return { kind: data === null ? "database" : "data", name, takenAt };
|
|
19
|
+
}
|
|
20
|
+
export function buildObjectKey(prefix, fileName) {
|
|
21
|
+
return `${prefix}/${fileName}${OBJECT_SUFFIX}`;
|
|
22
|
+
}
|
|
23
|
+
/** The backup file name of a key under `prefix`, or null for any other key. */
|
|
24
|
+
export function parseObjectKey(prefix, key) {
|
|
25
|
+
if (!key.startsWith(`${prefix}/`) || !key.endsWith(OBJECT_SUFFIX))
|
|
26
|
+
return null;
|
|
27
|
+
const name = key.slice(prefix.length + 1, -OBJECT_SUFFIX.length);
|
|
28
|
+
if (name.includes("/"))
|
|
29
|
+
return null;
|
|
30
|
+
return parseBackupFileName(name);
|
|
31
|
+
}
|
|
32
|
+
/** The newest key of `kind` under `prefix`, by the time in its name; null when there is none. */
|
|
33
|
+
export function selectNewestKey(prefix, keys, kind) {
|
|
34
|
+
let newest = null;
|
|
35
|
+
for (const key of keys) {
|
|
36
|
+
const parsed = parseObjectKey(prefix, key);
|
|
37
|
+
if (parsed?.kind !== kind)
|
|
38
|
+
continue;
|
|
39
|
+
if (newest === null || parsed.takenAt.getTime() > newest.takenAt.getTime())
|
|
40
|
+
newest = { key, takenAt: parsed.takenAt };
|
|
41
|
+
}
|
|
42
|
+
return newest;
|
|
43
|
+
}
|
|
44
|
+
//# sourceMappingURL=objects.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"objects.js","sourceRoot":"","sources":["../../src/offsite/objects.ts"],"names":[],"mappings":"AAKA,kFAAkF;AAClF,MAAM,aAAa,GAAG,gDAAgD,CAAC;AACvE,6DAA6D;AAC7D,MAAM,SAAS,GAAG,iCAAiC,CAAC;AACpD,MAAM,CAAC,MAAM,aAAa,GAAG,MAAM,CAAC;AASpC,SAAS,MAAM,CAAC,GAAW,EAAE,IAAY;IACvC,OAAO,IAAI,IAAI,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;AAC3I,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC9C,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACjE,IAAI,IAAI,KAAK,IAAI,IAAI,QAAQ,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACpD,MAAM,OAAO,GAAG,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACxH,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAAE,OAAO,IAAI,CAAC;IACjD,OAAO,EAAE,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AACtE,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAc,EAAE,QAAgB;IAC7D,OAAO,GAAG,MAAM,IAAI,QAAQ,GAAG,aAAa,EAAE,CAAC;AACjD,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,cAAc,CAAC,MAAc,EAAE,GAAW;IACxD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC;QAAE,OAAO,IAAI,CAAC;IAC/E,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACjE,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACpC,OAAO,mBAAmB,CAAC,IAAI,CAAC,CAAC;AACnC,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,eAAe,CAAC,MAAc,EAAE,IAAuB,EAAE,IAAgB;IACvF,IAAI,MAAM,GAA0C,IAAI,CAAC;IACzD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC3C,IAAI,MAAM,EAAE,IAAI,KAAK,IAAI;YAAE,SAAS;QACpC,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE;YAAE,MAAM,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;IACxH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export declare function isDatabaseName(name: string): boolean;
|
|
2
|
+
/** The URL with another database. */
|
|
3
|
+
export declare function withDatabase(url: string, database: string): string;
|
|
4
|
+
/** The database of a URL (`postgres` when it names none). */
|
|
5
|
+
export declare function getUrlDatabase(url: string): string;
|
|
6
|
+
/** `softure_offsite_verify_<8 hex>`: a fresh throwaway name. */
|
|
7
|
+
export declare function createVerifyDatabaseName(): string;
|
|
8
|
+
/** `<base>_restore_<YYYYMMDDHHMMSS>`, shortened so it stays a valid name. */
|
|
9
|
+
export declare function createRestoreDatabaseName(base: string, now: Date): string;
|
|
10
|
+
export declare function databaseExists(adminUrl: string, name: string): Promise<boolean>;
|
|
11
|
+
export declare function createDatabase(adminUrl: string, name: string): Promise<void>;
|
|
12
|
+
/** Drops `name`, ending its connections first (Postgres 13+). */
|
|
13
|
+
export declare function dropDatabase(adminUrl: string, name: string): Promise<void>;
|
|
14
|
+
export interface PgRestoreOptions {
|
|
15
|
+
readonly adminUrl: string;
|
|
16
|
+
readonly database: string;
|
|
17
|
+
/** Extra flags: `--no-owner`, `--no-acl`, `--clean --if-exists`. */
|
|
18
|
+
readonly flags: readonly string[];
|
|
19
|
+
readonly dump: ReadableStream<Uint8Array>;
|
|
20
|
+
/** The executable: a name on PATH or a path. */
|
|
21
|
+
readonly pgRestore: string;
|
|
22
|
+
readonly env: Readonly<Record<string, string | undefined>>;
|
|
23
|
+
}
|
|
24
|
+
export type PgRestoreResult = {
|
|
25
|
+
ok: true;
|
|
26
|
+
} | {
|
|
27
|
+
ok: false;
|
|
28
|
+
problem: string;
|
|
29
|
+
};
|
|
30
|
+
/** `pg_restore --exit-on-error` of the dump on stdin into `database`. */
|
|
31
|
+
export declare function runPgRestore(options: PgRestoreOptions): Promise<PgRestoreResult>;
|
|
32
|
+
//# sourceMappingURL=postgres.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"postgres.d.ts","sourceRoot":"","sources":["../../src/offsite/postgres.ts"],"names":[],"mappings":"AAYA,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEpD;AAED,qCAAqC;AACrC,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAIlE;AAED,6DAA6D;AAC7D,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAGlD;AAED,gEAAgE;AAChE,wBAAgB,wBAAwB,IAAI,MAAM,CAEjD;AAED,6EAA6E;AAC7E,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,GAAG,MAAM,CAIzE;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAKrF;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKlF;AAED,iEAAiE;AACjE,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAKhF;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,oEAAoE;IACpE,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;IAC1C,gDAAgD;IAChD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC;CAC5D;AAED,MAAM,MAAM,eAAe,GAAG;IAAE,EAAE,EAAE,IAAI,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5E,yEAAyE;AACzE,wBAAsB,YAAY,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CAqCtF"}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
// The Postgres side of restore and verify: an admin connection from a URL in the environment, databases created and
|
|
2
|
+
// dropped through it, and pg_restore fed from a decrypted stream with the URL as libpq variables, so the password
|
|
3
|
+
// never shows in the process list.
|
|
4
|
+
import { spawn } from "node:child_process";
|
|
5
|
+
import { randomBytes } from "node:crypto";
|
|
6
|
+
import { toLibpqEnv, withPgClient } from "@softure-ai/deploy";
|
|
7
|
+
import { pipeStreamTo } from "./crypto.js";
|
|
8
|
+
/** Database names this package creates or accepts: lower-case identifiers within Postgres's 63 bytes. */
|
|
9
|
+
const DATABASE_NAME = /^[a-z_][a-z0-9_]{0,62}$/;
|
|
10
|
+
const MAX_STDERR = 4000;
|
|
11
|
+
export function isDatabaseName(name) {
|
|
12
|
+
return DATABASE_NAME.test(name);
|
|
13
|
+
}
|
|
14
|
+
/** The URL with another database. */
|
|
15
|
+
export function withDatabase(url, database) {
|
|
16
|
+
const parsed = new URL(url);
|
|
17
|
+
parsed.pathname = `/${database}`;
|
|
18
|
+
return parsed.toString();
|
|
19
|
+
}
|
|
20
|
+
/** The database of a URL (`postgres` when it names none). */
|
|
21
|
+
export function getUrlDatabase(url) {
|
|
22
|
+
const name = decodeURIComponent(new URL(url).pathname.replace(/^\//, ""));
|
|
23
|
+
return name === "" ? "postgres" : name;
|
|
24
|
+
}
|
|
25
|
+
/** `softure_offsite_verify_<8 hex>`: a fresh throwaway name. */
|
|
26
|
+
export function createVerifyDatabaseName() {
|
|
27
|
+
return `softure_offsite_verify_${randomBytes(4).toString("hex")}`;
|
|
28
|
+
}
|
|
29
|
+
/** `<base>_restore_<YYYYMMDDHHMMSS>`, shortened so it stays a valid name. */
|
|
30
|
+
export function createRestoreDatabaseName(base, now) {
|
|
31
|
+
const stamp = now.toISOString().replace(/[-:T]/g, "").slice(0, 14);
|
|
32
|
+
const safeBase = base.replace(/[^a-z0-9_]/g, "_").replace(/^[^a-z_]/, "_").slice(0, 63 - "_restore_".length - stamp.length);
|
|
33
|
+
return `${safeBase}_restore_${stamp}`;
|
|
34
|
+
}
|
|
35
|
+
export async function databaseExists(adminUrl, name) {
|
|
36
|
+
return withPgClient(adminUrl, async (client) => {
|
|
37
|
+
const result = await client.query("SELECT EXISTS (SELECT 1 FROM pg_database WHERE datname = $1) AS present", [name]);
|
|
38
|
+
return result.rows[0]?.present === true;
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
export async function createDatabase(adminUrl, name) {
|
|
42
|
+
if (!isDatabaseName(name))
|
|
43
|
+
throw new Error(`${name} is not a database name this tool creates`);
|
|
44
|
+
await withPgClient(adminUrl, async (client) => {
|
|
45
|
+
await client.query(`CREATE DATABASE "${name}" TEMPLATE template0`);
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
/** Drops `name`, ending its connections first (Postgres 13+). */
|
|
49
|
+
export async function dropDatabase(adminUrl, name) {
|
|
50
|
+
if (!isDatabaseName(name))
|
|
51
|
+
throw new Error(`${name} is not a database name this tool drops`);
|
|
52
|
+
await withPgClient(adminUrl, async (client) => {
|
|
53
|
+
await client.query(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`);
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
/** `pg_restore --exit-on-error` of the dump on stdin into `database`. */
|
|
57
|
+
export async function runPgRestore(options) {
|
|
58
|
+
const libpq = toLibpqEnv(options.adminUrl);
|
|
59
|
+
if (!libpq.ok)
|
|
60
|
+
return { ok: false, problem: libpq.problem };
|
|
61
|
+
const env = { ...libpq.env, PGDATABASE: options.database };
|
|
62
|
+
for (const name of ["PATH", "HOME", "LANG"]) {
|
|
63
|
+
const value = options.env[name];
|
|
64
|
+
if (value !== undefined)
|
|
65
|
+
env[name] = value;
|
|
66
|
+
}
|
|
67
|
+
const child = spawn(options.pgRestore, ["--exit-on-error", "--no-password", ...options.flags, "--dbname", options.database], {
|
|
68
|
+
env,
|
|
69
|
+
stdio: ["pipe", "ignore", "pipe"],
|
|
70
|
+
});
|
|
71
|
+
let stderr = "";
|
|
72
|
+
child.stderr.on("data", (chunk) => {
|
|
73
|
+
stderr = (stderr + chunk.toString("utf8")).slice(-MAX_STDERR);
|
|
74
|
+
});
|
|
75
|
+
const exited = new Promise((resolve, reject) => {
|
|
76
|
+
child.on("error", reject);
|
|
77
|
+
child.on("close", (code) => {
|
|
78
|
+
resolve(code);
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
// A pg_restore that stops early closes its stdin; the pipe error is then the exit status's to report.
|
|
82
|
+
const fed = pipeStreamTo(options.dump, child.stdin).then(() => null, (error) => error);
|
|
83
|
+
let code;
|
|
84
|
+
try {
|
|
85
|
+
code = await exited;
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
return { ok: false, problem: `cannot run ${options.pgRestore} (${error instanceof Error ? error.message : String(error)})` };
|
|
89
|
+
}
|
|
90
|
+
const feedError = await fed;
|
|
91
|
+
if (code !== 0)
|
|
92
|
+
return { ok: false, problem: `pg_restore exited with ${String(code)}: ${stderr.trim().split("\n").slice(-3).join(" / ")}` };
|
|
93
|
+
if (feedError !== null)
|
|
94
|
+
return { ok: false, problem: `the backup stream failed (${feedError instanceof Error ? feedError.message : "unknown error"})` };
|
|
95
|
+
return { ok: true };
|
|
96
|
+
}
|
|
97
|
+
//# sourceMappingURL=postgres.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"postgres.js","sourceRoot":"","sources":["../../src/offsite/postgres.ts"],"names":[],"mappings":"AAAA,oHAAoH;AACpH,kHAAkH;AAClH,mCAAmC;AACnC,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,yGAAyG;AACzG,MAAM,aAAa,GAAG,yBAAyB,CAAC;AAChD,MAAM,UAAU,GAAG,IAAI,CAAC;AAExB,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAClC,CAAC;AAED,qCAAqC;AACrC,MAAM,UAAU,YAAY,CAAC,GAAW,EAAE,QAAgB;IACxD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,CAAC,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAC;IACjC,OAAO,MAAM,CAAC,QAAQ,EAAE,CAAC;AAC3B,CAAC;AAED,6DAA6D;AAC7D,MAAM,UAAU,cAAc,CAAC,GAAW;IACxC,MAAM,IAAI,GAAG,kBAAkB,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IAC1E,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;AACzC,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,wBAAwB;IACtC,OAAO,0BAA0B,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;AACpE,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,yBAAyB,CAAC,IAAY,EAAE,GAAS;IAC/D,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,WAAW,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAC5H,OAAO,GAAG,QAAQ,YAAY,KAAK,EAAE,CAAC;AACxC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,QAAgB,EAAE,IAAY;IACjE,OAAO,YAAY,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QAC7C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAuB,yEAAyE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3I,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,CAAC;IAC1C,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,QAAgB,EAAE,IAAY;IACjE,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,2CAA2C,CAAC,CAAC;IAC/F,MAAM,YAAY,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,CAAC,KAAK,CAAC,oBAAoB,IAAI,sBAAsB,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;AACL,CAAC;AAED,iEAAiE;AACjE,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB,EAAE,IAAY;IAC/D,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,yCAAyC,CAAC,CAAC;IAC7F,MAAM,YAAY,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,CAAC,KAAK,CAAC,4BAA4B,IAAI,gBAAgB,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;AACL,CAAC;AAeD,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAyB;IAC1D,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC3C,IAAI,CAAC,KAAK,CAAC,EAAE;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;IAC5D,MAAM,GAAG,GAA2B,EAAE,GAAG,KAAK,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;IACnF,KAAK,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;QAC5C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,KAAK,KAAK,SAAS;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;IAC7C,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,iBAAiB,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE;QAC3H,GAAG;QACH,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC;KAClC,CAAC,CAAC;IACH,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;QACxC,MAAM,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,IAAI,OAAO,CAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC5D,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,OAAO,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,sGAAsG;IACtG,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CACtD,GAAG,EAAE,CAAC,IAAI,EACV,CAAC,KAAc,EAAE,EAAE,CAAC,KAAK,CAC1B,CAAC;IACF,IAAI,IAAmB,CAAC;IACxB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,MAAM,CAAC;IACtB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,cAAc,OAAO,CAAC,SAAS,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAC/H,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC;IAC5B,IAAI,IAAI,KAAK,CAAC;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,0BAA0B,MAAM,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC;IAC5I,IAAI,SAAS,KAAK,IAAI;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,6BAA6B,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC;IACxJ,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;AACtB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"record.d.ts","sourceRoot":"","sources":["../../src/offsite/record.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAI7E"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// The run's row in `backup.runs`, written over a connection: the same SQL deploy.sh pipes into psql on a VPS
|
|
2
|
+
// (`buildStatusSql`), sent as one simple query so the insert and the prune share its transaction.
|
|
3
|
+
import { withPgClient } from "@softure-ai/deploy";
|
|
4
|
+
import { buildStatusSql } from "./status-sql.js";
|
|
5
|
+
export async function recordRun(url, record) {
|
|
6
|
+
await withPgClient(url, async (client) => {
|
|
7
|
+
await client.query(buildStatusSql(record));
|
|
8
|
+
});
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=record.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"record.js","sourceRoot":"","sources":["../../src/offsite/record.ts"],"names":[],"mappings":"AAAA,6GAA6G;AAC7G,kGAAkG;AAClG,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAGjD,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,MAAiB;IAC5D,MAAM,YAAY,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QACvC,MAAM,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { StoreConfig } from "./config.js";
|
|
2
|
+
import { type S3Options } from "./s3.js";
|
|
3
|
+
import type { BackupKind } from "../contract.js";
|
|
4
|
+
export type OffsiteResult<T> = {
|
|
5
|
+
ok: true;
|
|
6
|
+
value: T;
|
|
7
|
+
} | {
|
|
8
|
+
ok: false;
|
|
9
|
+
problem: string;
|
|
10
|
+
};
|
|
11
|
+
/** The key to read: the one named (checked to be a backup of `kind` under the prefix), else the newest of `kind`. */
|
|
12
|
+
export declare function resolveKey(store: StoreConfig, kind: BackupKind, key: string | undefined, s3?: S3Options): Promise<OffsiteResult<{
|
|
13
|
+
key: string;
|
|
14
|
+
takenAt: Date;
|
|
15
|
+
}>>;
|
|
16
|
+
export interface RestoreOptions {
|
|
17
|
+
readonly store: StoreConfig;
|
|
18
|
+
readonly identities: readonly string[];
|
|
19
|
+
/** A superuser's URL on the target server (any database of it). */
|
|
20
|
+
readonly adminUrl: string;
|
|
21
|
+
/** An object key; the newest database backup by default. */
|
|
22
|
+
readonly key?: string;
|
|
23
|
+
/** The database to restore into; a new `<url database>_restore_<time>` by default. */
|
|
24
|
+
readonly into?: string;
|
|
25
|
+
/** Restore over an existing `into` (pg_restore --clean --if-exists). */
|
|
26
|
+
readonly overwrite?: boolean;
|
|
27
|
+
/** Leave out ownership and grants, for a server without the app's roles. */
|
|
28
|
+
readonly noOwner?: boolean;
|
|
29
|
+
readonly pgRestore: string;
|
|
30
|
+
readonly env: Readonly<Record<string, string | undefined>>;
|
|
31
|
+
readonly now?: Date;
|
|
32
|
+
readonly s3?: S3Options;
|
|
33
|
+
}
|
|
34
|
+
export interface RestoreOutcome {
|
|
35
|
+
readonly key: string;
|
|
36
|
+
readonly database: string;
|
|
37
|
+
readonly created: boolean;
|
|
38
|
+
}
|
|
39
|
+
export declare function restoreBackup(options: RestoreOptions): Promise<OffsiteResult<RestoreOutcome>>;
|
|
40
|
+
export interface FetchOptions {
|
|
41
|
+
readonly store: StoreConfig;
|
|
42
|
+
readonly identities: readonly string[];
|
|
43
|
+
readonly kind: BackupKind;
|
|
44
|
+
readonly key?: string;
|
|
45
|
+
/** The decrypted file to write (0600). */
|
|
46
|
+
readonly out: string;
|
|
47
|
+
readonly s3?: S3Options;
|
|
48
|
+
}
|
|
49
|
+
/** Downloads and decrypts one object to a file, e.g. a data archive to unpack into the new volume. */
|
|
50
|
+
export declare function fetchBackup(options: FetchOptions): Promise<OffsiteResult<{
|
|
51
|
+
key: string;
|
|
52
|
+
out: string;
|
|
53
|
+
}>>;
|
|
54
|
+
//# sourceMappingURL=restore.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"restore.d.ts","sourceRoot":"","sources":["../../src/offsite/restore.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAW/C,OAAO,EAAgC,KAAK,SAAS,EAAE,MAAM,SAAS,CAAC;AACvE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAEjD,MAAM,MAAM,aAAa,CAAC,CAAC,IAAI;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,CAAC,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvF,qHAAqH;AACrH,wBAAsB,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,GAAG,SAAS,EAAE,EAAE,CAAC,EAAE,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,IAAI,CAAA;CAAE,CAAC,CAAC,CAUtK;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,mEAAmE;IACnE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,4DAA4D;IAC5D,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IACtB,sFAAsF;IACtF,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,wEAAwE;IACxE,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;IAC7B,4EAA4E;IAC5E,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC;IAC3D,QAAQ,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC;IACpB,QAAQ,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED,wBAAsB,aAAa,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC,CAwBnG;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IACtB,0CAA0C;IAC1C,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC;CACzB;AAED,sGAAsG;AACtG,wBAAsB,WAAW,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,aAAa,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAa7G"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { decryptStream, writeStreamToFile } from "./crypto.js";
|
|
2
|
+
import { parseObjectKey, selectNewestKey } from "./objects.js";
|
|
3
|
+
import { createDatabase, createRestoreDatabaseName, databaseExists, getUrlDatabase, isDatabaseName, runPgRestore, } from "./postgres.js";
|
|
4
|
+
import { getObjectStream, listObjects } from "./s3.js";
|
|
5
|
+
/** The key to read: the one named (checked to be a backup of `kind` under the prefix), else the newest of `kind`. */
|
|
6
|
+
export async function resolveKey(store, kind, key, s3) {
|
|
7
|
+
if (key !== undefined) {
|
|
8
|
+
const parsed = parseObjectKey(store.prefix, key);
|
|
9
|
+
if (parsed?.kind !== kind)
|
|
10
|
+
return { ok: false, problem: `${key} is not a ${kind} backup under ${store.prefix}/` };
|
|
11
|
+
return { ok: true, value: { key, takenAt: parsed.takenAt } };
|
|
12
|
+
}
|
|
13
|
+
const objects = await listObjects(store, `${store.prefix}/`, s3);
|
|
14
|
+
const newest = selectNewestKey(store.prefix, objects.map((object) => object.key), kind);
|
|
15
|
+
if (newest === null)
|
|
16
|
+
return { ok: false, problem: `no ${kind} backup is stored under ${store.prefix}/` };
|
|
17
|
+
return { ok: true, value: newest };
|
|
18
|
+
}
|
|
19
|
+
export async function restoreBackup(options) {
|
|
20
|
+
const resolved = await resolveKey(options.store, "database", options.key, options.s3);
|
|
21
|
+
if (!resolved.ok)
|
|
22
|
+
return resolved;
|
|
23
|
+
const target = options.into ?? createRestoreDatabaseName(getUrlDatabase(options.adminUrl), options.now ?? new Date());
|
|
24
|
+
if (!isDatabaseName(target))
|
|
25
|
+
return { ok: false, problem: `${target} is not a database name (lower-case letters, digits and _)` };
|
|
26
|
+
const exists = await databaseExists(options.adminUrl, target);
|
|
27
|
+
if (exists && options.overwrite !== true) {
|
|
28
|
+
return { ok: false, problem: `the database ${target} exists; pass --overwrite to restore over it, or restore into a new one` };
|
|
29
|
+
}
|
|
30
|
+
if (!exists && options.overwrite === true)
|
|
31
|
+
return { ok: false, problem: `--overwrite needs an existing database; ${target} does not exist` };
|
|
32
|
+
// Decrypt before creating anything: a wrong key fails here, at the header.
|
|
33
|
+
const encrypted = await getObjectStream(options.store, resolved.value.key, options.s3);
|
|
34
|
+
let dump;
|
|
35
|
+
try {
|
|
36
|
+
dump = await decryptStream(encrypted, options.identities);
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
await encrypted.cancel().catch(() => undefined);
|
|
40
|
+
return { ok: false, problem: `cannot decrypt ${resolved.value.key} with this identity (${error instanceof Error ? error.message : String(error)})` };
|
|
41
|
+
}
|
|
42
|
+
if (!exists)
|
|
43
|
+
await createDatabase(options.adminUrl, target);
|
|
44
|
+
const flags = [...(options.overwrite === true ? ["--clean", "--if-exists"] : []), ...(options.noOwner === true ? ["--no-owner", "--no-acl"] : [])];
|
|
45
|
+
const restored = await runPgRestore({ adminUrl: options.adminUrl, database: target, flags, dump, pgRestore: options.pgRestore, env: options.env });
|
|
46
|
+
if (!restored.ok)
|
|
47
|
+
return { ok: false, problem: `${resolved.value.key} did not restore into ${target}: ${restored.problem}` };
|
|
48
|
+
return { ok: true, value: { key: resolved.value.key, database: target, created: !exists } };
|
|
49
|
+
}
|
|
50
|
+
/** Downloads and decrypts one object to a file, e.g. a data archive to unpack into the new volume. */
|
|
51
|
+
export async function fetchBackup(options) {
|
|
52
|
+
const resolved = await resolveKey(options.store, options.kind, options.key, options.s3);
|
|
53
|
+
if (!resolved.ok)
|
|
54
|
+
return resolved;
|
|
55
|
+
const encrypted = await getObjectStream(options.store, resolved.value.key, options.s3);
|
|
56
|
+
let plain;
|
|
57
|
+
try {
|
|
58
|
+
plain = await decryptStream(encrypted, options.identities);
|
|
59
|
+
}
|
|
60
|
+
catch (error) {
|
|
61
|
+
await encrypted.cancel().catch(() => undefined);
|
|
62
|
+
return { ok: false, problem: `cannot decrypt ${resolved.value.key} with this identity (${error instanceof Error ? error.message : String(error)})` };
|
|
63
|
+
}
|
|
64
|
+
await writeStreamToFile(plain, options.out);
|
|
65
|
+
return { ok: true, value: { key: resolved.value.key, out: options.out } };
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=restore.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"restore.js","sourceRoot":"","sources":["../../src/offsite/restore.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAC/D,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/D,OAAO,EACL,cAAc,EACd,yBAAyB,EACzB,cAAc,EACd,cAAc,EACd,cAAc,EACd,YAAY,GACb,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,eAAe,EAAE,WAAW,EAAkB,MAAM,SAAS,CAAC;AAKvE,qHAAqH;AACrH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAkB,EAAE,IAAgB,EAAE,GAAuB,EAAE,EAAc;IAC5G,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QACjD,IAAI,MAAM,EAAE,IAAI,KAAK,IAAI;YAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,GAAG,aAAa,IAAI,iBAAiB,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC;QAClH,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;IAC/D,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,GAAG,KAAK,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC,CAAC;IACjE,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IACxF,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,IAAI,2BAA2B,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC;IACzG,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AACrC,CAAC;AA2BD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAAuB;IACzD,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;IACtF,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,OAAO,QAAQ,CAAC;IAClC,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,IAAI,yBAAyB,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC;IACtH,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,MAAM,4DAA4D,EAAE,CAAC;IAClI,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC9D,IAAI,MAAM,IAAI,OAAO,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;QACzC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,gBAAgB,MAAM,yEAAyE,EAAE,CAAC;IACjI,CAAC;IACD,IAAI,CAAC,MAAM,IAAI,OAAO,CAAC,SAAS,KAAK,IAAI;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,2CAA2C,MAAM,iBAAiB,EAAE,CAAC;IAC7I,2EAA2E;IAC3E,MAAM,SAAS,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;IACvF,IAAI,IAAgC,CAAC;IACrC,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAC5D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,SAAS,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAChD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,QAAQ,CAAC,KAAK,CAAC,GAAG,wBAAwB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IACvJ,CAAC;IACD,IAAI,CAAC,MAAM;QAAE,MAAM,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC5D,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnJ,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACnJ,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,yBAAyB,MAAM,KAAK,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;IAC7H,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC;AAC9F,CAAC;AAYD,sGAAsG;AACtG,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAqB;IACrD,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;IACxF,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,OAAO,QAAQ,CAAC;IAClC,MAAM,SAAS,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC;IACvF,IAAI,KAAiC,CAAC;IACtC,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAC7D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,SAAS,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAChD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,QAAQ,CAAC,KAAK,CAAC,GAAG,wBAAwB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IACvJ,CAAC;IACD,MAAM,iBAAiB,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAC5C,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC;AAC5E,CAAC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { StoreConfig } from "./config.js";
|
|
2
|
+
/** A failed request, with what S3 said (its error code) but never a header or a signature. */
|
|
3
|
+
export declare class S3RequestError extends Error {
|
|
4
|
+
readonly status: number | null;
|
|
5
|
+
readonly s3Code: string | null;
|
|
6
|
+
constructor(message: string, status: number | null, s3Code: string | null);
|
|
7
|
+
/** 401 and 403: the token is wrong or lacks the permission. */
|
|
8
|
+
get isDenied(): boolean;
|
|
9
|
+
}
|
|
10
|
+
export interface StoredObject {
|
|
11
|
+
readonly key: string;
|
|
12
|
+
readonly bytes: number;
|
|
13
|
+
}
|
|
14
|
+
export interface S3Options {
|
|
15
|
+
/** The clock of the signature. */
|
|
16
|
+
readonly now?: () => Date;
|
|
17
|
+
readonly fetch?: typeof fetch;
|
|
18
|
+
/** Per request; an upload of a large dump needs minutes. Default 30 minutes. */
|
|
19
|
+
readonly timeoutMs?: number;
|
|
20
|
+
}
|
|
21
|
+
/** The URL of a key (or of the bucket, for a list), path style under the endpoint's own path. */
|
|
22
|
+
export declare function getObjectUrl(store: StoreConfig, key: string | null): URL;
|
|
23
|
+
/** PUTs the file at `path` (its sha256 and size known) as `key`. */
|
|
24
|
+
export declare function putObjectFile(store: StoreConfig, input: {
|
|
25
|
+
key: string;
|
|
26
|
+
path: string;
|
|
27
|
+
bytes: number;
|
|
28
|
+
sha256: string;
|
|
29
|
+
}, options?: S3Options): Promise<void>;
|
|
30
|
+
/** The body of `key` as a stream. */
|
|
31
|
+
export declare function getObjectStream(store: StoreConfig, key: string, options?: S3Options): Promise<ReadableStream<Uint8Array>>;
|
|
32
|
+
/** Every object under `prefix/`, through ListObjectsV2 pages. */
|
|
33
|
+
export declare function listObjects(store: StoreConfig, prefix: string, options?: S3Options): Promise<StoredObject[]>;
|
|
34
|
+
//# sourceMappingURL=s3.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../../src/offsite/s3.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG/C,8FAA8F;AAC9F,qBAAa,cAAe,SAAQ,KAAK;IACvC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAE/B,YAAY,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,EAKxE;IAED,+DAA+D;IAC/D,IAAI,QAAQ,IAAI,OAAO,CAEtB;CACF;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,SAAS;IACxB,kCAAkC;IAClC,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IAC9B,gFAAgF;IAChF,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAYD,iGAAiG;AACjG,wBAAgB,YAAY,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,GAAG,CAIxE;AAmDD,oEAAoE;AACpE,wBAAsB,aAAa,CACjC,KAAK,EAAE,WAAW,EAClB,KAAK,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EACnE,OAAO,GAAE,SAAc,GACtB,OAAO,CAAC,IAAI,CAAC,CAKf;AAED,qCAAqC;AACrC,wBAAsB,eAAe,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,SAAc,GAAG,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAKnI;AAED,iEAAiE;AACjE,wBAAsB,WAAW,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,SAAc,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAkBtH"}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
// The three S3 requests: PUT of an encrypted file (the nightly run, a write-only token suffices), GET of an object
|
|
2
|
+
// and ListObjectsV2 under the prefix (restore and verify, a read token). Path style: <endpoint>/<bucket>/<key>.
|
|
3
|
+
import { createReadStream } from "node:fs";
|
|
4
|
+
import { Readable } from "node:stream";
|
|
5
|
+
import { EMPTY_SHA256, encodeRfc3986, signRequest } from "./sigv4.js";
|
|
6
|
+
/** A failed request, with what S3 said (its error code) but never a header or a signature. */
|
|
7
|
+
export class S3RequestError extends Error {
|
|
8
|
+
status;
|
|
9
|
+
s3Code;
|
|
10
|
+
constructor(message, status, s3Code) {
|
|
11
|
+
super(message);
|
|
12
|
+
this.name = "S3RequestError";
|
|
13
|
+
this.status = status;
|
|
14
|
+
this.s3Code = s3Code;
|
|
15
|
+
}
|
|
16
|
+
/** 401 and 403: the token is wrong or lacks the permission. */
|
|
17
|
+
get isDenied() {
|
|
18
|
+
return this.status === 401 || this.status === 403;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
const DEFAULT_TIMEOUT_MS = 30 * 60 * 1000;
|
|
22
|
+
const MAX_ERROR_TEXT = 200;
|
|
23
|
+
/** `/a/b//` → `/a/b`, by index: a regular expression here would backtrack on a path of many slashes. */
|
|
24
|
+
function trimTrailingSlashes(path) {
|
|
25
|
+
let end = path.length;
|
|
26
|
+
while (end > 0 && path.charCodeAt(end - 1) === 47)
|
|
27
|
+
end -= 1;
|
|
28
|
+
return path.slice(0, end);
|
|
29
|
+
}
|
|
30
|
+
/** The URL of a key (or of the bucket, for a list), path style under the endpoint's own path. */
|
|
31
|
+
export function getObjectUrl(store, key) {
|
|
32
|
+
const base = trimTrailingSlashes(store.endpoint.pathname);
|
|
33
|
+
const path = key === null ? `${base}/${encodeRfc3986(store.bucket)}` : `${base}/${encodeRfc3986(store.bucket)}/${encodeRfc3986(key, true)}`;
|
|
34
|
+
return new URL(`${store.endpoint.origin}${path}`);
|
|
35
|
+
}
|
|
36
|
+
function readXmlTag(xml, tag) {
|
|
37
|
+
const match = new RegExp(`<${tag}>([^<]*)</${tag}>`).exec(xml);
|
|
38
|
+
return match?.[1] === undefined ? null : decodeXml(match[1]);
|
|
39
|
+
}
|
|
40
|
+
function decodeXml(text) {
|
|
41
|
+
return text
|
|
42
|
+
.replace(/</g, "<")
|
|
43
|
+
.replace(/>/g, ">")
|
|
44
|
+
.replace(/"/g, '"')
|
|
45
|
+
.replace(/'/g, "'")
|
|
46
|
+
.replace(/&/g, "&");
|
|
47
|
+
}
|
|
48
|
+
async function toError(operation, response) {
|
|
49
|
+
const body = await response.text().catch(() => "");
|
|
50
|
+
const code = readXmlTag(body, "Code");
|
|
51
|
+
const detail = readXmlTag(body, "Message")?.slice(0, MAX_ERROR_TEXT);
|
|
52
|
+
const said = [code, detail].filter((part) => part !== null && part !== undefined && part !== "").join(": ");
|
|
53
|
+
return new S3RequestError(`${operation} answered HTTP ${String(response.status)}${said === "" ? "" : ` (${said})`}`, response.status, code);
|
|
54
|
+
}
|
|
55
|
+
async function send(operation, url, init, options) {
|
|
56
|
+
const doFetch = options.fetch ?? fetch;
|
|
57
|
+
let response;
|
|
58
|
+
try {
|
|
59
|
+
response = await doFetch(url, { ...init, signal: AbortSignal.timeout(options.timeoutMs ?? DEFAULT_TIMEOUT_MS) });
|
|
60
|
+
}
|
|
61
|
+
catch (error) {
|
|
62
|
+
// The cause names the host or the errno, never a header.
|
|
63
|
+
const cause = error instanceof Error ? (error.cause instanceof Error ? error.cause.message : error.message) : String(error);
|
|
64
|
+
throw new S3RequestError(`${operation} could not reach ${url.host} (${cause})`, null, null);
|
|
65
|
+
}
|
|
66
|
+
if (!response.ok)
|
|
67
|
+
throw await toError(operation, response);
|
|
68
|
+
return response;
|
|
69
|
+
}
|
|
70
|
+
function sign(store, method, url, payloadSha256, options, headers) {
|
|
71
|
+
return signRequest({
|
|
72
|
+
method,
|
|
73
|
+
url,
|
|
74
|
+
payloadSha256,
|
|
75
|
+
region: store.region,
|
|
76
|
+
accessKeyId: store.accessKeyId,
|
|
77
|
+
secretAccessKey: store.secretAccessKey,
|
|
78
|
+
now: (options.now ?? (() => new Date()))(),
|
|
79
|
+
...(headers === undefined ? {} : { headers }),
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
/** PUTs the file at `path` (its sha256 and size known) as `key`. */
|
|
83
|
+
export async function putObjectFile(store, input, options = {}) {
|
|
84
|
+
const url = getObjectUrl(store, input.key);
|
|
85
|
+
const headers = sign(store, "PUT", url, input.sha256, options, { "content-type": "application/octet-stream" });
|
|
86
|
+
const body = Readable.toWeb(createReadStream(input.path));
|
|
87
|
+
await send(`PUT ${input.key}`, url, { method: "PUT", headers: { ...headers, "content-length": String(input.bytes) }, body, duplex: "half" }, options);
|
|
88
|
+
}
|
|
89
|
+
/** The body of `key` as a stream. */
|
|
90
|
+
export async function getObjectStream(store, key, options = {}) {
|
|
91
|
+
const url = getObjectUrl(store, key);
|
|
92
|
+
const response = await send(`GET ${key}`, url, { method: "GET", headers: sign(store, "GET", url, EMPTY_SHA256, options) }, options);
|
|
93
|
+
if (response.body === null)
|
|
94
|
+
throw new S3RequestError(`GET ${key} answered without a body`, response.status, null);
|
|
95
|
+
return response.body;
|
|
96
|
+
}
|
|
97
|
+
/** Every object under `prefix/`, through ListObjectsV2 pages. */
|
|
98
|
+
export async function listObjects(store, prefix, options = {}) {
|
|
99
|
+
const objects = [];
|
|
100
|
+
let token = null;
|
|
101
|
+
do {
|
|
102
|
+
const url = getObjectUrl(store, null);
|
|
103
|
+
url.searchParams.set("list-type", "2");
|
|
104
|
+
url.searchParams.set("prefix", prefix);
|
|
105
|
+
if (token !== null)
|
|
106
|
+
url.searchParams.set("continuation-token", token);
|
|
107
|
+
const response = await send(`LIST ${prefix}`, url, { method: "GET", headers: sign(store, "GET", url, EMPTY_SHA256, options) }, options);
|
|
108
|
+
const xml = await response.text();
|
|
109
|
+
for (const match of xml.matchAll(/<Contents>([\s\S]*?)<\/Contents>/g)) {
|
|
110
|
+
const entry = match[1] ?? "";
|
|
111
|
+
const key = readXmlTag(entry, "Key");
|
|
112
|
+
if (key !== null)
|
|
113
|
+
objects.push({ key, bytes: Number(readXmlTag(entry, "Size") ?? 0) });
|
|
114
|
+
}
|
|
115
|
+
token = readXmlTag(xml, "IsTruncated") === "true" ? readXmlTag(xml, "NextContinuationToken") : null;
|
|
116
|
+
} while (token !== null);
|
|
117
|
+
return objects;
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=s3.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"s3.js","sourceRoot":"","sources":["../../src/offsite/s3.ts"],"names":[],"mappings":"AAAA,mHAAmH;AACnH,gHAAgH;AAChH,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAEtE,8FAA8F;AAC9F,MAAM,OAAO,cAAe,SAAQ,KAAK;IAC9B,MAAM,CAAgB;IACtB,MAAM,CAAgB;IAE/B,YAAY,OAAe,EAAE,MAAqB,EAAE,MAAqB;QACvE,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,+DAA+D;IAC/D,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,CAAC;IACpD,CAAC;CACF;AAeD,MAAM,kBAAkB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC1C,MAAM,cAAc,GAAG,GAAG,CAAC;AAE3B,wGAAwG;AACxG,SAAS,mBAAmB,CAAC,IAAY;IACvC,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC;IACtB,OAAO,GAAG,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,EAAE;QAAE,GAAG,IAAI,CAAC,CAAC;IAC5D,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC5B,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,YAAY,CAAC,KAAkB,EAAE,GAAkB;IACjE,MAAM,IAAI,GAAG,mBAAmB,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC1D,MAAM,IAAI,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC;IAC5I,OAAO,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC,CAAC;AACpD,CAAC;AAED,SAAS,UAAU,CAAC,GAAW,EAAE,GAAW;IAC1C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,IAAI,GAAG,aAAa,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/D,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI;SACR,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;AAC5B,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,SAAiB,EAAE,QAAkB;IAC1D,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;IACrE,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5G,OAAO,IAAI,cAAc,CAAC,GAAG,SAAS,kBAAkB,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,EAAE,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AAC9I,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,SAAiB,EAAE,GAAQ,EAAE,IAAiB,EAAE,OAAkB;IACpF,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC;IACvC,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC,EAAE,CAAC,CAAC;IACnH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,yDAAyD;QACzD,MAAM,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC5H,MAAM,IAAI,cAAc,CAAC,GAAG,SAAS,oBAAoB,GAAG,CAAC,IAAI,KAAK,KAAK,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;IAC9F,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,MAAM,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAC3D,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,IAAI,CAAC,KAAkB,EAAE,MAAqB,EAAE,GAAQ,EAAE,aAAqB,EAAE,OAAkB,EAAE,OAAgC;IAC5I,OAAO,WAAW,CAAC;QACjB,MAAM;QACN,GAAG;QACH,aAAa;QACb,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,EAAE;QAC1C,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;KAC9C,CAAC,CAAC;AACL,CAAC;AAED,oEAAoE;AACpE,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAkB,EAClB,KAAmE,EACnE,OAAO,GAAc,EAAE;IAEvB,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,cAAc,EAAE,0BAA0B,EAAE,CAAC,CAAC;IAC/G,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAA+B,CAAC;IACxF,MAAM,IAAI,CAAC,OAAO,KAAK,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,gBAAgB,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAiB,EAAE,OAAO,CAAC,CAAC;AACvK,CAAC;AAED,qCAAqC;AACrC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,KAAkB,EAAE,GAAW,EAAE,OAAO,GAAc,EAAE;IAC5F,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,GAAG,EAAE,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACpI,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI;QAAE,MAAM,IAAI,cAAc,CAAC,OAAO,GAAG,0BAA0B,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAClH,OAAO,QAAQ,CAAC,IAAI,CAAC;AACvB,CAAC;AAED,iEAAiE;AACjE,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,KAAkB,EAAE,MAAc,EAAE,OAAO,GAAc,EAAE;IAC3F,MAAM,OAAO,GAAmB,EAAE,CAAC;IACnC,IAAI,KAAK,GAAkB,IAAI,CAAC;IAChC,GAAG,CAAC;QACF,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACtC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;QACvC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACvC,IAAI,KAAK,KAAK,IAAI;YAAE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAC;QACtE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QACxI,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAClC,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,mCAAmC,CAAC,EAAE,CAAC;YACtE,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YACrC,IAAI,GAAG,KAAK,IAAI;gBAAE,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;QACzF,CAAC;QACD,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,uBAAuB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACtG,CAAC,QAAQ,KAAK,KAAK,IAAI,EAAE;IACzB,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export declare const EMPTY_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
|
|
2
|
+
export interface SignInput {
|
|
3
|
+
readonly method: "GET" | "PUT" | "HEAD";
|
|
4
|
+
readonly url: URL;
|
|
5
|
+
/** Headers to send and sign besides host, x-amz-date and x-amz-content-sha256; names in any case. */
|
|
6
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
7
|
+
readonly payloadSha256: string;
|
|
8
|
+
readonly region: string;
|
|
9
|
+
readonly accessKeyId: string;
|
|
10
|
+
readonly secretAccessKey: string;
|
|
11
|
+
readonly now: Date;
|
|
12
|
+
}
|
|
13
|
+
/** RFC 3986 encoding of every byte except the unreserved ones, as SigV4 wants; `/` kept when `keepSlash`. */
|
|
14
|
+
export declare function encodeRfc3986(text: string, keepSlash?: boolean): string;
|
|
15
|
+
/** `20130524T000000Z` */
|
|
16
|
+
export declare function formatAmzDate(now: Date): string;
|
|
17
|
+
/** The headers to send, the `Authorization` one included. */
|
|
18
|
+
export declare function signRequest(input: SignInput): Record<string, string>;
|
|
19
|
+
//# sourceMappingURL=sigv4.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sigv4.d.ts","sourceRoot":"","sources":["../../src/offsite/sigv4.ts"],"names":[],"mappings":"AAKA,eAAO,MAAM,YAAY,qEAAqE,CAAC;AAE/F,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,KAAK,GAAG,KAAK,GAAG,MAAM,CAAC;IACxC,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC;IAClB,qGAAqG;IACrG,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC;CACpB;AAED,6GAA6G;AAC7G,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,UAAQ,GAAG,MAAM,CAQrE;AAED,yBAAyB;AACzB,wBAAgB,aAAa,CAAC,GAAG,EAAE,IAAI,GAAG,MAAM,CAE/C;AAkBD,6DAA6D;AAC7D,wBAAgB,WAAW,CAAC,KAAK,EAAE,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CA+BpE"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// AWS Signature Version 4 for the three S3 requests this package sends (PUT, GET, ListObjectsV2), path style, so it
|
|
2
|
+
// works the same with Cloudflare R2, Backblaze B2, Hetzner Object Storage, MinIO and AWS. The payload hash is the
|
|
3
|
+
// real sha256 of the body (the empty body's for GET and list), never UNSIGNED-PAYLOAD.
|
|
4
|
+
import { createHash, createHmac } from "node:crypto";
|
|
5
|
+
export const EMPTY_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
|
|
6
|
+
/** RFC 3986 encoding of every byte except the unreserved ones, as SigV4 wants; `/` kept when `keepSlash`. */
|
|
7
|
+
export function encodeRfc3986(text, keepSlash = false) {
|
|
8
|
+
let encoded = "";
|
|
9
|
+
for (const byte of Buffer.from(text, "utf8")) {
|
|
10
|
+
const char = String.fromCharCode(byte);
|
|
11
|
+
if (/[A-Za-z0-9\-._~]/.test(char) || (keepSlash && char === "/"))
|
|
12
|
+
encoded += char;
|
|
13
|
+
else
|
|
14
|
+
encoded += `%${byte.toString(16).toUpperCase().padStart(2, "0")}`;
|
|
15
|
+
}
|
|
16
|
+
return encoded;
|
|
17
|
+
}
|
|
18
|
+
/** `20130524T000000Z` */
|
|
19
|
+
export function formatAmzDate(now) {
|
|
20
|
+
return now.toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z");
|
|
21
|
+
}
|
|
22
|
+
function sha256Hex(text) {
|
|
23
|
+
return createHash("sha256").update(text, "utf8").digest("hex");
|
|
24
|
+
}
|
|
25
|
+
function hmac(key, text) {
|
|
26
|
+
return createHmac("sha256", key).update(text, "utf8").digest();
|
|
27
|
+
}
|
|
28
|
+
function getCanonicalQuery(url) {
|
|
29
|
+
return [...url.searchParams.entries()]
|
|
30
|
+
.map(([key, value]) => [encodeRfc3986(key), encodeRfc3986(value)])
|
|
31
|
+
.sort(([left, leftValue], [right, rightValue]) => (left === right ? (leftValue < rightValue ? -1 : 1) : left < right ? -1 : 1))
|
|
32
|
+
.map(([key, value]) => `${key}=${value}`)
|
|
33
|
+
.join("&");
|
|
34
|
+
}
|
|
35
|
+
/** The headers to send, the `Authorization` one included. */
|
|
36
|
+
export function signRequest(input) {
|
|
37
|
+
const amzDate = formatAmzDate(input.now);
|
|
38
|
+
const day = amzDate.slice(0, 8);
|
|
39
|
+
const headers = {
|
|
40
|
+
host: input.url.host,
|
|
41
|
+
"x-amz-content-sha256": input.payloadSha256,
|
|
42
|
+
"x-amz-date": amzDate,
|
|
43
|
+
};
|
|
44
|
+
for (const [name, value] of Object.entries(input.headers ?? {}))
|
|
45
|
+
headers[name.toLowerCase()] = value.trim().replace(/\s+/g, " ");
|
|
46
|
+
const names = Object.keys(headers).sort();
|
|
47
|
+
const canonicalHeaders = names.map((name) => `${name}:${headers[name] ?? ""}\n`).join("");
|
|
48
|
+
const signedHeaders = names.join(";");
|
|
49
|
+
// The path as the URL holds it is already percent-encoded once; S3 signs it without a second encoding.
|
|
50
|
+
const canonicalRequest = [
|
|
51
|
+
input.method,
|
|
52
|
+
input.url.pathname,
|
|
53
|
+
getCanonicalQuery(input.url),
|
|
54
|
+
canonicalHeaders,
|
|
55
|
+
signedHeaders,
|
|
56
|
+
input.payloadSha256,
|
|
57
|
+
].join("\n");
|
|
58
|
+
const scope = `${day}/${input.region}/s3/aws4_request`;
|
|
59
|
+
const stringToSign = ["AWS4-HMAC-SHA256", amzDate, scope, sha256Hex(canonicalRequest)].join("\n");
|
|
60
|
+
const signingKey = hmac(hmac(hmac(hmac(`AWS4${input.secretAccessKey}`, day), input.region), "s3"), "aws4_request");
|
|
61
|
+
const signature = createHmac("sha256", signingKey).update(stringToSign, "utf8").digest("hex");
|
|
62
|
+
// fetch sets Host itself from the URL.
|
|
63
|
+
const sent = Object.fromEntries(Object.entries(headers).filter(([name]) => name !== "host"));
|
|
64
|
+
return {
|
|
65
|
+
...sent,
|
|
66
|
+
authorization: `AWS4-HMAC-SHA256 Credential=${input.accessKeyId}/${scope}, SignedHeaders=${signedHeaders}, Signature=${signature}`,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=sigv4.js.map
|