@rebasepro/server-postgres 0.10.0 → 0.10.1-canary.14e53ae
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/PostgresBootstrapper.d.ts +7 -3
- package/dist/auth/services.d.ts +43 -4
- package/dist/backup/backup-logic.d.ts +23 -0
- package/dist/backup/backup-service.d.ts +44 -2
- package/dist/backup/pg-tools.d.ts +41 -1
- package/dist/chunk-DSJWtz9O.js +40 -0
- package/dist/cli-helpers.d.ts +33 -1
- package/dist/ensure-collection-tables-CNlIONzj.js +304 -0
- package/dist/ensure-collection-tables-CNlIONzj.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.es.js +1472 -4640
- package/dist/index.es.js.map +1 -1
- package/dist/schema/auth-schema.d.ts +170 -0
- package/dist/schema/destructive-sql.d.ts +49 -0
- package/dist/schema/ensure-collection-tables.d.ts +79 -0
- package/dist/schema/generate-postgres-ddl-logic.d.ts +4 -1
- package/dist/services/cdc/CdcListener.d.ts +7 -14
- package/dist/services/channel-bus/ChannelBus.d.ts +29 -0
- package/dist/services/channel-bus/PostgresChannelBus.d.ts +111 -0
- package/dist/services/channel-bus/index.d.ts +55 -0
- package/dist/services/channel-history.d.ts +11 -0
- package/dist/services/channel-presence.d.ts +66 -0
- package/dist/services/pg-notify-listener.d.ts +47 -0
- package/dist/services/realtimeService.d.ts +114 -6
- package/dist/src-B0v4IKaI.js +329 -0
- package/dist/src-B0v4IKaI.js.map +1 -0
- package/dist/src-DmsRg8MR.js +4056 -0
- package/dist/src-DmsRg8MR.js.map +1 -0
- package/package.json +6 -6
- package/src/PostgresBootstrapper.ts +72 -3
- package/src/auth/ensure-tables.ts +91 -3
- package/src/auth/services.ts +186 -48
- package/src/backup/backup-cli.ts +60 -1
- package/src/backup/backup-cron.ts +24 -1
- package/src/backup/backup-logic.ts +62 -0
- package/src/backup/backup-service.ts +132 -13
- package/src/backup/pg-tools.ts +70 -2
- package/src/cli-helpers.ts +82 -27
- package/src/cli.ts +152 -6
- package/src/index.ts +4 -0
- package/src/schema/auth-schema.ts +41 -3
- package/src/schema/destructive-sql.ts +94 -0
- package/src/schema/ensure-collection-tables.test.ts +156 -0
- package/src/schema/ensure-collection-tables.ts +297 -0
- package/src/schema/generate-postgres-ddl-logic.ts +3 -3
- package/src/services/cdc/CdcListener.ts +27 -91
- package/src/services/channel-bus/ChannelBus.ts +44 -0
- package/src/services/channel-bus/PostgresChannelBus.ts +299 -0
- package/src/services/channel-bus/index.ts +123 -0
- package/src/services/channel-history.ts +35 -0
- package/src/services/channel-presence.ts +148 -0
- package/src/services/pg-notify-listener.ts +137 -0
- package/src/services/realtimeService.ts +383 -14
package/src/backup/backup-cli.ts
CHANGED
|
@@ -13,19 +13,22 @@ import { logger } from "@rebasepro/server";
|
|
|
13
13
|
import type { StorageController } from "@rebasepro/server";
|
|
14
14
|
import {
|
|
15
15
|
BackupDestination,
|
|
16
|
+
globalsFileForDump,
|
|
16
17
|
parseBackupDestination,
|
|
17
18
|
parseDbNameFromUrl,
|
|
18
19
|
resolveConnectionString,
|
|
19
20
|
withDatabaseName
|
|
20
21
|
} from "./pg-tools";
|
|
21
22
|
import {
|
|
23
|
+
applyGlobals,
|
|
22
24
|
BackupToolError,
|
|
23
25
|
createDump,
|
|
24
26
|
ensureDatabaseExists,
|
|
25
27
|
listBackups,
|
|
26
28
|
preflight,
|
|
27
29
|
restoreDump,
|
|
28
|
-
uploadBackup
|
|
30
|
+
uploadBackup,
|
|
31
|
+
validateDump
|
|
29
32
|
} from "./backup-service";
|
|
30
33
|
|
|
31
34
|
function formatBytes(bytes: number): string {
|
|
@@ -140,8 +143,12 @@ export async function backupCommand(rawArgs: string[]): Promise<void> {
|
|
|
140
143
|
noOwner: args["--no-owner"],
|
|
141
144
|
inheritStdio: true
|
|
142
145
|
});
|
|
146
|
+
await assertDumpValid(dump.localFile);
|
|
143
147
|
logger.info("");
|
|
144
148
|
logger.info(chalk.green(` ✓ Backup written to ${dump.localFile} (${formatBytes(dump.sizeBytes)})`));
|
|
149
|
+
if (dump.globalsFile) {
|
|
150
|
+
logger.info(chalk.gray(` ✓ Roles captured to ${dump.globalsFile} (needed so RLS survives a restore).`));
|
|
151
|
+
}
|
|
145
152
|
} else {
|
|
146
153
|
const storage = await resolveStorageForDestination(dest, process.env);
|
|
147
154
|
const dump = await createDump({
|
|
@@ -152,12 +159,20 @@ export async function backupCommand(rawArgs: string[]): Promise<void> {
|
|
|
152
159
|
inheritStdio: true
|
|
153
160
|
});
|
|
154
161
|
try {
|
|
162
|
+
await assertDumpValid(dump.localFile);
|
|
155
163
|
const uploaded = await uploadBackup(storage!, dump.localFile, dest);
|
|
156
164
|
logger.info("");
|
|
157
165
|
logger.info(chalk.green(` ✓ Backup uploaded to ${uploaded.storageUrl} (${formatBytes(dump.sizeBytes)})`));
|
|
166
|
+
// Upload the roles sidecar next to the dump so a restore can
|
|
167
|
+
// recreate roles the dump's GRANT/RLS statements depend on.
|
|
168
|
+
if (dump.globalsFile && fs.existsSync(dump.globalsFile)) {
|
|
169
|
+
const globalsUpload = await uploadBackup(storage!, dump.globalsFile, dest);
|
|
170
|
+
logger.info(chalk.gray(` ✓ Roles uploaded to ${globalsUpload.storageUrl} (needed so RLS survives a restore).`));
|
|
171
|
+
}
|
|
158
172
|
logger.info(chalk.gray(" Ensure this bucket is private — backups may contain secrets and PII."));
|
|
159
173
|
} finally {
|
|
160
174
|
if (fs.existsSync(dump.localFile)) fs.unlinkSync(dump.localFile);
|
|
175
|
+
if (dump.globalsFile && fs.existsSync(dump.globalsFile)) fs.unlinkSync(dump.globalsFile);
|
|
161
176
|
}
|
|
162
177
|
}
|
|
163
178
|
logger.info("");
|
|
@@ -177,6 +192,7 @@ export async function restoreCommand(rawArgs: string[]): Promise<void> {
|
|
|
177
192
|
"--create-db": Boolean,
|
|
178
193
|
"--clean": Boolean,
|
|
179
194
|
"--no-owner": Boolean,
|
|
195
|
+
"--continue-on-error": Boolean,
|
|
180
196
|
"--yes": Boolean,
|
|
181
197
|
"-y": "--yes"
|
|
182
198
|
},
|
|
@@ -206,7 +222,11 @@ export async function restoreCommand(rawArgs: string[]): Promise<void> {
|
|
|
206
222
|
logger.info("");
|
|
207
223
|
|
|
208
224
|
// Resolve the local file to restore from (download object-storage keys).
|
|
225
|
+
// Also resolve the `.globals.sql` roles sidecar so cluster roles can be
|
|
226
|
+
// recreated before the restore — without them the dump's GRANT/RLS
|
|
227
|
+
// statements fail and RLS is silently lost.
|
|
209
228
|
let localFile: string;
|
|
229
|
+
let globalsSql: string | null = null;
|
|
210
230
|
let cleanupTemp = false;
|
|
211
231
|
try {
|
|
212
232
|
if (/^(s3|gs):\/\//.test(backupArg)) {
|
|
@@ -221,12 +241,16 @@ export async function restoreCommand(rawArgs: string[]): Promise<void> {
|
|
|
221
241
|
localFile = path.join(tmpDir, path.basename(key));
|
|
222
242
|
fs.writeFileSync(localFile, Buffer.from(await file.arrayBuffer()));
|
|
223
243
|
cleanupTemp = true;
|
|
244
|
+
const globalsObj = await storage.getObject(globalsFileForDump(key), bucket);
|
|
245
|
+
if (globalsObj) globalsSql = Buffer.from(await globalsObj.arrayBuffer()).toString("utf-8");
|
|
224
246
|
} else {
|
|
225
247
|
localFile = path.resolve(backupArg);
|
|
226
248
|
if (!fs.existsSync(localFile)) {
|
|
227
249
|
logger.error(chalk.red(` ✗ Backup file not found: ${localFile}`));
|
|
228
250
|
process.exit(1);
|
|
229
251
|
}
|
|
252
|
+
const globalsPath = globalsFileForDump(localFile);
|
|
253
|
+
if (fs.existsSync(globalsPath)) globalsSql = fs.readFileSync(globalsPath, "utf-8");
|
|
230
254
|
}
|
|
231
255
|
|
|
232
256
|
// Version pre-flight. Check against the base connection — the server
|
|
@@ -261,11 +285,29 @@ export async function restoreCommand(rawArgs: string[]): Promise<void> {
|
|
|
261
285
|
}
|
|
262
286
|
}
|
|
263
287
|
|
|
288
|
+
// Recreate cluster roles before restoring so GRANT/RLS statements in
|
|
289
|
+
// the dump apply. Best-effort and idempotent (see applyGlobals).
|
|
290
|
+
if (globalsSql) {
|
|
291
|
+
logger.info(chalk.gray(" Recreating cluster roles from the backup's roles sidecar…"));
|
|
292
|
+
const { applied, skipped } = await applyGlobals(
|
|
293
|
+
targetConnection,
|
|
294
|
+
globalsSql,
|
|
295
|
+
(m) => logger.info(chalk.gray(m))
|
|
296
|
+
);
|
|
297
|
+
logger.info(chalk.gray(` ✓ Roles: ${applied} applied, ${skipped} skipped (already present or not permitted).`));
|
|
298
|
+
} else {
|
|
299
|
+
logger.warn(chalk.yellow(" ⚠️ No roles sidecar (.globals.sql) accompanies this backup."));
|
|
300
|
+
logger.warn(chalk.yellow(" If the dump grants to roles that don't exist (e.g. rebase_user), the restore"));
|
|
301
|
+
logger.warn(chalk.yellow(" will fail — recreate those roles first, or use a backup that includes its globals."));
|
|
302
|
+
}
|
|
303
|
+
|
|
264
304
|
await restoreDump({
|
|
265
305
|
connectionString: targetConnection,
|
|
266
306
|
inputFile: localFile,
|
|
267
307
|
clean: args["--clean"],
|
|
268
308
|
noOwner: args["--no-owner"],
|
|
309
|
+
// Fail loudly by default so a skipped GRANT never leaves RLS off.
|
|
310
|
+
exitOnError: !args["--continue-on-error"],
|
|
269
311
|
inheritStdio: true
|
|
270
312
|
});
|
|
271
313
|
|
|
@@ -322,6 +364,17 @@ export async function backupsCommand(rawArgs: string[]): Promise<void> {
|
|
|
322
364
|
}
|
|
323
365
|
}
|
|
324
366
|
|
|
367
|
+
/** Verify a freshly written dump; abort the command if it looks corrupt. */
|
|
368
|
+
async function assertDumpValid(localFile: string): Promise<void> {
|
|
369
|
+
const check = await validateDump(localFile);
|
|
370
|
+
if (!check.ok) {
|
|
371
|
+
throw new BackupToolError(
|
|
372
|
+
`The backup failed validation and was not trusted: ${check.reason}`,
|
|
373
|
+
"The dump may be corrupt or truncated. Investigate before relying on it."
|
|
374
|
+
);
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
|
|
325
378
|
function reportError(err: unknown): void {
|
|
326
379
|
if (err instanceof BackupToolError) {
|
|
327
380
|
logger.error(chalk.red(` ✗ ${err.message}`));
|
|
@@ -364,12 +417,18 @@ ${chalk.green.bold("Options")}
|
|
|
364
417
|
${chalk.blue("--create-db")} Create the target database first if it doesn't exist
|
|
365
418
|
${chalk.blue("--clean")} Drop existing objects before recreating them
|
|
366
419
|
${chalk.blue("--no-owner")} Ignore ownership from the dump
|
|
420
|
+
${chalk.blue("--continue-on-error")} Log and continue past errors ${chalk.red("(may leave RLS un-enforced!)")}
|
|
367
421
|
${chalk.blue("--yes, -y")} Skip the interactive confirmation ${chalk.red("(destructive!)")}
|
|
368
422
|
|
|
369
423
|
${chalk.red.bold("Warning")}
|
|
370
424
|
Restore is destructive and never runs automatically. Without --yes it
|
|
371
425
|
requires an interactive 'yes'. Prefer --create-db/--target-db to restore
|
|
372
426
|
into a fresh database rather than overwriting a live one.
|
|
427
|
+
|
|
428
|
+
By default the restore aborts on the first error (--exit-on-error) so a
|
|
429
|
+
skipped GRANT never silently leaves RLS un-enforced. Roles are recreated
|
|
430
|
+
from the backup's .globals.sql sidecar first; keep that file next to the
|
|
431
|
+
dump. Use --continue-on-error only when you understand the consequences.
|
|
373
432
|
`);
|
|
374
433
|
}
|
|
375
434
|
|
|
@@ -125,7 +125,7 @@ export function createBackupCron(config: BackupCronConfig): CronJobDefinition {
|
|
|
125
125
|
// Backups of a large database can take a while; allow up to an hour.
|
|
126
126
|
timeoutSeconds: 3600,
|
|
127
127
|
async handler({ log }) {
|
|
128
|
-
const { createDump, pruneBackups, uploadBackup } = await import("./backup-service");
|
|
128
|
+
const { createDump, pruneBackups, uploadBackup, validateDump } = await import("./backup-service");
|
|
129
129
|
const { destination } = config;
|
|
130
130
|
|
|
131
131
|
if (destination.kind !== "local" && !config.storage) {
|
|
@@ -145,12 +145,32 @@ export function createBackupCron(config: BackupCronConfig): CronJobDefinition {
|
|
|
145
145
|
});
|
|
146
146
|
log(`Dump created: ${dump.fileName} (${formatBytes(dump.sizeBytes)})`);
|
|
147
147
|
|
|
148
|
+
// Validate BEFORE pruning: a corrupt-but-exit-0 dump must never be
|
|
149
|
+
// the reason the last good backup gets deleted.
|
|
150
|
+
const check = await validateDump(dump.localFile);
|
|
151
|
+
if (!check.ok) {
|
|
152
|
+
// Clean up the bad temp file for object destinations.
|
|
153
|
+
if (destination.kind !== "local" && fs.existsSync(dump.localFile)) {
|
|
154
|
+
fs.unlinkSync(dump.localFile);
|
|
155
|
+
}
|
|
156
|
+
if (dump.globalsFile && destination.kind !== "local" && fs.existsSync(dump.globalsFile)) {
|
|
157
|
+
fs.unlinkSync(dump.globalsFile);
|
|
158
|
+
}
|
|
159
|
+
throw new Error(`New backup failed validation — skipping upload and pruning to protect existing backups. ${check.reason}`);
|
|
160
|
+
}
|
|
161
|
+
|
|
148
162
|
let storedKey = dump.localFile;
|
|
149
163
|
try {
|
|
150
164
|
if (destination.kind !== "local") {
|
|
151
165
|
const uploaded = await uploadBackup(config.storage!, dump.localFile, destination);
|
|
152
166
|
storedKey = uploaded.storageUrl;
|
|
153
167
|
log(`Uploaded to ${uploaded.storageUrl}`);
|
|
168
|
+
// Upload the roles sidecar so a restore can recreate the
|
|
169
|
+
// roles the dump's GRANT/RLS statements depend on.
|
|
170
|
+
if (dump.globalsFile && fs.existsSync(dump.globalsFile)) {
|
|
171
|
+
const g = await uploadBackup(config.storage!, dump.globalsFile, destination);
|
|
172
|
+
log(`Uploaded roles sidecar to ${g.storageUrl}`);
|
|
173
|
+
}
|
|
154
174
|
}
|
|
155
175
|
} finally {
|
|
156
176
|
// For object-storage destinations the local dump was a temp
|
|
@@ -158,6 +178,9 @@ export function createBackupCron(config: BackupCronConfig): CronJobDefinition {
|
|
|
158
178
|
if (destination.kind !== "local" && fs.existsSync(dump.localFile)) {
|
|
159
179
|
fs.unlinkSync(dump.localFile);
|
|
160
180
|
}
|
|
181
|
+
if (dump.globalsFile && destination.kind !== "local" && fs.existsSync(dump.globalsFile)) {
|
|
182
|
+
fs.unlinkSync(dump.globalsFile);
|
|
183
|
+
}
|
|
161
184
|
}
|
|
162
185
|
|
|
163
186
|
let pruned: string[] = [];
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure-ish orchestration for the backup/restore paths, kept free of `execa`
|
|
3
|
+
* and `pg` value imports so it can be unit-tested under jest (the impure
|
|
4
|
+
* edges — spawning processes, opening connections — live in
|
|
5
|
+
* `backup-service.ts`, which is vitest/runtime only).
|
|
6
|
+
*
|
|
7
|
+
* The functions here take their side-effecting dependency as an argument
|
|
8
|
+
* (a statement runner, an object deleter) so tests can inject fakes.
|
|
9
|
+
*/
|
|
10
|
+
import { globalsFileForDump, splitGlobalsStatements } from "./pg-tools";
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Replay a `pg_dumpall --globals-only` script one statement at a time,
|
|
14
|
+
* tolerating per-statement failures. On a same-cluster restore the roles
|
|
15
|
+
* usually already exist (`CREATE ROLE` → "already exists") and on a managed
|
|
16
|
+
* provider an `ALTER ROLE <superuser>` may be refused; neither should abort
|
|
17
|
+
* recreation of the roles that *are* missing. Returns how many statements
|
|
18
|
+
* applied vs were skipped.
|
|
19
|
+
*
|
|
20
|
+
* `runStatement` executes one SQL statement and rejects on error.
|
|
21
|
+
*/
|
|
22
|
+
export async function applyGlobalsWith(
|
|
23
|
+
runStatement: (sql: string) => Promise<void>,
|
|
24
|
+
globalsSql: string,
|
|
25
|
+
log: (message: string) => void = () => {}
|
|
26
|
+
): Promise<{ applied: number; skipped: number }> {
|
|
27
|
+
let applied = 0;
|
|
28
|
+
let skipped = 0;
|
|
29
|
+
for (const statement of splitGlobalsStatements(globalsSql)) {
|
|
30
|
+
try {
|
|
31
|
+
await runStatement(statement);
|
|
32
|
+
applied++;
|
|
33
|
+
} catch (err) {
|
|
34
|
+
skipped++;
|
|
35
|
+
const firstLine = statement.split("\n")[0].slice(0, 80);
|
|
36
|
+
log(` • Skipped global: ${firstLine} (${err instanceof Error ? err.message : String(err)})`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return { applied, skipped };
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Delete each pruned dump together with its `.globals.sql` roles sidecar, so
|
|
44
|
+
* pruning never orphans the roles file. The sidecar is best-effort — older
|
|
45
|
+
* backups predate it, so a missing-sidecar failure is swallowed while a
|
|
46
|
+
* failure deleting the dump itself propagates.
|
|
47
|
+
*
|
|
48
|
+
* `deleteObject` removes one key and rejects if it cannot (e.g. not found).
|
|
49
|
+
*/
|
|
50
|
+
export async function pruneWith(
|
|
51
|
+
keys: string[],
|
|
52
|
+
deleteObject: (key: string) => Promise<void>
|
|
53
|
+
): Promise<void> {
|
|
54
|
+
for (const key of keys) {
|
|
55
|
+
await deleteObject(key);
|
|
56
|
+
try {
|
|
57
|
+
await deleteObject(globalsFileForDump(key));
|
|
58
|
+
} catch {
|
|
59
|
+
// Sidecar may not exist for older backups — ignore.
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
@@ -17,8 +17,11 @@ import {
|
|
|
17
17
|
BackupDestination,
|
|
18
18
|
buildBackupFilename,
|
|
19
19
|
buildPgDumpArgs,
|
|
20
|
+
buildPgDumpallGlobalsArgs,
|
|
20
21
|
buildPgRestoreArgs,
|
|
22
|
+
buildPgRestoreListArgs,
|
|
21
23
|
checkToolServerCompatibility,
|
|
24
|
+
globalsFileForDump,
|
|
22
25
|
joinStorageKey,
|
|
23
26
|
parsePgToolMajor,
|
|
24
27
|
parseBackupTimestamp,
|
|
@@ -26,6 +29,7 @@ import {
|
|
|
26
29
|
VersionCompatibility
|
|
27
30
|
} from "./pg-tools";
|
|
28
31
|
import { BackupObject, RetentionOptions, selectBackupsToPrune } from "./retention";
|
|
32
|
+
import { applyGlobalsWith, pruneWith } from "./backup-logic";
|
|
29
33
|
|
|
30
34
|
export class BackupToolError extends Error {
|
|
31
35
|
constructor(message: string, readonly hint?: string) {
|
|
@@ -34,13 +38,16 @@ export class BackupToolError extends Error {
|
|
|
34
38
|
}
|
|
35
39
|
}
|
|
36
40
|
|
|
37
|
-
/** Locate `pg_dump` / `pg_restore`, honouring an env override. */
|
|
41
|
+
/** Locate `pg_dump` / `pg_restore` / `pg_dumpall`, honouring an env override. */
|
|
38
42
|
export function resolvePgBinary(
|
|
39
|
-
tool: "pg_dump" | "pg_restore",
|
|
43
|
+
tool: "pg_dump" | "pg_restore" | "pg_dumpall",
|
|
40
44
|
env: Record<string, string | undefined> = process.env
|
|
41
45
|
): string | null {
|
|
42
|
-
const
|
|
43
|
-
|
|
46
|
+
const overrideVar =
|
|
47
|
+
tool === "pg_dump" ? env.PG_DUMP_PATH
|
|
48
|
+
: tool === "pg_restore" ? env.PG_RESTORE_PATH
|
|
49
|
+
: env.PG_DUMPALL_PATH;
|
|
50
|
+
if (overrideVar && fs.existsSync(overrideVar)) return overrideVar;
|
|
44
51
|
return resolveLocalBin(tool);
|
|
45
52
|
}
|
|
46
53
|
|
|
@@ -105,12 +112,23 @@ export interface BackupResult {
|
|
|
105
112
|
localFile: string;
|
|
106
113
|
fileName: string;
|
|
107
114
|
sizeBytes: number;
|
|
115
|
+
/**
|
|
116
|
+
* Absolute path of the `.globals.sql` sidecar holding cluster-wide roles
|
|
117
|
+
* (present unless globals capture was disabled or unavailable).
|
|
118
|
+
*/
|
|
119
|
+
globalsFile?: string;
|
|
120
|
+
globalsSizeBytes?: number;
|
|
108
121
|
}
|
|
109
122
|
|
|
110
123
|
/**
|
|
111
124
|
* Produce a custom-format dump on local disk. When `outDir` is omitted the
|
|
112
125
|
* file is written to the OS temp directory (used by the upload path, which
|
|
113
126
|
* cleans it up afterwards).
|
|
127
|
+
*
|
|
128
|
+
* Alongside the `-Fc` dump it writes a `<name>.globals.sql` sidecar via
|
|
129
|
+
* `pg_dumpall --globals-only` so the roles the dump's GRANT/RLS statements
|
|
130
|
+
* depend on can be recreated on restore. Set `includeGlobals: false` to skip
|
|
131
|
+
* it (e.g. when the caller has no privilege to read cluster globals).
|
|
114
132
|
*/
|
|
115
133
|
export async function createDump(opts: {
|
|
116
134
|
connectionString: string;
|
|
@@ -120,6 +138,7 @@ export async function createDump(opts: {
|
|
|
120
138
|
excludeSchemas?: string[];
|
|
121
139
|
noOwner?: boolean;
|
|
122
140
|
inheritStdio?: boolean;
|
|
141
|
+
includeGlobals?: boolean;
|
|
123
142
|
env?: Record<string, string | undefined>;
|
|
124
143
|
}): Promise<BackupResult> {
|
|
125
144
|
const env = opts.env ?? process.env;
|
|
@@ -149,7 +168,91 @@ export async function createDump(opts: {
|
|
|
149
168
|
});
|
|
150
169
|
|
|
151
170
|
const sizeBytes = fs.existsSync(localFile) ? fs.statSync(localFile).size : 0;
|
|
152
|
-
|
|
171
|
+
|
|
172
|
+
const result: BackupResult = { localFile, fileName, sizeBytes };
|
|
173
|
+
|
|
174
|
+
if (opts.includeGlobals !== false) {
|
|
175
|
+
const dumpallBin = resolvePgBinary("pg_dumpall", env);
|
|
176
|
+
if (!dumpallBin) {
|
|
177
|
+
throw new BackupToolError(
|
|
178
|
+
"Could not find the 'pg_dumpall' binary needed to capture cluster roles.",
|
|
179
|
+
"Install the PostgreSQL client tools or set PG_DUMPALL_PATH. " +
|
|
180
|
+
"To take a role-incomplete backup anyway, pass includeGlobals: false."
|
|
181
|
+
);
|
|
182
|
+
}
|
|
183
|
+
const globalsFile = globalsFileForDump(localFile);
|
|
184
|
+
await execa(
|
|
185
|
+
dumpallBin,
|
|
186
|
+
buildPgDumpallGlobalsArgs({ connectionString: opts.connectionString, outFile: globalsFile }),
|
|
187
|
+
{ stdio: opts.inheritStdio ? "inherit" : "pipe", env: { ...(env as Record<string, string>) } }
|
|
188
|
+
);
|
|
189
|
+
result.globalsFile = globalsFile;
|
|
190
|
+
result.globalsSizeBytes = fs.existsSync(globalsFile) ? fs.statSync(globalsFile).size : 0;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
return result;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Cheap integrity check on a freshly written dump: it must be non-empty and
|
|
198
|
+
* `pg_restore --list` must parse its table of contents without error. Used
|
|
199
|
+
* before pruning older backups so a corrupt-but-exit-0 dump never becomes
|
|
200
|
+
* the reason the last good backup is deleted.
|
|
201
|
+
*/
|
|
202
|
+
export async function validateDump(
|
|
203
|
+
localFile: string,
|
|
204
|
+
env: Record<string, string | undefined> = process.env
|
|
205
|
+
): Promise<{ ok: boolean; reason?: string }> {
|
|
206
|
+
if (!fs.existsSync(localFile)) {
|
|
207
|
+
return { ok: false, reason: `Dump file does not exist: ${localFile}` };
|
|
208
|
+
}
|
|
209
|
+
if (fs.statSync(localFile).size === 0) {
|
|
210
|
+
return { ok: false, reason: "Dump file is empty (0 bytes)." };
|
|
211
|
+
}
|
|
212
|
+
const bin = resolvePgBinary("pg_restore", env);
|
|
213
|
+
if (!bin) {
|
|
214
|
+
// Can't verify without pg_restore; treat as inconclusive-but-fail so
|
|
215
|
+
// pruning doesn't proceed on an unverified dump.
|
|
216
|
+
return { ok: false, reason: "Could not find 'pg_restore' to verify the dump." };
|
|
217
|
+
}
|
|
218
|
+
try {
|
|
219
|
+
await execa(bin, buildPgRestoreListArgs(localFile), {
|
|
220
|
+
stdio: "pipe",
|
|
221
|
+
env: { ...(env as Record<string, string>) }
|
|
222
|
+
});
|
|
223
|
+
return { ok: true };
|
|
224
|
+
} catch (err) {
|
|
225
|
+
return { ok: false, reason: `pg_restore --list failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Replay a `pg_dumpall --globals-only` script to recreate cluster roles
|
|
231
|
+
* before a restore, so the dump's GRANT/RLS statements (which reference
|
|
232
|
+
* `rebase_user` and any owner roles) actually apply. Runs statement by
|
|
233
|
+
* statement and tolerates per-statement failures — on a same-cluster restore
|
|
234
|
+
* the roles usually already exist (`CREATE ROLE` → "already exists"), and on
|
|
235
|
+
* a managed provider an `ALTER ROLE <superuser>` may be refused; neither
|
|
236
|
+
* should abort role recreation. Returns how many statements applied vs were
|
|
237
|
+
* skipped.
|
|
238
|
+
*/
|
|
239
|
+
export async function applyGlobals(
|
|
240
|
+
connectionString: string,
|
|
241
|
+
globalsSql: string,
|
|
242
|
+
log: (message: string) => void = () => {}
|
|
243
|
+
): Promise<{ applied: number; skipped: number }> {
|
|
244
|
+
const { Client } = await import("pg");
|
|
245
|
+
const client = new Client({ connectionString });
|
|
246
|
+
await client.connect();
|
|
247
|
+
try {
|
|
248
|
+
return await applyGlobalsWith(
|
|
249
|
+
async (sql) => { await client.query(sql); },
|
|
250
|
+
globalsSql,
|
|
251
|
+
log
|
|
252
|
+
);
|
|
253
|
+
} finally {
|
|
254
|
+
await client.end();
|
|
255
|
+
}
|
|
153
256
|
}
|
|
154
257
|
|
|
155
258
|
/**
|
|
@@ -157,12 +260,18 @@ export async function createDump(opts: {
|
|
|
157
260
|
* `connectionString`. Destructive when `clean` is set (drops objects
|
|
158
261
|
* first). Never called automatically — the CLI gates it behind explicit
|
|
159
262
|
* confirmation.
|
|
263
|
+
*
|
|
264
|
+
* Runs with `--exit-on-error` by default: a restore that logs-and-continues
|
|
265
|
+
* past a failed GRANT (because a role was missing) reports success with RLS
|
|
266
|
+
* un-enforced. Callers should recreate roles first (see {@link applyGlobals})
|
|
267
|
+
* and only set `exitOnError: false` deliberately.
|
|
160
268
|
*/
|
|
161
269
|
export async function restoreDump(opts: {
|
|
162
270
|
connectionString: string;
|
|
163
271
|
inputFile: string;
|
|
164
272
|
clean?: boolean;
|
|
165
273
|
noOwner?: boolean;
|
|
274
|
+
exitOnError?: boolean;
|
|
166
275
|
inheritStdio?: boolean;
|
|
167
276
|
env?: Record<string, string | undefined>;
|
|
168
277
|
}): Promise<void> {
|
|
@@ -178,7 +287,8 @@ export async function restoreDump(opts: {
|
|
|
178
287
|
connectionString: opts.connectionString,
|
|
179
288
|
inputFile: opts.inputFile,
|
|
180
289
|
clean: opts.clean,
|
|
181
|
-
noOwner: opts.noOwner
|
|
290
|
+
noOwner: opts.noOwner,
|
|
291
|
+
exitOnError: opts.exitOnError
|
|
182
292
|
});
|
|
183
293
|
await execa(bin, args, {
|
|
184
294
|
stdio: opts.inheritStdio ? "inherit" : "pipe",
|
|
@@ -288,12 +398,21 @@ export async function pruneBackups(
|
|
|
288
398
|
): Promise<string[]> {
|
|
289
399
|
const backups = await listBackups(dest, storage);
|
|
290
400
|
const toDelete = selectBackupsToPrune(backups, options);
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
401
|
+
|
|
402
|
+
// Delete each dump together with its `.globals.sql` sidecar (pruneWith
|
|
403
|
+
// handles the pairing + best-effort sidecar). The deleter rejects on a
|
|
404
|
+
// missing key so pruneWith can swallow an absent sidecar.
|
|
405
|
+
const deleteObject =
|
|
406
|
+
dest.kind === "local"
|
|
407
|
+
? async (key: string) => {
|
|
408
|
+
if (!fs.existsSync(key)) throw new Error(`not found: ${key}`);
|
|
409
|
+
fs.unlinkSync(key);
|
|
410
|
+
}
|
|
411
|
+
: async (key: string) => {
|
|
412
|
+
if (!storage) throw new BackupToolError("Storage backend required to prune object backups.");
|
|
413
|
+
await storage.deleteObject(key, dest.bucket);
|
|
414
|
+
};
|
|
415
|
+
|
|
416
|
+
await pruneWith(toDelete, deleteObject);
|
|
298
417
|
return toDelete;
|
|
299
418
|
}
|
package/src/backup/pg-tools.ts
CHANGED
|
@@ -202,7 +202,12 @@ export function buildPgRestoreArgs(opts: {
|
|
|
202
202
|
inputFile: string;
|
|
203
203
|
/** Drop objects before recreating them (destructive but idempotent). */
|
|
204
204
|
clean?: boolean;
|
|
205
|
-
/**
|
|
205
|
+
/**
|
|
206
|
+
* Abort on the first error instead of logging and continuing. Defaults
|
|
207
|
+
* ON: a restore that silently skips failed GRANT/RLS statements (because
|
|
208
|
+
* a role is missing) "succeeds" with RLS un-enforced — a security hole.
|
|
209
|
+
* Fail loudly instead so the operator knows the restore is incomplete.
|
|
210
|
+
*/
|
|
206
211
|
exitOnError?: boolean;
|
|
207
212
|
noOwner?: boolean;
|
|
208
213
|
}): string[] {
|
|
@@ -213,13 +218,76 @@ export function buildPgRestoreArgs(opts: {
|
|
|
213
218
|
if (opts.noOwner) {
|
|
214
219
|
args.push("--no-owner");
|
|
215
220
|
}
|
|
216
|
-
|
|
221
|
+
// Default to --exit-on-error unless explicitly disabled.
|
|
222
|
+
if (opts.exitOnError !== false) {
|
|
217
223
|
args.push("--exit-on-error");
|
|
218
224
|
}
|
|
219
225
|
args.push(opts.inputFile);
|
|
220
226
|
return args;
|
|
221
227
|
}
|
|
222
228
|
|
|
229
|
+
/**
|
|
230
|
+
* Assemble the `pg_restore --list` argument vector. Reading a dump's table
|
|
231
|
+
* of contents parses the whole archive without touching a database, so it is
|
|
232
|
+
* a cheap integrity check that the file isn't truncated or corrupt.
|
|
233
|
+
*/
|
|
234
|
+
export function buildPgRestoreListArgs(inputFile: string): string[] {
|
|
235
|
+
return ["--list", inputFile];
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Assemble the `pg_dumpall --globals-only` argument vector. Roles (and other
|
|
240
|
+
* cluster-wide objects) live outside any single database, so a per-database
|
|
241
|
+
* `pg_dump` omits them. Without the `rebase_user` role the RLS GRANT
|
|
242
|
+
* statements in the main dump fail on restore and RLS is silently lost — so
|
|
243
|
+
* every backup captures the globals into a sidecar `.globals.sql`.
|
|
244
|
+
*
|
|
245
|
+
* `--no-role-passwords` keeps role secrets out of the artifact (backups may
|
|
246
|
+
* be shipped off-box); roles are recreated password-less and re-secured by
|
|
247
|
+
* the operator.
|
|
248
|
+
*/
|
|
249
|
+
export function buildPgDumpallGlobalsArgs(opts: {
|
|
250
|
+
connectionString: string;
|
|
251
|
+
outFile: string;
|
|
252
|
+
}): string[] {
|
|
253
|
+
return [
|
|
254
|
+
"--globals-only",
|
|
255
|
+
"--no-role-passwords",
|
|
256
|
+
"--no-password",
|
|
257
|
+
`--file=${opts.outFile}`,
|
|
258
|
+
`--dbname=${opts.connectionString}`
|
|
259
|
+
];
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* Derive the globals sidecar path/key for a given `.dump` file. Keeps the
|
|
264
|
+
* two artifacts adjacent so listing, uploading and pruning can find one from
|
|
265
|
+
* the other. A name that doesn't end in `.dump` is returned unchanged with a
|
|
266
|
+
* `.globals.sql` suffix appended.
|
|
267
|
+
*/
|
|
268
|
+
export function globalsFileForDump(dumpPath: string): string {
|
|
269
|
+
return dumpPath.endsWith(".dump")
|
|
270
|
+
? dumpPath.slice(0, -".dump".length) + ".globals.sql"
|
|
271
|
+
: dumpPath + ".globals.sql";
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Split a `pg_dumpall --globals-only` script into individual statements.
|
|
276
|
+
* Used when replaying globals on restore so each `CREATE ROLE` / `GRANT`
|
|
277
|
+
* can run independently and a benign "role already exists" on one doesn't
|
|
278
|
+
* abort the rest. Drops `--` comment lines and blank statements.
|
|
279
|
+
*/
|
|
280
|
+
export function splitGlobalsStatements(sql: string): string[] {
|
|
281
|
+
const withoutComments = sql
|
|
282
|
+
.split("\n")
|
|
283
|
+
.filter((line) => !line.trim().startsWith("--"))
|
|
284
|
+
.join("\n");
|
|
285
|
+
return withoutComments
|
|
286
|
+
.split(";")
|
|
287
|
+
.map((s) => s.trim())
|
|
288
|
+
.filter((s) => s.length > 0);
|
|
289
|
+
}
|
|
290
|
+
|
|
223
291
|
/**
|
|
224
292
|
* Resolve the Postgres connection string the backup commands should use,
|
|
225
293
|
* mirroring the precedence the branch command already relies on.
|