@rebasepro/server-postgres 0.12.1-canary.gf5f1d39 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/PostgresBackendDriver.d.ts +1 -1
- package/dist/PostgresBootstrapper.d.ts +25 -1
- package/dist/auth/services.d.ts +21 -0
- package/dist/backup/backup-service.d.ts +10 -1
- package/dist/backup/pg-tools.d.ts +47 -0
- package/dist/backup-service-CD8o_1Sl.js +8999 -0
- package/dist/backup-service-CD8o_1Sl.js.map +1 -0
- package/dist/cli-helpers.d.ts +39 -0
- package/dist/connection-BuZ97wsr.js +250 -0
- package/dist/connection-BuZ97wsr.js.map +1 -0
- package/dist/connection.d.ts +42 -0
- package/dist/ensure-collection-policies-ViG8XiPn.js +57 -0
- package/dist/ensure-collection-policies-ViG8XiPn.js.map +1 -0
- package/dist/ensure-collection-tables-CBQdOETu.js +650 -0
- package/dist/ensure-collection-tables-CBQdOETu.js.map +1 -0
- package/dist/index.es.js +1231 -10031
- package/dist/index.es.js.map +1 -1
- package/dist/policy-CeA1JcxP.js +105 -0
- package/dist/policy-CeA1JcxP.js.map +1 -0
- package/dist/schema/auth-schema.d.ts +83 -144
- package/dist/schema/ensure-collection-policies.d.ts +60 -0
- package/dist/schema/ensure-collection-tables.d.ts +44 -2
- package/dist/schema/generate-postgres-ddl-logic.d.ts +135 -1
- package/dist/schema/introspect-db-constraints.d.ts +57 -0
- package/dist/schema/introspect-db-logic.d.ts +94 -5
- package/dist/schema/introspect-db-queries.d.ts +119 -0
- package/dist/schema/introspect-db-structure.d.ts +263 -0
- package/dist/schema/introspect-db-types.d.ts +11 -0
- package/dist/services/FetchService.d.ts +4 -1
- package/dist/services/RelationService.d.ts +24 -1
- package/dist/services/channel-bus/index.d.ts +1 -7
- package/dist/services/collection-helpers.d.ts +36 -2
- package/dist/services/dataService.d.ts +3 -1
- package/dist/services/row-pipeline.d.ts +1 -1
- package/dist/{src-BbFOPJ1S.js → src-DlPBctw_.js} +299 -173
- package/dist/src-DlPBctw_.js.map +1 -0
- package/dist/{src-Zqwaw3P5.js → src-DoU9yPqq.js} +3 -159
- package/dist/src-DoU9yPqq.js.map +1 -0
- package/dist/utils/connection-string.d.ts +29 -0
- package/dist/utils/drizzle-conditions.d.ts +5 -4
- package/dist/utils/pg-error-utils.d.ts +35 -0
- package/dist/websocket-B2LsrINK.js +530 -0
- package/dist/websocket-B2LsrINK.js.map +1 -0
- package/package.json +14 -14
- package/src/PostgresAdapter.ts +21 -2
- package/src/PostgresBackendDriver.ts +4 -0
- package/src/PostgresBootstrapper.ts +192 -33
- package/src/auth/ensure-tables.ts +164 -9
- package/src/auth/services.ts +24 -2
- package/src/backup/backup-cli.ts +41 -2
- package/src/backup/backup-service.ts +38 -5
- package/src/backup/pg-tools.ts +96 -3
- package/src/cli-helpers.ts +70 -0
- package/src/cli.ts +44 -26
- package/src/collections/validate-relations.ts +15 -0
- package/src/connection.ts +73 -0
- package/src/data-transformer.ts +9 -3
- package/src/databasePoolManager.ts +5 -2
- package/src/schema/auth-schema.ts +30 -19
- package/src/schema/ensure-collection-policies.ts +105 -0
- package/src/schema/ensure-collection-tables.test.ts +105 -9
- package/src/schema/ensure-collection-tables.ts +220 -32
- package/src/schema/generate-drizzle-schema-logic.ts +42 -6
- package/src/schema/generate-postgres-ddl-logic.ts +382 -19
- package/src/schema/introspect-db-constraints.ts +385 -0
- package/src/schema/introspect-db-inference.ts +18 -8
- package/src/schema/introspect-db-logic.ts +385 -71
- package/src/schema/introspect-db-queries.ts +326 -0
- package/src/schema/introspect-db-structure.ts +670 -0
- package/src/schema/introspect-db-types.ts +56 -0
- package/src/schema/introspect-db.ts +37 -80
- package/src/schema/introspect-runtime.test.ts +56 -8
- package/src/schema/introspect-runtime.ts +31 -9
- package/src/security/policy-drift.test.ts +11 -3
- package/src/services/BranchService.ts +66 -28
- package/src/services/FetchService.ts +90 -14
- package/src/services/PersistService.ts +20 -6
- package/src/services/RelationService.ts +249 -48
- package/src/services/channel-bus/index.ts +0 -9
- package/src/services/collection-helpers.ts +69 -3
- package/src/services/dataService.ts +3 -1
- package/src/services/realtimeService.ts +3 -3
- package/src/services/row-pipeline.ts +1 -1
- package/src/utils/connection-string.ts +58 -0
- package/src/utils/drizzle-conditions.ts +31 -6
- package/src/utils/pg-error-utils.ts +65 -0
- package/src/websocket.ts +18 -9
- package/dist/chunk-DSJWtz9O.js +0 -40
- package/dist/ensure-collection-tables-CNTcZGvn.js +0 -304
- package/dist/ensure-collection-tables-CNTcZGvn.js.map +0 -1
- package/dist/src-BbFOPJ1S.js.map +0 -1
- package/dist/src-Zqwaw3P5.js.map +0 -1
|
@@ -107,16 +107,38 @@ export async function ensureAuthTablesExist(db: NodePgDatabase, collection?: Col
|
|
|
107
107
|
? "GENERATED ALWAYS AS IDENTITY"
|
|
108
108
|
: "DEFAULT gen_random_uuid()::text";
|
|
109
109
|
|
|
110
|
+
// Identifiers for the constraint and indexes reconciled further down.
|
|
111
|
+
// Derived from the resolved table name so two auth tables in different
|
|
112
|
+
// schemas cannot collide, and truncated to Postgres's 63-byte identifier
|
|
113
|
+
// limit here rather than letting the server truncate silently — the
|
|
114
|
+
// `IF NOT EXISTS` guards below have to compare against the same name
|
|
115
|
+
// Postgres actually stored, or they re-run forever.
|
|
116
|
+
const authIdentifier = (suffix: string) => `${resolvedTable}_${suffix}`.slice(0, 63);
|
|
117
|
+
const emailLengthConstraint = `"${authIdentifier("email_length_check")}"`;
|
|
118
|
+
const emailLowerUniqueIndex = authIdentifier("email_lower_key");
|
|
119
|
+
const verificationTokenIndex = authIdentifier("email_verification_token_idx");
|
|
120
|
+
|
|
121
|
+
// Every string column here is TEXT, deliberately. In Postgres VARCHAR(n)
|
|
122
|
+
// and TEXT are the same type with the same storage and the same
|
|
123
|
+
// performance; the only difference is a length check, and none of these
|
|
124
|
+
// columns wants one. The widths this table used to carry were inherited
|
|
125
|
+
// MySQL habit (255) and they were all wrong in the same direction —
|
|
126
|
+
// `password_hash VARCHAR(255)` against a 193-char scrypt string left 62
|
|
127
|
+
// characters of headroom in front of a KEY_LENGTH constant living in
|
|
128
|
+
// another package, and `photo_url VARCHAR(500)` rejected the `data:` URIs
|
|
129
|
+
// and long signed URLs that OAuth providers hand back. A limit worth
|
|
130
|
+
// having is a CHECK — alterable without a table rewrite, unlike a type
|
|
131
|
+
// modifier — which is why `email` has one and nothing else does.
|
|
110
132
|
await db.execute(sql`
|
|
111
133
|
CREATE TABLE IF NOT EXISTS ${sql.raw(usersTableName)} (
|
|
112
134
|
id ${sql.raw(userIdType)} PRIMARY KEY ${sql.raw(idDefault)},
|
|
113
|
-
email
|
|
114
|
-
display_name
|
|
115
|
-
photo_url
|
|
135
|
+
email TEXT NOT NULL CONSTRAINT ${sql.raw(emailLengthConstraint)} CHECK (length(email) <= 320),
|
|
136
|
+
display_name TEXT,
|
|
137
|
+
photo_url TEXT,
|
|
116
138
|
roles TEXT[] DEFAULT '{}' NOT NULL,
|
|
117
|
-
password_hash
|
|
139
|
+
password_hash TEXT,
|
|
118
140
|
email_verified BOOLEAN DEFAULT FALSE NOT NULL,
|
|
119
|
-
email_verification_token
|
|
141
|
+
email_verification_token TEXT,
|
|
120
142
|
email_verification_sent_at TIMESTAMP WITH TIME ZONE,
|
|
121
143
|
is_anonymous BOOLEAN DEFAULT FALSE NOT NULL,
|
|
122
144
|
metadata JSONB DEFAULT '{}' NOT NULL,
|
|
@@ -359,12 +381,12 @@ export async function ensureAuthTablesExist(db: NodePgDatabase, collection?: Col
|
|
|
359
381
|
// statement that references it. `email` is deliberately absent: it has
|
|
360
382
|
// existed since the first era and cannot be added NOT NULL safely.
|
|
361
383
|
const userColumnBackfills = [
|
|
362
|
-
"display_name
|
|
363
|
-
"photo_url
|
|
384
|
+
"display_name TEXT",
|
|
385
|
+
"photo_url TEXT",
|
|
364
386
|
"roles TEXT[] DEFAULT '{}' NOT NULL",
|
|
365
|
-
"password_hash
|
|
387
|
+
"password_hash TEXT",
|
|
366
388
|
"email_verified BOOLEAN DEFAULT FALSE NOT NULL",
|
|
367
|
-
"email_verification_token
|
|
389
|
+
"email_verification_token TEXT",
|
|
368
390
|
"email_verification_sent_at TIMESTAMP WITH TIME ZONE",
|
|
369
391
|
"is_anonymous BOOLEAN DEFAULT FALSE NOT NULL",
|
|
370
392
|
"metadata JSONB DEFAULT '{}' NOT NULL",
|
|
@@ -379,6 +401,139 @@ export async function ensureAuthTablesExist(db: NodePgDatabase, collection?: Col
|
|
|
379
401
|
`);
|
|
380
402
|
}
|
|
381
403
|
|
|
404
|
+
// Which of the columns below the table actually has. An adopted table —
|
|
405
|
+
// one this framework did not create, which the column-name resolution in
|
|
406
|
+
// `services.ts` exists to support — may be missing any of them, and every
|
|
407
|
+
// statement past this point has to tolerate that rather than abort the
|
|
408
|
+
// whole migration block.
|
|
409
|
+
const usersColumns = await db.execute(sql`
|
|
410
|
+
SELECT column_name, data_type
|
|
411
|
+
FROM information_schema.columns
|
|
412
|
+
WHERE table_schema = ${usersSchema} AND table_name = ${resolvedTable}
|
|
413
|
+
`);
|
|
414
|
+
const usersColumnTypes = new Map(
|
|
415
|
+
(usersColumns.rows as { column_name: string; data_type: string }[])
|
|
416
|
+
.map(row => [row.column_name, row.data_type])
|
|
417
|
+
);
|
|
418
|
+
|
|
419
|
+
// ── Migration: VARCHAR(n) → TEXT on the users string columns ────────
|
|
420
|
+
// Tables created before the widths came off still carry them. Postgres
|
|
421
|
+
// treats varchar(n) → text as binary-coercible with no stricter
|
|
422
|
+
// constraint, so this is a catalogue-only change: no table rewrite, no
|
|
423
|
+
// index rebuild, just a brief ACCESS EXCLUSIVE lock. Guarded on the
|
|
424
|
+
// current type so it runs once and is a pure catalogue read thereafter.
|
|
425
|
+
for (const column of ["email", "display_name", "photo_url", "password_hash", "email_verification_token"]) {
|
|
426
|
+
if (usersColumnTypes.get(column) !== "character varying") continue;
|
|
427
|
+
await db.execute(sql`
|
|
428
|
+
ALTER TABLE ${sql.raw(usersTableName)}
|
|
429
|
+
ALTER COLUMN ${sql.raw(`"${column}"`)} TYPE TEXT
|
|
430
|
+
`);
|
|
431
|
+
logger.info(`🔧 Widened ${usersTableName}.${column} from VARCHAR(n) to TEXT`);
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
// ── Migration: case-insensitive email identity ──────────────────────
|
|
435
|
+
// `getUserByEmail` has always searched `email.toLowerCase()` while the
|
|
436
|
+
// write path stored whatever it was handed, leaving normalisation to a
|
|
437
|
+
// convention every caller had to remember. A row that reached the table
|
|
438
|
+
// with mixed case is then invisible to every lookup — the account exists,
|
|
439
|
+
// login reports no such user, and the plain UNIQUE on `email` does not
|
|
440
|
+
// stop a second row differing only in case, because it compares bytes.
|
|
441
|
+
//
|
|
442
|
+
// Fixed on both sides: `mapPayload` now folds on write, and this index
|
|
443
|
+
// makes the database agree. A unique index on lower(email) is strictly
|
|
444
|
+
// stronger than the byte-exact UNIQUE that older tables carry, so the
|
|
445
|
+
// old constraint is left alone — it can no longer fire on anything the
|
|
446
|
+
// new one would allow.
|
|
447
|
+
//
|
|
448
|
+
// Deliberately no AUTH_SCHEMA_VERSION bump: a runtime that predates this
|
|
449
|
+
// migration keeps working against the migrated table (all of its own
|
|
450
|
+
// write paths already lower-cased), which is exactly the additive case
|
|
451
|
+
// the version stamp is documented not to cover.
|
|
452
|
+
if (usersColumnTypes.has("email")) {
|
|
453
|
+
const indexPresent = await db.execute(sql`
|
|
454
|
+
SELECT 1 FROM pg_class c
|
|
455
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
456
|
+
WHERE n.nspname = ${usersSchema} AND c.relname = ${emailLowerUniqueIndex} AND c.relkind = 'i'
|
|
457
|
+
`);
|
|
458
|
+
if (indexPresent.rows.length === 0) {
|
|
459
|
+
// Case-collisions already in the table would make the unique
|
|
460
|
+
// index impossible to build. Report them and leave the table
|
|
461
|
+
// alone: the next boot retries, so fixing the rows is all the
|
|
462
|
+
// operator has to do. Failing loudly beats folding the emails
|
|
463
|
+
// and letting CREATE INDEX pick which account survives.
|
|
464
|
+
const collisions = await db.execute(sql`
|
|
465
|
+
SELECT lower(email) AS normalized, count(*)::int AS occurrences
|
|
466
|
+
FROM ${sql.raw(usersTableName)}
|
|
467
|
+
WHERE email IS NOT NULL
|
|
468
|
+
GROUP BY lower(email)
|
|
469
|
+
HAVING count(*) > 1
|
|
470
|
+
LIMIT 10
|
|
471
|
+
`);
|
|
472
|
+
if (collisions.rows.length > 0) {
|
|
473
|
+
const sample = (collisions.rows as { normalized: string; occurrences: number }[])
|
|
474
|
+
.map(row => `${row.normalized} (×${row.occurrences})`)
|
|
475
|
+
.join(", ");
|
|
476
|
+
logger.error(
|
|
477
|
+
`❌ Cannot enforce case-insensitive email uniqueness on ${usersTableName}: ` +
|
|
478
|
+
`these addresses already exist more than once, differing only in case — ${sample}. ` +
|
|
479
|
+
"Merge or delete the duplicates and restart; until then two accounts can share " +
|
|
480
|
+
"one address and only the lower-cased one is reachable by login."
|
|
481
|
+
);
|
|
482
|
+
} else {
|
|
483
|
+
const folded = await db.execute(sql`
|
|
484
|
+
UPDATE ${sql.raw(usersTableName)}
|
|
485
|
+
SET email = lower(email)
|
|
486
|
+
WHERE email IS NOT NULL AND email <> lower(email)
|
|
487
|
+
`);
|
|
488
|
+
if (folded.rowCount) {
|
|
489
|
+
logger.info(`🔧 Lower-cased ${folded.rowCount} email address(es) in ${usersTableName}`);
|
|
490
|
+
}
|
|
491
|
+
await db.execute(sql`
|
|
492
|
+
CREATE UNIQUE INDEX IF NOT EXISTS ${sql.raw(`"${emailLowerUniqueIndex}"`)}
|
|
493
|
+
ON ${sql.raw(usersTableName)} (lower(email))
|
|
494
|
+
`);
|
|
495
|
+
logger.info(`✅ Email uniqueness on ${usersTableName} is now case-insensitive`);
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
// ── Migration: bound the email column's length ──────────────────────
|
|
501
|
+
// The only length limit on this table worth keeping. 320 is the RFC 5321
|
|
502
|
+
// maximum (64-char local part + @ + 255-char domain), and it matters here
|
|
503
|
+
// beyond tidiness: `email` carries a btree index, and a sufficiently long
|
|
504
|
+
// value fails index insertion with an error that says nothing about
|
|
505
|
+
// email. NOT VALID so an adopted table with a long row still migrates —
|
|
506
|
+
// it binds all new writes, which is the part that matters.
|
|
507
|
+
if (usersColumnTypes.has("email")) {
|
|
508
|
+
const checkPresent = await db.execute(sql`
|
|
509
|
+
SELECT 1 FROM pg_constraint c
|
|
510
|
+
JOIN pg_class t ON t.oid = c.conrelid
|
|
511
|
+
JOIN pg_namespace n ON n.oid = t.relnamespace
|
|
512
|
+
WHERE n.nspname = ${usersSchema}
|
|
513
|
+
AND t.relname = ${resolvedTable}
|
|
514
|
+
AND c.conname = ${authIdentifier("email_length_check")}
|
|
515
|
+
`);
|
|
516
|
+
if (checkPresent.rows.length === 0) {
|
|
517
|
+
await db.execute(sql`
|
|
518
|
+
ALTER TABLE ${sql.raw(usersTableName)}
|
|
519
|
+
ADD CONSTRAINT ${sql.raw(emailLengthConstraint)} CHECK (length(email) <= 320) NOT VALID
|
|
520
|
+
`);
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
// ── Index: email verification token lookups ─────────────────────────
|
|
525
|
+
// `getUserByVerificationToken` filters on this column, which had no
|
|
526
|
+
// index — every click of a verification link was a sequential scan of
|
|
527
|
+
// the whole users table. Partial, because the column is NULL for every
|
|
528
|
+
// user who is not mid-verification, which is nearly all of them.
|
|
529
|
+
if (usersColumnTypes.has("email_verification_token")) {
|
|
530
|
+
await db.execute(sql`
|
|
531
|
+
CREATE INDEX IF NOT EXISTS ${sql.raw(`"${verificationTokenIndex}"`)}
|
|
532
|
+
ON ${sql.raw(usersTableName)} (email_verification_token)
|
|
533
|
+
WHERE email_verification_token IS NOT NULL
|
|
534
|
+
`);
|
|
535
|
+
}
|
|
536
|
+
|
|
382
537
|
// ── Migration: refresh_tokens become session-scoped, rotation-safe ──
|
|
383
538
|
// Two shapes are reconciled here, on EVERY table named refresh_tokens
|
|
384
539
|
// in whatever schema it lives (a database provisioned by an older era
|
package/src/auth/services.ts
CHANGED
|
@@ -54,6 +54,28 @@ function getColumn(table: RebasePgTable | undefined, ...keys: string[]): RebaseP
|
|
|
54
54
|
return key ? table[key] : undefined;
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
/**
|
|
58
|
+
* The single definition of what an email address looks like in storage.
|
|
59
|
+
*
|
|
60
|
+
* Reads have always folded case; writes did not, and normalising was left to
|
|
61
|
+
* each caller. That asymmetry is only ever one forgotten `.toLowerCase()` away
|
|
62
|
+
* from a row no lookup can find — the account exists, every sign-in path
|
|
63
|
+
* reports no such user, and the byte-exact UNIQUE on the column does not stop a
|
|
64
|
+
* duplicate differing only in case. Applied on both sides here so the guarantee
|
|
65
|
+
* belongs to the repository rather than to its callers' discipline; the
|
|
66
|
+
* `lower(email)` unique index added in `ensureAuthTablesExist` is the database
|
|
67
|
+
* half of the same rule.
|
|
68
|
+
*
|
|
69
|
+
* Whitespace goes too: a trailing space survives the fold and reproduces the
|
|
70
|
+
* problem exactly.
|
|
71
|
+
*
|
|
72
|
+
* Re-exported rather than defined here: `@rebasepro/server` and
|
|
73
|
+
* `@rebasepro/server-mongo` write this column too, and a second copy of this
|
|
74
|
+
* rule is the defect it exists to prevent.
|
|
75
|
+
*/
|
|
76
|
+
import { normalizeEmail } from "@rebasepro/common";
|
|
77
|
+
export { normalizeEmail };
|
|
78
|
+
|
|
57
79
|
/**
|
|
58
80
|
* PostgreSQL implementation of UserRepository.
|
|
59
81
|
* Handles all user-related database operations using Drizzle ORM.
|
|
@@ -184,7 +206,7 @@ export class UserService implements UserRepository {
|
|
|
184
206
|
const metadataKey = getColumnKey(this.usersTable, "metadata") || "metadata";
|
|
185
207
|
|
|
186
208
|
if ("id" in data) payload[idKey] = data.id;
|
|
187
|
-
if ("email" in data) payload[emailKey] = data.email;
|
|
209
|
+
if ("email" in data) payload[emailKey] = normalizeEmail(data.email);
|
|
188
210
|
if ("passwordHash" in data) payload[passwordHashKey] = data.passwordHash;
|
|
189
211
|
if ("displayName" in data) payload[displayNameKey] = data.displayName;
|
|
190
212
|
if ("photoUrl" in data) payload[photoUrlKey] = data.photoUrl;
|
|
@@ -244,7 +266,7 @@ export class UserService implements UserRepository {
|
|
|
244
266
|
async getUserByEmail(email: string): Promise<UserData | null> {
|
|
245
267
|
const emailCol = getColumn(this.usersTable, "email");
|
|
246
268
|
if (!emailCol) return null;
|
|
247
|
-
const [row] = await this.db.select().from(this.usersTable).where(eq(emailCol, email
|
|
269
|
+
const [row] = await this.db.select().from(this.usersTable).where(eq(emailCol, normalizeEmail(email)));
|
|
248
270
|
return row ? this.mapRowToUser(row as Record<string, unknown>) : null;
|
|
249
271
|
}
|
|
250
272
|
|
package/src/backup/backup-cli.ts
CHANGED
|
@@ -100,6 +100,8 @@ export async function backupCommand(rawArgs: string[]): Promise<void> {
|
|
|
100
100
|
"--out": String,
|
|
101
101
|
"--exclude-schema": [String],
|
|
102
102
|
"--no-owner": Boolean,
|
|
103
|
+
"--enable-row-security": Boolean,
|
|
104
|
+
"--row-security-role": String,
|
|
103
105
|
"-o": "--out"
|
|
104
106
|
},
|
|
105
107
|
{ argv: rawArgs.slice(2), permissive: true }
|
|
@@ -129,6 +131,23 @@ export async function backupCommand(rawArgs: string[]): Promise<void> {
|
|
|
129
131
|
}
|
|
130
132
|
logger.info(chalk.gray(` Using pg_dump ${pf.toolMajor} against server ${pf.serverMajor}.`));
|
|
131
133
|
|
|
134
|
+
// Opt-in, and loud. With row security on, pg_dump stops refusing to read
|
|
135
|
+
// rows it cannot see and starts leaving them out — an exit-0 backup that
|
|
136
|
+
// is quietly short. Anyone choosing that should know they chose it.
|
|
137
|
+
const rowSecurity = args["--enable-row-security"]
|
|
138
|
+
? { uid: "rebase-db-backup", roles: [args["--row-security-role"] || "admin"] }
|
|
139
|
+
: undefined;
|
|
140
|
+
|
|
141
|
+
if (rowSecurity) {
|
|
142
|
+
logger.warn("");
|
|
143
|
+
logger.warn(chalk.yellow(" ⚠ Dumping with row-level security ON."));
|
|
144
|
+
logger.warn(chalk.gray(` Reading as roles [${rowSecurity.roles.join(", ")}], which satisfies the generated`));
|
|
145
|
+
logger.warn(chalk.gray(" `admin_full_access` policy. This backup contains exactly the rows those"));
|
|
146
|
+
logger.warn(chalk.gray(" policies admit — a table whose policies have no admin rule comes out short,"));
|
|
147
|
+
logger.warn(chalk.gray(" and pg_dump will not say so. Prefer granting the dumping role BYPASSRLS."));
|
|
148
|
+
logger.warn("");
|
|
149
|
+
}
|
|
150
|
+
|
|
132
151
|
try {
|
|
133
152
|
if (dest.kind === "local") {
|
|
134
153
|
// Honour an explicit `…/name.dump` path; otherwise treat it as a
|
|
@@ -141,7 +160,8 @@ export async function backupCommand(rawArgs: string[]): Promise<void> {
|
|
|
141
160
|
fileName: explicitFile ? path.basename(dest.path) : undefined,
|
|
142
161
|
excludeSchemas: args["--exclude-schema"],
|
|
143
162
|
noOwner: args["--no-owner"],
|
|
144
|
-
inheritStdio: true
|
|
163
|
+
inheritStdio: true,
|
|
164
|
+
rowSecurity
|
|
145
165
|
});
|
|
146
166
|
await assertDumpValid(dump.localFile);
|
|
147
167
|
logger.info("");
|
|
@@ -156,7 +176,8 @@ export async function backupCommand(rawArgs: string[]): Promise<void> {
|
|
|
156
176
|
dbName,
|
|
157
177
|
excludeSchemas: args["--exclude-schema"],
|
|
158
178
|
noOwner: args["--no-owner"],
|
|
159
|
-
inheritStdio: true
|
|
179
|
+
inheritStdio: true,
|
|
180
|
+
rowSecurity
|
|
160
181
|
});
|
|
161
182
|
try {
|
|
162
183
|
await assertDumpValid(dump.localFile);
|
|
@@ -395,6 +416,24 @@ ${chalk.green.bold("Options")}
|
|
|
395
416
|
${chalk.blue("--out, -o")} <dest> Local path or s3://…/gs://… URL (default: ./backups)
|
|
396
417
|
${chalk.blue("--exclude-schema")} <s> Exclude a schema (repeatable)
|
|
397
418
|
${chalk.blue("--no-owner")} Omit ownership commands from the dump
|
|
419
|
+
${chalk.blue("--enable-row-security")} Dump as an admin subject instead of failing on RLS
|
|
420
|
+
${chalk.red("(may produce a partial dump — see below)")}
|
|
421
|
+
${chalk.blue("--row-security-role")} <r> Role to read as with the flag above (default: admin)
|
|
422
|
+
|
|
423
|
+
${chalk.green.bold("Row-level security")}
|
|
424
|
+
On a managed Postgres the dumping role usually owns nothing and has no
|
|
425
|
+
BYPASSRLS, so pg_dump refuses:
|
|
426
|
+
|
|
427
|
+
ERROR: query would be affected by row-level security policy for table "..."
|
|
428
|
+
|
|
429
|
+
That refusal is the safe behaviour. --enable-row-security replaces it by
|
|
430
|
+
reading as an admin subject: Rebase sets app.uid/app.user_roles so the
|
|
431
|
+
generated admin_full_access policy admits the dump. The dump then contains
|
|
432
|
+
exactly what those policies admit ${chalk.red("and no error is raised for what they do not")} —
|
|
433
|
+
a table whose policies lack an admin rule comes out short, silently.
|
|
434
|
+
|
|
435
|
+
Granting the dumping role BYPASSRLS is the option that keeps a backup
|
|
436
|
+
meaning "every row".
|
|
398
437
|
|
|
399
438
|
${chalk.green.bold("Notes")}
|
|
400
439
|
Backups may contain secrets and PII. Use private storage destinations and
|
|
@@ -18,6 +18,9 @@ import {
|
|
|
18
18
|
buildBackupFilename,
|
|
19
19
|
buildPgDumpArgs,
|
|
20
20
|
buildPgDumpallGlobalsArgs,
|
|
21
|
+
buildRowSecurityPgOptions,
|
|
22
|
+
diagnoseRowSecurityDumpFailure,
|
|
23
|
+
type RowSecurityIdentity,
|
|
21
24
|
buildPgRestoreArgs,
|
|
22
25
|
buildPgRestoreListArgs,
|
|
23
26
|
checkToolServerCompatibility,
|
|
@@ -140,6 +143,15 @@ export async function createDump(opts: {
|
|
|
140
143
|
inheritStdio?: boolean;
|
|
141
144
|
includeGlobals?: boolean;
|
|
142
145
|
env?: Record<string, string | undefined>;
|
|
146
|
+
/**
|
|
147
|
+
* Dump with row security left on, reading as this identity.
|
|
148
|
+
*
|
|
149
|
+
* The escape hatch for a managed Postgres, where the dumping role owns
|
|
150
|
+
* nothing and has no `BYPASSRLS`. Off by default, and deliberately so: with
|
|
151
|
+
* row security on, `pg_dump` stops erroring on rows it cannot see and
|
|
152
|
+
* simply omits them. See {@link RowSecurityIdentity}.
|
|
153
|
+
*/
|
|
154
|
+
rowSecurity?: RowSecurityIdentity;
|
|
143
155
|
}): Promise<BackupResult> {
|
|
144
156
|
const env = opts.env ?? process.env;
|
|
145
157
|
const bin = resolvePgBinary("pg_dump", env);
|
|
@@ -159,13 +171,34 @@ export async function createDump(opts: {
|
|
|
159
171
|
connectionString: opts.connectionString,
|
|
160
172
|
outFile: localFile,
|
|
161
173
|
excludeSchemas: opts.excludeSchemas,
|
|
162
|
-
noOwner: opts.noOwner
|
|
174
|
+
noOwner: opts.noOwner,
|
|
175
|
+
rowSecurity: opts.rowSecurity
|
|
163
176
|
});
|
|
164
177
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
}
|
|
178
|
+
// `PGOPTIONS` is the only way to set a GUC on a tool that takes no SQL.
|
|
179
|
+
// Built from the same object that added `--enable-row-security`, so the
|
|
180
|
+
// flag cannot travel without the identity that makes it safe.
|
|
181
|
+
const dumpEnv: Record<string, string> = { ...(env as Record<string, string>) };
|
|
182
|
+
if (opts.rowSecurity) {
|
|
183
|
+
dumpEnv.PGOPTIONS = [env.PGOPTIONS, buildRowSecurityPgOptions(opts.rowSecurity)]
|
|
184
|
+
.filter(Boolean).join(" ");
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
try {
|
|
188
|
+
await execa(bin, args, {
|
|
189
|
+
stdio: opts.inheritStdio ? "inherit" : "pipe",
|
|
190
|
+
env: dumpEnv
|
|
191
|
+
});
|
|
192
|
+
} catch (error) {
|
|
193
|
+
// The RLS failure names a table and no cause. Replace it with the
|
|
194
|
+
// cause and the two ways out; anything else is re-thrown untouched.
|
|
195
|
+
const diagnosis = diagnoseRowSecurityDumpFailure(error);
|
|
196
|
+
if (!diagnosis) throw error;
|
|
197
|
+
throw new BackupToolError(
|
|
198
|
+
diagnosis,
|
|
199
|
+
"Run `rebase db backup --help` for the flag, and read what it says about partial dumps."
|
|
200
|
+
);
|
|
201
|
+
}
|
|
169
202
|
|
|
170
203
|
const sizeBytes = fs.existsSync(localFile) ? fs.statSync(localFile).size : 0;
|
|
171
204
|
|
package/src/backup/pg-tools.ts
CHANGED
|
@@ -6,6 +6,8 @@
|
|
|
6
6
|
* not require a database.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
+
import { forLibpq } from "../utils/connection-string";
|
|
10
|
+
|
|
9
11
|
/**
|
|
10
12
|
* A parsed backup destination. `--out` (and the scheduled-backup config)
|
|
11
13
|
* accepts either a local filesystem path or an object-storage URL.
|
|
@@ -171,6 +173,45 @@ export function joinStorageKey(prefix: string, fileName: string): string {
|
|
|
171
173
|
return clean.length > 0 ? `${clean}/${fileName}` : fileName;
|
|
172
174
|
}
|
|
173
175
|
|
|
176
|
+
/**
|
|
177
|
+
* The identity `pg_dump` reads rows as, when row security is left on.
|
|
178
|
+
*
|
|
179
|
+
* Not optional, and that is the whole design. `pg_dump --enable-row-security`
|
|
180
|
+
* on its own is the dangerous command in this file: it turns the "query would
|
|
181
|
+
* be affected by row-level security policy" *error* into a dump that exits 0
|
|
182
|
+
* and is silently missing every row the dumping role's policies exclude. A
|
|
183
|
+
* backup that looks fine and restores most of your data is worse than one that
|
|
184
|
+
* refused to run.
|
|
185
|
+
*
|
|
186
|
+
* So the flag is unreachable without a subject to evaluate the policies
|
|
187
|
+
* against. Rebase's generated policies read `app.uid` and `app.user_roles`;
|
|
188
|
+
* supplying an admin role satisfies the `admin_full_access` rule and the dump
|
|
189
|
+
* sees everything that rule sees.
|
|
190
|
+
*/
|
|
191
|
+
export interface RowSecurityIdentity {
|
|
192
|
+
/** Written to `app.uid`. Any non-empty value — it is only an audit trail. */
|
|
193
|
+
uid: string;
|
|
194
|
+
/** Written to `app.user_roles`. Must include a role the policies admit. */
|
|
195
|
+
roles: string[];
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* `PGOPTIONS` carrying an identity, for a libpq tool that has no other way to
|
|
200
|
+
* set a GUC.
|
|
201
|
+
*
|
|
202
|
+
* A backslash escape rather than quoting, which is what libpq's `-c` parsing
|
|
203
|
+
* takes: a space inside a value ends the option otherwise, so a role list is
|
|
204
|
+
* comma-joined and never spaced.
|
|
205
|
+
*/
|
|
206
|
+
export function buildRowSecurityPgOptions(identity: RowSecurityIdentity): string {
|
|
207
|
+
const escape = (value: string) => value.replace(/([\\ ])/g, "\\$1");
|
|
208
|
+
return [
|
|
209
|
+
`-c app.uid=${escape(identity.uid)}`,
|
|
210
|
+
`-c app.user_id=${escape(identity.uid)}`,
|
|
211
|
+
`-c app.user_roles=${escape(identity.roles.join(","))}`
|
|
212
|
+
].join(" ");
|
|
213
|
+
}
|
|
214
|
+
|
|
174
215
|
/**
|
|
175
216
|
* Assemble the `pg_dump` argument vector. Uses the custom format (`-Fc`),
|
|
176
217
|
* which is compressed and restorable selectively via `pg_restore`.
|
|
@@ -182,18 +223,70 @@ export function buildPgDumpArgs(opts: {
|
|
|
182
223
|
excludeSchemas?: string[];
|
|
183
224
|
/** Number of parallel jobs (directory format only; ignored for -Fc). */
|
|
184
225
|
noOwner?: boolean;
|
|
226
|
+
/**
|
|
227
|
+
* Dump with row security on, as this identity. Omit — which is the default
|
|
228
|
+
* — and `pg_dump` errors rather than skipping rows it cannot see.
|
|
229
|
+
*/
|
|
230
|
+
rowSecurity?: RowSecurityIdentity;
|
|
185
231
|
}): string[] {
|
|
186
232
|
const args = ["--format=custom", "--no-password", `--file=${opts.outFile}`];
|
|
187
233
|
if (opts.noOwner) {
|
|
188
234
|
args.push("--no-owner");
|
|
189
235
|
}
|
|
236
|
+
if (opts.rowSecurity) {
|
|
237
|
+
args.push("--enable-row-security");
|
|
238
|
+
}
|
|
190
239
|
for (const schema of opts.excludeSchemas ?? []) {
|
|
191
240
|
args.push(`--exclude-schema=${schema}`);
|
|
192
241
|
}
|
|
193
|
-
args.push(opts.connectionString);
|
|
242
|
+
args.push(forLibpq(opts.connectionString));
|
|
194
243
|
return args;
|
|
195
244
|
}
|
|
196
245
|
|
|
246
|
+
/**
|
|
247
|
+
* Whether a `pg_dump` failure is the row-security one, and what to do about it.
|
|
248
|
+
*
|
|
249
|
+
* The error text names the table and nothing else, so the first read of it is
|
|
250
|
+
* "why would a backup be affected by RLS at all?" — the answer being that the
|
|
251
|
+
* dumping role is not the tables' owner and has no `BYPASSRLS`, which is the
|
|
252
|
+
* normal state of the `postgres` user on Cloud SQL, RDS and every other managed
|
|
253
|
+
* Postgres. Nothing about that is visible from the message.
|
|
254
|
+
*
|
|
255
|
+
* Returns `null` for any other failure, so the caller reports it unchanged.
|
|
256
|
+
*/
|
|
257
|
+
export function diagnoseRowSecurityDumpFailure(error: unknown): string | null {
|
|
258
|
+
const text = [
|
|
259
|
+
(error as { stderr?: unknown })?.stderr,
|
|
260
|
+
(error as { message?: unknown })?.message
|
|
261
|
+
].map(part => (typeof part === "string" ? part : "")).join("\n");
|
|
262
|
+
|
|
263
|
+
if (!/row-level security policy/i.test(text)) return null;
|
|
264
|
+
|
|
265
|
+
const table = text.match(/for table "([^"]+)"/)?.[1];
|
|
266
|
+
|
|
267
|
+
return [
|
|
268
|
+
`pg_dump cannot read ${table ? `"${table}"` : "one of the tables"} because row-level security applies to it.`,
|
|
269
|
+
"",
|
|
270
|
+
" The dumping role is neither the table's owner nor `BYPASSRLS`, which is the normal",
|
|
271
|
+
" state of the `postgres` user on Cloud SQL, RDS and other managed Postgres — there is",
|
|
272
|
+
" no superuser to hand out.",
|
|
273
|
+
"",
|
|
274
|
+
" Two ways out:",
|
|
275
|
+
"",
|
|
276
|
+
" • Grant the dumping role BYPASSRLS, or make it the owner, and run this again. The",
|
|
277
|
+
" dump then contains every row, which is what a backup should mean.",
|
|
278
|
+
"",
|
|
279
|
+
" • Re-run with --enable-row-security to dump as an admin subject instead. Rebase",
|
|
280
|
+
" sets `app.uid`/`app.user_roles` so the generated `admin_full_access` policy",
|
|
281
|
+
" admits the dump. Read the warning it prints: the result contains exactly the",
|
|
282
|
+
" rows those policies admit, and any table whose policies do not include an",
|
|
283
|
+
" admin rule comes out short — with no error.",
|
|
284
|
+
"",
|
|
285
|
+
" Do not reach for a bare `pg_dump --enable-row-security` by hand. Without the",
|
|
286
|
+
" settings above it succeeds and silently omits rows."
|
|
287
|
+
].join("\n");
|
|
288
|
+
}
|
|
289
|
+
|
|
197
290
|
/**
|
|
198
291
|
* Assemble the `pg_restore` argument vector for a custom-format dump.
|
|
199
292
|
*/
|
|
@@ -211,7 +304,7 @@ export function buildPgRestoreArgs(opts: {
|
|
|
211
304
|
exitOnError?: boolean;
|
|
212
305
|
noOwner?: boolean;
|
|
213
306
|
}): string[] {
|
|
214
|
-
const args = ["--format=custom", "--no-password", `--dbname=${opts.connectionString}`];
|
|
307
|
+
const args = ["--format=custom", "--no-password", `--dbname=${forLibpq(opts.connectionString)}`];
|
|
215
308
|
if (opts.clean) {
|
|
216
309
|
args.push("--clean", "--if-exists");
|
|
217
310
|
}
|
|
@@ -255,7 +348,7 @@ export function buildPgDumpallGlobalsArgs(opts: {
|
|
|
255
348
|
"--no-role-passwords",
|
|
256
349
|
"--no-password",
|
|
257
350
|
`--file=${opts.outFile}`,
|
|
258
|
-
`--dbname=${opts.connectionString}`
|
|
351
|
+
`--dbname=${forLibpq(opts.connectionString)}`
|
|
259
352
|
];
|
|
260
353
|
}
|
|
261
354
|
|
package/src/cli-helpers.ts
CHANGED
|
@@ -2,6 +2,8 @@ import { isManyToMany } from "@rebasepro/types";
|
|
|
2
2
|
import path from "path";
|
|
3
3
|
import fs from "fs";
|
|
4
4
|
import { execSync } from "child_process";
|
|
5
|
+
import { createRequire } from "module";
|
|
6
|
+
import readline from "readline";
|
|
5
7
|
import { pathToFileURL } from "url";
|
|
6
8
|
import chalk from "chalk";
|
|
7
9
|
import { logger } from "@rebasepro/server";
|
|
@@ -10,6 +12,49 @@ import { moduleDir as __helpersDirname } from "./module-dir";
|
|
|
10
12
|
|
|
11
13
|
|
|
12
14
|
|
|
15
|
+
/**
|
|
16
|
+
* Why is a dependency's binary missing — never installed, or installed with its
|
|
17
|
+
* build script blocked?
|
|
18
|
+
*
|
|
19
|
+
* These need opposite advice, and getting it wrong is not a cosmetic miss. pnpm
|
|
20
|
+
* 10+ refuses to run a dependency's lifecycle scripts unless it is allowlisted
|
|
21
|
+
* (`pnpm.onlyBuiltDependencies`, or `allowBuilds` in `pnpm-workspace.yaml`).
|
|
22
|
+
* `@ariga/atlas` downloads its platform binary in `preinstall`, so a blocked
|
|
23
|
+
* script leaves a state that looks like a successful install: the package is on
|
|
24
|
+
* disk with its `install.js` and `package.json`, `node_modules/.bin` is empty,
|
|
25
|
+
* the install exits 0, and the only signal is `ERR_PNPM_IGNORED_BUILDS` several
|
|
26
|
+
* screens up.
|
|
27
|
+
*
|
|
28
|
+
* Telling somebody in that state to install the package again sends them round
|
|
29
|
+
* the same loop forever — the add succeeds, the script is blocked again,
|
|
30
|
+
* nothing changes. Verified by doing it: `pnpm add @ariga/atlas` into a bare
|
|
31
|
+
* project yields exactly this, three "Failed to create bin … ENOENT" warnings
|
|
32
|
+
* and no binary.
|
|
33
|
+
*
|
|
34
|
+
* Resolution is attempted from the user's project first and this package
|
|
35
|
+
* second, matching the order {@link resolveLocalBin} searches — the driver may
|
|
36
|
+
* be installed a level up from where the command runs.
|
|
37
|
+
*/
|
|
38
|
+
export function diagnoseMissingBin(packageName: string): "not-installed" | "build-script-blocked" {
|
|
39
|
+
const bases = [
|
|
40
|
+
pathToFileURL(path.join(process.cwd(), "package.json")),
|
|
41
|
+
pathToFileURL(path.join(__helpersDirname, "package.json"))
|
|
42
|
+
];
|
|
43
|
+
|
|
44
|
+
for (const base of bases) {
|
|
45
|
+
try {
|
|
46
|
+
// `package.json` rather than the package root: a package with an
|
|
47
|
+
// `exports` map that omits `.` is unresolvable by name even when it
|
|
48
|
+
// is perfectly installed, which would misreport it as absent.
|
|
49
|
+
createRequire(base).resolve(`${packageName}/package.json`);
|
|
50
|
+
return "build-script-blocked";
|
|
51
|
+
} catch {
|
|
52
|
+
// Try the next base.
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return "not-installed";
|
|
56
|
+
}
|
|
57
|
+
|
|
13
58
|
export function resolveLocalBin(binName: string): string | null {
|
|
14
59
|
// Try to find node_modules/.bin upwards from __helpersDirname first (package-relative)
|
|
15
60
|
let dir = __helpersDirname;
|
|
@@ -232,3 +277,28 @@ export async function getTableExcludes(
|
|
|
232
277
|
return excludes;
|
|
233
278
|
}
|
|
234
279
|
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* Ask a yes/no question on an interactive terminal.
|
|
283
|
+
*
|
|
284
|
+
* The `isTTY` guard is the contract, not an optimisation: non-interactive
|
|
285
|
+
* shells (CI, pipes, agents) can never answer, and `readline` on a
|
|
286
|
+
* non-TTY stdin resolves with whatever the pipe happens to contain — or never
|
|
287
|
+
* resolves at all. Returning false there is what makes an unattended
|
|
288
|
+
* `db push` abort instead of auto-confirming a destructive change. Callers
|
|
289
|
+
* should already have gone through {@link decidePushSafety}, which decides
|
|
290
|
+
* whether interactive confirmation is even possible; this is the backstop.
|
|
291
|
+
*
|
|
292
|
+
* Lives here rather than in cli.ts so it is reachable from a test — cli.ts uses
|
|
293
|
+
* `import.meta` and cannot be imported by the jest runner.
|
|
294
|
+
*/
|
|
295
|
+
export async function promptConfirm(question: string): Promise<boolean> {
|
|
296
|
+
if (!process.stdin.isTTY) return false;
|
|
297
|
+
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
298
|
+
try {
|
|
299
|
+
const answer: string = await new Promise((resolve) => rl.question(question, resolve));
|
|
300
|
+
return /^y(es)?$/i.test(answer.trim());
|
|
301
|
+
} finally {
|
|
302
|
+
rl.close();
|
|
303
|
+
}
|
|
304
|
+
}
|