@everystack/cli 0.4.84 → 0.4.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -1
- package/package.json +2 -2
- package/src/cli/apply-execute.ts +61 -1
- package/src/cli/apply-refusals.ts +74 -0
- package/src/cli/commands/db-apply.ts +20 -4
- package/src/cli/commands/db-plan.ts +1 -0
- package/src/cli/derived-apply.ts +17 -0
- package/src/cli/edge-plan.ts +10 -1
- package/src/cli/fk-preflight.ts +331 -0
- package/src/cli/migration-generate.ts +8 -7
- package/src/cli/plain-output.ts +10 -2
- package/src/cli/schema-diff.ts +98 -11
- package/src/cli/schema-fingerprint-v12.ts +239 -0
- package/src/cli/schema-fingerprint.ts +26 -3
- package/src/cli/schema-introspect.ts +23 -0
- package/src/cli/state-apply.ts +160 -13
package/CHANGELOG.md
CHANGED
|
@@ -4,11 +4,17 @@ All notable changes to `@everystack/cli` are documented here.
|
|
|
4
4
|
|
|
5
5
|
Format based on [Keep a Changelog](https://keepachangelog.com/).
|
|
6
6
|
|
|
7
|
-
## [
|
|
7
|
+
## [0.4.85] - 2026-09-16
|
|
8
8
|
|
|
9
9
|
### Added
|
|
10
10
|
|
|
11
11
|
- automate the version-drift check — a manual note is not a gate
|
|
12
|
+
- add foreign keys in two steps, and check the rows before the DDL
|
|
13
|
+
|
|
14
|
+
### Fixed
|
|
15
|
+
|
|
16
|
+
- the two-step FK add's gate could OOM the database, and its record could go missing
|
|
17
|
+
- db:plan --verbose actually expands the truncated change list
|
|
12
18
|
|
|
13
19
|
## [0.4.84] - 2026-09-16
|
|
14
20
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@everystack/cli",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.85",
|
|
4
4
|
"description": "CLI and OTA updates for Expo apps on everystack",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"author": "Scalable Technology, Inc. <licensing@scalable.technology>",
|
|
@@ -182,7 +182,7 @@
|
|
|
182
182
|
"react": "19.2.0",
|
|
183
183
|
"ts-jest": "29.4.9",
|
|
184
184
|
"zod": "3.25.67",
|
|
185
|
-
"@everystack/server": "0.4.
|
|
185
|
+
"@everystack/server": "0.4.31"
|
|
186
186
|
},
|
|
187
187
|
"scripts": {
|
|
188
188
|
"test": "jest",
|
package/src/cli/apply-execute.ts
CHANGED
|
@@ -57,6 +57,8 @@ import {
|
|
|
57
57
|
import { FUNCTIONS_SQL, contractFunctionRow } from './security-catalog';
|
|
58
58
|
import { fingerprintLive, FINGERPRINT_VERSION } from './schema-fingerprint';
|
|
59
59
|
import { applyGeneratedStatements } from './state-apply';
|
|
60
|
+
import { fkOrphanChecks, fkOrphanSql, fkOrphanRefusal, fkPreflightUnavailableRefusal, checkOrphanRowShape } from './fk-preflight';
|
|
61
|
+
import { validationPendingReason } from './apply-refusals';
|
|
60
62
|
import { ENSURE_RECONCILER_SQL, renderSchemaLogInsert, renderSchemaLogFingerprintUpdate } from './derived-apply';
|
|
61
63
|
import { checkPlanPrecondition, planHash, defaultAclHash, type EdgePlan } from './edge-plan';
|
|
62
64
|
import type { DescentLedgerRow } from './git-descent';
|
|
@@ -140,7 +142,19 @@ export interface ApplyPlanOptions {
|
|
|
140
142
|
}
|
|
141
143
|
|
|
142
144
|
export interface ApplyPlanResult {
|
|
143
|
-
status:
|
|
145
|
+
status:
|
|
146
|
+
| 'applied'
|
|
147
|
+
| 'already-applied'
|
|
148
|
+
| 'refused'
|
|
149
|
+
| 'descent-refused'
|
|
150
|
+
| 'authority-refused'
|
|
151
|
+
| 'snapshot-refused'
|
|
152
|
+
| 'authz-refused'
|
|
153
|
+
/** Rows already in the table do not satisfy a foreign key this plan installs — nothing ran. */
|
|
154
|
+
| 'preflight-refused'
|
|
155
|
+
/** The edge COMMITTED; a post-commit `VALIDATE CONSTRAINT` did not pass. Not a rollback. */
|
|
156
|
+
| 'validation-pending'
|
|
157
|
+
| 'verify-failed';
|
|
144
158
|
liveFingerprint: string;
|
|
145
159
|
reason?: string;
|
|
146
160
|
logId?: number;
|
|
@@ -359,6 +373,31 @@ export async function executeApplyPlan(
|
|
|
359
373
|
}
|
|
360
374
|
}
|
|
361
375
|
|
|
376
|
+
// The LAST gate before any DDL, and the only one that asks about DATA. A foreign key this plan
|
|
377
|
+
// installs is only installable if the rows already in the table satisfy it; PostgreSQL would
|
|
378
|
+
// answer that during the VALIDATE, one row at a time, with the lock already taken. Asked here,
|
|
379
|
+
// the answer arrives with a count and a sample and nothing has run. The snapshot is the one the
|
|
380
|
+
// lock was just computed from, so the question and the lock describe one instant.
|
|
381
|
+
const orphanChecks = fkOrphanChecks(plan.statements, before);
|
|
382
|
+
if (orphanChecks.length > 0) {
|
|
383
|
+
let refusal: string | null;
|
|
384
|
+
try {
|
|
385
|
+
const rows = (await runner(fkOrphanSql(orphanChecks))) as unknown;
|
|
386
|
+
// The read answers with exactly one row per key, zero orphans included. A short or
|
|
387
|
+
// missing answer is not "clean" — it is the gate not having run, and it goes down the
|
|
388
|
+
// same fail-closed path as a read that could not run at all.
|
|
389
|
+
checkOrphanRowShape(rows, orphanChecks.length);
|
|
390
|
+
refusal = fkOrphanRefusal(rows);
|
|
391
|
+
} catch (err: any) {
|
|
392
|
+
// Fail CLOSED: a check that could not run has not passed.
|
|
393
|
+
refusal = fkPreflightUnavailableRefusal(String(err?.message ?? err));
|
|
394
|
+
}
|
|
395
|
+
if (refusal) {
|
|
396
|
+
await recordRefusal(runner, plan, live, 'foreign key preflight', refusal, options);
|
|
397
|
+
return { status: 'preflight-refused', liveFingerprint: live, reason: refusal };
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
|
|
362
401
|
const result = await applyGeneratedStatements(runner, plan.statements, {
|
|
363
402
|
actor: options.actor,
|
|
364
403
|
gitRef: options.gitRef ?? plan.gitRef,
|
|
@@ -367,6 +406,11 @@ export async function executeApplyPlan(
|
|
|
367
406
|
fpVersion: FINGERPRINT_VERSION,
|
|
368
407
|
dirty: options.dirty ?? false,
|
|
369
408
|
now: options.now,
|
|
409
|
+
// The hash to stamp on the record written the moment the batch commits — the live state as
|
|
410
|
+
// it is THEN, with any new key installed and not yet proven. The plan's own endpoint is
|
|
411
|
+
// stamped below, after the last check has actually passed; predicting it here would put a
|
|
412
|
+
// state the database is not in onto the row every later apply descends from.
|
|
413
|
+
readFingerprint: async () => (await liveBaseFingerprint(session)).hash,
|
|
370
414
|
...(options.descentNote ? { descentNote: options.descentNote } : {}),
|
|
371
415
|
});
|
|
372
416
|
|
|
@@ -377,6 +421,22 @@ export async function executeApplyPlan(
|
|
|
377
421
|
await runner(renderSchemaLogFingerprintUpdate(result.logId, landed));
|
|
378
422
|
}
|
|
379
423
|
|
|
424
|
+
// A key left NOT VALID is reported as ITSELF, before the endpoint comparison it would otherwise
|
|
425
|
+
// fail. The edge committed and the schema is correct; what is missing is the proof that the
|
|
426
|
+
// existing rows satisfy the key, and the fix is a data fix plus a re-plan (which will emit the
|
|
427
|
+
// VALIDATE alone). Reporting that as `verify-failed` would tell the operator to stop touching a
|
|
428
|
+
// database that is in a perfectly well-defined state.
|
|
429
|
+
if (result.pendingValidations.length > 0) {
|
|
430
|
+
return {
|
|
431
|
+
status: 'validation-pending',
|
|
432
|
+
liveFingerprint: landed,
|
|
433
|
+
// The next move follows the CAUSE: bad rows are a data fix, a check that was cut off by a
|
|
434
|
+
// clock is a lane change. See validationPendingReason.
|
|
435
|
+
reason: validationPendingReason(result.pendingValidations),
|
|
436
|
+
...(result.logId !== undefined ? { logId: result.logId } : {}),
|
|
437
|
+
};
|
|
438
|
+
}
|
|
439
|
+
|
|
380
440
|
// The full `to` equals the post-apply raw live hash, so verify-after compares them directly.
|
|
381
441
|
if (landed !== plan.toFingerprint) {
|
|
382
442
|
return {
|
|
@@ -30,6 +30,72 @@ export function snapshotMovedRefusal(ref: string, captured: string, planFrom: st
|
|
|
30
30
|
return `the rollback point ${ref} captures ${captured.slice(0, 12)}, not the plan's start ${planFrom.slice(0, 12)} — the database moved since the plan was minted. Re-plan (db:plan) and apply again.`;
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
+
/**
|
|
34
|
+
* VALIDATION PENDING: the edge committed and a post-commit `VALIDATE CONSTRAINT` did not pass.
|
|
35
|
+
* Neither class above — nothing was refused and nothing is in an unknown state — but it still has
|
|
36
|
+
* to name a next move, and WHICH move depends on why the check did not pass:
|
|
37
|
+
*
|
|
38
|
+
* ORPHAN DATA — the rows genuinely do not satisfy the key. The key is the truth and the data
|
|
39
|
+
* is wrong, so the fix is the DATA (db:exec), then re-plan; the next plan emits
|
|
40
|
+
* the VALIDATE alone.
|
|
41
|
+
* CUT OFF — a wall clock or a dropped connection ended the check before it answered.
|
|
42
|
+
* NOTHING is known about the rows. Sending that operator to go fix data is a
|
|
43
|
+
* wild goose chase; the move is to run the same check where no clock ends it,
|
|
44
|
+
* which is the direct lane.
|
|
45
|
+
*
|
|
46
|
+
* An error neither pattern recognises names BOTH moves rather than picking one — the same rule
|
|
47
|
+
* the rest of this file follows about guessing.
|
|
48
|
+
*/
|
|
49
|
+
export interface PendingValidationRecord {
|
|
50
|
+
/** The exact `VALIDATE CONSTRAINT` statement that did not pass. */
|
|
51
|
+
sql: string;
|
|
52
|
+
/** PostgreSQL's (or the driver's) message. */
|
|
53
|
+
error: string;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** The rows really do violate the key: SQLSTATE 23503 and the message pg renders for it. */
|
|
57
|
+
const ORPHAN_DATA = /violates foreign key constraint|is violated by some row|\b23503\b/i;
|
|
58
|
+
/** The check never answered: a statement timeout, a lock timeout, or a connection that died. */
|
|
59
|
+
const CUT_OFF = /timeout|canceling statement|terminat(ed|ing)|connection (closed|reset|refused|lost)|socket|ECONNRESET|EPIPE|\b(57014|55P03|08006|08003)\b/i;
|
|
60
|
+
|
|
61
|
+
type PendingCause = 'data' | 'cut-off' | 'unknown';
|
|
62
|
+
|
|
63
|
+
function causeOf(error: string): PendingCause {
|
|
64
|
+
if (ORPHAN_DATA.test(error)) return 'data';
|
|
65
|
+
if (CUT_OFF.test(error)) return 'cut-off';
|
|
66
|
+
return 'unknown';
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function validationPendingReason(pending: readonly PendingValidationRecord[]): string {
|
|
70
|
+
const causes = new Set(pending.map((p) => causeOf(p.error)));
|
|
71
|
+
const unknown = causes.has('unknown');
|
|
72
|
+
|
|
73
|
+
const moves: string[] = [];
|
|
74
|
+
if (causes.has('data') || unknown) {
|
|
75
|
+
moves.push(
|
|
76
|
+
'Rows already in the table do not satisfy the key: fix the data (everystack db:exec runs a '
|
|
77
|
+
+ 'DML-only file against the target, credential-free), then db:plan and db:apply, which will '
|
|
78
|
+
+ 'emit the VALIDATE alone.',
|
|
79
|
+
);
|
|
80
|
+
}
|
|
81
|
+
if (causes.has('cut-off') || unknown) {
|
|
82
|
+
moves.push(
|
|
83
|
+
'The check was cut off rather than answered (a timeout or a dropped connection), so nothing '
|
|
84
|
+
+ 'is yet known about the rows: re-plan and apply again naming the direct lane '
|
|
85
|
+
+ '(--stage <name> --direct), which runs with no wall clock over it.',
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
if (unknown) moves.unshift('The error below is not one this command recognises, so both moves are named:');
|
|
89
|
+
|
|
90
|
+
const names = pending.map((p) => p.sql.replace(/\s+/g, ' ').trim()).join(' | ');
|
|
91
|
+
return [
|
|
92
|
+
`the edge applied, but ${pending.length} constraint(s) could not be validated against the rows `
|
|
93
|
+
+ 'already in the table — they are installed and enforced on new writes, but unproven.',
|
|
94
|
+
...moves,
|
|
95
|
+
`The check(s) still to pass: ${names}`,
|
|
96
|
+
].join(' ');
|
|
97
|
+
}
|
|
98
|
+
|
|
33
99
|
// ---------------------------------------------------------------------------
|
|
34
100
|
// The USER-FACING refusal message (apply-legibility Phase 3, p3-action-refusals).
|
|
35
101
|
//
|
|
@@ -55,6 +121,8 @@ export type ApplyRefusalStatus =
|
|
|
55
121
|
| 'authority-refused'
|
|
56
122
|
| 'snapshot-refused'
|
|
57
123
|
| 'authz-refused'
|
|
124
|
+
| 'preflight-refused'
|
|
125
|
+
| 'validation-pending'
|
|
58
126
|
| 'verify-failed';
|
|
59
127
|
|
|
60
128
|
/**
|
|
@@ -72,7 +140,13 @@ export function formatApplyRefusal(status: ApplyRefusalStatus, reason: string):
|
|
|
72
140
|
case 'authority-refused': // the destructive-approver set — hand the plan to an approver
|
|
73
141
|
case 'snapshot-refused': // no confirmed rollback point before a destructive statement
|
|
74
142
|
case 'authz-refused': // a foreign grantee arrived after the plan was minted
|
|
143
|
+
case 'preflight-refused': // rows already in the table do not satisfy a foreign key — nothing ran
|
|
75
144
|
return `REFUSED: ${reason}`;
|
|
145
|
+
case 'validation-pending':
|
|
146
|
+
// NOT a refusal and NOT a fault: the edge committed, and a constraint is installed but
|
|
147
|
+
// unproven against the rows that were already there. Its own word, because "refused" would
|
|
148
|
+
// say nothing ran and "verify failed" would say the database is in an unknown state.
|
|
149
|
+
return `Applied, validation PENDING: ${reason}`;
|
|
76
150
|
case 'verify-failed':
|
|
77
151
|
return `Verify FAILED: ${reason}`;
|
|
78
152
|
default: {
|
|
@@ -227,11 +227,27 @@ async function runCatchUp(
|
|
|
227
227
|
}
|
|
228
228
|
}
|
|
229
229
|
|
|
230
|
+
/**
|
|
231
|
+
* The statuses where the table and column state is COMMITTED on the target. Everything else
|
|
232
|
+
* either refused before any write or ended in a state nothing may be built on.
|
|
233
|
+
*/
|
|
234
|
+
const LANDED: ReadonlySet<ApplyPlanResult['status']> = new Set<ApplyPlanResult['status']>([
|
|
235
|
+
'applied',
|
|
236
|
+
'already-applied',
|
|
237
|
+
'validation-pending',
|
|
238
|
+
]);
|
|
239
|
+
|
|
230
240
|
/**
|
|
231
241
|
* The chain: a plan that LANDED is followed by the catch-up, on the same target, always —
|
|
232
242
|
* unless --state-only asked for the tables alone. A refusal or a verify fault chains nothing
|
|
233
243
|
* (there is no applied state to build on); reportApplyResult renders those and exits.
|
|
234
244
|
*
|
|
245
|
+
* `validation-pending` LANDED. It is not a refusal: the tables and columns committed and only a
|
|
246
|
+
* `VALIDATE CONSTRAINT` is outstanding. Functions, views, matviews and triggers do not depend on
|
|
247
|
+
* whether a foreign key has been PROVEN against the old rows — they compile and run identically
|
|
248
|
+
* either way — so skipping the catch-up there deployed half the checkout and said nothing about
|
|
249
|
+
* it, which is the exact incident this whole chain was built to end.
|
|
250
|
+
*
|
|
235
251
|
* THE POST-WRITE CALLER, and the only reason `'throw'` exists: the table and column changes are
|
|
236
252
|
* committed by the time this runs. Nothing downstream may end the process on its own, because
|
|
237
253
|
* the operator is owed one sentence first — your changes are safe, here is what to rerun. So a
|
|
@@ -244,11 +260,11 @@ async function catchUpAfterApply(
|
|
|
244
260
|
venue: CatchUpVenue,
|
|
245
261
|
flags: Record<string, string>,
|
|
246
262
|
): Promise<CatchUpReport> {
|
|
247
|
-
if (
|
|
263
|
+
if (!LANDED.has(result.status)) return { ok: true, statements: [] };
|
|
248
264
|
if (flags['state-only'] === 'true') return { ok: true, statements: [] };
|
|
249
|
-
const applied = result.status === 'applied'
|
|
250
|
-
?
|
|
251
|
-
:
|
|
265
|
+
const applied = result.status === 'already-applied'
|
|
266
|
+
? 0
|
|
267
|
+
: classifyGeneratedStatements(plan.statements ?? []).executable.length;
|
|
252
268
|
return runCatchUp(venue, flags, applied, 'throw');
|
|
253
269
|
}
|
|
254
270
|
|
package/src/cli/derived-apply.ts
CHANGED
|
@@ -586,3 +586,20 @@ RETURNING id`;
|
|
|
586
586
|
export function renderSchemaLogFingerprintUpdate(id: number, toFingerprint: string): string {
|
|
587
587
|
return `UPDATE everystack.schema_log SET to_fingerprint = ${escapeLiteral(toFingerprint)} WHERE id = ${Math.floor(id)}`;
|
|
588
588
|
}
|
|
589
|
+
|
|
590
|
+
/**
|
|
591
|
+
* Finish a row that was written BEFORE the work it describes had finished.
|
|
592
|
+
*
|
|
593
|
+
* The `state apply` row goes in the moment the batch commits, so a death during the post-commit
|
|
594
|
+
* `VALIDATE CONSTRAINT` pass cannot leave committed DDL with nothing on the record. That means
|
|
595
|
+
* the row exists while its own outcome is still open — and when the pass ends, what changes is
|
|
596
|
+
* the outcome and the statement text, not the identity of the apply. So it is AMENDED. A second
|
|
597
|
+
* row would put two applies in a ledger the descent witness reads as a series of live states.
|
|
598
|
+
*/
|
|
599
|
+
export function renderSchemaLogCompletionUpdate(
|
|
600
|
+
id: number,
|
|
601
|
+
entry: { outcome: string; sql: string; durationMs?: number },
|
|
602
|
+
): string {
|
|
603
|
+
const duration = entry.durationMs === undefined ? '' : `, duration_ms = ${Math.round(entry.durationMs)}`;
|
|
604
|
+
return `UPDATE everystack.schema_log SET outcome = ${escapeLiteral(entry.outcome)}, sql = ${escapeLiteral(entry.sql)}${duration} WHERE id = ${Math.floor(id)}`;
|
|
605
|
+
}
|
package/src/cli/edge-plan.ts
CHANGED
|
@@ -64,8 +64,17 @@ export const PLAN_VERSION = 2;
|
|
|
64
64
|
* the gate — not a false pass of the apply (it degrades to the post-DDL verify failure), but a
|
|
65
65
|
* silently dropped gate the operator was told is in force. The BUMP RULE says a gate the server
|
|
66
66
|
* runs is part of the contract, so the peers refuse until the stage is redeployed.
|
|
67
|
+
*
|
|
68
|
+
* v4 (the two-step foreign key): the server runs a NEW gate — the foreign-key orphan preflight,
|
|
69
|
+
* which counts the rows that would fail each key the plan installs or validates and refuses
|
|
70
|
+
* BEFORE any DDL — and the write path changed shape: `VALIDATE CONSTRAINT` is held out of the
|
|
71
|
+
* edge's transaction and run after it commits, so a failed check is a PENDING constraint rather
|
|
72
|
+
* than a rolled-back edge (`validation-pending`, a status a v3 ops Lambda cannot return). A v3
|
|
73
|
+
* stage would skip the preflight and run the VALIDATE inside the edge — a silently dropped gate
|
|
74
|
+
* and a different failure mode, both of which the BUMP RULE covers. Peers refuse until the stage
|
|
75
|
+
* is redeployed.
|
|
67
76
|
*/
|
|
68
|
-
export const APPLY_CONTRACT_VERSION =
|
|
77
|
+
export const APPLY_CONTRACT_VERSION = 4;
|
|
69
78
|
|
|
70
79
|
/** Classification counts (brick 9, decision 11): destructive = drops + narrowings + strips. */
|
|
71
80
|
export interface PlanClassification {
|
|
@@ -0,0 +1,331 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* fk-preflight — the orphan check that runs BEFORE a foreign key's DDL, not during it.
|
|
3
|
+
*
|
|
4
|
+
* A foreign key is only installable if the rows already in the table satisfy it. PostgreSQL
|
|
5
|
+
* finds that out DURING `VALIDATE CONSTRAINT`, by raising on the first row that does not — one
|
|
6
|
+
* row, no count, no second offender, and (before the two-step split) after the apply had already
|
|
7
|
+
* taken an ACCESS EXCLUSIVE lock and scanned most of the table. The operator learned "there is a
|
|
8
|
+
* problem" at the worst possible moment and learned nothing else.
|
|
9
|
+
*
|
|
10
|
+
* So the apply asks the question first, the same way the declared-owner preflight in
|
|
11
|
+
* derived-apply does: ONE read that answers it for EVERY key the edge touches, so the refusal
|
|
12
|
+
* can name the COMPLETE list — how many rows, on which key, and what a few of them look like —
|
|
13
|
+
* and NOTHING has run when it is raised. Fix the data (`db:exec` is the credential-free lane for
|
|
14
|
+
* exactly that), re-plan, apply.
|
|
15
|
+
*
|
|
16
|
+
* WHICH KEYS NEED THE QUESTION ASKED. Only the ones where an existing row could already violate
|
|
17
|
+
* the constraint, which is a property of the LIVE database, not of the statement:
|
|
18
|
+
*
|
|
19
|
+
* - a key on a table this edge CREATES → no. The table has no rows yet, and it does
|
|
20
|
+
* not exist to be queried before the edge runs.
|
|
21
|
+
* - a key on a column this edge ADDS → no. Every existing row's value is NULL, which
|
|
22
|
+
* satisfies a foreign key (MATCH SIMPLE), and
|
|
23
|
+
* the column cannot be queried yet either.
|
|
24
|
+
* - a key whose TARGET this edge creates → no, and this one is a deliberate gap: the
|
|
25
|
+
* parent is empty until the edge runs, so the
|
|
26
|
+
* answer cannot be read beforehand. The VALIDATE
|
|
27
|
+
* then fails and the key stays NOT VALID — a
|
|
28
|
+
* pending state, not a rolled-back apply.
|
|
29
|
+
* - a key already live but NOT VALID → YES. This is the interrupted two-step add, and
|
|
30
|
+
* the rows it was never checked against are
|
|
31
|
+
* exactly what VALIDATE is about to scan.
|
|
32
|
+
* - a key on an existing table and columns → YES.
|
|
33
|
+
*
|
|
34
|
+
* An `ADD … NOT VALID` and the `VALIDATE` that follows it in the same edge are ONE question, not
|
|
35
|
+
* two — the key is the unit, not the statement.
|
|
36
|
+
*
|
|
37
|
+
* Pure: the parsing, the SQL, and the refusal text. The caller runs the query.
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
import type { SchemaSnapshot, TableSchema, ForeignKey } from './schema-introspect';
|
|
41
|
+
import { statementBody } from './sql-statement';
|
|
42
|
+
import { quoteIdent, quoteQualified } from './pg-ident';
|
|
43
|
+
import { escapeLiteral } from './derived-apply';
|
|
44
|
+
|
|
45
|
+
/** One foreign key whose orphans must be counted before any DDL runs. */
|
|
46
|
+
export interface FkOrphanCheck {
|
|
47
|
+
/** Schema-qualified, as the live snapshot spells it (`public.posts`). */
|
|
48
|
+
table: string;
|
|
49
|
+
/** The constraint name as it will exist when the VALIDATE runs. */
|
|
50
|
+
constraint: string;
|
|
51
|
+
columns: string[];
|
|
52
|
+
/** Schema-qualified target table. */
|
|
53
|
+
refTable: string;
|
|
54
|
+
refColumns: string[];
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Strip quoting from one identifier token. */
|
|
58
|
+
const unquote = (id: string): string => (/^".*"$/.test(id) ? id.slice(1, -1).replace(/""/g, '"') : id);
|
|
59
|
+
|
|
60
|
+
/** `"public"."posts"` / `posts` / `public.posts` → `public.posts` (the snapshot's spelling). */
|
|
61
|
+
export function normalizeTableName(raw: string): string {
|
|
62
|
+
const parts = raw.trim().replace(/;$/, '').match(/("(?:[^"]|"")*"|[^.\s]+)/g) ?? [];
|
|
63
|
+
const names = parts.map(unquote);
|
|
64
|
+
if (names.length === 0) return raw;
|
|
65
|
+
return names.length === 1 ? `public.${names[0]}` : names.slice(-2).join('.');
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** `("a", "b")` → `['a','b']`. */
|
|
69
|
+
const parseColumns = (list: string): string[] =>
|
|
70
|
+
(list.replace(/^\(|\)$/g, '').match(/("(?:[^"]|"")*"|[^,\s]+)/g) ?? []).map(unquote);
|
|
71
|
+
|
|
72
|
+
const CREATE_TABLE_RE = /^CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?((?:"(?:[^"]|"")*"|[A-Za-z_][\w$]*)(?:\.(?:"(?:[^"]|"")*"|[A-Za-z_][\w$]*))?)/i;
|
|
73
|
+
const ADD_COLUMN_RE = /^ALTER\s+TABLE\s+(?:ONLY\s+)?(\S+)\s+ADD\s+COLUMN\s+(?:IF\s+NOT\s+EXISTS\s+)?("(?:[^"]|"")*"|[A-Za-z_][\w$]*)/i;
|
|
74
|
+
const ADD_FK_RE = /^ALTER\s+TABLE\s+(?:ONLY\s+)?(\S+)\s+ADD\s+CONSTRAINT\s+("(?:[^"]|"")*"|[A-Za-z_][\w$]*)\s+FOREIGN\s+KEY\s*(\([^)]*\))\s*REFERENCES\s+((?:"(?:[^"]|"")*"|[^\s(]+)(?:\.(?:"(?:[^"]|"")*"|[^\s(]+))?)\s*(\([^)]*\))/i;
|
|
75
|
+
const VALIDATE_RE = /^ALTER\s+TABLE\s+(?:ONLY\s+)?(\S+)\s+VALIDATE\s+CONSTRAINT\s+("(?:[^"]|"")*"|[A-Za-z_][\w$]*)/i;
|
|
76
|
+
const RENAME_CONSTRAINT_RE = /^ALTER\s+TABLE\s+(?:ONLY\s+)?(\S+)\s+RENAME\s+CONSTRAINT\s+("(?:[^"]|"")*"|[A-Za-z_][\w$]*)\s+TO\s+("(?:[^"]|"")*"|[A-Za-z_][\w$]*)/i;
|
|
77
|
+
|
|
78
|
+
/** The live table under its qualified name, or undefined. */
|
|
79
|
+
const liveTable = (live: SchemaSnapshot, table: string): TableSchema | undefined =>
|
|
80
|
+
live.tables.find((t) => t.table === table);
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The foreign keys an edge is about to install or validate that could ALREADY be violated by
|
|
84
|
+
* rows in the live database — see the rules in the module docblock.
|
|
85
|
+
*
|
|
86
|
+
* `statements` is the edge exactly as it will execute (held drops and notices included; they are
|
|
87
|
+
* comments and match nothing). `live` is the target's state as the apply just read it — the same
|
|
88
|
+
* snapshot the concurrency lock was computed from, so the two cannot describe different databases.
|
|
89
|
+
*/
|
|
90
|
+
export function fkOrphanChecks(statements: readonly string[], live: SchemaSnapshot): FkOrphanCheck[] {
|
|
91
|
+
const bodies = statements.map(statementBody);
|
|
92
|
+
|
|
93
|
+
// Tables this edge creates: their rows do not exist yet, and neither do they.
|
|
94
|
+
const created = new Set<string>();
|
|
95
|
+
for (const body of bodies) {
|
|
96
|
+
const m = CREATE_TABLE_RE.exec(body);
|
|
97
|
+
if (m) created.add(normalizeTableName(m[1]));
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Columns this edge adds: every existing row holds NULL there, which satisfies the key.
|
|
101
|
+
const addedColumns = new Set<string>();
|
|
102
|
+
for (const body of bodies) {
|
|
103
|
+
const m = ADD_COLUMN_RE.exec(body);
|
|
104
|
+
if (m) addedColumns.add(`${normalizeTableName(m[1])}.${unquote(m[2])}`);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// A constraint renamed by this edge is validated under its NEW name, but it is LIVE under the
|
|
108
|
+
// old one — so the live lookup has to walk back through the rename.
|
|
109
|
+
const renamedFrom = new Map<string, string>();
|
|
110
|
+
for (const body of bodies) {
|
|
111
|
+
const m = RENAME_CONSTRAINT_RE.exec(body);
|
|
112
|
+
if (m) renamedFrom.set(`${normalizeTableName(m[1])}::${unquote(m[3])}`, unquote(m[2]));
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const checks: FkOrphanCheck[] = [];
|
|
116
|
+
const seen = new Set<string>();
|
|
117
|
+
|
|
118
|
+
const push = (check: FkOrphanCheck): void => {
|
|
119
|
+
const key = `${check.table}::${check.constraint}`;
|
|
120
|
+
if (seen.has(key)) return;
|
|
121
|
+
seen.add(key);
|
|
122
|
+
checks.push(check);
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
for (const body of bodies) {
|
|
126
|
+
const add = ADD_FK_RE.exec(body);
|
|
127
|
+
if (add) {
|
|
128
|
+
const table = normalizeTableName(add[1]);
|
|
129
|
+
const columns = parseColumns(add[3]);
|
|
130
|
+
const target = liveTable(live, table);
|
|
131
|
+
// Not live yet, or one of its columns is not: nothing to scan, and nothing to scan it with.
|
|
132
|
+
if (!target) continue;
|
|
133
|
+
const liveColumns = new Set(target.columns.map((c) => c.name));
|
|
134
|
+
if (columns.some((c) => !liveColumns.has(c) || addedColumns.has(`${table}.${c}`))) continue;
|
|
135
|
+
const refTable = normalizeTableName(add[4]);
|
|
136
|
+
if (created.has(refTable) || !liveTable(live, refTable)) continue;
|
|
137
|
+
push({ table, constraint: unquote(add[2]), columns, refTable, refColumns: parseColumns(add[5]) });
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const validate = VALIDATE_RE.exec(body);
|
|
142
|
+
if (validate) {
|
|
143
|
+
const table = normalizeTableName(validate[1]);
|
|
144
|
+
const constraint = unquote(validate[2]);
|
|
145
|
+
// Already covered when this edge also ADDs it — one key, one question.
|
|
146
|
+
if (seen.has(`${table}::${constraint}`)) continue;
|
|
147
|
+
const target = liveTable(live, table);
|
|
148
|
+
if (!target) continue;
|
|
149
|
+
const liveName = renamedFrom.get(`${table}::${constraint}`) ?? constraint;
|
|
150
|
+
const fk: ForeignKey | undefined = target.foreignKeys.find((f) => f.name === liveName);
|
|
151
|
+
// A VALIDATE naming a key the live database does not have is not a question this preflight
|
|
152
|
+
// can ask — it would be guessing at columns. The statement's own failure is the answer.
|
|
153
|
+
if (!fk) continue;
|
|
154
|
+
push({ table, constraint, columns: fk.columns, refTable: normalizeTableName(fk.refTable), refColumns: fk.refColumns });
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
return checks;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** One row of {@link fkOrphanSql}'s result. */
|
|
162
|
+
export interface FkOrphanRow {
|
|
163
|
+
constraint_name: unknown;
|
|
164
|
+
table_name: unknown;
|
|
165
|
+
columns: unknown;
|
|
166
|
+
ref_table: unknown;
|
|
167
|
+
orphans: unknown;
|
|
168
|
+
samples: unknown;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* ONE read answering the question for every key at once — a first-failure refusal would make the
|
|
173
|
+
* operator fix one table, re-run, and discover the next.
|
|
174
|
+
*
|
|
175
|
+
* The scan is a real anti-join over the child table, which costs about what the VALIDATE it is
|
|
176
|
+
* protecting would cost. That is the price of knowing BEFORE the lock instead of during it, and
|
|
177
|
+
* it buys an exact count rather than pg's one-row-and-stop.
|
|
178
|
+
*
|
|
179
|
+
* NULLs are excluded, not counted: under MATCH SIMPLE (the only mode everystack emits) a row with
|
|
180
|
+
* a NULL in any key column satisfies the constraint. Counting them would refuse an apply that
|
|
181
|
+
* PostgreSQL would have accepted, which is the worse failure — a gate nobody can pass gets
|
|
182
|
+
* disabled.
|
|
183
|
+
*
|
|
184
|
+
* THE COUNT IS UNBOUNDED AND THE SAMPLE IS NOT, and the asymmetry is deliberate. The count is
|
|
185
|
+
* the full scan, which is the price of the answer and is paid either way. The SAMPLE is three
|
|
186
|
+
* values for the operator to recognise a row by — so it is taken from a `LIMIT 3` subquery and
|
|
187
|
+
* aggregated over those three rows alone. Aggregating the whole set and slicing it afterwards
|
|
188
|
+
* (`(array_agg(…))[1:3]`) builds the ENTIRE array in work memory first, which array_agg cannot
|
|
189
|
+
* spill to disk: on a table with millions of orphans the gate that exists to protect the
|
|
190
|
+
* database is the thing that takes it down.
|
|
191
|
+
*/
|
|
192
|
+
export function fkOrphanSql(checks: readonly FkOrphanCheck[]): string {
|
|
193
|
+
if (checks.length === 0) throw new Error('fkOrphanSql: no checks — the caller must skip the read');
|
|
194
|
+
return checks
|
|
195
|
+
.map((c) => {
|
|
196
|
+
// The columns come out of a REGEX over the ADD CONSTRAINT text. A short or empty parse is
|
|
197
|
+
// not a shape this read can paper over: joining the child's own column name against the
|
|
198
|
+
// parent errors on most schemas but SUCCEEDS on a parent that happens to carry the same
|
|
199
|
+
// name — and then counts zero, reporting a clean pass on a key nobody actually checked.
|
|
200
|
+
// Refuse, exactly as an empty check list is refused; the caller fails closed on the throw.
|
|
201
|
+
if (c.columns.length === 0) {
|
|
202
|
+
throw new Error(`fkOrphanSql: ${c.table}.${c.constraint} has no columns — the read cannot be built`);
|
|
203
|
+
}
|
|
204
|
+
if (c.refColumns.length !== c.columns.length) {
|
|
205
|
+
throw new Error(
|
|
206
|
+
`fkOrphanSql: ${c.table}.${c.constraint} names ${c.columns.length} column(s) but `
|
|
207
|
+
+ `${c.refColumns.length} reference column(s) in ${c.refTable} — the read cannot be built `
|
|
208
|
+
+ 'without guessing the join, and a guessed join can report a false clean.',
|
|
209
|
+
);
|
|
210
|
+
}
|
|
211
|
+
const child = quoteQualified(c.table);
|
|
212
|
+
const parent = quoteQualified(c.refTable);
|
|
213
|
+
const notNull = c.columns.map((col) => `c.${quoteIdent(col)} IS NOT NULL`).join(' AND ');
|
|
214
|
+
const join = c.columns.map((col, i) => `p.${quoteIdent(c.refColumns[i]!)} = c.${quoteIdent(col)}`).join(' AND ');
|
|
215
|
+
const where = (pad: string): string =>
|
|
216
|
+
`WHERE ${notNull}\n${pad} AND NOT EXISTS (SELECT 1 FROM ${parent} p WHERE ${join})`;
|
|
217
|
+
// One printable value per offending row: a single-column key prints the value, a composite
|
|
218
|
+
// one prints the tuple. Capped, because a sample is for recognising the row, not for
|
|
219
|
+
// reading its contents into a refusal message.
|
|
220
|
+
const sample = c.columns.length === 1
|
|
221
|
+
? `left(c.${quoteIdent(c.columns[0]!)}::text, 80)`
|
|
222
|
+
: `left(concat_ws(', ', ${c.columns.map((col) => `c.${quoteIdent(col)}::text`).join(', ')}), 80)`;
|
|
223
|
+
// Two scalar subqueries over no FROM: exactly ONE row per arm, at any orphan count —
|
|
224
|
+
// zero orphans gives 0 and a NULL sample array rather than no row at all. The caller
|
|
225
|
+
// asserts that shape (checkOrphanRowShape) and fails closed if it does not hold.
|
|
226
|
+
return `SELECT ${escapeLiteral(c.constraint)} AS constraint_name,
|
|
227
|
+
${escapeLiteral(c.table)} AS table_name,
|
|
228
|
+
${escapeLiteral(c.columns.join(', '))} AS columns,
|
|
229
|
+
${escapeLiteral(c.refTable)} AS ref_table,
|
|
230
|
+
(SELECT count(*)::bigint
|
|
231
|
+
FROM ${child} c
|
|
232
|
+
${where(' ')}) AS orphans,
|
|
233
|
+
(SELECT array_agg(s.sample)
|
|
234
|
+
FROM (SELECT ${sample} AS sample
|
|
235
|
+
FROM ${child} c
|
|
236
|
+
${where(' ')}
|
|
237
|
+
ORDER BY 1
|
|
238
|
+
LIMIT 3) s) AS samples`;
|
|
239
|
+
})
|
|
240
|
+
.join('\nUNION ALL\n');
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* The read returns exactly one row per key — see {@link fkOrphanSql}. Anything else is an answer
|
|
245
|
+
* nobody can read, and an unreadable answer is not "no orphans": it is the gate not running.
|
|
246
|
+
*
|
|
247
|
+
* FAIL CLOSED. A driver that hands back `[]` (a shape quirk, a transport that dropped the rows,
|
|
248
|
+
* a future change to the read) would otherwise walk straight through `fkOrphanRefusal`, which
|
|
249
|
+
* reports null — "every key is satisfied" — for an empty list. Asserted here, in the module that
|
|
250
|
+
* owns the read's shape, so the assertion cannot drift from the SQL it is about.
|
|
251
|
+
*/
|
|
252
|
+
export function checkOrphanRowShape(rows: unknown, expected: number): asserts rows is FkOrphanRow[] {
|
|
253
|
+
const got = Array.isArray(rows) ? rows.length : null;
|
|
254
|
+
if (got === expected) return;
|
|
255
|
+
throw new Error(
|
|
256
|
+
`the foreign-key row check answered with ${got === null ? 'something that is not a row set' : `${got} row(s)`} `
|
|
257
|
+
+ `for ${expected} key(s) — the read produces exactly one row per key, so this answer cannot be read.`,
|
|
258
|
+
);
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* The orphan count as a number, or a THROW.
|
|
263
|
+
*
|
|
264
|
+
* It used to coerce null / unreadable text to 0, which is the answer "this key is satisfied" —
|
|
265
|
+
* the single most dangerous default in this module. `0` is a finding; `null` is a missing
|
|
266
|
+
* answer, and the two must never be spelled the same way. The caller turns the throw into the
|
|
267
|
+
* fail-closed refusal an unreachable database gets.
|
|
268
|
+
*/
|
|
269
|
+
const toCount = (v: unknown, where: string): number => {
|
|
270
|
+
if (v === null || v === undefined) throw new Error(`the orphan count for ${where} could not be read (it came back empty).`);
|
|
271
|
+
if (typeof v === 'object') throw new Error(`the orphan count for ${where} could not be read (unexpected shape).`);
|
|
272
|
+
if (typeof v === 'string' && v.trim() === '') throw new Error(`the orphan count for ${where} could not be read (it came back blank).`);
|
|
273
|
+
const n = typeof v === 'bigint' ? Number(v) : Number(v);
|
|
274
|
+
if (!Number.isFinite(n)) throw new Error(`the orphan count for ${where} could not be read (got ${JSON.stringify(String(v))}).`);
|
|
275
|
+
return n;
|
|
276
|
+
};
|
|
277
|
+
|
|
278
|
+
const toSamples = (v: unknown): string[] =>
|
|
279
|
+
Array.isArray(v) ? v.filter((s) => s != null).map((s) => String(s)) : [];
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* The refusal when the read itself could not run (no privilege on a referenced table, a lock
|
|
283
|
+
* timeout, a dropped connection).
|
|
284
|
+
*
|
|
285
|
+
* FAIL CLOSED, the same rule the reconciler applies to grants it cannot see: a gate that did not
|
|
286
|
+
* run has not passed, and an apply that proceeds on an unanswered question is exactly the
|
|
287
|
+
* mid-VALIDATE surprise this preflight exists to remove. Nothing has run when this is raised.
|
|
288
|
+
*/
|
|
289
|
+
export function fkPreflightUnavailableRefusal(error: string): string {
|
|
290
|
+
return [
|
|
291
|
+
`the foreign-key row check could not run against this target — refused before any DDL, because a check that did not run has not passed.`,
|
|
292
|
+
` ${error}`,
|
|
293
|
+
`Grant the applying role SELECT on the tables the keys join, or run the apply from a role that has it, then re-plan and apply.`,
|
|
294
|
+
].join('\n');
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/**
|
|
298
|
+
* The refusal text, or null when every key's rows already satisfy it.
|
|
299
|
+
*
|
|
300
|
+
* It names the COUNT and a SAMPLE per key, because those are the two things the operator needs
|
|
301
|
+
* and neither is in PostgreSQL's own error: how much work the fix is, and what the broken rows
|
|
302
|
+
* look like. The fix is data, so the escape is a data fix and a re-plan — never a flag. There is
|
|
303
|
+
* deliberately no `--force`: an unsatisfiable foreign key is not an intent someone can attest to.
|
|
304
|
+
*
|
|
305
|
+
* THROWS when a row's count cannot be read — see {@link toCount}. `null` returned from here means
|
|
306
|
+
* "asked and satisfied", so an unreadable count must not be able to produce it.
|
|
307
|
+
*/
|
|
308
|
+
export function fkOrphanRefusal(rows: readonly FkOrphanRow[]): string | null {
|
|
309
|
+
const where = (r: FkOrphanRow): string => `${String(r.table_name)}.${String(r.constraint_name)}`;
|
|
310
|
+
const counted = rows.map((r) => ({ row: r, count: toCount(r.orphans, where(r)) }));
|
|
311
|
+
const offenders = counted.filter((c) => c.count > 0);
|
|
312
|
+
if (offenders.length === 0) return null;
|
|
313
|
+
|
|
314
|
+
const lines = [
|
|
315
|
+
`${offenders.length} foreign key(s) cannot be validated — rows already in the table do not satisfy them. Refused before any DDL, so nothing is half-applied.`,
|
|
316
|
+
`Fix the data first (everystack db:exec runs a DML-only file against the target, credential-free), then re-plan and apply.`,
|
|
317
|
+
];
|
|
318
|
+
for (const { row, count } of offenders) {
|
|
319
|
+
const samples = toSamples(row.samples);
|
|
320
|
+
lines.push(
|
|
321
|
+
` ORPHANED ROWS ${String(row.table_name)}.${String(row.constraint_name)}: ${count} row(s) hold a value in (${String(row.columns)}) with no matching row in ${String(row.ref_table)}.`,
|
|
322
|
+
);
|
|
323
|
+
if (samples.length > 0) {
|
|
324
|
+
lines.push(` sample value(s): ${samples.join(', ')}${count > samples.length ? ', …' : ''}`);
|
|
325
|
+
}
|
|
326
|
+
lines.push(
|
|
327
|
+
` the rows: SELECT * FROM ${row.table_name} c WHERE NOT EXISTS (SELECT 1 FROM ${row.ref_table} p WHERE p.<key> = c.<fk>);`,
|
|
328
|
+
);
|
|
329
|
+
}
|
|
330
|
+
return lines.join('\n');
|
|
331
|
+
}
|