@everystack/cli 0.4.58 → 0.4.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/cli/commands/db-backup.ts +19 -30
- package/src/cli/commands/db-fork.ts +10 -10
- package/src/cli/commands/db-reconcile.ts +110 -24
- package/src/cli/commands/db-swap.ts +24 -12
- package/src/cli/commands/db-sync.ts +103 -8
- package/src/cli/db-build.ts +16 -24
- package/src/cli/derived-apply.ts +19 -1
- package/src/cli/derived-compile.ts +59 -1
- package/src/cli/derived-plan.ts +12 -0
- package/src/cli/derived-source.ts +20 -0
- package/src/cli/stage-backup.ts +229 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@everystack/cli",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.60",
|
|
4
4
|
"description": "CLI and OTA updates for Expo apps on everystack",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"author": "Scalable Technology, Inc. <licensing@scalable.technology>",
|
|
@@ -160,7 +160,7 @@
|
|
|
160
160
|
"jest": "29.7.0",
|
|
161
161
|
"react": "19.2.0",
|
|
162
162
|
"ts-jest": "29.4.9",
|
|
163
|
-
"@everystack/server": "0.4.
|
|
163
|
+
"@everystack/server": "0.4.21"
|
|
164
164
|
},
|
|
165
165
|
"scripts": {
|
|
166
166
|
"test": "jest",
|
|
@@ -21,6 +21,7 @@ import { pipeline } from 'node:stream/promises';
|
|
|
21
21
|
import { randomBytes } from 'node:crypto';
|
|
22
22
|
import { resolveConfig, opsFunction, type CliConfig } from '../config.js';
|
|
23
23
|
import { invokeAction, presignGet, getS3 } from '../aws.js';
|
|
24
|
+
import { runStageBackup, stageBackupDeps } from '../stage-backup.js';
|
|
24
25
|
import {
|
|
25
26
|
parseBackupRef, keyForId, crossStageGuard, restoreTargetGuard,
|
|
26
27
|
backupKey, backupId, metaKey, utcStamp,
|
|
@@ -226,39 +227,27 @@ export async function dbBackupCommand(flags: Record<string, string>): Promise<vo
|
|
|
226
227
|
info(`Region: ${config.region}, Function: ${opsFunction(config)}`);
|
|
227
228
|
const fn = opsFunction(config);
|
|
228
229
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
}
|
|
237
|
-
if (dispatched?.error) {
|
|
238
|
-
fail(`Backup failed: ${dispatched.error}`);
|
|
239
|
-
process.exit(1);
|
|
240
|
-
}
|
|
241
|
-
const { runId, taskArn, id, warning } = dispatched as { runId: string; taskArn: string; id: string; warning?: string };
|
|
242
|
-
info(`backup ${id} — task ${taskArn}`);
|
|
243
|
-
info(`run id: ${runId} (everystack.task_log)`);
|
|
244
|
-
if (warning) info(`note: ${warning}`);
|
|
230
|
+
const result = await runStageBackup(stageBackupDeps(config, fn, flags.stage, {
|
|
231
|
+
onDispatch: (d) => {
|
|
232
|
+
info(`backup ${d.id} — task ${d.taskArn}`);
|
|
233
|
+
info(`run id: ${d.runId} (everystack.task_log)`);
|
|
234
|
+
// The dispatch warning is folded into result.warning and printed at the end, so printing it
|
|
235
|
+
// here too says the same thing twice.
|
|
236
|
+
},
|
|
237
|
+
}));
|
|
245
238
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
}
|
|
252
|
-
if (poll.outcome === 'error') {
|
|
253
|
-
fail(`task:status failed repeatedly: ${poll.status.error}. The backup may still be running — reconcile run id ${runId}.`);
|
|
254
|
-
process.exit(1);
|
|
255
|
-
}
|
|
256
|
-
if (poll.status.exitCode !== 0) {
|
|
257
|
-
fail(`Backup failed (exit ${poll.status.exitCode ?? 'unknown'})${poll.status.stoppedReason ? ` — ${poll.status.stoppedReason}` : ''}. Read the task logs (CloudWatch).`);
|
|
239
|
+
if (!result.ok) {
|
|
240
|
+
fail(`Backup failed: ${result.reason}`);
|
|
241
|
+
if (result.orphanKey) {
|
|
242
|
+
info(`The artifact is still in the bucket: s3://${config.backupsBucket ?? '<backups-bucket>'}/${result.orphanKey}`);
|
|
243
|
+
}
|
|
258
244
|
process.exit(1);
|
|
259
245
|
}
|
|
260
|
-
|
|
261
|
-
|
|
246
|
+
|
|
247
|
+
// Render it. The helper produced this and nothing printed it, so the one signal that an artifact
|
|
248
|
+
// was not verified against what the operator sent was invisible at the only moment it mattered.
|
|
249
|
+
if (result.warning) warn(result.warning);
|
|
250
|
+
success(`Backup ${result.id} (${fmtBytes(result.bytes)}). Restore with: everystack db:restore --from ${result.id} --confirm`);
|
|
262
251
|
}
|
|
263
252
|
|
|
264
253
|
/** db:backups — list a stage's logical backups. */
|
|
@@ -23,6 +23,7 @@ import { resolveConfig, opsFunction, type CliConfig } from '../config.js';
|
|
|
23
23
|
import { invokeAction, presignGet } from '../aws.js';
|
|
24
24
|
import { keyForId, parseBackupRef, isProductionTier, crossStageGuard } from '../backup.js';
|
|
25
25
|
import { pollTaskUntilStopped } from '../task-poll.js';
|
|
26
|
+
import { runStageBackup, stageBackupDeps } from '../stage-backup.js';
|
|
26
27
|
import { step, success, fail, info, warn } from '../output.js';
|
|
27
28
|
|
|
28
29
|
export type ForkGuardVerdict =
|
|
@@ -94,18 +95,17 @@ export async function dbForkCommand(flags: Record<string, string>): Promise<void
|
|
|
94
95
|
info(`Reusing backup ${backupId}.`);
|
|
95
96
|
} else {
|
|
96
97
|
step(`Backing up ${from} (pg_dump → S3 in the Task; may take a while for large databases)...`);
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
if (poll.outcome !== 'stopped' || poll.status.exitCode !== 0) {
|
|
105
|
-
const why = poll.outcome === 'stopped' ? `exit ${poll.status.exitCode}${poll.status.stoppedReason ? ` — ${poll.status.stoppedReason}` : ''}` : poll.outcome;
|
|
106
|
-
fail(`Source backup failed (${why}). Reconcile run ${dispatched.runId}.`);
|
|
98
|
+
// The shared sandwich: a fork built from a dump that spans a migration copies a schema that
|
|
99
|
+
// never existed. Same helper db:backup and db:swap use, so all three agree on what a proven
|
|
100
|
+
// artifact is.
|
|
101
|
+
const backup = await runStageBackup(stageBackupDeps(source, opsFunction(source), from, { onStep: step }));
|
|
102
|
+
if (!backup.ok) {
|
|
103
|
+
fail(`Source backup failed: ${backup.reason} The fork was NOT created.`);
|
|
104
|
+
if (backup.orphanKey) info(`The artifact is still in the bucket: s3://${source.backupsBucket}/${backup.orphanKey}`);
|
|
107
105
|
process.exit(1);
|
|
108
106
|
}
|
|
107
|
+
backupId = backup.id;
|
|
108
|
+
if (backup.warning) warn(backup.warning);
|
|
109
109
|
info(`Backup ${backupId}.`);
|
|
110
110
|
}
|
|
111
111
|
|
|
@@ -194,11 +194,19 @@ export async function executeReconcile(
|
|
|
194
194
|
// renderEnsureObjectSchemas.
|
|
195
195
|
const ensureSchemas = renderEnsureObjectSchemas(parsed.objects);
|
|
196
196
|
|
|
197
|
-
// Which phase we are in when something throws.
|
|
198
|
-
//
|
|
197
|
+
// Which phase we are in when something throws. Anything after the DDL rolls the DDL back with
|
|
198
|
+
// it inside the atomic path, so the operator sees no plan output and no applied statements —
|
|
199
199
|
// indistinguishable from a batch that never started, and the wrong thing to go debugging.
|
|
200
200
|
// The phase is the difference between "your DDL is bad" and "your DDL was fine".
|
|
201
|
-
|
|
201
|
+
//
|
|
202
|
+
// THREE phases, not two, because the post-apply work has two halves that fail for different
|
|
203
|
+
// reasons and are fixed differently. `verify` is everything between the DDL and the write: the
|
|
204
|
+
// catalog re-read, and the checks that stand between it and recording (a recorded identity with
|
|
205
|
+
// no source object, an object the catalog cannot see afterwards, a backend pid that moved). The
|
|
206
|
+
// DDL ran and NO bookkeeping was ever attempted. `bookkeeping` is the single provenance +
|
|
207
|
+
// schema_log write, plus the COMMIT that seals it. Reporting a re-read failure as `bookkeeping`
|
|
208
|
+
// sent the operator looking at a write that never happened.
|
|
209
|
+
let phase: 'ddl' | 'verify' | 'bookkeeping' = 'ddl';
|
|
202
210
|
|
|
203
211
|
if (atomic) await runner('BEGIN');
|
|
204
212
|
try {
|
|
@@ -236,11 +244,47 @@ export async function executeReconcile(
|
|
|
236
244
|
// the canonical search_path inside a savepoint it always rolls back, so the def hashes are
|
|
237
245
|
// stable regardless of the create-time wide path above — and that wide path survives the read.
|
|
238
246
|
// The unwrapped path has no transaction to borrow, so it takes a session of its own.
|
|
239
|
-
phase = '
|
|
247
|
+
phase = 'verify';
|
|
240
248
|
const after = await introspectDerived(atomic ? borrowedSessionRunner(runner) : session);
|
|
241
249
|
const liveById = new Map(after.objects.map((o) => [o.identity, o]));
|
|
242
250
|
const srcById = new Map(parsed.objects.map((o) => [o.identity, o]));
|
|
243
251
|
|
|
252
|
+
// THE ORPHAN CHECK — an outcome assertion, not a plan-shape one. Every identity whose
|
|
253
|
+
// provenance we are about to DELETE must be gone from the database. The read above is the
|
|
254
|
+
// real post-DDL catalog, so this asks the only question that matters: after this batch, is
|
|
255
|
+
// there a live object whose bookkeeping we are about to throw away?
|
|
256
|
+
//
|
|
257
|
+
// A live object with no provenance row reads as `unmanaged` — "not in source, never touched"
|
|
258
|
+
// — so it is never dropped, keeps its grants, and `--check` stays green. Nothing downstream
|
|
259
|
+
// can tell that state apart from an object that was genuinely never ours, which is why this
|
|
260
|
+
// has to be caught at the moment the row is lost rather than found afterwards.
|
|
261
|
+
//
|
|
262
|
+
// Reachable, and the mechanism is a search_path disagreement, NOT a declared spelling (drop
|
|
263
|
+
// identities come from the live catalog, never from the declaration). Introspection pins
|
|
264
|
+
// `search_path = public`, so a public-schema type renders BARE inside a function identity;
|
|
265
|
+
// the DDL batch omits its own SET when every declared schema is public; and under an ambient
|
|
266
|
+
// `ALTER DATABASE … SET search_path = app` the rendered `DROP FUNCTION IF EXISTS
|
|
267
|
+
// public.f(my_enum)` cannot resolve `my_enum`. PostgreSQL answers `NOTICE: type "my_enum"
|
|
268
|
+
// does not exist, skipping` and drops nothing. IF EXISTS makes that silent.
|
|
269
|
+
//
|
|
270
|
+
// A `defineSql` object never trips this — not because it is catalog-invisible (a defineSql
|
|
271
|
+
// function or procedure is perfectly visible) but because its identity is `schema.name`
|
|
272
|
+
// while the catalog reports `schema.name(argtypes)`, so the two never join.
|
|
273
|
+
const orphaned = rendered.remove.filter((identity) => liveById.has(identity));
|
|
274
|
+
if (orphaned.length > 0) {
|
|
275
|
+
throw new Error(
|
|
276
|
+
`refusing to drop provenance for ${orphaned.map((i) => `${i} (${liveById.get(i)!.kind})`).join(', ')}: `
|
|
277
|
+
+ `the object is STILL LIVE after the batch, so its DROP did not remove it. DROP ... IF EXISTS is `
|
|
278
|
+
+ `silent when a name in the target cannot be resolved — check the database's search_path `
|
|
279
|
+
+ `(SHOW search_path; ALTER DATABASE ... SET search_path) against the identity above. Deleting the `
|
|
280
|
+
+ `row anyway would leave the object running with its grants, reported as "unmanaged", and never `
|
|
281
|
+
+ `dropped again.\n\nNo provenance was written. There is no automated recovery for this state yet — `
|
|
282
|
+
+ `the generic guidance below names --baseline and --rebuild, and NEITHER applies here: both act on `
|
|
283
|
+
+ `objects the source still declares, and this one is absent from source. Drop the object by hand `
|
|
284
|
+
+ `once its name resolves, then re-run.`,
|
|
285
|
+
);
|
|
286
|
+
}
|
|
287
|
+
|
|
244
288
|
const bookkeeping: string[] = [];
|
|
245
289
|
// ORDER MATTERS: migrate, then record, then remove. This ordering is LOAD-BEARING — it is the
|
|
246
290
|
// fix, not the belt. `renderProvenanceMigrate`'s own guard is the belt (see its docstring).
|
|
@@ -254,11 +298,12 @@ export async function executeReconcile(
|
|
|
254
298
|
// Migrating first leaves the row already keyed by its new identity, so the upsert updates it.
|
|
255
299
|
//
|
|
256
300
|
// The guard alone would NOT be enough. Under record-first it makes one case actively worse: a
|
|
257
|
-
// source 'sql'-kind object sharing a legacy function identity (derived-plan.ts
|
|
258
|
-
// `prov.kind === 'sql'` but not `src.kind`
|
|
259
|
-
//
|
|
260
|
-
//
|
|
261
|
-
//
|
|
301
|
+
// source 'sql'-kind object sharing a legacy function identity (derived-plan.ts skips on
|
|
302
|
+
// `prov.kind === 'sql'` but not `src.kind` — deliberately, see the DELIBERATELY NOT SKIPPED
|
|
303
|
+
// note there) gets its freshly recorded row deleted by the guard, orphaning its provenance for
|
|
304
|
+
// good. Migrating first is what prevents that.
|
|
305
|
+
// PINNED BY __tests__/integration/provenance-migrate.test.ts — that scenario is the only one
|
|
306
|
+
// where the two orders diverge, and it is the ONLY test that reverting this reorder turns red.
|
|
262
307
|
for (const m of rendered.migrate) bookkeeping.push(renderProvenanceMigrate(m.from, m.to));
|
|
263
308
|
for (const identity of rendered.record) {
|
|
264
309
|
const src = srcById.get(identity);
|
|
@@ -299,6 +344,9 @@ export async function executeReconcile(
|
|
|
299
344
|
);
|
|
300
345
|
}
|
|
301
346
|
}
|
|
347
|
+
// Everything after the DDL has so far only read and rendered. From here the phase is the
|
|
348
|
+
// bookkeeping write itself, and the COMMIT that seals it.
|
|
349
|
+
phase = 'bookkeeping';
|
|
302
350
|
await runner(bookkeeping.join(';\n'));
|
|
303
351
|
|
|
304
352
|
if (atomic) await runner('COMMIT');
|
|
@@ -330,24 +378,36 @@ export async function executeReconcile(
|
|
|
330
378
|
* cryptic unique-violation on `pg_type`/`pg_class` instead of anything actionable. Turn that into
|
|
331
379
|
* a message that tells the operator what to do.
|
|
332
380
|
*
|
|
333
|
-
* `phase` says WHICH
|
|
381
|
+
* `phase` says WHICH part died. A post-DDL failure rolls the DDL back with it, so the operator's
|
|
334
382
|
* only evidence is an absence — no plan output, nothing applied — which reads as a batch that never
|
|
335
|
-
* started. Say plainly that the DDL was fine, or the next person debugs the wrong half.
|
|
383
|
+
* started. Say plainly that the DDL was fine, or the next person debugs the wrong half. `verify` and
|
|
384
|
+
* `bookkeeping` are named apart because they fail for different reasons: the first is the catalog
|
|
385
|
+
* re-read and the checks that stand between it and recording, the second is the provenance/schema_log
|
|
386
|
+
* write and its COMMIT.
|
|
336
387
|
*/
|
|
337
|
-
export function explainReconcileError(err: unknown, phase: 'ddl' | 'bookkeeping' = 'ddl', atomic = true): Error {
|
|
388
|
+
export function explainReconcileError(err: unknown, phase: 'ddl' | 'verify' | 'bookkeeping' = 'ddl', atomic = true): Error {
|
|
338
389
|
const msg = String((err as any)?.message ?? err);
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
390
|
+
// PHASE FIRST — the order is load-bearing. The name-collision advice below tells the operator to
|
|
391
|
+
// DROP … CASCADE, which is destructive and simply wrong for a failure that happened after the DDL
|
|
392
|
+
// (there, the create already succeeded).
|
|
393
|
+
//
|
|
394
|
+
// No post-DDL error reaches that text TODAY, and the reason is what to re-check before trusting
|
|
395
|
+
// this: neither post-DDL phase runs a CREATE. `verify` is SELECTs plus in-process throws;
|
|
396
|
+
// `bookkeeping` is INSERT/UPDATE/DELETE on two everystack tables. Nothing there can raise 42P07
|
|
397
|
+
// or 42710. (`already exists` IS a primary message for 42P07 — the regex would match it fine.)
|
|
398
|
+
// So the ordering is insurance. It stops being insurance the moment a phase gains a CREATE —
|
|
399
|
+
// moving ENSURE_RECONCILER_SQL, with its CREATE … IF NOT EXISTS, inside the try would do it.
|
|
400
|
+
if (phase === 'verify' || phase === 'bookkeeping') {
|
|
348
401
|
return new Error(
|
|
349
|
-
`${msg}\n\nThis failed AFTER the DDL, not in it.
|
|
350
|
-
+
|
|
402
|
+
`${msg}\n\nThis failed AFTER the DDL, not in it. `
|
|
403
|
+
+ (phase === 'verify'
|
|
404
|
+
? `The failure came from the verify half: the post-apply catalog re-read, or one of the checks `
|
|
405
|
+
+ `over it — a recorded identity with no source object, an object the catalog cannot see `
|
|
406
|
+
+ `afterwards, or a backend pid that moved. No provenance and no 'applied' schema_log row `
|
|
407
|
+
+ `was written; the bookkeeping write is never reached`
|
|
408
|
+
: `The failure came from the bookkeeping write that records what was built — the provenance and `
|
|
409
|
+
+ `schema_log statements, or the COMMIT that seals them. The post-apply catalog re-read `
|
|
410
|
+
+ `succeeded, so every recorded object was introspectable when it ran`)
|
|
351
411
|
+ (atomic
|
|
352
412
|
? `. That work shares the DDL's transaction, so the transaction rolled EVERYTHING back and nothing is `
|
|
353
413
|
+ `applied — the database is exactly as it was. Resolve the cause and re-run.`
|
|
@@ -360,6 +420,14 @@ export function explainReconcileError(err: unknown, phase: 'ddl' | 'bookkeeping'
|
|
|
360
420
|
+ `WHERE outcome LIKE 'failed%' ORDER BY applied_at DESC LIMIT 3;`,
|
|
361
421
|
);
|
|
362
422
|
}
|
|
423
|
+
if (/pg_type_typname_nsp_index|pg_class_relname_nsp_index|already exists/i.test(msg)) {
|
|
424
|
+
return new Error(
|
|
425
|
+
`${msg}\n\nreconcile tried to CREATE a derived object whose name already exists — usually a partial `
|
|
426
|
+
+ `earlier run or a hand-created object (its create is not idempotent). Resolve by explicitly dropping `
|
|
427
|
+
+ `the colliding object (DROP MATERIALIZED VIEW / VIEW … CASCADE) and re-running, or, if the plan reports `
|
|
428
|
+
+ `it as drift, re-run with --overwrite-drift.`,
|
|
429
|
+
);
|
|
430
|
+
}
|
|
363
431
|
return err instanceof Error ? err : new Error(msg);
|
|
364
432
|
}
|
|
365
433
|
|
|
@@ -385,6 +453,15 @@ export function buildReconcileReport(plan: ReconcilePlan): string[] {
|
|
|
385
453
|
const cost = a.bytes !== undefined ? `; ${formatBytes(a.bytes)}${a.rows ? `, ~${a.rows} rows` : ''}` : '';
|
|
386
454
|
lines.push(`${VERB_GLYPH[a.action] ?? '?'} ${a.action} ${a.identity} (${a.reason}${cost})`);
|
|
387
455
|
}
|
|
456
|
+
// Pending identity migrations. Bookkeeping only — no DDL, no object changed — but they ARE a
|
|
457
|
+
// write, and they were invisible here while also being excluded from `checkFails`. Silence in
|
|
458
|
+
// both places is how a re-key nobody asked about becomes a surprise.
|
|
459
|
+
for (const m of plan.migrations) {
|
|
460
|
+
lines.push(
|
|
461
|
+
`⇢ re-key ${m.from} → ${m.to} (provenance only — the identity spelling this cli reports; `
|
|
462
|
+
+ `pending until --apply, and it does NOT fail --check)`,
|
|
463
|
+
);
|
|
464
|
+
}
|
|
388
465
|
const rebuilt = plan.actions.filter((a) => (a.action === 'create' || a.action === 'refresh') && a.bytes !== undefined);
|
|
389
466
|
if (plan.totalRebuildBytes > 0) {
|
|
390
467
|
lines.push(`estimated rebuild: ${formatBytes(plan.totalRebuildBytes)} across ${rebuilt.length} matview(s)`);
|
|
@@ -416,7 +493,16 @@ export function buildReconcileReport(plan: ReconcilePlan): string[] {
|
|
|
416
493
|
* Baseline and rebaseline are explicit adoption the operator asked for, not findings; a
|
|
417
494
|
* backfill is record-only bookkeeping (body_hash arming) — neither is a CI failure. An
|
|
418
495
|
* UNAPPLIED authz-only grant change still fails here via `regrants` below (the real signal);
|
|
419
|
-
* an empty regrant delta means live already matches declared, so green is correct.
|
|
496
|
+
* an empty regrant delta means live already matches declared, so green is correct.
|
|
497
|
+
*
|
|
498
|
+
* `plan.migrations` is DELIBERATELY not here, for the backfill's reason. An identity migration
|
|
499
|
+
* re-keys a provenance row a cli upgrade under-keyed; it emits no DDL, changes no object, and
|
|
500
|
+
* converges in one apply. Failing on it would turn CI red for every consumer on the release that
|
|
501
|
+
* introduces the new identity spelling, and the only fix on offer would be "run apply" — a gate
|
|
502
|
+
* whose remedy is the thing the gate is supposed to be checking for. The live layer still matches
|
|
503
|
+
* the declared source, which is what `--check` asserts. Decided 2026-08-12; reversible by adding
|
|
504
|
+
* one clause. Green here is only defensible because the plan report PRINTS every pending
|
|
505
|
+
* migration (see buildReconcileReport) — silent in both places would be a write nobody can see. */
|
|
420
506
|
export function checkFails(plan: ReconcilePlan): boolean {
|
|
421
507
|
return plan.actions.some((a) => a.action !== 'baseline' && a.action !== 'rebaseline' && a.action !== 'backfill')
|
|
422
508
|
|| plan.drift.length > 0
|
|
@@ -45,6 +45,7 @@ import { keyForArtifactId, metaKey, utcStamp } from '../backup.js';
|
|
|
45
45
|
import { rdsSnapshotIdentifier } from '../rds-snapshot.js';
|
|
46
46
|
import { pollTaskUntilStopped } from '../task-poll.js';
|
|
47
47
|
import { decideSnapshotMode, confirmPhysicalSnapshot, interpretBackupPoll, type SnapshotModeRequest } from '../swap-snapshot.js';
|
|
48
|
+
import { runStageBackup, stageBackupDeps } from '../stage-backup.js';
|
|
48
49
|
import { pgEnvFromUrl, pgKeepaliveConninfo } from './db.js';
|
|
49
50
|
import { step, success, fail, warn, info } from '../output.js';
|
|
50
51
|
|
|
@@ -475,19 +476,30 @@ async function takePreSwapSnapshot(
|
|
|
475
476
|
// Logical: dispatch the Task and WAIT. The dispatch returning is not the backup existing — that
|
|
476
477
|
// conflation is what let a pg_dump run concurrently with the restore it was supposed to precede.
|
|
477
478
|
step('Snapshotting the stage before the swap (db:backup — waiting for the dump to finish)...');
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
479
|
+
// The shared sandwich. A rollback point taken across a concurrent migration holds a schema that
|
|
480
|
+
// matches neither endpoint — restoring from it would not return the stage to where it was, which
|
|
481
|
+
// is the one thing a pre-swap snapshot has to guarantee. An unproven artifact fails the swap.
|
|
482
|
+
const backup = await runStageBackup({
|
|
483
|
+
...stageBackupDeps({ region: ctx.region! }, ctx.opsFn!, ctx.stage, {
|
|
484
|
+
onStep: step,
|
|
485
|
+
onDispatch: (d: { id: string; runId: string }) => info(`backup ${d.id} dispatched (run ${d.runId}) — waiting for the task to stop before the restore starts.`),
|
|
486
|
+
}),
|
|
487
|
+
// Swap keeps its OWN reading of a finished poll: it is the only caller that can tell the
|
|
488
|
+
// operator to re-run with `--snapshot-ref <id>` once the dump lands, and each outcome needs
|
|
489
|
+
// different advice. The shared helper takes injected deps for exactly this.
|
|
490
|
+
awaitTask: async (ids) => {
|
|
491
|
+
const poll = await pollTaskUntilStopped(ctx.region!, ctx.opsFn!, { runId: ids.runId, taskArn: ids.taskArn });
|
|
492
|
+
const verdict = interpretBackupPoll(poll, { runId: ids.runId, id: ids.id });
|
|
493
|
+
if (!verdict.ok) return verdict;
|
|
494
|
+
// Carry the stamp through, or this caller silently opts out of the image-skew check.
|
|
495
|
+
return { ok: true, stampedFingerprint: (poll as any).status?.result?.fingerprint as string | undefined };
|
|
496
|
+
},
|
|
481
497
|
});
|
|
482
|
-
if (
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
{ runId, id },
|
|
488
|
-
);
|
|
489
|
-
if (!verdict.ok) throw new Error(verdict.reason);
|
|
490
|
-
info(`rollback point CONFIRMED: backup ${id} complete — restore with db:restore --from ${id} --confirm.`);
|
|
498
|
+
if (!backup.ok) {
|
|
499
|
+
throw new Error(`the pre-swap backup is not a usable rollback point, so the swap was NOT applied and live is untouched: ${backup.reason}`);
|
|
500
|
+
}
|
|
501
|
+
if (backup.warning) warn(backup.warning);
|
|
502
|
+
info(`rollback point CONFIRMED: backup ${backup.id} complete — restore with db:restore --from ${backup.id} --confirm.`);
|
|
491
503
|
}
|
|
492
504
|
|
|
493
505
|
export async function dbSwapCommand(flags: Record<string, string>): Promise<void> {
|
|
@@ -88,8 +88,9 @@ export interface SyncOptions {
|
|
|
88
88
|
}
|
|
89
89
|
|
|
90
90
|
export interface SyncHooks {
|
|
91
|
-
/** Fires after the state diff is computed, BEFORE anything executes — the review seam.
|
|
92
|
-
|
|
91
|
+
/** Fires after the state diff is computed, BEFORE anything executes — the review seam.
|
|
92
|
+
* `pass` is 1, or 2 on the settling pass (see {@link executeSync}). */
|
|
93
|
+
onStatePlan?: (statements: string[], classified: ClassifiedStatements, pass: number) => void;
|
|
93
94
|
/** Fires after the state layer applied and verified, before the compute reconcile. */
|
|
94
95
|
onStateDone?: (state: StateSyncOutcome) => void;
|
|
95
96
|
}
|
|
@@ -114,12 +115,13 @@ export interface SyncRun {
|
|
|
114
115
|
* compute, then the verdict. Pure orchestration over an injected QueryRunner;
|
|
115
116
|
* the CLI shell below owns flags, files, and exits.
|
|
116
117
|
*/
|
|
117
|
-
|
|
118
|
+
async function syncOnce(
|
|
118
119
|
runner: QueryRunner,
|
|
119
120
|
session: SessionRunner,
|
|
120
121
|
models: ModelDescriptor[],
|
|
121
|
-
options: SyncOptions
|
|
122
|
-
hooks: SyncHooks
|
|
122
|
+
options: SyncOptions,
|
|
123
|
+
hooks: SyncHooks,
|
|
124
|
+
pass: number,
|
|
123
125
|
): Promise<SyncRun> {
|
|
124
126
|
const current = await introspectSchema(session);
|
|
125
127
|
const liveAuthz = await introspectContract(session, contractFunctionRow, FUNCTIONS_SQL);
|
|
@@ -127,7 +129,7 @@ export async function executeSync(
|
|
|
127
129
|
allowDrops: options.allowDrops, liveAuthz, sequences: options.sequences,
|
|
128
130
|
governedRoles: options.governedRoles, extensions: options.extensions,
|
|
129
131
|
});
|
|
130
|
-
hooks.onStatePlan?.(statements, classifyGeneratedStatements(statements));
|
|
132
|
+
hooks.onStatePlan?.(statements, classifyGeneratedStatements(statements), pass);
|
|
131
133
|
|
|
132
134
|
const state = await applyStateAndVerify(runner, session, models, statements, current, liveAuthz, {
|
|
133
135
|
allowDrops: options.allowDrops, sequences: options.sequences, extensions: options.extensions,
|
|
@@ -167,6 +169,92 @@ export async function executeSync(
|
|
|
167
169
|
};
|
|
168
170
|
}
|
|
169
171
|
|
|
172
|
+
/**
|
|
173
|
+
* A sync that SETTLES — at most one extra pass, and the reason it is needed is inherent to
|
|
174
|
+
* creating things rather than a patch over one bug.
|
|
175
|
+
*
|
|
176
|
+
* `syncOnce` is a single diff-and-verify: it reads the live authz contract, computes the delta,
|
|
177
|
+
* applies it. On a FROM-SCRATCH run some objects the authz layer must grant on do not exist at
|
|
178
|
+
* read time — most sharply the sequence behind a serial column, which our own CREATE TABLE makes
|
|
179
|
+
* moments later. The first pass cannot see it, the sequence inheritance rule (A6) has nothing to
|
|
180
|
+
* grant on, and the run ends with a role that may INSERT into a table but cannot draw its
|
|
181
|
+
* sequence value. Measured on a consumer's 251-model schema: pass one reported `MATCH` and
|
|
182
|
+
* `NOT converged` in the same breath, and pass two applied 205 statements, nearly all of them
|
|
183
|
+
* `GRANT … ON SEQUENCE`.
|
|
184
|
+
*
|
|
185
|
+
* `buildIntoDatabase` has carried this second pass since A12, so `db:build`, `db:check`'s
|
|
186
|
+
* compose ring and `db:template:refresh` always settled. The `db:sync` VERB did not — it ran
|
|
187
|
+
* once and told the operator to run it again. That put "run db:sync twice" in a consumer's
|
|
188
|
+
* runbook as folklore, and its exit 1 aborted their documented one-verb from-scratch path
|
|
189
|
+
* before any data loaded. The retry belongs HERE, where every caller gets it and none has to
|
|
190
|
+
* know.
|
|
191
|
+
*
|
|
192
|
+
* Bounded at exactly one extra pass. The second read sees everything the first one created, so
|
|
193
|
+
* a third could only differ if the apply were non-convergent — and that is a real failure the
|
|
194
|
+
* `converged` bar must report, never something to loop away.
|
|
195
|
+
*
|
|
196
|
+
* An already-synced database converges on pass one and never pays for the second read.
|
|
197
|
+
*/
|
|
198
|
+
export async function executeSync(
|
|
199
|
+
runner: QueryRunner,
|
|
200
|
+
session: SessionRunner,
|
|
201
|
+
models: ModelDescriptor[],
|
|
202
|
+
options: SyncOptions = {},
|
|
203
|
+
hooks: SyncHooks = {},
|
|
204
|
+
): Promise<SyncRun> {
|
|
205
|
+
const first = await syncOnce(runner, session, models, options, hooks, 1);
|
|
206
|
+
// The trigger is the STATE bar, not the combined verdict. The cause above is state-layer
|
|
207
|
+
// only, so a compute refusal or an unadopted object cannot be settled by looking again —
|
|
208
|
+
// retrying on those would make every brownfield sync pay a second full read (and print its
|
|
209
|
+
// refusal twice) to reach the same answer.
|
|
210
|
+
if (first.stateConverged) return first;
|
|
211
|
+
const second = await syncOnce(runner, session, models, options, hooks, 2);
|
|
212
|
+
return {
|
|
213
|
+
...second,
|
|
214
|
+
state: mergeStateOutcomes(first.state, second.state),
|
|
215
|
+
// Report the reconcile that DID the work. The settling pass re-reads the derived layer the
|
|
216
|
+
// first pass created and correctly finds it managed and unchanged — reporting THAT as the
|
|
217
|
+
// outcome tells the operator "N up to date" about objects this very command just created,
|
|
218
|
+
// and hands `--json` a `compute.applied: false` for a successful build. Findings are not
|
|
219
|
+
// lost either way: a refusal, drift and needsBaseline are recomputed from live state on
|
|
220
|
+
// every pass, so the second run reports them again.
|
|
221
|
+
compute: first.compute?.applied && !second.compute?.applied ? first.compute : second.compute,
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Two passes, reported as the one operation the operator ran.
|
|
227
|
+
*
|
|
228
|
+
* The split is WORK DONE versus WORK PENDING, and each field belongs to one of them.
|
|
229
|
+
*
|
|
230
|
+
* Work done accumulates: `executable` is what actually ran, in the order it ran, and
|
|
231
|
+
* `fromFingerprint` is where the database ACTUALLY started — take the settling pass's and the
|
|
232
|
+
* record reads as though the first pass never happened.
|
|
233
|
+
*
|
|
234
|
+
* Work pending is whatever is STILL pending at the end, so it comes from the second pass alone.
|
|
235
|
+
* `heldDrops` and `notices` are regenerated from live state on every pass and are never
|
|
236
|
+
* executed, so concatenating them counts the same held `DROP TABLE` twice — and a destructive
|
|
237
|
+
* change reported at 2x is the wrong direction to be wrong in.
|
|
238
|
+
*/
|
|
239
|
+
function mergeStateOutcomes(first: StateSyncOutcome, second: StateSyncOutcome): StateSyncOutcome {
|
|
240
|
+
return {
|
|
241
|
+
// `remaining`, `toFingerprint`, `after` and the pending buckets all come from the second
|
|
242
|
+
// pass through this spread — the END state is the bar.
|
|
243
|
+
...second,
|
|
244
|
+
statements: [...first.statements, ...second.statements],
|
|
245
|
+
classified: {
|
|
246
|
+
executable: [...first.classified.executable, ...second.classified.executable],
|
|
247
|
+
heldDrops: second.classified.heldDrops,
|
|
248
|
+
notices: second.classified.notices,
|
|
249
|
+
},
|
|
250
|
+
applied: first.applied || second.applied,
|
|
251
|
+
fromFingerprint: first.fromFingerprint,
|
|
252
|
+
// Two passes that both ran DDL wrote TWO schema_log rows, correctly chained (A→B, B→C).
|
|
253
|
+
// This carries the later one; `schema_log` itself is the record of the pair, not this field.
|
|
254
|
+
logId: second.logId ?? first.logId,
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
|
|
170
258
|
// ---------------------------------------------------------------------------
|
|
171
259
|
// Report rendering (pure).
|
|
172
260
|
// ---------------------------------------------------------------------------
|
|
@@ -327,9 +415,16 @@ export async function dbSyncCommand(flags: Record<string, string>): Promise<void
|
|
|
327
415
|
gitRef: currentGitRef(),
|
|
328
416
|
},
|
|
329
417
|
{
|
|
330
|
-
onStatePlan: (statements, classified) => {
|
|
418
|
+
onStatePlan: (statements, classified, pass) => {
|
|
331
419
|
if (classified.executable.length > 0) {
|
|
332
|
-
|
|
420
|
+
// The settling pass is named, not hidden — an operator who sees a second plan with
|
|
421
|
+
// no explanation reads it as the sync failing. It says WHAT it is, never why: on a
|
|
422
|
+
// fresh database this is the grants for sequences the first pass created, but a
|
|
423
|
+
// genuinely non-convergent apply lands here too, and labelling that as routine
|
|
424
|
+
// settling would explain away the one failure this pass exists to surface.
|
|
425
|
+
info(pass > 1
|
|
426
|
+
? `settling pass — ${classified.executable.length} statement(s) still differing after the first pass:`
|
|
427
|
+
: `state plan — ${classified.executable.length} statement(s):`);
|
|
333
428
|
const { drops, narrowings } = classifyDestructive(classified.executable);
|
|
334
429
|
if (drops.length + narrowings.length > 0) {
|
|
335
430
|
warn(`${drops.length + narrowings.length} DESTRUCTIVE — ${drops.length} drop(s), ${narrowings.length} narrowing type change(s). Dev sync runs them; protected stages gate them (db:plan → db:apply).`);
|
package/src/cli/db-build.ts
CHANGED
|
@@ -168,7 +168,17 @@ export async function buildIntoDatabase(
|
|
|
168
168
|
//
|
|
169
169
|
// Provenance is recorded by this pass, so executeSync's own reconcile below sees the
|
|
170
170
|
// functions already managed and unchanged, and plans nothing for them.
|
|
171
|
-
|
|
171
|
+
// …and only the functions that CAN go first. A function's return and argument types are
|
|
172
|
+
// resolved by PostgreSQL when the function is created, and `check_function_bodies` does not
|
|
173
|
+
// reach them — it relaxes the BODY, which is why a body reading a not-yet-created table is
|
|
174
|
+
// fine here and `RETURNS SETOF stats_view.x_row` is not. A function whose SIGNATURE names a
|
|
175
|
+
// relation the declared state has not built yet stays behind and is created by the sync
|
|
176
|
+
// pass below, after state, in the compiler's dependency order. Measured on a consumer's
|
|
177
|
+
// declared state: 20 such pairs, `type "…_row" does not exist`, from-scratch only. Their
|
|
178
|
+
// explicit `dependsOn` could not have helped — this pass dropped the view from the wave, so
|
|
179
|
+
// no edge could order it back in.
|
|
180
|
+
const declaredFunctions = (options.declared ?? [])
|
|
181
|
+
.filter((o) => o.kind === 'function' && !o.signatureDeps?.length);
|
|
172
182
|
if (declaredFunctions.length > 0) {
|
|
173
183
|
await runner('SET check_function_bodies = off');
|
|
174
184
|
try {
|
|
@@ -183,34 +193,16 @@ export async function buildIntoDatabase(
|
|
|
183
193
|
await runner('RESET check_function_bodies');
|
|
184
194
|
}
|
|
185
195
|
}
|
|
186
|
-
|
|
196
|
+
// A12's second pass used to live HERE, and that is exactly why `db:sync` never settled: the
|
|
197
|
+
// from-scratch build got it and the verb an operator runs did not. `executeSync` owns it now
|
|
198
|
+
// (see its own comment for the sequence-behind-a-serial-column cause), so every caller
|
|
199
|
+
// settles and none has to know.
|
|
200
|
+
const run = await executeSync(runner, session, models, {
|
|
187
201
|
declared: options.declared,
|
|
188
202
|
sequences: options.sequences,
|
|
189
203
|
actor: options.actor ?? 'db-build',
|
|
190
204
|
gitRef: options.gitRef ?? null,
|
|
191
205
|
});
|
|
192
|
-
// A12 — THE SECOND PASS, and it is inherent to building from nothing rather than a patch
|
|
193
|
-
// over one bug. `executeSync` is a single diff-and-verify: it reads the live authz contract,
|
|
194
|
-
// computes the delta, applies it. On a FROM-SCRATCH build some objects the authz layer must
|
|
195
|
-
// grant on do not exist at read time — most sharply the sequence behind a serial column,
|
|
196
|
-
// which our own CREATE TABLE makes moments later. So the first pass cannot see it, the
|
|
197
|
-
// sequence inheritance rule (A6) has nothing to grant on, and the build lands with a role
|
|
198
|
-
// that may INSERT into a table but cannot draw its sequence value.
|
|
199
|
-
//
|
|
200
|
-
// Caught by the round-trip oracle (B7), not by A6's own tests: the rule converges perfectly
|
|
201
|
-
// against an EXISTING database, which is what those tests exercise.
|
|
202
|
-
//
|
|
203
|
-
// Bounded at exactly one extra pass. The second read sees everything the first one created,
|
|
204
|
-
// so a third could only differ if the apply were non-convergent — and that is a real failure
|
|
205
|
-
// the `converged` bar must report, never something to loop away.
|
|
206
|
-
if (!run.converged) {
|
|
207
|
-
run = await executeSync(runner, session, models, {
|
|
208
|
-
declared: options.declared,
|
|
209
|
-
sequences: options.sequences,
|
|
210
|
-
actor: options.actor ?? 'db-build',
|
|
211
|
-
gitRef: options.gitRef ?? null,
|
|
212
|
-
});
|
|
213
|
-
}
|
|
214
206
|
return {
|
|
215
207
|
converged: run.converged,
|
|
216
208
|
fingerprintMatch: run.fingerprintMatch,
|
package/src/cli/derived-apply.ts
CHANGED
|
@@ -487,6 +487,14 @@ export function renderReconcileSql(plan: ReconcilePlan, source: SourceObject[],
|
|
|
487
487
|
// REVOKE/GRANT — no provenance to record (ACLs are not in any hash).
|
|
488
488
|
for (const r of plan.regrants) statements.push(...r.statements);
|
|
489
489
|
|
|
490
|
+
// NOTE — a `remove` entry that is also a migration's TARGET is CORRECT, not a contradiction.
|
|
491
|
+
// A legacy-keyed function the source has removed migrates onto its signature identity and is
|
|
492
|
+
// then dropped through it: re-key, drop the object, delete the row. That is the documented path
|
|
493
|
+
// (the pre-pass reads the LIVE catalog precisely so a removed function drops instead of reading
|
|
494
|
+
// as unmanaged) and it converges. A guard here refusing that overlap was tried and reverted —
|
|
495
|
+
// it turned a right answer into a failed apply, and on `db:sync` it threw after the state layer
|
|
496
|
+
// had already committed. Pinned by "a removed legacy function renders as migrate-then-remove"
|
|
497
|
+
// in __tests__/cli/derived-apply.test.ts.
|
|
490
498
|
return { statements, record, remove, migrate: [...plan.migrations] };
|
|
491
499
|
}
|
|
492
500
|
|
|
@@ -514,7 +522,17 @@ ON CONFLICT (identity) DO UPDATE SET src_hash = EXCLUDED.src_hash, def_hash = EX
|
|
|
514
522
|
* provenance. See the ORDER MATTERS comment in db-reconcile.ts.
|
|
515
523
|
*
|
|
516
524
|
* Both branches are mutually exclusive under one snapshot, and an unreferenced data-modifying CTE
|
|
517
|
-
* still executes exactly once
|
|
525
|
+
* still executes exactly once. All three branches (target free / target occupied / nothing to
|
|
526
|
+
* migrate) run against a real database in __tests__/integration/provenance-migrate.test.ts.
|
|
527
|
+
*
|
|
528
|
+
* A KIND GUARD WAS TRIED HERE AND REVERTED. It refused the DELETE when the two rows' kinds
|
|
529
|
+
* differed, on the theory that a `defineSql` domain and a function can share a name and the
|
|
530
|
+
* DELETE would then orphan one of them. The occupied-target branch IS reachable — the ops-Lambda
|
|
531
|
+
* apply takes no mutation lease, so a second writer can occupy the target between plan and
|
|
532
|
+
* statement — but in that reachable state the two rows are the SAME object: a pre-`kind`-column
|
|
533
|
+
* legacy row (kind NULL) and the row the other apply just wrote. Measured on PostgreSQL 16: the
|
|
534
|
+
* guard aborted the apply and left two rows, where this statement converges to one. It was
|
|
535
|
+
* negative value on precisely the upgrade path it would have shipped to. */
|
|
518
536
|
export function renderProvenanceMigrate(from: string, to: string): string {
|
|
519
537
|
const fromLit = escapeLiteral(from);
|
|
520
538
|
const toLit = escapeLiteral(to);
|
|
@@ -377,6 +377,57 @@ export function compileDerived(models: readonly ModelDescriptor[], derived: read
|
|
|
377
377
|
const declaredIdentities = (refs: readonly DependsOnRef[]): string[] =>
|
|
378
378
|
[...new Set(refs.map((r) => ('table' in r ? `${r.schema}.${r.table}` : derivedIdentity(r))))].sort();
|
|
379
379
|
|
|
380
|
+
// Every RELATION the declared state creates — the tables, views and matviews a function's
|
|
381
|
+
// signature could name. A function's return and argument types are resolved by PostgreSQL
|
|
382
|
+
// when the function is created, so `db:build`'s functions-ahead-of-state pass has to know
|
|
383
|
+
// which functions cannot go first. See SourceObject.signatureDeps.
|
|
384
|
+
const relationIdentity = (ref: SetofReturn['setof']): string =>
|
|
385
|
+
'table' in ref ? `${ref.schema}.${ref.table}` : derivedIdentity(ref);
|
|
386
|
+
const relationIdentities = new Map<string, string>(
|
|
387
|
+
[
|
|
388
|
+
...models.map((m) => `${m.schema}.${m.table}`),
|
|
389
|
+
...derived
|
|
390
|
+
.filter((d) => d.kind === 'view' || d.kind === 'materialized view')
|
|
391
|
+
.map((d) => derivedIdentity(d)),
|
|
392
|
+
].map((id) => [id.toLowerCase(), id]),
|
|
393
|
+
);
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* A type name as written in a signature → the relation identity it names, if it names one.
|
|
397
|
+
*
|
|
398
|
+
* The quotes come off, and that is not cosmetic: `db:pull` writes the return type as
|
|
399
|
+
* `pg_get_function_result` spells it, and PostgreSQL quotes any identifier that needs it
|
|
400
|
+
* (mixed case, a reserved word, anything non-word). A pulled `SETOF "Stats_View"."x_row"`
|
|
401
|
+
* that read as no relation at all would go back into the ahead-of-state wave and fail exactly
|
|
402
|
+
* as before, silently.
|
|
403
|
+
*/
|
|
404
|
+
const relationNamed = (type: string): string | null => {
|
|
405
|
+
const bare = type
|
|
406
|
+
.trim().replace(/^setof\s+/i, '').replace(/(\s*\[\s*\])+$/, '')
|
|
407
|
+
.split('.').map((part) => part.trim().replace(/^"(.*)"$/s, '$1')).join('.');
|
|
408
|
+
if (!bare) return null;
|
|
409
|
+
// A quoted identifier is case-SENSITIVE and an unquoted one folds to lower — but the
|
|
410
|
+
// declared names this compares against are the plain lowercase ones the model accepts,
|
|
411
|
+
// so folding both sides is right here and a quoted mixed-case name simply matches nothing.
|
|
412
|
+
const key = bare.toLowerCase();
|
|
413
|
+
return relationIdentities.get(key.includes('.') ? key : `public.${key}`) ?? null;
|
|
414
|
+
};
|
|
415
|
+
|
|
416
|
+
const signatureDeps = (fn: FunctionDescriptor): string[] => {
|
|
417
|
+
const out = new Set<string>();
|
|
418
|
+
// `setof(ref)` is the typed spelling — the ref IS a relation descriptor, so it always counts.
|
|
419
|
+
if (typeof fn.returns !== 'string') out.add(relationIdentity(fn.returns.setof));
|
|
420
|
+
else {
|
|
421
|
+
const named = relationNamed(fn.returns);
|
|
422
|
+
if (named) out.add(named);
|
|
423
|
+
}
|
|
424
|
+
for (const a of fn.args) {
|
|
425
|
+
const named = relationNamed(a.type);
|
|
426
|
+
if (named) out.add(named);
|
|
427
|
+
}
|
|
428
|
+
return [...out].sort();
|
|
429
|
+
};
|
|
430
|
+
|
|
380
431
|
for (const d of derived) {
|
|
381
432
|
const identity = derivedIdentity(d);
|
|
382
433
|
switch (d.kind) {
|
|
@@ -398,7 +449,14 @@ export function compileDerived(models: readonly ModelDescriptor[], derived: read
|
|
|
398
449
|
// The pin rides STRUCTURALLY as well as inside `sql`, so the differ can compare it
|
|
399
450
|
// against the live catalog directly instead of only against a recorded hash. It is not
|
|
400
451
|
// in the hash (it is already inside `sql`), so carrying it moves nobody's fingerprint.
|
|
401
|
-
|
|
452
|
+
// The signature is an ORDERING edge as well as a record. `setofDep` covers the typed
|
|
453
|
+
// `setof(ref)` spelling only; a return or argument type written as a plain string is
|
|
454
|
+
// the shape `db:pull` renders, and it carries no `dependsOn` of its own — so without
|
|
455
|
+
// this the function and its view order by identity sort, and whether the build works
|
|
456
|
+
// comes down to which name happens to sort first.
|
|
457
|
+
const sigDeps = signatureDeps(d);
|
|
458
|
+
const sigEdges = sigDeps.filter((id) => identities.has(id));
|
|
459
|
+
nodes.push({ identity, deps: [...depIdentities(d.dependsOn), ...setofDep, ...sigEdges], build: make('function', d.name, sql, attachments, { identity, ...(sigDeps.length ? { signatureDeps: sigDeps } : {}), ...(d.owner ? { owner: d.owner } : {}), ...(d.searchPath !== undefined ? { searchPath: d.searchPath } : {}) }) });
|
|
402
460
|
break;
|
|
403
461
|
}
|
|
404
462
|
case 'sql': {
|
package/src/cli/derived-plan.ts
CHANGED
|
@@ -177,6 +177,18 @@ export function planReconcile(
|
|
|
177
177
|
// two objects — those baseline honestly), a row a live NON-function already owns (a view
|
|
178
178
|
// and a function can share `schema.name`), and a 'sql'-kind row (its identity is its
|
|
179
179
|
// declared name, not a catalog signature).
|
|
180
|
+
//
|
|
181
|
+
// DELIBERATELY NOT SKIPPED: a legacy identity the SOURCE also claims with a different object —
|
|
182
|
+
// a `defineSql` domain named `pmo_label` alongside a function whose legacy identity is
|
|
183
|
+
// `public.pmo_label`. The kind read below is the PROVENANCE row's, never the source's, and that
|
|
184
|
+
// is the intent. Types and functions live in different PostgreSQL catalogs, so both objects are
|
|
185
|
+
// legal; the collision exists only while the function's row is under-keyed. Migrating is what
|
|
186
|
+
// VACATES the bare name for the object that legitimately owns it — the run converges in one
|
|
187
|
+
// apply and the next is a clean no-op (pinned by __tests__/integration/provenance-migrate.test.ts).
|
|
188
|
+
// Skipping instead would keep the collision and let the domain's upsert overwrite the function's
|
|
189
|
+
// row; refusing would block a state that is legal, self-healing, and ambiguous for exactly one
|
|
190
|
+
// apply. Adjudicated 2026-08-12 (issue G7). What this rests on is the caller's
|
|
191
|
+
// migrate-before-record ordering — see the ORDER MATTERS comment in db-reconcile.ts.
|
|
180
192
|
const liveFnByLegacy = new Map<string, LiveObject[]>();
|
|
181
193
|
for (const o of live.objects) {
|
|
182
194
|
if (o.kind !== 'function') continue;
|
|
@@ -96,6 +96,26 @@ export interface SourceObject {
|
|
|
96
96
|
* "declared, then verified". Absent on kinds whose live edges pg_depend can't see.
|
|
97
97
|
*/
|
|
98
98
|
declaredDeps?: string[];
|
|
99
|
+
/**
|
|
100
|
+
* kind 'function' only: the relations this function's SIGNATURE names — a `setof(ref)`
|
|
101
|
+
* return, or an argument/return type spelled as a declared table, view or matview.
|
|
102
|
+
*
|
|
103
|
+
* Distinct from `dependsOn`, and the distinction is load-bearing for the from-scratch build.
|
|
104
|
+
* A body reference is a FORWARD reference the build can defer (`check_function_bodies = off`
|
|
105
|
+
* is exactly that escape), so a function whose body reads a not-yet-created table still
|
|
106
|
+
* creates. A SIGNATURE reference is not deferrable: PostgreSQL resolves the return and
|
|
107
|
+
* argument types when the function is created, so `RETURNS SETOF stats_view.x_row` needs
|
|
108
|
+
* that view to exist already. `db:build` runs functions ahead of state (an RLS policy's
|
|
109
|
+
* predicate must resolve the function it calls) and uses this to hold back the ones that
|
|
110
|
+
* cannot go first; the compiler also orders the function after them.
|
|
111
|
+
*
|
|
112
|
+
* RELATIONS only — tables, views, matviews. A signature naming a declared ENUM, domain or
|
|
113
|
+
* composite type has the same problem and is NOT covered here, so it still fails in the
|
|
114
|
+
* ahead-of-state pass. Absent/empty means "names no declared relation", never "is safe".
|
|
115
|
+
*
|
|
116
|
+
* Not in the hash — it is derived from `sql`, so carrying it moves no fingerprint.
|
|
117
|
+
*/
|
|
118
|
+
signatureDeps?: string[];
|
|
99
119
|
}
|
|
100
120
|
|
|
101
121
|
/** The provenance marker descriptor-compiled objects carry in `file` — the compiler
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The fingerprint sandwich around a stage backup — ONE implementation, three callers.
|
|
3
|
+
*
|
|
4
|
+
* A `db:backup` artifact carries a live base-schema fingerprint in its `.meta.json`, and the
|
|
5
|
+
* destructive-apply gate compares it against `plan.from`. That comparison is only meaningful if the
|
|
6
|
+
* fingerprint describes the schema the dump ACTUALLY holds. A dump taken across a concurrent
|
|
7
|
+
* migration holds a schema matching NEITHER endpoint, so its stamped fingerprint is a lie the gate
|
|
8
|
+
* would later trust. The sandwich reads the schema before the dump and again after it, and keeps
|
|
9
|
+
* the artifact only if the two agree.
|
|
10
|
+
*
|
|
11
|
+
* WHY THIS LIVES IN THE CLI: it used to run inside the task image, which imported
|
|
12
|
+
* `@everystack/cli/apply` to do it. The image is published and digest-pinned, so that froze a copy
|
|
13
|
+
* of the very CLI internals the gate depends on while the operator's checkout kept moving — a
|
|
14
|
+
* silent version skew on a safety-critical input. Here it runs from the operator's own checkout,
|
|
15
|
+
* the same code `db:apply` uses, so the two can never disagree about what a fingerprint means.
|
|
16
|
+
*
|
|
17
|
+
* WHY IT IS SHARED: `fingerprint` is a REQUIRED field on the backup job. Three commands dispatch
|
|
18
|
+
* backups (db:backup, db:fork, db:swap --snapshot logical), and when the field was added to one of
|
|
19
|
+
* them the other two silently broke. One implementation, one contract.
|
|
20
|
+
*
|
|
21
|
+
* The dispatch/poll/retract wiring is dependency-injected so the whole decision table is testable
|
|
22
|
+
* without AWS.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { invokeAction, lambdaSessionRunner } from './aws.js';
|
|
26
|
+
import { pollTaskUntilStopped } from './task-poll.js';
|
|
27
|
+
|
|
28
|
+
/** The verdict on a completed dump's fingerprint pair. Pure — the heart of the sandwich. */
|
|
29
|
+
export type SandwichVerdict = { ok: true } | { ok: false; reason: string };
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Did the schema hold still across the dump window?
|
|
33
|
+
*
|
|
34
|
+
* Agreement proves no DDL landed ANYWHERE between the two reads, so the fingerprint genuinely
|
|
35
|
+
* describes the dump's schema. Disagreement means the artifact is unusable as a safety net —
|
|
36
|
+
* better a refused backup than a false one.
|
|
37
|
+
*/
|
|
38
|
+
export function judgeSandwich(fpBefore: string, fpAfter: string): SandwichVerdict {
|
|
39
|
+
if (fpBefore === fpAfter) return { ok: true };
|
|
40
|
+
return {
|
|
41
|
+
ok: false,
|
|
42
|
+
reason:
|
|
43
|
+
`the schema changed during the dump (${fpBefore.slice(0, 12)} → ${fpAfter.slice(0, 12)}), `
|
|
44
|
+
+ 'so the backup spans a migration and its fingerprint cannot be proven. Re-run when no migration is concurrent.',
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface StageBackupDeps {
|
|
49
|
+
/** Read the live base-schema fingerprint. Called twice, bracketing the dump. */
|
|
50
|
+
readFingerprint: () => Promise<string>;
|
|
51
|
+
/** Dispatch `db:backup` with the operator-computed fingerprint. */
|
|
52
|
+
dispatch: (fingerprint: string) => Promise<{ runId: string; taskArn: string; id: string; key: string; warning?: string }>;
|
|
53
|
+
/**
|
|
54
|
+
* Wait for the task to stop; resolve ok, or a human reason it did not.
|
|
55
|
+
*
|
|
56
|
+
* `stampedFingerprint` is what the task reported writing into the artifact's `.meta.json`. It is
|
|
57
|
+
* the evidence the image honored the job — see the skew check in runStageBackup.
|
|
58
|
+
*/
|
|
59
|
+
awaitTask: (ids: { runId: string; taskArn: string; id: string }) => Promise<{ ok: true; bytes?: number; stampedFingerprint?: string } | { ok: false; reason: string }>;
|
|
60
|
+
/** Delete a retracted artifact (the ops Lambda's db:backup:retract). */
|
|
61
|
+
retract: (key: string) => Promise<void>;
|
|
62
|
+
/** Progress output. Optional so tests stay quiet. */
|
|
63
|
+
onStep?: (message: string) => void;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export type StageBackupResult =
|
|
67
|
+
| { ok: true; id: string; key: string; bytes?: number; fingerprint: string; warning?: string }
|
|
68
|
+
| { ok: false; reason: string; retracted?: boolean; orphanKey?: string };
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Take a stage backup whose fingerprint is PROVEN stable, or refuse.
|
|
72
|
+
*
|
|
73
|
+
* Never throws for an expected failure — every outcome is a verdict the caller renders, so the
|
|
74
|
+
* three commands can word their own consequences ("the swap was NOT applied", "the fork stopped")
|
|
75
|
+
* without re-deriving the logic.
|
|
76
|
+
*/
|
|
77
|
+
export async function runStageBackup(deps: StageBackupDeps): Promise<StageBackupResult> {
|
|
78
|
+
const step = deps.onStep ?? (() => {});
|
|
79
|
+
|
|
80
|
+
step('Reading the live schema fingerprint (before the dump)...');
|
|
81
|
+
let fpBefore: string;
|
|
82
|
+
try {
|
|
83
|
+
fpBefore = await deps.readFingerprint();
|
|
84
|
+
} catch (err: any) {
|
|
85
|
+
// Refuse BEFORE dispatching: a dump whose fingerprint cannot be proven is not a safety net, and
|
|
86
|
+
// taking one anyway would burn the dump window and still leave nothing trustworthy.
|
|
87
|
+
return { ok: false, reason: `the schema fingerprint could not be read, so no backup was dispatched: ${err?.message ?? String(err)}` };
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
let dispatched: { runId: string; taskArn: string; id: string; key: string; warning?: string };
|
|
91
|
+
try {
|
|
92
|
+
dispatched = await deps.dispatch(fpBefore);
|
|
93
|
+
} catch (err: any) {
|
|
94
|
+
return { ok: false, reason: `the backup would not dispatch: ${err?.message ?? String(err)}` };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const finished = await deps.awaitTask(dispatched);
|
|
98
|
+
if (!finished.ok) return { ok: false, reason: finished.reason };
|
|
99
|
+
|
|
100
|
+
// Did the image actually USE the fingerprint we sent?
|
|
101
|
+
//
|
|
102
|
+
// This is the version-skew guard, and it deliberately does not rely on the image cooperating. An
|
|
103
|
+
// older task image ignores the job's `fingerprint` field entirely and computes its own, stamping
|
|
104
|
+
// the manifest with a value this CLI never produced — the exact stale-internals problem the
|
|
105
|
+
// operator-side move exists to remove, arriving through a pinned digest nobody re-published. A
|
|
106
|
+
// protocol handshake cannot catch that, because an old image does not know to check it. Comparing
|
|
107
|
+
// what came back against what we sent works against ANY image, old or new.
|
|
108
|
+
if (finished.stampedFingerprint !== undefined && finished.stampedFingerprint !== fpBefore) {
|
|
109
|
+
step('Retracting the backup (the task stamped a fingerprint we did not send)...');
|
|
110
|
+
const mismatch =
|
|
111
|
+
`the task stamped fingerprint ${finished.stampedFingerprint.slice(0, 12)} but the operator sent `
|
|
112
|
+
+ `${fpBefore.slice(0, 12)} — the task image computed its own instead of using the job's, which means it `
|
|
113
|
+
+ 'predates the operator-side fingerprint contract. Re-publish the Task image (or pin a newer digest).';
|
|
114
|
+
try {
|
|
115
|
+
await deps.retract(dispatched.key);
|
|
116
|
+
return { ok: false, reason: `${mismatch} Backup ${dispatched.id} was retracted.`, retracted: true };
|
|
117
|
+
} catch (err: any) {
|
|
118
|
+
return {
|
|
119
|
+
ok: false,
|
|
120
|
+
reason: `${mismatch} Backup ${dispatched.id} could ALSO not be retracted (${err?.message ?? String(err)}) — delete it by hand.`,
|
|
121
|
+
retracted: false,
|
|
122
|
+
orphanKey: dispatched.key,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
step('Re-reading the schema fingerprint (after the dump)...');
|
|
128
|
+
let fpAfter: string;
|
|
129
|
+
try {
|
|
130
|
+
fpAfter = await deps.readFingerprint();
|
|
131
|
+
} catch (err: any) {
|
|
132
|
+
// The dump exists but is unproven. Do NOT retract — an unreadable fingerprint is not evidence
|
|
133
|
+
// the schema moved, and silently destroying a good backup is the worse error.
|
|
134
|
+
return {
|
|
135
|
+
ok: false,
|
|
136
|
+
reason: `the dump finished but the schema fingerprint could not be re-read (${err?.message ?? String(err)}), so backup ${dispatched.id} is NOT proven stable. Verify no migration ran during the dump, or re-run.`,
|
|
137
|
+
orphanKey: dispatched.key,
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const verdict = judgeSandwich(fpBefore, fpAfter);
|
|
142
|
+
if (verdict.ok) {
|
|
143
|
+
// A task that reported no fingerprint at all is not the skew case above — the result-ledger
|
|
144
|
+
// write is explicitly best-effort, so a blip there loses the whole result row. Failing the
|
|
145
|
+
// backup for that would turn a cosmetic outage into a lost safety net, so say it and continue.
|
|
146
|
+
const unverified = finished.stampedFingerprint === undefined
|
|
147
|
+
? 'the task did not report the fingerprint it stamped, so the artifact could not be verified against what the operator sent (the result-ledger write is best-effort)'
|
|
148
|
+
: undefined;
|
|
149
|
+
const warning = [dispatched.warning, unverified].filter(Boolean).join('; ') || undefined;
|
|
150
|
+
return { ok: true, id: dispatched.id, key: dispatched.key, bytes: finished.bytes, fingerprint: fpBefore, warning };
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
step('Retracting the backup (its fingerprint cannot be proven)...');
|
|
154
|
+
try {
|
|
155
|
+
await deps.retract(dispatched.key);
|
|
156
|
+
return { ok: false, reason: `${verdict.reason} Backup ${dispatched.id} was retracted.`, retracted: true };
|
|
157
|
+
} catch (err: any) {
|
|
158
|
+
// The artifact outlived the failure. Say so loudly with the key — an unproven backup left in
|
|
159
|
+
// the bucket is exactly what the destructive-apply gate must never pick up.
|
|
160
|
+
return {
|
|
161
|
+
ok: false,
|
|
162
|
+
reason: `${verdict.reason} Backup ${dispatched.id} could ALSO not be retracted (${err?.message ?? String(err)}) — delete it by hand so the destructive-apply gate never trusts it.`,
|
|
163
|
+
retracted: false,
|
|
164
|
+
orphanKey: dispatched.key,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Bind the sandwich to a real stage: the ops Lambda supplies the session the fingerprint is read
|
|
171
|
+
* over, dispatches the task, reports status, and performs the retraction — so the operator never
|
|
172
|
+
* holds a database URL or an S3 credential for any of it.
|
|
173
|
+
*
|
|
174
|
+
* The ONE place the wiring is written. A caller that hand-rolled this is how `fingerprint` became
|
|
175
|
+
* required in db:backup while db:fork and db:swap kept dispatching without it.
|
|
176
|
+
*/
|
|
177
|
+
export function stageBackupDeps(
|
|
178
|
+
config: { region: string },
|
|
179
|
+
opsFn: string,
|
|
180
|
+
stage: string | undefined,
|
|
181
|
+
hooks: { onStep?: (message: string) => void; onDispatch?: (d: { runId: string; taskArn: string; id: string; warning?: string }) => void } = {},
|
|
182
|
+
): StageBackupDeps {
|
|
183
|
+
return {
|
|
184
|
+
// BOTH ends of the sandwich read over this ONE closure, so they cannot land on different lanes.
|
|
185
|
+
// That matters: a before-read over a direct session and an after-read over the two-invoke
|
|
186
|
+
// ops-Lambda blend could manufacture a false mismatch — or, worse, a false match. The symmetry
|
|
187
|
+
// is structural here rather than asserted; if a caller ever needs to override one end, it has to
|
|
188
|
+
// override both.
|
|
189
|
+
readFingerprint: async () => {
|
|
190
|
+
// The SAME helper db:apply's concurrency lock uses — that shared definition is the point.
|
|
191
|
+
const { liveBaseFingerprint } = await import('./apply-execute.js');
|
|
192
|
+
return (await liveBaseFingerprint(lambdaSessionRunner(config.region, opsFn))).hash;
|
|
193
|
+
},
|
|
194
|
+
dispatch: async (fingerprint) => {
|
|
195
|
+
const dispatched: any = await invokeAction(config.region, opsFn, 'db:backup', {
|
|
196
|
+
stage, actor: process.env.USER ?? null, fingerprint,
|
|
197
|
+
});
|
|
198
|
+
if (dispatched?.error) throw new Error(dispatched.error);
|
|
199
|
+
hooks.onDispatch?.(dispatched);
|
|
200
|
+
return dispatched;
|
|
201
|
+
},
|
|
202
|
+
awaitTask: async (ids) => {
|
|
203
|
+
const poll = await pollTaskUntilStopped(config.region, opsFn, { runId: ids.runId, taskArn: ids.taskArn });
|
|
204
|
+
if (poll.outcome === 'timeout') {
|
|
205
|
+
return { ok: false, reason: `the dump did not finish in time (last status: ${poll.lastStatus}). Run id ${ids.runId} — reconcile via ECS / everystack.task_log.` };
|
|
206
|
+
}
|
|
207
|
+
if (poll.outcome === 'error') {
|
|
208
|
+
return { ok: false, reason: `the task status could not be read (${poll.status.error}). The dump may still be running — reconcile run id ${ids.runId}.` };
|
|
209
|
+
}
|
|
210
|
+
if (poll.status.exitCode !== 0) {
|
|
211
|
+
return { ok: false, reason: `the dump exited ${poll.status.exitCode ?? 'unknown'}${poll.status.stoppedReason ? ` — ${poll.status.stoppedReason}` : ''}. Read the task logs (CloudWatch).` };
|
|
212
|
+
}
|
|
213
|
+
return {
|
|
214
|
+
ok: true,
|
|
215
|
+
bytes: poll.status.result?.bytes as number | undefined,
|
|
216
|
+
// What the task says it wrote into the artifact's .meta.json — the skew evidence.
|
|
217
|
+
stampedFingerprint: poll.status.result?.fingerprint as string | undefined,
|
|
218
|
+
};
|
|
219
|
+
},
|
|
220
|
+
retract: async (key) => {
|
|
221
|
+
// `stage` scopes what the action is willing to delete — it refuses any key outside this
|
|
222
|
+
// stage's backup prefix, so a retraction cannot reach an export artifact or a restore intent.
|
|
223
|
+
const res: any = await invokeAction(config.region, opsFn, 'db:backup:retract', { key, stage });
|
|
224
|
+
if (res?.error) throw new Error(res.error);
|
|
225
|
+
if (!res?.retracted) throw new Error(`the ops function did not confirm the retraction (returned ${JSON.stringify(res)})`);
|
|
226
|
+
},
|
|
227
|
+
onStep: hooks.onStep,
|
|
228
|
+
};
|
|
229
|
+
}
|