@everystack/cli 0.4.36 → 0.4.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,52 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * The `everystack` entry point.
4
+ *
5
+ * This package ships TypeScript source deliberately — every `exports` target is a
6
+ * `./src/*.ts` file — so the CLI needs a TypeScript loader at runtime. It used to get one
7
+ * from `#!/usr/bin/env tsx` on `src/cli/index.ts`, which cannot work once the package is
8
+ * installed: `env` searches the CONSUMER's PATH, and tsx is a dependency of
9
+ * @everystack/cli, so under pnpm's isolated layout it lives in this package's own
10
+ * node_modules and is never on that PATH. `everystack --help` died with
11
+ * "env: tsx: No such file or directory"; only `pnpm exec everystack` worked, because that
12
+ * puts the local .bin on PATH first.
13
+ *
14
+ * So: boot under plain `node` — always present, it is what runs npm — and find tsx by
15
+ * MODULE RESOLUTION from this file rather than by PATH lookup.
16
+ *
17
+ * The loader is installed by re-executing node with `--import`, not by calling tsx's
18
+ * register() in-process. register() followed by `import()` of the TypeScript entry makes
19
+ * the entry load through a require(esm) path and Node rejects it with
20
+ * ERR_REQUIRE_CYCLE_MODULE. `--import` installs the hooks before any module graph exists,
21
+ * which is the only ordering that works. The extra process is the price of shipping
22
+ * source, and it is paid once per invocation.
23
+ */
24
+ import { createRequire } from 'node:module';
25
+ import { spawnSync } from 'node:child_process';
26
+ import { pathToFileURL, fileURLToPath } from 'node:url';
27
+
28
+ const require = createRequire(import.meta.url);
29
+
30
+ let tsx;
31
+ try {
32
+ tsx = pathToFileURL(require.resolve('tsx')).href;
33
+ } catch {
34
+ console.error('everystack: could not resolve the "tsx" TypeScript loader from this package.');
35
+ console.error('This usually means a partial install — try reinstalling @everystack/cli.');
36
+ process.exit(1);
37
+ }
38
+
39
+ const entry = fileURLToPath(new URL('../src/cli/index.ts', import.meta.url));
40
+
41
+ const result = spawnSync(process.execPath, ['--import', tsx, entry, ...process.argv.slice(2)], {
42
+ stdio: 'inherit',
43
+ });
44
+
45
+ // Re-raise a signal death as a signal death so `everystack … &` + Ctrl-C behaves, and
46
+ // otherwise pass the child's exit code through unchanged — the CLI's non-zero exits are
47
+ // load-bearing (db:check, db:authz:diff and friends are CI gates).
48
+ if (result.signal) {
49
+ process.kill(process.pid, result.signal);
50
+ } else {
51
+ process.exit(result.status ?? 1);
52
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@everystack/cli",
3
- "version": "0.4.36",
3
+ "version": "0.4.38",
4
4
  "description": "CLI and OTA updates for Expo apps on everystack",
5
5
  "license": "AGPL-3.0-only",
6
6
  "author": "Scalable Technology, Inc. <licensing@scalable.technology>",
@@ -17,6 +17,7 @@
17
17
  "access": "public"
18
18
  },
19
19
  "files": [
20
+ "bin",
20
21
  "src",
21
22
  "README.md"
22
23
  ],
@@ -91,7 +92,7 @@
91
92
  }
92
93
  },
93
94
  "bin": {
94
- "everystack": "./src/cli/index.ts"
95
+ "everystack": "./bin/everystack.mjs"
95
96
  },
96
97
  "dependencies": {
97
98
  "@aws-sdk/client-cloudfront": "3.1053.0",
@@ -111,7 +112,7 @@
111
112
  "@everystack/model": "0.4.5"
112
113
  },
113
114
  "peerDependencies": {
114
- "@everystack/server": ">=0.1.0",
115
+ "@everystack/server": ">=0.4.0",
115
116
  "@aws-sdk/client-cloudwatch": "3.1053.0",
116
117
  "@aws-sdk/client-rds": "3.1053.0",
117
118
  "@aws-sdk/s3-request-presigner": "3.1053.0",
@@ -9,9 +9,20 @@
9
9
  * can diff a deployed database against. `pull` is the export + on-ramp; `diff` is the
10
10
  * audit and the CI gate (non-zero exit on any drift).
11
11
  *
12
- * Both introspect through the ops Lambda `db:query` action (read-only SQL), the same
13
- * path security:audit uses. The core (introspect/assemble/diff) is backend-agnostic and
14
- * pure; only the runner here is AWS-bound.
12
+ * Two venues, one verdict. The core (introspect/assemble/diff/evaluate) is backend-agnostic
13
+ * and pure only the VENUE differs, and both satisfy the same `AuthzVenue` contract, so
14
+ * every command below runs identical evaluation code either way:
15
+ *
16
+ * - **Deployed** (`--stage`, the default): the ops Lambda's `db:query` + `db:authz:probe`
17
+ * actions. Credentials never leave AWS.
18
+ * - **Direct** (`--database-url`, or an inherited ADMIN_DATABASE_URL / DATABASE_URL): a
19
+ * postgres.js connection from this process.
20
+ *
21
+ * The direct venue is what makes a brownfield authz migration rehearsable. Translating
22
+ * hand-written RLS into Model abilities is the hardest part of an adoption, and until the
23
+ * local venue existed it was the ONLY part that required a deployed stage to iterate
24
+ * against. Every run names its venue, because a security verdict against an unintended
25
+ * database is worse than no verdict.
15
26
  */
16
27
 
17
28
  import path from 'node:path';
@@ -43,6 +54,7 @@ import { renderContractMarkdown } from '../authz-render.js';
43
54
  import { FUNCTIONS_SQL, contractFunctionRow } from '../security-catalog.js';
44
55
  import { resolveConfig, opsFunction } from '../config.js';
45
56
  import { resolveModelsPath } from '../models-path.js';
57
+ import { resolveDbSource, connectingVia, createUrlProbeRunner } from '../db-source.js';
46
58
  import { invokeAction } from '../aws.js';
47
59
  import { step, success, fail, info, warn } from '../output.js';
48
60
  import { opsAdviceLines, IAM_ADVICE } from '../ops-advice.js';
@@ -59,20 +71,65 @@ function lambdaRunner(region: string, fn: string): QueryRunner {
59
71
  };
60
72
  }
61
73
 
62
- async function introspectStage(flags: Record<string, string>): Promise<AuthzContract> {
74
+ /**
75
+ * Where an authz command runs. Both venues expose the same two capabilities, so the
76
+ * commands never branch on venue after this point — that is what keeps the local
77
+ * rehearsal honest: same SQL, same evaluation, same verdict.
78
+ */
79
+ interface AuthzVenue {
80
+ /** Read-only introspection. */
81
+ runner: QueryRunner;
82
+ /** Self-reverting red-team probe (writes, always rolled back). */
83
+ probe: (setup: string, read: string) => Promise<any[]>;
84
+ /** Human-readable venue, printed with every verdict. */
85
+ label: string;
86
+ /** Release any connection this venue holds. */
87
+ end: () => Promise<void>;
88
+ }
89
+
90
+ /**
91
+ * Precedence is `resolveDbSource`'s, shared with db:pull / db:generate / db:check:
92
+ * `--database-url` > explicit `--stage` > ADMIN_DATABASE_URL > DATABASE_URL > default stage.
93
+ */
94
+ async function resolveVenue(flags: Record<string, string>): Promise<AuthzVenue> {
95
+ const source = resolveDbSource(flags);
96
+ if (source.kind === 'url') {
97
+ step(connectingVia(source));
98
+ const { runner, probe, end } = await createUrlProbeRunner(source.url);
99
+ return { runner, probe, end, label: 'direct connection' };
100
+ }
63
101
  step('Resolving deployed config...');
64
102
  const config = await resolveConfig(flags.stage);
65
- info(`Region: ${config.region}, Function: ${opsFunction(config)}`);
66
- step('Introspecting authorization (rls + grants + policies + secdef)...');
67
- const runner = lambdaRunner(config.region, opsFunction(config));
68
- return introspectContract(runner, contractFunctionRow, FUNCTIONS_SQL);
103
+ const fn = opsFunction(config);
104
+ info(`Region: ${config.region}, Function: ${fn}`);
105
+ return {
106
+ runner: lambdaRunner(config.region, fn),
107
+ probe: async (setup: string, read: string) => {
108
+ const result: any = await invokeAction(config.region, fn, 'db:authz:probe', { setup, read });
109
+ if (result?.error) throw new Error(result.error);
110
+ return result?.rows ?? [];
111
+ },
112
+ label: flags.stage ? `stage ${flags.stage}` : 'deployed stage (.sst/outputs.json)',
113
+ end: async () => {},
114
+ };
115
+ }
116
+
117
+ async function introspectVenue(flags: Record<string, string>): Promise<{ contract: AuthzContract; label: string }> {
118
+ const venue = await resolveVenue(flags);
119
+ try {
120
+ step('Introspecting authorization (rls + grants + policies + secdef)...');
121
+ return { contract: await introspectContract(venue.runner, contractFunctionRow, FUNCTIONS_SQL), label: venue.label };
122
+ } finally {
123
+ await venue.end();
124
+ }
69
125
  }
70
126
 
71
127
  export async function dbAuthzPullCommand(flags: Record<string, string>): Promise<void> {
72
128
  const dir = path.resolve(flags.dir || flags.out || DEFAULT_DIR);
73
129
  let contract: AuthzContract;
130
+ let venueLabel: string;
74
131
  try {
75
- contract = await introspectStage(flags);
132
+ ({ contract, label: venueLabel } = await introspectVenue(flags));
76
133
  } catch (err: any) {
77
134
  fail(err.message);
78
135
  for (const line of opsAdviceLines(err, [IAM_ADVICE])) info(line);
@@ -82,7 +139,7 @@ export async function dbAuthzPullCommand(flags: Record<string, string>): Promise
82
139
  const files = await writeContract(dir, contract);
83
140
  const policies = contract.tables.reduce((n, t) => n + t.policies.length, 0);
84
141
  console.log('');
85
- success(`Wrote ${files.length} file(s) to ${dir}`);
142
+ success(`Wrote ${files.length} file(s) to ${dir} (from ${venueLabel})`);
86
143
  info(`${contract.tables.length} table(s), ${policies} policy(ies), ${contract.functions.length} SECDEF function(s).`);
87
144
  console.log('');
88
145
  warn('pull OVERWROTE the committed contract with the live state — `git diff` IS your drift report.');
@@ -97,9 +154,10 @@ export async function dbAuthzDiffCommand(flags: Record<string, string>): Promise
97
154
 
98
155
  let declared: AuthzContract;
99
156
  let live: AuthzContract;
157
+ let venueLabel: string;
100
158
  try {
101
159
  declared = await loadContract(dir);
102
- live = await introspectStage(flags);
160
+ ({ contract: live, label: venueLabel } = await introspectVenue(flags));
103
161
  } catch (err: any) {
104
162
  fail(err.message);
105
163
  process.exit(1);
@@ -113,11 +171,11 @@ export async function dbAuthzDiffCommand(flags: Record<string, string>): Promise
113
171
  const findings = diffContracts(declared, live);
114
172
  console.log('');
115
173
  if (findings.length === 0) {
116
- success(`db:authz:diff — live database matches the declared contract (${declared.tables.length} tables)`);
174
+ success(`db:authz:diff — ${venueLabel} matches the declared contract (${declared.tables.length} tables)`);
117
175
  process.exit(0);
118
176
  }
119
177
 
120
- fail(`db:authz:diff — ${findings.length} drift finding(s): the live database does NOT match the declared contract`);
178
+ fail(`db:authz:diff — ${findings.length} drift finding(s): ${venueLabel} does NOT match the declared contract`);
121
179
  console.log('');
122
180
  for (const f of findings) {
123
181
  warn(`[${f.kind}] ${f.subject} — ${f.detail}`);
@@ -163,24 +221,23 @@ export async function dbAuthzTestCommand(flags: Record<string, string>): Promise
163
221
 
164
222
  let rows: any[];
165
223
  let gapRows: any[];
224
+ let venueLabel: string;
225
+ let venue: AuthzVenue | undefined;
166
226
  try {
167
- step('Resolving deployed config...');
168
- const config = await resolveConfig(flags.stage);
169
- const fn = opsFunction(config);
170
- info(`Region: ${config.region}, Function: ${fn}`);
227
+ venue = await resolveVenue(flags);
228
+ venueLabel = venue.label;
171
229
  step('Red-teaming enforcement (SET ROLE + attempt per role/table/command, rolled back)...');
172
230
  const setup = buildProbeSql(probeRoles(contract), contract.tables.map((t) => t.table));
173
- const result: any = await invokeAction(config.region, fn, 'db:authz:probe', { setup, read: PROBE_SELECT_SQL });
174
- if (result?.error) throw new Error(result.error);
175
- rows = result?.rows ?? [];
231
+ rows = await venue.probe(setup, PROBE_SELECT_SQL);
176
232
  step('Checking SECDEF grant-completeness (owner can EXECUTE every helper it calls)...');
177
- const gc: any = await invokeAction(config.region, fn, 'db:query', { sql: GRANT_COMPLETENESS_SQL });
178
- if (gc?.error) throw new Error(gc.error);
179
- gapRows = gc?.rows ?? [];
233
+ gapRows = await venue.runner(GRANT_COMPLETENESS_SQL);
180
234
  } catch (err: any) {
181
235
  fail(`db:authz:test failed: ${err.message}`);
182
- info('Ensure the stage is deployed with @everystack/server >= the version that adds db:authz:probe.');
236
+ info('Ensure the stage is deployed with @everystack/server >= the version that adds db:authz:probe,');
237
+ info('or rehearse locally: db:authz:test --database-url <url>.');
183
238
  process.exit(1);
239
+ } finally {
240
+ await venue?.end();
184
241
  }
185
242
 
186
243
  const findings = evaluateRedTeam(contract, rows.map(toProbeResult));
@@ -200,10 +257,10 @@ export async function dbAuthzTestCommand(flags: Record<string, string>): Promise
200
257
  console.log('');
201
258
 
202
259
  if (holes.length === 0 && broken.length === 0 && gaps.length === 0) {
203
- success(`db:authz:test — enforcement matches the contract (${contract.tables.length} tables probed, default-deny holds, SECDEF grants complete)`);
260
+ success(`db:authz:test — ${venueLabel} enforces the contract (${contract.tables.length} tables probed, default-deny holds, SECDEF grants complete)`);
204
261
  process.exit(0);
205
262
  }
206
- fail(`db:authz:test — ${holes.length} enforcement hole(s), ${broken.length} broken grant(s), ${gaps.length} SECDEF grant gap(s). The database does not enforce the contract.`);
263
+ fail(`db:authz:test — ${holes.length} enforcement hole(s), ${broken.length} broken grant(s), ${gaps.length} SECDEF grant gap(s). ${venueLabel} does not enforce the contract.`);
207
264
  process.exit(1);
208
265
  }
209
266
 
@@ -223,6 +280,8 @@ export async function dbAuthzOwnerCommand(flags: Record<string, string>): Promis
223
280
  let probes: OwnerProbe[];
224
281
  let publicReadTables = new Set<string>();
225
282
  let rows: any[];
283
+ let venueLabel = '';
284
+ let venue: AuthzVenue | undefined;
226
285
  try {
227
286
  step(`Loading models from ${modelsPath}...`);
228
287
  const models = await loadModels(modelsPath);
@@ -240,20 +299,17 @@ export async function dbAuthzOwnerCommand(flags: Record<string, string>): Promis
240
299
  success('db:authz:owner — no owner-scoped models (no `can({ owner })`); nothing to probe.');
241
300
  process.exit(0);
242
301
  }
243
- step('Resolving deployed config...');
244
- const config = await resolveConfig(flags.stage);
245
- const fn = opsFunction(config);
246
- info(`Region: ${config.region}, Function: ${fn}`);
302
+ venue = await resolveVenue(flags);
303
+ venueLabel = venue.label;
247
304
  step('Red-teaming owner isolation (two JWT identities per table, rolled back)...');
248
- const result: any = await invokeAction(config.region, fn, 'db:authz:probe', {
249
- setup: buildOwnerProbeSql(probes), read: OWNER_PROBE_SELECT_SQL,
250
- });
251
- if (result?.error) throw new Error(result.error);
252
- rows = result?.rows ?? [];
305
+ rows = await venue.probe(buildOwnerProbeSql(probes), OWNER_PROBE_SELECT_SQL);
253
306
  } catch (err: any) {
254
307
  fail(`db:authz:owner failed: ${err.message}`);
255
- info('Ensure the stage is deployed with @everystack/server >= the version that adds db:authz:probe.');
308
+ info('Ensure the stage is deployed with @everystack/server >= the version that adds db:authz:probe,');
309
+ info('or rehearse locally: db:authz:owner --database-url <url>.');
256
310
  process.exit(1);
311
+ } finally {
312
+ await venue?.end();
257
313
  }
258
314
 
259
315
  const findings = evaluateOwnerProbe(rows.map(toOwnerProbeResult), publicReadTables);
@@ -270,9 +326,9 @@ export async function dbAuthzOwnerCommand(flags: Record<string, string>): Promis
270
326
  console.log('');
271
327
 
272
328
  if (leaks.length === 0 && vacuous.length === 0) {
273
- success(`db:authz:owner — owner isolation holds (${probes.length} table(s) probed${unprobed.length ? `, ${unprobed.length} unprobed` : ''}).`);
329
+ success(`db:authz:owner — owner isolation holds on ${venueLabel} (${probes.length} table(s) probed${unprobed.length ? `, ${unprobed.length} unprobed` : ''}).`);
274
330
  process.exit(0);
275
331
  }
276
- fail(`db:authz:owner — ${leaks.length} IDOR leak(s), ${vacuous.length} vacuous policy(ies). One user can reach another's rows.`);
332
+ fail(`db:authz:owner — ${leaks.length} IDOR leak(s), ${vacuous.length} vacuous policy(ies) on ${venueLabel}. One user can reach another's rows.`);
277
333
  process.exit(1);
278
334
  }
@@ -37,9 +37,12 @@ import { formatBytes } from '../bundle-weight.js';
37
37
  import { rewriteStatementLine, opensCopyData, closesCopyData } from '../schema-rewrite.js';
38
38
  import { resolveOperatorUrlViaStage } from '../direct-venue.js';
39
39
  import { withMutationLease, MutationLeaseError } from '../mutation-lease.js';
40
- import { resolveConfig, opsFunction } from '../config.js';
41
- import { invokeAction, presignGet } from '../aws.js';
42
- import { keyForArtifactId, metaKey } from '../backup.js';
40
+ import { resolveConfig, opsFunction, type CliConfig } from '../config.js';
41
+ import { invokeAction, presignGet, createRdsSnapshot, describeRdsSnapshots } from '../aws.js';
42
+ import { keyForArtifactId, metaKey, utcStamp } from '../backup.js';
43
+ import { rdsSnapshotIdentifier } from '../rds-snapshot.js';
44
+ import { pollTaskUntilStopped } from '../task-poll.js';
45
+ import { decideSnapshotMode, confirmPhysicalSnapshot, interpretBackupPoll, type SnapshotModeRequest } from '../swap-snapshot.js';
43
46
  import { pgEnvFromUrl, pgKeepaliveConninfo } from './db.js';
44
47
  import { step, success, fail, warn, info } from '../output.js';
45
48
 
@@ -264,11 +267,29 @@ async function restoreIntoIncoming(
264
267
  // never argv (libpq also REJECTS non-keyword URI params like the `search_path` the operator URL
265
268
  // bakes in — fine for postgres.js, fatal for a libpq URI). `-d` carries ONLY keepalives, which
266
269
  // have no PG* env equivalent and are what keep this connection from dying in the index phase.
270
+ //
271
+ // psql reads the file from STDIN (`-f -`) rather than opening it itself, purely so the restore
272
+ // knows its own write position. That position is the fact the heartbeat was missing: a server
273
+ // parked in `Client/ClientRead` is a STALL when bytes remain unsent and the successful TAIL when
274
+ // they do not, and those two used to print the same line. `-f -` keeps psql's `psql:<stdin>:N:`
275
+ // error prefixes, so the line number of a failing statement survives the change (verified
276
+ // against psql 16).
267
277
  io.log(`restore phase B: psql streaming ${formatBytes(written)} to the target — heartbeat every 10s.`);
268
- const psql = spawn('psql', ['-d', pgKeepaliveConninfo(), '-v', 'ON_ERROR_STOP=1', '-f', sqlPath], {
269
- stdio: ['ignore', 'ignore', 'pipe'],
278
+ const psql = spawn('psql', ['-d', pgKeepaliveConninfo(), '-v', 'ON_ERROR_STOP=1', '-f', '-'], {
279
+ stdio: ['pipe', 'ignore', 'pipe'],
270
280
  env: { ...process.env, ...pgEnvFromUrl(url) },
271
281
  });
282
+ let fedBytes = 0;
283
+ let feedDone = false;
284
+ // A psql that exits early (ON_ERROR_STOP) makes this pipeline fail with EPIPE. That is a
285
+ // DOWNSTREAM symptom — psql's own exit code and stderr are the authority on what went wrong, and
286
+ // a previous version of this code mistook the EPIPE for the cause and chased the wrong bug for
287
+ // two sessions. So the feed's error is swallowed here and psql's exit decides.
288
+ const feeding = pipeline(
289
+ fs.createReadStream(sqlPath),
290
+ countingTap((n) => { fedBytes = n; }),
291
+ psql.stdin!,
292
+ ).then(() => { feedDone = true; }).catch(() => { /* psql's exit is the authority */ });
272
293
  let pErr = '';
273
294
  psql.stderr.on('data', (d) => {
274
295
  const s = d.toString();
@@ -285,6 +306,9 @@ async function restoreIntoIncoming(
285
306
  incoming,
286
307
  log: io.log,
287
308
  warn: io.warn,
309
+ // The client half of the picture. Without it the heartbeat cried "deadlock signature" over the
310
+ // last poll of a run that had landed every row and was about to succeed.
311
+ clientFeed: () => ({ fedBytes, totalBytes: written, done: feedDone }),
288
312
  onSample: (sample) => {
289
313
  if (sample.state === null) deadBackendPolls += 1;
290
314
  else deadBackendPolls = 0;
@@ -319,8 +343,11 @@ async function restoreIntoIncoming(
319
343
  });
320
344
  } finally {
321
345
  await stopHeartbeat();
346
+ // The feed is already finished on the success path; on a failure path it is rejecting with
347
+ // EPIPE. Either way, await it so no stream work outlives the phase.
348
+ await feeding;
322
349
  }
323
- io.log(`restore phase B done in ${humanElapsed(Date.now() - bStart)} (restore total ${humanElapsed(Date.now() - t0)}).`);
350
+ io.log(`restore phase B done in ${humanElapsed(Date.now() - bStart)} (restore total ${humanElapsed(Date.now() - t0)}); fed ${formatBytes(fedBytes)} of ${formatBytes(written)}.`);
324
351
  } finally {
325
352
  await fs.promises.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
326
353
  }
@@ -407,6 +434,60 @@ async function resolveSwapArtifact(
407
434
  return fetchArtifactFromS3(from, stage, fingerprintFlag);
408
435
  }
409
436
 
437
+ /**
438
+ * Take (or account for) the pre-swap rollback point, and do not return until it EXISTS.
439
+ *
440
+ * Every branch here either produces a confirmed rollback point or throws — and a throw at this point
441
+ * means executeSwap never reaches the restore, so live is untouched. That property is the entire
442
+ * reason this is not a fire-and-forget dispatch any more.
443
+ */
444
+ async function takePreSwapSnapshot(
445
+ plan: Exclude<ReturnType<typeof decideSnapshotMode>, { mode: 'refuse' }>,
446
+ ctx: { stage?: string; region?: string; opsFn?: string },
447
+ ): Promise<void> {
448
+ if (plan.mode === 'attested') {
449
+ info(`pre-swap rollback point: ${plan.ref} (attested via --snapshot-ref — no new snapshot taken).`);
450
+ return;
451
+ }
452
+
453
+ if (plan.mode === 'none') {
454
+ warn('NO pre-swap snapshot (--snapshot none). If this swap lands bad data there is no rollback point — the retiring schema is dropped once verify passes.');
455
+ return;
456
+ }
457
+
458
+ if (plan.mode === 'physical') {
459
+ step(`Snapshotting the instance before the swap (RDS physical snapshot of ${plan.instanceId})...`);
460
+ const snapshotId = rdsSnapshotIdentifier(`${ctx.stage ?? 'swap'}-swap`, utcStamp(new Date()));
461
+ const region = ctx.region!;
462
+ const { id } = await confirmPhysicalSnapshot({
463
+ create: (sid) => createRdsSnapshot(region, plan.instanceId, sid),
464
+ describe: () => describeRdsSnapshots(region, plan.instanceId),
465
+ log: (m) => info(m),
466
+ sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
467
+ now: () => Date.now(),
468
+ }, { instanceId: plan.instanceId, snapshotId });
469
+ info(`rollback point CONFIRMED: RDS snapshot ${id} is available — restore the instance from it if this swap goes wrong.`);
470
+ return;
471
+ }
472
+
473
+ // Logical: dispatch the Task and WAIT. The dispatch returning is not the backup existing — that
474
+ // conflation is what let a pg_dump run concurrently with the restore it was supposed to precede.
475
+ step('Snapshotting the stage before the swap (db:backup — waiting for the dump to finish)...');
476
+ const dispatched: any = await invokeAction(ctx.region!, ctx.opsFn!, 'db:backup', {
477
+ stage: ctx.stage,
478
+ actor: process.env.USER ?? null,
479
+ });
480
+ if (dispatched?.error) throw new Error(`the pre-swap backup would not dispatch, so the swap was NOT applied: ${dispatched.error}`);
481
+ const { runId, taskArn, id } = dispatched as { runId: string; taskArn: string; id: string };
482
+ info(`backup ${id} dispatched (run ${runId}) — waiting for the task to stop before the restore starts.`);
483
+ const verdict = interpretBackupPoll(
484
+ await pollTaskUntilStopped(ctx.region!, ctx.opsFn!, { runId, taskArn }),
485
+ { runId, id },
486
+ );
487
+ if (!verdict.ok) throw new Error(verdict.reason);
488
+ info(`rollback point CONFIRMED: backup ${id} complete — restore with db:restore --from ${id} --confirm.`);
489
+ }
490
+
410
491
  export async function dbSwapCommand(flags: Record<string, string>): Promise<void> {
411
492
  const schema = flags.schema;
412
493
  const from = flags.from;
@@ -445,9 +526,9 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
445
526
  process.exit(1);
446
527
  }
447
528
  let url = urlFlag;
448
- let snapshotViaStage = false;
449
529
  let region: string | undefined;
450
530
  let opsFn: string | undefined;
531
+ let stageConfig: CliConfig | undefined;
451
532
 
452
533
  if (stage) {
453
534
  if (!direct) {
@@ -459,13 +540,12 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
459
540
  process.exit(1);
460
541
  }
461
542
  try {
462
- const config = await resolveConfig(stage);
463
- region = config.region;
464
- opsFn = opsFunction(config);
543
+ stageConfig = await resolveConfig(stage);
544
+ region = stageConfig.region;
545
+ opsFn = opsFunction(stageConfig);
465
546
  step('Resolving the operator connection from the stage (--direct)...');
466
547
  const op = await resolveOperatorUrlViaStage(stage);
467
548
  url = op.url;
468
- snapshotViaStage = true;
469
549
  info(`operator credential resolved (${op.source}) — swapping CLI-side, unbounded clock.`);
470
550
  } catch (err: any) { fail(err.message); process.exit(1); }
471
551
  }
@@ -475,6 +555,29 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
475
555
  process.exit(1);
476
556
  }
477
557
 
558
+ // THE ROLLBACK POINT. Decided here — before the models load, before the artifact is fetched, and
559
+ // long before anything is renamed — so a refusal costs nothing but a second.
560
+ //
561
+ // This step used to be a `db:backup` the swap did not wait for. The ops action dispatches a Task
562
+ // and returns, so the swap printed "snapshot on record" and began restoring while the pg_dump was
563
+ // still running: a consumer measured the restore blocked ~5 minutes on
564
+ // `Lock/relation HELD BY pid [pg_dump]`, the swap contending with its own backup. And a task that
565
+ // failed to start left a destructive swap running against a rollback point that did not exist.
566
+ //
567
+ // A physical RDS snapshot is now the default where the target is RDS (a control-plane call: no
568
+ // locks, no buffer-cache read, no client connection), the logical backup is the non-RDS fallback
569
+ // and is now WAITED ON, and the bare `--database-url` venue refuses rather than warning.
570
+ const snapshotPlan = decideSnapshotMode({
571
+ venue: stage ? 'stage' : 'url',
572
+ instanceId: flags.instance ?? stageConfig?.databaseInstanceId,
573
+ snapshotRef: flags['snapshot-ref'],
574
+ requested: flags.snapshot as SnapshotModeRequest | undefined,
575
+ });
576
+ if (snapshotPlan.mode === 'refuse') {
577
+ fail(snapshotPlan.reason);
578
+ process.exit(1);
579
+ }
580
+
478
581
  // --rebuild-derived carries a real outage window: the derived layer does not exist between the
479
582
  // swap committing and db:reconcile --apply finishing. State it BEFORE the work starts — saying it
480
583
  // only afterward tells the operator about an outage they are already in. It is now the OPT-OUT:
@@ -609,14 +712,7 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
609
712
  declaredIdentities: declaredDerivedObjects.map((o) => o.identity),
610
713
  });
611
714
  },
612
- snapshot: snapshotViaStage
613
- ? async () => {
614
- step('Snapshotting the stage before the swap (db:backup)...');
615
- const r: any = await invokeAction(region!, opsFn!, 'db:backup', { stage });
616
- if (r?.error) throw new Error(`pre-swap snapshot failed, so the swap was NOT applied: ${r.error}`);
617
- info(`snapshot on record: ${r?.id ?? 'backup complete'} — restore with db:restore --from ${r?.id ?? '<id>'} --confirm.`);
618
- }
619
- : async () => { warn('no snapshot taken (direct v1) — take one first: everystack db:backup --database-url … before a production swap.'); },
715
+ snapshot: () => takePreSwapSnapshot(snapshotPlan, { stage, region, opsFn }),
620
716
  }),
621
717
  );
622
718
 
@@ -70,13 +70,10 @@ export interface UrlRunner {
70
70
  }
71
71
 
72
72
  /**
73
- * A QueryRunner over a direct postgres.js connection. One connection is enough —
74
- * introspection is a handful of sequential catalog queries.
73
+ * Load the postgres.js driver, or explain how to get it. Shared by every direct-connection
74
+ * runner below so the missing-driver instructions can never drift between them.
75
75
  */
76
- export async function createUrlRunner(
77
- url: string,
78
- load: () => Promise<any> = () => import('postgres'),
79
- ): Promise<UrlRunner> {
76
+ async function loadPostgresDriver(load: () => Promise<any>): Promise<any> {
80
77
  let mod: any;
81
78
  try {
82
79
  mod = await load();
@@ -85,7 +82,18 @@ export async function createUrlRunner(
85
82
  'The direct-connection path needs the "postgres" driver. It ships with @everystack/server; in a project without it: pnpm add -D postgres',
86
83
  );
87
84
  }
88
- const postgres = mod.default ?? mod;
85
+ return mod.default ?? mod;
86
+ }
87
+
88
+ /**
89
+ * A QueryRunner over a direct postgres.js connection. One connection is enough —
90
+ * introspection is a handful of sequential catalog queries.
91
+ */
92
+ export async function createUrlRunner(
93
+ url: string,
94
+ load: () => Promise<any> = () => import('postgres'),
95
+ ): Promise<UrlRunner> {
96
+ const postgres = await loadPostgresDriver(load);
89
97
  // max_lifetime: null — the driver's default recycles a connection after a random 30–60
90
98
  // minutes, resolving the in-flight query and THEN killing the session. Under reconcile's
91
99
  // BEGIN-across-calls transaction that is a silent session swap mid-batch (the reconcile
@@ -114,15 +122,7 @@ export async function createUrlPipelineRunner(
114
122
  url: string,
115
123
  load: () => Promise<any> = () => import('postgres'),
116
124
  ): Promise<UrlPipelineRunner> {
117
- let mod: any;
118
- try {
119
- mod = await load();
120
- } catch {
121
- throw new Error(
122
- 'The direct-connection path needs the "postgres" driver. It ships with @everystack/server; in a project without it: pnpm add -D postgres',
123
- );
124
- }
125
- const postgres = mod.default ?? mod;
125
+ const postgres = await loadPostgresDriver(load);
126
126
  const sql = postgres(url, { max: 1, onnotice: () => {}, ...sslDefaults(url) });
127
127
  return {
128
128
  query: async (query: string) => Array.from(await sql.unsafe(query)),
@@ -132,6 +132,57 @@ export async function createUrlPipelineRunner(
132
132
  };
133
133
  }
134
134
 
135
+ export interface UrlProbeRunner {
136
+ /** Read-only introspection, for the contract pull/diff. */
137
+ runner: QueryRunner;
138
+ /** The self-reverting red-team probe. See `probe` below. */
139
+ probe: (setup: string, read: string) => Promise<any[]>;
140
+ /** Close the client so the process can exit cleanly. */
141
+ end: () => Promise<void>;
142
+ }
143
+
144
+ /** Thrown to abort the probe transaction; never escapes `probe`. */
145
+ const PROBE_ROLLBACK = Symbol('authz_probe_rollback');
146
+
147
+ /**
148
+ * The direct-connection twin of the server's `db:authz:probe` action
149
+ * (`@everystack/server` plugin.ts) — the local venue for `db:authz:test` / `db:authz:owner`.
150
+ *
151
+ * The probe SQL must WRITE to test INSERT/UPDATE/DELETE privileges, so the only thing
152
+ * standing between a red-team run and a mutated developer database is the rollback. It is
153
+ * therefore unconditional: the transaction body always throws `PROBE_ROLLBACK` after
154
+ * reading the outcome rows, so postgres.js aborts it on every path — success included.
155
+ * There is no code path that commits. `SET ROLE` is transactional (its effect disappears
156
+ * when the transaction aborts), so the session's role is restored by the same rollback.
157
+ *
158
+ * `max: 1` is load-bearing, not tuning: the probe's SET ROLE / savepoint state only makes
159
+ * sense on the single connection that ran the setup.
160
+ */
161
+ export async function createUrlProbeRunner(
162
+ url: string,
163
+ load: () => Promise<any> = () => import('postgres'),
164
+ ): Promise<UrlProbeRunner> {
165
+ const postgres = await loadPostgresDriver(load);
166
+ const sql = postgres(url, { max: 1, max_lifetime: null, onnotice: () => {}, ...sslDefaults(url) });
167
+ return {
168
+ runner: async (query: string) => Array.from(await sql.unsafe(query)),
169
+ probe: async (setup: string, read: string) => {
170
+ let rows: any[] = [];
171
+ try {
172
+ await sql.begin(async (tx: any) => {
173
+ await tx.unsafe(setup);
174
+ rows = Array.from(await tx.unsafe(read));
175
+ throw PROBE_ROLLBACK; // discard every probe write — unconditional
176
+ });
177
+ } catch (err: unknown) {
178
+ if (err !== PROBE_ROLLBACK) throw err;
179
+ }
180
+ return rows;
181
+ },
182
+ end: () => sql.end({ timeout: 5 }),
183
+ };
184
+ }
185
+
135
186
  /**
136
187
  * Managed Postgres (RDS / Supabase / Neon) requires TLS — a direct connection without it
137
188
  * is rejected with pg_hba "no encryption". Default `ssl: 'require'` for remote hosts so the
package/src/cli/index.ts CHANGED
@@ -373,7 +373,7 @@ Usage:
373
373
  everystack db:restore --from <id> [--stage <name>] --confirm Restore a backup INTO the stage's DB (DESTRUCTIVE)
374
374
  everystack db:backup:download <id> [--stage <name>] Presigned URL to download a backup's dump (valid 1h)
375
375
  everystack db:export --schema <name> [--stage <name> | --database-url <url> [--out <file.dump>]] [--models <barrel>] Schema-scoped pg_dump artifact, stamped with the DECLARED schema fingerprint (the canonical-sync export; db:swap gates on that stamp). --stage dumps the stage's private DB via the ops Lambda → S3; --database-url (explicit flag, never the env) dumps a reachable DB to a local .dump + .meta.json — the build-locally → publish → swap on-ramp
376
- everystack db:swap --schema <name> --database-url <url> --from <artifact.dump> [--fingerprint <hash>] Land a schema artifact atomically: fingerprint gate → restore into <schema>_incoming (COPY-safe rewrite) → one txn (drop+rename+recreate app→schema FKs, re-apply authz) → verify → drop retiring. Refresh-free; app.* untouched. DESTRUCTIVE (--stage/--direct ops venue rides stage-write-lanes)
376
+ everystack db:swap --schema <name> --from <artifact.dump | artifact-id> [--stage <name> --direct | --database-url <url>] --confirm [--fingerprint <hash>] [--snapshot physical|logical|none] [--snapshot-ref <id>] [--rebuild-derived] [--dump-build <file.json>] Land a schema artifact atomically: fingerprint gate → pre-flight refusals → CONFIRMED snapshot → restore into <schema>_incoming (COPY-safe rewrite) → build the paired derived layer → one txn (drop+rename+recreate app→schema FKs, re-apply authz + schema USAGE) → assertions → drop retiring. Refresh-free; app.* untouched; the derived layer is never absent. DESTRUCTIVE. The venue is EXPLICIT — DATABASE_URL in the env is refused, and --stage requires --direct (a multi-GB restore exceeds the ops-Lambda 900s clock). The rollback point defaults to a PHYSICAL RDS snapshot on a stage that exposes databaseInstanceId (no locks, no pg_dump contending with the restore) and a WAITED logical db:backup otherwise; a bare --database-url refuses without --snapshot-ref <id> or --snapshot none. docs/schema-swap.md
377
377
  everystack db:generate [--stage <name> | --database-url <url>] [--name <label>] [--models db/models/index.ts] [--schema-out db/schema.generated.ts] [--allow-drops] [--apply] [--dry-run] Diff models vs the live DB → next migration file, or with --apply execute it directly (one transaction, schema_log recorded, verified by re-diff — no drizzle folder needed; direct connection only; DROPs held back unless --allow-drops). --dry-run prints the edge and writes NOTHING (no migration, no journal entry, no schema refresh) — the preview verb; db:diff computes a models-vs-models edge with no database at all. The resolved --schema-out is recorded in the migration journal: later flag-less runs reuse it (flag > recorded > default), a differing flag updates the record and says so
378
378
  everystack db:pull [--stage <name> | --database-url <url>] [--schema public] [--out <dir | file.ts>] [--derived-out <file.ts>] [--abilities public-read] Introspect the live DB → render field() Models (the brownfield on-ramp). --out <dir> writes one file per model + index.ts (the default shape); --out <file.ts> writes a single module; stdout otherwise. --derived-out <file.ts> extracts the derived layer (descriptors + sequences) as its own self-contained module — alone it leaves the models untouched (the hand-maintained-barrel splice); with --out the models render omits the now-external derived layer. Every model scaffolds its authz decision as comments (db:check fails until authored); --abilities public-read stamps the common stanza (public read, admin write) uncommented — explicit generated code, never a runtime default. --matviews-as-tables renders every matview as defineMaterializedTable with INTROSPECTED fields (the canonical-sync flip: a pipeline-owned table everystack migrates but never refreshes) — names land in an exported materializedTables array to spread into your models; fields come back nullable/unkeyed (matviews carry no PK/NOT NULL) — tighten on review; add --suggest-keys to probe the LIVE rows for functionally-unique columns (one scan per matview) and surface each as a commented .primaryKey() suggestion. docs/derived-objects.md#flipping-a-matview-to-a-materialized-table---matviews-as-tables
379
379
  Both introspect via the deployed ops Lambda by default; --database-url (or an inherited DATABASE_URL) connects directly — for a schema that exists only on a local Postgres.
@@ -394,10 +394,11 @@ Usage:
394
394
  everystack db:template:refresh [--database-url <url>] [--models <barrel>] [--seed "<cmd>"] [--no-seed] (Re)build the dev template <base>_tpl FROM THE DECLARED STATE (both layers, fingerprint MATCH bar) + seed-as-code (the app's db:seed script, run with DATABASE_URL pointed at the template; --seed overrides). All or nothing: a failed build/seed leaves NO template. Never a data copy
395
395
  everystack db:branch [--list | --drop --confirm | --prune --confirm] [--database-url <url>] Mint (or find) the current git branch's database from the template (CREATE DATABASE … TEMPLATE — schema, authz, derived layer, and seed rows inherited), print its DATABASE_URL, then db:sync evolves it with the checkout. --list maps every branch DB to its branch; --prune drops the ones whose branch is gone (never guesses: unknown mappings are kept)
396
396
  everystack db:fork --from-stage <src> --stage <target> --confirm [--backup <id>] Fork one DEPLOYED stage's database into another: back up the source (or reuse --backup <id>), presign the dump (the operator's credentials ARE the cross-stage authorization; expires in 1h), restore into the target via its ops Lambda. Production is never a target (that's db:restore); forking FROM production warns about PII; the branch's schema edge then lands via db:plan → db:apply (descent composes). Teardown: sst remove --stage <target>
397
- everystack db:authz:pull [--stage <name>] [--dir authz] Introspect live authz (rls/grants/policies/secdef) → reviewable contract files
398
- everystack db:authz:diff [--stage <name>] [--dir authz] Validate the live DB against the committed contract (non-zero exit on drift)
399
- everystack db:authz:test [--stage <name>] [--dir authz] Red-team enforcement: SET ROLE + attempt per role/table/command (non-zero exit on a hole)
400
- everystack db:authz:owner [--stage <name>] [--models db/models/index.ts] Red-team owner isolation: two JWT identities per owner-scoped table — catches IDOR (non-zero exit on a leak)
397
+ everystack db:authz:pull [--stage <name> | --database-url <url>] [--dir authz] Introspect live authz (rls/grants/policies/secdef) → reviewable contract files
398
+ everystack db:authz:diff [--stage <name> | --database-url <url>] [--dir authz] Validate the live DB against the committed contract (non-zero exit on drift)
399
+ everystack db:authz:test [--stage <name> | --database-url <url>] [--dir authz] Red-team enforcement: SET ROLE + attempt per role/table/command (non-zero exit on a hole)
400
+ everystack db:authz:owner [--stage <name> | --database-url <url>] [--models db/models/index.ts] Red-team owner isolation: two JWT identities per owner-scoped table — catches IDOR (non-zero exit on a leak)
401
+ All four accept a DIRECT venue (--database-url, or an inherited ADMIN_DATABASE_URL/DATABASE_URL) so a brownfield authz migration can be rehearsed against a local database instead of a deployed stage. Same SQL, same evaluation, same verdict — every run names the venue it judged. The red-team probes WRITE to test INSERT/UPDATE/DELETE privileges and are always rolled back.
401
402
  everystack db:authz:report [--dir authz] Render the committed contract as a human-readable authorization review (no DB)
402
403
  everystack console --stage <name> [--sandbox] Interactive REPL on deployed Lambda
403
404
  everystack status [--stage <name>] [--hours <n>] Platform health: CDN, Lambda, rollup summary
@@ -78,6 +78,24 @@ export interface HeartbeatSample {
78
78
  blockedBy: string | null;
79
79
  }
80
80
 
81
+ /**
82
+ * What the CLIENT has sent, to sit alongside what the server reports.
83
+ *
84
+ * The server side alone cannot tell a stalled load from a finished one. Both look like
85
+ * `Client/ClientRead` with flat progress: the server is waiting for the next command. The difference
86
+ * is whether there is anything left to send, and only the client knows that. Without this the
87
+ * detector called the deadlock signature on a run that had landed all 5,984,956 rows and was one
88
+ * second from completing.
89
+ */
90
+ export interface ClientFeed {
91
+ /** Bytes handed to psql's stdin so far. */
92
+ fedBytes: number;
93
+ /** Bytes the restore has to feed in total (the rewritten SQL file's size). */
94
+ totalBytes: number;
95
+ /** The local reader has written everything — nothing more is coming from our side. */
96
+ done: boolean;
97
+ }
98
+
81
99
  export type Liveness =
82
100
  /** Rows or COPY bytes grew since the last poll. */
83
101
  | 'progressing'
@@ -85,6 +103,12 @@ export type Liveness =
85
103
  | 'blocked'
86
104
  /** Flat progress AND the server is waiting on the client: the deadlock signature. */
87
105
  | 'client-stall'
106
+ /**
107
+ * The client has sent everything and the server is waiting on it. That is not a stall — it is the
108
+ * tail of a successful load (the final commit, the connection winding down). Distinguishing this
109
+ * from `client-stall` is the whole reason ClientFeed exists.
110
+ */
111
+ | 'finishing'
88
112
  /** Connected and working, but nothing measurable moved this poll (DDL, index build). */
89
113
  | 'busy'
90
114
  /** No psql backend connected — not started yet, or already gone. */
@@ -218,9 +242,14 @@ function movedForward(prev: HeartbeatSample, cur: HeartbeatSample): boolean {
218
242
  * 3. no predecessor → busy (cannot claim progress OR a stall on the first poll)
219
243
  * 4. rows/bytes grew → progressing, EVEN in ClientRead (the healthy high-latency case)
220
244
  * 5. schema not created → busy (DDL phase, nothing to measure)
221
- * 6. flat + ClientRead → client-stall (the deadlock signature)
245
+ * 6. flat + ClientRead + the client still has bytes to send → client-stall (the deadlock signature)
246
+ * 7. flat + ClientRead + the client has sent everything → finishing (the successful tail)
247
+ *
248
+ * Step 7 is the B5 fix. "Nothing is moving" was treated as sufficient evidence of a stall, and it is
249
+ * not: at the end of a healthy load nothing moves either. A stall claim now requires UNLANDED WORK —
250
+ * bytes the client still owes the server — which is why `feed` is threaded this far down.
222
251
  */
223
- export function classify(prev: HeartbeatSample | null, cur: HeartbeatSample): Liveness {
252
+ export function classify(prev: HeartbeatSample | null, cur: HeartbeatSample, feed?: ClientFeed): Liveness {
224
253
  if (cur.state === null) return 'absent';
225
254
  if (!prev) return 'busy';
226
255
  // PROGRESS OUTRANKS EVERY SCARY WAIT STATE, including Lock. A restore takes relation locks
@@ -231,7 +260,7 @@ export function classify(prev: HeartbeatSample | null, cur: HeartbeatSample): Li
231
260
  if (movedForward(prev, cur)) return 'progressing';
232
261
  if (cur.waitEventType === 'Lock') return 'blocked';
233
262
  if (!cur.schemaExists) return 'busy';
234
- if (cur.waitEvent === 'ClientRead') return 'client-stall';
263
+ if (cur.waitEvent === 'ClientRead') return feed?.done ? 'finishing' : 'client-stall';
235
264
  return 'busy';
236
265
  }
237
266
 
@@ -251,16 +280,70 @@ function groupNum(n: number): string {
251
280
  return n.toLocaleString('en-US');
252
281
  }
253
282
 
254
- /** One operator-readable line per poll. Says what moved, what it is waiting on, and how long in. */
255
- export function formatSample(prev: HeartbeatSample | null, cur: HeartbeatSample, liveness: Liveness): string {
283
+ /** `1.9 GB`, `512 KB` bytes at poll-line resolution. */
284
+ function fmtBytes(n: number): string {
285
+ const units = ['B', 'KB', 'MB', 'GB', 'TB'];
286
+ let v = n;
287
+ let u = 0;
288
+ while (v >= 1024 && u < units.length - 1) { v /= 1024; u += 1; }
289
+ return `${u === 0 ? v : v.toFixed(1)} ${units[u]}`;
290
+ }
291
+
292
+ /**
293
+ * WHICH signal moved between two samples, named honestly.
294
+ *
295
+ * The bug (B6): the progressing line said "COPY advancing" whenever the row delta was zero,
296
+ * regardless of what had actually moved. So a restore in its index phase — relations growing, no
297
+ * COPY anywhere — printed "COPY advancing", and so did the poll right after the final COPY finished.
298
+ * An idle tail read as a live load. Each branch below reports only what it actually observed.
299
+ */
300
+ export function progressLabel(prev: HeartbeatSample, cur: HeartbeatSample): string {
301
+ const dRows = (cur.rows ?? 0) - (prev.rows ?? 0);
302
+ if (dRows > 0) {
303
+ const secs = Math.max(1, (cur.elapsedMs - prev.elapsedMs) / 1000);
304
+ return `+${groupNum(dRows)} row(s) (${groupNum(Math.round(dRows / secs))} rows/s)`;
305
+ }
306
+ const copyMoved = cur.copyBytes > prev.copyBytes || cur.copyRows > prev.copyRows;
307
+ // "Advancing" is a claim about NOW, so it needs a COPY running now — not merely one that moved.
308
+ if (copyMoved && cur.copies > 0) return 'COPY advancing';
309
+ const dRel = cur.relations - prev.relations;
310
+ if (dRel > 0) return `+${groupNum(dRel)} relation(s) created`;
311
+ if (copyMoved) return 'a COPY completed since the last poll';
312
+ return 'something moved';
313
+ }
314
+
315
+ /** The client-side position clause — `client fed 1.9 GB/2.0 GB (95%)`. Empty when unknown. */
316
+ function clientClause(feed?: ClientFeed): string {
317
+ if (!feed) return '';
318
+ const pct = feed.totalBytes > 0 ? Math.floor((feed.fedBytes / feed.totalBytes) * 100) : 0;
319
+ const sent = feed.done ? 'client has fed ALL' : 'client fed';
320
+ return `, ${sent} ${fmtBytes(feed.fedBytes)}/${fmtBytes(feed.totalBytes)} (${pct}%)`;
321
+ }
322
+
323
+ /**
324
+ * One operator-readable line per poll. Says what moved, what it is waiting on, how long in — and, when
325
+ * known, what the CLIENT has sent.
326
+ *
327
+ * That last part is B7. A `Client/ClientRead` wait with the client 20% through the file and the same
328
+ * wait with the file fully fed are completely different situations, and they used to print
329
+ * identically: two debugging sessions went down the wrong path because "the client is not sending"
330
+ * and "the instance has no I/O left" were indistinguishable in the log.
331
+ */
332
+ export function formatSample(
333
+ prev: HeartbeatSample | null,
334
+ cur: HeartbeatSample,
335
+ liveness: Liveness,
336
+ feed?: ClientFeed,
337
+ ): string {
256
338
  const at = `t+${humanElapsed(cur.elapsedMs)}`;
257
339
  const wait = cur.waitEventType ? `${cur.waitEventType}/${cur.waitEvent ?? '?'}` : (cur.state ?? 'idle');
340
+ const client = clientClause(feed);
258
341
 
259
342
  if (liveness === 'absent') {
260
343
  return `restore: no psql backend connected to the target (${at}) — the loader has not started yet, or has already exited.`;
261
344
  }
262
345
  if (!cur.schemaExists) {
263
- return `restore: schema not created yet, no tables yet — DDL phase, ${wait} (${at}).`;
346
+ return `restore: schema not created yet, no tables yet — DDL phase, ${wait}${client} (${at}).`;
264
347
  }
265
348
 
266
349
  const rows = cur.rows === null ? 'unknown' : groupNum(cur.rows);
@@ -275,20 +358,18 @@ export function formatSample(prev: HeartbeatSample | null, cur: HeartbeatSample,
275
358
  if (liveness === 'blocked') {
276
359
  // Naming the holder is the difference between an actionable report and a shrug.
277
360
  const by = cur.blockedBy ? ` HELD BY ${cur.blockedBy}` : ' (holder not visible — it may belong to another role)';
278
- return `restore: waiting on ${wait} with no progress this poll${by} — ${scope} (${at}).`;
361
+ return `restore: waiting on ${wait} with no progress this poll${by} — ${scope}${client} (${at}).`;
362
+ }
363
+ if (liveness === 'finishing') {
364
+ return `restore: ${scope}${client} — FINISHING: everything has been sent, the server is winding the load down (${wait}), ${at}.`;
279
365
  }
280
366
  if (liveness === 'client-stall') {
281
- return `restore: nothing landed since the last poll and the server is waiting on the client (${wait}) — ${scope} (${at}).`;
367
+ return `restore: nothing landed since the last poll and the server is waiting on the client (${wait}) — ${scope}${client} (${at}).`;
282
368
  }
283
369
  if (liveness === 'progressing' && prev) {
284
- const dRows = (cur.rows ?? 0) - (prev.rows ?? 0);
285
- const secs = Math.max(1, (cur.elapsedMs - prev.elapsedMs) / 1000);
286
- const moved = dRows > 0
287
- ? `+${groupNum(dRows)} row(s) (${groupNum(Math.round(dRows / secs))} rows/s)`
288
- : 'COPY advancing';
289
- return `restore: ${scope}, ${moved} — ${wait} (${at}).`;
370
+ return `restore: ${scope}, ${progressLabel(prev, cur)} ${wait}${client} (${at}).`;
290
371
  }
291
- return `restore: ${scope}, nothing new this poll — ${wait} (${at}).`;
372
+ return `restore: ${scope}, nothing new this poll — ${wait}${client} (${at}).`;
292
373
  }
293
374
 
294
375
  /**
@@ -335,6 +416,12 @@ export interface HeartbeatOptions {
335
416
  * backend for. Must not throw; a sink that does is ignored so it cannot kill the heartbeat.
336
417
  */
337
418
  onSample?: (sample: HeartbeatSample) => void;
419
+ /**
420
+ * The client's current write position, read fresh each poll. Supplying it is what lets the
421
+ * heartbeat tell a stalled load from a finishing one (see ClientFeed) — without it the detector
422
+ * falls back to server-side-only reasoning and will call a quiet tail a stall.
423
+ */
424
+ clientFeed?: () => ClientFeed;
338
425
  }
339
426
 
340
427
  /**
@@ -363,8 +450,10 @@ export function startHeartbeat(runner: QueryRunner, opts: HeartbeatOptions): ()
363
450
  // Raw sample first: a caller acting on the probe (the dead-backend watchdog) must see every
364
451
  // sample, and must never be able to break the heartbeat by throwing.
365
452
  try { opts.onSample?.(cur); } catch { /* a sink's failure is not the probe's problem */ }
366
- const liveness = classify(prev, cur);
367
- const line = formatSample(prev, cur, liveness);
453
+ let feed: ClientFeed | undefined;
454
+ try { feed = opts.clientFeed?.(); } catch { /* same: a bad reader must not kill the probe */ }
455
+ const liveness = classify(prev, cur, feed);
456
+ const line = formatSample(prev, cur, liveness, feed);
368
457
 
369
458
  if (liveness === 'blocked') {
370
459
  warn(line);
@@ -0,0 +1,260 @@
1
+ /**
2
+ * db:swap's pre-swap rollback point — which snapshot to take, and proof that it exists.
3
+ *
4
+ * The swap's step 2 is "SNAPSHOT — the rollback point, before anything destructive". It was taking a
5
+ * LOGICAL backup (`db:backup` via the ops Lambda) and that turned out to be wrong twice over:
6
+ *
7
+ * - It never waited. The ops action DISPATCHES a Fargate task and returns the run id; the swap
8
+ * printed "snapshot on record" and started restoring immediately. So the pg_dump ran alongside
9
+ * the restore — a consumer measured the restore blocked ~5 minutes on
10
+ * `Lock/relation HELD BY pid [pg_dump] COPY <schema>.<table>`, the swap contending with its own
11
+ * backup. On a small instance that makes the swap's duration a coin flip unrelated to data volume.
12
+ * - Worse than slow: unproven. A task that failed to start, or died on its credential, left the
13
+ * swap proceeding into a destructive rename believing it had a rollback point it did not have.
14
+ *
15
+ * A PHYSICAL RDS snapshot is the better rollback point on every axis that matters here: it is a
16
+ * control-plane call, so it holds no relation locks, reads nothing through the buffer cache, and
17
+ * needs no client connection. It is also RDS-only, hence the mode selection below rather than a
18
+ * straight replacement.
19
+ *
20
+ * The decision is pure and the confirmation takes its IO injected, so both are provable without an
21
+ * AWS account. See docs/schema-swap.md.
22
+ */
23
+
24
+ /** The values `--snapshot` accepts. `none` is the explicit opt-out; there is no implicit one. */
25
+ export const SNAPSHOT_MODES = ['physical', 'logical', 'none'] as const;
26
+ export type SnapshotModeRequest = (typeof SNAPSHOT_MODES)[number];
27
+
28
+ /** What db:swap should do about a rollback point before it touches anything. */
29
+ export type SnapshotDecision =
30
+ /** Take an RDS snapshot of this instance (no locks, no pg_dump, no connection). */
31
+ | { mode: 'physical'; instanceId: string }
32
+ /** Dispatch db:backup and WAIT for the task to finish before restoring. */
33
+ | { mode: 'logical' }
34
+ /** The operator already took one and named it — nothing to do but record it. */
35
+ | { mode: 'attested'; ref: string }
36
+ /** The operator explicitly accepted no rollback point. */
37
+ | { mode: 'none' }
38
+ /** Nothing safe is available and no consent was given — refuse before anything moves. */
39
+ | { mode: 'refuse'; reason: string };
40
+
41
+ export interface SnapshotDecisionInput {
42
+ /**
43
+ * `stage` = `--stage <name> --direct`, which has an ops Lambda and deployed outputs behind it.
44
+ * `url` = a bare `--database-url`, which has neither: no stage config to read an instance id from
45
+ * and no ops function to dispatch a backup to.
46
+ */
47
+ venue: 'stage' | 'url';
48
+ /** `config.databaseInstanceId`. `placeholder` is what a non-RDS or dev-mode stage carries. */
49
+ instanceId?: string;
50
+ /** `--snapshot-ref <id>` — a snapshot the operator took themselves (db:apply's precedent). */
51
+ snapshotRef?: string;
52
+ /** `--snapshot <physical|logical|none>`. */
53
+ requested?: SnapshotModeRequest;
54
+ }
55
+
56
+ /** Is this instance id something we can actually snapshot, or an absent/placeholder output? */
57
+ function usableInstanceId(id?: string): id is string {
58
+ return !!id && id !== 'placeholder';
59
+ }
60
+
61
+ /**
62
+ * Choose the rollback point.
63
+ *
64
+ * Precedence, and the reasoning for it:
65
+ * 1. `--snapshot-ref` — the operator attests one exists. Taking a second is waste, and this is the
66
+ * shape db:apply's direct lane already uses, so the two destructive verbs read the same.
67
+ * 2. `--snapshot none` — informed consent to have no rollback point. Explicit only.
68
+ * 3. `--snapshot physical|logical` — an explicit choice, honoured or refused with the reason.
69
+ * 4. The default: physical on an RDS stage, logical on any other stage, refuse on a bare URL.
70
+ *
71
+ * The bare-URL default is a REFUSAL rather than a warning. It used to warn and carry on, which is
72
+ * the same class of hole as the `--stage` bypass 0.4.36 closed: a destructive verb whose safety step
73
+ * is optional in practice.
74
+ */
75
+ export function decideSnapshotMode(input: SnapshotDecisionInput): SnapshotDecision {
76
+ const { venue, instanceId, snapshotRef, requested } = input;
77
+
78
+ if (requested !== undefined && !SNAPSHOT_MODES.includes(requested)) {
79
+ return {
80
+ mode: 'refuse',
81
+ reason: `--snapshot ${requested} is not a snapshot mode. Pass one of: ${SNAPSHOT_MODES.join(', ')} `
82
+ + `(physical = an RDS snapshot, logical = a db:backup pg_dump, none = explicitly no rollback point).`,
83
+ };
84
+ }
85
+
86
+ if (snapshotRef) return { mode: 'attested', ref: snapshotRef };
87
+ if (requested === 'none') return { mode: 'none' };
88
+
89
+ if (requested === 'physical') {
90
+ // A physical snapshot is an RDS control-plane call, which needs a REGION as well as an instance
91
+ // id — and the only source of a region here is the stage's deployed config. A bare
92
+ // --database-url has none, so honouring the request would mean calling AWS with an undefined
93
+ // region and reporting a confusing SDK error instead of the real problem.
94
+ if (venue === 'url') {
95
+ return {
96
+ mode: 'refuse',
97
+ reason: 'A physical snapshot needs the stage\'s region and instance id, and a bare --database-url carries neither. '
98
+ + 'Take one against the stage (everystack db:snapshot --stage <name>) and attest it here: --snapshot-ref <id>. '
99
+ + 'Or accept the risk explicitly with --snapshot none.',
100
+ };
101
+ }
102
+ if (!usableInstanceId(instanceId)) {
103
+ return {
104
+ mode: 'refuse',
105
+ reason: 'A physical snapshot needs the RDS instance id, and this stage does not expose one. '
106
+ + 'Add `databaseInstanceId: database.id` to the outputs return block in sst.config.ts and redeploy '
107
+ + '(run db:swap from the app directory so .sst/outputs.json is readable), or pass --snapshot logical '
108
+ + 'for a pg_dump rollback point instead.',
109
+ };
110
+ }
111
+ return { mode: 'physical', instanceId };
112
+ }
113
+
114
+ if (requested === 'logical') {
115
+ if (venue === 'url') {
116
+ return {
117
+ mode: 'refuse',
118
+ reason: 'A logical snapshot runs in the stage\'s Task lane via the ops Lambda, and a bare --database-url has no stage behind it. '
119
+ + 'Take one yourself and name it: --snapshot-ref <id> (everystack db:backup --database-url … or db:snapshot), '
120
+ + 'or accept the risk explicitly with --snapshot none.',
121
+ };
122
+ }
123
+ return { mode: 'logical' };
124
+ }
125
+
126
+ // No explicit request — the default per venue.
127
+ if (venue === 'stage') {
128
+ return usableInstanceId(instanceId) ? { mode: 'physical', instanceId } : { mode: 'logical' };
129
+ }
130
+ return {
131
+ mode: 'refuse',
132
+ reason: 'db:swap over a bare --database-url does not take a snapshot for you, and it will not run destructively without one. '
133
+ + 'Take a rollback point and name it: --snapshot-ref <id> (everystack db:snapshot, or db:backup --database-url …). '
134
+ + 'If you genuinely want no rollback point, say so: --snapshot none.',
135
+ };
136
+ }
137
+
138
+ /**
139
+ * A finished `pollTaskUntilStopped` result, read as "is there a rollback point or not".
140
+ *
141
+ * Kept pure and separate because this is the judgement the old code never made: it treated the
142
+ * DISPATCH as the answer. Every non-success outcome here has to abort the swap, and each one needs
143
+ * different advice, so the wording is worth pinning in a test.
144
+ */
145
+ export function interpretBackupPoll(
146
+ poll:
147
+ | { outcome: 'timeout'; lastStatus: string }
148
+ | { outcome: 'error'; status: { error?: string } }
149
+ | { outcome: 'stopped'; status: { exitCode?: number | null; stoppedReason?: string | null } },
150
+ ids: { runId: string; id: string },
151
+ ): { ok: true } | { ok: false; reason: string } {
152
+ const untouched = 'so the swap was NOT applied and live is untouched.';
153
+ if (poll.outcome === 'timeout') {
154
+ return {
155
+ ok: false,
156
+ reason: `the pre-swap backup did not finish in time (last status: ${poll.lastStatus}), ${untouched} `
157
+ + `Run id ${ids.runId} — check everystack.task_log / ECS, then re-run with --snapshot-ref ${ids.id} once the backup is on record.`,
158
+ };
159
+ }
160
+ if (poll.outcome === 'error') {
161
+ return {
162
+ ok: false,
163
+ reason: `the pre-swap backup's status could not be read (${poll.status.error}), so the swap was NOT applied. `
164
+ + `It may still be running — reconcile run id ${ids.runId} before retrying.`,
165
+ };
166
+ }
167
+ if (poll.status.exitCode !== 0) {
168
+ return {
169
+ ok: false,
170
+ reason: `the pre-swap backup FAILED (exit ${poll.status.exitCode ?? 'unknown'})`
171
+ + `${poll.status.stoppedReason ? ` — ${poll.status.stoppedReason}` : ''}, ${untouched} `
172
+ + `Read the task logs (CloudWatch) before retrying.`,
173
+ };
174
+ }
175
+ return { ok: true };
176
+ }
177
+
178
+ /** The RDS control-plane calls confirmPhysicalSnapshot needs, injected so the wait is testable. */
179
+ export interface PhysicalSnapshotIO {
180
+ /** CreateDBSnapshot — returns the new snapshot's identifier and initial status. */
181
+ create: (snapshotId: string) => Promise<{ identifier: string; status: string }>;
182
+ /** DescribeDBSnapshots for the instance (manual snapshots). */
183
+ describe: () => Promise<Array<{ identifier: string; status: string }>>;
184
+ log: (msg: string) => void;
185
+ sleep: (ms: number) => Promise<void>;
186
+ now: () => number;
187
+ }
188
+
189
+ export interface ConfirmPhysicalOptions {
190
+ instanceId: string;
191
+ snapshotId: string;
192
+ /** How long to wait for `available` before refusing. */
193
+ deadlineMs?: number;
194
+ /** How often to re-read the status. */
195
+ pollIntervalMs?: number;
196
+ }
197
+
198
+ /**
199
+ * A snapshot's status is polled to a TERMINAL state before the swap is allowed to continue. 15
200
+ * minutes is generous for a dev-sized instance and short enough that a stuck snapshot surfaces as a
201
+ * refusal rather than an hour of silence.
202
+ */
203
+ const DEFAULT_SNAPSHOT_DEADLINE_MS = 15 * 60_000;
204
+ const DEFAULT_SNAPSHOT_POLL_MS = 10_000;
205
+
206
+ /**
207
+ * Take an RDS snapshot and do not return until RDS says it is `available`.
208
+ *
209
+ * Why wait at all, when the snapshot's consistency point is fixed the moment CreateDBSnapshot is
210
+ * accepted: because "accepted" is not "exists". A snapshot can go to `failed` (instance state,
211
+ * storage), and the entire value of this step is that the operator can get back. Proceeding into a
212
+ * destructive rename on an unconfirmed rollback point is the defect this module was written to
213
+ * remove — waiting is the only thing that turns the printed id into a fact.
214
+ *
215
+ * On the deadline it THROWS naming the snapshot id, because the re-run is one flag: the snapshot is
216
+ * still coming, so `--snapshot-ref <id>` reuses it instead of starting another.
217
+ */
218
+ export async function confirmPhysicalSnapshot(
219
+ io: PhysicalSnapshotIO,
220
+ opts: ConfirmPhysicalOptions,
221
+ ): Promise<{ id: string }> {
222
+ const deadlineMs = opts.deadlineMs ?? DEFAULT_SNAPSHOT_DEADLINE_MS;
223
+ const pollMs = opts.pollIntervalMs ?? DEFAULT_SNAPSHOT_POLL_MS;
224
+ const created = await io.create(opts.snapshotId);
225
+ const id = created.identifier;
226
+ io.log(`physical snapshot ${id} of ${opts.instanceId} — status ${created.status}.`);
227
+ if (created.status === 'available') return { id };
228
+ if (created.status === 'failed') {
229
+ throw new Error(`the pre-swap RDS snapshot ${id} failed immediately, so the swap was NOT applied and live is untouched.`);
230
+ }
231
+
232
+ const start = io.now();
233
+ let lastStatus = created.status;
234
+ for (;;) {
235
+ if (io.now() - start >= deadlineMs) {
236
+ throw new Error(
237
+ `the pre-swap RDS snapshot ${id} is still "${lastStatus}" after ${Math.round(deadlineMs / 60_000)} minutes, so the swap was NOT applied and live is untouched. `
238
+ + `The snapshot is still being taken — it is a real rollback point once it reaches available. `
239
+ + `Watch it with \`everystack db:snapshots\`, then re-run this swap with --snapshot-ref ${id} to reuse it instead of taking another.`,
240
+ );
241
+ }
242
+ await io.sleep(pollMs);
243
+ const snaps = await io.describe();
244
+ const mine = snaps.find((s) => s.identifier === id);
245
+ if (!mine) {
246
+ throw new Error(
247
+ `the pre-swap RDS snapshot ${id} is no longer listed on ${opts.instanceId} — it was deleted or never registered, so there is no rollback point. `
248
+ + `The swap was NOT applied and live is untouched.`,
249
+ );
250
+ }
251
+ if (mine.status !== lastStatus) {
252
+ lastStatus = mine.status;
253
+ io.log(`physical snapshot ${id}: ${lastStatus}`);
254
+ }
255
+ if (mine.status === 'available') return { id };
256
+ if (mine.status === 'failed') {
257
+ throw new Error(`the pre-swap RDS snapshot ${id} FAILED, so the swap was NOT applied and live is untouched. Check the instance's state and storage, then retry.`);
258
+ }
259
+ }
260
+ }