@everystack/cli 0.4.27 → 0.4.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@everystack/cli",
3
- "version": "0.4.27",
3
+ "version": "0.4.29",
4
4
  "description": "CLI and OTA updates for Expo apps on everystack",
5
5
  "license": "AGPL-3.0-only",
6
6
  "author": "Scalable Technology, Inc. <licensing@scalable.technology>",
@@ -45,6 +45,10 @@
45
45
  "types": "./src/refresh.ts",
46
46
  "default": "./src/refresh.ts"
47
47
  },
48
+ "./backfill": {
49
+ "types": "./src/backfill.ts",
50
+ "default": "./src/backfill.ts"
51
+ },
48
52
  "./audit/source": {
49
53
  "types": "./src/cli/audit-source-api.ts",
50
54
  "default": "./src/cli/audit-source-api.ts"
@@ -0,0 +1,23 @@
1
+ /**
2
+ * @everystack/cli/backfill — the one-shot data-job core, for the ops-Lambda lane.
3
+ *
4
+ * db:backfill runs committed, content-addressed *.sql jobs against a database and records each
5
+ * in everystack.backfill_log. It was direct-connection only; this barrel lets the ops `db:backfill`
6
+ * action (stage-write-lanes brick 4) load the same core and run it on the operator connection, so
7
+ * `db:backfill --stage --apply` needs no raw admin URL on the operator's machine. The CLI ships the
8
+ * *.sql files; this plans (by content identity), executes pending jobs in order, and records them.
9
+ */
10
+
11
+ export {
12
+ planBackfills,
13
+ readBackfillLog,
14
+ executeBackfills,
15
+ markBackfillApplied,
16
+ } from './cli/backfill.js';
17
+ export type {
18
+ BackfillPlan,
19
+ BackfillRecord,
20
+ BackfillRunOptions,
21
+ BackfillRunResult,
22
+ } from './cli/backfill.js';
23
+ export type { SourceFile } from './cli/derived-source.js';
package/src/cli/backup.ts CHANGED
@@ -49,6 +49,28 @@ export function keyForId(id: string): string | null {
49
49
  return ref ? `${backupPrefix(ref.stage)}/${ref.name}.dump.gz` : null;
50
50
  }
51
51
 
52
+ // --- Artifact key scheme (schema-scoped exports for the hot-swap) — mirrors server/backup.ts ----
53
+ // A backup is the whole database; an ARTIFACT is one schema, content-addressed, carrying the schema
54
+ // fingerprint it was built against. Separate prefix so the two never collide. db:swap --stage --direct
55
+ // resolves an artifact id (from db:export --stage) to its S3 key, presigns, and downloads it.
56
+
57
+ /** The S3 prefix a schema's artifacts live under (admin-only, encrypted). */
58
+ export function artifactPrefix(schema: string, stage: string): string {
59
+ return `artifacts/${schema}/${stage}`;
60
+ }
61
+
62
+ /** Parse a schema-qualified artifact id `stats/dev/20260716T120000Z-ab12cd`, or null when malformed. */
63
+ export function parseArtifactRef(id: string): { schema: string; stage: string; name: string } | null {
64
+ const m = id.match(/^([A-Za-z0-9_]+)\/([A-Za-z0-9_-]+)\/([A-Za-z0-9_.:-]+)$/);
65
+ return m ? { schema: m[1], stage: m[2], name: m[3] } : null;
66
+ }
67
+
68
+ /** The S3 dump key for an artifact id (inverse of the server's artifactId), or null when malformed. */
69
+ export function keyForArtifactId(id: string): string | null {
70
+ const ref = parseArtifactRef(id);
71
+ return ref ? `${artifactPrefix(ref.schema, ref.stage)}/${ref.name}.dump.gz` : null;
72
+ }
73
+
52
74
  /**
53
75
  * Format a Date as the UTC `YYYYMMDDTHHMMSSZ` stamp used in backup keys/ids. The clock is the
54
76
  * caller's — pass `new Date()` at the call site so this stays pure and unit-testable.
@@ -35,6 +35,9 @@ import { resolveDbSource, createUrlRunner, connectingVia, type DbSource } from '
35
35
  import { resolveConfig, opsFunction } from '../config.js';
36
36
  import { invokeAction, lambdaQueryRunner } from '../aws.js';
37
37
  import { executeApplyPlan, type ApplyPlanResult } from '../apply-execute.js';
38
+ import { estimateOpsRuntimeFit } from '../ops-fit.js';
39
+ import { resolveOperatorUrlViaStage } from '../direct-venue.js';
40
+ import { withMutationLease, MutationLeaseError } from '../mutation-lease.js';
38
41
  import { pgDumpPreflightError } from './db-backup.js';
39
42
  import { step, success, fail, info, warn } from '../output.js';
40
43
 
@@ -253,9 +256,32 @@ export async function dbApplyCommand(flags: Record<string, string>): Promise<voi
253
256
  }
254
257
  }
255
258
 
259
+ // `--stage --direct` (lane 1): resolve the stage's OPERATOR connection from its ops
260
+ // Lambda, then execute CLI-side with an unbounded clock — the same ceremony as the
261
+ // direct path below (which is stage-aware: approver check + auto-snapshot when --stage
262
+ // is set), just a longer clock. The operator never holds a URL; it lives in memory only.
263
+ if (dbSource.kind === 'stage' && flags.direct === 'true') {
264
+ try {
265
+ step('Resolving the operator connection from the stage (--direct)...');
266
+ const op = await resolveOperatorUrlViaStage(flags.stage);
267
+ info(`operator credential resolved (${op.source}) — executing CLI-side, unbounded clock.`);
268
+ dbSource = { kind: 'url', url: op.url, from: 'operator' };
269
+ } catch (err: any) {
270
+ fail(err.message);
271
+ process.exit(1);
272
+ }
273
+ }
274
+
256
275
  // Deployed stage, no direct URL: the write runs in the ops Lambda — the
257
276
  // operator never holds a database URL. --database-url stays local-only.
258
277
  if (dbSource.kind === 'stage') {
278
+ // Will this edge fit the ops-Lambda's 900-second clock? If not, refuse up front and
279
+ // name --direct (credential-free, unbounded) instead of burning 15 minutes to learn it.
280
+ const fit = estimateOpsRuntimeFit(plan);
281
+ if (!fit.fits) {
282
+ fail(`REFUSED (ops-Lambda runtime): ${fit.reason}`);
283
+ process.exit(1);
284
+ }
259
285
  await applyPlanViaStage(plan, flags, forceDescent);
260
286
  return;
261
287
  }
@@ -315,7 +341,13 @@ export async function dbApplyCommand(flags: Record<string, string>): Promise<voi
315
341
  warn(`DESCENT FORCED — the fast-forward rule is bypassed for this apply. Snapshot on record: ${forceDescent}.`);
316
342
  }
317
343
  step('Verifying the target is where the plan started...');
318
- const result = await executeApplyPlan(runner, plan, {
344
+ // The mutation lease: one operator mutates a database at a time. Acquired on this apply's
345
+ // own session (the max:1 createUrlRunner) before any write; a second operator is refused
346
+ // by name. Self-releases on disconnect.
347
+ const result = await withMutationLease(
348
+ runner,
349
+ { verb: 'db:apply', actor: process.env.USER ?? 'unknown' },
350
+ () => executeApplyPlan(runner, plan, {
319
351
  actor: process.env.USER ?? null,
320
352
  gitRef: currentGitRef() ?? plan.gitRef,
321
353
  ...(verifyAuthority ? { verifyAuthority } : {}),
@@ -340,10 +372,16 @@ export async function dbApplyCommand(flags: Record<string, string>): Promise<voi
340
372
  }
341
373
  },
342
374
  } : {}),
343
- });
375
+ }),
376
+ );
344
377
 
345
378
  reportApplyResult(result, plan);
346
379
  } catch (err: any) {
380
+ if (err instanceof MutationLeaseError) {
381
+ // Nothing was applied — the lease refused before any write. Not a rollback.
382
+ fail(err.message);
383
+ process.exit(1);
384
+ }
347
385
  fail(`Apply failed and rolled back: ${err.message}`);
348
386
  process.exit(1);
349
387
  } finally {
@@ -28,6 +28,10 @@ import {
28
28
  } from '../backfill.js';
29
29
  import { currentGitRef } from '../state-apply.js';
30
30
  import { resolveDbSource, createUrlRunner, connectingVia, type DbSource } from '../db-source.js';
31
+ import { resolveOperatorUrlViaStage } from '../direct-venue.js';
32
+ import { withMutationLease, MutationLeaseError } from '../mutation-lease.js';
33
+ import { resolveConfig, opsFunction } from '../config.js';
34
+ import { invokeAction } from '../aws.js';
31
35
  import { readSqlDirIfPresent } from './db-sync.js';
32
36
  import { step, success, fail, info, warn } from '../output.js';
33
37
 
@@ -49,6 +53,71 @@ export function buildBackfillReport(plan: BackfillPlan): string[] {
49
53
  return lines;
50
54
  }
51
55
 
56
+ /**
57
+ * db:backfill --stage: ship the *.sql files to the ops Lambda's db:backfill action, which
58
+ * plans + (under --apply) runs them on the operator connection and records them. The operator
59
+ * never holds a database URL. --direct is the escape for a backfill too big for the 900s clock.
60
+ */
61
+ async function runBackfillViaStage(
62
+ files: { file: string; sql: string }[],
63
+ flags: Record<string, string>,
64
+ ): Promise<void> {
65
+ step('Resolving deployed config...');
66
+ const config = await resolveConfig(flags.stage);
67
+
68
+ let markApplied: string | undefined;
69
+ if (flags['mark-applied'] !== undefined) {
70
+ if (flags['mark-applied'] === 'true') {
71
+ fail('--mark-applied needs a file name from the backfills dir.');
72
+ process.exit(1);
73
+ }
74
+ markApplied = flags['mark-applied'];
75
+ }
76
+
77
+ const payload = {
78
+ files: files.map((f) => ({ file: f.file, sql: f.sql })),
79
+ apply: flags.apply === 'true',
80
+ ...(markApplied ? { markApplied } : {}),
81
+ actor: process.env.USER ?? null,
82
+ gitRef: currentGitRef(),
83
+ };
84
+
85
+ step('Running the backfill lane in the ops Lambda (credential-free)...');
86
+ const res: any = await invokeAction(config.region, opsFunction(config), 'db:backfill', payload);
87
+ if (res?.error) {
88
+ fail(`db:backfill failed: ${res.error}`);
89
+ if (/Unknown action/i.test(String(res.error))) {
90
+ info('The deployed handler predates the db:backfill ops action. Upgrade @everystack/server, or run direct: db:backfill --apply --database-url <url> (or --stage --direct).');
91
+ }
92
+ process.exit(1);
93
+ }
94
+
95
+ if (res?.markedApplied) {
96
+ success(`Recorded ${res.markedApplied} as applied WITHOUT running it (identity ${String(res.identity).slice(0, 12)}).`);
97
+ return;
98
+ }
99
+
100
+ if (res?.plan) for (const line of buildBackfillReport(res.plan)) info(line);
101
+ if (flags.json === 'true') console.log(JSON.stringify(res.plan ?? res, null, 2));
102
+
103
+ if (!res?.applied) {
104
+ if (res?.plan?.pending?.length) info('Run them: everystack db:backfill --stage ' + (flags.stage ?? '<stage>') + ' --apply');
105
+ if (res?.plan?.blocked?.length) process.exit(1);
106
+ return;
107
+ }
108
+
109
+ for (const ran of res.ran ?? []) info(`~ ${ran.file} applied in ${ran.durationMs}ms`);
110
+ if (res.failed) {
111
+ fail(`${res.failed.file} FAILED and rolled back (recorded; it stays pending for the retry): ${res.failed.error}`);
112
+ process.exit(1);
113
+ }
114
+ if (res?.plan?.blocked?.length) {
115
+ fail('Pending jobs ran, but blocked file(s) remain above — fix them (new file name, or --mark-applied).');
116
+ process.exit(1);
117
+ }
118
+ success(`${(res.ran ?? []).length} backfill(s) applied and recorded in everystack.backfill_log.`);
119
+ }
120
+
52
121
  export async function dbBackfillCommand(flags: Record<string, string>): Promise<void> {
53
122
  const dir = flags.dir || DEFAULT_DIR;
54
123
  const files = await readSqlDirIfPresent(dir);
@@ -64,9 +133,28 @@ export async function dbBackfillCommand(flags: Record<string, string>): Promise<
64
133
  fail(err.message);
65
134
  process.exit(1);
66
135
  }
67
- if (dbSource.kind !== 'url') {
68
- fail('db:backfill needs a direct connection (--database-url or DATABASE_URL) the lane reads and writes its per-database record.');
69
- process.exit(1);
136
+
137
+ // `--stage --direct` (lane 1): a backfill too big for the ops-Lambda's 900-second clock runs
138
+ // CLI-side with an unbounded clock. Resolve the stage's operator connection and fall through
139
+ // to the direct path — credential-free, the operator never holds a URL.
140
+ if (dbSource.kind === 'stage' && flags.direct === 'true') {
141
+ try {
142
+ step('Resolving the operator connection from the stage (--direct)...');
143
+ const op = await resolveOperatorUrlViaStage(flags.stage);
144
+ info(`operator credential resolved (${op.source}) — running backfills CLI-side, unbounded clock.`);
145
+ dbSource = { kind: 'url', url: op.url, from: 'operator' };
146
+ } catch (err: any) {
147
+ fail(err.message);
148
+ process.exit(1);
149
+ }
150
+ }
151
+
152
+ // `--stage` (ops path): ship the *.sql files to the ops Lambda's db:backfill action — it plans
153
+ // (by content identity, against everystack.backfill_log) and, under --apply, runs the pending
154
+ // jobs on the operator connection and records them. The operator holds no URL.
155
+ if (dbSource.kind === 'stage') {
156
+ await runBackfillViaStage(files, flags);
157
+ return;
70
158
  }
71
159
 
72
160
  step(connectingVia(dbSource));
@@ -110,7 +198,12 @@ export async function dbBackfillCommand(flags: Record<string, string>): Promise<
110
198
  }
111
199
 
112
200
  step(`Running ${plan.pending.length} backfill(s), each as its own transaction...`);
113
- const result = await executeBackfills(runner, plan, opts);
201
+ // A data-write operator mutation takes the lease so a concurrent operator is refused.
202
+ const result = await withMutationLease(
203
+ runner,
204
+ { verb: 'db:backfill', actor: process.env.USER ?? 'unknown' },
205
+ () => executeBackfills(runner, plan, opts),
206
+ );
114
207
  for (const ran of result.ran) info(`~ ${ran.file} applied in ${ran.durationMs}ms`);
115
208
  if (result.failed) {
116
209
  fail(`${result.failed.file} FAILED and rolled back (recorded; it stays pending for the retry): ${result.failed.error}`);
@@ -121,6 +214,13 @@ export async function dbBackfillCommand(flags: Record<string, string>): Promise<
121
214
  process.exit(1);
122
215
  }
123
216
  success(`${result.ran.length} backfill(s) applied and recorded in everystack.backfill_log.`);
217
+ } catch (err: any) {
218
+ if (err instanceof MutationLeaseError) {
219
+ // The lease refused before any job ran — nothing was written.
220
+ fail(err.message);
221
+ process.exit(1);
222
+ }
223
+ throw err;
124
224
  } finally {
125
225
  await end?.();
126
226
  }
@@ -215,5 +215,5 @@ export async function dbExportCommand(flags: Record<string, string>): Promise<vo
215
215
  fail(`Export failed: ${result.error}`);
216
216
  process.exit(1);
217
217
  }
218
- success(`Artifact ${result.id} (${fmtBytes(result.bytes)}, fingerprint ${String(result.fingerprint).slice(0, 12)}). Deploy with: everystack db:swap --schema ${schema} --stage <target>`);
218
+ success(`Artifact ${result.id} (${fmtBytes(result.bytes)}, fingerprint ${String(result.fingerprint).slice(0, 12)}). Deploy with: everystack db:swap --schema ${schema} --from ${result.id} --stage <target> --direct --confirm`);
219
219
  }
@@ -46,6 +46,8 @@ import {
46
46
  ENSURE_RECONCILER_SQL,
47
47
  } from '../derived-apply.js';
48
48
  import { resolveDbSource, createUrlRunner, connectingVia, type DbSource } from '../db-source.js';
49
+ import { resolveOperatorUrlViaStage } from '../direct-venue.js';
50
+ import { withMutationLease, MutationLeaseError } from '../mutation-lease.js';
49
51
  import { loadDeclaredDerived, retiredSqlDirAnywhere, retiredSqlDirFlagRefusal, type DeclaredDerived } from '../declared-derived.js';
50
52
  import { currentGitRef } from '../state-apply.js';
51
53
  import { resolveConfig, opsFunction } from '../config.js';
@@ -380,6 +382,27 @@ export async function dbReconcileCommand(flags: Record<string, string>): Promise
380
382
  process.exit(1);
381
383
  }
382
384
 
385
+ // `--stage --direct` (lane 1): a 54-object derived rebuild runs 15-25 minutes and blows
386
+ // the ops-Lambda's 900-second clock. Resolve the stage's OPERATOR connection from its ops
387
+ // Lambda and execute CLI-side with an unbounded clock — credential-free, the operator never
388
+ // holds a URL. Only meaningful under --apply (a dry-run just reads). Fall through as a url
389
+ // source so the local-runner branch below runs executeReconcile against it, under the lease.
390
+ if (dbSource.kind === 'stage' && flags.direct === 'true') {
391
+ if (!apply) {
392
+ fail('--direct is a write venue — it only applies with --apply. For a dry-run drop --direct (the stage plans read-only via the ops Lambda).');
393
+ process.exit(1);
394
+ }
395
+ try {
396
+ step('Resolving the operator connection from the stage (--direct)...');
397
+ const op = await resolveOperatorUrlViaStage(flags.stage);
398
+ info(`operator credential resolved (${op.source}) — reconciling CLI-side, unbounded clock.`);
399
+ dbSource = { kind: 'url', url: op.url, from: 'operator' };
400
+ } catch (err: any) {
401
+ fail(err.message);
402
+ process.exit(1);
403
+ }
404
+ }
405
+
383
406
  // --baseline only writes under --apply (executeReconcile returns the plan and executes nothing
384
407
  // when apply=false). Passing it alone used to print "recording provenance…" and exit 0 while
385
408
  // persisting nothing — a silent no-op. Fail loudly instead; the plan already lists needsBaseline.
@@ -459,21 +482,33 @@ export async function dbReconcileCommand(flags: Record<string, string>): Promise
459
482
  fail(`db:reconcile failed: ${result.error}`);
460
483
  if (/Unknown action/i.test(String(result.error))) {
461
484
  info('The deployed handler predates the db:reconcile ops action (needs @everystack/server >= 0.4.8). Upgrade the server, or apply direct: db:reconcile --apply --database-url <url>.');
485
+ } else if (/timed out|timeout|task timed out/i.test(String(result.error))) {
486
+ info('A large derived rebuild (dozens of objects) can exceed the ops-Lambda 900-second clock. Re-run credential-free with an unbounded clock: db:reconcile --apply --stage ' + (flags.stage ?? '<stage>') + ' --direct.');
462
487
  }
463
488
  process.exit(1);
464
489
  }
465
490
  run = { plan: result.plan, applied: result.applied, statements: result.statements ?? [], refusal: result.refusal ?? undefined };
466
491
  } else {
467
492
  let runner: QueryRunner;
493
+ // A write over a direct connection (--database-url or --direct) takes the mutation
494
+ // lease; a read-only stage dry-run over the ops Lambda does not (reads never lease).
495
+ let leased = false;
468
496
  if (dbSource.kind === 'url') {
469
497
  step(connectingVia(dbSource));
470
498
  ({ runner, end } = await createUrlRunner(dbSource.url));
499
+ leased = apply;
471
500
  } else {
472
501
  step('Resolving deployed config...');
473
502
  const config = await resolveConfig(flags.stage);
474
503
  runner = lambdaRunner(config.region, opsFunction(config));
475
504
  }
476
- run = await executeReconcile(runner, reconcileOptions);
505
+ run = leased
506
+ ? await withMutationLease(
507
+ runner,
508
+ { verb: 'db:reconcile', actor: process.env.USER ?? 'unknown' },
509
+ () => executeReconcile(runner, reconcileOptions),
510
+ )
511
+ : await executeReconcile(runner, reconcileOptions);
477
512
  }
478
513
 
479
514
  if (flags.json === 'true') {
@@ -499,6 +534,13 @@ export async function dbReconcileCommand(flags: Record<string, string>): Promise
499
534
 
500
535
  if (run.refusal) process.exit(1);
501
536
  if (check && checkFails(run.plan)) process.exit(1);
537
+ } catch (err: any) {
538
+ if (err instanceof MutationLeaseError) {
539
+ // The lease refused before any DDL — nothing was reconciled.
540
+ fail(err.message);
541
+ process.exit(1);
542
+ }
543
+ throw err;
502
544
  } finally {
503
545
  await end?.();
504
546
  }
@@ -14,9 +14,13 @@
14
14
  */
15
15
 
16
16
  import fs from 'node:fs';
17
+ import os from 'node:os';
18
+ import path from 'node:path';
19
+ import { createGunzip } from 'node:zlib';
17
20
  import { spawn } from 'node:child_process';
18
21
  import { Transform } from 'node:stream';
19
22
  import { pipeline } from 'node:stream/promises';
23
+ import { Readable } from 'node:stream';
20
24
  import type { ModelDescriptor } from '@everystack/model';
21
25
  import { fingerprintModels } from '../schema-fingerprint.js';
22
26
  import { resolveModelsPath } from '../models-path.js';
@@ -25,7 +29,13 @@ import { loadDeclaredDerived } from '../declared-derived.js';
25
29
  import { createUrlRunner } from '../db-source.js';
26
30
  import { executeSwap, type SwapVerdict } from '../swap-execute.js';
27
31
  import { rewriteStatementLine, opensCopyData, closesCopyData } from '../schema-rewrite.js';
28
- import { step, success, fail, warn } from '../output.js';
32
+ import { resolveOperatorUrlViaStage } from '../direct-venue.js';
33
+ import { withMutationLease, MutationLeaseError } from '../mutation-lease.js';
34
+ import { resolveConfig, opsFunction } from '../config.js';
35
+ import { invokeAction, presignGet } from '../aws.js';
36
+ import { keyForArtifactId, metaKey } from '../backup.js';
37
+ import { pgEnvFromUrl } from './db.js';
38
+ import { step, success, fail, warn, info } from '../output.js';
29
39
 
30
40
  /** A COPY-aware line transform that rewrites the schema token on statement lines only. */
31
41
  function schemaRewriteStream(from: string, to: string): Transform {
@@ -64,10 +74,17 @@ function schemaRewriteStream(from: string, to: string): Transform {
64
74
  * rewrite → psql. The archive names `<schema>`; the rewrite lands it as `<incoming>`, COPY-data-safe.
65
75
  */
66
76
  async function restoreIntoIncoming(url: string, artifactPath: string, schema: string, incoming: string): Promise<void> {
67
- // psql takes the connection URL directly (credential parsed by libpq, not on the process table
68
- // beyond argv-as-URL); pg_restore -f - just reads the archive file to SQL on stdout.
77
+ // Connect via PG* env, not `-d <url>`. libpq VALIDATES URI query params against its keyword
78
+ // list and REJECTS non-keywords like `search_path` ("invalid URI query parameter") and the
79
+ // operator URL db:operator-url mints bakes search_path in (fine for postgres.js, fatal for a
80
+ // libpq client). pgEnvFromUrl extracts only libpq keywords (dropping search_path, routing
81
+ // sslmode→PGSSLMODE) and puts the password in PGPASSWORD, off the process argv. pg_restore -f -
82
+ // just reads the archive file to SQL on stdout (no connection).
69
83
  const restore = spawn('pg_restore', ['-f', '-', artifactPath], { stdio: ['ignore', 'pipe', 'pipe'] });
70
- const psql = spawn('psql', ['-v', 'ON_ERROR_STOP=1', '-d', url], { stdio: ['pipe', 'ignore', 'pipe'] });
84
+ const psql = spawn('psql', ['-v', 'ON_ERROR_STOP=1'], {
85
+ stdio: ['pipe', 'ignore', 'pipe'],
86
+ env: { ...process.env, ...pgEnvFromUrl(url) },
87
+ });
71
88
  let rErr = '', pErr = '';
72
89
  restore.stderr.on('data', (d) => { rErr += d.toString(); });
73
90
  psql.stderr.on('data', (d) => { pErr += d.toString(); });
@@ -100,14 +117,115 @@ export function readArtifactFingerprint(artifactPath: string, flag?: string): st
100
117
  }
101
118
  }
102
119
 
120
+ /** A resolved artifact ready for restore: a local plain `-Fc` .dump plus its stamped fingerprint. */
121
+ interface ResolvedArtifact {
122
+ dumpPath: string;
123
+ fingerprint: string | null;
124
+ /** Remove any temp files fetched from S3 (no-op for a local artifact). */
125
+ cleanup: () => Promise<void>;
126
+ }
127
+
128
+ /**
129
+ * Fetch a schema-export artifact from S3 to a local temp `-Fc` .dump for the --direct swap.
130
+ * db:export --stage stores a gzipped -Fc archive + a sibling .meta.json (fingerprint). This
131
+ * presigns both with the CLI's own IAM, streams the dump down, gunzips it (pg_restore reads a
132
+ * plain archive), and reads the stamped fingerprint from the meta. The operator holds no DB
133
+ * credential; the artifact just transits the caller's presigned S3 read.
134
+ */
135
+ async function fetchArtifactFromS3(
136
+ id: string,
137
+ stage: string | undefined,
138
+ fingerprintFlag: string | undefined,
139
+ ): Promise<ResolvedArtifact> {
140
+ const key = keyForArtifactId(id);
141
+ if (!key) throw new Error(`--from is neither a local file nor a valid artifact id (expected schema/stage/stamp, got ${id}).`);
142
+
143
+ const config = await resolveConfig(stage);
144
+ if (!config.backupsBucket) {
145
+ throw new Error('No backupsBucket in the deployed config — cannot fetch the S3 artifact. Add `backupsBucket` to the sst.config outputs and redeploy, or pass a local --from <file.dump>.');
146
+ }
147
+
148
+ const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'everystack-swap-'));
149
+ const dumpPath = path.join(tmpDir, 'artifact.dump');
150
+ const cleanup = async () => { await fs.promises.rm(tmpDir, { recursive: true, force: true }).catch(() => {}); };
151
+
152
+ try {
153
+ step(`Fetching artifact ${id} from S3 (presigned, then gunzip)...`);
154
+ const dumpUrl = await presignGet(config.region, config.backupsBucket, key, 3600);
155
+ const res = await fetch(dumpUrl);
156
+ if (!res.ok || !res.body) throw new Error(`artifact download failed: HTTP ${res.status}`);
157
+ await pipeline(Readable.fromWeb(res.body as any), createGunzip(), fs.createWriteStream(dumpPath));
158
+
159
+ // The fingerprint: an explicit flag wins; otherwise read the sibling .meta.json.
160
+ let fingerprint = fingerprintFlag ?? null;
161
+ if (!fingerprint) {
162
+ try {
163
+ const metaUrl = await presignGet(config.region, config.backupsBucket, metaKey(key), 3600);
164
+ const metaRes = await fetch(metaUrl);
165
+ if (metaRes.ok) fingerprint = (await metaRes.json() as any)?.fingerprint ?? null;
166
+ } catch { /* fall through — the caller reports a missing fingerprint */ }
167
+ }
168
+ return { dumpPath, fingerprint, cleanup };
169
+ } catch (err) {
170
+ await cleanup();
171
+ throw err;
172
+ }
173
+ }
174
+
175
+ /** Resolve --from to a local plain -Fc dump: a local file as-is, else an S3 artifact id fetched down. */
176
+ async function resolveSwapArtifact(
177
+ from: string,
178
+ stage: string | undefined,
179
+ fingerprintFlag: string | undefined,
180
+ ): Promise<ResolvedArtifact> {
181
+ if (fs.existsSync(from)) {
182
+ return { dumpPath: from, fingerprint: readArtifactFingerprint(from, fingerprintFlag), cleanup: async () => {} };
183
+ }
184
+ return fetchArtifactFromS3(from, stage, fingerprintFlag);
185
+ }
186
+
103
187
  export async function dbSwapCommand(flags: Record<string, string>): Promise<void> {
104
188
  const schema = flags.schema;
105
- const url = flags['database-url'] || process.env.DATABASE_URL;
106
189
  const from = flags.from;
190
+ const stage = flags.stage;
191
+ const direct = flags.direct === 'true';
107
192
  if (!schema) { fail('db:swap needs --schema <name>.'); process.exit(1); }
108
- if (!from) { fail('db:swap needs --from <artifact.dump> (the schema-scoped -Fc archive to land).'); process.exit(1); }
193
+ if (!from) { fail('db:swap needs --from <artifact.dump | artifact-id> (the schema-scoped -Fc archive to land).'); process.exit(1); }
194
+
195
+ // Resolve the venue.
196
+ // - --database-url (or DATABASE_URL): a local/direct operator connection.
197
+ // - --stage --direct: resolve the stage's OPERATOR connection from its ops Lambda and execute
198
+ // CLI-side with an unbounded clock (a multi-GB restore blows the 900s Lambda ceiling). The
199
+ // operator never holds a URL; the swap snapshots the stage via db:backup before it lands.
200
+ // - --stage alone: refuse, naming --direct — the ops-Lambda venue can't hold the restore clock.
201
+ let url = flags['database-url'] || process.env.DATABASE_URL;
202
+ let snapshotViaStage = false;
203
+ let region: string | undefined;
204
+ let opsFn: string | undefined;
205
+
206
+ if (!url && stage) {
207
+ if (!direct) {
208
+ fail('db:swap --stage needs --direct: a schema restore can exceed the ops-Lambda 900-second clock, so the swap runs CLI-side with an unbounded clock (credential-free — the operator never holds a URL). Re-run with --stage ' + stage + ' --direct.');
209
+ process.exit(1);
210
+ }
211
+ if (flags.confirm !== 'true') {
212
+ fail('db:swap --stage --direct is destructive (it drops the retiring schema after the swap). Confirm explicitly: --confirm.');
213
+ process.exit(1);
214
+ }
215
+ try {
216
+ const config = await resolveConfig(stage);
217
+ region = config.region;
218
+ opsFn = opsFunction(config);
219
+ step('Resolving the operator connection from the stage (--direct)...');
220
+ const op = await resolveOperatorUrlViaStage(stage);
221
+ url = op.url;
222
+ snapshotViaStage = true;
223
+ info(`operator credential resolved (${op.source}) — swapping CLI-side, unbounded clock.`);
224
+ } catch (err: any) { fail(err.message); process.exit(1); }
225
+ }
226
+
109
227
  if (!url) {
110
- fail('db:swap v1 needs a direct --database-url (the operator connection). The credential-free --stage venue rides the stage-write-lanes bricks; a large restore needs its unbounded clock.');
228
+ fail('db:swap needs a target: --database-url <url> (local/direct), or --stage <name> --direct (credential-free the operator never holds a URL).');
111
229
  process.exit(1);
112
230
  }
113
231
 
@@ -121,24 +239,41 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
121
239
  declaredFingerprint = fingerprintModels(models, { schemas: [schema], sequences: declaredDb?.sequences }).hash;
122
240
  } catch (err: any) { fail(err.message); process.exit(1); }
123
241
 
124
- const artifactFingerprint = readArtifactFingerprint(from, flags.fingerprint);
125
- if (!artifactFingerprint) {
126
- fail(`db:swap can't find the artifact fingerprint — expected a sibling .meta.json next to ${from}, or pass --fingerprint. Without it the gate can't run, and an unchecked swap is exactly what the gate prevents.`);
242
+ // Resolve --from to a local plain -Fc dump (a local file, or an S3 export id fetched down).
243
+ let artifact: ResolvedArtifact;
244
+ try {
245
+ artifact = await resolveSwapArtifact(from, stage, flags.fingerprint);
246
+ } catch (err: any) { fail(err.message); process.exit(1); }
247
+
248
+ if (!artifact.fingerprint) {
249
+ await artifact.cleanup();
250
+ fail(`db:swap can't find the artifact fingerprint — expected a sibling .meta.json next to ${from} (or the artifact's S3 meta), or pass --fingerprint. Without it the gate can't run, and an unchecked swap is exactly what the gate prevents.`);
127
251
  process.exit(1);
128
252
  }
253
+ const artifactFingerprint = artifact.fingerprint;
129
254
 
130
255
  const { runner, end } = await createUrlRunner(url);
131
256
  try {
132
257
  step(`Swapping ${schema} — gate, land incoming, atomic swap, verify...`);
133
- const res = await executeSwap(runner, {
134
- models, schema,
135
- artifactFingerprint,
136
- declaredFingerprint,
137
- applyIncoming: async () => { await restoreIntoIncoming(url, from, schema, `${schema}_incoming`); },
138
- // Snapshot + verify-hook wiring land with the ops venue; a direct v1 swap warns rather than
139
- // silently skipping the safety net.
140
- snapshot: async () => { warn('no snapshot taken (direct v1) — take one first: everystack db:backup --database-url … before a production swap.'); },
141
- });
258
+ // One operator mutates a database at a time — the swap is a whole-schema replacement.
259
+ const res = await withMutationLease(
260
+ runner,
261
+ { verb: 'db:swap', actor: process.env.USER ?? 'unknown' },
262
+ () => executeSwap(runner, {
263
+ models, schema,
264
+ artifactFingerprint,
265
+ declaredFingerprint,
266
+ applyIncoming: async () => { await restoreIntoIncoming(url!, artifact.dumpPath, schema, `${schema}_incoming`); },
267
+ snapshot: snapshotViaStage
268
+ ? async () => {
269
+ step('Snapshotting the stage before the swap (db:backup)...');
270
+ const r: any = await invokeAction(region!, opsFn!, 'db:backup', { stage });
271
+ if (r?.error) throw new Error(`pre-swap snapshot failed, so the swap was NOT applied: ${r.error}`);
272
+ info(`snapshot on record: ${r?.id ?? 'backup complete'} — restore with db:restore --from ${r?.id ?? '<id>'} --confirm.`);
273
+ }
274
+ : async () => { warn('no snapshot taken (direct v1) — take one first: everystack db:backup --database-url … before a production swap.'); },
275
+ }),
276
+ );
142
277
 
143
278
  if (res.status === 'swapped') {
144
279
  success(`Swapped ${schema} — the artifact is live (no refresh ran).`);
@@ -148,9 +283,11 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
148
283
  process.exit(1);
149
284
  }
150
285
  } catch (err: any) {
286
+ if (err instanceof MutationLeaseError) { fail(err.message); process.exit(1); }
151
287
  fail(`Swap failed: ${err.message}`);
152
288
  process.exit(1);
153
289
  } finally {
154
290
  await end?.();
291
+ await artifact.cleanup();
155
292
  }
156
293
  }
@@ -211,7 +211,7 @@ export function buildProvisionSecretPlan(args: {
211
211
  * same seam as pipeline-loader). */
212
212
  export interface ServerProvision {
213
213
  runProvision(
214
- payload: { authPassword?: string; adminPassword?: string },
214
+ payload: { authPassword?: string; adminPassword?: string; searchPath?: string[] },
215
215
  deps: {
216
216
  execute: (sql: string) => Promise<unknown>;
217
217
  connection?: { host: string; port?: number | string; database: string } | null;
@@ -473,6 +473,31 @@ export async function dbProvisionCommand(flags: Record<string, string>): Promise
473
473
  info('Creating the least-privilege role chain on your EXISTING database (no database is created).');
474
474
  step('Provisioning roles...');
475
475
 
476
+ // The declared schema set — resolved BEFORE provisioning so it can be set as the ROLE default
477
+ // (ALTER ROLE … SET search_path), the right layer for bare-ref resolution (search-path-ownership).
478
+ // Union the DERIVED-object schemas (matviews/views live in schemas the barrel never names) so the
479
+ // authenticator flip doesn't go dark on bare-ref matview reads. Best-effort: a modelless or
480
+ // all-public app sets no path (unchanged behavior).
481
+ let declaredSchemas: string[] = [];
482
+ try {
483
+ const { resolveModelsPath } = await import('../models-path.js');
484
+ const { loadModels } = await import('./db-generate.js');
485
+ const models = await loadModels(resolveModelsPath(flags.models));
486
+ let derivedObjects: Array<{ schema: string }> = [];
487
+ try {
488
+ const { loadDeclaredDerived } = await import('../declared-derived.js');
489
+ const declared = await loadDeclaredDerived(flags.models);
490
+ if (declared) derivedObjects = declared.objects;
491
+ } catch {
492
+ // No derived layer resolvable — the table schemas still resolve correctly.
493
+ }
494
+ declaredSchemas = collectDeclaredSchemas({ models, derivedObjects });
495
+ } catch {
496
+ // No models resolvable here — leave the roles schema-agnostic.
497
+ }
498
+ const searchPath = declaredSearchPath(declaredSchemas);
499
+ if (searchPath.length) info(` Declared search_path (set as the role default via ALTER ROLE): ${searchPath.join(', ')}`);
500
+
476
501
  // Generate BOTH login passwords and keep them in memory only — they are written straight
477
502
  // into the secret store and are NEVER printed, logged, or returned to a human.
478
503
  const { randomBytes } = await import('node:crypto');
@@ -486,7 +511,7 @@ export async function dbProvisionCommand(flags: Record<string, string>): Promise
486
511
  const { createUrlRunner } = await import('../db-source.js');
487
512
  const { runner, end } = await createUrlRunner(url);
488
513
  try {
489
- return await runProvision({ authPassword, adminPassword }, {
514
+ return await runProvision({ authPassword, adminPassword, searchPath }, {
490
515
  execute: async (statement) => runner(statement),
491
516
  connection: parseUrlConnection(url),
492
517
  // Same probe as the ops-Lambda venue: a real login as the new role, DDL-capable.
@@ -519,7 +544,7 @@ export async function dbProvisionCommand(flags: Record<string, string>): Promise
519
544
  conn = parseUrlConnection(directUrl);
520
545
  } else {
521
546
  try {
522
- result = await invokeAction(config.region, opsFunction(config), 'db:provision', { authPassword, adminPassword });
547
+ result = await invokeAction(config.region, opsFunction(config), 'db:provision', { authPassword, adminPassword, searchPath });
523
548
  } catch (err: any) {
524
549
  // Auto-fallback: no Ops Lambda (Unknown action), but an ADMIN_DATABASE_URL secret exists —
525
550
  // provision directly over it. Master never crosses the CLI; the operator sets nothing new.
@@ -575,31 +600,6 @@ export async function dbProvisionCommand(flags: Record<string, string>): Promise
575
600
  // wrong-database scare, and that check meant re-handling the master credential).
576
601
  success(`Target database: ${conn.database} @ ${conn.host}:${conn.port ?? 5432} — secrets for stage "${flags.stage}" will point here.`);
577
602
 
578
- // Bake the app's declared schemas into the minted URLs, so a multi-schema API connects with
579
- // the right search_path instead of going dark on bare refs (best-effort: a modelless app or
580
- // an all-public one changes nothing).
581
- let declaredSchemas: string[] = [];
582
- try {
583
- const { resolveModelsPath } = await import('../models-path.js');
584
- const { loadModels } = await import('./db-generate.js');
585
- const models = await loadModels(resolveModelsPath(flags.models));
586
- // Union the DERIVED-object schemas (matviews/views live in schemas the barrel never
587
- // names) so the authenticator flip doesn't go dark on bare-ref matview reads.
588
- let derivedObjects: Array<{ schema: string }> = [];
589
- try {
590
- const { loadDeclaredDerived } = await import('../declared-derived.js');
591
- const declared = await loadDeclaredDerived(flags.models);
592
- if (declared) derivedObjects = declared.objects;
593
- } catch {
594
- // No derived layer resolvable — the table schemas still bake correctly.
595
- }
596
- declaredSchemas = collectDeclaredSchemas({ models, derivedObjects });
597
- } catch {
598
- // No models resolvable here — leave the URLs schema-agnostic.
599
- }
600
- const searchPath = declaredSearchPath(declaredSchemas);
601
- if (searchPath.length) info(` search_path baked into DATABASE_URL: ${searchPath.join(', ')}`);
602
-
603
603
  // Preserve the connection params (sslmode, …) from the URL the app already connects with — the
604
604
  // minted URL is built from bare host/port/db components, so without this a `?sslmode=require`
605
605
  // would be dropped and the new roles could fail to connect on an SSL-forced RDS. Prefer the
@@ -608,11 +608,12 @@ export async function dbProvisionCommand(flags: Record<string, string>): Promise
608
608
  ?? stageSecrets.ADMIN_DATABASE_URL ?? stageSecrets.AdminDatabaseUrl
609
609
  ?? stageSecrets.DATABASE_URL ?? stageSecrets.DatabaseUrl;
610
610
 
611
+ // The minted secret is credentials-only: search_path now lives on the ROLE (ALTER ROLE, above),
612
+ // not baked into the URL. This keeps the secret a pure credential and — critically — yields a
613
+ // libpq-clean URL (no ?search_path=, which psql/pg_restore reject as an unknown keyword). Real
614
+ // connection params (sslmode, …) are still carried from the source URL.
611
615
  const mint = (role: string, password: string): string =>
612
- withSearchPath(
613
- preserveConnParams(`postgresql://${role}:${password}@${conn.host}:${conn.port ?? 5432}/${conn.database}`, sourceUrl),
614
- declaredSchemas,
615
- );
616
+ preserveConnParams(`postgresql://${role}:${password}@${conn.host}:${conn.port ?? 5432}/${conn.database}`, sourceUrl);
616
617
 
617
618
  const authUrl = mint(result.loginRole, authPassword);
618
619
  const adminUrl = result.adminRole ? mint(result.adminRole, adminPassword) : undefined;
@@ -27,7 +27,7 @@
27
27
  import type { QueryRunner } from './authz-contract.js';
28
28
 
29
29
  export type DbSource =
30
- | { kind: 'url'; url: string; from: 'flag' | 'env' | 'admin-env' }
30
+ | { kind: 'url'; url: string; from: 'flag' | 'env' | 'admin-env' | 'operator' }
31
31
  | { kind: 'stage' };
32
32
 
33
33
  /** Decide which path serves this invocation. Pure; env injectable for tests. */
@@ -57,6 +57,9 @@ export function resolveDbSource(
57
57
  export function connectingVia(source: Extract<DbSource, { kind: 'url' }>): string {
58
58
  if (source.from === 'flag') return 'Connecting via --database-url...';
59
59
  if (source.from === 'admin-env') return 'Connecting via ADMIN_DATABASE_URL...';
60
+ if (source.from === 'operator') {
61
+ return 'Connecting --direct (operator credential resolved from the stage\'s ops Lambda, held in memory only)...';
62
+ }
60
63
  return 'Connecting via DATABASE_URL...';
61
64
  }
62
65
 
@@ -0,0 +1,43 @@
1
+ /**
2
+ * The `--direct` venue (stage-write-lanes lane 1).
3
+ *
4
+ * A write verb run with `--stage <name> --direct` resolves the OPERATOR connection from the
5
+ * stage's IAM-gated ops Lambda (the `db:operator-url` action, brick 1), holds the URL in
6
+ * process memory only, and executes CLI-side with an unbounded clock. The ceremony is
7
+ * untouched — only the execution venue moves. Shared by db:apply, db:reconcile, db:backfill,
8
+ * and db:swap so every lane resolves the operator credential the same way.
9
+ *
10
+ * The URL travels only in the Lambda invoke response and this process's memory: never printed,
11
+ * never written, never exported. Callers pass it straight to createUrlRunner and drop it.
12
+ */
13
+
14
+ import { resolveConfig, opsFunction } from './config.js';
15
+ import { invokeAction } from './aws.js';
16
+
17
+ export interface OperatorConnection {
18
+ /** The operator connection string — hold in memory, never print. */
19
+ url: string;
20
+ /** Which credential the stage resolved: 'admin' (migrator) or 'master'. */
21
+ source: string;
22
+ }
23
+
24
+ /**
25
+ * Resolve a stage's operator connection for the `--direct` lane via its ops Lambda.
26
+ * Throws a remediation-named error when no operator connection exists or the ops Lambda
27
+ * predates this action (an older server that does not know `db:operator-url`).
28
+ */
29
+ export async function resolveOperatorUrlViaStage(
30
+ stage: string | undefined,
31
+ invoke: typeof invokeAction = invokeAction,
32
+ ): Promise<OperatorConnection> {
33
+ const config = await resolveConfig(stage);
34
+ const fn = opsFunction(config);
35
+ const res: any = await invoke(config.region, fn, 'db:operator-url', {});
36
+ if (res?.error) throw new Error(res.error);
37
+ if (!res?.url) {
38
+ throw new Error(
39
+ 'the ops Lambda did not return an operator connection for --direct. Deploy a server build that ships the db:operator-url action, or use --database-url for a local connection.',
40
+ );
41
+ }
42
+ return { url: res.url, source: res.source ?? 'operator' };
43
+ }
package/src/cli/index.ts CHANGED
@@ -357,7 +357,7 @@ Usage:
357
357
  everystack db:psql --stage <name> Interactive ADMIN psql (IAM-gated; resolves the admin URL in-process)
358
358
  everystack db:psql [--stage <name>] -c <command> Run one query via Lambda (works for private RDS)
359
359
  everystack db:doctor [--stage <name>] [--direct | --database-url <api> [--admin-database-url <ops>]] Check the DB is least-privilege + RLS-subject (api vs operator connection). No flag = ops-Lambda venue (auto-falls to direct via the stage secrets if the handler has no dbPlugin). --direct = probe both connections from the stage's DATABASE_URL + ADMIN_DATABASE_URL secrets; --database-url = explicit local venue (a lone URL is probed as both)
360
- everystack db:provision --stage <name> [--direct | --database-url <url>] Create the least-privilege role chain on an EXISTING database (idempotent; creates no DB). No flag = ops-Lambda venue (auto-falls to direct via the ADMIN_DATABASE_URL secret if the handler has no dbPlugin). --direct = direct connection reading that secret (master never on argv); --database-url = explicit URL. Declared schemas are baked into the minted DATABASE_URL as search_path
360
+ everystack db:provision --stage <name> [--direct | --database-url <url>] Create the least-privilege role chain on an EXISTING database (idempotent; creates no DB). No flag = ops-Lambda venue (auto-falls to direct via the ADMIN_DATABASE_URL secret if the handler has no dbPlugin). --direct = direct connection reading that secret (master never on argv); --database-url = explicit URL. Declared schemas are set as the ROLE default (ALTER ROLE … SET search_path) — the secret stays credentials-only, and the URL is libpq-clean
361
361
  everystack db:snapshot [--stage <name>] [--instance <id>] Take a physical RDS snapshot (instant DR point; RDS only — use db:backup for portable logical backups)
362
362
  everystack db:snapshots [--stage <name>] [--instance <id>] List manual RDS snapshots for the instance
363
363
  everystack db:backup:probe [--stage <name>] Verify the pg_dump layer is attached + version-compatible with the server
@@ -0,0 +1,228 @@
1
+ /**
2
+ * The universal mutation lease (docs/plans/mutation-lease.md).
3
+ *
4
+ * One lease per database, session-scoped, self-releasing, refuse-don't-coordinate.
5
+ * A PostgreSQL advisory SESSION lock with a fixed documented key, acquired on the SAME
6
+ * session that performs the mutation and held for the verb's whole body. Two operators
7
+ * cannot co-mutate one stage; the second is refused with the holder named, and re-tries
8
+ * nothing.
9
+ *
10
+ * This is the minimal core the stage-write-lanes `--direct` verbs need (mutation-lease
11
+ * brick 1). The full subsystem — the `db:lease` break-glass verb, the connection-layer
12
+ * enforcement of "one session, no second connection", the environment-state board — is a
13
+ * strict superset that converges onto this key later. Because the key is fixed, a session
14
+ * lock taken here contends correctly with any future acquirer (PostgreSQL advisory locks
15
+ * contend on the key, not on who took it).
16
+ *
17
+ * Why an advisory session lock and not a lease table: it self-releases on disconnect (a
18
+ * crashed CLI, a killed Lambda) with no TTL, no janitor, no stale row; `pg_try_advisory_lock`
19
+ * is atomic try-acquire so refuse-don't-coordinate falls out of the primitive; and the truth
20
+ * lives in `pg_locks`, discoverable from the catalog rather than bookkeeping that can lie.
21
+ */
22
+
23
+ import type { QueryRunner } from './authz-contract.js';
24
+
25
+ /**
26
+ * The fixed lease key: `('ES','TK')` as two 16-bit ints — 0x4553, 0x544B. One key covers
27
+ * every mutation verb (schema applies, reconciles, syncs, restores) so the cross-lane
28
+ * collision the lease exists to prevent (a rebuild during a refresh, a restore during a
29
+ * sync) cannot slip between per-lane keys. Documented, never computed.
30
+ */
31
+ export const MUTATION_LEASE_KEY = { classid: 0x4553, objid: 0x544b } as const;
32
+
33
+ /** PostgreSQL truncates `application_name` to NAMEDATALEN-1 = 63 bytes silently. */
34
+ const APPLICATION_NAME_MAX_BYTES = 63;
35
+
36
+ /**
37
+ * Build the mutation session's identity: `everystack:<verb>:<actor>+<session>`. The
38
+ * `<actor>` alone is not enough — two agents on the same stage this week both run as the
39
+ * same OS user, so a refusal that names only the actor tells the second agent nothing. The
40
+ * `+<session>` discriminator is what makes the refusal actionable: the second agent reads a
41
+ * session id that is not its own and knows it is colliding with a live peer, not staring at
42
+ * its own orphaned backend. Truncated to 63 bytes on a char boundary (PostgreSQL would
43
+ * truncate silently; we do it deterministically so the stored name is knowable).
44
+ */
45
+ export function leaseApplicationName(verb: string, actor: string, session: string): string {
46
+ const full = `everystack:${verb}:${actor}+${session}`;
47
+ return truncateToBytes(full, APPLICATION_NAME_MAX_BYTES);
48
+ }
49
+
50
+ /** Truncate to at most `maxBytes` UTF-8 bytes without splitting a multibyte char. */
51
+ function truncateToBytes(s: string, maxBytes: number): string {
52
+ if (Buffer.byteLength(s, 'utf8') <= maxBytes) return s;
53
+ let out = '';
54
+ let bytes = 0;
55
+ for (const ch of s) {
56
+ const chBytes = Buffer.byteLength(ch, 'utf8');
57
+ if (bytes + chBytes > maxBytes) break;
58
+ out += ch;
59
+ bytes += chBytes;
60
+ }
61
+ return out;
62
+ }
63
+
64
+ /** Single-quote-escape a string for inline SQL (doubles embedded quotes). */
65
+ function sqlQuote(s: string): string {
66
+ return `'${s.replace(/'/g, "''")}'`;
67
+ }
68
+
69
+ /** `SET application_name` for this session, via set_config so the value is quote-safe. */
70
+ export function setApplicationNameSql(name: string): string {
71
+ return `SELECT set_config('application_name', ${sqlQuote(name)}, false)`;
72
+ }
73
+
74
+ /** Try-acquire the session lease. Returns a boolean column `acquired`. Atomic, non-blocking. */
75
+ export const TRY_ACQUIRE_LEASE_SQL =
76
+ `SELECT pg_try_advisory_lock(${MUTATION_LEASE_KEY.classid}, ${MUTATION_LEASE_KEY.objid}) AS acquired`;
77
+
78
+ /** Release the session lease. Idempotent-safe to call in a finally. */
79
+ export const RELEASE_LEASE_SQL =
80
+ `SELECT pg_advisory_unlock(${MUTATION_LEASE_KEY.classid}, ${MUTATION_LEASE_KEY.objid}) AS released`;
81
+
82
+ /**
83
+ * Read the current holder of the lease from the catalog — the truth, not bookkeeping.
84
+ * Joins `pg_locks` to `pg_stat_activity` on the holding backend and returns the database,
85
+ * the holder's `application_name`, its pid, when the backend started, and its state.
86
+ */
87
+ export const HOLDER_SQL = `
88
+ SELECT
89
+ current_database() AS database,
90
+ a.application_name AS holder,
91
+ a.pid AS pid,
92
+ to_char(a.backend_start AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS') || ' UTC' AS since,
93
+ a.state AS state
94
+ FROM pg_locks l
95
+ JOIN pg_stat_activity a ON a.pid = l.pid
96
+ WHERE l.locktype = 'advisory'
97
+ AND l.classid = ${MUTATION_LEASE_KEY.classid}
98
+ AND l.objid = ${MUTATION_LEASE_KEY.objid}
99
+ AND l.granted
100
+ LIMIT 1;
101
+ `.trim();
102
+
103
+ /** The structured holder facts the refusal sentence renders from. Pure, testable. */
104
+ export interface LeaseHolder {
105
+ database: string;
106
+ holder: string | null;
107
+ pid: number | null;
108
+ since: string | null;
109
+ state: string | null;
110
+ }
111
+
112
+ /**
113
+ * Render the refusal sentence from the holder facts. Pure — the impure path assembles a
114
+ * `LeaseHolder` from `HOLDER_SQL` and hands it here, so the exact wording is unit-tested
115
+ * against a fixed row shape.
116
+ *
117
+ * Deliberately does NOT advertise `everystack db:lease`: that break-glass verb is a later
118
+ * brick and does not exist yet. Naming a command the CLI would then reject is the very
119
+ * paper-cut this whole effort is closing — so the hint stays true to what exists today.
120
+ */
121
+ export function renderLeaseRefusal(h: LeaseHolder): string {
122
+ const holder = h.holder && h.holder.length > 0 ? h.holder : 'an unnamed session';
123
+ const parts: string[] = [];
124
+ if (h.pid != null) parts.push(`pid ${h.pid}`);
125
+ if (h.since) parts.push(`since ${h.since}`);
126
+ if (h.state) parts.push(`state: ${h.state}`);
127
+ const where = parts.length ? ` (${parts.join(', ')})` : '';
128
+ return (
129
+ `refused: ${h.database} is being mutated by ${holder}${where}. ` +
130
+ `One mutation at a time — wait for it to finish, or if the holder is dead-but-connected, ` +
131
+ `terminate that backend and retry.`
132
+ );
133
+ }
134
+
135
+ /** Thrown when the lease is held by another session. Carries the holder for callers. */
136
+ export class MutationLeaseError extends Error {
137
+ readonly holder: LeaseHolder;
138
+ constructor(message: string, holder: LeaseHolder) {
139
+ super(message);
140
+ this.name = 'MutationLeaseError';
141
+ this.holder = holder;
142
+ }
143
+ }
144
+
145
+ export interface LeaseIdentity {
146
+ /** The verb holding the lease, e.g. `db:apply`, `db:reconcile`. */
147
+ verb: string;
148
+ /** The operator identity (OS user / STS identity). */
149
+ actor: string;
150
+ /**
151
+ * A short per-invocation discriminator so the two-agents-same-user refusal is actionable.
152
+ * Defaults to a fresh 6-hex-char nonce.
153
+ */
154
+ session?: string;
155
+ }
156
+
157
+ /** A fresh 6-hex-char session discriminator. */
158
+ export function leaseSession(): string {
159
+ // Node's crypto is always present; 3 bytes -> 6 hex chars, ample for the discriminator.
160
+ // eslint-disable-next-line @typescript-eslint/no-var-requires
161
+ const { randomBytes } = require('node:crypto');
162
+ return randomBytes(3).toString('hex');
163
+ }
164
+
165
+ /**
166
+ * Own the lease for the whole of `fn`. Stamps `application_name`, try-acquires the session
167
+ * lock, and on contention throws `MutationLeaseError` naming the holder. Releases in a
168
+ * `finally` — and because the lock is session-scoped, a crash mid-body frees it too.
169
+ *
170
+ * Must run on the mutation's OWN session — the same max:1 connection that performs every
171
+ * write of the verb. A verb that opens a SECOND connection mid-body has a writer the lease
172
+ * does not cover.
173
+ *
174
+ * HONEST LIMITATION (do not overstate): this is CONVENTIONALLY enforced, not runtime-enforced.
175
+ * A PostgreSQL connection carries no read/write intent the connection layer can inspect, so
176
+ * "reject a second connection" is unrealizable in-process. The counterexample is already in the
177
+ * tree: `createUrlPipelineRunner` (db-source.ts) opens a second, independent max:1 session with
178
+ * its own transaction, distinct from the `createUrlRunner` session this lease is handed — a verb
179
+ * that leases on one and writes on the other has a mutating connection the lease does not cover,
180
+ * and nothing rejects it. The only REAL enforcement is Postgres-side: reads connect as a
181
+ * read-only role that physically cannot write (redesign tracked in mutation-lease.md B1). Until
182
+ * that lands, treat this as a code-review convention with a named residual hole, not a guarantee.
183
+ */
184
+ export async function withMutationLease<T>(
185
+ runner: QueryRunner,
186
+ identity: LeaseIdentity,
187
+ fn: () => Promise<T>,
188
+ ): Promise<T> {
189
+ const session = identity.session ?? leaseSession();
190
+ const appName = leaseApplicationName(identity.verb, identity.actor, session);
191
+ await runner(setApplicationNameSql(appName));
192
+
193
+ const rows = await runner(TRY_ACQUIRE_LEASE_SQL);
194
+ const acquired = rows[0]?.acquired === true || rows[0]?.acquired === 't';
195
+ if (!acquired) {
196
+ const holder = await readHolder(runner);
197
+ throw new MutationLeaseError(renderLeaseRefusal(holder), holder);
198
+ }
199
+
200
+ try {
201
+ return await fn();
202
+ } finally {
203
+ // Best-effort release; the session lock also frees on disconnect, so a throw here
204
+ // (e.g. the connection already died) must not mask the body's outcome.
205
+ try {
206
+ await runner(RELEASE_LEASE_SQL);
207
+ } catch {
208
+ /* session gone — the lock died with it */
209
+ }
210
+ }
211
+ }
212
+
213
+ /** Assemble a `LeaseHolder` from `HOLDER_SQL`; a missing row still yields a usable sentence. */
214
+ async function readHolder(runner: QueryRunner): Promise<LeaseHolder> {
215
+ let row: any;
216
+ try {
217
+ row = (await runner(HOLDER_SQL))[0];
218
+ } catch {
219
+ row = undefined;
220
+ }
221
+ return {
222
+ database: row?.database ?? 'the database',
223
+ holder: row?.holder ?? null,
224
+ pid: row?.pid != null ? Number(row.pid) : null,
225
+ since: row?.since ?? null,
226
+ state: row?.state ?? null,
227
+ };
228
+ }
@@ -0,0 +1,78 @@
1
+ /**
2
+ * Will this edge fit the ops-Lambda clock? (stage-write-lanes brick 2.)
3
+ *
4
+ * `db:apply --stage` executes the write inside the ops Lambda, which has a hard 900-second
5
+ * ceiling. A large brownfield edge — 1855 statements with 117 index builds, on a t4g.small —
6
+ * hit that ceiling in the field: the transaction rolled back clean, but the operator burned
7
+ * 15 minutes to learn it wouldn't fit. This estimates fit from the plan itself (at mint time,
8
+ * no live probe) so the ops path can REFUSE up front and name `--direct`, the credential-free
9
+ * escape with an unbounded clock.
10
+ *
11
+ * The heuristic is deliberately CONSERVATIVE, and that is safe by construction: a false
12
+ * "won't fit" routes the operator to `--direct`, which is itself credential-free and strictly
13
+ * more capable than the ops-Lambda path. Over-refusing costs nothing but a flag; under-refusing
14
+ * costs a 15-minute timeout. So we lean toward refusing.
15
+ *
16
+ * The bottleneck is the DATABASE, not the Lambda's memory/CPU — index builds scale with table
17
+ * size — so a raw statement count is only a proxy. Two signals: total executable statements,
18
+ * and the count of non-CONCURRENT index builds (the expensive class that dominated the incident).
19
+ */
20
+
21
+ export interface OpsFitPlan {
22
+ statements?: string[];
23
+ executable?: number;
24
+ }
25
+
26
+ /**
27
+ * Thresholds tuned to the incident (1855 statements / 117 index builds timed out at 900s on a
28
+ * t4g.small) with generous headroom below it. Named so the reasoning is legible and one edit
29
+ * retunes the gate.
30
+ */
31
+ export const OPS_FIT_LIMITS = {
32
+ /** Total executable statements above which the ops-Lambda clock is at risk. */
33
+ maxStatements: 500,
34
+ /** Non-CONCURRENT index builds above which a single edge likely blows the budget. */
35
+ maxIndexBuilds: 30,
36
+ } as const;
37
+
38
+ /** A CREATE INDEX that will hold a lock and scan the table (CONCURRENTLY is the slow-but-online form). */
39
+ function isBlockingIndexBuild(stmt: string): boolean {
40
+ return /^\s*CREATE\s+(UNIQUE\s+)?INDEX\b/i.test(stmt) && !/\bCONCURRENTLY\b/i.test(stmt);
41
+ }
42
+
43
+ export interface OpsFitVerdict {
44
+ fits: boolean;
45
+ /** Present when !fits — the sentence the ops path refuses with, already naming --direct. */
46
+ reason?: string;
47
+ }
48
+
49
+ /**
50
+ * Estimate whether `plan` fits the ops-Lambda runtime. Pure; no database, no clock.
51
+ * `db:apply --stage` (the ops-Lambda path) calls this BEFORE invoking; `--direct` skips it
52
+ * (it has no Lambda ceiling).
53
+ */
54
+ export function estimateOpsRuntimeFit(plan: OpsFitPlan): OpsFitVerdict {
55
+ const statements = plan.statements ?? [];
56
+ const total = plan.executable ?? statements.length;
57
+ const indexBuilds = statements.filter(isBlockingIndexBuild).length;
58
+
59
+ if (total > OPS_FIT_LIMITS.maxStatements) {
60
+ return {
61
+ fits: false,
62
+ reason:
63
+ `this edge is ${total} statements — past the ~${OPS_FIT_LIMITS.maxStatements}-statement ops-Lambda budget ` +
64
+ `(the write runs inside a 900-second Lambda; a large edge rolls back clean but wastes the wait). ` +
65
+ `Re-run with --direct — same ceremony, executed CLI-side with an unbounded clock, still credential-free.`,
66
+ };
67
+ }
68
+ if (indexBuilds > OPS_FIT_LIMITS.maxIndexBuilds) {
69
+ return {
70
+ fits: false,
71
+ reason:
72
+ `this edge builds ${indexBuilds} indexes — index builds scale with table size and dominate the ops-Lambda ` +
73
+ `runtime (past ~${OPS_FIT_LIMITS.maxIndexBuilds} the 900-second ceiling is at risk). ` +
74
+ `Re-run with --direct — same ceremony, executed CLI-side with an unbounded clock, still credential-free.`,
75
+ };
76
+ }
77
+ return { fits: true };
78
+ }