@everystack/cli 0.4.25 → 0.4.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -1
- package/src/backfill.ts +23 -0
- package/src/cli/backup.ts +22 -0
- package/src/cli/commands/db-apply.ts +40 -2
- package/src/cli/commands/db-backfill.ts +104 -4
- package/src/cli/commands/db-export.ts +1 -1
- package/src/cli/commands/db-reconcile.ts +43 -1
- package/src/cli/commands/db-refresh.ts +1 -1
- package/src/cli/commands/db-swap.ts +145 -16
- package/src/cli/commands/db.ts +43 -12
- package/src/cli/db-source.ts +4 -1
- package/src/cli/direct-venue.ts +43 -0
- package/src/cli/discover.ts +21 -7
- package/src/cli/mutation-lease.ts +228 -0
- package/src/cli/ops-fit.ts +78 -0
- package/src/cli/refresh-execute.ts +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@everystack/cli",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.28",
|
|
4
4
|
"description": "CLI and OTA updates for Expo apps on everystack",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"author": "Scalable Technology, Inc. <licensing@scalable.technology>",
|
|
@@ -45,6 +45,10 @@
|
|
|
45
45
|
"types": "./src/refresh.ts",
|
|
46
46
|
"default": "./src/refresh.ts"
|
|
47
47
|
},
|
|
48
|
+
"./backfill": {
|
|
49
|
+
"types": "./src/backfill.ts",
|
|
50
|
+
"default": "./src/backfill.ts"
|
|
51
|
+
},
|
|
48
52
|
"./audit/source": {
|
|
49
53
|
"types": "./src/cli/audit-source-api.ts",
|
|
50
54
|
"default": "./src/cli/audit-source-api.ts"
|
package/src/backfill.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @everystack/cli/backfill — the one-shot data-job core, for the ops-Lambda lane.
|
|
3
|
+
*
|
|
4
|
+
* db:backfill runs committed, content-addressed *.sql jobs against a database and records each
|
|
5
|
+
* in everystack.backfill_log. It was direct-connection only; this barrel lets the ops `db:backfill`
|
|
6
|
+
* action (stage-write-lanes brick 4) load the same core and run it on the operator connection, so
|
|
7
|
+
* `db:backfill --stage --apply` needs no raw admin URL on the operator's machine. The CLI ships the
|
|
8
|
+
* *.sql files; this plans (by content identity), executes pending jobs in order, and records them.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
export {
|
|
12
|
+
planBackfills,
|
|
13
|
+
readBackfillLog,
|
|
14
|
+
executeBackfills,
|
|
15
|
+
markBackfillApplied,
|
|
16
|
+
} from './cli/backfill.js';
|
|
17
|
+
export type {
|
|
18
|
+
BackfillPlan,
|
|
19
|
+
BackfillRecord,
|
|
20
|
+
BackfillRunOptions,
|
|
21
|
+
BackfillRunResult,
|
|
22
|
+
} from './cli/backfill.js';
|
|
23
|
+
export type { SourceFile } from './cli/derived-source.js';
|
package/src/cli/backup.ts
CHANGED
|
@@ -49,6 +49,28 @@ export function keyForId(id: string): string | null {
|
|
|
49
49
|
return ref ? `${backupPrefix(ref.stage)}/${ref.name}.dump.gz` : null;
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
+
// --- Artifact key scheme (schema-scoped exports for the hot-swap) — mirrors server/backup.ts ----
|
|
53
|
+
// A backup is the whole database; an ARTIFACT is one schema, content-addressed, carrying the schema
|
|
54
|
+
// fingerprint it was built against. Separate prefix so the two never collide. db:swap --stage --direct
|
|
55
|
+
// resolves an artifact id (from db:export --stage) to its S3 key, presigns, and downloads it.
|
|
56
|
+
|
|
57
|
+
/** The S3 prefix a schema's artifacts live under (admin-only, encrypted). */
|
|
58
|
+
export function artifactPrefix(schema: string, stage: string): string {
|
|
59
|
+
return `artifacts/${schema}/${stage}`;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Parse a schema-qualified artifact id `stats/dev/20260716T120000Z-ab12cd`, or null when malformed. */
|
|
63
|
+
export function parseArtifactRef(id: string): { schema: string; stage: string; name: string } | null {
|
|
64
|
+
const m = id.match(/^([A-Za-z0-9_]+)\/([A-Za-z0-9_-]+)\/([A-Za-z0-9_.:-]+)$/);
|
|
65
|
+
return m ? { schema: m[1], stage: m[2], name: m[3] } : null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** The S3 dump key for an artifact id (inverse of the server's artifactId), or null when malformed. */
|
|
69
|
+
export function keyForArtifactId(id: string): string | null {
|
|
70
|
+
const ref = parseArtifactRef(id);
|
|
71
|
+
return ref ? `${artifactPrefix(ref.schema, ref.stage)}/${ref.name}.dump.gz` : null;
|
|
72
|
+
}
|
|
73
|
+
|
|
52
74
|
/**
|
|
53
75
|
* Format a Date as the UTC `YYYYMMDDTHHMMSSZ` stamp used in backup keys/ids. The clock is the
|
|
54
76
|
* caller's — pass `new Date()` at the call site so this stays pure and unit-testable.
|
|
@@ -35,6 +35,9 @@ import { resolveDbSource, createUrlRunner, connectingVia, type DbSource } from '
|
|
|
35
35
|
import { resolveConfig, opsFunction } from '../config.js';
|
|
36
36
|
import { invokeAction, lambdaQueryRunner } from '../aws.js';
|
|
37
37
|
import { executeApplyPlan, type ApplyPlanResult } from '../apply-execute.js';
|
|
38
|
+
import { estimateOpsRuntimeFit } from '../ops-fit.js';
|
|
39
|
+
import { resolveOperatorUrlViaStage } from '../direct-venue.js';
|
|
40
|
+
import { withMutationLease, MutationLeaseError } from '../mutation-lease.js';
|
|
38
41
|
import { pgDumpPreflightError } from './db-backup.js';
|
|
39
42
|
import { step, success, fail, info, warn } from '../output.js';
|
|
40
43
|
|
|
@@ -253,9 +256,32 @@ export async function dbApplyCommand(flags: Record<string, string>): Promise<voi
|
|
|
253
256
|
}
|
|
254
257
|
}
|
|
255
258
|
|
|
259
|
+
// `--stage --direct` (lane 1): resolve the stage's OPERATOR connection from its ops
|
|
260
|
+
// Lambda, then execute CLI-side with an unbounded clock — the same ceremony as the
|
|
261
|
+
// direct path below (which is stage-aware: approver check + auto-snapshot when --stage
|
|
262
|
+
// is set), just a longer clock. The operator never holds a URL; it lives in memory only.
|
|
263
|
+
if (dbSource.kind === 'stage' && flags.direct === 'true') {
|
|
264
|
+
try {
|
|
265
|
+
step('Resolving the operator connection from the stage (--direct)...');
|
|
266
|
+
const op = await resolveOperatorUrlViaStage(flags.stage);
|
|
267
|
+
info(`operator credential resolved (${op.source}) — executing CLI-side, unbounded clock.`);
|
|
268
|
+
dbSource = { kind: 'url', url: op.url, from: 'operator' };
|
|
269
|
+
} catch (err: any) {
|
|
270
|
+
fail(err.message);
|
|
271
|
+
process.exit(1);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
256
275
|
// Deployed stage, no direct URL: the write runs in the ops Lambda — the
|
|
257
276
|
// operator never holds a database URL. --database-url stays local-only.
|
|
258
277
|
if (dbSource.kind === 'stage') {
|
|
278
|
+
// Will this edge fit the ops-Lambda's 900-second clock? If not, refuse up front and
|
|
279
|
+
// name --direct (credential-free, unbounded) instead of burning 15 minutes to learn it.
|
|
280
|
+
const fit = estimateOpsRuntimeFit(plan);
|
|
281
|
+
if (!fit.fits) {
|
|
282
|
+
fail(`REFUSED (ops-Lambda runtime): ${fit.reason}`);
|
|
283
|
+
process.exit(1);
|
|
284
|
+
}
|
|
259
285
|
await applyPlanViaStage(plan, flags, forceDescent);
|
|
260
286
|
return;
|
|
261
287
|
}
|
|
@@ -315,7 +341,13 @@ export async function dbApplyCommand(flags: Record<string, string>): Promise<voi
|
|
|
315
341
|
warn(`DESCENT FORCED — the fast-forward rule is bypassed for this apply. Snapshot on record: ${forceDescent}.`);
|
|
316
342
|
}
|
|
317
343
|
step('Verifying the target is where the plan started...');
|
|
318
|
-
|
|
344
|
+
// The mutation lease: one operator mutates a database at a time. Acquired on this apply's
|
|
345
|
+
// own session (the max:1 createUrlRunner) before any write; a second operator is refused
|
|
346
|
+
// by name. Self-releases on disconnect.
|
|
347
|
+
const result = await withMutationLease(
|
|
348
|
+
runner,
|
|
349
|
+
{ verb: 'db:apply', actor: process.env.USER ?? 'unknown' },
|
|
350
|
+
() => executeApplyPlan(runner, plan, {
|
|
319
351
|
actor: process.env.USER ?? null,
|
|
320
352
|
gitRef: currentGitRef() ?? plan.gitRef,
|
|
321
353
|
...(verifyAuthority ? { verifyAuthority } : {}),
|
|
@@ -340,10 +372,16 @@ export async function dbApplyCommand(flags: Record<string, string>): Promise<voi
|
|
|
340
372
|
}
|
|
341
373
|
},
|
|
342
374
|
} : {}),
|
|
343
|
-
|
|
375
|
+
}),
|
|
376
|
+
);
|
|
344
377
|
|
|
345
378
|
reportApplyResult(result, plan);
|
|
346
379
|
} catch (err: any) {
|
|
380
|
+
if (err instanceof MutationLeaseError) {
|
|
381
|
+
// Nothing was applied — the lease refused before any write. Not a rollback.
|
|
382
|
+
fail(err.message);
|
|
383
|
+
process.exit(1);
|
|
384
|
+
}
|
|
347
385
|
fail(`Apply failed and rolled back: ${err.message}`);
|
|
348
386
|
process.exit(1);
|
|
349
387
|
} finally {
|
|
@@ -28,6 +28,10 @@ import {
|
|
|
28
28
|
} from '../backfill.js';
|
|
29
29
|
import { currentGitRef } from '../state-apply.js';
|
|
30
30
|
import { resolveDbSource, createUrlRunner, connectingVia, type DbSource } from '../db-source.js';
|
|
31
|
+
import { resolveOperatorUrlViaStage } from '../direct-venue.js';
|
|
32
|
+
import { withMutationLease, MutationLeaseError } from '../mutation-lease.js';
|
|
33
|
+
import { resolveConfig, opsFunction } from '../config.js';
|
|
34
|
+
import { invokeAction } from '../aws.js';
|
|
31
35
|
import { readSqlDirIfPresent } from './db-sync.js';
|
|
32
36
|
import { step, success, fail, info, warn } from '../output.js';
|
|
33
37
|
|
|
@@ -49,6 +53,71 @@ export function buildBackfillReport(plan: BackfillPlan): string[] {
|
|
|
49
53
|
return lines;
|
|
50
54
|
}
|
|
51
55
|
|
|
56
|
+
/**
|
|
57
|
+
* db:backfill --stage: ship the *.sql files to the ops Lambda's db:backfill action, which
|
|
58
|
+
* plans + (under --apply) runs them on the operator connection and records them. The operator
|
|
59
|
+
* never holds a database URL. --direct is the escape for a backfill too big for the 900s clock.
|
|
60
|
+
*/
|
|
61
|
+
async function runBackfillViaStage(
|
|
62
|
+
files: { file: string; sql: string }[],
|
|
63
|
+
flags: Record<string, string>,
|
|
64
|
+
): Promise<void> {
|
|
65
|
+
step('Resolving deployed config...');
|
|
66
|
+
const config = await resolveConfig(flags.stage);
|
|
67
|
+
|
|
68
|
+
let markApplied: string | undefined;
|
|
69
|
+
if (flags['mark-applied'] !== undefined) {
|
|
70
|
+
if (flags['mark-applied'] === 'true') {
|
|
71
|
+
fail('--mark-applied needs a file name from the backfills dir.');
|
|
72
|
+
process.exit(1);
|
|
73
|
+
}
|
|
74
|
+
markApplied = flags['mark-applied'];
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const payload = {
|
|
78
|
+
files: files.map((f) => ({ file: f.file, sql: f.sql })),
|
|
79
|
+
apply: flags.apply === 'true',
|
|
80
|
+
...(markApplied ? { markApplied } : {}),
|
|
81
|
+
actor: process.env.USER ?? null,
|
|
82
|
+
gitRef: currentGitRef(),
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
step('Running the backfill lane in the ops Lambda (credential-free)...');
|
|
86
|
+
const res: any = await invokeAction(config.region, opsFunction(config), 'db:backfill', payload);
|
|
87
|
+
if (res?.error) {
|
|
88
|
+
fail(`db:backfill failed: ${res.error}`);
|
|
89
|
+
if (/Unknown action/i.test(String(res.error))) {
|
|
90
|
+
info('The deployed handler predates the db:backfill ops action. Upgrade @everystack/server, or run direct: db:backfill --apply --database-url <url> (or --stage --direct).');
|
|
91
|
+
}
|
|
92
|
+
process.exit(1);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
if (res?.markedApplied) {
|
|
96
|
+
success(`Recorded ${res.markedApplied} as applied WITHOUT running it (identity ${String(res.identity).slice(0, 12)}).`);
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if (res?.plan) for (const line of buildBackfillReport(res.plan)) info(line);
|
|
101
|
+
if (flags.json === 'true') console.log(JSON.stringify(res.plan ?? res, null, 2));
|
|
102
|
+
|
|
103
|
+
if (!res?.applied) {
|
|
104
|
+
if (res?.plan?.pending?.length) info('Run them: everystack db:backfill --stage ' + (flags.stage ?? '<stage>') + ' --apply');
|
|
105
|
+
if (res?.plan?.blocked?.length) process.exit(1);
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
for (const ran of res.ran ?? []) info(`~ ${ran.file} applied in ${ran.durationMs}ms`);
|
|
110
|
+
if (res.failed) {
|
|
111
|
+
fail(`${res.failed.file} FAILED and rolled back (recorded; it stays pending for the retry): ${res.failed.error}`);
|
|
112
|
+
process.exit(1);
|
|
113
|
+
}
|
|
114
|
+
if (res?.plan?.blocked?.length) {
|
|
115
|
+
fail('Pending jobs ran, but blocked file(s) remain above — fix them (new file name, or --mark-applied).');
|
|
116
|
+
process.exit(1);
|
|
117
|
+
}
|
|
118
|
+
success(`${(res.ran ?? []).length} backfill(s) applied and recorded in everystack.backfill_log.`);
|
|
119
|
+
}
|
|
120
|
+
|
|
52
121
|
export async function dbBackfillCommand(flags: Record<string, string>): Promise<void> {
|
|
53
122
|
const dir = flags.dir || DEFAULT_DIR;
|
|
54
123
|
const files = await readSqlDirIfPresent(dir);
|
|
@@ -64,9 +133,28 @@ export async function dbBackfillCommand(flags: Record<string, string>): Promise<
|
|
|
64
133
|
fail(err.message);
|
|
65
134
|
process.exit(1);
|
|
66
135
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
136
|
+
|
|
137
|
+
// `--stage --direct` (lane 1): a backfill too big for the ops-Lambda's 900-second clock runs
|
|
138
|
+
// CLI-side with an unbounded clock. Resolve the stage's operator connection and fall through
|
|
139
|
+
// to the direct path — credential-free, the operator never holds a URL.
|
|
140
|
+
if (dbSource.kind === 'stage' && flags.direct === 'true') {
|
|
141
|
+
try {
|
|
142
|
+
step('Resolving the operator connection from the stage (--direct)...');
|
|
143
|
+
const op = await resolveOperatorUrlViaStage(flags.stage);
|
|
144
|
+
info(`operator credential resolved (${op.source}) — running backfills CLI-side, unbounded clock.`);
|
|
145
|
+
dbSource = { kind: 'url', url: op.url, from: 'operator' };
|
|
146
|
+
} catch (err: any) {
|
|
147
|
+
fail(err.message);
|
|
148
|
+
process.exit(1);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// `--stage` (ops path): ship the *.sql files to the ops Lambda's db:backfill action — it plans
|
|
153
|
+
// (by content identity, against everystack.backfill_log) and, under --apply, runs the pending
|
|
154
|
+
// jobs on the operator connection and records them. The operator holds no URL.
|
|
155
|
+
if (dbSource.kind === 'stage') {
|
|
156
|
+
await runBackfillViaStage(files, flags);
|
|
157
|
+
return;
|
|
70
158
|
}
|
|
71
159
|
|
|
72
160
|
step(connectingVia(dbSource));
|
|
@@ -110,7 +198,12 @@ export async function dbBackfillCommand(flags: Record<string, string>): Promise<
|
|
|
110
198
|
}
|
|
111
199
|
|
|
112
200
|
step(`Running ${plan.pending.length} backfill(s), each as its own transaction...`);
|
|
113
|
-
|
|
201
|
+
// A data-write operator mutation takes the lease so a concurrent operator is refused.
|
|
202
|
+
const result = await withMutationLease(
|
|
203
|
+
runner,
|
|
204
|
+
{ verb: 'db:backfill', actor: process.env.USER ?? 'unknown' },
|
|
205
|
+
() => executeBackfills(runner, plan, opts),
|
|
206
|
+
);
|
|
114
207
|
for (const ran of result.ran) info(`~ ${ran.file} applied in ${ran.durationMs}ms`);
|
|
115
208
|
if (result.failed) {
|
|
116
209
|
fail(`${result.failed.file} FAILED and rolled back (recorded; it stays pending for the retry): ${result.failed.error}`);
|
|
@@ -121,6 +214,13 @@ export async function dbBackfillCommand(flags: Record<string, string>): Promise<
|
|
|
121
214
|
process.exit(1);
|
|
122
215
|
}
|
|
123
216
|
success(`${result.ran.length} backfill(s) applied and recorded in everystack.backfill_log.`);
|
|
217
|
+
} catch (err: any) {
|
|
218
|
+
if (err instanceof MutationLeaseError) {
|
|
219
|
+
// The lease refused before any job ran — nothing was written.
|
|
220
|
+
fail(err.message);
|
|
221
|
+
process.exit(1);
|
|
222
|
+
}
|
|
223
|
+
throw err;
|
|
124
224
|
} finally {
|
|
125
225
|
await end?.();
|
|
126
226
|
}
|
|
@@ -215,5 +215,5 @@ export async function dbExportCommand(flags: Record<string, string>): Promise<vo
|
|
|
215
215
|
fail(`Export failed: ${result.error}`);
|
|
216
216
|
process.exit(1);
|
|
217
217
|
}
|
|
218
|
-
success(`Artifact ${result.id} (${fmtBytes(result.bytes)}, fingerprint ${String(result.fingerprint).slice(0, 12)}). Deploy with: everystack db:swap --schema ${schema} --stage <target
|
|
218
|
+
success(`Artifact ${result.id} (${fmtBytes(result.bytes)}, fingerprint ${String(result.fingerprint).slice(0, 12)}). Deploy with: everystack db:swap --schema ${schema} --from ${result.id} --stage <target> --direct --confirm`);
|
|
219
219
|
}
|
|
@@ -46,6 +46,8 @@ import {
|
|
|
46
46
|
ENSURE_RECONCILER_SQL,
|
|
47
47
|
} from '../derived-apply.js';
|
|
48
48
|
import { resolveDbSource, createUrlRunner, connectingVia, type DbSource } from '../db-source.js';
|
|
49
|
+
import { resolveOperatorUrlViaStage } from '../direct-venue.js';
|
|
50
|
+
import { withMutationLease, MutationLeaseError } from '../mutation-lease.js';
|
|
49
51
|
import { loadDeclaredDerived, retiredSqlDirAnywhere, retiredSqlDirFlagRefusal, type DeclaredDerived } from '../declared-derived.js';
|
|
50
52
|
import { currentGitRef } from '../state-apply.js';
|
|
51
53
|
import { resolveConfig, opsFunction } from '../config.js';
|
|
@@ -380,6 +382,27 @@ export async function dbReconcileCommand(flags: Record<string, string>): Promise
|
|
|
380
382
|
process.exit(1);
|
|
381
383
|
}
|
|
382
384
|
|
|
385
|
+
// `--stage --direct` (lane 1): a 54-object derived rebuild runs 15-25 minutes and blows
|
|
386
|
+
// the ops-Lambda's 900-second clock. Resolve the stage's OPERATOR connection from its ops
|
|
387
|
+
// Lambda and execute CLI-side with an unbounded clock — credential-free, the operator never
|
|
388
|
+
// holds a URL. Only meaningful under --apply (a dry-run just reads). Fall through as a url
|
|
389
|
+
// source so the local-runner branch below runs executeReconcile against it, under the lease.
|
|
390
|
+
if (dbSource.kind === 'stage' && flags.direct === 'true') {
|
|
391
|
+
if (!apply) {
|
|
392
|
+
fail('--direct is a write venue — it only applies with --apply. For a dry-run drop --direct (the stage plans read-only via the ops Lambda).');
|
|
393
|
+
process.exit(1);
|
|
394
|
+
}
|
|
395
|
+
try {
|
|
396
|
+
step('Resolving the operator connection from the stage (--direct)...');
|
|
397
|
+
const op = await resolveOperatorUrlViaStage(flags.stage);
|
|
398
|
+
info(`operator credential resolved (${op.source}) — reconciling CLI-side, unbounded clock.`);
|
|
399
|
+
dbSource = { kind: 'url', url: op.url, from: 'operator' };
|
|
400
|
+
} catch (err: any) {
|
|
401
|
+
fail(err.message);
|
|
402
|
+
process.exit(1);
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
383
406
|
// --baseline only writes under --apply (executeReconcile returns the plan and executes nothing
|
|
384
407
|
// when apply=false). Passing it alone used to print "recording provenance…" and exit 0 while
|
|
385
408
|
// persisting nothing — a silent no-op. Fail loudly instead; the plan already lists needsBaseline.
|
|
@@ -459,21 +482,33 @@ export async function dbReconcileCommand(flags: Record<string, string>): Promise
|
|
|
459
482
|
fail(`db:reconcile failed: ${result.error}`);
|
|
460
483
|
if (/Unknown action/i.test(String(result.error))) {
|
|
461
484
|
info('The deployed handler predates the db:reconcile ops action (needs @everystack/server >= 0.4.8). Upgrade the server, or apply direct: db:reconcile --apply --database-url <url>.');
|
|
485
|
+
} else if (/timed out|timeout|task timed out/i.test(String(result.error))) {
|
|
486
|
+
info('A large derived rebuild (dozens of objects) can exceed the ops-Lambda 900-second clock. Re-run credential-free with an unbounded clock: db:reconcile --apply --stage ' + (flags.stage ?? '<stage>') + ' --direct.');
|
|
462
487
|
}
|
|
463
488
|
process.exit(1);
|
|
464
489
|
}
|
|
465
490
|
run = { plan: result.plan, applied: result.applied, statements: result.statements ?? [], refusal: result.refusal ?? undefined };
|
|
466
491
|
} else {
|
|
467
492
|
let runner: QueryRunner;
|
|
493
|
+
// A write over a direct connection (--database-url or --direct) takes the mutation
|
|
494
|
+
// lease; a read-only stage dry-run over the ops Lambda does not (reads never lease).
|
|
495
|
+
let leased = false;
|
|
468
496
|
if (dbSource.kind === 'url') {
|
|
469
497
|
step(connectingVia(dbSource));
|
|
470
498
|
({ runner, end } = await createUrlRunner(dbSource.url));
|
|
499
|
+
leased = apply;
|
|
471
500
|
} else {
|
|
472
501
|
step('Resolving deployed config...');
|
|
473
502
|
const config = await resolveConfig(flags.stage);
|
|
474
503
|
runner = lambdaRunner(config.region, opsFunction(config));
|
|
475
504
|
}
|
|
476
|
-
run =
|
|
505
|
+
run = leased
|
|
506
|
+
? await withMutationLease(
|
|
507
|
+
runner,
|
|
508
|
+
{ verb: 'db:reconcile', actor: process.env.USER ?? 'unknown' },
|
|
509
|
+
() => executeReconcile(runner, reconcileOptions),
|
|
510
|
+
)
|
|
511
|
+
: await executeReconcile(runner, reconcileOptions);
|
|
477
512
|
}
|
|
478
513
|
|
|
479
514
|
if (flags.json === 'true') {
|
|
@@ -499,6 +534,13 @@ export async function dbReconcileCommand(flags: Record<string, string>): Promise
|
|
|
499
534
|
|
|
500
535
|
if (run.refusal) process.exit(1);
|
|
501
536
|
if (check && checkFails(run.plan)) process.exit(1);
|
|
537
|
+
} catch (err: any) {
|
|
538
|
+
if (err instanceof MutationLeaseError) {
|
|
539
|
+
// The lease refused before any DDL — nothing was reconciled.
|
|
540
|
+
fail(err.message);
|
|
541
|
+
process.exit(1);
|
|
542
|
+
}
|
|
543
|
+
throw err;
|
|
502
544
|
} finally {
|
|
503
545
|
await end?.();
|
|
504
546
|
}
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* (the `db:refresh` action), so a least-privileged operator never holds a URL. `--database-url`
|
|
11
11
|
* / `--direct` refresh over a direct connection (dev). `--list` previews the topo-ordered
|
|
12
12
|
* matviews without connecting. Plain REFRESH (ACCESS EXCLUSIVE) — CONCURRENTLY is a follow-on.
|
|
13
|
-
* (
|
|
13
|
+
* (consumer field report 2026-07-19.)
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
16
|
import { resolveDbSource, createUrlRunner, connectingVia, type DbSource } from '../db-source.js';
|
|
@@ -14,9 +14,13 @@
|
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
16
|
import fs from 'node:fs';
|
|
17
|
+
import os from 'node:os';
|
|
18
|
+
import path from 'node:path';
|
|
19
|
+
import { createGunzip } from 'node:zlib';
|
|
17
20
|
import { spawn } from 'node:child_process';
|
|
18
21
|
import { Transform } from 'node:stream';
|
|
19
22
|
import { pipeline } from 'node:stream/promises';
|
|
23
|
+
import { Readable } from 'node:stream';
|
|
20
24
|
import type { ModelDescriptor } from '@everystack/model';
|
|
21
25
|
import { fingerprintModels } from '../schema-fingerprint.js';
|
|
22
26
|
import { resolveModelsPath } from '../models-path.js';
|
|
@@ -25,7 +29,12 @@ import { loadDeclaredDerived } from '../declared-derived.js';
|
|
|
25
29
|
import { createUrlRunner } from '../db-source.js';
|
|
26
30
|
import { executeSwap, type SwapVerdict } from '../swap-execute.js';
|
|
27
31
|
import { rewriteStatementLine, opensCopyData, closesCopyData } from '../schema-rewrite.js';
|
|
28
|
-
import {
|
|
32
|
+
import { resolveOperatorUrlViaStage } from '../direct-venue.js';
|
|
33
|
+
import { withMutationLease, MutationLeaseError } from '../mutation-lease.js';
|
|
34
|
+
import { resolveConfig, opsFunction } from '../config.js';
|
|
35
|
+
import { invokeAction, presignGet } from '../aws.js';
|
|
36
|
+
import { keyForArtifactId, metaKey } from '../backup.js';
|
|
37
|
+
import { step, success, fail, warn, info } from '../output.js';
|
|
29
38
|
|
|
30
39
|
/** A COPY-aware line transform that rewrites the schema token on statement lines only. */
|
|
31
40
|
function schemaRewriteStream(from: string, to: string): Transform {
|
|
@@ -100,14 +109,115 @@ export function readArtifactFingerprint(artifactPath: string, flag?: string): st
|
|
|
100
109
|
}
|
|
101
110
|
}
|
|
102
111
|
|
|
112
|
+
/** A resolved artifact ready for restore: a local plain `-Fc` .dump plus its stamped fingerprint. */
|
|
113
|
+
interface ResolvedArtifact {
|
|
114
|
+
dumpPath: string;
|
|
115
|
+
fingerprint: string | null;
|
|
116
|
+
/** Remove any temp files fetched from S3 (no-op for a local artifact). */
|
|
117
|
+
cleanup: () => Promise<void>;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Fetch a schema-export artifact from S3 to a local temp `-Fc` .dump for the --direct swap.
|
|
122
|
+
* db:export --stage stores a gzipped -Fc archive + a sibling .meta.json (fingerprint). This
|
|
123
|
+
* presigns both with the CLI's own IAM, streams the dump down, gunzips it (pg_restore reads a
|
|
124
|
+
* plain archive), and reads the stamped fingerprint from the meta. The operator holds no DB
|
|
125
|
+
* credential; the artifact just transits the caller's presigned S3 read.
|
|
126
|
+
*/
|
|
127
|
+
async function fetchArtifactFromS3(
|
|
128
|
+
id: string,
|
|
129
|
+
stage: string | undefined,
|
|
130
|
+
fingerprintFlag: string | undefined,
|
|
131
|
+
): Promise<ResolvedArtifact> {
|
|
132
|
+
const key = keyForArtifactId(id);
|
|
133
|
+
if (!key) throw new Error(`--from is neither a local file nor a valid artifact id (expected schema/stage/stamp, got ${id}).`);
|
|
134
|
+
|
|
135
|
+
const config = await resolveConfig(stage);
|
|
136
|
+
if (!config.backupsBucket) {
|
|
137
|
+
throw new Error('No backupsBucket in the deployed config — cannot fetch the S3 artifact. Add `backupsBucket` to the sst.config outputs and redeploy, or pass a local --from <file.dump>.');
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'everystack-swap-'));
|
|
141
|
+
const dumpPath = path.join(tmpDir, 'artifact.dump');
|
|
142
|
+
const cleanup = async () => { await fs.promises.rm(tmpDir, { recursive: true, force: true }).catch(() => {}); };
|
|
143
|
+
|
|
144
|
+
try {
|
|
145
|
+
step(`Fetching artifact ${id} from S3 (presigned, then gunzip)...`);
|
|
146
|
+
const dumpUrl = await presignGet(config.region, config.backupsBucket, key, 3600);
|
|
147
|
+
const res = await fetch(dumpUrl);
|
|
148
|
+
if (!res.ok || !res.body) throw new Error(`artifact download failed: HTTP ${res.status}`);
|
|
149
|
+
await pipeline(Readable.fromWeb(res.body as any), createGunzip(), fs.createWriteStream(dumpPath));
|
|
150
|
+
|
|
151
|
+
// The fingerprint: an explicit flag wins; otherwise read the sibling .meta.json.
|
|
152
|
+
let fingerprint = fingerprintFlag ?? null;
|
|
153
|
+
if (!fingerprint) {
|
|
154
|
+
try {
|
|
155
|
+
const metaUrl = await presignGet(config.region, config.backupsBucket, metaKey(key), 3600);
|
|
156
|
+
const metaRes = await fetch(metaUrl);
|
|
157
|
+
if (metaRes.ok) fingerprint = (await metaRes.json() as any)?.fingerprint ?? null;
|
|
158
|
+
} catch { /* fall through — the caller reports a missing fingerprint */ }
|
|
159
|
+
}
|
|
160
|
+
return { dumpPath, fingerprint, cleanup };
|
|
161
|
+
} catch (err) {
|
|
162
|
+
await cleanup();
|
|
163
|
+
throw err;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/** Resolve --from to a local plain -Fc dump: a local file as-is, else an S3 artifact id fetched down. */
|
|
168
|
+
async function resolveSwapArtifact(
|
|
169
|
+
from: string,
|
|
170
|
+
stage: string | undefined,
|
|
171
|
+
fingerprintFlag: string | undefined,
|
|
172
|
+
): Promise<ResolvedArtifact> {
|
|
173
|
+
if (fs.existsSync(from)) {
|
|
174
|
+
return { dumpPath: from, fingerprint: readArtifactFingerprint(from, fingerprintFlag), cleanup: async () => {} };
|
|
175
|
+
}
|
|
176
|
+
return fetchArtifactFromS3(from, stage, fingerprintFlag);
|
|
177
|
+
}
|
|
178
|
+
|
|
103
179
|
export async function dbSwapCommand(flags: Record<string, string>): Promise<void> {
|
|
104
180
|
const schema = flags.schema;
|
|
105
|
-
const url = flags['database-url'] || process.env.DATABASE_URL;
|
|
106
181
|
const from = flags.from;
|
|
182
|
+
const stage = flags.stage;
|
|
183
|
+
const direct = flags.direct === 'true';
|
|
107
184
|
if (!schema) { fail('db:swap needs --schema <name>.'); process.exit(1); }
|
|
108
|
-
if (!from) { fail('db:swap needs --from <artifact.dump> (the schema-scoped -Fc archive to land).'); process.exit(1); }
|
|
185
|
+
if (!from) { fail('db:swap needs --from <artifact.dump | artifact-id> (the schema-scoped -Fc archive to land).'); process.exit(1); }
|
|
186
|
+
|
|
187
|
+
// Resolve the venue.
|
|
188
|
+
// - --database-url (or DATABASE_URL): a local/direct operator connection.
|
|
189
|
+
// - --stage --direct: resolve the stage's OPERATOR connection from its ops Lambda and execute
|
|
190
|
+
// CLI-side with an unbounded clock (a multi-GB restore blows the 900s Lambda ceiling). The
|
|
191
|
+
// operator never holds a URL; the swap snapshots the stage via db:backup before it lands.
|
|
192
|
+
// - --stage alone: refuse, naming --direct — the ops-Lambda venue can't hold the restore clock.
|
|
193
|
+
let url = flags['database-url'] || process.env.DATABASE_URL;
|
|
194
|
+
let snapshotViaStage = false;
|
|
195
|
+
let region: string | undefined;
|
|
196
|
+
let opsFn: string | undefined;
|
|
197
|
+
|
|
198
|
+
if (!url && stage) {
|
|
199
|
+
if (!direct) {
|
|
200
|
+
fail('db:swap --stage needs --direct: a schema restore can exceed the ops-Lambda 900-second clock, so the swap runs CLI-side with an unbounded clock (credential-free — the operator never holds a URL). Re-run with --stage ' + stage + ' --direct.');
|
|
201
|
+
process.exit(1);
|
|
202
|
+
}
|
|
203
|
+
if (flags.confirm !== 'true') {
|
|
204
|
+
fail('db:swap --stage --direct is destructive (it drops the retiring schema after the swap). Confirm explicitly: --confirm.');
|
|
205
|
+
process.exit(1);
|
|
206
|
+
}
|
|
207
|
+
try {
|
|
208
|
+
const config = await resolveConfig(stage);
|
|
209
|
+
region = config.region;
|
|
210
|
+
opsFn = opsFunction(config);
|
|
211
|
+
step('Resolving the operator connection from the stage (--direct)...');
|
|
212
|
+
const op = await resolveOperatorUrlViaStage(stage);
|
|
213
|
+
url = op.url;
|
|
214
|
+
snapshotViaStage = true;
|
|
215
|
+
info(`operator credential resolved (${op.source}) — swapping CLI-side, unbounded clock.`);
|
|
216
|
+
} catch (err: any) { fail(err.message); process.exit(1); }
|
|
217
|
+
}
|
|
218
|
+
|
|
109
219
|
if (!url) {
|
|
110
|
-
fail('db:swap
|
|
220
|
+
fail('db:swap needs a target: --database-url <url> (local/direct), or --stage <name> --direct (credential-free — the operator never holds a URL).');
|
|
111
221
|
process.exit(1);
|
|
112
222
|
}
|
|
113
223
|
|
|
@@ -121,24 +231,41 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
|
|
|
121
231
|
declaredFingerprint = fingerprintModels(models, { schemas: [schema], sequences: declaredDb?.sequences }).hash;
|
|
122
232
|
} catch (err: any) { fail(err.message); process.exit(1); }
|
|
123
233
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
234
|
+
// Resolve --from to a local plain -Fc dump (a local file, or an S3 export id fetched down).
|
|
235
|
+
let artifact: ResolvedArtifact;
|
|
236
|
+
try {
|
|
237
|
+
artifact = await resolveSwapArtifact(from, stage, flags.fingerprint);
|
|
238
|
+
} catch (err: any) { fail(err.message); process.exit(1); }
|
|
239
|
+
|
|
240
|
+
if (!artifact.fingerprint) {
|
|
241
|
+
await artifact.cleanup();
|
|
242
|
+
fail(`db:swap can't find the artifact fingerprint — expected a sibling .meta.json next to ${from} (or the artifact's S3 meta), or pass --fingerprint. Without it the gate can't run, and an unchecked swap is exactly what the gate prevents.`);
|
|
127
243
|
process.exit(1);
|
|
128
244
|
}
|
|
245
|
+
const artifactFingerprint = artifact.fingerprint;
|
|
129
246
|
|
|
130
247
|
const { runner, end } = await createUrlRunner(url);
|
|
131
248
|
try {
|
|
132
249
|
step(`Swapping ${schema} — gate, land incoming, atomic swap, verify...`);
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
250
|
+
// One operator mutates a database at a time — the swap is a whole-schema replacement.
|
|
251
|
+
const res = await withMutationLease(
|
|
252
|
+
runner,
|
|
253
|
+
{ verb: 'db:swap', actor: process.env.USER ?? 'unknown' },
|
|
254
|
+
() => executeSwap(runner, {
|
|
255
|
+
models, schema,
|
|
256
|
+
artifactFingerprint,
|
|
257
|
+
declaredFingerprint,
|
|
258
|
+
applyIncoming: async () => { await restoreIntoIncoming(url!, artifact.dumpPath, schema, `${schema}_incoming`); },
|
|
259
|
+
snapshot: snapshotViaStage
|
|
260
|
+
? async () => {
|
|
261
|
+
step('Snapshotting the stage before the swap (db:backup)...');
|
|
262
|
+
const r: any = await invokeAction(region!, opsFn!, 'db:backup', { stage });
|
|
263
|
+
if (r?.error) throw new Error(`pre-swap snapshot failed, so the swap was NOT applied: ${r.error}`);
|
|
264
|
+
info(`snapshot on record: ${r?.id ?? 'backup complete'} — restore with db:restore --from ${r?.id ?? '<id>'} --confirm.`);
|
|
265
|
+
}
|
|
266
|
+
: async () => { warn('no snapshot taken (direct v1) — take one first: everystack db:backup --database-url … before a production swap.'); },
|
|
267
|
+
}),
|
|
268
|
+
);
|
|
142
269
|
|
|
143
270
|
if (res.status === 'swapped') {
|
|
144
271
|
success(`Swapped ${schema} — the artifact is live (no refresh ran).`);
|
|
@@ -148,9 +275,11 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
|
|
|
148
275
|
process.exit(1);
|
|
149
276
|
}
|
|
150
277
|
} catch (err: any) {
|
|
278
|
+
if (err instanceof MutationLeaseError) { fail(err.message); process.exit(1); }
|
|
151
279
|
fail(`Swap failed: ${err.message}`);
|
|
152
280
|
process.exit(1);
|
|
153
281
|
} finally {
|
|
154
282
|
await end?.();
|
|
283
|
+
await artifact.cleanup();
|
|
155
284
|
}
|
|
156
285
|
}
|
package/src/cli/commands/db.ts
CHANGED
|
@@ -363,7 +363,7 @@ export function declaredSearchPath(schemas: string[]): string[] {
|
|
|
363
363
|
* descriptors, NOT the barrel, so a table-only scan structurally misses it — and the
|
|
364
364
|
* authenticator flip then goes dark on every bare-ref matview read ("relation does not
|
|
365
365
|
* exist"). Baking the union into DATABASE_URL's search_path keeps those reads resolving.
|
|
366
|
-
* (
|
|
366
|
+
* (consumer field report 2026-07-19: 54 matviews in `stats_view`, absent from the table-only bake.)
|
|
367
367
|
*/
|
|
368
368
|
export function collectDeclaredSchemas(args: {
|
|
369
369
|
models: Array<{ schema?: string }>;
|
|
@@ -379,7 +379,7 @@ export function collectDeclaredSchemas(args: {
|
|
|
379
379
|
* defaults to `"$user",public`, so every bare-ref query 404s until a hand ALTER ROLE pin
|
|
380
380
|
* (recorded nowhere, repeated every stage) — writing it into the minted DATABASE_URL is
|
|
381
381
|
* declarative and survives redeploys. postgres.js forwards unknown URL params as connection
|
|
382
|
-
* startup params (
|
|
382
|
+
* startup params (a consumer proved search_path this way). No-op for an all-public app.
|
|
383
383
|
*/
|
|
384
384
|
export function withSearchPath(url: string, schemas: string[]): string {
|
|
385
385
|
const path = declaredSearchPath(schemas);
|
|
@@ -389,6 +389,31 @@ export function withSearchPath(url: string, schemas: string[]): string {
|
|
|
389
389
|
return u.toString();
|
|
390
390
|
}
|
|
391
391
|
|
|
392
|
+
/**
|
|
393
|
+
* Carry the CONNECTION params (sslmode, connect_timeout, …) from the URL the app already connects
|
|
394
|
+
* with onto a freshly minted one. provision mints the new URL from host/port/database components,
|
|
395
|
+
* which silently drops any `?sslmode=require` on the source — on an `rds.force_ssl=1` instance the
|
|
396
|
+
* new authenticator/migrator roles then can't connect. `search_path` is provision's OWN concern
|
|
397
|
+
* (withSearchPath owns it), so it is never copied; existing params on the minted URL win. No-op
|
|
398
|
+
* when there is no source or it doesn't parse.
|
|
399
|
+
*/
|
|
400
|
+
export function preserveConnParams(mintedUrl: string, sourceUrl?: string): string {
|
|
401
|
+
if (!sourceUrl) return mintedUrl;
|
|
402
|
+
let src: URL;
|
|
403
|
+
let out: URL;
|
|
404
|
+
try {
|
|
405
|
+
src = new URL(sourceUrl);
|
|
406
|
+
out = new URL(mintedUrl);
|
|
407
|
+
} catch {
|
|
408
|
+
return mintedUrl;
|
|
409
|
+
}
|
|
410
|
+
for (const [key, value] of src.searchParams) {
|
|
411
|
+
if (key === 'search_path') continue; // provision derives search_path from the declared schemas
|
|
412
|
+
if (!out.searchParams.has(key)) out.searchParams.set(key, value);
|
|
413
|
+
}
|
|
414
|
+
return out.toString();
|
|
415
|
+
}
|
|
416
|
+
|
|
392
417
|
/** host/port/database from a postgres URL — the direct venue's connection info. */
|
|
393
418
|
export function parseUrlConnection(url: string): { host: string; port: string; database: string } {
|
|
394
419
|
const u = new URL(url);
|
|
@@ -575,16 +600,22 @@ export async function dbProvisionCommand(flags: Record<string, string>): Promise
|
|
|
575
600
|
const searchPath = declaredSearchPath(declaredSchemas);
|
|
576
601
|
if (searchPath.length) info(` search_path baked into DATABASE_URL: ${searchPath.join(', ')}`);
|
|
577
602
|
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
)
|
|
582
|
-
const
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
603
|
+
// Preserve the connection params (sslmode, …) from the URL the app already connects with — the
|
|
604
|
+
// minted URL is built from bare host/port/db components, so without this a `?sslmode=require`
|
|
605
|
+
// would be dropped and the new roles could fail to connect on an SSL-forced RDS. Prefer the
|
|
606
|
+
// venue URL provision connected through, else the existing DATABASE_URL secret (either spelling).
|
|
607
|
+
const sourceUrl = directUrl
|
|
608
|
+
?? stageSecrets.ADMIN_DATABASE_URL ?? stageSecrets.AdminDatabaseUrl
|
|
609
|
+
?? stageSecrets.DATABASE_URL ?? stageSecrets.DatabaseUrl;
|
|
610
|
+
|
|
611
|
+
const mint = (role: string, password: string): string =>
|
|
612
|
+
withSearchPath(
|
|
613
|
+
preserveConnParams(`postgresql://${role}:${password}@${conn.host}:${conn.port ?? 5432}/${conn.database}`, sourceUrl),
|
|
614
|
+
declaredSchemas,
|
|
615
|
+
);
|
|
616
|
+
|
|
617
|
+
const authUrl = mint(result.loginRole, authPassword);
|
|
618
|
+
const adminUrl = result.adminRole ? mint(result.adminRole, adminPassword) : undefined;
|
|
588
619
|
|
|
589
620
|
// Write the secrets directly to the SST secret store — never display them. One blob
|
|
590
621
|
// write updates every key together, so there is no window where the API side is the
|
package/src/cli/db-source.ts
CHANGED
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
import type { QueryRunner } from './authz-contract.js';
|
|
28
28
|
|
|
29
29
|
export type DbSource =
|
|
30
|
-
| { kind: 'url'; url: string; from: 'flag' | 'env' | 'admin-env' }
|
|
30
|
+
| { kind: 'url'; url: string; from: 'flag' | 'env' | 'admin-env' | 'operator' }
|
|
31
31
|
| { kind: 'stage' };
|
|
32
32
|
|
|
33
33
|
/** Decide which path serves this invocation. Pure; env injectable for tests. */
|
|
@@ -57,6 +57,9 @@ export function resolveDbSource(
|
|
|
57
57
|
export function connectingVia(source: Extract<DbSource, { kind: 'url' }>): string {
|
|
58
58
|
if (source.from === 'flag') return 'Connecting via --database-url...';
|
|
59
59
|
if (source.from === 'admin-env') return 'Connecting via ADMIN_DATABASE_URL...';
|
|
60
|
+
if (source.from === 'operator') {
|
|
61
|
+
return 'Connecting --direct (operator credential resolved from the stage\'s ops Lambda, held in memory only)...';
|
|
62
|
+
}
|
|
60
63
|
return 'Connecting via DATABASE_URL...';
|
|
61
64
|
}
|
|
62
65
|
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `--direct` venue (stage-write-lanes lane 1).
|
|
3
|
+
*
|
|
4
|
+
* A write verb run with `--stage <name> --direct` resolves the OPERATOR connection from the
|
|
5
|
+
* stage's IAM-gated ops Lambda (the `db:operator-url` action, brick 1), holds the URL in
|
|
6
|
+
* process memory only, and executes CLI-side with an unbounded clock. The ceremony is
|
|
7
|
+
* untouched — only the execution venue moves. Shared by db:apply, db:reconcile, db:backfill,
|
|
8
|
+
* and db:swap so every lane resolves the operator credential the same way.
|
|
9
|
+
*
|
|
10
|
+
* The URL travels only in the Lambda invoke response and this process's memory: never printed,
|
|
11
|
+
* never written, never exported. Callers pass it straight to createUrlRunner and drop it.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { resolveConfig, opsFunction } from './config.js';
|
|
15
|
+
import { invokeAction } from './aws.js';
|
|
16
|
+
|
|
17
|
+
export interface OperatorConnection {
|
|
18
|
+
/** The operator connection string — hold in memory, never print. */
|
|
19
|
+
url: string;
|
|
20
|
+
/** Which credential the stage resolved: 'admin' (migrator) or 'master'. */
|
|
21
|
+
source: string;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Resolve a stage's operator connection for the `--direct` lane via its ops Lambda.
|
|
26
|
+
* Throws a remediation-named error when no operator connection exists or the ops Lambda
|
|
27
|
+
* predates this action (an older server that does not know `db:operator-url`).
|
|
28
|
+
*/
|
|
29
|
+
export async function resolveOperatorUrlViaStage(
|
|
30
|
+
stage: string | undefined,
|
|
31
|
+
invoke: typeof invokeAction = invokeAction,
|
|
32
|
+
): Promise<OperatorConnection> {
|
|
33
|
+
const config = await resolveConfig(stage);
|
|
34
|
+
const fn = opsFunction(config);
|
|
35
|
+
const res: any = await invoke(config.region, fn, 'db:operator-url', {});
|
|
36
|
+
if (res?.error) throw new Error(res.error);
|
|
37
|
+
if (!res?.url) {
|
|
38
|
+
throw new Error(
|
|
39
|
+
'the ops Lambda did not return an operator connection for --direct. Deploy a server build that ships the db:operator-url action, or use --database-url for a local connection.',
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
return { url: res.url, source: res.source ?? 'operator' };
|
|
43
|
+
}
|
package/src/cli/discover.ts
CHANGED
|
@@ -26,6 +26,7 @@ interface DiscoveredConfig {
|
|
|
26
26
|
updatesBucket: string;
|
|
27
27
|
clientBundlesBucket: string;
|
|
28
28
|
mediaBucket?: string;
|
|
29
|
+
backupsBucket?: string;
|
|
29
30
|
kvsArn?: string;
|
|
30
31
|
distributionId?: string;
|
|
31
32
|
}
|
|
@@ -209,6 +210,7 @@ interface BucketsResult {
|
|
|
209
210
|
updatesBucket?: string;
|
|
210
211
|
clientBundlesBucket?: string;
|
|
211
212
|
mediaBucket?: string;
|
|
213
|
+
backupsBucket?: string;
|
|
212
214
|
}
|
|
213
215
|
|
|
214
216
|
/**
|
|
@@ -231,10 +233,15 @@ export async function discoverBuckets(
|
|
|
231
233
|
// Media bucket resource is named 'Media' (not 'MediaBucket'), so match both patterns
|
|
232
234
|
const mediaNeedle = `${prefix}media-`.toLowerCase();
|
|
233
235
|
const mediaBucketNeedle = `${prefix}mediabucket-`.toLowerCase();
|
|
236
|
+
// Backups bucket (db:backup/restore/backup:download). Discovered like every other bucket, so the
|
|
237
|
+
// --stage path carries it — db:backup:download presigns CLIENT-side and needs the name locally
|
|
238
|
+
// (db:backup/backups run server-side in the ops Lambda, which is why only download failed).
|
|
239
|
+
const backupsNeedle = `${prefix}backupsbucket-`.toLowerCase();
|
|
234
240
|
|
|
235
241
|
let updatesBucket: string | undefined;
|
|
236
242
|
let clientBundlesBucket: string | undefined;
|
|
237
243
|
let mediaBucket: string | undefined;
|
|
244
|
+
let backupsBucket: string | undefined;
|
|
238
245
|
|
|
239
246
|
for (const b of buckets) {
|
|
240
247
|
if (!b.Name) continue;
|
|
@@ -242,10 +249,11 @@ export async function discoverBuckets(
|
|
|
242
249
|
if (lower.startsWith(updatesNeedle)) updatesBucket = b.Name;
|
|
243
250
|
if (lower.startsWith(clientNeedle)) clientBundlesBucket = b.Name;
|
|
244
251
|
if (!mediaBucket && (lower.startsWith(mediaNeedle) || lower.startsWith(mediaBucketNeedle))) mediaBucket = b.Name;
|
|
245
|
-
if (
|
|
252
|
+
if (lower.startsWith(backupsNeedle)) backupsBucket = b.Name;
|
|
253
|
+
if (updatesBucket && clientBundlesBucket && mediaBucket && backupsBucket) break;
|
|
246
254
|
}
|
|
247
255
|
|
|
248
|
-
return { updatesBucket, clientBundlesBucket, mediaBucket };
|
|
256
|
+
return { updatesBucket, clientBundlesBucket, mediaBucket, backupsBucket };
|
|
249
257
|
}
|
|
250
258
|
|
|
251
259
|
// ---------------------------------------------------------------------------
|
|
@@ -304,14 +312,16 @@ export async function discoverConfig(
|
|
|
304
312
|
);
|
|
305
313
|
}
|
|
306
314
|
|
|
307
|
-
// Media bucket fallback: if not found via naming convention (custom/pre-existing
|
|
308
|
-
// try reading from .sst/outputs.json where sst.config.ts return values land.
|
|
315
|
+
// Media/backups bucket fallback: if not found via naming convention (custom/pre-existing
|
|
316
|
+
// bucket), try reading from .sst/outputs.json where sst.config.ts return values land.
|
|
309
317
|
let mediaBucket = bucketsResult.mediaBucket;
|
|
310
|
-
|
|
318
|
+
let backupsBucket = bucketsResult.backupsBucket;
|
|
319
|
+
if (!mediaBucket || !backupsBucket) {
|
|
311
320
|
try {
|
|
312
321
|
const raw = await fs.readFile(path.resolve('.sst', 'outputs.json'), 'utf8');
|
|
313
322
|
const outputs = JSON.parse(raw);
|
|
314
|
-
if (outputs.mediaBucket) mediaBucket = outputs.mediaBucket;
|
|
323
|
+
if (!mediaBucket && outputs.mediaBucket) mediaBucket = outputs.mediaBucket;
|
|
324
|
+
if (!backupsBucket && outputs.backupsBucket) backupsBucket = outputs.backupsBucket;
|
|
315
325
|
} catch {
|
|
316
326
|
// outputs.json not available — non-fatal
|
|
317
327
|
}
|
|
@@ -327,6 +337,7 @@ export async function discoverConfig(
|
|
|
327
337
|
updatesBucket: bucketsResult.updatesBucket,
|
|
328
338
|
clientBundlesBucket: bucketsResult.clientBundlesBucket,
|
|
329
339
|
mediaBucket,
|
|
340
|
+
backupsBucket,
|
|
330
341
|
kvsArn: cfResult.kvsArn,
|
|
331
342
|
distributionId,
|
|
332
343
|
};
|
|
@@ -337,7 +348,7 @@ export async function discoverConfig(
|
|
|
337
348
|
// ---------------------------------------------------------------------------
|
|
338
349
|
|
|
339
350
|
const CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
|
|
340
|
-
const CACHE_VERSION =
|
|
351
|
+
const CACHE_VERSION = 5; // Bump when CachedOutputs schema changes to invalidate stale caches (5: +backupsBucket)
|
|
341
352
|
|
|
342
353
|
interface CachedOutputs {
|
|
343
354
|
_cachedAt: number;
|
|
@@ -350,6 +361,7 @@ interface CachedOutputs {
|
|
|
350
361
|
updatesBucket?: string;
|
|
351
362
|
clientBundlesBucket?: string;
|
|
352
363
|
mediaBucket?: string;
|
|
364
|
+
backupsBucket?: string;
|
|
353
365
|
kvsArn?: string;
|
|
354
366
|
distributionId?: string;
|
|
355
367
|
}
|
|
@@ -377,6 +389,7 @@ export async function getCachedConfig(stage: string): Promise<DiscoveredConfig |
|
|
|
377
389
|
updatesBucket: cached.updatesBucket,
|
|
378
390
|
clientBundlesBucket: cached.clientBundlesBucket,
|
|
379
391
|
mediaBucket: cached.mediaBucket,
|
|
392
|
+
backupsBucket: cached.backupsBucket,
|
|
380
393
|
kvsArn: cached.kvsArn,
|
|
381
394
|
distributionId: cached.distributionId,
|
|
382
395
|
};
|
|
@@ -452,6 +465,7 @@ export async function setCachedConfig(stage: string, config: DiscoveredConfig):
|
|
|
452
465
|
updatesBucket: config.updatesBucket,
|
|
453
466
|
clientBundlesBucket: config.clientBundlesBucket,
|
|
454
467
|
mediaBucket: config.mediaBucket,
|
|
468
|
+
backupsBucket: config.backupsBucket,
|
|
455
469
|
kvsArn: config.kvsArn,
|
|
456
470
|
distributionId: config.distributionId,
|
|
457
471
|
};
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The universal mutation lease (docs/plans/mutation-lease.md).
|
|
3
|
+
*
|
|
4
|
+
* One lease per database, session-scoped, self-releasing, refuse-don't-coordinate.
|
|
5
|
+
* A PostgreSQL advisory SESSION lock with a fixed documented key, acquired on the SAME
|
|
6
|
+
* session that performs the mutation and held for the verb's whole body. Two operators
|
|
7
|
+
* cannot co-mutate one stage; the second is refused with the holder named, and re-tries
|
|
8
|
+
* nothing.
|
|
9
|
+
*
|
|
10
|
+
* This is the minimal core the stage-write-lanes `--direct` verbs need (mutation-lease
|
|
11
|
+
* brick 1). The full subsystem — the `db:lease` break-glass verb, the connection-layer
|
|
12
|
+
* enforcement of "one session, no second connection", the environment-state board — is a
|
|
13
|
+
* strict superset that converges onto this key later. Because the key is fixed, a session
|
|
14
|
+
* lock taken here contends correctly with any future acquirer (PostgreSQL advisory locks
|
|
15
|
+
* contend on the key, not on who took it).
|
|
16
|
+
*
|
|
17
|
+
* Why an advisory session lock and not a lease table: it self-releases on disconnect (a
|
|
18
|
+
* crashed CLI, a killed Lambda) with no TTL, no janitor, no stale row; `pg_try_advisory_lock`
|
|
19
|
+
* is atomic try-acquire so refuse-don't-coordinate falls out of the primitive; and the truth
|
|
20
|
+
* lives in `pg_locks`, discoverable from the catalog rather than bookkeeping that can lie.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import type { QueryRunner } from './authz-contract.js';
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* The fixed lease key: `('ES','TK')` as two 16-bit ints — 0x4553, 0x544B. One key covers
|
|
27
|
+
* every mutation verb (schema applies, reconciles, syncs, restores) so the cross-lane
|
|
28
|
+
* collision the lease exists to prevent (a rebuild during a refresh, a restore during a
|
|
29
|
+
* sync) cannot slip between per-lane keys. Documented, never computed.
|
|
30
|
+
*/
|
|
31
|
+
export const MUTATION_LEASE_KEY = { classid: 0x4553, objid: 0x544b } as const;
|
|
32
|
+
|
|
33
|
+
/** PostgreSQL truncates `application_name` to NAMEDATALEN-1 = 63 bytes silently. */
|
|
34
|
+
const APPLICATION_NAME_MAX_BYTES = 63;
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Build the mutation session's identity: `everystack:<verb>:<actor>+<session>`. The
|
|
38
|
+
* `<actor>` alone is not enough — two agents on the same stage this week both run as the
|
|
39
|
+
* same OS user, so a refusal that names only the actor tells the second agent nothing. The
|
|
40
|
+
* `+<session>` discriminator is what makes the refusal actionable: the second agent reads a
|
|
41
|
+
* session id that is not its own and knows it is colliding with a live peer, not staring at
|
|
42
|
+
* its own orphaned backend. Truncated to 63 bytes on a char boundary (PostgreSQL would
|
|
43
|
+
* truncate silently; we do it deterministically so the stored name is knowable).
|
|
44
|
+
*/
|
|
45
|
+
export function leaseApplicationName(verb: string, actor: string, session: string): string {
|
|
46
|
+
const full = `everystack:${verb}:${actor}+${session}`;
|
|
47
|
+
return truncateToBytes(full, APPLICATION_NAME_MAX_BYTES);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** Truncate to at most `maxBytes` UTF-8 bytes without splitting a multibyte char. */
|
|
51
|
+
function truncateToBytes(s: string, maxBytes: number): string {
|
|
52
|
+
if (Buffer.byteLength(s, 'utf8') <= maxBytes) return s;
|
|
53
|
+
let out = '';
|
|
54
|
+
let bytes = 0;
|
|
55
|
+
for (const ch of s) {
|
|
56
|
+
const chBytes = Buffer.byteLength(ch, 'utf8');
|
|
57
|
+
if (bytes + chBytes > maxBytes) break;
|
|
58
|
+
out += ch;
|
|
59
|
+
bytes += chBytes;
|
|
60
|
+
}
|
|
61
|
+
return out;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Single-quote-escape a string for inline SQL (doubles embedded quotes). */
|
|
65
|
+
function sqlQuote(s: string): string {
|
|
66
|
+
return `'${s.replace(/'/g, "''")}'`;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** `SET application_name` for this session, via set_config so the value is quote-safe. */
|
|
70
|
+
export function setApplicationNameSql(name: string): string {
|
|
71
|
+
return `SELECT set_config('application_name', ${sqlQuote(name)}, false)`;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Try-acquire the session lease. Returns a boolean column `acquired`. Atomic, non-blocking. */
|
|
75
|
+
export const TRY_ACQUIRE_LEASE_SQL =
|
|
76
|
+
`SELECT pg_try_advisory_lock(${MUTATION_LEASE_KEY.classid}, ${MUTATION_LEASE_KEY.objid}) AS acquired`;
|
|
77
|
+
|
|
78
|
+
/** Release the session lease. Idempotent-safe to call in a finally. */
|
|
79
|
+
export const RELEASE_LEASE_SQL =
|
|
80
|
+
`SELECT pg_advisory_unlock(${MUTATION_LEASE_KEY.classid}, ${MUTATION_LEASE_KEY.objid}) AS released`;
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Read the current holder of the lease from the catalog — the truth, not bookkeeping.
|
|
84
|
+
* Joins `pg_locks` to `pg_stat_activity` on the holding backend and returns the database,
|
|
85
|
+
* the holder's `application_name`, its pid, when the backend started, and its state.
|
|
86
|
+
*/
|
|
87
|
+
export const HOLDER_SQL = `
|
|
88
|
+
SELECT
|
|
89
|
+
current_database() AS database,
|
|
90
|
+
a.application_name AS holder,
|
|
91
|
+
a.pid AS pid,
|
|
92
|
+
to_char(a.backend_start AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS') || ' UTC' AS since,
|
|
93
|
+
a.state AS state
|
|
94
|
+
FROM pg_locks l
|
|
95
|
+
JOIN pg_stat_activity a ON a.pid = l.pid
|
|
96
|
+
WHERE l.locktype = 'advisory'
|
|
97
|
+
AND l.classid = ${MUTATION_LEASE_KEY.classid}
|
|
98
|
+
AND l.objid = ${MUTATION_LEASE_KEY.objid}
|
|
99
|
+
AND l.granted
|
|
100
|
+
LIMIT 1;
|
|
101
|
+
`.trim();
|
|
102
|
+
|
|
103
|
+
/** The structured holder facts the refusal sentence renders from. Pure, testable. */
|
|
104
|
+
export interface LeaseHolder {
|
|
105
|
+
database: string;
|
|
106
|
+
holder: string | null;
|
|
107
|
+
pid: number | null;
|
|
108
|
+
since: string | null;
|
|
109
|
+
state: string | null;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Render the refusal sentence from the holder facts. Pure — the impure path assembles a
|
|
114
|
+
* `LeaseHolder` from `HOLDER_SQL` and hands it here, so the exact wording is unit-tested
|
|
115
|
+
* against a fixed row shape.
|
|
116
|
+
*
|
|
117
|
+
* Deliberately does NOT advertise `everystack db:lease`: that break-glass verb is a later
|
|
118
|
+
* brick and does not exist yet. Naming a command the CLI would then reject is the very
|
|
119
|
+
* paper-cut this whole effort is closing — so the hint stays true to what exists today.
|
|
120
|
+
*/
|
|
121
|
+
export function renderLeaseRefusal(h: LeaseHolder): string {
|
|
122
|
+
const holder = h.holder && h.holder.length > 0 ? h.holder : 'an unnamed session';
|
|
123
|
+
const parts: string[] = [];
|
|
124
|
+
if (h.pid != null) parts.push(`pid ${h.pid}`);
|
|
125
|
+
if (h.since) parts.push(`since ${h.since}`);
|
|
126
|
+
if (h.state) parts.push(`state: ${h.state}`);
|
|
127
|
+
const where = parts.length ? ` (${parts.join(', ')})` : '';
|
|
128
|
+
return (
|
|
129
|
+
`refused: ${h.database} is being mutated by ${holder}${where}. ` +
|
|
130
|
+
`One mutation at a time — wait for it to finish, or if the holder is dead-but-connected, ` +
|
|
131
|
+
`terminate that backend and retry.`
|
|
132
|
+
);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Thrown when the lease is held by another session. Carries the holder for callers. */
|
|
136
|
+
export class MutationLeaseError extends Error {
|
|
137
|
+
readonly holder: LeaseHolder;
|
|
138
|
+
constructor(message: string, holder: LeaseHolder) {
|
|
139
|
+
super(message);
|
|
140
|
+
this.name = 'MutationLeaseError';
|
|
141
|
+
this.holder = holder;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export interface LeaseIdentity {
|
|
146
|
+
/** The verb holding the lease, e.g. `db:apply`, `db:reconcile`. */
|
|
147
|
+
verb: string;
|
|
148
|
+
/** The operator identity (OS user / STS identity). */
|
|
149
|
+
actor: string;
|
|
150
|
+
/**
|
|
151
|
+
* A short per-invocation discriminator so the two-agents-same-user refusal is actionable.
|
|
152
|
+
* Defaults to a fresh 6-hex-char nonce.
|
|
153
|
+
*/
|
|
154
|
+
session?: string;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** A fresh 6-hex-char session discriminator. */
|
|
158
|
+
export function leaseSession(): string {
|
|
159
|
+
// Node's crypto is always present; 3 bytes -> 6 hex chars, ample for the discriminator.
|
|
160
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
161
|
+
const { randomBytes } = require('node:crypto');
|
|
162
|
+
return randomBytes(3).toString('hex');
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Own the lease for the whole of `fn`. Stamps `application_name`, try-acquires the session
|
|
167
|
+
* lock, and on contention throws `MutationLeaseError` naming the holder. Releases in a
|
|
168
|
+
* `finally` — and because the lock is session-scoped, a crash mid-body frees it too.
|
|
169
|
+
*
|
|
170
|
+
* Must run on the mutation's OWN session — the same max:1 connection that performs every
|
|
171
|
+
* write of the verb. A verb that opens a SECOND connection mid-body has a writer the lease
|
|
172
|
+
* does not cover.
|
|
173
|
+
*
|
|
174
|
+
* HONEST LIMITATION (do not overstate): this is CONVENTIONALLY enforced, not runtime-enforced.
|
|
175
|
+
* A PostgreSQL connection carries no read/write intent the connection layer can inspect, so
|
|
176
|
+
* "reject a second connection" is unrealizable in-process. The counterexample is already in the
|
|
177
|
+
* tree: `createUrlPipelineRunner` (db-source.ts) opens a second, independent max:1 session with
|
|
178
|
+
* its own transaction, distinct from the `createUrlRunner` session this lease is handed — a verb
|
|
179
|
+
* that leases on one and writes on the other has a mutating connection the lease does not cover,
|
|
180
|
+
* and nothing rejects it. The only REAL enforcement is Postgres-side: reads connect as a
|
|
181
|
+
* read-only role that physically cannot write (redesign tracked in mutation-lease.md B1). Until
|
|
182
|
+
* that lands, treat this as a code-review convention with a named residual hole, not a guarantee.
|
|
183
|
+
*/
|
|
184
|
+
export async function withMutationLease<T>(
|
|
185
|
+
runner: QueryRunner,
|
|
186
|
+
identity: LeaseIdentity,
|
|
187
|
+
fn: () => Promise<T>,
|
|
188
|
+
): Promise<T> {
|
|
189
|
+
const session = identity.session ?? leaseSession();
|
|
190
|
+
const appName = leaseApplicationName(identity.verb, identity.actor, session);
|
|
191
|
+
await runner(setApplicationNameSql(appName));
|
|
192
|
+
|
|
193
|
+
const rows = await runner(TRY_ACQUIRE_LEASE_SQL);
|
|
194
|
+
const acquired = rows[0]?.acquired === true || rows[0]?.acquired === 't';
|
|
195
|
+
if (!acquired) {
|
|
196
|
+
const holder = await readHolder(runner);
|
|
197
|
+
throw new MutationLeaseError(renderLeaseRefusal(holder), holder);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
try {
|
|
201
|
+
return await fn();
|
|
202
|
+
} finally {
|
|
203
|
+
// Best-effort release; the session lock also frees on disconnect, so a throw here
|
|
204
|
+
// (e.g. the connection already died) must not mask the body's outcome.
|
|
205
|
+
try {
|
|
206
|
+
await runner(RELEASE_LEASE_SQL);
|
|
207
|
+
} catch {
|
|
208
|
+
/* session gone — the lock died with it */
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** Assemble a `LeaseHolder` from `HOLDER_SQL`; a missing row still yields a usable sentence. */
|
|
214
|
+
async function readHolder(runner: QueryRunner): Promise<LeaseHolder> {
|
|
215
|
+
let row: any;
|
|
216
|
+
try {
|
|
217
|
+
row = (await runner(HOLDER_SQL))[0];
|
|
218
|
+
} catch {
|
|
219
|
+
row = undefined;
|
|
220
|
+
}
|
|
221
|
+
return {
|
|
222
|
+
database: row?.database ?? 'the database',
|
|
223
|
+
holder: row?.holder ?? null,
|
|
224
|
+
pid: row?.pid != null ? Number(row.pid) : null,
|
|
225
|
+
since: row?.since ?? null,
|
|
226
|
+
state: row?.state ?? null,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Will this edge fit the ops-Lambda clock? (stage-write-lanes brick 2.)
|
|
3
|
+
*
|
|
4
|
+
* `db:apply --stage` executes the write inside the ops Lambda, which has a hard 900-second
|
|
5
|
+
* ceiling. A large brownfield edge — 1855 statements with 117 index builds, on a t4g.small —
|
|
6
|
+
* hit that ceiling in the field: the transaction rolled back clean, but the operator burned
|
|
7
|
+
* 15 minutes to learn it wouldn't fit. This estimates fit from the plan itself (at mint time,
|
|
8
|
+
* no live probe) so the ops path can REFUSE up front and name `--direct`, the credential-free
|
|
9
|
+
* escape with an unbounded clock.
|
|
10
|
+
*
|
|
11
|
+
* The heuristic is deliberately CONSERVATIVE, and that is safe by construction: a false
|
|
12
|
+
* "won't fit" routes the operator to `--direct`, which is itself credential-free and strictly
|
|
13
|
+
* more capable than the ops-Lambda path. Over-refusing costs nothing but a flag; under-refusing
|
|
14
|
+
* costs a 15-minute timeout. So we lean toward refusing.
|
|
15
|
+
*
|
|
16
|
+
* The bottleneck is the DATABASE, not the Lambda's memory/CPU — index builds scale with table
|
|
17
|
+
* size — so a raw statement count is only a proxy. Two signals: total executable statements,
|
|
18
|
+
* and the count of non-CONCURRENT index builds (the expensive class that dominated the incident).
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
export interface OpsFitPlan {
|
|
22
|
+
statements?: string[];
|
|
23
|
+
executable?: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Thresholds tuned to the incident (1855 statements / 117 index builds timed out at 900s on a
|
|
28
|
+
* t4g.small) with generous headroom below it. Named so the reasoning is legible and one edit
|
|
29
|
+
* retunes the gate.
|
|
30
|
+
*/
|
|
31
|
+
export const OPS_FIT_LIMITS = {
|
|
32
|
+
/** Total executable statements above which the ops-Lambda clock is at risk. */
|
|
33
|
+
maxStatements: 500,
|
|
34
|
+
/** Non-CONCURRENT index builds above which a single edge likely blows the budget. */
|
|
35
|
+
maxIndexBuilds: 30,
|
|
36
|
+
} as const;
|
|
37
|
+
|
|
38
|
+
/** A CREATE INDEX that will hold a lock and scan the table (CONCURRENTLY is the slow-but-online form). */
|
|
39
|
+
function isBlockingIndexBuild(stmt: string): boolean {
|
|
40
|
+
return /^\s*CREATE\s+(UNIQUE\s+)?INDEX\b/i.test(stmt) && !/\bCONCURRENTLY\b/i.test(stmt);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface OpsFitVerdict {
|
|
44
|
+
fits: boolean;
|
|
45
|
+
/** Present when !fits — the sentence the ops path refuses with, already naming --direct. */
|
|
46
|
+
reason?: string;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Estimate whether `plan` fits the ops-Lambda runtime. Pure; no database, no clock.
|
|
51
|
+
* `db:apply --stage` (the ops-Lambda path) calls this BEFORE invoking; `--direct` skips it
|
|
52
|
+
* (it has no Lambda ceiling).
|
|
53
|
+
*/
|
|
54
|
+
export function estimateOpsRuntimeFit(plan: OpsFitPlan): OpsFitVerdict {
|
|
55
|
+
const statements = plan.statements ?? [];
|
|
56
|
+
const total = plan.executable ?? statements.length;
|
|
57
|
+
const indexBuilds = statements.filter(isBlockingIndexBuild).length;
|
|
58
|
+
|
|
59
|
+
if (total > OPS_FIT_LIMITS.maxStatements) {
|
|
60
|
+
return {
|
|
61
|
+
fits: false,
|
|
62
|
+
reason:
|
|
63
|
+
`this edge is ${total} statements — past the ~${OPS_FIT_LIMITS.maxStatements}-statement ops-Lambda budget ` +
|
|
64
|
+
`(the write runs inside a 900-second Lambda; a large edge rolls back clean but wastes the wait). ` +
|
|
65
|
+
`Re-run with --direct — same ceremony, executed CLI-side with an unbounded clock, still credential-free.`,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
if (indexBuilds > OPS_FIT_LIMITS.maxIndexBuilds) {
|
|
69
|
+
return {
|
|
70
|
+
fits: false,
|
|
71
|
+
reason:
|
|
72
|
+
`this edge builds ${indexBuilds} indexes — index builds scale with table size and dominate the ops-Lambda ` +
|
|
73
|
+
`runtime (past ~${OPS_FIT_LIMITS.maxIndexBuilds} the 900-second ceiling is at risk). ` +
|
|
74
|
+
`Re-run with --direct — same ceremony, executed CLI-side with an unbounded clock, still credential-free.`,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
return { fits: true };
|
|
78
|
+
}
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* path. This is that verb: refresh the declared matviews in dependency order, over whatever
|
|
7
7
|
* runner the venue supplies. The ops `db:refresh` action loads executeRefresh here and runs it
|
|
8
8
|
* on the operator connection, so `db:refresh --stage` needs no raw URL on the operator's machine
|
|
9
|
-
* (
|
|
9
|
+
* (consumer field report 2026-07-19, the data-lane twin of the reconcile lane).
|
|
10
10
|
*
|
|
11
11
|
* Plain `REFRESH MATERIALIZED VIEW` (ACCESS EXCLUSIVE) — the faithful twin of the direct-connect
|
|
12
12
|
* data script it replaces. CONCURRENTLY (needs a unique index; doesn't block reads) is a
|