@everystack/cli 0.3.31 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +2 -2
- package/src/.pdr-tmp-20277-1783706384842.ts +59 -0
- package/src/.roundtrip-tmp-20275-1783706385459.ts +121 -0
- package/src/cli/authz-compile.ts +5 -2
- package/src/cli/aws.ts +12 -3
- package/src/cli/backfill.ts +1 -1
- package/src/cli/commands/db-authz.ts +3 -17
- package/src/cli/commands/db-branch.ts +28 -4
- package/src/cli/commands/db-check.ts +84 -33
- package/src/cli/commands/db-fingerprint.ts +7 -2
- package/src/cli/commands/db-generate.ts +102 -34
- package/src/cli/commands/db-pull.ts +82 -6
- package/src/cli/commands/db-reconcile.ts +132 -41
- package/src/cli/commands/db-sync.ts +54 -19
- package/src/cli/commands/db.ts +7 -6
- package/src/cli/commands/update.ts +2 -1
- package/src/cli/db-build.ts +9 -3
- package/src/cli/db-source.ts +5 -1
- package/src/cli/declared-derived.ts +173 -0
- package/src/cli/derived-apply.ts +40 -6
- package/src/cli/derived-compile.ts +377 -0
- package/src/cli/derived-grants.ts +96 -0
- package/src/cli/derived-introspect.ts +258 -21
- package/src/cli/derived-lint.ts +261 -0
- package/src/cli/derived-plan.ts +176 -10
- package/src/cli/derived-render.ts +366 -0
- package/src/cli/derived-source.ts +53 -125
- package/src/cli/edge-plan.ts +4 -1
- package/src/cli/git-descent.ts +15 -2
- package/src/cli/index.ts +6 -6
- package/src/cli/migration-compile.ts +38 -7
- package/src/cli/migration-generate.ts +43 -4
- package/src/cli/model-render.ts +125 -16
- package/src/cli/models-path.ts +1 -1
- package/src/cli/ops-advice.ts +66 -0
- package/src/cli/pipeline-path.ts +1 -1
- package/src/cli/schema-compile.ts +41 -10
- package/src/cli/schema-diff.ts +123 -17
- package/src/cli/schema-fingerprint.ts +28 -18
- package/src/cli/schema-introspect.ts +175 -8
- package/src/cli/schema-source.ts +75 -6
- package/src/cli/state-apply.ts +52 -1
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* `everystack db:reconcile` — deploy the derived layer (functions, views,
|
|
3
|
-
* matviews) by reconciling the live database
|
|
3
|
+
* matviews, triggers, escape-hatch objects) by reconciling the live database
|
|
4
|
+
* against the DECLARED descriptors (defineView / defineMaterializedView /
|
|
5
|
+
* defineFunction / defineSql / trigger() on the models). Descriptors are the
|
|
6
|
+
* single home — a checkout still carrying db/sql *.sql files fails loudly with
|
|
7
|
+
* the migration path (B7).
|
|
4
8
|
*
|
|
5
|
-
* db:reconcile [--
|
|
6
|
-
* [--apply] [--check] [--baseline] [--overwrite-drift] [--json]
|
|
9
|
+
* db:reconcile [--stage <name> | --database-url <url>]
|
|
10
|
+
* [--apply] [--check] [--baseline] [--rebaseline] [--overwrite-drift] [--json]
|
|
7
11
|
*
|
|
8
12
|
* Derived objects do not migrate; they deploy. The plan is computed fresh
|
|
9
13
|
* every run: source objects (hashed, comment/whitespace-insensitive) joined
|
|
@@ -16,33 +20,35 @@
|
|
|
16
20
|
*
|
|
17
21
|
* Hand-edited live objects are DRIFT — reported, applied over only with
|
|
18
22
|
* `--overwrite-drift`. Never-reconciled matches need `--baseline` once (the
|
|
19
|
-
* explicit trust that live == source, recorded).
|
|
23
|
+
* explicit trust that live == source, recorded). A source RE-RENDER over an
|
|
24
|
+
* untouched live object — the db/sql-era → descriptor migration — needs
|
|
25
|
+
* `--rebaseline` once: live is verified (defHash match), the new source hash
|
|
26
|
+
* is re-recorded, nothing rebuilds. `--apply` needs a direct
|
|
20
27
|
* connection (`--database-url` / DATABASE_URL): the ops Lambda `db:query`
|
|
21
28
|
* path is read-only by design, so plan/check work anywhere but writes go
|
|
22
29
|
* through a credentialed connection you chose to hold.
|
|
23
30
|
*/
|
|
24
31
|
|
|
25
|
-
import fs from 'node:fs/promises';
|
|
26
|
-
import path from 'node:path';
|
|
27
32
|
import type { QueryRunner } from '../authz-contract.js';
|
|
28
|
-
import {
|
|
33
|
+
import type { SourceObject } from '../derived-source.js';
|
|
29
34
|
import { introspectDerived } from '../derived-introspect.js';
|
|
30
35
|
import { planReconcile, type ReconcilePlan, type ReconcileOptions } from '../derived-plan.js';
|
|
31
36
|
import {
|
|
32
37
|
renderReconcileSql,
|
|
33
38
|
renderProvenanceUpsert,
|
|
39
|
+
renderProvenanceMigrate,
|
|
40
|
+
triggerDropSql,
|
|
34
41
|
renderProvenanceDelete,
|
|
35
42
|
renderSchemaLogInsert,
|
|
36
43
|
ENSURE_RECONCILER_SQL,
|
|
37
44
|
} from '../derived-apply.js';
|
|
38
45
|
import { resolveDbSource, createUrlRunner, connectingVia, type DbSource } from '../db-source.js';
|
|
46
|
+
import { loadDeclaredDerived, retiredSqlDirAnywhere, retiredSqlDirFlagRefusal, type DeclaredDerived } from '../declared-derived.js';
|
|
39
47
|
import { currentGitRef } from '../state-apply.js';
|
|
40
48
|
import { resolveConfig, opsFunction } from '../config.js';
|
|
41
49
|
import { invokeAction } from '../aws.js';
|
|
42
50
|
import { step, success, fail, info, warn } from '../output.js';
|
|
43
51
|
|
|
44
|
-
const DEFAULT_SQL_DIR = 'db/sql';
|
|
45
|
-
|
|
46
52
|
// ---------------------------------------------------------------------------
|
|
47
53
|
// The testable core.
|
|
48
54
|
// ---------------------------------------------------------------------------
|
|
@@ -52,6 +58,8 @@ export interface ExecuteOptions extends ReconcileOptions {
|
|
|
52
58
|
actor?: string | null;
|
|
53
59
|
gitRef?: string | null;
|
|
54
60
|
planRef?: string | null;
|
|
61
|
+
/** Descriptor-compiled objects (the declared derived layer) — the ONE source stream. */
|
|
62
|
+
declared?: SourceObject[];
|
|
55
63
|
/** Injectable clock for tests. */
|
|
56
64
|
now?: () => number;
|
|
57
65
|
}
|
|
@@ -73,22 +81,35 @@ export interface ReconcileRun {
|
|
|
73
81
|
/** Statements Postgres forbids inside a transaction block — they implicitly commit. */
|
|
74
82
|
export const NON_TRANSACTIONAL_RE = /\bCONCURRENTLY\b|\bVACUUM\b/i;
|
|
75
83
|
|
|
84
|
+
/** Session identity — the guard against postgres-js's silent reconnect (see the apply path). */
|
|
85
|
+
export const BACKEND_PID_SQL = 'SELECT pg_backend_pid() AS pid';
|
|
86
|
+
|
|
87
|
+
// Dollar-quoted bodies and comments are CONTENT, not batch shape — a plpgsql helper whose
|
|
88
|
+
// body says `REFRESH MATERIALIZED VIEW CONCURRENTLY …` is perfectly transactional to CREATE.
|
|
89
|
+
// Stripping errs loud, never silent: over-stripping would wrongly WRAP a truly-concurrent
|
|
90
|
+
// statement, and PostgreSQL then refuses it with its own clear error at apply time.
|
|
91
|
+
const DOLLAR_BODY_RE = /\$([A-Za-z_][A-Za-z0-9_]*)?\$[\s\S]*?\$\1\$/g;
|
|
92
|
+
const LINE_COMMENT_RE = /--[^\n]*/g;
|
|
93
|
+
const BLOCK_COMMENT_RE = /\/\*[\s\S]*?\*\//g;
|
|
94
|
+
|
|
76
95
|
/**
|
|
77
96
|
* Whether the whole batch can run inside one transaction. Almost always yes — the escape hatch
|
|
78
97
|
* is a consumer whose matview source carries a `CREATE INDEX CONCURRENTLY` (rendered verbatim) or
|
|
79
|
-
* a `VACUUM`, which self-commit and cannot live in a transaction block.
|
|
98
|
+
* a `VACUUM`, which self-commit and cannot live in a transaction block. Only the statement's own
|
|
99
|
+
* shape decides: dollar-quoted bodies and comments are stripped before the test.
|
|
80
100
|
*/
|
|
81
101
|
export function isTransactionalBatch(statements: string[]): boolean {
|
|
82
|
-
return statements.every((s) => !NON_TRANSACTIONAL_RE.test(
|
|
102
|
+
return statements.every((s) => !NON_TRANSACTIONAL_RE.test(
|
|
103
|
+
s.replace(DOLLAR_BODY_RE, '').replace(BLOCK_COMMENT_RE, '').replace(LINE_COMMENT_RE, ''),
|
|
104
|
+
));
|
|
83
105
|
}
|
|
84
106
|
|
|
85
107
|
export async function executeReconcile(
|
|
86
108
|
runner: QueryRunner,
|
|
87
|
-
sources: SourceFile[],
|
|
88
109
|
options: ExecuteOptions = {},
|
|
89
110
|
): Promise<ReconcileRun> {
|
|
90
111
|
const live = await introspectDerived(runner);
|
|
91
|
-
const parsed =
|
|
112
|
+
const parsed = { objects: options.declared ?? [], warnings: [] as string[] };
|
|
92
113
|
const plan = planReconcile(parsed, live, options);
|
|
93
114
|
const rendered = renderReconcileSql(plan, parsed.objects);
|
|
94
115
|
|
|
@@ -106,7 +127,7 @@ export async function executeReconcile(
|
|
|
106
127
|
refusal: `drift on ${plan.drift.map((d) => d.identity).join(', ')} — inspect, then re-run with --overwrite-drift to rebuild from source`,
|
|
107
128
|
};
|
|
108
129
|
}
|
|
109
|
-
if (rendered.statements.length === 0 && rendered.record.length === 0 && rendered.remove.length === 0) {
|
|
130
|
+
if (rendered.statements.length === 0 && rendered.record.length === 0 && rendered.remove.length === 0 && rendered.migrate.length === 0) {
|
|
110
131
|
return { plan, applied: false, statements: [] };
|
|
111
132
|
}
|
|
112
133
|
|
|
@@ -124,8 +145,26 @@ export async function executeReconcile(
|
|
|
124
145
|
|
|
125
146
|
if (atomic) await runner('BEGIN');
|
|
126
147
|
try {
|
|
148
|
+
// The session guard: postgres-js `max: 1` is a pool of one, not a session lease — a
|
|
149
|
+
// connection that dies between runner calls silently reconnects, the open transaction
|
|
150
|
+
// aborts server-side, and every later call runs in autocommit on a fresh session. The
|
|
151
|
+
// bookkeeping would then record a state the DDL never reached, and the object would read
|
|
152
|
+
// as converged forever (the red-team's false-convergence finding). Pin the backend pid
|
|
153
|
+
// at BEGIN; refuse the bookkeeping if it moved. createUrlRunner also disables the
|
|
154
|
+
// driver's max_lifetime recycling — the deterministic trigger during long applies.
|
|
155
|
+
const pidAtBegin = atomic ? Number((await runner(BACKEND_PID_SQL))[0]?.pid) : null;
|
|
156
|
+
|
|
127
157
|
if (rendered.statements.length > 0) {
|
|
128
|
-
|
|
158
|
+
if (atomic) {
|
|
159
|
+
await runner(rendered.statements.join(';\n'));
|
|
160
|
+
} else {
|
|
161
|
+
// Self-committing statements (CONCURRENTLY, VACUUM) refuse ANY transaction block —
|
|
162
|
+
// including the implicit one a joined multi-statement simple query runs in, so the
|
|
163
|
+
// batch MUST go one statement per protocol message. A failure mid-way leaves the
|
|
164
|
+
// earlier statements committed: the documented cost of the unwrapped path; the next
|
|
165
|
+
// plan re-converges from what actually landed.
|
|
166
|
+
for (const statement of rendered.statements) await runner(statement);
|
|
167
|
+
}
|
|
129
168
|
}
|
|
130
169
|
|
|
131
170
|
// Re-read the catalog so provenance records the def hashes of what NOW exists, not what we hoped
|
|
@@ -137,20 +176,42 @@ export async function executeReconcile(
|
|
|
137
176
|
const bookkeeping: string[] = [];
|
|
138
177
|
for (const identity of rendered.record) {
|
|
139
178
|
const src = srcById.get(identity);
|
|
179
|
+
if (!src) {
|
|
180
|
+
throw new Error(`reconcile applied but ${identity} has no source object — provenance not recorded, investigate`);
|
|
181
|
+
}
|
|
182
|
+
// 'sql'-kind objects are invisible to the catalog: provenance records the source
|
|
183
|
+
// hash on both sides plus HOW TO REMOVE it (the escape hatch's whole contract).
|
|
184
|
+
if (src.kind === 'sql') {
|
|
185
|
+
bookkeeping.push(renderProvenanceUpsert(identity, src.hash, src.hash, 'sql', src.drop));
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
140
188
|
const liveObj = liveById.get(identity);
|
|
141
|
-
if (!
|
|
189
|
+
if (!liveObj) {
|
|
142
190
|
throw new Error(`reconcile applied but ${identity} is not introspectable afterwards — provenance not recorded, investigate`);
|
|
143
191
|
}
|
|
144
|
-
|
|
192
|
+
const dropSql = src.kind === 'trigger' && src.table ? triggerDropSql(src.name, src.table) : undefined;
|
|
193
|
+
bookkeeping.push(renderProvenanceUpsert(identity, src.hash, liveObj.defHash, src.kind, dropSql));
|
|
145
194
|
}
|
|
195
|
+
for (const m of rendered.migrate) bookkeeping.push(renderProvenanceMigrate(m.from, m.to));
|
|
146
196
|
for (const identity of rendered.remove) bookkeeping.push(renderProvenanceDelete(identity));
|
|
147
197
|
bookkeeping.push(renderSchemaLogInsert({
|
|
148
198
|
kind: 'compute reconcile',
|
|
149
|
-
sql: rendered.statements.join(';\n') || '-- provenance-only (baseline/prune)',
|
|
199
|
+
sql: rendered.statements.join(';\n') || '-- provenance-only (baseline/rebaseline/prune)',
|
|
150
200
|
outcome: 'applied',
|
|
151
201
|
actor: options.actor, gitRef: options.gitRef, planRef: options.planRef,
|
|
152
202
|
durationMs: now() - started,
|
|
153
203
|
}));
|
|
204
|
+
|
|
205
|
+
if (atomic) {
|
|
206
|
+
const pidNow = Number((await runner(BACKEND_PID_SQL))[0]?.pid);
|
|
207
|
+
if (pidNow !== pidAtBegin) {
|
|
208
|
+
throw new Error(
|
|
209
|
+
`the session changed mid-transaction (backend pid ${pidAtBegin} → ${pidNow}) — the connection `
|
|
210
|
+
+ `was swapped or recycled under the open transaction (pooler restart? server restart?), PostgreSQL `
|
|
211
|
+
+ `aborted it, and NOTHING from this batch is applied. No bookkeeping was written; re-run the apply.`,
|
|
212
|
+
);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
154
215
|
await runner(bookkeeping.join(';\n'));
|
|
155
216
|
|
|
156
217
|
if (atomic) await runner('COMMIT');
|
|
@@ -205,7 +266,7 @@ export function formatBytes(bytes: number): string {
|
|
|
205
266
|
}
|
|
206
267
|
|
|
207
268
|
const VERB_GLYPH: Record<string, string> = {
|
|
208
|
-
create: '+', replace: '~', drop: '-', refresh: '↻', baseline: '◦', prune: '·',
|
|
269
|
+
create: '+', replace: '~', drop: '-', refresh: '↻', baseline: '◦', rebaseline: '≈', prune: '·',
|
|
209
270
|
};
|
|
210
271
|
|
|
211
272
|
export function buildReconcileReport(plan: ReconcilePlan): string[] {
|
|
@@ -220,6 +281,16 @@ export function buildReconcileReport(plan: ReconcilePlan): string[] {
|
|
|
220
281
|
lines.push(`estimated rebuild: ${formatBytes(plan.totalRebuildBytes)} across ${rebuilt.length} matview(s)`);
|
|
221
282
|
}
|
|
222
283
|
for (const d of plan.drift) lines.push(`! drift ${d.identity} — ${d.detail}`);
|
|
284
|
+
for (const r of plan.regrants) {
|
|
285
|
+
lines.push(`≠ regrant ${r.identity} — live grants differ from source (${r.statements.join('; ')})`);
|
|
286
|
+
}
|
|
287
|
+
for (const d of plan.dependencyDrift) {
|
|
288
|
+
const parts = [
|
|
289
|
+
...(d.missing.length ? [`reads ${d.missing.join(', ')} undeclared`] : []),
|
|
290
|
+
...(d.stale.length ? [`declares ${d.stale.join(', ')} but never reads it`] : []),
|
|
291
|
+
];
|
|
292
|
+
lines.push(`! depends ${d.identity} — ${parts.join('; ')} (fix dependsOn in the descriptor)`);
|
|
293
|
+
}
|
|
223
294
|
if (plan.needsBaseline.length > 0) {
|
|
224
295
|
lines.push(`! needs baseline (${plan.needsBaseline.length}): ${plan.needsBaseline.join(', ')} — run with --baseline to record trust, once`);
|
|
225
296
|
}
|
|
@@ -232,12 +303,15 @@ export function buildReconcileReport(plan: ReconcilePlan): string[] {
|
|
|
232
303
|
return lines;
|
|
233
304
|
}
|
|
234
305
|
|
|
235
|
-
/** Anything that makes `--check` fail: pending work or unresolved findings.
|
|
306
|
+
/** Anything that makes `--check` fail: pending work or unresolved findings.
|
|
307
|
+
* Baseline and rebaseline are explicit adoption the operator asked for, not findings. */
|
|
236
308
|
export function checkFails(plan: ReconcilePlan): boolean {
|
|
237
|
-
return plan.actions.some((a) => a.action !== 'baseline')
|
|
309
|
+
return plan.actions.some((a) => a.action !== 'baseline' && a.action !== 'rebaseline')
|
|
238
310
|
|| plan.drift.length > 0
|
|
239
311
|
|| plan.needsBaseline.length > 0
|
|
240
|
-
|| plan.blocked.length > 0
|
|
312
|
+
|| plan.blocked.length > 0
|
|
313
|
+
|| plan.regrants.length > 0
|
|
314
|
+
|| plan.dependencyDrift.length > 0;
|
|
241
315
|
}
|
|
242
316
|
|
|
243
317
|
// ---------------------------------------------------------------------------
|
|
@@ -253,25 +327,27 @@ function lambdaRunner(region: string, fn: string): QueryRunner {
|
|
|
253
327
|
};
|
|
254
328
|
}
|
|
255
329
|
|
|
256
|
-
async function readSqlDir(dir: string): Promise<SourceFile[]> {
|
|
257
|
-
let entries: string[];
|
|
258
|
-
try {
|
|
259
|
-
entries = await fs.readdir(dir);
|
|
260
|
-
} catch {
|
|
261
|
-
throw new Error(`No SQL source directory at ${dir} (set --sql-dir).`);
|
|
262
|
-
}
|
|
263
|
-
const files = entries.filter((f) => f.endsWith('.sql')).sort();
|
|
264
|
-
return Promise.all(files.map(async (file) => ({
|
|
265
|
-
file: path.join(dir, file),
|
|
266
|
-
sql: await fs.readFile(path.join(dir, file), 'utf-8'),
|
|
267
|
-
})));
|
|
268
|
-
}
|
|
269
|
-
|
|
270
330
|
export async function dbReconcileCommand(flags: Record<string, string>): Promise<void> {
|
|
271
|
-
const sqlDir = flags['sql-dir'] || DEFAULT_SQL_DIR;
|
|
272
331
|
const apply = flags.apply === 'true';
|
|
273
332
|
const check = flags.check === 'true';
|
|
274
333
|
|
|
334
|
+
// B7 tombstone — db/sql is retired and no longer read. Failing beats silently
|
|
335
|
+
// skipping: skipped files would leave live objects unmanaged.
|
|
336
|
+
try {
|
|
337
|
+
if (flags['sql-dir']) {
|
|
338
|
+
fail(await retiredSqlDirFlagRefusal(flags['sql-dir']));
|
|
339
|
+
process.exit(1);
|
|
340
|
+
}
|
|
341
|
+
const retired = await retiredSqlDirAnywhere(flags.models);
|
|
342
|
+
if (retired) {
|
|
343
|
+
fail(retired);
|
|
344
|
+
process.exit(1);
|
|
345
|
+
}
|
|
346
|
+
} catch (err: any) {
|
|
347
|
+
fail(err.message);
|
|
348
|
+
process.exit(1);
|
|
349
|
+
}
|
|
350
|
+
|
|
275
351
|
let dbSource: DbSource;
|
|
276
352
|
try {
|
|
277
353
|
dbSource = resolveDbSource(flags);
|
|
@@ -292,6 +368,10 @@ export async function dbReconcileCommand(flags: Record<string, string>): Promise
|
|
|
292
368
|
fail('--baseline only persists with --apply — on its own it records nothing (it silently no-op\'d before). Re-run with --apply, or drop --baseline to just see the plan (it already lists what needs baseline).');
|
|
293
369
|
process.exit(1);
|
|
294
370
|
}
|
|
371
|
+
if (flags.rebaseline === 'true' && !apply) {
|
|
372
|
+
fail('--rebaseline only persists with --apply — on its own it records nothing. Re-run with --apply, or drop --rebaseline to see the plan (source re-renders show as drop+create with reason "source changed").');
|
|
373
|
+
process.exit(1);
|
|
374
|
+
}
|
|
295
375
|
|
|
296
376
|
// --rebuild and --baseline are opposite answers to the same "can't verify first contact" question.
|
|
297
377
|
if (flags.rebuild === 'true' && flags.baseline === 'true') {
|
|
@@ -299,13 +379,18 @@ export async function dbReconcileCommand(flags: Record<string, string>): Promise
|
|
|
299
379
|
process.exit(1);
|
|
300
380
|
}
|
|
301
381
|
|
|
302
|
-
|
|
382
|
+
// The DECLARED derived layer (descriptors from the models barrel) — the one source
|
|
383
|
+
// stream — plus the pending table renames, so trigger provenance migrates.
|
|
384
|
+
let declared: DeclaredDerived | null = null;
|
|
303
385
|
try {
|
|
304
|
-
|
|
386
|
+
declared = await loadDeclaredDerived(flags.models);
|
|
305
387
|
} catch (err: any) {
|
|
306
388
|
fail(err.message);
|
|
307
389
|
process.exit(1);
|
|
308
390
|
}
|
|
391
|
+
if (declared && declared.objects.length > 0) {
|
|
392
|
+
info(`${declared.objects.length} declared derived object(s) from the models barrel.`);
|
|
393
|
+
}
|
|
309
394
|
|
|
310
395
|
let runner: QueryRunner;
|
|
311
396
|
let end: (() => Promise<void>) | undefined;
|
|
@@ -319,9 +404,12 @@ export async function dbReconcileCommand(flags: Record<string, string>): Promise
|
|
|
319
404
|
}
|
|
320
405
|
|
|
321
406
|
try {
|
|
322
|
-
const run = await executeReconcile(runner,
|
|
407
|
+
const run = await executeReconcile(runner, {
|
|
323
408
|
apply,
|
|
409
|
+
declared: declared?.objects,
|
|
410
|
+
renamedTables: declared?.renamedTables,
|
|
324
411
|
baseline: flags.baseline === 'true',
|
|
412
|
+
rebaseline: flags.rebaseline === 'true',
|
|
325
413
|
rebuild: flags.rebuild === 'true',
|
|
326
414
|
overwriteDrift: flags['overwrite-drift'] === 'true',
|
|
327
415
|
actor: process.env.USER ?? null,
|
|
@@ -337,7 +425,10 @@ export async function dbReconcileCommand(flags: Record<string, string>): Promise
|
|
|
337
425
|
} else if (run.applied) {
|
|
338
426
|
success(`Applied ${run.statements.length} statement(s); provenance and schema_log recorded.`);
|
|
339
427
|
if (run.plan.actions.some((a) => a.action === 'baseline')) {
|
|
340
|
-
warn('baseline recorded trust WITHOUT verifying live matches source — on first contact it CANNOT compare a source hash to a live deparse, so it trusts your assertion, it does not check. If you need a guarantee that live ==
|
|
428
|
+
warn('baseline recorded trust WITHOUT verifying live matches source — on first contact it CANNOT compare a source hash to a live deparse, so it trusts your assertion, it does not check. If you need a guarantee that live == the declared source, drop the object and let reconcile recreate it (or --overwrite-drift when the plan reports drift).');
|
|
429
|
+
}
|
|
430
|
+
if (run.plan.actions.some((a) => a.action === 'rebaseline')) {
|
|
431
|
+
warn('rebaseline VERIFIED the live side (each object is byte-identical to what the reconciler last applied) and TRUSTED the source side (that your re-rendered source produces the same object). If a descriptor actually differs — a changed body, a renamed index — that difference is now recorded as applied and will NOT deploy until the source changes again. When unsure, rebuild instead: drop the flag and let the plan drop+create.');
|
|
341
432
|
}
|
|
342
433
|
} else if (apply) {
|
|
343
434
|
success('Nothing to apply — derived layer matches source.');
|
|
@@ -2,15 +2,15 @@
|
|
|
2
2
|
* `everystack db:sync` — make the database match your checkout. One verb,
|
|
3
3
|
* both layers, for dev-stage databases (brick 5's core).
|
|
4
4
|
*
|
|
5
|
-
* db:sync [--database-url <url>] [--models <barrel>]
|
|
6
|
-
* [--allow-drops] [--overwrite-drift] [--baseline] [--json]
|
|
5
|
+
* db:sync [--database-url <url>] [--models <barrel>]
|
|
6
|
+
* [--allow-drops] [--overwrite-drift] [--baseline] [--rebaseline] [--json]
|
|
7
7
|
*
|
|
8
8
|
* The edit loop's verb: fingerprint the live base state, apply the state
|
|
9
9
|
* diff (tables + authz, one transaction, verified by re-diff), reconcile
|
|
10
|
-
* the derived layer
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
10
|
+
* the derived layer against the declared descriptors, and report where the
|
|
11
|
+
* database now is — "at fingerprint X, matching your checkout" when
|
|
12
|
+
* everything converged. Hot-reload for the database; no artifact produced,
|
|
13
|
+
* every act recorded in `everystack.schema_log`.
|
|
14
14
|
*
|
|
15
15
|
* The verdict is two-tier, honestly: CONVERGED means the state re-diff is
|
|
16
16
|
* empty and the compute layer is clean — that is the exit-0 bar. The
|
|
@@ -34,6 +34,8 @@ import { introspectContract, type QueryRunner } from '../authz-contract.js';
|
|
|
34
34
|
import { introspectSchema } from '../schema-introspect.js';
|
|
35
35
|
import { FUNCTIONS_SQL, contractFunctionRow } from '../security-catalog.js';
|
|
36
36
|
import { generateMigrationSql, unmodeledTables } from '../migration-generate.js';
|
|
37
|
+
import type { SequenceDescriptor } from '@everystack/model';
|
|
38
|
+
import type { SourceObject } from '../derived-source.js';
|
|
37
39
|
import {
|
|
38
40
|
applyStateAndVerify,
|
|
39
41
|
classifyGeneratedStatements,
|
|
@@ -50,10 +52,10 @@ import { planBackfills, readBackfillLog } from '../backfill.js';
|
|
|
50
52
|
import { resolveModelsPath } from '../models-path.js';
|
|
51
53
|
import { executeReconcile, buildReconcileReport, type ReconcileRun } from './db-reconcile.js';
|
|
52
54
|
import { loadModels } from './db-generate.js';
|
|
55
|
+
import { loadDeclaredDerived, retiredSqlDirAnywhere, retiredSqlDirFlagRefusal, type DeclaredDerived } from '../declared-derived.js';
|
|
53
56
|
import { reportPipelineLastRun } from './pipeline-run.js';
|
|
54
57
|
import { step, success, fail, info, warn } from '../output.js';
|
|
55
58
|
|
|
56
|
-
const DEFAULT_SQL_DIR = 'db/sql';
|
|
57
59
|
const DEFAULT_SCHEMA_OUT = 'db/schema.generated.ts';
|
|
58
60
|
|
|
59
61
|
// ---------------------------------------------------------------------------
|
|
@@ -64,8 +66,17 @@ export interface SyncOptions {
|
|
|
64
66
|
allowDrops?: boolean;
|
|
65
67
|
overwriteDrift?: boolean;
|
|
66
68
|
baseline?: boolean;
|
|
69
|
+
/** Re-record provenance for source re-renders over verifiably-untouched live objects
|
|
70
|
+
* (the db/sql-era migration) — see ReconcileOptions.rebaseline. */
|
|
71
|
+
rebaseline?: boolean;
|
|
67
72
|
actor?: string | null;
|
|
68
73
|
gitRef?: string | null;
|
|
74
|
+
/** Descriptor-compiled derived objects (the declared layer) — the one compute stream. */
|
|
75
|
+
declared?: SourceObject[];
|
|
76
|
+
/** Standalone sequences the modules declare (state — created before tables, fingerprinted). */
|
|
77
|
+
sequences?: SequenceDescriptor[];
|
|
78
|
+
/** Pending table renames — trigger provenance migrates instead of drop+create. */
|
|
79
|
+
renamedTables?: Record<string, string>;
|
|
69
80
|
/** Injectable clock for tests. */
|
|
70
81
|
now?: () => number;
|
|
71
82
|
}
|
|
@@ -79,7 +90,7 @@ export interface SyncHooks {
|
|
|
79
90
|
|
|
80
91
|
export interface SyncRun {
|
|
81
92
|
state: StateSyncOutcome;
|
|
82
|
-
/** null = no
|
|
93
|
+
/** null = no declared derived layer — the compute step is skipped. */
|
|
83
94
|
compute: ReconcileRun | null;
|
|
84
95
|
declaredFingerprint: string;
|
|
85
96
|
fingerprintMatch: boolean;
|
|
@@ -100,31 +111,34 @@ export interface SyncRun {
|
|
|
100
111
|
export async function executeSync(
|
|
101
112
|
runner: QueryRunner,
|
|
102
113
|
models: ModelDescriptor[],
|
|
103
|
-
sources: SourceFile[] | null,
|
|
104
114
|
options: SyncOptions = {},
|
|
105
115
|
hooks: SyncHooks = {},
|
|
106
116
|
): Promise<SyncRun> {
|
|
107
117
|
const current = await introspectSchema(runner);
|
|
108
118
|
const liveAuthz = await introspectContract(runner, contractFunctionRow, FUNCTIONS_SQL);
|
|
109
119
|
const statements = generateMigrationSql(models, current, {
|
|
110
|
-
allowDrops: options.allowDrops, liveAuthz,
|
|
120
|
+
allowDrops: options.allowDrops, liveAuthz, sequences: options.sequences,
|
|
111
121
|
});
|
|
112
122
|
hooks.onStatePlan?.(statements, classifyGeneratedStatements(statements));
|
|
113
123
|
|
|
114
124
|
const state = await applyStateAndVerify(runner, models, statements, current, liveAuthz, {
|
|
115
|
-
allowDrops: options.allowDrops,
|
|
125
|
+
allowDrops: options.allowDrops, sequences: options.sequences,
|
|
116
126
|
actor: options.actor, gitRef: options.gitRef, now: options.now,
|
|
117
127
|
});
|
|
118
128
|
hooks.onStateDone?.(state);
|
|
119
129
|
|
|
120
|
-
|
|
130
|
+
// The declared derived layer is the one compute stream; nothing declared = skipped.
|
|
131
|
+
const compute = !options.declared?.length ? null : await executeReconcile(runner, {
|
|
121
132
|
apply: true,
|
|
122
133
|
baseline: options.baseline,
|
|
134
|
+
rebaseline: options.rebaseline,
|
|
123
135
|
overwriteDrift: options.overwriteDrift,
|
|
136
|
+
declared: options.declared,
|
|
137
|
+
renamedTables: options.renamedTables,
|
|
124
138
|
actor: options.actor, gitRef: options.gitRef, now: options.now,
|
|
125
139
|
});
|
|
126
140
|
|
|
127
|
-
const declaredFingerprint = fingerprintModels(models).hash;
|
|
141
|
+
const declaredFingerprint = fingerprintModels(models, { sequences: options.sequences }).hash;
|
|
128
142
|
const stateConverged = state.remaining.length === 0;
|
|
129
143
|
// The compute bar, judged AFTER the apply: no refusal (drift/blocked stop the
|
|
130
144
|
// whole batch) and nothing left unadopted (needsBaseline objects were not
|
|
@@ -182,7 +196,7 @@ export function buildSyncReport(run: SyncRun): string[] {
|
|
|
182
196
|
|
|
183
197
|
// Compute.
|
|
184
198
|
if (compute === null) {
|
|
185
|
-
lines.push('· compute: no
|
|
199
|
+
lines.push('· compute: no derived layer declared — nothing to reconcile');
|
|
186
200
|
} else {
|
|
187
201
|
lines.push('compute:');
|
|
188
202
|
for (const line of buildReconcileReport(compute.plan)) lines.push(` ${line}`);
|
|
@@ -250,21 +264,39 @@ export async function dbSyncCommand(flags: Record<string, string>): Promise<void
|
|
|
250
264
|
const modelsPath = resolveModelsPath(flags.models);
|
|
251
265
|
const schemaOut = path.resolve(flags['schema-out'] || DEFAULT_SCHEMA_OUT);
|
|
252
266
|
let models: ModelDescriptor[];
|
|
267
|
+
// The declared derived layer + sequences + renames — the same stream db:reconcile
|
|
268
|
+
// consumes. Loaded before the schema write: typed derived relations emit into it.
|
|
269
|
+
let declaredDb: DeclaredDerived | null = null;
|
|
253
270
|
try {
|
|
254
271
|
step(`Loading models from ${modelsPath}...`);
|
|
255
272
|
models = await loadModels(modelsPath);
|
|
273
|
+
declaredDb = await loadDeclaredDerived(flags.models);
|
|
256
274
|
info(`${models.length} model(s).`);
|
|
257
275
|
// Same contract as db:generate: the drizzle runtime schema is a derived
|
|
258
276
|
// artifact, refreshed from the models on every run.
|
|
259
277
|
step(`Writing the drizzle schema → ${path.relative(process.cwd(), schemaOut)}...`);
|
|
260
|
-
await fs.writeFile(schemaOut, compileDrizzleSource(models), 'utf8');
|
|
278
|
+
await fs.writeFile(schemaOut, compileDrizzleSource(models, { derived: declaredDb?.derived }), 'utf8');
|
|
261
279
|
} catch (err: any) {
|
|
262
280
|
fail(err.message);
|
|
263
281
|
process.exit(1);
|
|
264
282
|
}
|
|
265
283
|
|
|
266
|
-
|
|
267
|
-
|
|
284
|
+
// B7 tombstone — db/sql is retired and no longer read; failing beats silently
|
|
285
|
+
// skipping (skipped files would leave live objects unmanaged).
|
|
286
|
+
try {
|
|
287
|
+
if (flags['sql-dir']) {
|
|
288
|
+
fail(await retiredSqlDirFlagRefusal(flags['sql-dir']));
|
|
289
|
+
process.exit(1);
|
|
290
|
+
}
|
|
291
|
+
const retired = await retiredSqlDirAnywhere(flags.models);
|
|
292
|
+
if (retired) {
|
|
293
|
+
fail(retired);
|
|
294
|
+
process.exit(1);
|
|
295
|
+
}
|
|
296
|
+
} catch (err: any) {
|
|
297
|
+
fail(err.message);
|
|
298
|
+
process.exit(1);
|
|
299
|
+
}
|
|
268
300
|
|
|
269
301
|
step(connectingVia(dbSource));
|
|
270
302
|
const { runner, end } = await createUrlRunner(dbSource.url);
|
|
@@ -273,11 +305,14 @@ export async function dbSyncCommand(flags: Record<string, string>): Promise<void
|
|
|
273
305
|
const run = await executeSync(
|
|
274
306
|
runner,
|
|
275
307
|
models,
|
|
276
|
-
sources,
|
|
277
308
|
{
|
|
309
|
+
declared: declaredDb?.objects,
|
|
310
|
+
sequences: declaredDb?.sequences,
|
|
311
|
+
renamedTables: declaredDb?.renamedTables,
|
|
278
312
|
allowDrops: flags['allow-drops'] === 'true',
|
|
279
313
|
overwriteDrift: flags['overwrite-drift'] === 'true',
|
|
280
314
|
baseline: flags.baseline === 'true',
|
|
315
|
+
rebaseline: flags.rebaseline === 'true',
|
|
281
316
|
actor: process.env.USER ?? null,
|
|
282
317
|
gitRef: currentGitRef(),
|
|
283
318
|
},
|
|
@@ -296,7 +331,7 @@ export async function dbSyncCommand(flags: Record<string, string>): Promise<void
|
|
|
296
331
|
}
|
|
297
332
|
},
|
|
298
333
|
onStateDone: () => {
|
|
299
|
-
if (
|
|
334
|
+
if (declaredDb?.objects.length) step('Reconciling the declared derived layer...');
|
|
300
335
|
},
|
|
301
336
|
},
|
|
302
337
|
);
|
package/src/cli/commands/db.ts
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { resolveConfig, opsFunction } from '../config.js';
|
|
9
9
|
import { invokeAction } from '../aws.js';
|
|
10
10
|
import { step, success, fail, info, warn } from '../output.js';
|
|
11
|
+
import { opsAdviceLines, IAM_ADVICE } from '../ops-advice.js';
|
|
11
12
|
|
|
12
13
|
/** Secret-store keys that may hold the privileged (operator) connection URL,
|
|
13
14
|
* in precedence order. Names vary by deployment; --secret-name overrides. */
|
|
@@ -69,7 +70,7 @@ export async function dbMigrateCommand(flags: Record<string, string>): Promise<v
|
|
|
69
70
|
}
|
|
70
71
|
} catch (err: any) {
|
|
71
72
|
fail(`Migration failed: ${err.message}`);
|
|
72
|
-
|
|
73
|
+
for (const line of opsAdviceLines(err, [IAM_ADVICE])) info(line);
|
|
73
74
|
process.exit(1);
|
|
74
75
|
}
|
|
75
76
|
}
|
|
@@ -105,7 +106,7 @@ export async function dbSeedCommand(flags: Record<string, string>): Promise<void
|
|
|
105
106
|
}
|
|
106
107
|
} catch (err: any) {
|
|
107
108
|
fail(`Seed failed: ${err.message}`);
|
|
108
|
-
|
|
109
|
+
for (const line of opsAdviceLines(err, [IAM_ADVICE])) info(line);
|
|
109
110
|
process.exit(1);
|
|
110
111
|
}
|
|
111
112
|
}
|
|
@@ -138,7 +139,7 @@ export async function dbResetCommand(flags: Record<string, string>): Promise<voi
|
|
|
138
139
|
success(result?.message || 'Database reset and migrations applied');
|
|
139
140
|
} catch (err: any) {
|
|
140
141
|
fail(`Reset failed: ${err.message}`);
|
|
141
|
-
|
|
142
|
+
for (const line of opsAdviceLines(err, [IAM_ADVICE])) info(line);
|
|
142
143
|
process.exit(1);
|
|
143
144
|
}
|
|
144
145
|
}
|
|
@@ -172,7 +173,7 @@ export async function dbProvisionCommand(flags: Record<string, string>): Promise
|
|
|
172
173
|
result = await invokeAction(config.region, opsFunction(config), 'db:provision', { authPassword });
|
|
173
174
|
} catch (err: any) {
|
|
174
175
|
fail(`db:provision failed: ${err.message}`);
|
|
175
|
-
|
|
176
|
+
for (const line of opsAdviceLines(err, [IAM_ADVICE])) info(line);
|
|
176
177
|
process.exit(1);
|
|
177
178
|
}
|
|
178
179
|
if (result?.error) {
|
|
@@ -236,7 +237,7 @@ export async function dbDoctorCommand(flags: Record<string, string>): Promise<vo
|
|
|
236
237
|
report = await invokeAction(config.region, opsFunction(config), 'db:doctor', {});
|
|
237
238
|
} catch (err: any) {
|
|
238
239
|
fail(`db:doctor failed: ${err.message}`);
|
|
239
|
-
|
|
240
|
+
for (const line of opsAdviceLines(err, [IAM_ADVICE])) info(line);
|
|
240
241
|
process.exit(1);
|
|
241
242
|
}
|
|
242
243
|
|
|
@@ -327,7 +328,7 @@ export async function dbPsqlCommand(flags: Record<string, string>): Promise<void
|
|
|
327
328
|
process.exit(0);
|
|
328
329
|
} catch (err: any) {
|
|
329
330
|
fail(`Query failed: ${err.message}`);
|
|
330
|
-
|
|
331
|
+
for (const line of opsAdviceLines(err, [IAM_ADVICE])) info(line);
|
|
331
332
|
process.exit(1);
|
|
332
333
|
}
|
|
333
334
|
}
|
|
@@ -8,6 +8,7 @@ import { spawn } from 'node:child_process';
|
|
|
8
8
|
import { resolveConfig, type CliConfig } from '../config.js';
|
|
9
9
|
import { uploadToS3, headS3, invokeAction } from '../aws.js';
|
|
10
10
|
import { step, success, warn, fail, info } from '../output.js';
|
|
11
|
+
import { opsAdviceLines, IAM_ADVICE } from '../ops-advice.js';
|
|
11
12
|
import { exportApp } from '../utils/export.js';
|
|
12
13
|
import { walkDirectory } from '../utils/walk.js';
|
|
13
14
|
import { loadSstSecrets } from '../utils/secrets.js';
|
|
@@ -301,7 +302,7 @@ export async function updateCommand(flags: UpdateFlags & Record<string, string>)
|
|
|
301
302
|
}
|
|
302
303
|
} catch (err: any) {
|
|
303
304
|
fail(`Registration failed for ${platform}: ${err.message}`);
|
|
304
|
-
|
|
305
|
+
for (const line of opsAdviceLines(err, [IAM_ADVICE])) info(line);
|
|
305
306
|
process.exit(1);
|
|
306
307
|
}
|
|
307
308
|
}
|
package/src/cli/db-build.ts
CHANGED
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
|
|
13
13
|
import type { ModelDescriptor } from '@everystack/model';
|
|
14
14
|
import type { TableContract, QueryRunner } from './authz-contract.js';
|
|
15
|
-
import type {
|
|
15
|
+
import type { SourceObject } from './derived-source.js';
|
|
16
|
+
import type { SequenceDescriptor } from '@everystack/model';
|
|
16
17
|
import { compileDeclaredState } from './declared-diff.js';
|
|
17
18
|
import { createUrlRunner } from './db-source.js';
|
|
18
19
|
import { executeSync, buildSyncReport } from './commands/db-sync.js';
|
|
@@ -88,7 +89,10 @@ export interface BuildResult {
|
|
|
88
89
|
}
|
|
89
90
|
|
|
90
91
|
export interface BuildOptions {
|
|
91
|
-
|
|
92
|
+
/** Descriptor-compiled derived objects (the declared layer). */
|
|
93
|
+
declared?: SourceObject[];
|
|
94
|
+
/** Standalone sequences (state — created before tables, in the fingerprint bar). */
|
|
95
|
+
sequences?: SequenceDescriptor[];
|
|
92
96
|
actor?: string | null;
|
|
93
97
|
gitRef?: string | null;
|
|
94
98
|
}
|
|
@@ -106,7 +110,9 @@ export async function buildIntoDatabase(
|
|
|
106
110
|
const { runner, end } = await createUrlRunner(url);
|
|
107
111
|
try {
|
|
108
112
|
const createdRoles = await ensureContractRoles(runner, models);
|
|
109
|
-
const run = await executeSync(runner, models,
|
|
113
|
+
const run = await executeSync(runner, models, {
|
|
114
|
+
declared: options.declared,
|
|
115
|
+
sequences: options.sequences,
|
|
110
116
|
actor: options.actor ?? 'db-build',
|
|
111
117
|
gitRef: options.gitRef ?? null,
|
|
112
118
|
});
|
package/src/cli/db-source.ts
CHANGED
|
@@ -83,7 +83,11 @@ export async function createUrlRunner(
|
|
|
83
83
|
);
|
|
84
84
|
}
|
|
85
85
|
const postgres = mod.default ?? mod;
|
|
86
|
-
|
|
86
|
+
// max_lifetime: null — the driver's default recycles a connection after a random 30–60
|
|
87
|
+
// minutes, resolving the in-flight query and THEN killing the session. Under reconcile's
|
|
88
|
+
// BEGIN-across-calls transaction that is a silent session swap mid-batch (the reconcile
|
|
89
|
+
// apply path also pins the backend pid and refuses bookkeeping if it moved).
|
|
90
|
+
const sql = postgres(url, { max: 1, max_lifetime: null, onnotice: () => {}, ...sslDefaults(url) });
|
|
87
91
|
return {
|
|
88
92
|
runner: async (query: string) => Array.from(await sql.unsafe(query)),
|
|
89
93
|
end: () => sql.end({ timeout: 5 }),
|