@everystack/cli 0.3.28 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +6 -2
- package/src/cli/apply-execute.ts +185 -0
- package/src/cli/authz-compile.ts +5 -2
- package/src/cli/authz-lint.ts +48 -0
- package/src/cli/aws.ts +102 -4
- package/src/cli/backfill.ts +1 -1
- package/src/cli/commands/db-apply.ts +159 -169
- package/src/cli/commands/db-backfill.ts +2 -2
- package/src/cli/commands/db-backup.ts +34 -0
- package/src/cli/commands/db-branch.ts +28 -4
- package/src/cli/commands/db-check.ts +97 -34
- package/src/cli/commands/db-fingerprint.ts +9 -4
- package/src/cli/commands/db-generate.ts +73 -22
- package/src/cli/commands/db-plan.ts +7 -13
- package/src/cli/commands/db-pull.ts +39 -7
- package/src/cli/commands/db-reconcile.ts +219 -66
- package/src/cli/commands/db-sync.ts +58 -21
- package/src/cli/commands/deploy.ts +4 -0
- package/src/cli/commands/pipeline-run.ts +269 -0
- package/src/cli/db-build.ts +9 -3
- package/src/cli/db-source.ts +83 -7
- package/src/cli/declared-derived.ts +136 -0
- package/src/cli/derived-apply.ts +40 -6
- package/src/cli/derived-compile.ts +327 -0
- package/src/cli/derived-grants.ts +96 -0
- package/src/cli/derived-introspect.ts +258 -21
- package/src/cli/derived-lint.ts +261 -0
- package/src/cli/derived-plan.ts +194 -12
- package/src/cli/derived-render.ts +320 -0
- package/src/cli/derived-source.ts +53 -125
- package/src/cli/discover.ts +16 -0
- package/src/cli/git-descent.ts +15 -2
- package/src/cli/index.ts +26 -6
- package/src/cli/migration-compile.ts +38 -7
- package/src/cli/migration-generate.ts +43 -4
- package/src/cli/model-render.ts +106 -13
- package/src/cli/models-path.ts +1 -1
- package/src/cli/ops-diagnostics.ts +71 -0
- package/src/cli/pipeline-loader.ts +68 -0
- package/src/cli/pipeline-path.ts +25 -0
- package/src/cli/schema-compile.ts +41 -10
- package/src/cli/schema-diff.ts +123 -17
- package/src/cli/schema-fingerprint.ts +28 -18
- package/src/cli/schema-introspect.ts +175 -8
- package/src/cli/schema-source.ts +75 -6
- package/src/cli/state-apply.ts +4 -1
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* invariant: the merged declared state must compose, and generated
|
|
4
4
|
* artifacts must match regeneration).
|
|
5
5
|
*
|
|
6
|
-
* db:check [--models <barrel>] [--
|
|
6
|
+
* db:check [--models <barrel>] [--schema-out <file.ts>]
|
|
7
7
|
* [--database-url <url>] [--json]
|
|
8
8
|
*
|
|
9
9
|
* Two rings, both from the checkout alone (no target database is ever
|
|
@@ -13,7 +13,8 @@
|
|
|
13
13
|
* git can't see, like two branches declaring the same table, fails
|
|
14
14
|
* here); the generated drizzle schema matches a byte-for-byte
|
|
15
15
|
* regeneration (hand-edited artifacts and forgotten `db:generate` runs
|
|
16
|
-
* fail loudly);
|
|
16
|
+
* fail loudly); a leftover db/sql directory fails with the migration
|
|
17
|
+
* path (B7 — descriptors are the single home).
|
|
17
18
|
*
|
|
18
19
|
* EPHEMERAL COMPOSE — with a scratch PostgreSQL (`--database-url` /
|
|
19
20
|
* DATABASE_URL), build the merged declared state FROM SCRATCH on an
|
|
@@ -28,23 +29,25 @@
|
|
|
28
29
|
*/
|
|
29
30
|
|
|
30
31
|
import fs from 'node:fs/promises';
|
|
31
|
-
import type { ModelDescriptor } from '@everystack/model';
|
|
32
|
+
import type { ModelDescriptor, DerivedDescriptor } from '@everystack/model';
|
|
32
33
|
import { compileDeclaredState } from '../declared-diff.js';
|
|
33
34
|
import { compileMigration } from '../migration-compile.js';
|
|
34
35
|
import { compileDrizzleSource } from '../schema-source.js';
|
|
35
|
-
import { parseSqlSources, type SourceFile } from '../derived-source.js';
|
|
36
36
|
import { currentGitRef } from '../state-apply.js';
|
|
37
37
|
import { createDatabase, dropDatabase, buildIntoDatabase, withDatabase } from '../db-build.js';
|
|
38
38
|
import { resolveModelsPath } from '../models-path.js';
|
|
39
|
-
import { readSqlDirIfPresent } from './db-sync.js';
|
|
40
39
|
import { loadModels } from './db-generate.js';
|
|
40
|
+
import { loadDeclaredDerived, retiredSqlDirAnywhere, retiredSqlDirFlagRefusal, type DeclaredDerived } from '../declared-derived.js';
|
|
41
|
+
import type { SequenceDescriptor } from '@everystack/model';
|
|
42
|
+
import type { SourceObject } from '../derived-source.js';
|
|
43
|
+
import { findReadAuthzGaps } from '../authz-lint.js';
|
|
44
|
+
import { findDerivedReadGaps, findSecdefExecuteGaps, findMatviewSnapshotWarnings } from '../derived-lint.js';
|
|
41
45
|
import { step, success, fail, info, warn } from '../output.js';
|
|
42
46
|
|
|
43
47
|
// The role derivation moved to the shared builder (db-build) with brick
|
|
44
48
|
// 5-remainder; re-exported here because db:check is where it grew up.
|
|
45
49
|
export { rolesFromContract } from '../db-build.js';
|
|
46
50
|
|
|
47
|
-
const DEFAULT_SQL_DIR = 'db/sql';
|
|
48
51
|
const DEFAULT_SCHEMA_OUT = 'db/schema.generated.ts';
|
|
49
52
|
|
|
50
53
|
// ---------------------------------------------------------------------------
|
|
@@ -52,8 +55,9 @@ const DEFAULT_SCHEMA_OUT = 'db/schema.generated.ts';
|
|
|
52
55
|
// ---------------------------------------------------------------------------
|
|
53
56
|
|
|
54
57
|
export interface CheckFinding {
|
|
55
|
-
|
|
56
|
-
|
|
58
|
+
/** `warn` is visible-but-never-fatal — the matview snapshot gate's channel. */
|
|
59
|
+
level: 'ok' | 'note' | 'warn' | 'fail';
|
|
60
|
+
area: 'models' | 'compose' | 'artifact' | 'compute' | 'authz';
|
|
57
61
|
message: string;
|
|
58
62
|
}
|
|
59
63
|
|
|
@@ -65,8 +69,15 @@ export interface StaticCheckInput {
|
|
|
65
69
|
artifactPath: string;
|
|
66
70
|
/** null = no generated schema artifact on disk. */
|
|
67
71
|
artifactSource: string | null;
|
|
68
|
-
/**
|
|
69
|
-
|
|
72
|
+
/** Set when a retired db/sql directory still carries .sql files — the tombstone
|
|
73
|
+
* message (with the migration path); a hard fail, never a shrug. */
|
|
74
|
+
sqlDirRetired?: string | null;
|
|
75
|
+
/** Standalone sequences the modules declare (state — in the from-scratch compose). */
|
|
76
|
+
sequences?: SequenceDescriptor[];
|
|
77
|
+
/** The declared derived descriptors (raw — the B2 gates read abilities, not SQL). */
|
|
78
|
+
derived?: DerivedDescriptor[] | null;
|
|
79
|
+
/** Set when the derived layer failed to COMPILE (e.g. reachability) — a hard fail. */
|
|
80
|
+
derivedError?: string;
|
|
70
81
|
}
|
|
71
82
|
|
|
72
83
|
export function runStaticChecks(input: StaticCheckInput): CheckFinding[] {
|
|
@@ -97,20 +108,61 @@ export function runStaticChecks(input: StaticCheckInput): CheckFinding[] {
|
|
|
97
108
|
}
|
|
98
109
|
try {
|
|
99
110
|
compileDeclaredState(input.models);
|
|
100
|
-
const statements = compileMigration(input.models);
|
|
111
|
+
const statements = compileMigration(input.models, { sequences: input.sequences });
|
|
101
112
|
findings.push({ level: 'ok', area: 'compose', message: `declared state composes — ${statements.length} statement(s) from scratch` });
|
|
102
113
|
} catch (err: any) {
|
|
103
114
|
findings.push({ level: 'fail', area: 'compose', message: `the merged declared state does NOT compose: ${err.message}` });
|
|
104
115
|
return findings;
|
|
105
116
|
}
|
|
106
117
|
|
|
118
|
+
// Force-RLS with no read authz: an exposed, RLS-enabled table that declares no read ability reads
|
|
119
|
+
// empty for every app role once it is RLS-subject (the superuser-drop landmine).
|
|
120
|
+
const authzGaps = findReadAuthzGaps(input.models);
|
|
121
|
+
if (authzGaps.length === 0) {
|
|
122
|
+
findings.push({ level: 'ok', area: 'authz', message: 'every exposed table declares a read path — none goes dark under RLS' });
|
|
123
|
+
} else {
|
|
124
|
+
for (const gap of authzGaps) {
|
|
125
|
+
findings.push({ level: 'fail', area: 'authz', message: gap.message });
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// The derived layer failing to COMPILE (reachability, cycles, undeclared trigger
|
|
130
|
+
// functions) is a hard fail — CI cannot shrug at a compile error.
|
|
131
|
+
if (input.derivedError) {
|
|
132
|
+
findings.push({
|
|
133
|
+
level: 'fail',
|
|
134
|
+
area: 'compute',
|
|
135
|
+
message: `the declared derived layer does NOT compile: ${input.derivedError}`,
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// The B2 gates over the declared derived layer: every relation makes its read decision
|
|
140
|
+
// (fail), every SECURITY DEFINER declares its callers (fail), and a matview over
|
|
141
|
+
// row-scoped sources is named for what it is — a snapshot with the refresher's eyes (warn).
|
|
142
|
+
if (input.derived?.length) {
|
|
143
|
+
const derivedGaps = [...findDerivedReadGaps(input.derived), ...findSecdefExecuteGaps(input.derived)];
|
|
144
|
+
for (const gap of derivedGaps) {
|
|
145
|
+
findings.push({ level: 'fail', area: 'authz', message: gap.message });
|
|
146
|
+
}
|
|
147
|
+
for (const warning of findMatviewSnapshotWarnings(input.derived)) {
|
|
148
|
+
findings.push({ level: 'warn', area: 'authz', message: warning.message });
|
|
149
|
+
}
|
|
150
|
+
if (derivedGaps.length === 0) {
|
|
151
|
+
findings.push({
|
|
152
|
+
level: 'ok',
|
|
153
|
+
area: 'authz',
|
|
154
|
+
message: `every derived relation declares its read surface and every SECURITY DEFINER its callers (${input.derived.length} descriptor(s))`,
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
107
159
|
if (input.artifactSource === null) {
|
|
108
160
|
findings.push({
|
|
109
161
|
level: 'note',
|
|
110
162
|
area: 'artifact',
|
|
111
163
|
message: `no generated schema at ${input.artifactPath} — nothing to match (db:generate writes it)`,
|
|
112
164
|
});
|
|
113
|
-
} else if (input.artifactSource === compileDrizzleSource(input.models)) {
|
|
165
|
+
} else if (input.artifactSource === compileDrizzleSource(input.models, { derived: input.derived ?? undefined })) {
|
|
114
166
|
findings.push({ level: 'ok', area: 'artifact', message: `${input.artifactPath} matches regeneration byte-for-byte` });
|
|
115
167
|
} else {
|
|
116
168
|
findings.push({
|
|
@@ -120,20 +172,10 @@ export function runStaticChecks(input: StaticCheckInput): CheckFinding[] {
|
|
|
120
172
|
});
|
|
121
173
|
}
|
|
122
174
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
const parsed = parseSqlSources(input.sources);
|
|
128
|
-
findings.push({
|
|
129
|
-
level: 'ok',
|
|
130
|
-
area: 'compute',
|
|
131
|
-
message: `${parsed.objects.length} derived object(s) parsed from ${input.sources.length} db/sql file(s)`,
|
|
132
|
-
});
|
|
133
|
-
for (const w of parsed.warnings) findings.push({ level: 'note', area: 'compute', message: w });
|
|
134
|
-
} catch (err: any) {
|
|
135
|
-
findings.push({ level: 'fail', area: 'compute', message: `db/sql does not parse: ${err.message}` });
|
|
136
|
-
}
|
|
175
|
+
// B7 — db/sql is retired. Files still present would be silently unmanaged: fail
|
|
176
|
+
// with the migration path. (Absence is the normal state; nothing to report.)
|
|
177
|
+
if (input.sqlDirRetired) {
|
|
178
|
+
findings.push({ level: 'fail', area: 'compute', message: input.sqlDirRetired });
|
|
137
179
|
}
|
|
138
180
|
|
|
139
181
|
return findings;
|
|
@@ -165,14 +207,14 @@ export interface EphemeralComposeResult {
|
|
|
165
207
|
export async function executeEphemeralCompose(
|
|
166
208
|
adminUrl: string,
|
|
167
209
|
models: ModelDescriptor[],
|
|
168
|
-
|
|
169
|
-
opts: { actor?: string | null; gitRef?: string | null } = {},
|
|
210
|
+
opts: { actor?: string | null; gitRef?: string | null; declared?: SourceObject[]; sequences?: SequenceDescriptor[] } = {},
|
|
170
211
|
): Promise<EphemeralComposeResult> {
|
|
171
212
|
const database = `escheck_${process.pid}_${Date.now()}`;
|
|
172
213
|
await createDatabase(adminUrl, database);
|
|
173
214
|
try {
|
|
174
215
|
const built = await buildIntoDatabase(withDatabase(adminUrl, database), models, {
|
|
175
|
-
|
|
216
|
+
declared: opts.declared,
|
|
217
|
+
sequences: opts.sequences,
|
|
176
218
|
actor: opts.actor ?? 'db:check',
|
|
177
219
|
gitRef: opts.gitRef ?? null,
|
|
178
220
|
});
|
|
@@ -186,7 +228,7 @@ export async function executeEphemeralCompose(
|
|
|
186
228
|
// The CLI shell.
|
|
187
229
|
// ---------------------------------------------------------------------------
|
|
188
230
|
|
|
189
|
-
const MARK: Record<CheckFinding['level'], string> = { ok: '=', note: '·', fail: '!' };
|
|
231
|
+
const MARK: Record<CheckFinding['level'], string> = { ok: '=', note: '·', warn: '~', fail: '!' };
|
|
190
232
|
|
|
191
233
|
export async function dbCheckCommand(flags: Record<string, string>): Promise<void> {
|
|
192
234
|
const modelsPath = resolveModelsPath(flags.models);
|
|
@@ -206,11 +248,30 @@ export async function dbCheckCommand(flags: Record<string, string>): Promise<voi
|
|
|
206
248
|
} catch {
|
|
207
249
|
artifactSource = null;
|
|
208
250
|
}
|
|
209
|
-
|
|
251
|
+
// B7 tombstone — a retired db/sql directory still carrying .sql files is a FAIL
|
|
252
|
+
// finding (the message carries the migration path), never a silent skip. The retired
|
|
253
|
+
// --sql-dir flag is itself a FAIL finding here (the sibling verbs refuse it outright);
|
|
254
|
+
// detector errors (an unreadable directory) surface as the finding, never disarm it.
|
|
255
|
+
const sqlDirRetired = flags['sql-dir']
|
|
256
|
+
? await retiredSqlDirFlagRefusal(flags['sql-dir'])
|
|
257
|
+
: await retiredSqlDirAnywhere(flags.models).catch((err: any) => String(err?.message ?? err));
|
|
258
|
+
|
|
259
|
+
// The declared derived layer + sequences — the same stream every other verb consumes.
|
|
260
|
+
// A compile failure here (reachability, cycles) is a FAIL finding, not a shrug.
|
|
261
|
+
let declaredDb: DeclaredDerived | null = null;
|
|
262
|
+
let derivedError: string | undefined;
|
|
263
|
+
try {
|
|
264
|
+
declaredDb = await loadDeclaredDerived(flags.models);
|
|
265
|
+
} catch (err: any) {
|
|
266
|
+
derivedError = err.message;
|
|
267
|
+
}
|
|
210
268
|
|
|
211
|
-
const findings = runStaticChecks({
|
|
269
|
+
const findings = runStaticChecks({
|
|
270
|
+
modelsPath, models, modelsError, artifactPath, artifactSource, sqlDirRetired,
|
|
271
|
+
sequences: declaredDb?.sequences, derived: declaredDb?.derived, derivedError,
|
|
272
|
+
});
|
|
212
273
|
for (const f of findings) {
|
|
213
|
-
(f.level === 'fail' ? warn : info)(`${MARK[f.level]} ${f.area}: ${f.message}`);
|
|
274
|
+
(f.level === 'fail' || f.level === 'warn' ? warn : info)(`${MARK[f.level]} ${f.area}: ${f.message}`);
|
|
214
275
|
}
|
|
215
276
|
|
|
216
277
|
let ephemeral: EphemeralComposeResult | null = null;
|
|
@@ -224,9 +285,11 @@ export async function dbCheckCommand(flags: Record<string, string>): Promise<voi
|
|
|
224
285
|
} else {
|
|
225
286
|
step('Composing the declared state from scratch on an ephemeral database...');
|
|
226
287
|
try {
|
|
227
|
-
ephemeral = await executeEphemeralCompose(url, models!,
|
|
288
|
+
ephemeral = await executeEphemeralCompose(url, models!, {
|
|
228
289
|
actor: process.env.USER ?? 'db:check',
|
|
229
290
|
gitRef: currentGitRef(),
|
|
291
|
+
declared: declaredDb?.objects,
|
|
292
|
+
sequences: declaredDb?.sequences,
|
|
230
293
|
});
|
|
231
294
|
for (const line of ephemeral.report) info(` ${line}`);
|
|
232
295
|
} catch (err: any) {
|
|
@@ -25,11 +25,13 @@ import {
|
|
|
25
25
|
type UnfingerprintedObject,
|
|
26
26
|
} from '../schema-fingerprint.js';
|
|
27
27
|
import { resolveModelsPath } from '../models-path.js';
|
|
28
|
-
import { resolveDbSource, createUrlRunner, type DbSource } from '../db-source.js';
|
|
28
|
+
import { resolveDbSource, createUrlRunner, connectingVia, type DbSource } from '../db-source.js';
|
|
29
29
|
import { resolveConfig, opsFunction } from '../config.js';
|
|
30
30
|
import { invokeAction } from '../aws.js';
|
|
31
31
|
import { step, success, fail, info, warn } from '../output.js';
|
|
32
32
|
import { loadModels } from './db-generate.js';
|
|
33
|
+
import { loadDeclaredDerived } from '../declared-derived.js';
|
|
34
|
+
import type { SequenceDescriptor } from '@everystack/model';
|
|
33
35
|
|
|
34
36
|
export interface FingerprintStatus {
|
|
35
37
|
live: string;
|
|
@@ -42,11 +44,12 @@ export interface FingerprintStatus {
|
|
|
42
44
|
export async function computeFingerprintStatus(
|
|
43
45
|
runner: QueryRunner,
|
|
44
46
|
models: ModelDescriptor[] | null,
|
|
47
|
+
sequences?: SequenceDescriptor[],
|
|
45
48
|
): Promise<FingerprintStatus> {
|
|
46
49
|
const snapshot = await introspectSchema(runner);
|
|
47
50
|
const contract = await introspectContract(runner, contractFunctionRow, FUNCTIONS_SQL);
|
|
48
51
|
const live = fingerprintLive(snapshot, contract).hash;
|
|
49
|
-
const declared = models ? fingerprintModels(models).hash : null;
|
|
52
|
+
const declared = models ? fingerprintModels(models, { sequences }).hash : null;
|
|
50
53
|
const unfingerprinted = mapUnfingerprintedRows((await runner(UNFINGERPRINTED_SQL)) as any[]);
|
|
51
54
|
return { live, declared, match: declared === null ? null : live === declared, unfingerprinted };
|
|
52
55
|
}
|
|
@@ -71,8 +74,10 @@ export async function dbFingerprintCommand(flags: Record<string, string>): Promi
|
|
|
71
74
|
|
|
72
75
|
const modelsPath = resolveModelsPath(flags.models);
|
|
73
76
|
let models: ModelDescriptor[] | null = null;
|
|
77
|
+
let sequences: SequenceDescriptor[] | undefined;
|
|
74
78
|
try {
|
|
75
79
|
models = await loadModels(modelsPath);
|
|
80
|
+
sequences = (await loadDeclaredDerived(flags.models))?.sequences;
|
|
76
81
|
} catch {
|
|
77
82
|
// Live-only mode: no models barrel — still useful (what fingerprint is this DB at?).
|
|
78
83
|
}
|
|
@@ -80,7 +85,7 @@ export async function dbFingerprintCommand(flags: Record<string, string>): Promi
|
|
|
80
85
|
let runner: QueryRunner;
|
|
81
86
|
let end: (() => Promise<void>) | undefined;
|
|
82
87
|
if (dbSource.kind === 'url') {
|
|
83
|
-
step(dbSource
|
|
88
|
+
step(connectingVia(dbSource));
|
|
84
89
|
({ runner, end } = await createUrlRunner(dbSource.url));
|
|
85
90
|
} else {
|
|
86
91
|
step('Resolving deployed config...');
|
|
@@ -89,7 +94,7 @@ export async function dbFingerprintCommand(flags: Record<string, string>): Promi
|
|
|
89
94
|
}
|
|
90
95
|
|
|
91
96
|
try {
|
|
92
|
-
const status = await computeFingerprintStatus(runner, models);
|
|
97
|
+
const status = await computeFingerprintStatus(runner, models, sequences);
|
|
93
98
|
|
|
94
99
|
if (flags.json === 'true') {
|
|
95
100
|
console.log(JSON.stringify(status, null, 2));
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
*
|
|
4
4
|
* db:generate --stage <name> [--name <label>] introspect the live DB → diff the Models
|
|
5
5
|
* → write the next drizzle migration file
|
|
6
|
+
* db:generate --dry-run the same edge, printed — NOTHING written
|
|
7
|
+
* (no migration, no journal, no schema refresh)
|
|
6
8
|
*
|
|
7
9
|
* The one generator — both layers in one ordered migration. It compiles the app's Models,
|
|
8
10
|
* introspects the deployed database through the ops Lambda `db:query` action (the same
|
|
@@ -24,11 +26,12 @@ import type { ModelDescriptor, Module } from '@everystack/model';
|
|
|
24
26
|
import { introspectSchema } from '../schema-introspect.js';
|
|
25
27
|
import { compileDrizzleSource } from '../schema-source.js';
|
|
26
28
|
import { compileModuleMigration } from '../migration-compile.js';
|
|
27
|
-
import { generateMigrationSql, unmodeledTables, formatMigrationFile, planMigrationFile, HELD_DROP_PREFIX, type Journal } from '../migration-generate.js';
|
|
29
|
+
import { generateMigrationSql, unmodeledTables, formatMigrationFile, planMigrationFile, resolveSchemaOut, HELD_DROP_PREFIX, type Journal } from '../migration-generate.js';
|
|
28
30
|
import { introspectContract, type QueryRunner } from '../authz-contract.js';
|
|
29
31
|
import { FUNCTIONS_SQL, contractFunctionRow } from '../security-catalog.js';
|
|
30
|
-
import { resolveDbSource, createUrlRunner, type DbSource } from '../db-source.js';
|
|
32
|
+
import { resolveDbSource, createUrlRunner, connectingVia, type DbSource } from '../db-source.js';
|
|
31
33
|
import { resolveModelsPath } from '../models-path.js';
|
|
34
|
+
import { loadDeclaredDerived, type DeclaredDerived } from '../declared-derived.js';
|
|
32
35
|
import { applyStateAndVerify, classifyGeneratedStatements, currentGitRef, type StateSyncOutcome } from '../state-apply.js';
|
|
33
36
|
import { resolveConfig, opsFunction } from '../config.js';
|
|
34
37
|
import { invokeAction } from '../aws.js';
|
|
@@ -76,7 +79,7 @@ async function loadModules(modelsPath: string): Promise<Module[]> {
|
|
|
76
79
|
}
|
|
77
80
|
if (Array.isArray(mod.modules)) return mod.modules;
|
|
78
81
|
const models = mod.models ?? mod.default;
|
|
79
|
-
if (Array.isArray(models)) return [{ models, extensions: [], functions: [] }];
|
|
82
|
+
if (Array.isArray(models)) return [{ models, extensions: [], sequences: [], derived: [], functions: [] }];
|
|
80
83
|
throw new Error(`${modelsPath} must export a \`modules\` (or \`models\`) array.`);
|
|
81
84
|
}
|
|
82
85
|
|
|
@@ -87,14 +90,14 @@ async function loadModules(modelsPath: string): Promise<Module[]> {
|
|
|
87
90
|
* `drizzle/`. Roles are excluded by design (they are `db:provision`). Use it to (re)initialize
|
|
88
91
|
* a clean migration history: clear `drizzle/`, then `db:generate --init`.
|
|
89
92
|
*/
|
|
90
|
-
async function dbGenerateInit(modelsPath: string, migrationsDir: string, schemaOut: string, name: string): Promise<void> {
|
|
93
|
+
async function dbGenerateInit(modelsPath: string, migrationsDir: string, schemaOut: string, name: string, schemaOutRecord: string): Promise<void> {
|
|
91
94
|
step(`Loading modules from ${modelsPath}...`);
|
|
92
95
|
const modules = await loadModules(modelsPath);
|
|
93
96
|
const models = modules.flatMap((m) => m.models);
|
|
94
97
|
info(`${modules.length} module(s), ${models.length} model(s).`);
|
|
95
98
|
|
|
96
99
|
step(`Writing the drizzle schema → ${path.relative(process.cwd(), schemaOut)}...`);
|
|
97
|
-
await fs.writeFile(schemaOut, compileDrizzleSource(models), 'utf8');
|
|
100
|
+
await fs.writeFile(schemaOut, compileDrizzleSource(models, { derived: modules.flatMap((m) => m.derived) }), 'utf8');
|
|
98
101
|
|
|
99
102
|
const statements = compileModuleMigration(modules);
|
|
100
103
|
|
|
@@ -106,6 +109,7 @@ async function dbGenerateInit(modelsPath: string, migrationsDir: string, schemaO
|
|
|
106
109
|
journal = { version: '7', dialect: 'postgresql', entries: [] };
|
|
107
110
|
}
|
|
108
111
|
const { filename, journal: nextJournal } = planMigrationFile(journal, name, Date.now());
|
|
112
|
+
nextJournal.schemaOut = schemaOutRecord; // D2 — flag-less later runs reuse it
|
|
109
113
|
const filePath = path.join(migrationsDir, filename);
|
|
110
114
|
await fs.mkdir(path.join(migrationsDir, 'meta'), { recursive: true });
|
|
111
115
|
await fs.writeFile(filePath, formatMigrationFile(statements), 'utf8');
|
|
@@ -117,18 +121,43 @@ async function dbGenerateInit(modelsPath: string, migrationsDir: string, schemaO
|
|
|
117
121
|
process.exit(0);
|
|
118
122
|
}
|
|
119
123
|
|
|
124
|
+
/** Read the journal if present (null = no history yet). */
|
|
125
|
+
async function readJournal(migrationsDir: string): Promise<Journal | null> {
|
|
126
|
+
try {
|
|
127
|
+
return JSON.parse(await fs.readFile(path.join(migrationsDir, 'meta', '_journal.json'), 'utf8'));
|
|
128
|
+
} catch {
|
|
129
|
+
return null;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
120
133
|
export async function dbGenerateCommand(flags: Record<string, string>): Promise<void> {
|
|
121
134
|
const modelsPath = resolveModelsPath(flags.models);
|
|
122
135
|
const migrationsDir = path.resolve(flags.dir || DEFAULT_MIGRATIONS);
|
|
123
|
-
const schemaOut = path.resolve(flags['schema-out'] || DEFAULT_SCHEMA_OUT);
|
|
124
136
|
const name = flags.name || 'generated';
|
|
125
137
|
const allowDrops = flags['allow-drops'] === 'true';
|
|
138
|
+
const dryRun = flags['dry-run'] === 'true';
|
|
139
|
+
|
|
140
|
+
// D2 — schema-out resolution: flag > journal-recorded > default. Recording the
|
|
141
|
+
// resolved path in the journal makes the relocated-artifact orphan unmintable:
|
|
142
|
+
// a later run without the flag keeps writing where the project writes.
|
|
143
|
+
const journal = await readJournal(migrationsDir);
|
|
144
|
+
const schemaOutRes = resolveSchemaOut(flags['schema-out'], journal, DEFAULT_SCHEMA_OUT);
|
|
145
|
+
const schemaOut = path.resolve(schemaOutRes.path);
|
|
146
|
+
if (schemaOutRes.source === 'recorded') {
|
|
147
|
+
info(`schema-out: ${schemaOutRes.path} (recorded in the journal; override with --schema-out)`);
|
|
148
|
+
} else if (schemaOutRes.source === 'flag' && schemaOutRes.updateRecord && journal?.schemaOut) {
|
|
149
|
+
warn(`schema-out moves: ${journal.schemaOut} → ${schemaOutRes.path} — the journal record updates with this run; delete the old artifact.`);
|
|
150
|
+
}
|
|
126
151
|
|
|
127
152
|
// --init: greenfield, no live DB — the model emits the whole migration (extensions + schema +
|
|
128
153
|
// authz + package functions). The brownfield path below introspects the deployed DB and diffs.
|
|
129
154
|
if (flags.init === 'true') {
|
|
155
|
+
if (dryRun) {
|
|
156
|
+
fail('--dry-run previews the brownfield diff — --init is the greenfield generator (its output IS the whole migration; there is nothing cheaper to preview).');
|
|
157
|
+
process.exit(1);
|
|
158
|
+
}
|
|
130
159
|
try {
|
|
131
|
-
await dbGenerateInit(modelsPath, migrationsDir, schemaOut, flags.name || 'init');
|
|
160
|
+
await dbGenerateInit(modelsPath, migrationsDir, schemaOut, flags.name || 'init', schemaOutRes.path);
|
|
132
161
|
} catch (err: any) {
|
|
133
162
|
fail(err.message);
|
|
134
163
|
process.exit(1);
|
|
@@ -137,7 +166,12 @@ export async function dbGenerateCommand(flags: Record<string, string>): Promise<
|
|
|
137
166
|
}
|
|
138
167
|
|
|
139
168
|
const apply = flags.apply === 'true';
|
|
169
|
+
if (dryRun && apply) {
|
|
170
|
+
fail('--dry-run and --apply are opposites — a dry run writes and executes nothing. Pick one.');
|
|
171
|
+
process.exit(1);
|
|
172
|
+
}
|
|
140
173
|
let models: ModelDescriptor[];
|
|
174
|
+
let declaredDb: DeclaredDerived | null = null;
|
|
141
175
|
let current;
|
|
142
176
|
let liveAuthz;
|
|
143
177
|
let runner!: QueryRunner;
|
|
@@ -145,19 +179,23 @@ export async function dbGenerateCommand(flags: Record<string, string>): Promise<
|
|
|
145
179
|
try {
|
|
146
180
|
step(`Loading models from ${modelsPath}...`);
|
|
147
181
|
models = await loadModels(modelsPath);
|
|
182
|
+
declaredDb = await loadDeclaredDerived(flags.models);
|
|
148
183
|
info(`${models.length} model(s).`);
|
|
149
184
|
// The drizzle runtime schema is a derived artifact — refresh it from the models
|
|
150
185
|
// every run (it needs no DB), so a relation-only change with no DDL diff still
|
|
151
|
-
// updates it. The drift-guard test asserts this file matches the models
|
|
152
|
-
|
|
153
|
-
|
|
186
|
+
// updates it. The drift-guard test asserts this file matches the models (and the
|
|
187
|
+
// typed derived relations — B6a). A dry run refreshes NOTHING — not even this.
|
|
188
|
+
if (!dryRun) {
|
|
189
|
+
step(`Writing the drizzle schema → ${path.relative(process.cwd(), schemaOut)}...`);
|
|
190
|
+
await fs.writeFile(schemaOut, compileDrizzleSource(models, { derived: declaredDb?.derived }), 'utf8');
|
|
191
|
+
}
|
|
154
192
|
const dbSource: DbSource = resolveDbSource(flags);
|
|
155
193
|
if (apply && dbSource.kind !== 'url') {
|
|
156
194
|
fail('--apply needs a direct connection (--database-url or DATABASE_URL) — the ops Lambda query path is read-only by design.');
|
|
157
195
|
process.exit(1);
|
|
158
196
|
}
|
|
159
197
|
if (dbSource.kind === 'url') {
|
|
160
|
-
step(dbSource
|
|
198
|
+
step(connectingVia(dbSource));
|
|
161
199
|
({ runner, end } = await createUrlRunner(dbSource.url));
|
|
162
200
|
} else {
|
|
163
201
|
step('Resolving deployed config...');
|
|
@@ -179,14 +217,29 @@ export async function dbGenerateCommand(flags: Record<string, string>): Promise<
|
|
|
179
217
|
|
|
180
218
|
// One ordered migration carries both layers: data DDL first, then the authz reconcile
|
|
181
219
|
// (RLS/policies/grants) the Models' abilities declare, diffed against the live contract.
|
|
182
|
-
const statements = generateMigrationSql(models, current, { allowDrops, liveAuthz });
|
|
220
|
+
const statements = generateMigrationSql(models, current, { allowDrops, liveAuthz, sequences: declaredDb?.sequences });
|
|
183
221
|
const unmodeled = unmodeledTables(models, current);
|
|
184
222
|
if (unmodeled.length) {
|
|
185
223
|
info(`${unmodeled.length} table(s) in the database are not declared by any model — left untouched (db:generate manages only declared tables).`);
|
|
186
224
|
}
|
|
187
225
|
console.log('');
|
|
188
226
|
if (statements.length === 0) {
|
|
189
|
-
success(
|
|
227
|
+
success(`db:generate — the live database already matches the models. ${dryRun ? 'Nothing to preview.' : 'No migration written.'}`);
|
|
228
|
+
process.exit(0);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
// D1 — --dry-run: the edge as SQL, nothing else. No migration file, no journal
|
|
232
|
+
// entry, no schema refresh — preview traffic belongs here (and db:diff computes
|
|
233
|
+
// the models-vs-models edge with no database at all).
|
|
234
|
+
if (dryRun) {
|
|
235
|
+
const classified = classifyGeneratedStatements(statements);
|
|
236
|
+
info(`dry run — ${statements.length} statement(s), nothing written:`);
|
|
237
|
+
console.log('');
|
|
238
|
+
for (const s of statements) console.log(s.endsWith(';') ? s : `${s};`);
|
|
239
|
+
console.log('');
|
|
240
|
+
if (classified.heldDrops.length) warn(`${classified.heldDrops.length} destructive change(s) held back (shown as comments) — --allow-drops includes them.`);
|
|
241
|
+
if (classified.notices.length) info(`${classified.notices.length} notice(s) — manual follow-ups.`);
|
|
242
|
+
info('Nothing was written. Drop --dry-run to mint the migration, or `db:generate --apply` to run it directly on a dev database.');
|
|
190
243
|
process.exit(0);
|
|
191
244
|
}
|
|
192
245
|
|
|
@@ -209,7 +262,7 @@ export async function dbGenerateCommand(flags: Record<string, string>): Promise<
|
|
|
209
262
|
let outcome: StateSyncOutcome;
|
|
210
263
|
try {
|
|
211
264
|
outcome = await applyStateAndVerify(runner, models, statements, current, liveAuthz, {
|
|
212
|
-
allowDrops, actor: process.env.USER ?? null, gitRef: currentGitRef(),
|
|
265
|
+
allowDrops, sequences: declaredDb?.sequences, actor: process.env.USER ?? null, gitRef: currentGitRef(),
|
|
213
266
|
});
|
|
214
267
|
} catch (err: any) {
|
|
215
268
|
fail(`Apply failed and rolled back: ${err.message}`);
|
|
@@ -228,14 +281,12 @@ export async function dbGenerateCommand(flags: Record<string, string>): Promise<
|
|
|
228
281
|
}
|
|
229
282
|
|
|
230
283
|
const journalPath = path.join(migrationsDir, 'meta', '_journal.json');
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
const { filename, journal: nextJournal } = planMigrationFile(journal, name, Date.now());
|
|
284
|
+
const { filename, journal: nextJournal } = planMigrationFile(
|
|
285
|
+
journal ?? { version: '7', dialect: 'postgresql', entries: [] }, name, Date.now(),
|
|
286
|
+
);
|
|
287
|
+
// D2 — the resolved schema-out path rides the journal, so the next flag-less run
|
|
288
|
+
// keeps writing where this project writes.
|
|
289
|
+
nextJournal.schemaOut = schemaOutRes.path;
|
|
239
290
|
const filePath = path.join(migrationsDir, filename);
|
|
240
291
|
try {
|
|
241
292
|
await fs.mkdir(path.join(migrationsDir, 'meta'), { recursive: true });
|
|
@@ -28,24 +28,16 @@ import { verifyDescent } from '../git-descent.js';
|
|
|
28
28
|
import { planBackfills, readBackfillLog } from '../backfill.js';
|
|
29
29
|
import { readSqlDirIfPresent } from './db-sync.js';
|
|
30
30
|
import { currentGitRef } from '../state-apply.js';
|
|
31
|
-
import { resolveDbSource, createUrlRunner, type DbSource } from '../db-source.js';
|
|
31
|
+
import { resolveDbSource, createUrlRunner, connectingVia, type DbSource } from '../db-source.js';
|
|
32
32
|
import { resolveModelsPath } from '../models-path.js';
|
|
33
33
|
import { resolveConfig, opsFunction } from '../config.js';
|
|
34
|
-
import {
|
|
34
|
+
import { lambdaQueryRunner } from '../aws.js';
|
|
35
35
|
import { loadModels } from './db-generate.js';
|
|
36
|
+
import { reportPipelineLastRun } from './pipeline-run.js';
|
|
36
37
|
import { step, success, fail, info, warn } from '../output.js';
|
|
37
38
|
|
|
38
39
|
const DEFAULT_OUT = 'db.plan.json';
|
|
39
40
|
|
|
40
|
-
/** A QueryRunner backed by the ops Lambda `db:query` action (read-only). */
|
|
41
|
-
function lambdaRunner(region: string, fn: string): QueryRunner {
|
|
42
|
-
return async (sql: string) => {
|
|
43
|
-
const result: any = await invokeAction(region, fn, 'db:query', { sql });
|
|
44
|
-
if (result?.error) throw new Error(`Query failed: ${result.error}`);
|
|
45
|
-
return result?.rows ?? [];
|
|
46
|
-
};
|
|
47
|
-
}
|
|
48
|
-
|
|
49
41
|
export async function dbPlanCommand(flags: Record<string, string>): Promise<void> {
|
|
50
42
|
let dbSource: DbSource;
|
|
51
43
|
try {
|
|
@@ -69,12 +61,12 @@ export async function dbPlanCommand(flags: Record<string, string>): Promise<void
|
|
|
69
61
|
let runner: QueryRunner;
|
|
70
62
|
let end: (() => Promise<void>) | undefined;
|
|
71
63
|
if (dbSource.kind === 'url') {
|
|
72
|
-
step(dbSource
|
|
64
|
+
step(connectingVia(dbSource));
|
|
73
65
|
({ runner, end } = await createUrlRunner(dbSource.url));
|
|
74
66
|
} else {
|
|
75
67
|
step('Resolving deployed config...');
|
|
76
68
|
const config = await resolveConfig(flags.stage);
|
|
77
|
-
runner =
|
|
69
|
+
runner = lambdaQueryRunner(config.region, opsFunction(config));
|
|
78
70
|
}
|
|
79
71
|
|
|
80
72
|
try {
|
|
@@ -134,6 +126,8 @@ export async function dbPlanCommand(flags: Record<string, string>): Promise<void
|
|
|
134
126
|
warn(`backfill lane: could not read everystack.backfill_log (${err.message}) — pending-backfill visibility unavailable.`);
|
|
135
127
|
}
|
|
136
128
|
}
|
|
129
|
+
|
|
130
|
+
await reportPipelineLastRun(runner);
|
|
137
131
|
if (out !== '-') {
|
|
138
132
|
success(`Wrote ${out} — review it, then \`everystack db:apply --plan ${out}\`.`);
|
|
139
133
|
warn('Plans are ephemeral release artifacts — attach to the run, do NOT commit.');
|