@everystack/cli 0.3.12 → 0.3.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/package.json +1 -1
- package/src/cli/commands/db-reconcile.ts +300 -0
- package/src/cli/derived-apply.ts +149 -0
- package/src/cli/derived-introspect.ts +314 -0
- package/src/cli/derived-plan.ts +279 -0
- package/src/cli/derived-source.ts +373 -0
- package/src/cli/index.ts +5 -0
- package/src/cli/runbook/detect.ts +32 -0
- package/src/cli/runbook/model-section.ts +67 -1
- package/src/cli/runbook/sections.ts +59 -12
|
@@ -0,0 +1,373 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* derived-source — the source side of the derived-object reconciler.
|
|
3
|
+
*
|
|
4
|
+
* Parses an app's ordered db/sql files into structured derived objects
|
|
5
|
+
* (functions, views, materialized views), attaching the statements that ride
|
|
6
|
+
* with an object (its indexes, comments, grants) and hashing each object's
|
|
7
|
+
* normalized content. The reconciler diffs these hashes against the live
|
|
8
|
+
* catalog's provenance to decide what to create, replace, or drop.
|
|
9
|
+
*
|
|
10
|
+
* Everything here is pure and database-free. The lexer respects Postgres
|
|
11
|
+
* string syntax — single quotes with '' doubling, E-strings with backslash
|
|
12
|
+
* escapes, double-quoted identifiers, tagged dollar quoting, and nested block
|
|
13
|
+
* comments — because a matview body is exactly where naive splitting dies.
|
|
14
|
+
*
|
|
15
|
+
* Identity note: functions are identified by schema-qualified NAME, not full
|
|
16
|
+
* signature. Overloads are rare in app SQL layers; when they appear we warn
|
|
17
|
+
* rather than mis-model them (the honest limitation, revisit if a consumer
|
|
18
|
+
* actually needs overloads).
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { createHash } from 'node:crypto';
|
|
22
|
+
|
|
23
|
+
export type DerivedKind = 'function' | 'view' | 'materialized view';
|
|
24
|
+
|
|
25
|
+
/** A statement that belongs to an object: its index, comment, or grant. */
|
|
26
|
+
export interface Attachment {
|
|
27
|
+
kind: 'index' | 'comment' | 'grant';
|
|
28
|
+
sql: string;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface SourceObject {
|
|
32
|
+
kind: DerivedKind;
|
|
33
|
+
/** Schema name, defaulted to `public` when unqualified. */
|
|
34
|
+
schema: string;
|
|
35
|
+
name: string;
|
|
36
|
+
/** `schema.name` — the reconciler's join key against the live catalog. */
|
|
37
|
+
identity: string;
|
|
38
|
+
/** The CREATE statement as authored (no trailing semicolon). */
|
|
39
|
+
sql: string;
|
|
40
|
+
/** Indexes / comments / grants targeting this object, in source order. */
|
|
41
|
+
attachments: Attachment[];
|
|
42
|
+
/** sha256 of the normalized CREATE + attachments — comment/whitespace edits do not change it. */
|
|
43
|
+
hash: string;
|
|
44
|
+
/** Source file this object came from. */
|
|
45
|
+
file: string;
|
|
46
|
+
/** Position in the concatenated source — a valid dependency order by convention. */
|
|
47
|
+
seq: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export interface SourceFile {
|
|
51
|
+
file: string;
|
|
52
|
+
sql: string;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface ParsedSources {
|
|
56
|
+
objects: SourceObject[];
|
|
57
|
+
/** Statements the reconciler does not manage — surfaced, never silently dropped. */
|
|
58
|
+
warnings: string[];
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// ---------------------------------------------------------------------------
|
|
62
|
+
// Lexer.
|
|
63
|
+
// ---------------------------------------------------------------------------
|
|
64
|
+
|
|
65
|
+
type LexState =
|
|
66
|
+
| { in: 'normal' }
|
|
67
|
+
| { in: 'single'; escaped: boolean }
|
|
68
|
+
| { in: 'double' }
|
|
69
|
+
| { in: 'dollar'; tag: string }
|
|
70
|
+
| { in: 'line-comment' }
|
|
71
|
+
| { in: 'block-comment'; depth: number };
|
|
72
|
+
|
|
73
|
+
/** Matches a dollar-quote opener at position i; returns the full tag (`$$`, `$body$`) or null. */
|
|
74
|
+
function dollarTagAt(text: string, i: number): string | null {
|
|
75
|
+
if (text[i] !== '$') return null;
|
|
76
|
+
const m = /^\$[A-Za-z_][A-Za-z0-9_]*\$|^\$\$/.exec(text.slice(i, i + 64));
|
|
77
|
+
return m ? m[0] : null;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Walk the text, calling `emit` for every character with the state it is IN
|
|
82
|
+
* (comments/strings/normal). The shared core under both the statement splitter
|
|
83
|
+
* and the normalizer, so they can never disagree about where a string ends.
|
|
84
|
+
*/
|
|
85
|
+
function lex(
|
|
86
|
+
text: string,
|
|
87
|
+
emit: (ch: string, state: LexState['in'], index: number) => void,
|
|
88
|
+
): void {
|
|
89
|
+
let state: LexState = { in: 'normal' };
|
|
90
|
+
let i = 0;
|
|
91
|
+
while (i < text.length) {
|
|
92
|
+
const ch = text[i];
|
|
93
|
+
const next = text[i + 1];
|
|
94
|
+
|
|
95
|
+
if (state.in === 'normal') {
|
|
96
|
+
if (ch === '-' && next === '-') {
|
|
97
|
+
state = { in: 'line-comment' };
|
|
98
|
+
emit(ch, 'line-comment', i); i++;
|
|
99
|
+
emit(next, 'line-comment', i); i++;
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
if (ch === '/' && next === '*') {
|
|
103
|
+
state = { in: 'block-comment', depth: 1 };
|
|
104
|
+
emit(ch, 'block-comment', i); i++;
|
|
105
|
+
emit(next, 'block-comment', i); i++;
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
const tag = dollarTagAt(text, i);
|
|
109
|
+
if (tag) {
|
|
110
|
+
state = { in: 'dollar', tag };
|
|
111
|
+
for (const c of tag) { emit(c, 'dollar', i); i++; }
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
if (ch === "'") {
|
|
115
|
+
// E'...' / e'...' strings give backslash escape semantics.
|
|
116
|
+
const prev = text[i - 1];
|
|
117
|
+
state = { in: 'single', escaped: prev === 'E' || prev === 'e' };
|
|
118
|
+
emit(ch, 'single', i); i++;
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
if (ch === '"') {
|
|
122
|
+
state = { in: 'double' };
|
|
123
|
+
emit(ch, 'double', i); i++;
|
|
124
|
+
continue;
|
|
125
|
+
}
|
|
126
|
+
emit(ch, 'normal', i); i++;
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
if (state.in === 'line-comment') {
|
|
131
|
+
emit(ch, 'line-comment', i); i++;
|
|
132
|
+
if (ch === '\n') state = { in: 'normal' };
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if (state.in === 'block-comment') {
|
|
137
|
+
if (ch === '/' && next === '*') {
|
|
138
|
+
state = { in: 'block-comment', depth: state.depth + 1 };
|
|
139
|
+
emit(ch, 'block-comment', i); i++;
|
|
140
|
+
emit(next, 'block-comment', i); i++;
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
if (ch === '*' && next === '/') {
|
|
144
|
+
const newDepth: number = state.depth - 1;
|
|
145
|
+
emit(ch, 'block-comment', i); i++;
|
|
146
|
+
emit(next, 'block-comment', i); i++;
|
|
147
|
+
state = newDepth === 0 ? { in: 'normal' } : { in: 'block-comment', depth: newDepth };
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
emit(ch, 'block-comment', i); i++;
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
if (state.in === 'single') {
|
|
155
|
+
if (state.escaped && ch === '\\') {
|
|
156
|
+
emit(ch, 'single', i); i++;
|
|
157
|
+
if (i < text.length) { emit(text[i], 'single', i); i++; }
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
if (ch === "'" && next === "'") {
|
|
161
|
+
emit(ch, 'single', i); i++;
|
|
162
|
+
emit(next, 'single', i); i++;
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
emit(ch, 'single', i); i++;
|
|
166
|
+
if (ch === "'") state = { in: 'normal' };
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
if (state.in === 'double') {
|
|
171
|
+
if (ch === '"' && next === '"') {
|
|
172
|
+
emit(ch, 'double', i); i++;
|
|
173
|
+
emit(next, 'double', i); i++;
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
emit(ch, 'double', i); i++;
|
|
177
|
+
if (ch === '"') state = { in: 'normal' };
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// dollar
|
|
182
|
+
if (ch === '$') {
|
|
183
|
+
const tag = dollarTagAt(text, i);
|
|
184
|
+
if (tag === state.tag) {
|
|
185
|
+
for (const c of tag) { emit(c, 'dollar', i); i++; }
|
|
186
|
+
state = { in: 'normal' };
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
emit(ch, 'dollar', i); i++;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** Split SQL text into statements on top-level semicolons. Statements come back trimmed. */
|
|
195
|
+
export function splitSqlStatements(text: string): string[] {
|
|
196
|
+
const statements: string[] = [];
|
|
197
|
+
let current = '';
|
|
198
|
+
lex(text, (ch, state) => {
|
|
199
|
+
if (state === 'normal' && ch === ';') {
|
|
200
|
+
const s = current.trim();
|
|
201
|
+
if (s) statements.push(s);
|
|
202
|
+
current = '';
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
current += ch;
|
|
206
|
+
});
|
|
207
|
+
const last = current.trim();
|
|
208
|
+
if (last) statements.push(last);
|
|
209
|
+
return statements;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
const WORDISH = /[A-Za-z0-9_$]/;
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Normalize SQL for content hashing: comments stripped; outside strings,
|
|
216
|
+
* whitespace survives only as a single space between word tokens (`SELECT a`),
|
|
217
|
+
* never around punctuation (`a,b` ≡ `a, b`). A renumbered header comment or a
|
|
218
|
+
* reflowed SELECT hashes identically; any change to the actual SQL does not.
|
|
219
|
+
* The output is a canonical hashing form, not necessarily runnable SQL.
|
|
220
|
+
*/
|
|
221
|
+
export function normalizeSql(sql: string): string {
|
|
222
|
+
let out = '';
|
|
223
|
+
let pendingSpace = false;
|
|
224
|
+
lex(sql, (ch, state) => {
|
|
225
|
+
if (state === 'line-comment' || state === 'block-comment') return;
|
|
226
|
+
if (state === 'normal' && /\s/.test(ch)) {
|
|
227
|
+
pendingSpace = true;
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
if (pendingSpace) {
|
|
231
|
+
if (out.length > 0 && WORDISH.test(out[out.length - 1]) && WORDISH.test(ch)) out += ' ';
|
|
232
|
+
pendingSpace = false;
|
|
233
|
+
}
|
|
234
|
+
out += ch;
|
|
235
|
+
});
|
|
236
|
+
return out.trim();
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// ---------------------------------------------------------------------------
|
|
240
|
+
// Classification.
|
|
241
|
+
// ---------------------------------------------------------------------------
|
|
242
|
+
|
|
243
|
+
/** Strip quotes from an identifier and split an optionally schema-qualified name. */
|
|
244
|
+
function parseQualified(raw: string): { schema: string; name: string } {
|
|
245
|
+
const parts: string[] = [];
|
|
246
|
+
let current = '';
|
|
247
|
+
let quoted = false;
|
|
248
|
+
for (let i = 0; i < raw.length; i++) {
|
|
249
|
+
const ch = raw[i];
|
|
250
|
+
if (ch === '"') {
|
|
251
|
+
if (quoted && raw[i + 1] === '"') { current += '"'; i++; continue; }
|
|
252
|
+
quoted = !quoted;
|
|
253
|
+
continue;
|
|
254
|
+
}
|
|
255
|
+
if (ch === '.' && !quoted) { parts.push(current); current = ''; continue; }
|
|
256
|
+
current += ch;
|
|
257
|
+
}
|
|
258
|
+
parts.push(current);
|
|
259
|
+
if (parts.length >= 2) return { schema: parts[0], name: parts.slice(1).join('.') };
|
|
260
|
+
return { schema: 'public', name: parts[0] };
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
const FUNCTION_RE = /^CREATE\s+(?:OR\s+REPLACE\s+)?FUNCTION\s+((?:"[^"]*"|[^\s(])+)\s*\(/i;
|
|
264
|
+
const VIEW_RE = /^CREATE\s+(?:OR\s+REPLACE\s+)?VIEW\s+((?:"[^"]*"|[^\s(])+)/i;
|
|
265
|
+
const MATVIEW_RE = /^CREATE\s+MATERIALIZED\s+VIEW\s+(?:IF\s+NOT\s+EXISTS\s+)?((?:"[^"]*"|[^\s(])+)/i;
|
|
266
|
+
const INDEX_RE = /^CREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?(?:(?:"[^"]*"|[^\s(])+\s+)?ON\s+(?:ONLY\s+)?((?:"[^"]*"|[^\s(])+)/i;
|
|
267
|
+
const COMMENT_RE = /^COMMENT\s+ON\s+(MATERIALIZED\s+VIEW|VIEW|FUNCTION|COLUMN)\s+((?:"[^"]*"|[^\s(,])+)/i;
|
|
268
|
+
const GRANT_RE = /^GRANT\s+[\s\S]*?\bON\s+(?:TABLE\s+)?((?:"[^"]*"|[^\s(,])+)/i;
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* The first non-comment text of a statement, whitespace collapsed but token
|
|
272
|
+
* boundaries preserved (unlike the aggressive hash form — `VIEW "MyView"` must
|
|
273
|
+
* keep its space for the classification regexes).
|
|
274
|
+
*/
|
|
275
|
+
function statementHead(statement: string): string {
|
|
276
|
+
let out = '';
|
|
277
|
+
lex(statement, (ch, state) => {
|
|
278
|
+
if (state === 'line-comment' || state === 'block-comment') return;
|
|
279
|
+
if (state === 'normal' && /\s/.test(ch)) {
|
|
280
|
+
if (out.length > 0 && !out.endsWith(' ')) out += ' ';
|
|
281
|
+
return;
|
|
282
|
+
}
|
|
283
|
+
out += ch;
|
|
284
|
+
});
|
|
285
|
+
return out.trim().slice(0, 400);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* Parse ordered source files into derived objects. Attachments (indexes,
|
|
290
|
+
* comments, grants) join the object they target; statements that are neither a
|
|
291
|
+
* derived object nor an attachment become warnings — the reconciler manages
|
|
292
|
+
* the derived layer only, and it says so out loud.
|
|
293
|
+
*/
|
|
294
|
+
export function parseSqlSources(files: SourceFile[]): ParsedSources {
|
|
295
|
+
const objects: SourceObject[] = [];
|
|
296
|
+
const warnings: string[] = [];
|
|
297
|
+
const byIdentity = new Map<string, SourceObject>();
|
|
298
|
+
let seq = 0;
|
|
299
|
+
|
|
300
|
+
const identityOf = (raw: string): { schema: string; name: string; identity: string } => {
|
|
301
|
+
const { schema, name } = parseQualified(raw);
|
|
302
|
+
return { schema, name, identity: `${schema}.${name}` };
|
|
303
|
+
};
|
|
304
|
+
|
|
305
|
+
for (const { file, sql } of files) {
|
|
306
|
+
for (const statement of splitSqlStatements(sql)) {
|
|
307
|
+
const head = statementHead(statement);
|
|
308
|
+
|
|
309
|
+
const asObject = (kind: DerivedKind, rawName: string): void => {
|
|
310
|
+
const { schema, name, identity } = identityOf(rawName);
|
|
311
|
+
if (byIdentity.has(identity)) {
|
|
312
|
+
warnings.push(
|
|
313
|
+
`${file}: duplicate definition for ${identity} — function overloads are not modeled; ` +
|
|
314
|
+
`the reconciler tracks one definition per name (last one wins).`,
|
|
315
|
+
);
|
|
316
|
+
}
|
|
317
|
+
const obj: SourceObject = {
|
|
318
|
+
kind, schema, name, identity,
|
|
319
|
+
sql: statement, attachments: [], hash: '', file, seq: seq++,
|
|
320
|
+
};
|
|
321
|
+
byIdentity.set(identity, obj);
|
|
322
|
+
objects.push(obj);
|
|
323
|
+
};
|
|
324
|
+
|
|
325
|
+
const attach = (kind: Attachment['kind'], rawTarget: string, stripColumn = false): boolean => {
|
|
326
|
+
let { identity } = identityOf(rawTarget);
|
|
327
|
+
if (stripColumn) {
|
|
328
|
+
// COMMENT ON COLUMN schema.obj.col — the owning object is one segment up.
|
|
329
|
+
const parts = identity.split('.');
|
|
330
|
+
identity = parts.length > 2 ? parts.slice(0, -1).join('.') : `public.${parts[0]}`;
|
|
331
|
+
}
|
|
332
|
+
const target = byIdentity.get(identity);
|
|
333
|
+
if (!target) return false;
|
|
334
|
+
target.attachments.push({ kind, sql: statement });
|
|
335
|
+
return true;
|
|
336
|
+
};
|
|
337
|
+
|
|
338
|
+
let m: RegExpExecArray | null;
|
|
339
|
+
if ((m = FUNCTION_RE.exec(head))) { asObject('function', m[1]); continue; }
|
|
340
|
+
if ((m = MATVIEW_RE.exec(head))) { asObject('materialized view', m[1]); continue; }
|
|
341
|
+
if ((m = VIEW_RE.exec(head))) { asObject('view', m[1]); continue; }
|
|
342
|
+
|
|
343
|
+
if ((m = INDEX_RE.exec(head))) {
|
|
344
|
+
if (!attach('index', m[1])) {
|
|
345
|
+
warnings.push(`${file}: CREATE INDEX targets ${m[1]}, which is not a derived object in these sources — not managed here (base-table indexes belong to the model layer).`);
|
|
346
|
+
}
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
if ((m = COMMENT_RE.exec(head))) {
|
|
350
|
+
const isColumn = /^COLUMN$/i.test(m[1].replace(/\s+/g, ' ').split(' ')[0]) || /^COLUMN/i.test(m[1]);
|
|
351
|
+
if (!attach('comment', m[2], isColumn)) {
|
|
352
|
+
warnings.push(`${file}: COMMENT ON targets ${m[2]}, which is not a derived object in these sources.`);
|
|
353
|
+
}
|
|
354
|
+
continue;
|
|
355
|
+
}
|
|
356
|
+
if ((m = GRANT_RE.exec(head))) {
|
|
357
|
+
if (!attach('grant', m[1])) {
|
|
358
|
+
warnings.push(`${file}: GRANT targets ${m[1]}, which is not a derived object in these sources.`);
|
|
359
|
+
}
|
|
360
|
+
continue;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
warnings.push(`${file}: unmanaged statement (${head.slice(0, 60)}…) — the reconciler manages functions, views, and materialized views; move this to the appropriate layer.`);
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
for (const obj of objects) {
|
|
368
|
+
const content = [normalizeSql(obj.sql), ...obj.attachments.map((a) => normalizeSql(a.sql))].join('\n');
|
|
369
|
+
obj.hash = createHash('sha256').update(content).digest('hex');
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
return { objects, warnings };
|
|
373
|
+
}
|
package/src/cli/index.ts
CHANGED
|
@@ -11,6 +11,7 @@ import { dbMigrateCommand, dbSeedCommand, dbResetCommand, dbPsqlCommand, dbDocto
|
|
|
11
11
|
import { dbAuthzPullCommand, dbAuthzDiffCommand, dbAuthzTestCommand, dbAuthzOwnerCommand, dbAuthzReportCommand } from './commands/db-authz.js';
|
|
12
12
|
import { dbGenerateCommand } from './commands/db-generate.js';
|
|
13
13
|
import { dbPullCommand } from './commands/db-pull.js';
|
|
14
|
+
import { dbReconcileCommand } from './commands/db-reconcile.js';
|
|
14
15
|
import { dbSnapshotCommand, dbSnapshotsCommand } from './commands/db-snapshot.js';
|
|
15
16
|
import { dbBackupProbeCommand, dbBackupCommand, dbBackupsCommand, dbRestoreCommand, dbBackupDownloadCommand } from './commands/db-backup.js';
|
|
16
17
|
import { consoleCommand } from './commands/console.js';
|
|
@@ -172,6 +173,9 @@ async function main() {
|
|
|
172
173
|
case 'db:pull':
|
|
173
174
|
await dbPullCommand(flags);
|
|
174
175
|
break;
|
|
176
|
+
case 'db:reconcile':
|
|
177
|
+
await dbReconcileCommand(flags);
|
|
178
|
+
break;
|
|
175
179
|
case 'db:authz:pull':
|
|
176
180
|
await dbAuthzPullCommand(flags);
|
|
177
181
|
break;
|
|
@@ -294,6 +298,7 @@ Usage:
|
|
|
294
298
|
everystack db:generate [--stage <name> | --database-url <url>] [--name <label>] [--models models/index.ts] [--allow-drops] Diff models vs the live DB → write the next migration (never touches the DB; DROPs held back unless --allow-drops)
|
|
295
299
|
everystack db:pull [--stage <name> | --database-url <url>] [--schema public] [--out <dir | file.ts>] Introspect the live DB → render field() Models (the brownfield on-ramp). --out <dir> writes one file per model + index.ts (the default shape); --out <file.ts> writes a single module; stdout otherwise
|
|
296
300
|
Both introspect via the deployed ops Lambda by default; --database-url (or an inherited DATABASE_URL) connects directly — for a schema that exists only on a local Postgres.
|
|
301
|
+
everystack db:reconcile [--sql-dir db/sql] [--stage <name> | --database-url <url>] [--apply] [--check] [--baseline] [--overwrite-drift] [--json] Reconcile the derived layer (functions/views/matviews) against db/sql sources: plan with rebuild-cost estimates by default; --check is the CI gate; --apply executes (direct connection only) and records provenance + schema_log. Hand-edits are drift (never overwritten silently); first contact with existing objects wants --baseline, once.
|
|
297
302
|
everystack db:authz:pull [--stage <name>] [--dir authz] Introspect live authz (rls/grants/policies/secdef) → reviewable contract files
|
|
298
303
|
everystack db:authz:diff [--stage <name>] [--dir authz] Validate the live DB against the committed contract (non-zero exit on drift)
|
|
299
304
|
everystack db:authz:test [--stage <name>] [--dir authz] Red-team enforcement: SET ROLE + attempt per role/table/command (non-zero exit on a hole)
|
|
@@ -12,6 +12,14 @@ import path from 'node:path';
|
|
|
12
12
|
|
|
13
13
|
export type Tier = 'V1' | 'V2' | 'V3';
|
|
14
14
|
|
|
15
|
+
/**
|
|
16
|
+
* How this app's migrations actually happen — detection, not aspiration.
|
|
17
|
+
* The discriminator for 'everystack' is the db:generate artifact
|
|
18
|
+
* (db/schema.generated.ts), not the presence of drizzle.config.ts: the
|
|
19
|
+
* reference app keeps legacy drizzle-kit tooling alongside the generated flow.
|
|
20
|
+
*/
|
|
21
|
+
export type MigrationMode = 'everystack' | 'transitional' | 'hand-authored';
|
|
22
|
+
|
|
15
23
|
export interface ProjectReality {
|
|
16
24
|
root: string;
|
|
17
25
|
/** package.json name. */
|
|
@@ -36,6 +44,10 @@ export interface ProjectReality {
|
|
|
36
44
|
hasExpoApp: boolean;
|
|
37
45
|
/** Relative migrations dir ('drizzle' or 'db/migrations') or null. */
|
|
38
46
|
migrationsDir: string | null;
|
|
47
|
+
/** The detected migration story, or null when the app has none yet. */
|
|
48
|
+
migrationMode: MigrationMode | null;
|
|
49
|
+
/** package.json scripts, in declaration order. */
|
|
50
|
+
scripts: Record<string, string>;
|
|
39
51
|
}
|
|
40
52
|
|
|
41
53
|
const V2_PACKAGES = new Set(['api', 'auth', 'admin', 'logging', 'security', 'query']);
|
|
@@ -111,6 +123,24 @@ export function detectProjectReality(root: string): ProjectReality {
|
|
|
111
123
|
? 'db/migrations'
|
|
112
124
|
: null;
|
|
113
125
|
|
|
126
|
+
const hasModels = models.files.length > 0 || models.hasIndex;
|
|
127
|
+
const hasGeneratedSchema = isFile(path.join(abs, 'db', 'schema.generated.ts'));
|
|
128
|
+
const hasHandTooling = isFile(path.join(abs, 'drizzle.config.ts'));
|
|
129
|
+
const migrationMode: MigrationMode | null = hasModels
|
|
130
|
+
? hasGeneratedSchema || (!migrationsDir && !hasHandTooling)
|
|
131
|
+
? 'everystack'
|
|
132
|
+
: 'transitional'
|
|
133
|
+
: migrationsDir || hasHandTooling
|
|
134
|
+
? 'hand-authored'
|
|
135
|
+
: null;
|
|
136
|
+
|
|
137
|
+
const scripts: Record<string, string> = {};
|
|
138
|
+
if (pkg.scripts && typeof pkg.scripts === 'object') {
|
|
139
|
+
for (const [name, cmd] of Object.entries(pkg.scripts)) {
|
|
140
|
+
if (typeof cmd === 'string') scripts[name] = cmd;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
114
144
|
return {
|
|
115
145
|
root: abs,
|
|
116
146
|
name: typeof pkg.name === 'string' && pkg.name ? pkg.name : path.basename(abs),
|
|
@@ -123,5 +153,7 @@ export function detectProjectReality(root: string): ProjectReality {
|
|
|
123
153
|
hasSstConfig: isFile(path.join(abs, 'sst.config.ts')),
|
|
124
154
|
hasExpoApp: isDir(path.join(abs, 'app')),
|
|
125
155
|
migrationsDir,
|
|
156
|
+
migrationMode,
|
|
157
|
+
scripts,
|
|
126
158
|
};
|
|
127
159
|
}
|
|
@@ -80,7 +80,73 @@ function relationLine(name: string, rel: Relation): string {
|
|
|
80
80
|
return `- \`${name}\` — ${shape} ${target} (via \`${rel.column}\`)`;
|
|
81
81
|
}
|
|
82
82
|
|
|
83
|
-
export
|
|
83
|
+
export interface DataModelOptions {
|
|
84
|
+
/** Above this many models the section renders as a summary. Default 16. */
|
|
85
|
+
threshold?: number;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const DEFAULT_THRESHOLD = 16;
|
|
89
|
+
|
|
90
|
+
function nameList(tables: string[], cap = 15): string {
|
|
91
|
+
const shown = tables.slice(0, cap).map((t) => `\`${t}\``);
|
|
92
|
+
const rest = tables.length - cap;
|
|
93
|
+
return rest > 0 ? `${shown.join(', ')} (+${rest} more)` : shown.join(', ');
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* At scale (a 199-model brownfield pull) the full per-field render is reference
|
|
98
|
+
* material, not operations — ~5k lines nobody reads. Above the threshold we
|
|
99
|
+
* summarize per schema and keep the security-relevant signals (owner scoping,
|
|
100
|
+
* tables nobody can reach, operational tables), pointing at the real reference.
|
|
101
|
+
*/
|
|
102
|
+
function renderSummary(models: ModelDescriptor[]): string {
|
|
103
|
+
const bySchema = new Map<string, ModelDescriptor[]>();
|
|
104
|
+
for (const m of models) {
|
|
105
|
+
const schema = m.schema || 'public';
|
|
106
|
+
if (!bySchema.has(schema)) bySchema.set(schema, []);
|
|
107
|
+
bySchema.get(schema)!.push(m);
|
|
108
|
+
}
|
|
109
|
+
const lines: string[] = [
|
|
110
|
+
'## Your data model',
|
|
111
|
+
'',
|
|
112
|
+
`${models.length} models across ${bySchema.size} schema(s) — summarized; the full reference is`,
|
|
113
|
+
'the `models/` directory and `everystack db:authz:report`. Compiled from the Models — edit',
|
|
114
|
+
'them and regenerate; never edit this section.',
|
|
115
|
+
];
|
|
116
|
+
for (const schema of [...bySchema.keys()].sort()) {
|
|
117
|
+
const group = bySchema.get(schema)!;
|
|
118
|
+
lines.push('', `### ${schema} — ${group.length} table(s)`, '');
|
|
119
|
+
const ownerScoped = group.filter((m) => m.abilities.some((a) => a.condition?.owner || a.condition?.via));
|
|
120
|
+
const anonReadable = group.filter((m) =>
|
|
121
|
+
m.abilities.some(
|
|
122
|
+
(a) =>
|
|
123
|
+
(a.action === 'read' || a.action === 'manage') &&
|
|
124
|
+
!a.condition?.role && !a.condition?.owner && !a.condition?.via &&
|
|
125
|
+
!a.condition?.where && !a.condition?.sql,
|
|
126
|
+
),
|
|
127
|
+
);
|
|
128
|
+
const roleGated = group.filter((m) => m.abilities.some((a) => a.condition?.role));
|
|
129
|
+
const noAbilities = group.filter((m) => m.abilities.length === 0);
|
|
130
|
+
const operational = group.filter((m) => m.private || m.writtenBy !== 'app');
|
|
131
|
+
lines.push(
|
|
132
|
+
`- owner-scoped: ${ownerScoped.length} · anon-readable: ${anonReadable.length} · role-gated: ${roleGated.length}`,
|
|
133
|
+
);
|
|
134
|
+
if (noAbilities.length) {
|
|
135
|
+
lines.push(
|
|
136
|
+
`- no abilities declared (unreachable through the API): ${nameList(noAbilities.map((m) => m.table))}`,
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
if (operational.length) {
|
|
140
|
+
lines.push(
|
|
141
|
+
`- operational (private / non-app writers): ${nameList(operational.map((m) => m.table))}`,
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return lines.join('\n');
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export function renderDataModelSection(models: ModelDescriptor[], opts: DataModelOptions = {}): string {
|
|
149
|
+
if (models.length > (opts.threshold ?? DEFAULT_THRESHOLD)) return renderSummary(models);
|
|
84
150
|
const lines: string[] = [
|
|
85
151
|
'## Your data model',
|
|
86
152
|
'',
|
|
@@ -71,8 +71,24 @@ function projectMap(r: ProjectReality): Block {
|
|
|
71
71
|
`- \`${r.models.dir}/\` — \`defineModel\` tables (${r.models.files.length} model(s); the source of truth for schema + authz)`,
|
|
72
72
|
);
|
|
73
73
|
}
|
|
74
|
-
if (r.hasDatabase)
|
|
75
|
-
|
|
74
|
+
if (r.hasDatabase) {
|
|
75
|
+
lines.push(
|
|
76
|
+
r.migrationMode === 'everystack'
|
|
77
|
+
? '- `db/` — generated Drizzle schema (never edit by hand)'
|
|
78
|
+
: r.migrationMode === 'transitional'
|
|
79
|
+
? '- `db/` — Drizzle schema (migrating to Models; parts are still hand-maintained)'
|
|
80
|
+
: '- `db/` — Drizzle schema (hand-maintained)',
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
if (r.migrationsDir) {
|
|
84
|
+
lines.push(
|
|
85
|
+
r.migrationMode === 'everystack'
|
|
86
|
+
? `- \`${r.migrationsDir}/\` — generated migrations`
|
|
87
|
+
: r.migrationMode === 'transitional'
|
|
88
|
+
? `- \`${r.migrationsDir}/\` — migrations (hand-authored journal, moving to \`db:generate\`)`
|
|
89
|
+
: `- \`${r.migrationsDir}/\` — migrations (hand-authored via drizzle-kit)`,
|
|
90
|
+
);
|
|
91
|
+
}
|
|
76
92
|
if (r.handlers.api || r.handlers.worker || r.handlers.image || r.crons.length) {
|
|
77
93
|
const parts = [
|
|
78
94
|
r.handlers.api ? 'api.ts' : null,
|
|
@@ -87,14 +103,24 @@ function projectMap(r: ProjectReality): Block {
|
|
|
87
103
|
return gen('project-map', lines);
|
|
88
104
|
}
|
|
89
105
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
106
|
+
const SCRIPTS_MAX = 12;
|
|
107
|
+
const SCRIPT_CMD_MAX = 64;
|
|
108
|
+
|
|
109
|
+
function dailyWorkflow(r: ProjectReality): Block {
|
|
110
|
+
const lines = ['## Daily workflow', ''];
|
|
111
|
+
const entries = Object.entries(r.scripts);
|
|
112
|
+
for (const [name, cmd] of entries.slice(0, SCRIPTS_MAX)) {
|
|
113
|
+
const shown = cmd.length > SCRIPT_CMD_MAX ? `${cmd.slice(0, SCRIPT_CMD_MAX)}…` : cmd;
|
|
114
|
+
lines.push(`- \`pnpm ${name}\` — \`${shown}\``);
|
|
115
|
+
}
|
|
116
|
+
if (entries.length > SCRIPTS_MAX) {
|
|
117
|
+
lines.push(`- …and ${entries.length - SCRIPTS_MAX} more in \`package.json\``);
|
|
118
|
+
}
|
|
119
|
+
if (entries.length === 0) {
|
|
120
|
+
lines.push('- No scripts declared in `package.json` yet.');
|
|
121
|
+
}
|
|
122
|
+
lines.push('- Every feature starts with a failing test; conventional commits; tests pass before commit.');
|
|
123
|
+
return gen('daily-workflow', lines);
|
|
98
124
|
}
|
|
99
125
|
|
|
100
126
|
function deploy(r: ProjectReality): Block {
|
|
@@ -127,6 +153,13 @@ function extend(r: ProjectReality): Block {
|
|
|
127
153
|
'- **Add a table or field:** edit the Model in `models/`, run `everystack db:generate`,',
|
|
128
154
|
' review the emitted migration, then `everystack db:migrate`. Never hand-write a SQL',
|
|
129
155
|
' migration, never edit `db/schema.ts` — after any change `db:generate` must be a clean no-op.',
|
|
156
|
+
);
|
|
157
|
+
if (r.migrationMode === 'transitional') {
|
|
158
|
+
lines.push(
|
|
159
|
+
' _(Target flow — this app is migrating to Models and the hand-authored journal is being retired.)_',
|
|
160
|
+
);
|
|
161
|
+
}
|
|
162
|
+
lines.push(
|
|
130
163
|
'- **Change who can read/write:** declare `can()` abilities on the Model — they compile',
|
|
131
164
|
' to RLS + grants. Never hand-write `CREATE POLICY` or `GRANT`.',
|
|
132
165
|
);
|
|
@@ -158,10 +191,24 @@ function extend(r: ProjectReality): Block {
|
|
|
158
191
|
}
|
|
159
192
|
|
|
160
193
|
function database(r: ProjectReality): Block {
|
|
194
|
+
const flow =
|
|
195
|
+
r.migrationMode === 'everystack'
|
|
196
|
+
? ['- `everystack db:generate` — compile Models → next migration (schema + RLS + grants)']
|
|
197
|
+
: r.migrationMode === 'transitional'
|
|
198
|
+
? [
|
|
199
|
+
'- **Mid-migration to Models:** the journal is still hand-authored. Target flow is',
|
|
200
|
+
' `everystack db:generate`; it must produce a clean, empty data diff before the',
|
|
201
|
+
' generated flow takes over. Until then, review hand-written SQL like the liability it is.',
|
|
202
|
+
]
|
|
203
|
+
: [
|
|
204
|
+
'- Migrations are hand-authored (drizzle-kit). The Models on-ramp: `everystack db:pull`',
|
|
205
|
+
' writes `models/` from the live database; from there `everystack db:generate` compiles',
|
|
206
|
+
' migrations (schema + RLS + grants) and hand-written SQL retires.',
|
|
207
|
+
];
|
|
161
208
|
return gen('database', [
|
|
162
209
|
'## Database operations',
|
|
163
210
|
'',
|
|
164
|
-
|
|
211
|
+
...flow,
|
|
165
212
|
'- `everystack db:migrate --stage <name>` — apply migrations on the deployed Lambda',
|
|
166
213
|
'- `everystack db:seed --stage dev` — seed the database (dev only)',
|
|
167
214
|
'- `everystack db:psql --stage dev` — psql into the stage database (credentials never exposed)',
|
|
@@ -273,7 +320,7 @@ export function buildStructure(reality: ProjectReality, ctx: RunbookContext): Bl
|
|
|
273
320
|
),
|
|
274
321
|
prerequisites(reality),
|
|
275
322
|
projectMap(reality),
|
|
276
|
-
dailyWorkflow(),
|
|
323
|
+
dailyWorkflow(reality),
|
|
277
324
|
deploy(reality),
|
|
278
325
|
slot(
|
|
279
326
|
'stage-notes',
|