@miller-tech/uap 1.210.8 → 1.211.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/bin/cli.js +7 -0
- package/dist/bin/cli.js.map +1 -1
- package/dist/cli/schema-diff.d.ts +86 -5
- package/dist/cli/schema-diff.d.ts.map +1 -1
- package/dist/cli/schema-diff.js +523 -28
- package/dist/cli/schema-diff.js.map +1 -1
- package/package.json +3 -3
- package/src/policies/enforcers/__pycache__/_common.cpython-312.pyc +0 -0
- package/src/policies/enforcers/schema_diff_gate.py +558 -28
- package/templates/hooks/__pycache__/deliver_autoroute.cpython-312.pyc +0 -0
- package/tools/agents/scripts/__pycache__/toolcall_path_normalizer.cpython-312.pyc +0 -0
- package/tools/agents/tests/test_schema_diff_gate.py +193 -1
- package/tools/agents/tests/test_schema_diff_inline.py +373 -0
package/dist/cli/schema-diff.js
CHANGED
|
@@ -48,15 +48,87 @@ function extractTypeScriptFields(content) {
|
|
|
48
48
|
}
|
|
49
49
|
return fields;
|
|
50
50
|
}
|
|
51
|
+
/**
|
|
52
|
+
* Blank out SQL comments, preserving offsets and line structure.
|
|
53
|
+
*
|
|
54
|
+
* Everything below counts parentheses and splits on commas, and a comment is
|
|
55
|
+
* allowed to contain both. `-- natural key (see ADR-14` raised the paren depth
|
|
56
|
+
* and swallowed the rest of the file, so the table it belonged to contributed
|
|
57
|
+
* NO columns; if that happened on one side of the diff only, every column read
|
|
58
|
+
* as removed and an added comment became a BREAKING verdict. Replacing with
|
|
59
|
+
* spaces rather than deleting keeps every offset valid for the callers that
|
|
60
|
+
* index back into the string.
|
|
61
|
+
*/
|
|
62
|
+
function stripSqlComments(content) {
|
|
63
|
+
let out = '';
|
|
64
|
+
let i = 0;
|
|
65
|
+
while (i < content.length) {
|
|
66
|
+
const two = content.slice(i, i + 2);
|
|
67
|
+
if (two === '--') {
|
|
68
|
+
while (i < content.length && content[i] !== '\n') {
|
|
69
|
+
out += ' ';
|
|
70
|
+
i++;
|
|
71
|
+
}
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
if (two === '/*') {
|
|
75
|
+
while (i < content.length && content.slice(i, i + 2) !== '*/') {
|
|
76
|
+
out += content[i] === '\n' ? '\n' : ' ';
|
|
77
|
+
i++;
|
|
78
|
+
}
|
|
79
|
+
out += ' ';
|
|
80
|
+
i += 2;
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
// A quoted literal may legitimately contain -- or parens; copy it whole.
|
|
84
|
+
if (content[i] === "'" || content[i] === '"') {
|
|
85
|
+
const quote = content[i];
|
|
86
|
+
out += content[i];
|
|
87
|
+
i++;
|
|
88
|
+
while (i < content.length) {
|
|
89
|
+
out += content[i];
|
|
90
|
+
if (content[i] === quote) {
|
|
91
|
+
i++;
|
|
92
|
+
break;
|
|
93
|
+
}
|
|
94
|
+
i++;
|
|
95
|
+
}
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
out += content[i];
|
|
99
|
+
i++;
|
|
100
|
+
}
|
|
101
|
+
return out;
|
|
102
|
+
}
|
|
103
|
+
/** Contents between `content[open]` ('(') and its matching ')', or null. */
|
|
104
|
+
function tableBody(content, open) {
|
|
105
|
+
let depth = 0;
|
|
106
|
+
for (let i = open; i < content.length; i++) {
|
|
107
|
+
if (content[i] === '(')
|
|
108
|
+
depth++;
|
|
109
|
+
else if (content[i] === ')') {
|
|
110
|
+
depth--;
|
|
111
|
+
if (depth === 0)
|
|
112
|
+
return content.slice(open + 1, i);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return null;
|
|
116
|
+
}
|
|
51
117
|
/**
|
|
52
118
|
* Extract SQLite CREATE TABLE columns from SQL or source code.
|
|
53
119
|
*/
|
|
54
|
-
function extractSQLiteColumns(
|
|
120
|
+
function extractSQLiteColumns(raw) {
|
|
55
121
|
const columns = new Map();
|
|
56
|
-
const
|
|
122
|
+
const content = stripSqlComments(raw);
|
|
123
|
+
const createTables = content.matchAll(/CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?(\w+)\s*\(/gi);
|
|
57
124
|
for (const table of createTables) {
|
|
58
125
|
const tableName = table[1];
|
|
59
|
-
|
|
126
|
+
// Paren-matched, not `\(([\s\S]*?)\)`: the non-greedy form stopped at the
|
|
127
|
+
// FIRST `)`, so one `amt DECIMAL(10,2)` hid every column declared after it
|
|
128
|
+
// — including a later NOT NULL addition, which a blocking gate must see.
|
|
129
|
+
const body = tableBody(content, (table.index ?? 0) + table[0].length - 1);
|
|
130
|
+
if (body === null)
|
|
131
|
+
continue;
|
|
60
132
|
const colMatches = body.matchAll(/(\w+)\s+(INTEGER|TEXT|REAL|BLOB|DATETIME|BOOLEAN|VARCHAR[^,]*)/gi);
|
|
61
133
|
for (const col of colMatches) {
|
|
62
134
|
columns.set(`${tableName}.${col[1]}`, col[2].trim());
|
|
@@ -64,6 +136,135 @@ function extractSQLiteColumns(content) {
|
|
|
64
136
|
}
|
|
65
137
|
return columns;
|
|
66
138
|
}
|
|
139
|
+
/** Top-level comma split: `DECIMAL(10,2)` is one part, not two. */
|
|
140
|
+
function splitTopLevel(body) {
|
|
141
|
+
const parts = [];
|
|
142
|
+
let depth = 0;
|
|
143
|
+
let cur = '';
|
|
144
|
+
for (const ch of body) {
|
|
145
|
+
if (ch === '(')
|
|
146
|
+
depth++;
|
|
147
|
+
else if (ch === ')')
|
|
148
|
+
depth--;
|
|
149
|
+
if (ch === ',' && depth === 0) {
|
|
150
|
+
parts.push(cur);
|
|
151
|
+
cur = '';
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
cur += ch;
|
|
155
|
+
}
|
|
156
|
+
if (cur.trim())
|
|
157
|
+
parts.push(cur);
|
|
158
|
+
return parts;
|
|
159
|
+
}
|
|
160
|
+
/** Clauses that describe the TABLE, not a column. */
|
|
161
|
+
const TABLE_CONSTRAINT = /^\s*(PRIMARY|FOREIGN|UNIQUE|CHECK|CONSTRAINT|KEY|INDEX|EXCLUDE|LIKE|INHERITS)\b/i;
|
|
162
|
+
/**
|
|
163
|
+
* Columns the schema makes MANDATORY: NOT NULL with nothing to fill them.
|
|
164
|
+
*
|
|
165
|
+
* Needed because the generic added-field rule ("breaking unless the type says
|
|
166
|
+
* optional or ?") is Zod/TypeScript-shaped and reads every SQL type as
|
|
167
|
+
* required — so a plain `ADD COLUMN note TEXT`, the most common and most
|
|
168
|
+
* harmless migration there is, was reported BREAKING. Harmless while the
|
|
169
|
+
* checker only printed advice; not harmless now that a gate blocks on the
|
|
170
|
+
* verdict, where it would refuse ordinary additive migrations and teach people
|
|
171
|
+
* to reach for the waiver.
|
|
172
|
+
*
|
|
173
|
+
* A column is only genuinely breaking to add when existing rows and existing
|
|
174
|
+
* INSERTs cannot satisfy it: NOT NULL, no DEFAULT, and not self-filling
|
|
175
|
+
* (SERIAL/IDENTITY/AUTOINCREMENT/GENERATED).
|
|
176
|
+
*
|
|
177
|
+
* The table body is matched by counting parens rather than with the
|
|
178
|
+
* non-greedy `\(([\s\S]*?)\)` used elsewhere here, which stops at the first
|
|
179
|
+
* `)` and so loses every column after a `DECIMAL(10,2)`. Over-collecting is
|
|
180
|
+
* safe: this set only ever classifies names the diff already produced.
|
|
181
|
+
*/
|
|
182
|
+
function sqlRequiredColumns(raw) {
|
|
183
|
+
const required = new Set();
|
|
184
|
+
const content = stripSqlComments(raw);
|
|
185
|
+
const heads = content.matchAll(/CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?([\w."`\[\]]+)\s*\(/gi);
|
|
186
|
+
for (const head of heads) {
|
|
187
|
+
const table = (head[1] ?? '').replace(/["`[\]]/g, '').split('.').pop() ?? '';
|
|
188
|
+
const body = tableBody(content, (head.index ?? 0) + head[0].length - 1);
|
|
189
|
+
if (body === null)
|
|
190
|
+
continue;
|
|
191
|
+
for (const part of splitTopLevel(body)) {
|
|
192
|
+
if (TABLE_CONSTRAINT.test(part))
|
|
193
|
+
continue;
|
|
194
|
+
const name = part.trim().match(/^["`[]?(\w+)/)?.[1];
|
|
195
|
+
if (!name)
|
|
196
|
+
continue;
|
|
197
|
+
const notNull = /\bNOT\s+NULL\b/i.test(part);
|
|
198
|
+
const filled = /\b(DEFAULT|SERIAL|BIGSERIAL|SMALLSERIAL|AUTOINCREMENT|IDENTITY|GENERATED)\b/i.test(part);
|
|
199
|
+
if (notNull && !filled)
|
|
200
|
+
required.add(`${table}.${name}`);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return required;
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Apply the SQL rule to `added` verdicts produced by the generic differ.
|
|
207
|
+
*
|
|
208
|
+
* Only additions are re-scored: removals and type changes keep the generic
|
|
209
|
+
* judgement, which is right for SQL too.
|
|
210
|
+
*/
|
|
211
|
+
function rescoreSqlAdditions(changes, afterContent, context) {
|
|
212
|
+
const required = sqlRequiredColumns(afterContent);
|
|
213
|
+
return changes.map((c) => {
|
|
214
|
+
if (c.type !== 'added')
|
|
215
|
+
return c;
|
|
216
|
+
const column = c.path.startsWith(`${context}.`) ? c.path.slice(context.length + 1) : c.path;
|
|
217
|
+
const breaking = required.has(column);
|
|
218
|
+
if (breaking === c.breaking)
|
|
219
|
+
return c;
|
|
220
|
+
return {
|
|
221
|
+
...c,
|
|
222
|
+
breaking,
|
|
223
|
+
description: breaking
|
|
224
|
+
? `${c.description.replace(/ \(required — breaking\)$/, '')} (NOT NULL with no default — breaking)`
|
|
225
|
+
: c.description.replace(/ \(required — breaking\)$/, ''),
|
|
226
|
+
};
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
/**
|
|
230
|
+
* Destructive DDL statements, as human-readable descriptions.
|
|
231
|
+
*
|
|
232
|
+
* The CREATE TABLE differ can only speak about a file that HAS a previous
|
|
233
|
+
* version to compare against, which excludes the single most common breaking
|
|
234
|
+
* change there is: a brand-new migration whose whole content is `DROP TABLE
|
|
235
|
+
* users;`. Nothing was removed relative to the old file, because there is no
|
|
236
|
+
* old file -- so the differ said "no changes" and the gate cleared it.
|
|
237
|
+
*
|
|
238
|
+
* These statements are breaking on sight, no baseline required.
|
|
239
|
+
*/
|
|
240
|
+
function destructiveDdl(raw) {
|
|
241
|
+
const sql = stripSqlComments(raw);
|
|
242
|
+
const found = [];
|
|
243
|
+
const scan = (re, describe) => {
|
|
244
|
+
for (const m of sql.matchAll(re))
|
|
245
|
+
found.push(describe(m));
|
|
246
|
+
};
|
|
247
|
+
scan(/\bDROP\s+TABLE\s+(?:IF\s+EXISTS\s+)?([\w."`[\]]+)/gi, (m) => `DROP TABLE ${m[1]}`);
|
|
248
|
+
scan(/\bDROP\s+(?:MATERIALIZED\s+)?VIEW\s+(?:IF\s+EXISTS\s+)?([\w."`[\]]+)/gi, (m) => `DROP VIEW ${m[1]}`);
|
|
249
|
+
scan(/\bDROP\s+INDEX\s+(?:IF\s+EXISTS\s+)?([\w."`[\]]+)/gi, (m) => `DROP INDEX ${m[1]}`);
|
|
250
|
+
scan(/\bTRUNCATE\s+(?:TABLE\s+)?([\w."`[\]]+)/gi, (m) => `TRUNCATE ${m[1]}`);
|
|
251
|
+
scan(/\bALTER\s+TABLE\s+([\w."`[\]]+)\s+DROP\s+(?:COLUMN\s+)?(?:IF\s+EXISTS\s+)?([\w."`[\]]+)/gi, (m) => `ALTER TABLE ${m[1]} DROP COLUMN ${m[2]}`);
|
|
252
|
+
scan(/\bALTER\s+TABLE\s+([\w."`[\]]+)\s+RENAME\s+(?:COLUMN\s+)?([\w."`[\]]+)\s+TO\s+([\w."`[\]]+)/gi, (m) => `ALTER TABLE ${m[1]} RENAME ${m[2]} TO ${m[3]}`);
|
|
253
|
+
scan(/\bALTER\s+TABLE\s+([\w."`[\]]+)\s+RENAME\s+TO\s+([\w."`[\]]+)/gi, (m) => `ALTER TABLE ${m[1]} RENAME TO ${m[2]}`);
|
|
254
|
+
scan(/\bALTER\s+TABLE\s+([\w."`[\]]+)\s+ALTER\s+(?:COLUMN\s+)?([\w."`[\]]+)\s+(?:SET\s+DATA\s+)?TYPE\b/gi, (m) => `ALTER TABLE ${m[1]} ALTER COLUMN ${m[2]} TYPE`);
|
|
255
|
+
scan(/\bALTER\s+TABLE\s+([\w."`[\]]+)\s+ALTER\s+(?:COLUMN\s+)?([\w."`[\]]+)\s+SET\s+NOT\s+NULL/gi, (m) => `ALTER TABLE ${m[1]} SET NOT NULL on ${m[2]}`);
|
|
256
|
+
// ADD COLUMN is breaking only when existing rows cannot satisfy it, the same
|
|
257
|
+
// rule sqlRequiredColumns applies inside CREATE TABLE.
|
|
258
|
+
for (const m of sql.matchAll(/\bALTER\s+TABLE\s+([\w."`[\]]+)\s+ADD\s+(?:COLUMN\s+)?(?:IF\s+NOT\s+EXISTS\s+)?([\w."`[\]]+)([^;]*)/gi)) {
|
|
259
|
+
const tail = m[3] ?? '';
|
|
260
|
+
const notNull = /\bNOT\s+NULL\b/i.test(tail);
|
|
261
|
+
const filled = /\b(DEFAULT|SERIAL|BIGSERIAL|SMALLSERIAL|AUTOINCREMENT|IDENTITY|GENERATED)\b/i.test(tail);
|
|
262
|
+
if (notNull && !filled) {
|
|
263
|
+
found.push(`ALTER TABLE ${m[1]} ADD COLUMN ${m[2]} NOT NULL with no default`);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
return found;
|
|
267
|
+
}
|
|
67
268
|
/**
|
|
68
269
|
* Compare two field maps and produce a list of changes.
|
|
69
270
|
*/
|
|
@@ -125,11 +326,114 @@ function gitEnv() {
|
|
|
125
326
|
}
|
|
126
327
|
return env;
|
|
127
328
|
}
|
|
128
|
-
|
|
329
|
+
/**
|
|
330
|
+
* Both versions of a path, because a commit may take either.
|
|
331
|
+
*
|
|
332
|
+
* `git commit` stores the INDEX; `git commit -a` stores the WORKTREE. Callers
|
|
333
|
+
* need to know both to decide which one a given commit will take — but only
|
|
334
|
+
* the analysed one is ever RECORDED (see committedVersion), because a marker
|
|
335
|
+
* that names bytes nothing inspected is the bug this whole change exists to
|
|
336
|
+
* remove.
|
|
337
|
+
*/
|
|
338
|
+
async function fileVersions(path, cwd) {
|
|
339
|
+
const { execFileSync } = await import('child_process');
|
|
340
|
+
const opts = { cwd, env: gitEnv(), encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] };
|
|
341
|
+
const hex = (s) => (/^[0-9a-f]{40}$|^[0-9a-f]{64}$/.test(s) ? s : '');
|
|
342
|
+
let index = '';
|
|
343
|
+
let worktree = '';
|
|
344
|
+
try {
|
|
345
|
+
index = hex(execFileSync('git', ['rev-parse', `:${path}`], opts).trim());
|
|
346
|
+
}
|
|
347
|
+
catch {
|
|
348
|
+
/* not staged */
|
|
349
|
+
}
|
|
350
|
+
try {
|
|
351
|
+
worktree = hex(execFileSync('git', ['hash-object', '--', path], opts).trim());
|
|
352
|
+
}
|
|
353
|
+
catch {
|
|
354
|
+
/* deleted, unreadable, sparse */
|
|
355
|
+
}
|
|
356
|
+
return { index, worktree };
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* The version that will be committed, and the bytes to analyse.
|
|
360
|
+
*
|
|
361
|
+
* INDEX-preferred, because `git commit` stores the index. Recording the
|
|
362
|
+
* worktree instead is what let a staged-malicious / benign-worktree pair clear
|
|
363
|
+
* a gate claiming to cover "staged content"; and recording BOTH without
|
|
364
|
+
* analysing both would re-open exactly that, since the marker would vouch for
|
|
365
|
+
* bytes nothing inspected. So the analysed bytes and the recorded sha are the
|
|
366
|
+
* same object, always.
|
|
367
|
+
*
|
|
368
|
+
* When the worktree has since diverged, only the index version is vouched for.
|
|
369
|
+
* A subsequent `git commit -a` would take the worktree, which the enforcer then
|
|
370
|
+
* finds uncovered and blocks — clearable with `git add` and a re-run.
|
|
371
|
+
*/
|
|
372
|
+
async function committedVersion(path, cwd, source = 'index') {
|
|
373
|
+
const { execFileSync } = await import('child_process');
|
|
374
|
+
const opts = {
|
|
375
|
+
cwd,
|
|
376
|
+
env: gitEnv(),
|
|
377
|
+
encoding: 'utf-8',
|
|
378
|
+
stdio: ['ignore', 'pipe', 'ignore'],
|
|
379
|
+
};
|
|
380
|
+
const v = await fileVersions(path, cwd);
|
|
381
|
+
// `commit -a` / `--all` stores the worktree; everything else the index. The
|
|
382
|
+
// GATE decides which and tells us, so the bytes examined here are the bytes
|
|
383
|
+
// that command will store — no inference, no drift.
|
|
384
|
+
const preferWorktree = source === 'worktree';
|
|
385
|
+
if (!preferWorktree && v.index) {
|
|
386
|
+
try {
|
|
387
|
+
const content = execFileSync('git', ['cat-file', 'blob', v.index], {
|
|
388
|
+
...opts,
|
|
389
|
+
maxBuffer: 32 * 1024 * 1024,
|
|
390
|
+
});
|
|
391
|
+
return { sha: v.index, content };
|
|
392
|
+
}
|
|
393
|
+
catch {
|
|
394
|
+
return { sha: v.index, content: null };
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
if (v.worktree) {
|
|
398
|
+
const abs = isAbsolute(path) ? path : join(cwd, path);
|
|
399
|
+
try {
|
|
400
|
+
return { sha: v.worktree, content: readFileSync(abs, 'utf-8') };
|
|
401
|
+
}
|
|
402
|
+
catch {
|
|
403
|
+
return { sha: v.worktree, content: null };
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
return { sha: '', content: null };
|
|
407
|
+
}
|
|
408
|
+
async function hashFiles(paths, cwd, source = 'index') {
|
|
409
|
+
if (paths.length === 0)
|
|
410
|
+
return [];
|
|
411
|
+
const out = [];
|
|
412
|
+
for (const path of paths) {
|
|
413
|
+
// The source's OWN blob, with no cross-source fallback.
|
|
414
|
+
// committedVersion falls back to the worktree when the index has no entry,
|
|
415
|
+
// which is right for choosing bytes to analyse but wrong as an identity
|
|
416
|
+
// claim: for `git rm --cached x` it reported the worktree sha for a path
|
|
417
|
+
// the index no longer holds, so the gate's sha comparison disagreed and a
|
|
418
|
+
// precisely-detected staged deletion decayed into "could not answer".
|
|
419
|
+
// Empty means "this source holds nothing here" — itself a fact the gate
|
|
420
|
+
// needs, since that is exactly what a deletion looks like.
|
|
421
|
+
const v = await fileVersions(path, cwd);
|
|
422
|
+
out.push({ path, sha: source === 'worktree' ? v.worktree : v.index });
|
|
423
|
+
}
|
|
424
|
+
return out;
|
|
425
|
+
}
|
|
426
|
+
export async function diffFileSchema(filePath, baseBranch = 'HEAD~1', cwd = process.cwd(), source = 'index') {
|
|
129
427
|
const changes = [];
|
|
428
|
+
const isSql = filePath.endsWith('.sql');
|
|
429
|
+
// False until an analyser proves otherwise. Every early exit and the outer
|
|
430
|
+
// catch therefore report "not analysed", which the gate treats as uncovered
|
|
431
|
+
// rather than as clean.
|
|
432
|
+
let analysed = false;
|
|
130
433
|
try {
|
|
131
434
|
// Get the "before" version from git
|
|
132
435
|
let beforeContent;
|
|
436
|
+
let hasBaseline = true;
|
|
133
437
|
try {
|
|
134
438
|
// execFileSync with an argv array: the old template string ran through a
|
|
135
439
|
// shell, and `filePath` comes straight from `git diff --name-only`, which
|
|
@@ -144,8 +448,13 @@ export async function diffFileSchema(filePath, baseBranch = 'HEAD~1', cwd = proc
|
|
|
144
448
|
});
|
|
145
449
|
}
|
|
146
450
|
catch {
|
|
147
|
-
//
|
|
148
|
-
|
|
451
|
+
// No previous version. Adding fields to something that did not exist
|
|
452
|
+
// cannot break a reader of the OLD schema -- but the file's own
|
|
453
|
+
// statements still can, and a new migration is where they usually live.
|
|
454
|
+
// Returning "no changes" here is what cleared a brand-new
|
|
455
|
+
// migrations/002.sql whose entire content was DROP TABLE users;.
|
|
456
|
+
beforeContent = '';
|
|
457
|
+
hasBaseline = false;
|
|
149
458
|
}
|
|
150
459
|
// Get the "after" version from working tree
|
|
151
460
|
// Resolved against the run's cwd: `git diff --name-only` emits
|
|
@@ -153,8 +462,33 @@ export async function diffFileSchema(filePath, baseBranch = 'HEAD~1', cwd = proc
|
|
|
153
462
|
// every file look DELETED when run from a subdirectory — reporting
|
|
154
463
|
// breaking:true for each and blocking the very commit it was asked to
|
|
155
464
|
// clear. (join() leaves an absolute path untouched.)
|
|
465
|
+
// Analyse the version that will be COMMITTED (index-preferred), so the
|
|
466
|
+
// bytes inspected here are the same object whose sha the marker records.
|
|
467
|
+
// Reading the worktree while recording the index — or the reverse — is how
|
|
468
|
+
// a marker ends up vouching for bytes nothing looked at.
|
|
469
|
+
const committed = await committedVersion(filePath, cwd, source);
|
|
156
470
|
const absPath = isAbsolute(filePath) ? filePath : join(cwd, filePath);
|
|
157
|
-
|
|
471
|
+
// A path removed from the INDEX is a staged deletion even though the
|
|
472
|
+
// worktree copy is still sitting there. committedVersion falls back to the
|
|
473
|
+
// worktree when the index has no entry, so `git rm --cached x` was
|
|
474
|
+
// analysed as "worktree vs HEAD: unchanged" -- clean -- while the commit
|
|
475
|
+
// removed the file. Verified.
|
|
476
|
+
if (hasBaseline && source === 'index' && !(await fileVersions(filePath, cwd)).index) {
|
|
477
|
+
return {
|
|
478
|
+
file: filePath,
|
|
479
|
+
changes: [
|
|
480
|
+
{
|
|
481
|
+
type: 'removed',
|
|
482
|
+
path: filePath,
|
|
483
|
+
description: `File "${filePath}" was removed from the index (staged deletion)`,
|
|
484
|
+
breaking: true,
|
|
485
|
+
},
|
|
486
|
+
],
|
|
487
|
+
breaking: true,
|
|
488
|
+
analysed: true,
|
|
489
|
+
};
|
|
490
|
+
}
|
|
491
|
+
if (committed.content === null && !existsSync(absPath)) {
|
|
158
492
|
// File was deleted — all fields removed (breaking)
|
|
159
493
|
return {
|
|
160
494
|
file: filePath,
|
|
@@ -167,11 +501,28 @@ export async function diffFileSchema(filePath, baseBranch = 'HEAD~1', cwd = proc
|
|
|
167
501
|
},
|
|
168
502
|
],
|
|
169
503
|
breaking: true,
|
|
504
|
+
analysed: true,
|
|
170
505
|
};
|
|
171
506
|
}
|
|
172
|
-
const afterContent = readFileSync(absPath, 'utf-8');
|
|
507
|
+
const afterContent = committed.content ?? readFileSync(absPath, 'utf-8');
|
|
508
|
+
// Statements that are breaking on their own terms, baseline or not.
|
|
509
|
+
const before = new Set(hasBaseline ? destructiveDdl(beforeContent) : []);
|
|
510
|
+
for (const stmt of destructiveDdl(afterContent)) {
|
|
511
|
+
if (before.has(stmt))
|
|
512
|
+
continue; // already in the base; not new here
|
|
513
|
+
changes.push({
|
|
514
|
+
type: 'removed',
|
|
515
|
+
path: filePath,
|
|
516
|
+
description: `Destructive statement: ${stmt}`,
|
|
517
|
+
breaking: true,
|
|
518
|
+
});
|
|
519
|
+
}
|
|
173
520
|
// Detect schema type and extract fields
|
|
521
|
+
let understood = isSql; // .sql is covered: CREATE TABLE diff + DDL scan
|
|
174
522
|
if (filePath.endsWith('.ts') || filePath.endsWith('.js')) {
|
|
523
|
+
// Only meaningful against a baseline -- there is no TS analogue of the
|
|
524
|
+
// DDL scan, so a new file tells us nothing about what it breaks.
|
|
525
|
+
understood = hasBaseline;
|
|
175
526
|
// Check for Zod schemas
|
|
176
527
|
if (beforeContent.includes('z.object') || afterContent.includes('z.object')) {
|
|
177
528
|
const beforeFields = extractZodFields(beforeContent);
|
|
@@ -188,14 +539,14 @@ export async function diffFileSchema(filePath, baseBranch = 'HEAD~1', cwd = proc
|
|
|
188
539
|
if (beforeContent.includes('CREATE TABLE') || afterContent.includes('CREATE TABLE')) {
|
|
189
540
|
const beforeCols = extractSQLiteColumns(beforeContent);
|
|
190
541
|
const afterCols = extractSQLiteColumns(afterContent);
|
|
191
|
-
changes.push(...diffFields(beforeCols, afterCols, 'sqlite'));
|
|
542
|
+
changes.push(...rescoreSqlAdditions(diffFields(beforeCols, afterCols, 'sqlite'), afterContent, 'sqlite'));
|
|
192
543
|
}
|
|
193
544
|
}
|
|
194
545
|
// SQL files
|
|
195
546
|
if (filePath.endsWith('.sql')) {
|
|
196
547
|
const beforeCols = extractSQLiteColumns(beforeContent);
|
|
197
548
|
const afterCols = extractSQLiteColumns(afterContent);
|
|
198
|
-
changes.push(...diffFields(beforeCols, afterCols, 'sql'));
|
|
549
|
+
changes.push(...rescoreSqlAdditions(diffFields(beforeCols, afterCols, 'sql'), afterContent, 'sql'));
|
|
199
550
|
}
|
|
200
551
|
// JSON schema files
|
|
201
552
|
if (filePath.endsWith('.json') &&
|
|
@@ -203,6 +554,7 @@ export async function diffFileSchema(filePath, baseBranch = 'HEAD~1', cwd = proc
|
|
|
203
554
|
try {
|
|
204
555
|
const beforeObj = JSON.parse(beforeContent);
|
|
205
556
|
const afterObj = JSON.parse(afterContent);
|
|
557
|
+
understood = hasBaseline;
|
|
206
558
|
const flatBefore = flattenObject(beforeObj);
|
|
207
559
|
const flatAfter = flattenObject(afterObj);
|
|
208
560
|
const beforeKeys = new Map(Object.keys(flatBefore).map((k) => [k, typeof flatBefore[k]]));
|
|
@@ -210,17 +562,21 @@ export async function diffFileSchema(filePath, baseBranch = 'HEAD~1', cwd = proc
|
|
|
210
562
|
changes.push(...diffFields(beforeKeys, afterKeys, 'json'));
|
|
211
563
|
}
|
|
212
564
|
catch {
|
|
213
|
-
// Not valid JSON
|
|
565
|
+
// Not valid JSON — say so rather than reporting a clean parse.
|
|
566
|
+
understood = false;
|
|
214
567
|
}
|
|
215
568
|
}
|
|
569
|
+
analysed = understood;
|
|
216
570
|
}
|
|
217
571
|
catch {
|
|
218
|
-
// Git or file read error
|
|
572
|
+
// Git or file read error. `analysed` stays false: the gate must not read a
|
|
573
|
+
// failed read as an all-clear.
|
|
219
574
|
}
|
|
220
575
|
return {
|
|
221
576
|
file: filePath,
|
|
222
577
|
changes,
|
|
223
578
|
breaking: changes.some((c) => c.breaking),
|
|
579
|
+
analysed,
|
|
224
580
|
};
|
|
225
581
|
}
|
|
226
582
|
/**
|
|
@@ -266,8 +622,37 @@ async function baseResolves(base, cwd) {
|
|
|
266
622
|
return false;
|
|
267
623
|
}
|
|
268
624
|
}
|
|
625
|
+
/**
|
|
626
|
+
* What the GATE watches — kept identical to WATCHED_RE in
|
|
627
|
+
* src/policies/enforcers/schema_diff_gate.py.
|
|
628
|
+
*
|
|
629
|
+
* The two sets must agree, because the gate now demands that every watched path
|
|
630
|
+
* be covered by the marker, and only paths this CLI EXAMINES get into the
|
|
631
|
+
* marker. When the gate watched a superset, the difference was a permanent
|
|
632
|
+
* deadlock: `infra/helm_charts/pgdog/values.yaml` matched none of the filters
|
|
633
|
+
* below, so it could never be recorded, and the refusal told the operator to
|
|
634
|
+
* re-run a command that produced the identical marker. Verified before the fix.
|
|
635
|
+
*
|
|
636
|
+
* These files are hashed and recorded even though diffFileSchema cannot parse
|
|
637
|
+
* YAML — coverage is about "these bytes were seen", which is exactly the claim
|
|
638
|
+
* the marker needs to support. Any change to the enforcer's WATCHED_RE must be
|
|
639
|
+
* mirrored here; test/cli/gate-watched-mirror.test.ts pins that.
|
|
640
|
+
*/
|
|
641
|
+
export // `s` (dotAll) so `.` spans a newline: a path may legally contain one, and
|
|
642
|
+
|
|
643
|
+
// without the flag `migrations/a\nb.sql` matched nothing — the gate did not
|
|
644
|
+
// consider it watched at all, and a column drop in it was never examined.
|
|
645
|
+
const GATE_WATCHED_RE = /(migrations\/.*\.sql|infra\/postgres-spock\/|infra\/helm_charts\/[^/]*pgdog|infra\/helm_charts\/[^/]*cnpg|infra\/helm_charts\/[^/]*redis|infra\/helm_charts\/[^/]*envoy|infra\/helm_charts\/[^/]*sentinel)/is;
|
|
269
646
|
/** git's empty tree — diffing against it means "every tracked file is new". */
|
|
270
647
|
const EMPTY_TREE = '4b825dc642cb6eb9a060e54bf8d69288fbee4904';
|
|
648
|
+
/**
|
|
649
|
+
* Version of the --json verdict shape.
|
|
650
|
+
*
|
|
651
|
+
* Bump on any change to the fields the gate reads. The gate refuses a contract
|
|
652
|
+
* it does not recognise and falls back rather than guessing, so an old gate
|
|
653
|
+
* paired with a new CLI degrades instead of misreading.
|
|
654
|
+
*/
|
|
655
|
+
export const SCHEMA_DIFF_CONTRACT = 1;
|
|
271
656
|
export async function runSchemaDiff(baseBranch = 'HEAD~1', cwd = process.cwd(),
|
|
272
657
|
/**
|
|
273
658
|
* True when the caller did not pass -b, i.e. `HEAD~1` is our default rather
|
|
@@ -280,7 +665,8 @@ export async function runSchemaDiff(baseBranch = 'HEAD~1', cwd = process.cwd(),
|
|
|
280
665
|
* commit with no waiver and no override — verified. Falling back
|
|
281
666
|
* unconditionally would instead resurrect the stamp for `-b typo`.
|
|
282
667
|
*/
|
|
283
|
-
baseIsDefault = false) {
|
|
668
|
+
baseIsDefault = false, opts = {}) {
|
|
669
|
+
const { only, source = 'index', quiet = false } = opts;
|
|
284
670
|
const results = [];
|
|
285
671
|
let examined = [];
|
|
286
672
|
let ran = false;
|
|
@@ -323,7 +709,8 @@ baseIsDefault = false) {
|
|
|
323
709
|
return { ran, examined, effectiveBase, results };
|
|
324
710
|
}
|
|
325
711
|
// Filter to schema-relevant files
|
|
326
|
-
const schemaFiles = changedFiles.filter((f) =>
|
|
712
|
+
const schemaFiles = changedFiles.filter((f) => GATE_WATCHED_RE.test(f) ||
|
|
713
|
+
f.includes('schema') ||
|
|
327
714
|
f.includes('types') ||
|
|
328
715
|
f.includes('config') ||
|
|
329
716
|
f.includes('database') ||
|
|
@@ -335,26 +722,46 @@ baseIsDefault = false) {
|
|
|
335
722
|
f.includes('memory/short-term/schema') ||
|
|
336
723
|
f.includes('tasks/database') ||
|
|
337
724
|
f.includes('tasks/types'))));
|
|
725
|
+
// `only` REPLACES this enumeration rather than intersecting it. The list
|
|
726
|
+
// above comes from `git diff --name-only <base>`, which compares the
|
|
727
|
+
// WORKTREE — so a staged-only change whose worktree copy still matches the
|
|
728
|
+
// base does not appear in it at all. Intersecting would have dropped
|
|
729
|
+
// exactly those paths, handed the gate an empty `examined`, and let the
|
|
730
|
+
// staged change through as "covered": the same class of hole as verifying
|
|
731
|
+
// the index while `-a` commits the worktree. The gate derives its watched
|
|
732
|
+
// set from the source it is about to commit, and it is authoritative.
|
|
733
|
+
const scopedFiles = only ? [...new Set(only)] : schemaFiles;
|
|
338
734
|
// Past this point git has produced a real file list, so the run is genuine
|
|
339
735
|
// even if nothing schema-relevant changed.
|
|
340
736
|
ran = true;
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
737
|
+
// Hash what we are about to read, so the marker binds to content. Batched
|
|
738
|
+
// in one process rather than one per file: this runs inside a gate remedy.
|
|
739
|
+
examined = await hashFiles(scopedFiles, cwd, source);
|
|
740
|
+
for (const file of scopedFiles) {
|
|
741
|
+
const result = await diffFileSchema(file, effectiveBase, cwd, source);
|
|
742
|
+
// Every examined path gets a result, not just the ones with changes: the
|
|
743
|
+
// gate needs `analysed` for the quiet ones too, and "absent from the
|
|
744
|
+
// list" is precisely the ambiguity between "clean" and "never looked at"
|
|
745
|
+
// that this change exists to remove. The human report still prints only
|
|
746
|
+
// files with changes.
|
|
747
|
+
results.push(result);
|
|
748
|
+
}
|
|
749
|
+
// Print results. In --json mode the verdict IS the output: a stray human
|
|
750
|
+
// line here would break the gate's JSON parse, and a gate that cannot read
|
|
751
|
+
// its checker falls back to allow-with-warning — silently unarmed.
|
|
752
|
+
const changed = results.filter((r) => r.changes.length > 0);
|
|
753
|
+
if (quiet) {
|
|
754
|
+
// caller renders the verdict
|
|
347
755
|
}
|
|
348
|
-
|
|
349
|
-
if (results.length === 0) {
|
|
756
|
+
else if (changed.length === 0) {
|
|
350
757
|
console.log('No schema changes detected.');
|
|
351
758
|
}
|
|
352
759
|
else {
|
|
353
|
-
const hasBreaking =
|
|
354
|
-
console.log(`\nSchema Diff Results (${
|
|
760
|
+
const hasBreaking = changed.some((r) => r.breaking);
|
|
761
|
+
console.log(`\nSchema Diff Results (${changed.length} files with changes):`);
|
|
355
762
|
console.log(hasBreaking ? ' BREAKING CHANGES DETECTED' : ' No breaking changes');
|
|
356
763
|
console.log('');
|
|
357
|
-
for (const result of
|
|
764
|
+
for (const result of changed) {
|
|
358
765
|
console.log(` ${result.breaking ? 'BREAKING' : 'OK'} ${result.file}`);
|
|
359
766
|
for (const change of result.changes) {
|
|
360
767
|
const icon = change.breaking ? ' !!!' : ' ';
|
|
@@ -387,11 +794,36 @@ baseIsDefault = false) {
|
|
|
387
794
|
/** Bounded rendering: one branch touching hundreds of schema files otherwise
|
|
388
795
|
* writes a multi-kilobyte row into the shared short-term store, at high
|
|
389
796
|
* importance, where it crowds out real context and survives pruning. */
|
|
797
|
+
/**
|
|
798
|
+
* How many `path@sha` entries a marker may carry before it degrades to a
|
|
799
|
+
* legacy (time-only) marker.
|
|
800
|
+
*
|
|
801
|
+
* Content scoping needs EVERY examined path in the marker — a truncated list
|
|
802
|
+
* would leave the omitted paths uncovered and block them with no way for the
|
|
803
|
+
* operator to produce a covering marker. Rather than deadlock, an oversized
|
|
804
|
+
* change records `(truncated)` and the gate falls back to time-only. 100
|
|
805
|
+
* entries is roughly 5KB, above any realistic schema change and well under the
|
|
806
|
+
* size where one memory row starts crowding recall.
|
|
807
|
+
*/
|
|
808
|
+
const MAX_MARKER_ENTRIES = 100;
|
|
809
|
+
/** Marker rendering. `path@sha7` is what makes a pass content-scoped. */
|
|
390
810
|
function fileList(examined) {
|
|
391
811
|
if (examined.length === 0)
|
|
392
812
|
return '(none changed)';
|
|
393
|
-
|
|
394
|
-
|
|
813
|
+
if (examined.length > MAX_MARKER_ENTRIES) {
|
|
814
|
+
return `(truncated: ${examined.length} files)`;
|
|
815
|
+
}
|
|
816
|
+
return examined
|
|
817
|
+
.map((f) => {
|
|
818
|
+
// A path containing the delimiters can forge a second, parsable entry:
|
|
819
|
+
// a file named `xschema@<40hex>,migrations/002.sql@<benign sha>` parses
|
|
820
|
+
// into a bogus coverage claim for 002.sql that overwrites the real one.
|
|
821
|
+
// Such a path is recorded WITHOUT identity, so the enforcer treats that
|
|
822
|
+
// one path as uncovered instead of trusting an attacker-built entry.
|
|
823
|
+
const safe = !f.path.includes(',') && !f.path.includes('@');
|
|
824
|
+
return f.sha && safe ? `${f.path}@${f.sha}` : f.path;
|
|
825
|
+
})
|
|
826
|
+
.join(',');
|
|
395
827
|
}
|
|
396
828
|
export async function recordSchemaDiffPass(baseBranch, examined, cwd = process.cwd()) {
|
|
397
829
|
try {
|
|
@@ -457,11 +889,74 @@ export function registerSchemaDiffCommand(program) {
|
|
|
457
889
|
// No commander default: an omitted -b must be distinguishable from an
|
|
458
890
|
// explicit one, because only the default may fall back to the empty tree.
|
|
459
891
|
.option('-b, --base <branch>', 'Base branch/commit to compare against (default: HEAD~1)')
|
|
892
|
+
// --- the inline-gate contract -------------------------------------------
|
|
893
|
+
// The enforcer calls this directly instead of hunting for a marker written
|
|
894
|
+
// by some earlier run. That deletes an entire class of defect: a stored
|
|
895
|
+
// verdict can be stale, forged, truncated, format-drifted, scoped to a
|
|
896
|
+
// different file set, or written by another worktree — and every one of
|
|
897
|
+
// those actually happened here. A verdict computed now, over the exact
|
|
898
|
+
// paths the gate is about to allow, can be none of them.
|
|
899
|
+
.option('--json', 'Emit a machine-readable verdict for the gate (implies no marker write)')
|
|
900
|
+
.option('--paths-from <file>', 'Read NUL-separated paths from a file. Preferred over --paths: a comma or ' +
|
|
901
|
+
'newline in a filename cannot corrupt the list, and git C-quotes such ' +
|
|
902
|
+
'names in --name-only output, which made the checker examine a path ' +
|
|
903
|
+
'that does not exist and report it clean.')
|
|
904
|
+
.option('--paths <csv>', 'Restrict the check to these paths. The GATE passes its own watched set, ' +
|
|
905
|
+
'so the checker and the gate can never disagree about what was examined.')
|
|
906
|
+
.option('--source <which>', 'Which bytes to examine: "index" (what `git commit` stores) or "worktree" (what `commit -a` stores)')
|
|
460
907
|
.action(async (options) => {
|
|
461
908
|
const baseIsDefault = options.base === undefined;
|
|
462
909
|
const base = options.base ?? 'HEAD~1';
|
|
463
|
-
|
|
910
|
+
// No trimming on this path: a leading or trailing space is part of the
|
|
911
|
+
// filename, and silently trimming it produced a path the checker could
|
|
912
|
+
// not open while the gate still counted it as covered.
|
|
913
|
+
const only = options.pathsFrom
|
|
914
|
+
? readFileSync(options.pathsFrom, 'utf-8').split('\0').filter(Boolean)
|
|
915
|
+
: options.paths
|
|
916
|
+
? options.paths.split(',').map((p) => p.trim()).filter(Boolean)
|
|
917
|
+
: undefined;
|
|
918
|
+
const run = await runSchemaDiff(base, process.cwd(), baseIsDefault, {
|
|
919
|
+
only,
|
|
920
|
+
source: options.source === 'worktree' ? 'worktree' : 'index',
|
|
921
|
+
quiet: Boolean(options.json),
|
|
922
|
+
});
|
|
464
923
|
const hasBreaking = run.results.some((r) => r.breaking);
|
|
924
|
+
if (options.json) {
|
|
925
|
+
// The gate reads THIS, not an exit code: exit 1 conflates "breaking
|
|
926
|
+
// change found" with "the run never happened", and those must lead to
|
|
927
|
+
// opposite decisions (block vs fall back).
|
|
928
|
+
//
|
|
929
|
+
// One entry per requested path, each carrying the evidence the gate
|
|
930
|
+
// needs to decide whether to believe it:
|
|
931
|
+
// sha the blob actually read. The gate re-derives it and
|
|
932
|
+
// compares. Without it the old `examined` was a list of
|
|
933
|
+
// the gate's own argument echoed back -- it could only
|
|
934
|
+
// fail on a whitespace artefact, never on "nothing was
|
|
935
|
+
// read", which is exactly the case it existed to catch.
|
|
936
|
+
// analysed whether any analyser understood the file at all. An
|
|
937
|
+
// empty `breaking` from a helm chart is not an all-clear.
|
|
938
|
+
// `contract` lets a future gate refuse a shape it does not know
|
|
939
|
+
// instead of misreading it.
|
|
940
|
+
const byPath = new Map(run.results.map((r) => [r.file, r]));
|
|
941
|
+
console.log(JSON.stringify({
|
|
942
|
+
contract: SCHEMA_DIFF_CONTRACT,
|
|
943
|
+
ran: run.ran,
|
|
944
|
+
base: run.effectiveBase,
|
|
945
|
+
files: run.examined.map((f) => {
|
|
946
|
+
const r = byPath.get(f.path);
|
|
947
|
+
return {
|
|
948
|
+
path: f.path,
|
|
949
|
+
sha: f.sha,
|
|
950
|
+
analysed: r ? r.analysed : false,
|
|
951
|
+
breaking: r
|
|
952
|
+
? r.changes.filter((c) => c.breaking).map((c) => c.description)
|
|
953
|
+
: [],
|
|
954
|
+
};
|
|
955
|
+
}),
|
|
956
|
+
}));
|
|
957
|
+
process.exitCode = hasBreaking ? 1 : 0;
|
|
958
|
+
return;
|
|
959
|
+
}
|
|
465
960
|
if (hasBreaking) {
|
|
466
961
|
console.log('\nBreaking changes require explicit approval before proceeding.');
|
|
467
962
|
process.exitCode = 1;
|