eval-quality 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/dist/application/index.d.ts +4 -0
- package/dist/application/index.js +2 -0
- package/dist/core/emit/emit.js +4 -2
- package/dist/core/preflight/reduce.d.ts +1 -1
- package/dist/core/preflight/reduce.js +5 -2
- package/dist/core/schemas/evaluator-configuration.d.ts +9 -0
- package/dist/core/schemas/evaluator-configuration.js +9 -0
- package/dist/core/schemas/evidence-artifact.d.ts +9 -0
- package/dist/core/schemas/evidence-artifact.js +9 -0
- package/dist/core/schemas/isolation-manifest.d.ts +18 -0
- package/dist/core/schemas/isolation-manifest.js +18 -0
- package/dist/core/schemas/preflight-verdict.d.ts +9 -0
- package/dist/core/schemas/preflight-verdict.js +9 -0
- package/dist/core/schemas/private-artifact-manifest.d.ts +10 -0
- package/dist/core/schemas/private-artifact-manifest.js +10 -0
- package/dist/core/schemas/scoring-policy.d.ts +11 -0
- package/dist/core/schemas/scoring-policy.js +11 -0
- package/dist/core/schemas/sealed-evaluator-brief.d.ts +12 -0
- package/dist/core/schemas/sealed-evaluator-brief.js +12 -0
- package/dist/core/schemas/sealed-run-record.d.ts +11 -0
- package/dist/core/schemas/sealed-run-record.js +11 -0
- package/dist/core/seal/seal.js +4 -5
- package/dist/gates/audit-lockfile-age.mjs +295 -0
- package/dist/gates/check-dependency-direction.js +303 -0
- package/dist/gates/check-licenses.mjs +305 -0
- package/dist/gates/dependency-direction.js +555 -0
- package/dist/gates/discover-source-files.js +44 -0
- package/dist/gates/gate-config.js +251 -0
- package/dist/gates/gates-cli.js +410 -0
- package/dist/gates/lineage-ownership.js +364 -0
- package/dist/gates/package-boundary.js +388 -0
- package/dist/gates/token-scan.js +203 -0
- package/dist/index.d.ts +11 -1
- package/dist/index.js +20 -1
- package/dist/testing/probe-conformance.d.ts +23 -18
- package/package.json +20 -8
|
@@ -0,0 +1,364 @@
|
|
|
1
|
+
// A published gate: field ownership.
|
|
2
|
+
//
|
|
3
|
+
// The rule is "only these modules may write these fields, and each of them has
|
|
4
|
+
// to write every one it is named for". A consumer declares four things: the
|
|
5
|
+
// fields it owns, the path prefixes where declaring them is always allowed,
|
|
6
|
+
// the modules permitted to write them, and the helper identifiers that count as
|
|
7
|
+
// a write because they set the fields on a caller's behalf. Nothing in the gate
|
|
8
|
+
// knows what the fields mean.
|
|
9
|
+
//
|
|
10
|
+
// That is what makes it worth publishing. The rule shipped here holds two
|
|
11
|
+
// lineage fields against the stages allowed to mint them, and the same rule
|
|
12
|
+
// holds an `id` nothing but a factory may assign, an `updatedAt` one repository
|
|
13
|
+
// layer owns, a `tenantId` written only where a request is authorized, or any
|
|
14
|
+
// other field whose value is a claim rather than a convenience. A consumer with
|
|
15
|
+
// no lineage concept at all is the ordinary case.
|
|
16
|
+
//
|
|
17
|
+
// Both directions are checked, because both go wrong. A write outside the
|
|
18
|
+
// declared set is the obvious one. A declared writer that writes none of its
|
|
19
|
+
// fields is the likelier regression: a rename empties the list and every
|
|
20
|
+
// remaining check keeps passing over a rule nothing enforces.
|
|
21
|
+
//
|
|
22
|
+
// Token-anchored, so the gate reads TypeScript source through the typescript
|
|
23
|
+
// package's own scanner. `typescript` is loaded on first use rather than at
|
|
24
|
+
// import, so this module carries no `typescript` on its load path and the gate
|
|
25
|
+
// refuses by name when the dependency is absent instead of failing to load.
|
|
26
|
+
//
|
|
27
|
+
// Every rule is fail-closed: an ambiguous shape is reported, since an owned
|
|
28
|
+
// field outside its declared home is worth a human look either way.
|
|
29
|
+
//
|
|
30
|
+
// Run by `node` directly: Node's type stripping erases types only, so no
|
|
31
|
+
// TypeScript enum, namespace, parameter property, or non-type re-export may
|
|
32
|
+
// appear in this file or anything it imports.
|
|
33
|
+
import { z } from 'zod';
|
|
34
|
+
import { discoverEntries, RelativePath, RelativePrefix, ScannedPathList, } from './package-boundary.js';
|
|
35
|
+
/** The gate needs `typescript` and could not resolve it. */
|
|
36
|
+
export const TYPESCRIPT_UNAVAILABLE = 'EVAL_QUALITY_TYPESCRIPT_UNAVAILABLE';
|
|
37
|
+
const codedError = (code, message) => Object.assign(new Error(message), { code });
|
|
38
|
+
const importTokenScanner = async () => {
|
|
39
|
+
// `token-scan.ts` imports `typescript/unstable/ast` at its own top level, so
|
|
40
|
+
// this is the one place the dependency is reached and the one place its
|
|
41
|
+
// absence can be turned into a sentence.
|
|
42
|
+
const [ast, scan] = await Promise.all([
|
|
43
|
+
import('typescript/unstable/ast'),
|
|
44
|
+
import('./token-scan.js'),
|
|
45
|
+
]);
|
|
46
|
+
return {
|
|
47
|
+
scanTokens: scan.scanTokens,
|
|
48
|
+
computeLineStarts: scan.computeLineStarts,
|
|
49
|
+
lineOf: scan.lineOf,
|
|
50
|
+
syntax: ast.SyntaxKind,
|
|
51
|
+
};
|
|
52
|
+
};
|
|
53
|
+
/**
|
|
54
|
+
* The tokenizer, or a refusal naming the dependency and the gate that needs it.
|
|
55
|
+
* `load` is injectable so the refusal has a test that does not require
|
|
56
|
+
* uninstalling anything.
|
|
57
|
+
*/
|
|
58
|
+
export async function loadTokenScanner(gate, load = importTokenScanner) {
|
|
59
|
+
try {
|
|
60
|
+
return await load();
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
if (error.code !== 'ERR_MODULE_NOT_FOUND') {
|
|
64
|
+
throw error;
|
|
65
|
+
}
|
|
66
|
+
throw codedError(TYPESCRIPT_UNAVAILABLE, `the ${gate} gate reads your source through the typescript package's own scanner, and typescript did not resolve. Install typescript to run this gate; every other gate needs nothing beyond this package.`);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const Identifier = z
|
|
70
|
+
.string()
|
|
71
|
+
.min(1)
|
|
72
|
+
.regex(/^[A-Za-z_$][A-Za-z0-9_$]*$/, 'is not an identifier; the scan matches a name as the tokenizer sees it, so a member expression or a quoted key cannot be written here');
|
|
73
|
+
export const FieldOwnershipSection = z
|
|
74
|
+
.strictObject({
|
|
75
|
+
paths: ScannedPathList.describe('The source the scan reads. Give it the extensions your source uses; a file outside these paths is neither held nor counted.'),
|
|
76
|
+
fields: z
|
|
77
|
+
.array(Identifier)
|
|
78
|
+
.min(1)
|
|
79
|
+
.describe('The field names you own. A write to any of them outside the declared set is a violation.'),
|
|
80
|
+
declarations: z
|
|
81
|
+
.array(RelativePrefix)
|
|
82
|
+
.default([])
|
|
83
|
+
.describe('Path prefixes where naming a field is always allowed, whatever the writer list says: where the schema, the type, and the factory that defines the shape live. A file under one of these is exempt entirely.'),
|
|
84
|
+
writers: z
|
|
85
|
+
.array(RelativePath)
|
|
86
|
+
.min(1)
|
|
87
|
+
.describe('The modules permitted to write the fields. Each one has to write every field, so a rename that empties this list fails here instead of quietly disabling the rule.'),
|
|
88
|
+
helpers: z
|
|
89
|
+
.array(Identifier)
|
|
90
|
+
.default([])
|
|
91
|
+
.describe("Identifiers that write the fields on a caller's behalf. Naming one outside the writer list is the same write one line further out, so the scan reports the call, the import, and an aliased import alike. A write routed through a helper you have not named here is invisible to this gate."),
|
|
92
|
+
})
|
|
93
|
+
.superRefine((section, ctx) => {
|
|
94
|
+
const overlap = section.fields.filter((field) => section.helpers.includes(field));
|
|
95
|
+
if (overlap.length > 0) {
|
|
96
|
+
ctx.addIssue({
|
|
97
|
+
code: 'custom',
|
|
98
|
+
path: ['helpers'],
|
|
99
|
+
message: `names ${overlap.join(', ')}, which is also a field; one name cannot be both, since a field is reported by position and a helper by mention`,
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
})
|
|
103
|
+
.describe('Fails on a write to a field you own from a module you did not declare, and on a declared module that writes none of the fields it is named for.');
|
|
104
|
+
export const rulesOf = (section) => ({
|
|
105
|
+
fields: new Set(section.fields),
|
|
106
|
+
declarations: section.declarations,
|
|
107
|
+
writers: section.writers,
|
|
108
|
+
helpers: new Set(section.helpers),
|
|
109
|
+
});
|
|
110
|
+
/** How far back the enclosing-bracket search runs before giving up and reporting. */
|
|
111
|
+
const MAX_LOOKBACK = 500;
|
|
112
|
+
const kindsOf = (syntax) => ({
|
|
113
|
+
syntax,
|
|
114
|
+
binders: new Set([
|
|
115
|
+
syntax.ConstKeyword,
|
|
116
|
+
syntax.LetKeyword,
|
|
117
|
+
syntax.VarKeyword,
|
|
118
|
+
syntax.ImportKeyword,
|
|
119
|
+
]),
|
|
120
|
+
typeDeclarers: new Set([syntax.TypeKeyword, syntax.InterfaceKeyword]),
|
|
121
|
+
typeHeads: new Set([syntax.LessThanToken, syntax.ExtendsKeyword]),
|
|
122
|
+
});
|
|
123
|
+
function isPermitted(file, rules) {
|
|
124
|
+
if (rules.declarations.some((prefix) => file.startsWith(prefix)))
|
|
125
|
+
return true;
|
|
126
|
+
return rules.writers.includes(file);
|
|
127
|
+
}
|
|
128
|
+
const assigns = (kinds, kind) => kind !== undefined &&
|
|
129
|
+
kind >= kinds.syntax.FirstAssignment &&
|
|
130
|
+
kind <= kinds.syntax.LastAssignment;
|
|
131
|
+
/**
|
|
132
|
+
* True when the token at `index` starts a member of the literal around it. A
|
|
133
|
+
* formatter writes TS type members newline-separated with no separator and often
|
|
134
|
+
* behind `readonly`, so a line break counts alongside `{`, `,` and `;`.
|
|
135
|
+
*/
|
|
136
|
+
function opensMember(kinds, tokens, lines, index) {
|
|
137
|
+
const previous = tokens[index - 1];
|
|
138
|
+
if (previous === undefined)
|
|
139
|
+
return false;
|
|
140
|
+
if (previous.kind === kinds.syntax.OpenBraceToken ||
|
|
141
|
+
previous.kind === kinds.syntax.CommaToken ||
|
|
142
|
+
previous.kind === kinds.syntax.SemicolonToken ||
|
|
143
|
+
previous.kind === kinds.syntax.ReadonlyKeyword) {
|
|
144
|
+
return true;
|
|
145
|
+
}
|
|
146
|
+
return (lines[index - 1] ?? 0) < (lines[index] ?? 0);
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Walks back to the nearest unmatched opening bracket. A `{` is a literal, and
|
|
150
|
+
* a `:` before it or a `type`/`interface` in its statement makes it a type
|
|
151
|
+
* literal. A `{` a binder introduced is a destructuring pattern, and a `(` or
|
|
152
|
+
* `[` reached first is a parameter list or an index; both are reads.
|
|
153
|
+
*/
|
|
154
|
+
function enclosureOf(kinds, tokens, lines, index) {
|
|
155
|
+
const { syntax } = kinds;
|
|
156
|
+
let braces = 0;
|
|
157
|
+
let parens = 0;
|
|
158
|
+
let brackets = 0;
|
|
159
|
+
const floor = Math.max(0, index - MAX_LOOKBACK);
|
|
160
|
+
for (let i = index - 1; i >= floor; i--) {
|
|
161
|
+
const kind = tokens[i]?.kind;
|
|
162
|
+
if (kind === syntax.CloseBraceToken)
|
|
163
|
+
braces++;
|
|
164
|
+
else if (kind === syntax.CloseParenToken)
|
|
165
|
+
parens++;
|
|
166
|
+
else if (kind === syntax.CloseBracketToken)
|
|
167
|
+
brackets++;
|
|
168
|
+
else if (kind === syntax.OpenParenToken && parens-- === 0)
|
|
169
|
+
return 'read';
|
|
170
|
+
else if (kind === syntax.OpenBracketToken && brackets-- === 0) {
|
|
171
|
+
return 'read';
|
|
172
|
+
}
|
|
173
|
+
else if (kind === syntax.OpenBraceToken && braces-- === 0) {
|
|
174
|
+
const before = tokens[i - 1]?.kind ?? -1;
|
|
175
|
+
if (kinds.binders.has(before))
|
|
176
|
+
return 'read';
|
|
177
|
+
if (kinds.typeHeads.has(before))
|
|
178
|
+
return 'type-literal';
|
|
179
|
+
// A `{` after a colon is a type annotation, unless the name before that
|
|
180
|
+
// colon is itself a member of a value literal: `lineage: { id: null }`
|
|
181
|
+
// is a nested value, while `row: { id: Id }` in a parameter list is a
|
|
182
|
+
// shape.
|
|
183
|
+
if (before === syntax.ColonToken) {
|
|
184
|
+
if (kinds.binders.has(tokens[i - 3]?.kind ?? -1))
|
|
185
|
+
return 'type-literal';
|
|
186
|
+
return enclosureOf(kinds, tokens, lines, i - 2) === 'value-literal'
|
|
187
|
+
? 'value-literal'
|
|
188
|
+
: 'type-literal';
|
|
189
|
+
}
|
|
190
|
+
return declaresType(kinds, tokens, i) ? 'type-literal' : 'value-literal';
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
// Unresolved within the window: report it.
|
|
194
|
+
return 'value-literal';
|
|
195
|
+
}
|
|
196
|
+
/** True when a `type` or `interface` keyword opens the statement holding the `{` at `open`. */
|
|
197
|
+
function declaresType(kinds, tokens, open) {
|
|
198
|
+
const { syntax } = kinds;
|
|
199
|
+
const floor = Math.max(0, open - MAX_LOOKBACK);
|
|
200
|
+
for (let i = open - 1; i >= floor; i--) {
|
|
201
|
+
const kind = tokens[i]?.kind;
|
|
202
|
+
if (kind === undefined)
|
|
203
|
+
return false;
|
|
204
|
+
if (kinds.typeDeclarers.has(kind))
|
|
205
|
+
return true;
|
|
206
|
+
if (kind === syntax.SemicolonToken ||
|
|
207
|
+
kind === syntax.OpenBraceToken ||
|
|
208
|
+
kind === syntax.CloseBraceToken ||
|
|
209
|
+
kinds.binders.has(kind)) {
|
|
210
|
+
return false;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
return false;
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* What a bare-identifier occurrence is. Any assignment operator makes it an
|
|
217
|
+
* assignment wherever it appears. A name opening a member of an object or type
|
|
218
|
+
* literal declares the field; a type literal is reported too, and `scanFile`
|
|
219
|
+
* keeps it out of the write count.
|
|
220
|
+
*/
|
|
221
|
+
function writeKind(kinds, tokens, lines, index) {
|
|
222
|
+
const { syntax } = kinds;
|
|
223
|
+
const next = tokens[index + 1]?.kind;
|
|
224
|
+
if (assigns(kinds, next))
|
|
225
|
+
return 'assignment';
|
|
226
|
+
if (tokens[index - 1]?.kind === syntax.DotToken)
|
|
227
|
+
return undefined;
|
|
228
|
+
if (next !== syntax.ColonToken &&
|
|
229
|
+
next !== syntax.CommaToken &&
|
|
230
|
+
next !== syntax.CloseBraceToken) {
|
|
231
|
+
return undefined;
|
|
232
|
+
}
|
|
233
|
+
// `return count }` and `[id, x]` use a name bound elsewhere, so only a member
|
|
234
|
+
// start reaches the enclosure walk.
|
|
235
|
+
if (!opensMember(kinds, tokens, lines, index))
|
|
236
|
+
return undefined;
|
|
237
|
+
switch (enclosureOf(kinds, tokens, lines, index)) {
|
|
238
|
+
case 'value-literal':
|
|
239
|
+
return 'literal';
|
|
240
|
+
case 'type-literal':
|
|
241
|
+
return 'type';
|
|
242
|
+
default:
|
|
243
|
+
return undefined;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* True when this member is a field given a value inside a value literal. A
|
|
248
|
+
* shorthand binds a name and every type position declares a shape, so neither
|
|
249
|
+
* can stand in for the write a declared writer owes. The enclosure decides it,
|
|
250
|
+
* since a denylist of type names cannot be completed: an alias, a branded type,
|
|
251
|
+
* and a literal type all read like values.
|
|
252
|
+
*/
|
|
253
|
+
function mints(kinds, tokens, index) {
|
|
254
|
+
return tokens[index + 1]?.kind === kinds.syntax.ColonToken;
|
|
255
|
+
}
|
|
256
|
+
function scanFile(file, source, rules, scanner, kinds, violations) {
|
|
257
|
+
const { syntax } = kinds;
|
|
258
|
+
const tokens = scanner.scanTokens(source);
|
|
259
|
+
const lineStarts = scanner.computeLineStarts(source);
|
|
260
|
+
const lines = tokens.map((token) => scanner.lineOf(lineStarts, token.start));
|
|
261
|
+
const permitted = isPermitted(file, rules);
|
|
262
|
+
const written = new Set();
|
|
263
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
264
|
+
const token = tokens[i];
|
|
265
|
+
if (token === undefined)
|
|
266
|
+
continue;
|
|
267
|
+
const line = lines[i] ?? 1;
|
|
268
|
+
// A string spelling an owned field reaches it through a computed key, a
|
|
269
|
+
// bracket assignment, `Object.defineProperty`, or `Reflect.set`. All four
|
|
270
|
+
// look alike at this level, so any of them is reported.
|
|
271
|
+
if (token.kind === syntax.StringLiteral ||
|
|
272
|
+
token.kind === syntax.NoSubstitutionTemplateLiteral) {
|
|
273
|
+
if (permitted || !rules.fields.has(token.value))
|
|
274
|
+
continue;
|
|
275
|
+
violations.push({
|
|
276
|
+
file,
|
|
277
|
+
line,
|
|
278
|
+
subject: token.value,
|
|
279
|
+
rule: 'names an owned field as a string, which reaches it through a computed key or a reflective set',
|
|
280
|
+
});
|
|
281
|
+
continue;
|
|
282
|
+
}
|
|
283
|
+
if (token.kind !== syntax.Identifier)
|
|
284
|
+
continue;
|
|
285
|
+
if (rules.fields.has(token.value)) {
|
|
286
|
+
const kind = writeKind(kinds, tokens, lines, i);
|
|
287
|
+
if (kind === undefined)
|
|
288
|
+
continue;
|
|
289
|
+
if (kind === 'assignment' ||
|
|
290
|
+
(kind === 'literal' && mints(kinds, tokens, i))) {
|
|
291
|
+
written.add(token.value);
|
|
292
|
+
}
|
|
293
|
+
if (permitted)
|
|
294
|
+
continue;
|
|
295
|
+
violations.push({
|
|
296
|
+
file,
|
|
297
|
+
line,
|
|
298
|
+
subject: token.value,
|
|
299
|
+
rule: `only a declared path or a declared writer may set this field; this is a ${kind} position`,
|
|
300
|
+
});
|
|
301
|
+
continue;
|
|
302
|
+
}
|
|
303
|
+
if (rules.helpers.has(token.value) && !permitted) {
|
|
304
|
+
violations.push({
|
|
305
|
+
file,
|
|
306
|
+
line,
|
|
307
|
+
subject: token.value,
|
|
308
|
+
rule: `${token.value}() sets the owned fields, so naming it outside the writer list is the same write one line further out`,
|
|
309
|
+
});
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
return written;
|
|
313
|
+
}
|
|
314
|
+
/**
|
|
315
|
+
* Scans every file in `files` (repo-relative POSIX path -> source text) and
|
|
316
|
+
* returns every violation, in no particular cross-file order. Pure and
|
|
317
|
+
* synchronous over the map, so one function backs both the real scan and a
|
|
318
|
+
* synthetic test map.
|
|
319
|
+
*/
|
|
320
|
+
export function scanFieldOwnership(files, rules, scanner, options) {
|
|
321
|
+
const kinds = kindsOf(scanner.syntax);
|
|
322
|
+
const violations = [];
|
|
323
|
+
const writesByFile = new Map();
|
|
324
|
+
for (const [file, source] of files) {
|
|
325
|
+
writesByFile.set(file, scanFile(file, source, rules, scanner, kinds, violations));
|
|
326
|
+
}
|
|
327
|
+
for (const module of rules.writers) {
|
|
328
|
+
const written = writesByFile.get(module);
|
|
329
|
+
if (written === undefined) {
|
|
330
|
+
if (!options.wholeTree)
|
|
331
|
+
continue;
|
|
332
|
+
violations.push({
|
|
333
|
+
file: module,
|
|
334
|
+
line: 1,
|
|
335
|
+
subject: module,
|
|
336
|
+
rule: 'the configuration names this module as a writer and no such file was scanned; a rename emptied the writer list',
|
|
337
|
+
});
|
|
338
|
+
continue;
|
|
339
|
+
}
|
|
340
|
+
for (const field of rules.fields) {
|
|
341
|
+
if (written.has(field))
|
|
342
|
+
continue;
|
|
343
|
+
violations.push({
|
|
344
|
+
file: module,
|
|
345
|
+
line: 1,
|
|
346
|
+
subject: field,
|
|
347
|
+
rule: 'the configuration names this module as a writer of this field and it writes none',
|
|
348
|
+
});
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
return violations;
|
|
352
|
+
}
|
|
353
|
+
/**
|
|
354
|
+
* The gate, over a consumer's tree. `root` is the directory its configuration
|
|
355
|
+
* file sits in.
|
|
356
|
+
*/
|
|
357
|
+
export async function runFieldOwnership(root, section, gate = 'field-ownership', load) {
|
|
358
|
+
const scanner = await loadTokenScanner(gate, load);
|
|
359
|
+
const { entries } = await discoverEntries(root, section.paths, gate);
|
|
360
|
+
const violations = scanFieldOwnership(entries, rulesOf(section), scanner, {
|
|
361
|
+
wholeTree: true,
|
|
362
|
+
});
|
|
363
|
+
return { violations, scanned: entries.size };
|
|
364
|
+
}
|