eval-quality 3.0.0 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/README.md +4 -1
  2. package/dist/application/index.d.ts +4 -0
  3. package/dist/application/index.js +2 -0
  4. package/dist/core/emit/emit.js +4 -2
  5. package/dist/core/preflight/reduce.d.ts +1 -1
  6. package/dist/core/preflight/reduce.js +5 -2
  7. package/dist/core/schemas/evaluator-configuration.d.ts +9 -0
  8. package/dist/core/schemas/evaluator-configuration.js +9 -0
  9. package/dist/core/schemas/evidence-artifact.d.ts +9 -0
  10. package/dist/core/schemas/evidence-artifact.js +9 -0
  11. package/dist/core/schemas/isolation-manifest.d.ts +18 -0
  12. package/dist/core/schemas/isolation-manifest.js +18 -0
  13. package/dist/core/schemas/preflight-verdict.d.ts +9 -0
  14. package/dist/core/schemas/preflight-verdict.js +9 -0
  15. package/dist/core/schemas/private-artifact-manifest.d.ts +10 -0
  16. package/dist/core/schemas/private-artifact-manifest.js +10 -0
  17. package/dist/core/schemas/scoring-policy.d.ts +11 -0
  18. package/dist/core/schemas/scoring-policy.js +11 -0
  19. package/dist/core/schemas/sealed-evaluator-brief.d.ts +12 -0
  20. package/dist/core/schemas/sealed-evaluator-brief.js +12 -0
  21. package/dist/core/schemas/sealed-run-record.d.ts +11 -0
  22. package/dist/core/schemas/sealed-run-record.js +11 -0
  23. package/dist/core/seal/seal.js +4 -5
  24. package/dist/gates/audit-lockfile-age.mjs +295 -0
  25. package/dist/gates/check-dependency-direction.js +303 -0
  26. package/dist/gates/check-licenses.mjs +305 -0
  27. package/dist/gates/dependency-direction.js +555 -0
  28. package/dist/gates/discover-source-files.js +44 -0
  29. package/dist/gates/gate-config.js +251 -0
  30. package/dist/gates/gates-cli.js +410 -0
  31. package/dist/gates/lineage-ownership.js +364 -0
  32. package/dist/gates/package-boundary.js +388 -0
  33. package/dist/gates/token-scan.js +203 -0
  34. package/dist/index.d.ts +11 -1
  35. package/dist/index.js +20 -1
  36. package/dist/testing/probe-conformance.d.ts +23 -18
  37. package/package.json +20 -8
@@ -0,0 +1,364 @@
1
+ // A published gate: field ownership.
2
+ //
3
+ // The rule is "only these modules may write these fields, and each of them has
4
+ // to write every one it is named for". A consumer declares four things: the
5
+ // fields it owns, the path prefixes where declaring them is always allowed,
6
+ // the modules permitted to write them, and the helper identifiers that count as
7
+ // a write because they set the fields on a caller's behalf. Nothing in the gate
8
+ // knows what the fields mean.
9
+ //
10
+ // That is what makes it worth publishing. The rule shipped here holds two
11
+ // lineage fields against the stages allowed to mint them, and the same rule
12
+ // holds an `id` nothing but a factory may assign, an `updatedAt` one repository
13
+ // layer owns, a `tenantId` written only where a request is authorized, or any
14
+ // other field whose value is a claim rather than a convenience. A consumer with
15
+ // no lineage concept at all is the ordinary case.
16
+ //
17
+ // Both directions are checked, because both go wrong. A write outside the
18
+ // declared set is the obvious one. A declared writer that writes none of its
19
+ // fields is the likelier regression: a rename empties the list and every
20
+ // remaining check keeps passing over a rule nothing enforces.
21
+ //
22
+ // Token-anchored, so the gate reads TypeScript source through the typescript
23
+ // package's own scanner. `typescript` is loaded on first use rather than at
24
+ // import, so this module carries no `typescript` on its load path and the gate
25
+ // refuses by name when the dependency is absent instead of failing to load.
26
+ //
27
+ // Every rule is fail-closed: an ambiguous shape is reported, since an owned
28
+ // field outside its declared home is worth a human look either way.
29
+ //
30
+ // Run by `node` directly: Node's type stripping erases types only, so no
31
+ // TypeScript enum, namespace, parameter property, or non-type re-export may
32
+ // appear in this file or anything it imports.
33
+ import { z } from 'zod';
34
+ import { discoverEntries, RelativePath, RelativePrefix, ScannedPathList, } from './package-boundary.js';
35
+ /** The gate needs `typescript` and could not resolve it. */
36
+ export const TYPESCRIPT_UNAVAILABLE = 'EVAL_QUALITY_TYPESCRIPT_UNAVAILABLE';
37
+ const codedError = (code, message) => Object.assign(new Error(message), { code });
38
+ const importTokenScanner = async () => {
39
+ // `token-scan.ts` imports `typescript/unstable/ast` at its own top level, so
40
+ // this is the one place the dependency is reached and the one place its
41
+ // absence can be turned into a sentence.
42
+ const [ast, scan] = await Promise.all([
43
+ import('typescript/unstable/ast'),
44
+ import('./token-scan.js'),
45
+ ]);
46
+ return {
47
+ scanTokens: scan.scanTokens,
48
+ computeLineStarts: scan.computeLineStarts,
49
+ lineOf: scan.lineOf,
50
+ syntax: ast.SyntaxKind,
51
+ };
52
+ };
53
+ /**
54
+ * The tokenizer, or a refusal naming the dependency and the gate that needs it.
55
+ * `load` is injectable so the refusal has a test that does not require
56
+ * uninstalling anything.
57
+ */
58
+ export async function loadTokenScanner(gate, load = importTokenScanner) {
59
+ try {
60
+ return await load();
61
+ }
62
+ catch (error) {
63
+ if (error.code !== 'ERR_MODULE_NOT_FOUND') {
64
+ throw error;
65
+ }
66
+ throw codedError(TYPESCRIPT_UNAVAILABLE, `the ${gate} gate reads your source through the typescript package's own scanner, and typescript did not resolve. Install typescript to run this gate; every other gate needs nothing beyond this package.`);
67
+ }
68
+ }
69
+ const Identifier = z
70
+ .string()
71
+ .min(1)
72
+ .regex(/^[A-Za-z_$][A-Za-z0-9_$]*$/, 'is not an identifier; the scan matches a name as the tokenizer sees it, so a member expression or a quoted key cannot be written here');
73
+ export const FieldOwnershipSection = z
74
+ .strictObject({
75
+ paths: ScannedPathList.describe('The source the scan reads. Give it the extensions your source uses; a file outside these paths is neither held nor counted.'),
76
+ fields: z
77
+ .array(Identifier)
78
+ .min(1)
79
+ .describe('The field names you own. A write to any of them outside the declared set is a violation.'),
80
+ declarations: z
81
+ .array(RelativePrefix)
82
+ .default([])
83
+ .describe('Path prefixes where naming a field is always allowed, whatever the writer list says: where the schema, the type, and the factory that defines the shape live. A file under one of these is exempt entirely.'),
84
+ writers: z
85
+ .array(RelativePath)
86
+ .min(1)
87
+ .describe('The modules permitted to write the fields. Each one has to write every field, so a rename that empties this list fails here instead of quietly disabling the rule.'),
88
+ helpers: z
89
+ .array(Identifier)
90
+ .default([])
91
+ .describe("Identifiers that write the fields on a caller's behalf. Naming one outside the writer list is the same write one line further out, so the scan reports the call, the import, and an aliased import alike. A write routed through a helper you have not named here is invisible to this gate."),
92
+ })
93
+ .superRefine((section, ctx) => {
94
+ const overlap = section.fields.filter((field) => section.helpers.includes(field));
95
+ if (overlap.length > 0) {
96
+ ctx.addIssue({
97
+ code: 'custom',
98
+ path: ['helpers'],
99
+ message: `names ${overlap.join(', ')}, which is also a field; one name cannot be both, since a field is reported by position and a helper by mention`,
100
+ });
101
+ }
102
+ })
103
+ .describe('Fails on a write to a field you own from a module you did not declare, and on a declared module that writes none of the fields it is named for.');
104
+ export const rulesOf = (section) => ({
105
+ fields: new Set(section.fields),
106
+ declarations: section.declarations,
107
+ writers: section.writers,
108
+ helpers: new Set(section.helpers),
109
+ });
110
+ /** How far back the enclosing-bracket search runs before giving up and reporting. */
111
+ const MAX_LOOKBACK = 500;
112
+ const kindsOf = (syntax) => ({
113
+ syntax,
114
+ binders: new Set([
115
+ syntax.ConstKeyword,
116
+ syntax.LetKeyword,
117
+ syntax.VarKeyword,
118
+ syntax.ImportKeyword,
119
+ ]),
120
+ typeDeclarers: new Set([syntax.TypeKeyword, syntax.InterfaceKeyword]),
121
+ typeHeads: new Set([syntax.LessThanToken, syntax.ExtendsKeyword]),
122
+ });
123
+ function isPermitted(file, rules) {
124
+ if (rules.declarations.some((prefix) => file.startsWith(prefix)))
125
+ return true;
126
+ return rules.writers.includes(file);
127
+ }
128
+ const assigns = (kinds, kind) => kind !== undefined &&
129
+ kind >= kinds.syntax.FirstAssignment &&
130
+ kind <= kinds.syntax.LastAssignment;
131
+ /**
132
+ * True when the token at `index` starts a member of the literal around it. A
133
+ * formatter writes TS type members newline-separated with no separator and often
134
+ * behind `readonly`, so a line break counts alongside `{`, `,` and `;`.
135
+ */
136
+ function opensMember(kinds, tokens, lines, index) {
137
+ const previous = tokens[index - 1];
138
+ if (previous === undefined)
139
+ return false;
140
+ if (previous.kind === kinds.syntax.OpenBraceToken ||
141
+ previous.kind === kinds.syntax.CommaToken ||
142
+ previous.kind === kinds.syntax.SemicolonToken ||
143
+ previous.kind === kinds.syntax.ReadonlyKeyword) {
144
+ return true;
145
+ }
146
+ return (lines[index - 1] ?? 0) < (lines[index] ?? 0);
147
+ }
148
+ /**
149
+ * Walks back to the nearest unmatched opening bracket. A `{` is a literal, and
150
+ * a `:` before it or a `type`/`interface` in its statement makes it a type
151
+ * literal. A `{` a binder introduced is a destructuring pattern, and a `(` or
152
+ * `[` reached first is a parameter list or an index; both are reads.
153
+ */
154
+ function enclosureOf(kinds, tokens, lines, index) {
155
+ const { syntax } = kinds;
156
+ let braces = 0;
157
+ let parens = 0;
158
+ let brackets = 0;
159
+ const floor = Math.max(0, index - MAX_LOOKBACK);
160
+ for (let i = index - 1; i >= floor; i--) {
161
+ const kind = tokens[i]?.kind;
162
+ if (kind === syntax.CloseBraceToken)
163
+ braces++;
164
+ else if (kind === syntax.CloseParenToken)
165
+ parens++;
166
+ else if (kind === syntax.CloseBracketToken)
167
+ brackets++;
168
+ else if (kind === syntax.OpenParenToken && parens-- === 0)
169
+ return 'read';
170
+ else if (kind === syntax.OpenBracketToken && brackets-- === 0) {
171
+ return 'read';
172
+ }
173
+ else if (kind === syntax.OpenBraceToken && braces-- === 0) {
174
+ const before = tokens[i - 1]?.kind ?? -1;
175
+ if (kinds.binders.has(before))
176
+ return 'read';
177
+ if (kinds.typeHeads.has(before))
178
+ return 'type-literal';
179
+ // A `{` after a colon is a type annotation, unless the name before that
180
+ // colon is itself a member of a value literal: `lineage: { id: null }`
181
+ // is a nested value, while `row: { id: Id }` in a parameter list is a
182
+ // shape.
183
+ if (before === syntax.ColonToken) {
184
+ if (kinds.binders.has(tokens[i - 3]?.kind ?? -1))
185
+ return 'type-literal';
186
+ return enclosureOf(kinds, tokens, lines, i - 2) === 'value-literal'
187
+ ? 'value-literal'
188
+ : 'type-literal';
189
+ }
190
+ return declaresType(kinds, tokens, i) ? 'type-literal' : 'value-literal';
191
+ }
192
+ }
193
+ // Unresolved within the window: report it.
194
+ return 'value-literal';
195
+ }
196
+ /** True when a `type` or `interface` keyword opens the statement holding the `{` at `open`. */
197
+ function declaresType(kinds, tokens, open) {
198
+ const { syntax } = kinds;
199
+ const floor = Math.max(0, open - MAX_LOOKBACK);
200
+ for (let i = open - 1; i >= floor; i--) {
201
+ const kind = tokens[i]?.kind;
202
+ if (kind === undefined)
203
+ return false;
204
+ if (kinds.typeDeclarers.has(kind))
205
+ return true;
206
+ if (kind === syntax.SemicolonToken ||
207
+ kind === syntax.OpenBraceToken ||
208
+ kind === syntax.CloseBraceToken ||
209
+ kinds.binders.has(kind)) {
210
+ return false;
211
+ }
212
+ }
213
+ return false;
214
+ }
215
+ /**
216
+ * What a bare-identifier occurrence is. Any assignment operator makes it an
217
+ * assignment wherever it appears. A name opening a member of an object or type
218
+ * literal declares the field; a type literal is reported too, and `scanFile`
219
+ * keeps it out of the write count.
220
+ */
221
+ function writeKind(kinds, tokens, lines, index) {
222
+ const { syntax } = kinds;
223
+ const next = tokens[index + 1]?.kind;
224
+ if (assigns(kinds, next))
225
+ return 'assignment';
226
+ if (tokens[index - 1]?.kind === syntax.DotToken)
227
+ return undefined;
228
+ if (next !== syntax.ColonToken &&
229
+ next !== syntax.CommaToken &&
230
+ next !== syntax.CloseBraceToken) {
231
+ return undefined;
232
+ }
233
+ // `return count }` and `[id, x]` use a name bound elsewhere, so only a member
234
+ // start reaches the enclosure walk.
235
+ if (!opensMember(kinds, tokens, lines, index))
236
+ return undefined;
237
+ switch (enclosureOf(kinds, tokens, lines, index)) {
238
+ case 'value-literal':
239
+ return 'literal';
240
+ case 'type-literal':
241
+ return 'type';
242
+ default:
243
+ return undefined;
244
+ }
245
+ }
246
+ /**
247
+ * True when this member is a field given a value inside a value literal. A
248
+ * shorthand binds a name and every type position declares a shape, so neither
249
+ * can stand in for the write a declared writer owes. The enclosure decides it,
250
+ * since a denylist of type names cannot be completed: an alias, a branded type,
251
+ * and a literal type all read like values.
252
+ */
253
+ function mints(kinds, tokens, index) {
254
+ return tokens[index + 1]?.kind === kinds.syntax.ColonToken;
255
+ }
256
+ function scanFile(file, source, rules, scanner, kinds, violations) {
257
+ const { syntax } = kinds;
258
+ const tokens = scanner.scanTokens(source);
259
+ const lineStarts = scanner.computeLineStarts(source);
260
+ const lines = tokens.map((token) => scanner.lineOf(lineStarts, token.start));
261
+ const permitted = isPermitted(file, rules);
262
+ const written = new Set();
263
+ for (let i = 0; i < tokens.length; i++) {
264
+ const token = tokens[i];
265
+ if (token === undefined)
266
+ continue;
267
+ const line = lines[i] ?? 1;
268
+ // A string spelling an owned field reaches it through a computed key, a
269
+ // bracket assignment, `Object.defineProperty`, or `Reflect.set`. All four
270
+ // look alike at this level, so any of them is reported.
271
+ if (token.kind === syntax.StringLiteral ||
272
+ token.kind === syntax.NoSubstitutionTemplateLiteral) {
273
+ if (permitted || !rules.fields.has(token.value))
274
+ continue;
275
+ violations.push({
276
+ file,
277
+ line,
278
+ subject: token.value,
279
+ rule: 'names an owned field as a string, which reaches it through a computed key or a reflective set',
280
+ });
281
+ continue;
282
+ }
283
+ if (token.kind !== syntax.Identifier)
284
+ continue;
285
+ if (rules.fields.has(token.value)) {
286
+ const kind = writeKind(kinds, tokens, lines, i);
287
+ if (kind === undefined)
288
+ continue;
289
+ if (kind === 'assignment' ||
290
+ (kind === 'literal' && mints(kinds, tokens, i))) {
291
+ written.add(token.value);
292
+ }
293
+ if (permitted)
294
+ continue;
295
+ violations.push({
296
+ file,
297
+ line,
298
+ subject: token.value,
299
+ rule: `only a declared path or a declared writer may set this field; this is a ${kind} position`,
300
+ });
301
+ continue;
302
+ }
303
+ if (rules.helpers.has(token.value) && !permitted) {
304
+ violations.push({
305
+ file,
306
+ line,
307
+ subject: token.value,
308
+ rule: `${token.value}() sets the owned fields, so naming it outside the writer list is the same write one line further out`,
309
+ });
310
+ }
311
+ }
312
+ return written;
313
+ }
314
+ /**
315
+ * Scans every file in `files` (repo-relative POSIX path -> source text) and
316
+ * returns every violation, in no particular cross-file order. Pure and
317
+ * synchronous over the map, so one function backs both the real scan and a
318
+ * synthetic test map.
319
+ */
320
+ export function scanFieldOwnership(files, rules, scanner, options) {
321
+ const kinds = kindsOf(scanner.syntax);
322
+ const violations = [];
323
+ const writesByFile = new Map();
324
+ for (const [file, source] of files) {
325
+ writesByFile.set(file, scanFile(file, source, rules, scanner, kinds, violations));
326
+ }
327
+ for (const module of rules.writers) {
328
+ const written = writesByFile.get(module);
329
+ if (written === undefined) {
330
+ if (!options.wholeTree)
331
+ continue;
332
+ violations.push({
333
+ file: module,
334
+ line: 1,
335
+ subject: module,
336
+ rule: 'the configuration names this module as a writer and no such file was scanned; a rename emptied the writer list',
337
+ });
338
+ continue;
339
+ }
340
+ for (const field of rules.fields) {
341
+ if (written.has(field))
342
+ continue;
343
+ violations.push({
344
+ file: module,
345
+ line: 1,
346
+ subject: field,
347
+ rule: 'the configuration names this module as a writer of this field and it writes none',
348
+ });
349
+ }
350
+ }
351
+ return violations;
352
+ }
353
+ /**
354
+ * The gate, over a consumer's tree. `root` is the directory its configuration
355
+ * file sits in.
356
+ */
357
+ export async function runFieldOwnership(root, section, gate = 'field-ownership', load) {
358
+ const scanner = await loadTokenScanner(gate, load);
359
+ const { entries } = await discoverEntries(root, section.paths, gate);
360
+ const violations = scanFieldOwnership(entries, rulesOf(section), scanner, {
361
+ wholeTree: true,
362
+ });
363
+ return { violations, scanned: entries.size };
364
+ }