eval-quality 2.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/dist/application/index.d.ts +4 -0
- package/dist/application/index.js +2 -0
- package/dist/core/compile/compile.js +6 -1
- package/dist/core/compile/schema-version.d.ts +15 -2
- package/dist/core/compile/schema-version.js +11 -3
- package/dist/core/emit/emit.js +4 -2
- package/dist/core/preflight/plan.js +15 -0
- package/dist/core/preflight/reduce.d.ts +1 -1
- package/dist/core/preflight/reduce.js +5 -2
- package/dist/core/schemas/evaluator-configuration.d.ts +9 -0
- package/dist/core/schemas/evaluator-configuration.js +9 -0
- package/dist/core/schemas/evidence-artifact.d.ts +9 -0
- package/dist/core/schemas/evidence-artifact.js +9 -0
- package/dist/core/schemas/isolation-manifest.d.ts +18 -0
- package/dist/core/schemas/isolation-manifest.js +18 -0
- package/dist/core/schemas/preflight-verdict.d.ts +9 -0
- package/dist/core/schemas/preflight-verdict.js +9 -0
- package/dist/core/schemas/private-artifact-manifest.d.ts +10 -0
- package/dist/core/schemas/private-artifact-manifest.js +10 -0
- package/dist/core/schemas/probe.d.ts +41 -0
- package/dist/core/schemas/probe.js +43 -0
- package/dist/core/schemas/scoring-policy.d.ts +11 -0
- package/dist/core/schemas/scoring-policy.js +11 -0
- package/dist/core/schemas/sealed-evaluator-brief.d.ts +12 -0
- package/dist/core/schemas/sealed-evaluator-brief.js +12 -0
- package/dist/core/schemas/sealed-run-record.d.ts +11 -0
- package/dist/core/schemas/sealed-run-record.js +11 -0
- package/dist/core/score/score.d.ts +1 -1
- package/dist/core/score/score.js +23 -0
- package/dist/core/seal/seal.js +4 -5
- package/dist/gates/audit-lockfile-age.mjs +295 -0
- package/dist/gates/check-dependency-direction.js +303 -0
- package/dist/gates/check-licenses.mjs +305 -0
- package/dist/gates/dependency-direction.js +555 -0
- package/dist/gates/discover-source-files.js +44 -0
- package/dist/gates/gate-config.js +251 -0
- package/dist/gates/gates-cli.js +410 -0
- package/dist/gates/lineage-ownership.js +364 -0
- package/dist/gates/package-boundary.js +388 -0
- package/dist/gates/token-scan.js +203 -0
- package/dist/index.d.ts +11 -1
- package/dist/index.js +20 -1
- package/dist/testing/probe-conformance.d.ts +23 -18
- package/package.json +20 -8
|
@@ -0,0 +1,388 @@
|
|
|
1
|
+
import { lstat, readdir, readFile } from 'node:fs/promises';
|
|
2
|
+
import { resolve } from 'node:path';
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
/**
|
|
5
|
+
* The bound on a consumer-supplied regular expression, and what it does not
|
|
6
|
+
* cover.
|
|
7
|
+
*
|
|
8
|
+
* A pattern arrives as text from a file this package did not write, so the work
|
|
9
|
+
* one pattern can do is the consumer's to choose. JavaScript's `RegExp` exposes
|
|
10
|
+
* no step counter and no timeout, and `check-doc-claims.ts:943` records the same
|
|
11
|
+
* limit from the other side: `REGEX_STEP_BUDGET` is a number that exists because
|
|
12
|
+
* a step count is the bound you would want and nothing offers one.
|
|
13
|
+
*
|
|
14
|
+
* So the bound is on the two things that are measurable. The pattern side:
|
|
15
|
+
* `MAX_PATTERN_LENGTH` on the source, `MAX_PATTERNS` on the array, a flag set
|
|
16
|
+
* that excludes `g` and `y`, and a refusal of backreferences, which is the
|
|
17
|
+
* construct that turns a linear scan of one line into an exponential one. The
|
|
18
|
+
* input side: `MAX_SCANNED_LINE` on the text a pattern is matched against, so a
|
|
19
|
+
* minified bundle or an embedded data URI cannot hand a pattern a megabyte.
|
|
20
|
+
*
|
|
21
|
+
* 64 KiB, not the single kilobyte a first draft of this bound used. This
|
|
22
|
+
* package's own `corpus/` is a legitimate long-line source: an unminified
|
|
23
|
+
* behavioral contract is one JSON object per line and the longest committed
|
|
24
|
+
* one is 13,333 characters. A bound sized to the threat this constant exists
|
|
25
|
+
* for, an actual megabyte-scale bundle, and not to this repository's own
|
|
26
|
+
* content, is a gate that fails on the tree it ships with, which is worse
|
|
27
|
+
* than the DoS surface it is meant to close.
|
|
28
|
+
*
|
|
29
|
+
* What is left uncovered: a pattern with nested quantifiers over a line shorter
|
|
30
|
+
* than the bound can still take far longer than the file deserves. That work is
|
|
31
|
+
* bounded, since the input is, and it is the consumer's own pattern over the
|
|
32
|
+
* consumer's own tree. Running each match in a worker with a wall-clock kill
|
|
33
|
+
* would close it and would make the scanner asynchronous and impure, which costs
|
|
34
|
+
* more than the surface is worth.
|
|
35
|
+
*/
|
|
36
|
+
export const MAX_PATTERNS = 64;
|
|
37
|
+
export const MAX_PATTERN_LENGTH = 200;
|
|
38
|
+
export const MAX_SCANNED_LINE = 65_536;
|
|
39
|
+
/**
|
|
40
|
+
* The name an over-long line is reported under. Reserved, so a consumer pattern
|
|
41
|
+
* cannot take it and a reader of the report can tell a rule that fired from a
|
|
42
|
+
* line the gate declined to match.
|
|
43
|
+
*/
|
|
44
|
+
export const OVERLONG_LINE = 'line-exceeds-scan-bound';
|
|
45
|
+
/** A path or manifest field the configuration named and the tree does not have. */
|
|
46
|
+
export const SCAN_PATH_ERROR = 'EVAL_QUALITY_SCAN_PATH';
|
|
47
|
+
/** A tree the gate could not read to the end, so the scan proves nothing. */
|
|
48
|
+
export const SCAN_UNREADABLE = 'EVAL_QUALITY_SCAN_UNREADABLE';
|
|
49
|
+
const codedError = (code, message) => Object.assign(new Error(message), { code });
|
|
50
|
+
const detail = (error) => error instanceof Error ? error.message : String(error);
|
|
51
|
+
const NonEmpty = z.string().min(1);
|
|
52
|
+
const isSafeRelative = (value) => !value.startsWith('/') &&
|
|
53
|
+
!/^[A-Za-z]:/.test(value) &&
|
|
54
|
+
!value.includes('\\') &&
|
|
55
|
+
value.split('/').every((segment) => segment !== '' && segment !== '..');
|
|
56
|
+
const RELATIVE_PATH_MESSAGE = 'is not a repository-relative path: write it with forward slashes, no leading slash, no drive letter, and no ".." segment, so a configuration can only name files beneath itself';
|
|
57
|
+
/** Shared with `lineage-ownership.ts`; see the note on `ScannedPathList`. */
|
|
58
|
+
export const RelativePath = NonEmpty.max(400).refine(isSafeRelative, RELATIVE_PATH_MESSAGE);
|
|
59
|
+
/** The same shape with a trailing slash allowed, for a directory prefix. */
|
|
60
|
+
export const RelativePrefix = NonEmpty.max(400).refine((value) => isSafeRelative(value.endsWith('/') ? value.slice(0, -1) : value), RELATIVE_PATH_MESSAGE);
|
|
61
|
+
const ScannedPath = z.strictObject({
|
|
62
|
+
path: RelativePath.describe('A directory to walk, or a single file to read, relative to this configuration file.'),
|
|
63
|
+
recursive: z
|
|
64
|
+
.boolean()
|
|
65
|
+
.default(true)
|
|
66
|
+
.describe('Whether a directory is walked to the bottom. Ignored when the path names a file.'),
|
|
67
|
+
extensions: z
|
|
68
|
+
.array(z
|
|
69
|
+
.string()
|
|
70
|
+
.regex(/^\.[A-Za-z0-9][A-Za-z0-9.]*$/, 'is not a file extension; write it with its leading dot, as ".ts"'))
|
|
71
|
+
.min(1)
|
|
72
|
+
.optional()
|
|
73
|
+
.describe('Which files under this path are read. Leave it out and every file is read whatever its name, which is what a tree of generated data needs.'),
|
|
74
|
+
optional: z
|
|
75
|
+
.boolean()
|
|
76
|
+
.default(false)
|
|
77
|
+
.describe('Whether this path may contribute nothing. False, the default, fails when the path is absent or holds no matching file, so a generated tree nobody built cannot read as a clean scan.'),
|
|
78
|
+
});
|
|
79
|
+
/**
|
|
80
|
+
* The scanned-set declaration, shared with the field-ownership gate. It sits
|
|
81
|
+
* here because this is the gate whose headline is the scanned set; a third gate
|
|
82
|
+
* that needs it is the point at which it earns a module of its own.
|
|
83
|
+
*/
|
|
84
|
+
export const ScannedPathList = z.array(ScannedPath).min(1);
|
|
85
|
+
const ManifestField = NonEmpty.regex(/^[A-Za-z_$][A-Za-z0-9_$-]*(?:\.[A-Za-z_$][A-Za-z0-9_$-]*)*$/, 'is not a manifest field path; write one key, or several joined by dots');
|
|
86
|
+
const ManifestScan = z
|
|
87
|
+
.strictObject({
|
|
88
|
+
file: RelativePath.default('package.json').describe('The manifest a registry publishes verbatim.'),
|
|
89
|
+
fields: z
|
|
90
|
+
.array(ManifestField)
|
|
91
|
+
.min(1)
|
|
92
|
+
.describe('Which fields are scanned. A string is one entry, an array joins into one, and an object becomes one entry per key, so "scripts" covers every script by name. A field named here and absent from the manifest is refused, so a typo cannot read as a field with nothing in it.'),
|
|
93
|
+
})
|
|
94
|
+
.describe('The manifest fields scanned as synthetic entries. A JSON value has no line of its own, so every one of them reports at line 1.');
|
|
95
|
+
/**
|
|
96
|
+
* `\1` through `\9` and `\k<name>`. It over-refuses an escaped backslash
|
|
97
|
+
* followed by a digit, which is a literal backslash and not a backreference,
|
|
98
|
+
* and that spelling has no place in a boundary pattern anyway.
|
|
99
|
+
*/
|
|
100
|
+
const BACKREFERENCE = /\\[1-9]|\\k</;
|
|
101
|
+
const ForbiddenPattern = z
|
|
102
|
+
.strictObject({
|
|
103
|
+
name: NonEmpty.describe('What a violation is reported under. Unique across the array.'),
|
|
104
|
+
match: z
|
|
105
|
+
.string()
|
|
106
|
+
.min(1)
|
|
107
|
+
.max(MAX_PATTERN_LENGTH)
|
|
108
|
+
.describe('The regular expression, as source text. It is matched against one logical line at a time.'),
|
|
109
|
+
flags: z
|
|
110
|
+
.string()
|
|
111
|
+
.regex(/^[imsuv]*$/, 'admits only i, m, s, u and v. A g or a y carries a match position between calls, so a pattern holding either would match every second line it should have matched')
|
|
112
|
+
.default('')
|
|
113
|
+
.describe('Regular-expression flags. Empty by default.'),
|
|
114
|
+
reason: NonEmpty.describe('Why the package may not carry it. Printed beside every violation, so the report says what to do rather than only what fired.'),
|
|
115
|
+
})
|
|
116
|
+
.superRefine((pattern, ctx) => {
|
|
117
|
+
if (pattern.name === OVERLONG_LINE) {
|
|
118
|
+
ctx.addIssue({
|
|
119
|
+
code: 'custom',
|
|
120
|
+
path: ['name'],
|
|
121
|
+
message: `is reserved: the gate reports a line past its own matching bound under "${OVERLONG_LINE}"`,
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
if (BACKREFERENCE.test(pattern.match)) {
|
|
125
|
+
ctx.addIssue({
|
|
126
|
+
code: 'custom',
|
|
127
|
+
path: ['match'],
|
|
128
|
+
message: 'carries a backreference, which is the construct that turns a scan of one line into an exponential one; write the pattern without one',
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
try {
|
|
132
|
+
new RegExp(pattern.match, pattern.flags);
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
ctx.addIssue({
|
|
136
|
+
code: 'custom',
|
|
137
|
+
path: ['match'],
|
|
138
|
+
message: `is not a regular expression: ${detail(error)}`,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
export const PackageBoundarySection = z
|
|
143
|
+
.strictObject({
|
|
144
|
+
paths: ScannedPathList.describe('Everything the scan reads out of the tree. What you leave out is exempt, and leaving it out is the only exemption there is.'),
|
|
145
|
+
manifest: ManifestScan.optional(),
|
|
146
|
+
patterns: z
|
|
147
|
+
.array(ForbiddenPattern)
|
|
148
|
+
.min(1)
|
|
149
|
+
.max(MAX_PATTERNS)
|
|
150
|
+
.describe('The forbidden patterns, in precedence order. The first one that matches a logical line is the one the line is reported under, and no line is reported twice, so a specific spelling has to precede any shorter word contained in it: a pattern for a word that is a substring of a path you also forbid will otherwise take every one of those paths and the path pattern will never fire.'),
|
|
151
|
+
})
|
|
152
|
+
.superRefine((section, ctx) => {
|
|
153
|
+
const seen = new Set();
|
|
154
|
+
section.patterns.forEach((pattern, index) => {
|
|
155
|
+
if (seen.has(pattern.name)) {
|
|
156
|
+
ctx.addIssue({
|
|
157
|
+
code: 'custom',
|
|
158
|
+
path: ['patterns', index, 'name'],
|
|
159
|
+
message: `repeats "${pattern.name}"; a violation is reported under this name, so two patterns cannot share one`,
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
seen.add(pattern.name);
|
|
163
|
+
});
|
|
164
|
+
})
|
|
165
|
+
.describe('Fails on any line of any scanned file that matches a pattern you declared. The scanned set is what a registry would publish out of your tree, so what this holds is the package as an installer receives it.');
|
|
166
|
+
export const compileBoundaryPatterns = (patterns) => patterns.map((pattern) => ({
|
|
167
|
+
name: pattern.name,
|
|
168
|
+
regex: new RegExp(pattern.match, pattern.flags),
|
|
169
|
+
reason: pattern.reason,
|
|
170
|
+
}));
|
|
171
|
+
/** The comment markers a wrapped run is joined across. */
|
|
172
|
+
const COMMENT_START = /^\s*(\/\/+|\/\*+|\*+\/?|#)/;
|
|
173
|
+
const isCommentLine = (line) => COMMENT_START.test(line);
|
|
174
|
+
/** Strips the marker so `* Widget` and `* 1.5` join into `Widget 1.5`. */
|
|
175
|
+
const commentText = (line) => line
|
|
176
|
+
.replace(COMMENT_START, '')
|
|
177
|
+
.replace(/\*\/\s*$/, '')
|
|
178
|
+
.trim();
|
|
179
|
+
/**
|
|
180
|
+
* One entry per logical line: a run of consecutive comment lines joins with
|
|
181
|
+
* single spaces and is attributed to the first line of the run, because a
|
|
182
|
+
* comment wraps at some column and a reference split across two physical lines
|
|
183
|
+
* is the ordinary shape. Every other line stands on its own, and its two joins
|
|
184
|
+
* are the same string.
|
|
185
|
+
*/
|
|
186
|
+
export function logicalLines(source) {
|
|
187
|
+
const physical = source.split('\n');
|
|
188
|
+
const logical = [];
|
|
189
|
+
let index = 0;
|
|
190
|
+
while (index < physical.length) {
|
|
191
|
+
const current = physical[index];
|
|
192
|
+
if (!isCommentLine(current)) {
|
|
193
|
+
logical.push({ line: index + 1, text: current, tight: current });
|
|
194
|
+
index += 1;
|
|
195
|
+
continue;
|
|
196
|
+
}
|
|
197
|
+
const parts = [];
|
|
198
|
+
const start = index;
|
|
199
|
+
while (index < physical.length) {
|
|
200
|
+
const line = physical[index];
|
|
201
|
+
if (!isCommentLine(line))
|
|
202
|
+
break;
|
|
203
|
+
const stripped = commentText(line);
|
|
204
|
+
if (stripped !== '')
|
|
205
|
+
parts.push(stripped);
|
|
206
|
+
index += 1;
|
|
207
|
+
}
|
|
208
|
+
logical.push({
|
|
209
|
+
line: start + 1,
|
|
210
|
+
text: parts.join(' '),
|
|
211
|
+
tight: parts.join(''),
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
return logical;
|
|
215
|
+
}
|
|
216
|
+
export function scanPackageBoundary(files, patterns) {
|
|
217
|
+
const violations = [];
|
|
218
|
+
for (const [file, source] of files) {
|
|
219
|
+
for (const { line, text, tight } of logicalLines(source)) {
|
|
220
|
+
if (text === '')
|
|
221
|
+
continue;
|
|
222
|
+
if (text.length > MAX_SCANNED_LINE) {
|
|
223
|
+
// Reported rather than skipped: a line the gate declined to match is
|
|
224
|
+
// a line nobody held, and silence there is the pass this bound would
|
|
225
|
+
// otherwise buy.
|
|
226
|
+
violations.push({
|
|
227
|
+
file,
|
|
228
|
+
line,
|
|
229
|
+
pattern: OVERLONG_LINE,
|
|
230
|
+
reason: `a logical line of ${text.length} characters is past the ${MAX_SCANNED_LINE}-character bound a pattern is matched within, so no pattern was run against it`,
|
|
231
|
+
text: `${text.slice(0, 120)}...`,
|
|
232
|
+
});
|
|
233
|
+
continue;
|
|
234
|
+
}
|
|
235
|
+
for (const pattern of patterns) {
|
|
236
|
+
if (!pattern.regex.test(text) && !pattern.regex.test(tight))
|
|
237
|
+
continue;
|
|
238
|
+
// One violation per logical line: the `break` is what makes the
|
|
239
|
+
// declared order decide the reported name. A line carrying two
|
|
240
|
+
// forbidden references is one edit either way.
|
|
241
|
+
violations.push({
|
|
242
|
+
file,
|
|
243
|
+
line,
|
|
244
|
+
pattern: pattern.name,
|
|
245
|
+
reason: pattern.reason,
|
|
246
|
+
text,
|
|
247
|
+
});
|
|
248
|
+
break;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
return violations;
|
|
253
|
+
}
|
|
254
|
+
const matchesExtension = (declared, name) => declared.extensions === undefined ||
|
|
255
|
+
declared.extensions.some((extension) => name.endsWith(extension));
|
|
256
|
+
async function walk(root, posix, declared, entries) {
|
|
257
|
+
let dirents;
|
|
258
|
+
try {
|
|
259
|
+
dirents = await readdir(resolve(root, posix), { withFileTypes: true });
|
|
260
|
+
}
|
|
261
|
+
catch (error) {
|
|
262
|
+
throw codedError(SCAN_UNREADABLE, `${posix} could not be read: ${detail(error)}`);
|
|
263
|
+
}
|
|
264
|
+
let count = 0;
|
|
265
|
+
for (const entry of dirents) {
|
|
266
|
+
const child = `${posix}/${entry.name}`;
|
|
267
|
+
if (entry.isSymbolicLink()) {
|
|
268
|
+
throw codedError(SCAN_UNREADABLE, `${child} is a symbolic link; this scan does not follow links, and skipping one would leave a file unscanned while the run reported a clean tree`);
|
|
269
|
+
}
|
|
270
|
+
if (entry.isDirectory()) {
|
|
271
|
+
if (!declared.recursive)
|
|
272
|
+
continue;
|
|
273
|
+
count += await walk(root, child, declared, entries);
|
|
274
|
+
}
|
|
275
|
+
else if (entry.isFile() && matchesExtension(declared, entry.name)) {
|
|
276
|
+
entries.set(child, await readFile(resolve(root, child), 'utf8'));
|
|
277
|
+
count += 1;
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
return count;
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* Everything the declared paths hold, keyed by the path a report prints. Fails
|
|
284
|
+
* closed on the three ways a walk under-reports: a path that is not there, a
|
|
285
|
+
* path that matched no file, and a link whose target could sit anywhere.
|
|
286
|
+
*/
|
|
287
|
+
export async function discoverEntries(root, paths, gate) {
|
|
288
|
+
const entries = new Map();
|
|
289
|
+
const counts = [];
|
|
290
|
+
for (const declared of paths) {
|
|
291
|
+
let info;
|
|
292
|
+
try {
|
|
293
|
+
info = await lstat(resolve(root, declared.path));
|
|
294
|
+
}
|
|
295
|
+
catch (error) {
|
|
296
|
+
if (error.code !== 'ENOENT') {
|
|
297
|
+
throw codedError(SCAN_UNREADABLE, `${declared.path} could not be read: ${detail(error)}`);
|
|
298
|
+
}
|
|
299
|
+
if (declared.optional) {
|
|
300
|
+
counts.push({ path: declared.path, files: 0 });
|
|
301
|
+
continue;
|
|
302
|
+
}
|
|
303
|
+
throw codedError(SCAN_PATH_ERROR, `${declared.path} does not exist, and the "${gate}" section names it under paths; mark it optional if it may be absent`);
|
|
304
|
+
}
|
|
305
|
+
if (info.isSymbolicLink()) {
|
|
306
|
+
throw codedError(SCAN_UNREADABLE, `${declared.path} is a symbolic link, and this scan does not follow links`);
|
|
307
|
+
}
|
|
308
|
+
if (info.isFile()) {
|
|
309
|
+
entries.set(declared.path, await readFile(resolve(root, declared.path), 'utf8'));
|
|
310
|
+
counts.push({ path: declared.path, files: 1 });
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
313
|
+
const found = await walk(root, declared.path, declared, entries);
|
|
314
|
+
if (found === 0 && !declared.optional) {
|
|
315
|
+
throw codedError(SCAN_PATH_ERROR, `${declared.path} holds no file the "${gate}" section asked for, so a scan of nothing would report zero violations for the wrong reason; mark it optional if it may be empty`);
|
|
316
|
+
}
|
|
317
|
+
counts.push({ path: declared.path, files: found });
|
|
318
|
+
}
|
|
319
|
+
return { entries, counts };
|
|
320
|
+
}
|
|
321
|
+
function flattenField(key, value, into) {
|
|
322
|
+
if (typeof value === 'string') {
|
|
323
|
+
into.set(key, value);
|
|
324
|
+
return;
|
|
325
|
+
}
|
|
326
|
+
if (typeof value === 'number' || typeof value === 'boolean') {
|
|
327
|
+
into.set(key, String(value));
|
|
328
|
+
return;
|
|
329
|
+
}
|
|
330
|
+
if (Array.isArray(value)) {
|
|
331
|
+
into.set(key, value.map((each) => String(each)).join(' '));
|
|
332
|
+
return;
|
|
333
|
+
}
|
|
334
|
+
if (value !== null && typeof value === 'object') {
|
|
335
|
+
for (const [name, nested] of Object.entries(value)) {
|
|
336
|
+
flattenField(`${key}.${name}`, nested, into);
|
|
337
|
+
}
|
|
338
|
+
return;
|
|
339
|
+
}
|
|
340
|
+
throw codedError(SCAN_PATH_ERROR, `${key} is null, so there is nothing to scan`);
|
|
341
|
+
}
|
|
342
|
+
export async function manifestEntries(root, manifest, gate) {
|
|
343
|
+
let text;
|
|
344
|
+
try {
|
|
345
|
+
text = await readFile(resolve(root, manifest.file), 'utf8');
|
|
346
|
+
}
|
|
347
|
+
catch (error) {
|
|
348
|
+
throw codedError(SCAN_PATH_ERROR, `${manifest.file} could not be read: ${detail(error)}; the "${gate}" section names it under manifest.file`);
|
|
349
|
+
}
|
|
350
|
+
let parsed;
|
|
351
|
+
try {
|
|
352
|
+
parsed = JSON.parse(text);
|
|
353
|
+
}
|
|
354
|
+
catch (error) {
|
|
355
|
+
throw codedError(SCAN_PATH_ERROR, `${manifest.file} is not valid JSON: ${detail(error)}`);
|
|
356
|
+
}
|
|
357
|
+
const entries = new Map();
|
|
358
|
+
for (const field of manifest.fields) {
|
|
359
|
+
let value = parsed;
|
|
360
|
+
for (const segment of field.split('.')) {
|
|
361
|
+
value =
|
|
362
|
+
value !== null && typeof value === 'object'
|
|
363
|
+
? value[segment]
|
|
364
|
+
: undefined;
|
|
365
|
+
}
|
|
366
|
+
if (value === undefined) {
|
|
367
|
+
throw codedError(SCAN_PATH_ERROR, `${manifest.file} carries no "${field}", and the "${gate}" section names it under manifest.fields; a field that is not there would otherwise read as a field with nothing in it`);
|
|
368
|
+
}
|
|
369
|
+
flattenField(`${manifest.file}#${field}`, value, entries);
|
|
370
|
+
}
|
|
371
|
+
return entries;
|
|
372
|
+
}
|
|
373
|
+
/**
|
|
374
|
+
* The gate, over a consumer's tree. `root` is the directory its configuration
|
|
375
|
+
* file sits in, which is what makes a configuration self-contained wherever it
|
|
376
|
+
* is kept.
|
|
377
|
+
*/
|
|
378
|
+
export async function runPackageBoundary(root, section, gate = 'package-boundary') {
|
|
379
|
+
const { entries, counts } = await discoverEntries(root, section.paths, gate);
|
|
380
|
+
if (section.manifest !== undefined) {
|
|
381
|
+
const fields = await manifestEntries(root, section.manifest, gate);
|
|
382
|
+
for (const [key, value] of fields)
|
|
383
|
+
entries.set(key, value);
|
|
384
|
+
counts.push({ path: section.manifest.file, files: fields.size });
|
|
385
|
+
}
|
|
386
|
+
const violations = scanPackageBoundary(entries, compileBoundaryPatterns(section.patterns));
|
|
387
|
+
return { violations, counts, scanned: entries.size };
|
|
388
|
+
}
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
// The one tokenizer both source-scanning gates share. Pure and synchronous.
|
|
2
|
+
//
|
|
3
|
+
// TypeScript 7.0.2 ships no in-process parser, and spawning the native `tsgo`
|
|
4
|
+
// binary is too slow for a lint gate, so `createScanner` is the surface left.
|
|
5
|
+
// Two of its context-dependent decisions belong to the parser, and `scanTokens`
|
|
6
|
+
// makes them:
|
|
7
|
+
//
|
|
8
|
+
// A slash is division or the start of a regex depending on what came before.
|
|
9
|
+
// Left alone, every regex body leaks into the stream as code: `/^#/` emits a
|
|
10
|
+
// zero-width token forever, a backtick opens a template that runs to end of
|
|
11
|
+
// file, `\/` opens a line comment that eats the rest of its line, and `{`
|
|
12
|
+
// inside a character class shifts brace depth for every line after it. The
|
|
13
|
+
// re-scan below decides it the way the parser does, from the previous token.
|
|
14
|
+
//
|
|
15
|
+
// The closing `}` of a template substitution is a brace or a template
|
|
16
|
+
// continuation. Left alone the template's tail reads as code and its closing
|
|
17
|
+
// backtick opens a second template, so the file's whole tail is fiction.
|
|
18
|
+
//
|
|
19
|
+
// Three guards catch what is left: a token that makes no progress, an
|
|
20
|
+
// unterminated literal, and a stream that ends with an unbalanced brace or an
|
|
21
|
+
// open template. Each throws with an offset instead of returning a stream a
|
|
22
|
+
// gate would trust.
|
|
23
|
+
import { computeLineStarts, createScanner, SyntaxKind, } from 'typescript/unstable/ast';
|
|
24
|
+
/**
|
|
25
|
+
* Token kinds that can end an expression, so a following `/` is division. The
|
|
26
|
+
* type keywords are here for an `as` clause: `y as number / 2` is division.
|
|
27
|
+
*/
|
|
28
|
+
const ENDS_EXPRESSION = new Set([
|
|
29
|
+
SyntaxKind.Identifier,
|
|
30
|
+
SyntaxKind.NumericLiteral,
|
|
31
|
+
SyntaxKind.BigIntLiteral,
|
|
32
|
+
SyntaxKind.StringLiteral,
|
|
33
|
+
SyntaxKind.NoSubstitutionTemplateLiteral,
|
|
34
|
+
SyntaxKind.TemplateTail,
|
|
35
|
+
SyntaxKind.RegularExpressionLiteral,
|
|
36
|
+
SyntaxKind.CloseParenToken,
|
|
37
|
+
SyntaxKind.CloseBracketToken,
|
|
38
|
+
SyntaxKind.CloseBraceToken,
|
|
39
|
+
SyntaxKind.ThisKeyword,
|
|
40
|
+
SyntaxKind.SuperKeyword,
|
|
41
|
+
SyntaxKind.TrueKeyword,
|
|
42
|
+
SyntaxKind.FalseKeyword,
|
|
43
|
+
SyntaxKind.NullKeyword,
|
|
44
|
+
SyntaxKind.PlusPlusToken,
|
|
45
|
+
SyntaxKind.MinusMinusToken,
|
|
46
|
+
SyntaxKind.NumberKeyword,
|
|
47
|
+
SyntaxKind.StringKeyword,
|
|
48
|
+
SyntaxKind.BooleanKeyword,
|
|
49
|
+
SyntaxKind.AnyKeyword,
|
|
50
|
+
SyntaxKind.UnknownKeyword,
|
|
51
|
+
SyntaxKind.ObjectKeyword,
|
|
52
|
+
SyntaxKind.SymbolKeyword,
|
|
53
|
+
SyntaxKind.BigIntKeyword,
|
|
54
|
+
SyntaxKind.NeverKeyword,
|
|
55
|
+
SyntaxKind.VoidKeyword,
|
|
56
|
+
SyntaxKind.UndefinedKeyword,
|
|
57
|
+
SyntaxKind.ConstKeyword,
|
|
58
|
+
]);
|
|
59
|
+
/** A `(` after one of these heads a statement, so its `)` ends no expression. */
|
|
60
|
+
const CONTROL_HEADS = new Set([
|
|
61
|
+
SyntaxKind.IfKeyword,
|
|
62
|
+
SyntaxKind.WhileKeyword,
|
|
63
|
+
SyntaxKind.ForKeyword,
|
|
64
|
+
SyntaxKind.CatchKeyword,
|
|
65
|
+
SyntaxKind.WithKeyword,
|
|
66
|
+
]);
|
|
67
|
+
/**
|
|
68
|
+
* A `{` after one of these is an object or type literal; anywhere else it opens
|
|
69
|
+
* a block. `=>` is deliberately absent: an arrow's braces are always a body,
|
|
70
|
+
* and an object-literal body needs parentheses, which `(` already covers.
|
|
71
|
+
*/
|
|
72
|
+
const LITERAL_HEADS = new Set([
|
|
73
|
+
SyntaxKind.OpenParenToken,
|
|
74
|
+
SyntaxKind.CommaToken,
|
|
75
|
+
SyntaxKind.ColonToken,
|
|
76
|
+
SyntaxKind.OpenBracketToken,
|
|
77
|
+
SyntaxKind.ReturnKeyword,
|
|
78
|
+
SyntaxKind.QuestionToken,
|
|
79
|
+
SyntaxKind.ExtendsKeyword,
|
|
80
|
+
SyntaxKind.LessThanToken,
|
|
81
|
+
SyntaxKind.BarToken,
|
|
82
|
+
SyntaxKind.AmpersandToken,
|
|
83
|
+
]);
|
|
84
|
+
/** Where a statement can begin, which is where a label can. */
|
|
85
|
+
const STATEMENT_BOUNDARIES = new Set([
|
|
86
|
+
-1,
|
|
87
|
+
SyntaxKind.SemicolonToken,
|
|
88
|
+
SyntaxKind.OpenBraceToken,
|
|
89
|
+
SyntaxKind.CloseBraceToken,
|
|
90
|
+
]);
|
|
91
|
+
export function scanTokens(source) {
|
|
92
|
+
const scanner = createScanner(/* skipTrivia */ true, undefined, source);
|
|
93
|
+
const tokens = [];
|
|
94
|
+
// Brace depth at each open template's substitution, innermost last.
|
|
95
|
+
const templates = [];
|
|
96
|
+
// Whether each open `(` heads a statement and each open `{` opens a block,
|
|
97
|
+
// innermost last. A statement's `)` and a block's `}` end no expression, so
|
|
98
|
+
// a `/` after either starts a regex.
|
|
99
|
+
const controlParens = [];
|
|
100
|
+
const blockBraces = [];
|
|
101
|
+
let depth = 0;
|
|
102
|
+
let previousEnd = -1;
|
|
103
|
+
let previousKind = -1;
|
|
104
|
+
let beforePreviousKind = -1;
|
|
105
|
+
let thirdPreviousKind = -1;
|
|
106
|
+
let previousEnds = false;
|
|
107
|
+
/**
|
|
108
|
+
* True when the `:` just scanned labels a statement. A label needs an
|
|
109
|
+
* identifier at statement position, so a ternary's `g :` and a member's
|
|
110
|
+
* `a :` inside an open literal are both excluded.
|
|
111
|
+
*/
|
|
112
|
+
const labelColon = () => previousKind === SyntaxKind.ColonToken &&
|
|
113
|
+
beforePreviousKind === SyntaxKind.Identifier &&
|
|
114
|
+
STATEMENT_BOUNDARIES.has(thirdPreviousKind) &&
|
|
115
|
+
blockBraces[blockBraces.length - 1] !== false;
|
|
116
|
+
while (true) {
|
|
117
|
+
let kind = scanner.scan();
|
|
118
|
+
if (kind === SyntaxKind.EndOfFile)
|
|
119
|
+
break;
|
|
120
|
+
if ((kind === SyntaxKind.SlashToken ||
|
|
121
|
+
kind === SyntaxKind.SlashEqualsToken) &&
|
|
122
|
+
!previousEnds) {
|
|
123
|
+
kind = scanner.reScanSlashToken();
|
|
124
|
+
}
|
|
125
|
+
const end = scanner.getTokenEnd();
|
|
126
|
+
if (end === previousEnd) {
|
|
127
|
+
throw new Error(`token scan made no progress at offset ${scanner.getTokenStart()}; the source is outside what this tokenizer can read`);
|
|
128
|
+
}
|
|
129
|
+
if (scanner.isUnterminated()) {
|
|
130
|
+
throw new Error(`token scan hit an unterminated literal at offset ${scanner.getTokenStart()}; everything after it would be read as the wrong kind of token`);
|
|
131
|
+
}
|
|
132
|
+
previousEnd = end;
|
|
133
|
+
let ends = ENDS_EXPRESSION.has(kind);
|
|
134
|
+
if (kind === SyntaxKind.OpenParenToken) {
|
|
135
|
+
// `p.catch(f)` is a call and `for await (…)` is a statement, so the
|
|
136
|
+
// token before the keyword decides both.
|
|
137
|
+
controlParens.push((CONTROL_HEADS.has(previousKind) &&
|
|
138
|
+
beforePreviousKind !== SyntaxKind.DotToken) ||
|
|
139
|
+
(previousKind === SyntaxKind.AwaitKeyword &&
|
|
140
|
+
beforePreviousKind === SyntaxKind.ForKeyword));
|
|
141
|
+
}
|
|
142
|
+
else if (kind === SyntaxKind.CloseParenToken) {
|
|
143
|
+
if (controlParens.pop() === true)
|
|
144
|
+
ends = false;
|
|
145
|
+
}
|
|
146
|
+
else if (kind === SyntaxKind.OpenBraceToken) {
|
|
147
|
+
depth++;
|
|
148
|
+
blockBraces.push(!((LITERAL_HEADS.has(previousKind) ||
|
|
149
|
+
(previousKind >= SyntaxKind.FirstAssignment &&
|
|
150
|
+
previousKind <= SyntaxKind.LastAssignment)) &&
|
|
151
|
+
!labelColon()));
|
|
152
|
+
}
|
|
153
|
+
else if (kind === SyntaxKind.CloseBraceToken) {
|
|
154
|
+
if (templates[templates.length - 1] === depth) {
|
|
155
|
+
kind = scanner.reScanTemplateToken(/* isTaggedTemplate */ false);
|
|
156
|
+
if (kind === SyntaxKind.TemplateTail)
|
|
157
|
+
templates.pop();
|
|
158
|
+
ends = ENDS_EXPRESSION.has(kind);
|
|
159
|
+
}
|
|
160
|
+
else {
|
|
161
|
+
// A brace closing a real block or literal. Drop this and a later
|
|
162
|
+
// `}` is misread as a template continuation.
|
|
163
|
+
depth--;
|
|
164
|
+
if (blockBraces.pop() === true)
|
|
165
|
+
ends = false;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
if (kind === SyntaxKind.TemplateHead)
|
|
169
|
+
templates.push(depth);
|
|
170
|
+
// A postfix `!` inherits: `x! / 2` is division, `!/re/.test(s)` is a
|
|
171
|
+
// regex, and the token before the `!` is what separates them.
|
|
172
|
+
if (kind === SyntaxKind.ExclamationToken)
|
|
173
|
+
ends = previousEnds;
|
|
174
|
+
thirdPreviousKind = beforePreviousKind;
|
|
175
|
+
beforePreviousKind = previousKind;
|
|
176
|
+
previousKind = kind;
|
|
177
|
+
previousEnds = ends;
|
|
178
|
+
tokens.push({
|
|
179
|
+
kind,
|
|
180
|
+
text: scanner.getTokenText(),
|
|
181
|
+
value: scanner.getTokenValue(),
|
|
182
|
+
start: scanner.getTokenStart(),
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
if (depth !== 0 || templates.length > 0) {
|
|
186
|
+
throw new Error(`token scan ended with brace depth ${depth} and ${templates.length} open template(s); the stream desynced somewhere in this file`);
|
|
187
|
+
}
|
|
188
|
+
return tokens;
|
|
189
|
+
}
|
|
190
|
+
/** 1-indexed line holding `pos`, by binary search over `computeLineStarts`. */
|
|
191
|
+
export function lineOf(lineStarts, pos) {
|
|
192
|
+
let low = 0;
|
|
193
|
+
let high = lineStarts.length - 1;
|
|
194
|
+
while (low < high) {
|
|
195
|
+
const mid = (low + high + 1) >> 1;
|
|
196
|
+
if ((lineStarts[mid] ?? 0) <= pos)
|
|
197
|
+
low = mid;
|
|
198
|
+
else
|
|
199
|
+
high = mid - 1;
|
|
200
|
+
}
|
|
201
|
+
return low + 1;
|
|
202
|
+
}
|
|
203
|
+
export { computeLineStarts };
|
package/dist/index.d.ts
CHANGED
|
@@ -1,15 +1,25 @@
|
|
|
1
1
|
export * from './application/index.ts';
|
|
2
2
|
export type { ArtifactReference } from './core/schemas/artifact-reference.ts';
|
|
3
3
|
export type { EvalContract } from './core/schemas/eval-contract.ts';
|
|
4
|
+
export { EVAL_CONTRACT_SCHEMA_VERSION } from './core/schemas/eval-contract.ts';
|
|
4
5
|
export type { EvaluatorConfiguration } from './core/schemas/evaluator-configuration.ts';
|
|
6
|
+
export { EVALUATOR_CONFIGURATION_SCHEMA_VERSION } from './core/schemas/evaluator-configuration.ts';
|
|
5
7
|
export type { EvidenceArtifact } from './core/schemas/evidence-artifact.ts';
|
|
8
|
+
export { EVIDENCE_ARTIFACT_SCHEMA_VERSION } from './core/schemas/evidence-artifact.ts';
|
|
6
9
|
export type { IsolationManifest } from './core/schemas/isolation-manifest.ts';
|
|
10
|
+
export { ISOLATION_MANIFEST_SCHEMA_VERSION } from './core/schemas/isolation-manifest.ts';
|
|
7
11
|
export type { PreflightCheck, PreflightVerdict, } from './core/schemas/preflight-verdict.ts';
|
|
12
|
+
export { PREFLIGHT_VERDICT_SCHEMA_VERSION } from './core/schemas/preflight-verdict.ts';
|
|
8
13
|
export type { PrivateArtifactManifest } from './core/schemas/private-artifact-manifest.ts';
|
|
14
|
+
export { PRIVATE_ARTIFACT_MANIFEST_SCHEMA_VERSION } from './core/schemas/private-artifact-manifest.ts';
|
|
9
15
|
export type { Probe } from './core/schemas/probe.ts';
|
|
16
|
+
export { PROBE_SCHEMA_VERSION } from './core/schemas/probe.ts';
|
|
10
17
|
export type { Rubric } from './core/schemas/rubric.ts';
|
|
11
18
|
export type { ScoringPolicy } from './core/schemas/scoring-policy.ts';
|
|
19
|
+
export { SCORING_POLICY_SCHEMA_VERSION } from './core/schemas/scoring-policy.ts';
|
|
12
20
|
export type { SealedEvaluatorBrief } from './core/schemas/sealed-evaluator-brief.ts';
|
|
21
|
+
export { SEALED_EVALUATOR_BRIEF_SCHEMA_VERSION } from './core/schemas/sealed-evaluator-brief.ts';
|
|
13
22
|
export type { SealedRunRecord } from './core/schemas/sealed-run-record.ts';
|
|
23
|
+
export { SEALED_RUN_RECORD_SCHEMA_VERSION } from './core/schemas/sealed-run-record.ts';
|
|
14
24
|
export type { FixtureReset, ManifestationWitness, SensitivityWitness, SensitivityWitnessLeg, WitnessChannel, WitnessInputs, } from './core/schemas/sensitivity-witness.ts';
|
|
15
|
-
export declare const VERSION = "
|
|
25
|
+
export declare const VERSION = "3.1.0";
|
package/dist/index.js
CHANGED
|
@@ -18,5 +18,24 @@
|
|
|
18
18
|
// second edge. The port vocabulary stays at the `eval-quality/conformance`
|
|
19
19
|
// subpath, where AD-37 puts the conformance definition an adapter author
|
|
20
20
|
// reads; the reference adapters stay at `eval-quality/adapters`.
|
|
21
|
+
//
|
|
22
|
+
// The schema versions ride that second edge beside the artifact types they
|
|
23
|
+
// stamp, so a consumer satisfying AD-11's equality rule imports the number this
|
|
24
|
+
// build reads. Ten of the twelve artifacts carry one: the two with an in-package
|
|
25
|
+
// reader, the three this package stamps, and the five a caller assembles and
|
|
26
|
+
// `score` validates. `artifact-reference` carries no lineage at all. A rubric
|
|
27
|
+
// does carry a `schemaVersion`, and no constant here states it: this package
|
|
28
|
+
// never parses a standalone rubric, and the eval contract embeds `RubricBody`,
|
|
29
|
+
// the body without lineage.
|
|
21
30
|
export * from './application/index.js';
|
|
22
|
-
export
|
|
31
|
+
export { EVAL_CONTRACT_SCHEMA_VERSION } from './core/schemas/eval-contract.js';
|
|
32
|
+
export { EVALUATOR_CONFIGURATION_SCHEMA_VERSION } from './core/schemas/evaluator-configuration.js';
|
|
33
|
+
export { EVIDENCE_ARTIFACT_SCHEMA_VERSION } from './core/schemas/evidence-artifact.js';
|
|
34
|
+
export { ISOLATION_MANIFEST_SCHEMA_VERSION } from './core/schemas/isolation-manifest.js';
|
|
35
|
+
export { PREFLIGHT_VERDICT_SCHEMA_VERSION } from './core/schemas/preflight-verdict.js';
|
|
36
|
+
export { PRIVATE_ARTIFACT_MANIFEST_SCHEMA_VERSION } from './core/schemas/private-artifact-manifest.js';
|
|
37
|
+
export { PROBE_SCHEMA_VERSION } from './core/schemas/probe.js';
|
|
38
|
+
export { SCORING_POLICY_SCHEMA_VERSION } from './core/schemas/scoring-policy.js';
|
|
39
|
+
export { SEALED_EVALUATOR_BRIEF_SCHEMA_VERSION } from './core/schemas/sealed-evaluator-brief.js';
|
|
40
|
+
export { SEALED_RUN_RECORD_SCHEMA_VERSION } from './core/schemas/sealed-run-record.js';
|
|
41
|
+
export const VERSION = '3.1.0';
|