eval-quality 3.2.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/application/index.d.ts +6 -0
- package/dist/application/index.js +8 -0
- package/dist/gates/check-dependency-direction.js +10 -10
- package/dist/gates/check-doc-claims.js +253 -10
- package/dist/gates/check-doc-invocations.mjs +89 -30
- package/dist/gates/dependency-direction.js +130 -0
- package/dist/gates/gate-config.js +12 -9
- package/dist/gates/lineage-ownership.js +19 -22
- package/dist/gates/typescript-scanner.js +181 -0
- package/dist/index.d.ts +5 -3
- package/dist/index.js +1 -1
- package/dist/ports/environment-probe-port.d.ts +12 -12
- package/package.json +5 -2
package/README.md
CHANGED
|
@@ -134,7 +134,7 @@ Node.js 22.20.0 or newer. `zod` is the only production dependency.
|
|
|
134
134
|
npm install eval-quality
|
|
135
135
|
```
|
|
136
136
|
|
|
137
|
-
Every command runs through `npx
|
|
137
|
+
Every command runs through `npx`. The four below are the grammar rather than a sequence to copy: they name files your own harness produces. [The full walkthrough](https://bmad-code-org.github.io/bmad-eval-quality/how-to/author-behavioral-contracts/) runs the same four end to end over files this repository commits, with no placeholder in the path.
|
|
138
138
|
|
|
139
139
|
```bash
|
|
140
140
|
npx eval-quality compile --in contract.json --out ./eval-out
|
|
@@ -6,6 +6,11 @@
|
|
|
6
6
|
* unamended.
|
|
7
7
|
*/
|
|
8
8
|
export { digestArtifact, digestBytes, digestComposite, } from '../core/canonical/digest.ts';
|
|
9
|
+
export { makePointerDenotesCollection, makeResolveOperand, referenceSetKeysOf, } from '../core/evaluate/evidence-resolution.ts';
|
|
10
|
+
export type { PointerDenotesCollection, ReferenceSetKeys, ResolveOperand, } from '../core/evaluate/resolution.ts';
|
|
11
|
+
export { resolveCheck } from '../core/evaluate/resolution.ts';
|
|
12
|
+
export type { ResolvedValue } from '../core/evaluate/resolved-value.ts';
|
|
13
|
+
export { ABSENT } from '../core/evaluate/resolved-value.ts';
|
|
9
14
|
export type { FailureCode } from '../core/failure-codes.ts';
|
|
10
15
|
export { FAILURE_CODES, StructuralFailure } from '../core/failure-codes.ts';
|
|
11
16
|
export type { LineageChainReport, LineageFinding, } from '../core/lineage/chain.ts';
|
|
@@ -21,6 +26,7 @@ export type { QualificationFailure, QualificationFailureCode, QualificationResul
|
|
|
21
26
|
export { QUALIFICATION_FAILURES } from '../core/score/qualification.ts';
|
|
22
27
|
export type { ComparableResult, DominanceRelationValue, } from '../core/score/strength.ts';
|
|
23
28
|
export { compareDominance, DOMINANCE_RELATIONS, } from '../core/score/strength.ts';
|
|
29
|
+
export type { PlanIndex } from '../core/seal/plan-index.ts';
|
|
24
30
|
export { compile } from './compile.ts';
|
|
25
31
|
export type { Diagnostic, DiagnosticSink } from './diagnostics.ts';
|
|
26
32
|
export type { PreflightFromObservationsOptions, RunPreflightOptions, } from './preflight.ts';
|
|
@@ -6,6 +6,14 @@
|
|
|
6
6
|
* unamended.
|
|
7
7
|
*/
|
|
8
8
|
export { digestArtifact, digestBytes, digestComposite, } from '../core/canonical/digest.js';
|
|
9
|
+
// The evaluator, for a consumer that resolves a check itself: the resolver,
|
|
10
|
+
// the two factories that build its operand and collection predicates from a
|
|
11
|
+
// contract and its observations, and the reference-set keys. `ABSENT` ships
|
|
12
|
+
// beside them because a `ResolveOperand` a consumer writes has to return it,
|
|
13
|
+
// and a sentinel the type names and the barrel withholds cannot be returned.
|
|
14
|
+
export { makePointerDenotesCollection, makeResolveOperand, referenceSetKeysOf, } from '../core/evaluate/evidence-resolution.js';
|
|
15
|
+
export { resolveCheck } from '../core/evaluate/resolution.js';
|
|
16
|
+
export { ABSENT } from '../core/evaluate/resolved-value.js';
|
|
9
17
|
export { FAILURE_CODES, StructuralFailure } from '../core/failure-codes.js';
|
|
10
18
|
export { validateLineageChain } from '../core/lineage/chain.js';
|
|
11
19
|
export { INTERCHANGE_ARTIFACT_KEYS } from '../core/schemas/artifact.js';
|
|
@@ -18,6 +18,7 @@
|
|
|
18
18
|
// appear in this file or anything it imports, or the gate fails at load.
|
|
19
19
|
import { z } from 'zod';
|
|
20
20
|
import { discoverSourceFiles } from './discover-source-files.js';
|
|
21
|
+
import { isCode, loadTypeScriptScanner, TYPESCRIPT_UNAVAILABLE, } from './typescript-scanner.js';
|
|
21
22
|
/** The gate's key in the configuration file, and the token the binary dispatches on. */
|
|
22
23
|
export const DEPENDENCY_DIRECTION_GATE = 'dependency-direction';
|
|
23
24
|
/**
|
|
@@ -26,7 +27,7 @@ export const DEPENDENCY_DIRECTION_GATE = 'dependency-direction';
|
|
|
26
27
|
* description and asserted by `tests/architecture/dependency-direction.test.ts`,
|
|
27
28
|
* so the ordering property below is a measured fact rather than a warning.
|
|
28
29
|
*/
|
|
29
|
-
export const ORDERING_WITNESS_VIOLATIONS =
|
|
30
|
+
export const ORDERING_WITNESS_VIOLATIONS = 80;
|
|
30
31
|
/** The optional peer is absent. The consumer repairs it by installing it, so it takes the usage code. */
|
|
31
32
|
export const TYPESCRIPT_PEER_MISSING = 'EVAL_QUALITY_TYPESCRIPT_PEER_MISSING';
|
|
32
33
|
/** A declared scan root could not be walked. Also a usage code: nothing was scanned, so nothing was answered. */
|
|
@@ -228,25 +229,24 @@ const refuse = (code, message) => ({
|
|
|
228
229
|
/**
|
|
229
230
|
* The scanner needs `typescript/unstable/ast`, and `typescript` is an optional
|
|
230
231
|
* peer dependency so that a consumer running the other gates installs nothing.
|
|
231
|
-
*
|
|
232
|
-
* the dependency and the gate that wanted it
|
|
232
|
+
* `typescript-scanner.ts` is what turns a resolver stack trace into a sentence
|
|
233
|
+
* naming the dependency, the installed version and the gate that wanted it; a
|
|
234
|
+
* failure of any other shape is a bug in the scanner, not a missing peer, and
|
|
235
|
+
* is rethrown unchanged.
|
|
233
236
|
*
|
|
234
237
|
* `load` is injectable so a test can exercise the refusal without uninstalling
|
|
235
238
|
* the package the test runner itself needs.
|
|
236
239
|
*/
|
|
237
|
-
export async function probeTypeScript(load = () =>
|
|
240
|
+
export async function probeTypeScript(load = () => loadTypeScriptScanner(DEPENDENCY_DIRECTION_GATE)) {
|
|
238
241
|
try {
|
|
239
242
|
await load();
|
|
240
243
|
return { ok: true };
|
|
241
244
|
}
|
|
242
245
|
catch (error) {
|
|
243
|
-
if (error
|
|
244
|
-
|
|
246
|
+
if (isCode(error, TYPESCRIPT_UNAVAILABLE)) {
|
|
247
|
+
return { ok: false, message: error.message };
|
|
245
248
|
}
|
|
246
|
-
|
|
247
|
-
ok: false,
|
|
248
|
-
message: `the ${DEPENDENCY_DIRECTION_GATE} gate reads your source with the TypeScript scanner, and the optional peer dependency "typescript" is not installed here. Install it (npm install --save-dev typescript), or drop the "${DEPENDENCY_DIRECTION_GATE}" section from your configuration to stop invoking this gate. No other gate needs it.`,
|
|
249
|
-
};
|
|
249
|
+
throw error;
|
|
250
250
|
}
|
|
251
251
|
}
|
|
252
252
|
const orderViolations = (violations) => [...violations].sort((a, b) => a.file === b.file ? a.line - b.line : a.file < b.file ? -1 : 1);
|
|
@@ -50,6 +50,22 @@
|
|
|
50
50
|
// sentence was rewritten fails as a dead entry. That is weaker than deciding
|
|
51
51
|
// the claim and stronger than the nothing that precedes it.
|
|
52
52
|
//
|
|
53
|
+
// A "read" claim by itself proves only that the sentence was once registered,
|
|
54
|
+
// and nothing re-checks the reading afterward: the artifact it was read from
|
|
55
|
+
// can drift for years and the entry stays green. A claim may carry an `asOf`
|
|
56
|
+
// pin against that: a normalized-content sha256 of the file the human actually
|
|
57
|
+
// read, defaulting to the claim's own page. An edit to that file changes the
|
|
58
|
+
// hash and fails the entry, which is what turns "read once" into "read, and
|
|
59
|
+
// still current." A `settles` predicate needs none of this, because it already
|
|
60
|
+
// re-runs every check.
|
|
61
|
+
//
|
|
62
|
+
// The pin only works for a subject this run can read, so it is worth nothing
|
|
63
|
+
// against a fact about another repository or a live system; naming a subject
|
|
64
|
+
// in this tree that is not the actual evidence, such as the page merely
|
|
65
|
+
// stating the fact, buys a false sense of protection rather than none. Naming
|
|
66
|
+
// the real evidence file under `subject`, in-tree, is what makes the pin mean
|
|
67
|
+
// something.
|
|
68
|
+
//
|
|
53
69
|
// What stays outside all eight: editorial judgment, design rationale, anything
|
|
54
70
|
// about the world beyond the tree, any claim about runtime behaviour that only
|
|
55
71
|
// executing the code would settle, and whether a code a page names is the one
|
|
@@ -62,6 +78,7 @@
|
|
|
62
78
|
// Run by `node` directly: Node's type stripping erases types only, so no
|
|
63
79
|
// TypeScript enum, namespace, parameter property, or non-type re-export may
|
|
64
80
|
// appear in this file or anything it imports.
|
|
81
|
+
import { createHash } from 'node:crypto';
|
|
65
82
|
import { realpathSync } from 'node:fs';
|
|
66
83
|
import { lstat, readdir, readFile } from 'node:fs/promises';
|
|
67
84
|
import { resolve, sep } from 'node:path';
|
|
@@ -225,6 +242,41 @@ const VocabularyBlock = z
|
|
|
225
242
|
});
|
|
226
243
|
})
|
|
227
244
|
.describe('Every sentence saying a member of your vocabulary is accepted or refused agrees with the two sets.');
|
|
245
|
+
/**
|
|
246
|
+
* The 64-character lowercase-hex shape a sha256 digest prints as, so a
|
|
247
|
+
* truncated or upper-cased paste is refused at configuration load rather than
|
|
248
|
+
* comparing unequal to every subject forever.
|
|
249
|
+
*/
|
|
250
|
+
const Sha256Hex = z
|
|
251
|
+
.string()
|
|
252
|
+
.regex(/^[0-9a-f]{64}$/, 'is not a sha256 hex digest: 64 lowercase hex characters');
|
|
253
|
+
const DatedClaimAsOf = z
|
|
254
|
+
.strictObject({
|
|
255
|
+
subject: RelativePath.optional().describe("Which file the hash pins the claim to, when the judgment is about a file other than the one carrying the sentence. Defaults to the claim's own `file`. Has to be in this tree: a fact about another repository or a live system cannot be pinned, and naming the page that merely states such a fact is worse than naming nothing, since it reads as protected when it is not."),
|
|
256
|
+
hash: Sha256Hex.describe("The subject's normalized-content sha256, taken the moment a human confirmed this claim true against it."),
|
|
257
|
+
})
|
|
258
|
+
.describe('Pins a `read` claim to the content a human read it against, so an edit to that content fails the gate instead of a stale confirmation passing forever. A claim resting on more than one subject wants a predicate over the set, not several pins under one key.');
|
|
259
|
+
const DatedClaimEntry = z
|
|
260
|
+
.strictObject({
|
|
261
|
+
file: RelativePath,
|
|
262
|
+
key: NonEmpty.describe('A distinctive stretch of the sentence, matched literally. It names one sentence: a key short enough to match two lets a new and false claim ride in on an existing registration.'),
|
|
263
|
+
settles: z
|
|
264
|
+
.union([z.literal('read'), ModuleValue])
|
|
265
|
+
.describe('How the claim is settled. A predicate is run and a false answer fails the gate. "read" records that no artifact decides it.'),
|
|
266
|
+
reason: NonEmpty.describe('What the predicate reads, or why nothing in the tree can decide it.'),
|
|
267
|
+
asOf: DatedClaimAsOf.optional(),
|
|
268
|
+
})
|
|
269
|
+
.superRefine((entry, ctx) => {
|
|
270
|
+
// A predicate already re-runs every check; a content pin beside it would be
|
|
271
|
+
// a second staleness rule racing the first, and the two can disagree.
|
|
272
|
+
if (entry.asOf !== undefined && entry.settles !== 'read') {
|
|
273
|
+
ctx.addIssue({
|
|
274
|
+
code: 'custom',
|
|
275
|
+
path: ['asOf'],
|
|
276
|
+
message: 'is set, and settles is a predicate rather than "read"; a predicate is re-checked every run, so pinning a content hash beside it is redundant at best and contradictory at worst',
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
});
|
|
228
280
|
const DatedBlock = z
|
|
229
281
|
.strictObject({
|
|
230
282
|
triggers: z
|
|
@@ -233,15 +285,26 @@ const DatedBlock = z
|
|
|
233
285
|
.describe('The shapes a claim takes when its truth depends on when it was written. A sentence matching one has to be registered below.'),
|
|
234
286
|
headings: ProsePattern.optional().describe('A heading that says its section is about what has not happened, so every bullet under one is dated whatever words it uses.'),
|
|
235
287
|
claims: z
|
|
236
|
-
.array(
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
.
|
|
288
|
+
.array(DatedClaimEntry)
|
|
289
|
+
.min(1)
|
|
290
|
+
.superRefine((claims, ctx) => {
|
|
291
|
+
// One entry per sentence. Two entries sharing a `file` and `key` both
|
|
292
|
+
// pass the "seen" check below, since it is keyed on the same two
|
|
293
|
+
// values, so a duplicate is not caught there; left unrefused, it
|
|
294
|
+
// double-counts one sentence as two registered claims.
|
|
295
|
+
const seen = new Set();
|
|
296
|
+
claims.forEach((claim, index) => {
|
|
297
|
+
const key = compositeKey(claim.file, claim.key);
|
|
298
|
+
if (seen.has(key)) {
|
|
299
|
+
ctx.addIssue({
|
|
300
|
+
code: 'custom',
|
|
301
|
+
path: [index, 'key'],
|
|
302
|
+
message: `repeats the file and key of an earlier entry (${claim.file}: "${claim.key}"), and a dated claim is registered once; give the sentence one entry`,
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
seen.add(key);
|
|
306
|
+
});
|
|
307
|
+
}),
|
|
245
308
|
})
|
|
246
309
|
.describe('Every sentence whose truth depends on when it was written is registered with how it is settled.');
|
|
247
310
|
const TranscriptionEntry = z.strictObject({
|
|
@@ -365,6 +428,148 @@ const sentenceAround = (line, offset) => {
|
|
|
365
428
|
const end = stop === -1 ? line.length : offset + stop + 1;
|
|
366
429
|
return line.slice(start, end);
|
|
367
430
|
};
|
|
431
|
+
/**
|
|
432
|
+
* A fenced code block's opening delimiter: three or more backticks or tildes,
|
|
433
|
+
* CommonMark's own two fence characters. Captured so the close can require the
|
|
434
|
+
* same character and at least as many repeats, the way CommonMark itself does:
|
|
435
|
+
* a shorter or differently-charactered run inside the fence is content, not a
|
|
436
|
+
* close.
|
|
437
|
+
*/
|
|
438
|
+
const FENCE_OPEN = /^(`{3,}|~{3,})/;
|
|
439
|
+
const closesFence = (trimmedStart, marker) => new RegExp(`^${marker[0] === '`' ? '`' : '~'}{${marker.length},}\\s*$`).test(trimmedStart);
|
|
440
|
+
/**
|
|
441
|
+
* What an `asOf` hash is taken over. Outside a fenced code block, each line's
|
|
442
|
+
* leading indentation is read as a nesting depth rather than kept byte-exact,
|
|
443
|
+
* and everything after it is trimmed of trailing whitespace and has its
|
|
444
|
+
* internal whitespace runs collapsed to one space; a run of blank lines
|
|
445
|
+
* collapses to one. Depth, not width, is what a nested list or an indented
|
|
446
|
+
* block actually carries: a formatter that reindents an existing structure
|
|
447
|
+
* two spaces to four, say, does not change what the page says and must not
|
|
448
|
+
* trip the pin, but un-nesting a list item, or nesting a new one, is a real
|
|
449
|
+
* structural change and has to. Comparing raw indentation width cannot tell
|
|
450
|
+
* these apart, so depth is tracked with a stack the way an indentation-block
|
|
451
|
+
* language is: a deeper indent than the current top pushes a new level, a
|
|
452
|
+
* shallower one pops back to it, and the line is rewritten with a canonical
|
|
453
|
+
* two-space unit per level of depth rather than its own original width. A
|
|
454
|
+
* fenced block is left byte-exact (line-ending normalized), because
|
|
455
|
+
* indentation inside one is meaning a formatter is not free to move, and
|
|
456
|
+
* either collapsing or renormalizing it would let a broken code sample hide
|
|
457
|
+
* behind a passing gate. A stray shorter or wrongly-charactered fence-like
|
|
458
|
+
* line inside an open fence does not close it, so a nested example fence
|
|
459
|
+
* stays part of the outer block's protected content.
|
|
460
|
+
*
|
|
461
|
+
* What this does not do runs in both directions. Normalization only touches
|
|
462
|
+
* whitespace, so two shapes of real change stay invisible: a hard line
|
|
463
|
+
* break's trailing two spaces can be removed, and content outside a fence
|
|
464
|
+
* that is reindented without crossing a depth boundary, such as a non-fenced
|
|
465
|
+
* indented code sample's own internal width, changes without changing the
|
|
466
|
+
* hash either, since depth tracking only sees where a line sits relative to
|
|
467
|
+
* its neighbors and not what its own further indentation means. Every other
|
|
468
|
+
* markdown-syntax change is a non-whitespace byte and always shows, which is
|
|
469
|
+
* why catching either of those needs real markdown parsing and nothing else
|
|
470
|
+
* here does. In the other direction, a table's delimiter-row padding
|
|
471
|
+
* (`|---|---|` vs `| --- | --- |`) or a prose line rewrapped to a different
|
|
472
|
+
* width changes the hash even though nothing about what the page says
|
|
473
|
+
* changed, so a formatter doing either still trips the pin it was meant to
|
|
474
|
+
* spare. A consumer whose formatter rewraps prose can avoid that one with
|
|
475
|
+
* its own `proseWrap: "preserve"` setting; this gate has no equivalent knob.
|
|
476
|
+
*/
|
|
477
|
+
const normalizeForHash = (text) => {
|
|
478
|
+
const lines = [];
|
|
479
|
+
let fenced = false;
|
|
480
|
+
let fenceMarker = '';
|
|
481
|
+
let blank = false;
|
|
482
|
+
// A stack of indentation widths seen on the path to the current line, the
|
|
483
|
+
// same technique an indentation-block language's own lexer uses: its
|
|
484
|
+
// length, not the raw column number on top of it, is the depth a
|
|
485
|
+
// formatter cannot change just by picking a different unit width.
|
|
486
|
+
const indentStack = [0];
|
|
487
|
+
for (const withCr of text.split('\n')) {
|
|
488
|
+
const raw = withCr.endsWith('\r') ? withCr.slice(0, -1) : withCr;
|
|
489
|
+
const trimmedStart = raw.trimStart();
|
|
490
|
+
if (fenced) {
|
|
491
|
+
if (closesFence(trimmedStart, fenceMarker)) {
|
|
492
|
+
fenced = false;
|
|
493
|
+
lines.push(trimmedStart.trimEnd());
|
|
494
|
+
blank = false;
|
|
495
|
+
continue;
|
|
496
|
+
}
|
|
497
|
+
lines.push(raw);
|
|
498
|
+
blank = false;
|
|
499
|
+
continue;
|
|
500
|
+
}
|
|
501
|
+
const opened = FENCE_OPEN.exec(trimmedStart);
|
|
502
|
+
if (opened !== null) {
|
|
503
|
+
fenced = true;
|
|
504
|
+
fenceMarker = opened[1];
|
|
505
|
+
lines.push(trimmedStart.trimEnd());
|
|
506
|
+
blank = false;
|
|
507
|
+
continue;
|
|
508
|
+
}
|
|
509
|
+
const content = trimmedStart.replace(/\s+$/, '').replace(/[ \t]+/g, ' ');
|
|
510
|
+
if (content === '') {
|
|
511
|
+
if (blank)
|
|
512
|
+
continue;
|
|
513
|
+
blank = true;
|
|
514
|
+
lines.push('');
|
|
515
|
+
continue;
|
|
516
|
+
}
|
|
517
|
+
blank = false;
|
|
518
|
+
const indent = raw.length - trimmedStart.length;
|
|
519
|
+
while (indentStack.length > 1 &&
|
|
520
|
+
indent < indentStack[indentStack.length - 1]) {
|
|
521
|
+
indentStack.pop();
|
|
522
|
+
}
|
|
523
|
+
if (indent > indentStack[indentStack.length - 1]) {
|
|
524
|
+
indentStack.push(indent);
|
|
525
|
+
}
|
|
526
|
+
const depth = indentStack.length - 1;
|
|
527
|
+
lines.push(`${' '.repeat(depth)}${content}`);
|
|
528
|
+
}
|
|
529
|
+
// A document that ends without closing its last fence is malformed, and the
|
|
530
|
+
// trailing bytes inside that open fence are exactly the content the fenced
|
|
531
|
+
// branch above promises to keep byte-exact; trimming into them here would
|
|
532
|
+
// break that promise for the one shape that never legitimately arises in a
|
|
533
|
+
// well-formed page.
|
|
534
|
+
if (!fenced) {
|
|
535
|
+
while (lines.length > 0 && lines[0] === '')
|
|
536
|
+
lines.shift();
|
|
537
|
+
while (lines.length > 0 && lines[lines.length - 1] === '')
|
|
538
|
+
lines.pop();
|
|
539
|
+
}
|
|
540
|
+
return lines.join('\n');
|
|
541
|
+
};
|
|
542
|
+
/** What `dated.claims[].asOf.hash` holds: exported so a human confirming a claim can compute it. */
|
|
543
|
+
export const hashOfSubject = (text) => createHash('sha256').update(normalizeForHash(text)).digest('hex');
|
|
544
|
+
/**
|
|
545
|
+
* An `asOf.subject`'s content, read the way every other path this gate reads
|
|
546
|
+
* is read: a symbolic link is refused rather than followed, the same rule
|
|
547
|
+
* `walkPages` applies to a page and for the same reason, so a pin cannot be
|
|
548
|
+
* moved to point outside the tree the configuration names. A subject that is
|
|
549
|
+
* also a walked page is read from `pageText` rather than the disk a second
|
|
550
|
+
* time, which is both cheaper and, for that case, already covered by
|
|
551
|
+
* `walkPages`'s own symlink refusal.
|
|
552
|
+
*/
|
|
553
|
+
const readSubject = async (root, subject, pageText) => {
|
|
554
|
+
const cached = pageText.get(subject);
|
|
555
|
+
if (cached !== undefined)
|
|
556
|
+
return { kind: 'ok', body: cached.join('\n') };
|
|
557
|
+
const target = resolve(root, subject);
|
|
558
|
+
const info = await lstat(target).catch(() => null);
|
|
559
|
+
if (info === null)
|
|
560
|
+
return { kind: 'error', code: 'ENOENT' };
|
|
561
|
+
if (info.isSymbolicLink())
|
|
562
|
+
return { kind: 'symlink' };
|
|
563
|
+
try {
|
|
564
|
+
return { kind: 'ok', body: await readFile(target, 'utf8') };
|
|
565
|
+
}
|
|
566
|
+
catch (error) {
|
|
567
|
+
return {
|
|
568
|
+
kind: 'error',
|
|
569
|
+
code: error.code ?? 'EUNKNOWN',
|
|
570
|
+
};
|
|
571
|
+
}
|
|
572
|
+
};
|
|
368
573
|
/**
|
|
369
574
|
* Which backticked tokens in a captured stretch count as list members. A
|
|
370
575
|
* spelling rule covers a set whose members share a shape; a module export covers
|
|
@@ -480,7 +685,21 @@ export async function runDocClaims(root, section) {
|
|
|
480
685
|
// this the short form is ambiguous and the check would refuse a citation
|
|
481
686
|
// a reader resolves without effort.
|
|
482
687
|
const qualified = new Map();
|
|
688
|
+
// A fenced block is output rather than prose, and a diagnostic a page
|
|
689
|
+
// transcribes carries the file and line of whatever tree the command
|
|
690
|
+
// ran over, which is commonly a fixture the page itself created. Read
|
|
691
|
+
// as a citation that path resolves to nothing here and the class fails
|
|
692
|
+
// on a page that is correct. The invocation check already holds a
|
|
693
|
+
// transcribed block against the bytes the run really wrote, so the
|
|
694
|
+
// claim is held either way.
|
|
695
|
+
let fenced = false;
|
|
483
696
|
lines.forEach((line, index) => {
|
|
697
|
+
if (line.trimStart().startsWith('```')) {
|
|
698
|
+
fenced = !fenced;
|
|
699
|
+
return;
|
|
700
|
+
}
|
|
701
|
+
if (fenced)
|
|
702
|
+
return;
|
|
484
703
|
for (const match of line.matchAll(citation)) {
|
|
485
704
|
const cited = match[1];
|
|
486
705
|
const first = Number(match[2]);
|
|
@@ -655,6 +874,7 @@ export async function runDocClaims(root, section) {
|
|
|
655
874
|
const seen = new Set();
|
|
656
875
|
let read = 0;
|
|
657
876
|
let derived = 0;
|
|
877
|
+
let pinned = 0;
|
|
658
878
|
for (const page of authoredPages) {
|
|
659
879
|
const lines = pageText.get(page);
|
|
660
880
|
let owedSection = false;
|
|
@@ -701,6 +921,29 @@ export async function runDocClaims(root, section) {
|
|
|
701
921
|
}
|
|
702
922
|
if (entry.settles === 'read') {
|
|
703
923
|
read += 1;
|
|
924
|
+
if (entry.asOf !== undefined) {
|
|
925
|
+
pinned += 1;
|
|
926
|
+
const subject = entry.asOf.subject ?? entry.file;
|
|
927
|
+
const read = await readSubject(root, subject, pageText);
|
|
928
|
+
if (read.kind === 'symlink') {
|
|
929
|
+
fail(`${entry.file}: dated.claims holds "${entry.key}" with asOf.subject "${subject}", ` +
|
|
930
|
+
'which is a symbolic link; name the file it resolves to, so the pin cannot be ' +
|
|
931
|
+
'moved to point outside the tree the configuration names');
|
|
932
|
+
}
|
|
933
|
+
else if (read.kind === 'error') {
|
|
934
|
+
fail(`${entry.file}: dated.claims holds "${entry.key}" with asOf.subject "${subject}", ` +
|
|
935
|
+
`which could not be read (${read.code})`);
|
|
936
|
+
}
|
|
937
|
+
else {
|
|
938
|
+
const computed = hashOfSubject(read.body);
|
|
939
|
+
if (computed !== entry.asOf.hash) {
|
|
940
|
+
fail(`${entry.file}: "${entry.key}" was last confirmed against ${subject} at a ` +
|
|
941
|
+
`different content hash (stored ${entry.asOf.hash}, computed ${computed}); ` +
|
|
942
|
+
`run \`node scripts/hash-doc-claim-subject.ts ${subject}\` and update asOf.hash, ` +
|
|
943
|
+
'or fix/remove the entry');
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
}
|
|
704
947
|
continue;
|
|
705
948
|
}
|
|
706
949
|
derived += 1;
|
|
@@ -709,7 +952,7 @@ export async function runDocClaims(root, section) {
|
|
|
709
952
|
fail(`${entry.file}: "${entry.key}" is no longer true; the check that settles it ` +
|
|
710
953
|
`(${entry.reason}) now answers no`);
|
|
711
954
|
}
|
|
712
|
-
parts.push(`${derived + read} time-sensitive claims registered (${derived} settled by a predicate, ${read} by review)`);
|
|
955
|
+
parts.push(`${derived + read} time-sensitive claims registered (${derived} settled by a predicate, ${read} by review, ${pinned} pinned to a content hash)`);
|
|
713
956
|
}
|
|
714
957
|
// -----------------------------------------------------------------------
|
|
715
958
|
// Class 5: named codes
|
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
// A published gate: every fenced command-line invocation in the pages a
|
|
2
|
-
// consumer names is run against the binary
|
|
3
|
-
// compared with what the page claims.
|
|
2
|
+
// consumer names is run against the binary whose spelling opens the line, and
|
|
3
|
+
// the exit code is compared with what the page claims.
|
|
4
|
+
//
|
|
5
|
+
// A section names one binary, or several. A package that publishes two
|
|
6
|
+
// commands documents both, and each one carries its own built entry, its own
|
|
7
|
+
// spellings and its own installed-path prefix. Every spelling is matched
|
|
8
|
+
// against the same page and the longest one wins, because two published names
|
|
9
|
+
// commonly share a prefix and declaration order says nothing about which of
|
|
10
|
+
// them a line belongs to.
|
|
4
11
|
//
|
|
5
12
|
// The check exists because the documentation once described a product this
|
|
6
13
|
// repository does not contain. A usage exit is what a command line returns when
|
|
@@ -14,8 +21,12 @@
|
|
|
14
21
|
// reported no problems. So the exit code is judged too, wherever judging it
|
|
15
22
|
// means anything:
|
|
16
23
|
//
|
|
17
|
-
// * A
|
|
18
|
-
// about the command line alone, so a
|
|
24
|
+
// * A crash always fails, for every invocation, and so does a usage error the
|
|
25
|
+
// page did not declare. Both are about the command line alone, so a
|
|
26
|
+
// stand-in input cannot excuse them. A page that declares the usage exit
|
|
27
|
+
// for a faithful invocation is claiming that refusal on purpose, which is
|
|
28
|
+
// what a binary spending that code on a configuration it would not read
|
|
29
|
+
// needs.
|
|
19
30
|
// * An invocation is FAITHFUL when every input it names resolved to real
|
|
20
31
|
// bytes: a file the repository ships, a file the same page told the reader
|
|
21
32
|
// to create, or an artifact an earlier command on the page wrote. A
|
|
@@ -35,9 +46,11 @@
|
|
|
35
46
|
// The exit code alone is a weak claim, because it is shared. One code commonly
|
|
36
47
|
// covers a whole family of failures, so a page can name one failure while the
|
|
37
48
|
// binary reports another and the codes still agree. A page that declares its
|
|
38
|
-
// exit code may therefore transcribe the
|
|
39
|
-
// compared line for line against what the run wrote
|
|
40
|
-
//
|
|
49
|
+
// exit code may therefore transcribe the output beside it, and that block is
|
|
50
|
+
// compared line for line against what the run wrote: stderr when the run wrote
|
|
51
|
+
// any, and stdout otherwise, since a page documenting a command that worked is
|
|
52
|
+
// quoting the answer rather than a diagnostic. Four rules shape which block gets
|
|
53
|
+
// compared:
|
|
41
54
|
//
|
|
42
55
|
// * The block is a `text` fence separated from the command's fence by blank
|
|
43
56
|
// lines only. Prose between them detaches it, and a fence carrying any
|
|
@@ -311,11 +324,15 @@ function describesLine(documented, actual) {
|
|
|
311
324
|
* fence or the next unindented line.
|
|
312
325
|
*
|
|
313
326
|
* A `text` fence separated from a declared-exit invocation by nothing but blank
|
|
314
|
-
* lines is that invocation's transcribed
|
|
315
|
-
*
|
|
316
|
-
*
|
|
317
|
-
* page that documents a failure prints it on stderr. The fence has to have
|
|
327
|
+
* lines is that invocation's transcribed output, and it travels on the run as
|
|
328
|
+
* `expectStderr`. Only a declared-exit invocation collects one, so a page opts
|
|
329
|
+
* in to the comparison by declaring what the run returns. The fence has to have
|
|
318
330
|
* pushed exactly one such invocation, since both would carry its declaration.
|
|
331
|
+
*
|
|
332
|
+
* `spellings` is every spelling across every declared binary, already sorted
|
|
333
|
+
* longest first, and each one carries the index of the binary it belongs to.
|
|
334
|
+
* That index travels on the run, so the caller knows which entry to execute and
|
|
335
|
+
* whose installed-path prefix to resolve the arguments against.
|
|
319
336
|
*/
|
|
320
337
|
function extractActions(file, source, spellings) {
|
|
321
338
|
const lines = source.split('\n');
|
|
@@ -421,10 +438,12 @@ function extractActions(file, source, spellings) {
|
|
|
421
438
|
index += 1;
|
|
422
439
|
text = `${text.slice(0, -1).trim()} ${lines[index].trim()}`;
|
|
423
440
|
}
|
|
424
|
-
const
|
|
425
|
-
|
|
441
|
+
const matched = spellings
|
|
442
|
+
.map((spelling) => ({ spelling, match: text.match(spelling.pattern) }))
|
|
443
|
+
.find((candidate) => candidate.match !== null);
|
|
444
|
+
if (matched === undefined)
|
|
426
445
|
continue;
|
|
427
|
-
const tail = (match[1] ?? '').trim();
|
|
446
|
+
const tail = (matched.match[1] ?? '').trim();
|
|
428
447
|
if (tail === '')
|
|
429
448
|
continue;
|
|
430
449
|
const first = tokenize(tail)[0];
|
|
@@ -436,6 +455,7 @@ function extractActions(file, source, spellings) {
|
|
|
436
455
|
line: startLine,
|
|
437
456
|
invocation: text,
|
|
438
457
|
tail,
|
|
458
|
+
binary: matched.spelling.binary,
|
|
439
459
|
expectExit,
|
|
440
460
|
expectStderr: null,
|
|
441
461
|
};
|
|
@@ -453,12 +473,24 @@ function extractActions(file, source, spellings) {
|
|
|
453
473
|
* section names resolves against it.
|
|
454
474
|
*/
|
|
455
475
|
export function runDocInvocations(root, section) {
|
|
456
|
-
|
|
476
|
+
// One binary is the ordinary case and stays spelled as one object. The list
|
|
477
|
+
// is built once here, so everything below reads the same shape.
|
|
478
|
+
const declared = Array.isArray(section.binary)
|
|
479
|
+
? section.binary
|
|
480
|
+
: [section.binary];
|
|
481
|
+
const binaries = declared.map((binary) => ({
|
|
482
|
+
entry: resolve(root, binary.entry),
|
|
483
|
+
installedPrefix: binary.installedPrefix,
|
|
484
|
+
}));
|
|
457
485
|
// A build is a precondition rather than an excuse. Skipping here would let
|
|
458
486
|
// the gate exit 0 having executed nothing, which is the vacuous pass the
|
|
459
|
-
// whole check exists to prevent.
|
|
460
|
-
|
|
461
|
-
|
|
487
|
+
// whole check exists to prevent. The refusal names which binary is missing,
|
|
488
|
+
// since a reader with two of them has two build steps to choose between.
|
|
489
|
+
for (const [index, binary] of binaries.entries()) {
|
|
490
|
+
if (existsSync(binary.entry))
|
|
491
|
+
continue;
|
|
492
|
+
const field = declared.length === 1 ? 'binary.entry' : `binary[${index}].entry`;
|
|
493
|
+
throw codedError(DOC_PATH_ERROR, `${binary.entry} does not exist; the "doc-invocations" section names it under ${field}, so build it before the gate runs`);
|
|
462
494
|
}
|
|
463
495
|
const sampleInput = resolve(root, section.sampleInput);
|
|
464
496
|
if (!existsSync(sampleInput)) {
|
|
@@ -486,12 +518,26 @@ export function runDocInvocations(root, section) {
|
|
|
486
518
|
throw codedError(DOC_PATH_ERROR, `the "doc-invocations" section names "${page}" under pages, and that encloses the directory the configuration sits in; name a page or a directory inside it, since every fenced command under a whole repository is more than this gate should run`);
|
|
487
519
|
}
|
|
488
520
|
}
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
521
|
+
// Longest first, and the first match wins. Two published names commonly
|
|
522
|
+
// share a prefix, so matching in declaration order would let a short
|
|
523
|
+
// spelling belonging to one binary claim a line that opens with a longer
|
|
524
|
+
// spelling belonging to another, and the line would then run against the
|
|
525
|
+
// wrong entry and be judged against the wrong installed-path prefix.
|
|
526
|
+
const spellings = declared
|
|
527
|
+
.flatMap((binary, index) => binary.spellings.map((spelling) => ({
|
|
528
|
+
text: spelling.trim(),
|
|
529
|
+
pattern: spellingPattern(spelling),
|
|
530
|
+
binary: index,
|
|
531
|
+
})))
|
|
532
|
+
.sort((a, b) => b.text.length - a.text.length);
|
|
533
|
+
const repoRoot = resolve(root);
|
|
534
|
+
// One context per binary, because `installedPrefix` belongs to the binary a
|
|
535
|
+
// line matched. Two binaries on one page can map different installed paths.
|
|
536
|
+
const contexts = binaries.map((binary) => ({
|
|
537
|
+
repoRoot,
|
|
492
538
|
sampleInput,
|
|
493
|
-
installedPrefix:
|
|
494
|
-
};
|
|
539
|
+
installedPrefix: binary.installedPrefix,
|
|
540
|
+
}));
|
|
495
541
|
const files = section.pages
|
|
496
542
|
.flatMap((page) => collectMarkdown(resolve(root, page)))
|
|
497
543
|
.sort();
|
|
@@ -506,8 +552,8 @@ export function runDocInvocations(root, section) {
|
|
|
506
552
|
let compared = 0;
|
|
507
553
|
try {
|
|
508
554
|
for (const [index, absolute] of files.entries()) {
|
|
509
|
-
const file = absolute.startsWith(
|
|
510
|
-
? absolute.slice(
|
|
555
|
+
const file = absolute.startsWith(repoRoot)
|
|
556
|
+
? absolute.slice(repoRoot.length + 1)
|
|
511
557
|
: absolute;
|
|
512
558
|
const sandbox = createPageSandbox(workDir, index);
|
|
513
559
|
for (const action of extractActions(file, readFileSync(absolute, 'utf8'), spellings)) {
|
|
@@ -520,11 +566,11 @@ export function runDocInvocations(root, section) {
|
|
|
520
566
|
continue;
|
|
521
567
|
}
|
|
522
568
|
scanned += 1;
|
|
523
|
-
const { tokens, faithful } = realizeArguments(action.tail, sandbox,
|
|
569
|
+
const { tokens, faithful } = realizeArguments(action.tail, sandbox, contexts[action.binary]);
|
|
524
570
|
// The sandbox root is the working directory and stdin is closed: a
|
|
525
571
|
// relative write lands inside the sandbox, and a command that reads
|
|
526
572
|
// stdin sees an empty stream and returns at once.
|
|
527
|
-
const result = spawnSync(process.execPath, [entry, ...tokens], {
|
|
573
|
+
const result = spawnSync(process.execPath, [binaries[action.binary].entry, ...tokens], {
|
|
528
574
|
cwd: sandbox.root,
|
|
529
575
|
encoding: 'utf8',
|
|
530
576
|
input: '',
|
|
@@ -556,7 +602,14 @@ export function runDocInvocations(root, section) {
|
|
|
556
602
|
record('the binary crashed');
|
|
557
603
|
continue;
|
|
558
604
|
}
|
|
559
|
-
|
|
605
|
+
// A usage exit is a mistyped command or a flag that stopped existing,
|
|
606
|
+
// except where the page declares it. One binary can spend the same
|
|
607
|
+
// code on a configuration it refused to read, and a page teaching a
|
|
608
|
+
// reader to recognise that refusal is making a claim about it like
|
|
609
|
+
// any other. The declaration is what separates the two, so an
|
|
610
|
+
// undeclared usage exit still fails every invocation.
|
|
611
|
+
if (result.status === section.usageExit &&
|
|
612
|
+
action.expectExit !== section.usageExit) {
|
|
560
613
|
record('usage error: the documented command or flag does not exist');
|
|
561
614
|
continue;
|
|
562
615
|
}
|
|
@@ -580,7 +633,13 @@ export function runDocInvocations(root, section) {
|
|
|
580
633
|
if (action.expectStderr === null)
|
|
581
634
|
continue;
|
|
582
635
|
compared += 1;
|
|
583
|
-
|
|
636
|
+
// A page documenting a rejection quotes stderr, and a page
|
|
637
|
+
// documenting a command that worked quotes stdout. A run that wrote
|
|
638
|
+
// nothing to stderr is the second case, so the block is compared
|
|
639
|
+
// against what the run actually said rather than against an empty
|
|
640
|
+
// stream. Declaring the exit code is still what attaches a block at
|
|
641
|
+
// all, so no page acquires a comparison it did not ask for.
|
|
642
|
+
const written = (result.stderr.trim() === '' ? result.stdout : result.stderr).split('\n');
|
|
584
643
|
const documented = action.expectStderr;
|
|
585
644
|
const overElided = documented.findIndex((line) => line.split(ELISION).length - 1 > section.elisionLimit);
|
|
586
645
|
if (overElided !== -1) {
|
|
@@ -612,7 +671,7 @@ export function runDocInvocations(root, section) {
|
|
|
612
671
|
// no commands at all. The pages are there and the binary is there, so the one
|
|
613
672
|
// thing left to name is the spelling list.
|
|
614
673
|
if (scanned === 0) {
|
|
615
|
-
throw codedError(DOC_PATH_ERROR, `no fenced command in ${files.length} page(s) matched any spelling the "doc-invocations" section declares (${
|
|
674
|
+
throw codedError(DOC_PATH_ERROR, `no fenced command in ${files.length} page(s) matched any spelling the "doc-invocations" section declares (${spellings.map((spelling) => spelling.text).join(', ')}); a gate that extracted nothing reports a pass over nothing`);
|
|
616
675
|
}
|
|
617
676
|
return { failures, scanned, judged, compared, pages: files.length };
|
|
618
677
|
}
|
|
@@ -473,6 +473,20 @@ function scanFile(file, source, files, graph, violations) {
|
|
|
473
473
|
const openIndex = tokens[i + 1]?.kind === SyntaxKind.QuestionDotToken ? i + 2 : i + 1;
|
|
474
474
|
if (tokens[openIndex]?.kind !== SyntaxKind.OpenParenToken)
|
|
475
475
|
continue;
|
|
476
|
+
// The site this arm exists for is the free identifier `require` called
|
|
477
|
+
// with a specifier. Two shapes share its tokens and are ordinary
|
|
478
|
+
// JavaScript: a member call, `sandbox.require('fs')`, where the token
|
|
479
|
+
// before is `.` or `?.`, and a method named require, `require(name) {`
|
|
480
|
+
// in an object literal or a class, where the token after the matching
|
|
481
|
+
// `)` is `{`. A call's own `)` is never followed by `{`, so
|
|
482
|
+
// `if (require('x')) {` still reads as the call it is. A consumer
|
|
483
|
+
// renaming a mock to get past this arm is a gate teaching the wrong lesson.
|
|
484
|
+
const before = tokens[i - 1]?.kind;
|
|
485
|
+
if (before === SyntaxKind.DotToken ||
|
|
486
|
+
before === SyntaxKind.QuestionDotToken ||
|
|
487
|
+
isMethodDefinition(tokens, i, openIndex)) {
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
476
490
|
if (graph.commonjs === 'forbid') {
|
|
477
491
|
violations.push({
|
|
478
492
|
file,
|
|
@@ -545,6 +559,122 @@ function scanFile(file, source, files, graph, violations) {
|
|
|
545
559
|
* against `graph` and returns every violation found, in no particular cross-file
|
|
546
560
|
* order.
|
|
547
561
|
*/
|
|
562
|
+
/**
|
|
563
|
+
* A modifier or generator marker that can sit between a declaration boundary
|
|
564
|
+
* and the member name it modifies: `async`, `static`, `get`, `set`,
|
|
565
|
+
* `readonly`, an access modifier, or `*`. Skipped when walking backward from
|
|
566
|
+
* `require` to find what actually introduces it.
|
|
567
|
+
*/
|
|
568
|
+
const MEMBER_MODIFIERS = new Set([
|
|
569
|
+
SyntaxKind.AsyncKeyword,
|
|
570
|
+
SyntaxKind.StaticKeyword,
|
|
571
|
+
SyntaxKind.GetKeyword,
|
|
572
|
+
SyntaxKind.SetKeyword,
|
|
573
|
+
SyntaxKind.ReadonlyKeyword,
|
|
574
|
+
SyntaxKind.PrivateKeyword,
|
|
575
|
+
SyntaxKind.PublicKeyword,
|
|
576
|
+
SyntaxKind.ProtectedKeyword,
|
|
577
|
+
SyntaxKind.AsteriskToken,
|
|
578
|
+
]);
|
|
579
|
+
/**
|
|
580
|
+
* Whether `require`, immediately followed by a parameter list and then `{`,
|
|
581
|
+
* actually sits where a name is declared rather than where a call's result is
|
|
582
|
+
* followed by an unrelated block statement: `const mod = require('x')` and a
|
|
583
|
+
* stray `{` on the next line tokenize exactly like a method body, and only
|
|
584
|
+
* what precedes `require` tells them apart. Skips the modifiers above, then
|
|
585
|
+
* requires the next token to be the opening brace of the object, class or
|
|
586
|
+
* interface `require` is the first member of, a `,` or `;` separating it from
|
|
587
|
+
* a prior member, a `case`/`default` label, `function` for a function
|
|
588
|
+
* declaration, or the start of the file. Anything else -- `=`, `return`, or
|
|
589
|
+
* any other token an expression puts before a call -- means this is a call.
|
|
590
|
+
*
|
|
591
|
+
* A `;` or `}` immediately before `require` stays undecidable this way: both
|
|
592
|
+
* a prior class member and a prior unrelated statement end on one, and
|
|
593
|
+
* telling them apart needs knowing what kind of block `require` sits in,
|
|
594
|
+
* which a token stream does not carry. Rare enough, and specific enough to
|
|
595
|
+
* write on purpose, that it is left as the one shape this arm still misses.
|
|
596
|
+
*/
|
|
597
|
+
function isDeclarationPosition(tokens, requireIndex) {
|
|
598
|
+
let j = requireIndex - 1;
|
|
599
|
+
while (j >= 0 && MEMBER_MODIFIERS.has(tokens[j]?.kind))
|
|
600
|
+
j -= 1;
|
|
601
|
+
if (j < 0)
|
|
602
|
+
return true;
|
|
603
|
+
const kind = tokens[j]?.kind;
|
|
604
|
+
return (kind === SyntaxKind.OpenBraceToken ||
|
|
605
|
+
kind === SyntaxKind.CommaToken ||
|
|
606
|
+
kind === SyntaxKind.SemicolonToken ||
|
|
607
|
+
kind === SyntaxKind.CaseKeyword ||
|
|
608
|
+
kind === SyntaxKind.DefaultKeyword ||
|
|
609
|
+
kind === SyntaxKind.FunctionKeyword);
|
|
610
|
+
}
|
|
611
|
+
/**
|
|
612
|
+
* Whether the parenthesised list opening at `openIndex` is a parameter list,
|
|
613
|
+
* which is to say `require` here is a method or a signature and never a call.
|
|
614
|
+
* A `{` straight after the matching `)` defers to `isDeclarationPosition`. A
|
|
615
|
+
* `:` after it is either a return-type annotation, a ternary's else, or a
|
|
616
|
+
* `case`/`default` label, and the three are told apart by looking back from
|
|
617
|
+
* `require` at bracket depth zero: a ternary has its `?` before the call and
|
|
618
|
+
* inside the same expression, a `case`/`default` label has the keyword
|
|
619
|
+
* immediately before the call and nothing between them, and a member
|
|
620
|
+
* declaration has none of those before its own `{`, `,` or `;` -- or before
|
|
621
|
+
* reaching an enclosing `(`, `[` or `{` with nothing still open inside it,
|
|
622
|
+
* which is the same boundary one level up. An unclosed list reads as a call,
|
|
623
|
+
* which is what this arm already did with a stream it could not place.
|
|
624
|
+
*/
|
|
625
|
+
function isMethodDefinition(tokens, requireIndex, openIndex) {
|
|
626
|
+
let depth = 0;
|
|
627
|
+
let closeIndex = -1;
|
|
628
|
+
for (let j = openIndex; j < tokens.length; j++) {
|
|
629
|
+
const kind = tokens[j]?.kind;
|
|
630
|
+
if (kind === SyntaxKind.OpenParenToken)
|
|
631
|
+
depth += 1;
|
|
632
|
+
else if (kind === SyntaxKind.CloseParenToken) {
|
|
633
|
+
depth -= 1;
|
|
634
|
+
if (depth === 0) {
|
|
635
|
+
closeIndex = j;
|
|
636
|
+
break;
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
if (closeIndex === -1)
|
|
641
|
+
return false;
|
|
642
|
+
const after = tokens[closeIndex + 1]?.kind;
|
|
643
|
+
if (after === SyntaxKind.OpenBraceToken) {
|
|
644
|
+
return isDeclarationPosition(tokens, requireIndex);
|
|
645
|
+
}
|
|
646
|
+
if (after !== SyntaxKind.ColonToken)
|
|
647
|
+
return false;
|
|
648
|
+
depth = 0;
|
|
649
|
+
for (let j = requireIndex - 1; j >= 0; j--) {
|
|
650
|
+
const kind = tokens[j]?.kind;
|
|
651
|
+
if (kind === SyntaxKind.CloseParenToken ||
|
|
652
|
+
kind === SyntaxKind.CloseBracketToken ||
|
|
653
|
+
kind === SyntaxKind.CloseBraceToken) {
|
|
654
|
+
depth += 1;
|
|
655
|
+
continue;
|
|
656
|
+
}
|
|
657
|
+
if (kind === SyntaxKind.OpenParenToken ||
|
|
658
|
+
kind === SyntaxKind.OpenBracketToken ||
|
|
659
|
+
kind === SyntaxKind.OpenBraceToken) {
|
|
660
|
+
if (depth === 0)
|
|
661
|
+
return true;
|
|
662
|
+
depth -= 1;
|
|
663
|
+
continue;
|
|
664
|
+
}
|
|
665
|
+
if (depth > 0)
|
|
666
|
+
continue;
|
|
667
|
+
if (kind === SyntaxKind.QuestionToken)
|
|
668
|
+
return false;
|
|
669
|
+
if (kind === SyntaxKind.CaseKeyword || kind === SyntaxKind.DefaultKeyword) {
|
|
670
|
+
return false;
|
|
671
|
+
}
|
|
672
|
+
if (kind === SyntaxKind.SemicolonToken || kind === SyntaxKind.CommaToken) {
|
|
673
|
+
return true;
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
return true;
|
|
677
|
+
}
|
|
548
678
|
export function scanSources(files, graph) {
|
|
549
679
|
const fileSet = new Set(files.keys());
|
|
550
680
|
const violations = [];
|
|
@@ -279,6 +279,16 @@ const LicencesSection = z
|
|
|
279
279
|
* `node_modules`, which is what keeps the pre-install path open for the two
|
|
280
280
|
* gates that run before `npm ci`.
|
|
281
281
|
*/
|
|
282
|
+
const DocumentedBinary = z
|
|
283
|
+
.strictObject({
|
|
284
|
+
entry: RelativePath.describe('The built entry point every documented command is run against. It is a precondition: a gate that skipped when it was absent would exit 0 having executed nothing.'),
|
|
285
|
+
spellings: z
|
|
286
|
+
.array(NonEmpty)
|
|
287
|
+
.min(1)
|
|
288
|
+
.describe('How your documentation writes the command, as the literal text a reader types. Each is matched at the start of a fenced line, with whitespace or end of line after it, so a sample of your own diagnostic output is left alone.'),
|
|
289
|
+
installedPrefix: NonEmpty.optional().describe('The path prefix a page uses for a file inside the installed package, such as "node_modules/your-package/". Mapping it away is what lets those examples be checked against real bytes.'),
|
|
290
|
+
})
|
|
291
|
+
.describe('One binary a documented command line runs.');
|
|
282
292
|
const DocInvocationsSection = z
|
|
283
293
|
.strictObject({
|
|
284
294
|
pages: z
|
|
@@ -286,15 +296,8 @@ const DocInvocationsSection = z
|
|
|
286
296
|
.min(1)
|
|
287
297
|
.describe("The pages whose fenced commands are run, as files or directories to walk. Naming the configuration's own directory is refused: every fenced command in a whole repository is more than this gate should run."),
|
|
288
298
|
binary: z
|
|
289
|
-
.
|
|
290
|
-
|
|
291
|
-
spellings: z
|
|
292
|
-
.array(NonEmpty)
|
|
293
|
-
.min(1)
|
|
294
|
-
.describe('How your documentation writes the command, as the literal text a reader types. Each is matched at the start of a fenced line, with whitespace or end of line after it, so a sample of your own diagnostic output is left alone.'),
|
|
295
|
-
installedPrefix: NonEmpty.optional().describe('The path prefix a page uses for a file inside the installed package, such as "node_modules/your-package/". Mapping it away is what lets those examples be checked against real bytes.'),
|
|
296
|
-
})
|
|
297
|
-
.describe('What a documented command line runs.'),
|
|
299
|
+
.union([DocumentedBinary, z.array(DocumentedBinary).min(1)])
|
|
300
|
+
.describe("What a documented command line runs. One object for a package that publishes one binary, or an array for a package that publishes several, each carrying its own entry, spellings and installedPrefix. Every spelling across every binary is matched against the same page and the longest one wins, so a short spelling belonging to one binary never claims a line that opens with a longer spelling belonging to another. Each binary's entry is its own precondition and the refusal names which one is missing. An empty array is refused, since a gate with no spelling to match extracts nothing and reports a pass over nothing."),
|
|
298
301
|
sampleInput: RelativePath.describe('A file that stands in for an input only a reader has, such as `<path>`. A run that needed one is judged for usage errors and crashes and no more.'),
|
|
299
302
|
usageExit: z
|
|
300
303
|
.int()
|
|
@@ -32,39 +32,36 @@
|
|
|
32
32
|
// appear in this file or anything it imports.
|
|
33
33
|
import { z } from 'zod';
|
|
34
34
|
import { discoverEntries, RelativePath, RelativePrefix, ScannedPathList, } from './scanned-paths.js';
|
|
35
|
+
import { loadTypeScriptScanner, TYPESCRIPT_UNAVAILABLE, } from './typescript-scanner.js';
|
|
35
36
|
/** The gate needs `typescript` and could not resolve it. */
|
|
36
|
-
export
|
|
37
|
-
const
|
|
38
|
-
const importTokenScanner = async () => {
|
|
37
|
+
export { TYPESCRIPT_UNAVAILABLE };
|
|
38
|
+
const importTokenScanner = (gate) => async () => {
|
|
39
39
|
// `token-scan.ts` imports `typescript/unstable/ast` at its own top level, so
|
|
40
|
-
//
|
|
41
|
-
//
|
|
42
|
-
const
|
|
43
|
-
|
|
44
|
-
import('./token-scan.js'),
|
|
45
|
-
]);
|
|
40
|
+
// the loader runs first and its refusal is the one a consumer reads; the
|
|
41
|
+
// import of `token-scan.ts` follows only once the scanner is known to be there.
|
|
42
|
+
const ast = await loadTypeScriptScanner(gate);
|
|
43
|
+
const scan = await import('./token-scan.js');
|
|
46
44
|
return {
|
|
47
45
|
scanTokens: scan.scanTokens,
|
|
48
46
|
computeLineStarts: scan.computeLineStarts,
|
|
49
47
|
lineOf: scan.lineOf,
|
|
48
|
+
// `loadTypeScriptScanner` types `SyntaxKind` as a generic
|
|
49
|
+
// `Readonly<Record<string, number>>`, since its own shape check indexes
|
|
50
|
+
// it by whichever member name each of the three scanner modules reads.
|
|
51
|
+
// That check has already run and passed by the time this line executes,
|
|
52
|
+
// so every member this file reads off `Syntax` is verified present; the
|
|
53
|
+
// double cast is regaining the precise type the runtime check earned.
|
|
50
54
|
syntax: ast.SyntaxKind,
|
|
51
55
|
};
|
|
52
56
|
};
|
|
53
57
|
/**
|
|
54
|
-
* The tokenizer, or
|
|
55
|
-
* `load` is injectable so
|
|
56
|
-
* uninstalling anything
|
|
58
|
+
* The tokenizer, or the refusal `loadTypeScriptScanner` throws naming the
|
|
59
|
+
* dependency and the gate that needs it. `load` is injectable so a test can
|
|
60
|
+
* exercise that refusal without uninstalling anything the test runner itself
|
|
61
|
+
* needs.
|
|
57
62
|
*/
|
|
58
|
-
export async function loadTokenScanner(gate, load = importTokenScanner) {
|
|
59
|
-
|
|
60
|
-
return await load();
|
|
61
|
-
}
|
|
62
|
-
catch (error) {
|
|
63
|
-
if (error.code !== 'ERR_MODULE_NOT_FOUND') {
|
|
64
|
-
throw error;
|
|
65
|
-
}
|
|
66
|
-
throw codedError(TYPESCRIPT_UNAVAILABLE, `the ${gate} gate reads your source through the typescript package's own scanner, and typescript did not resolve. Install typescript to run this gate; every other gate needs nothing beyond this package.`);
|
|
67
|
-
}
|
|
63
|
+
export async function loadTokenScanner(gate, load = importTokenScanner(gate)) {
|
|
64
|
+
return load();
|
|
68
65
|
}
|
|
69
66
|
const Identifier = z
|
|
70
67
|
.string()
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
// The one place the two source-scanning gates reach TypeScript.
|
|
2
|
+
//
|
|
3
|
+
// The scanner lives at `typescript/unstable/ast`, a subpath whose own name says
|
|
4
|
+
// it may move, and it exists from TypeScript 7.0: the 5.x line has no such
|
|
5
|
+
// subpath and the 7.x main entry exports no scanner. The optional peer range
|
|
6
|
+
// says `>=5.7.0` and stays that wide on purpose. npm resolves an optional peer
|
|
7
|
+
// that is present, so a range of `>=7` would turn `npm install` red for every
|
|
8
|
+
// consumer with TypeScript 5 in its tree and no interest in these two gates.
|
|
9
|
+
// The version fact lives here instead, spoken at the one moment it matters:
|
|
10
|
+
// when a consumer runs one of the two gates.
|
|
11
|
+
//
|
|
12
|
+
// Three refusals, each its own because the repair is different. The package
|
|
13
|
+
// is absent; the package is a version with no such subpath; the subpath is
|
|
14
|
+
// there and lacks a member this build reads. The third is the quiet one: a
|
|
15
|
+
// renamed enum member reads as `undefined`, `token.kind === undefined` never
|
|
16
|
+
// matches, and the rule it guarded switches off with every gate green. So every
|
|
17
|
+
// `SyntaxKind` member the three scanner modules read is listed below, and
|
|
18
|
+
// `tests/architecture/typescript-scanner.test.ts` derives the same list from
|
|
19
|
+
// their sources, so the list is not a copy a hand maintains.
|
|
20
|
+
//
|
|
21
|
+
// Run by `node` directly: Node's type stripping erases types only, so no
|
|
22
|
+
// TypeScript enum, namespace, parameter property, or non-type re-export may
|
|
23
|
+
// appear in this file or anything it imports, or the gate fails at load.
|
|
24
|
+
/** `error.code` on every refusal here; the binary maps it to the usage exit. */
|
|
25
|
+
export const TYPESCRIPT_UNAVAILABLE = 'EVAL_QUALITY_TYPESCRIPT_UNAVAILABLE';
|
|
26
|
+
/** Where the scanner is read from, and the first TypeScript that ships it. */
|
|
27
|
+
export const SCANNER_SUBPATH = 'typescript/unstable/ast';
|
|
28
|
+
export const SCANNER_SHIPS_FROM = '7.0.0';
|
|
29
|
+
/** Every `SyntaxKind` member `token-scan.ts`, `dependency-direction.ts` and `lineage-ownership.ts` read. */
|
|
30
|
+
export const REQUIRED_SYNTAX_KINDS = [
|
|
31
|
+
'AmpersandToken',
|
|
32
|
+
'AnyKeyword',
|
|
33
|
+
'AsKeyword',
|
|
34
|
+
'AsteriskToken',
|
|
35
|
+
'AsyncKeyword',
|
|
36
|
+
'AwaitKeyword',
|
|
37
|
+
'BarToken',
|
|
38
|
+
'BigIntKeyword',
|
|
39
|
+
'BigIntLiteral',
|
|
40
|
+
'BooleanKeyword',
|
|
41
|
+
'CaseKeyword',
|
|
42
|
+
'CatchKeyword',
|
|
43
|
+
'ClassKeyword',
|
|
44
|
+
'CloseBraceToken',
|
|
45
|
+
'CloseBracketToken',
|
|
46
|
+
'CloseParenToken',
|
|
47
|
+
'ColonToken',
|
|
48
|
+
'CommaToken',
|
|
49
|
+
'ConstKeyword',
|
|
50
|
+
'DefaultKeyword',
|
|
51
|
+
'DotToken',
|
|
52
|
+
'EndOfFile',
|
|
53
|
+
'EqualsGreaterThanToken',
|
|
54
|
+
'EqualsToken',
|
|
55
|
+
'ExclamationToken',
|
|
56
|
+
'ExportKeyword',
|
|
57
|
+
'ExtendsKeyword',
|
|
58
|
+
'FalseKeyword',
|
|
59
|
+
'FirstAssignment',
|
|
60
|
+
'ForKeyword',
|
|
61
|
+
'FromKeyword',
|
|
62
|
+
'FunctionKeyword',
|
|
63
|
+
'GetKeyword',
|
|
64
|
+
'Identifier',
|
|
65
|
+
'IfKeyword',
|
|
66
|
+
'ImportKeyword',
|
|
67
|
+
'InterfaceKeyword',
|
|
68
|
+
'LastAssignment',
|
|
69
|
+
'LessThanToken',
|
|
70
|
+
'LetKeyword',
|
|
71
|
+
'MinusMinusToken',
|
|
72
|
+
'NeverKeyword',
|
|
73
|
+
'NewKeyword',
|
|
74
|
+
'NoSubstitutionTemplateLiteral',
|
|
75
|
+
'NullKeyword',
|
|
76
|
+
'NumberKeyword',
|
|
77
|
+
'NumericLiteral',
|
|
78
|
+
'ObjectKeyword',
|
|
79
|
+
'OpenBraceToken',
|
|
80
|
+
'OpenBracketToken',
|
|
81
|
+
'OpenParenToken',
|
|
82
|
+
'PlusPlusToken',
|
|
83
|
+
'PrivateKeyword',
|
|
84
|
+
'ProtectedKeyword',
|
|
85
|
+
'PublicKeyword',
|
|
86
|
+
'QuestionDotToken',
|
|
87
|
+
'QuestionToken',
|
|
88
|
+
'ReadonlyKeyword',
|
|
89
|
+
'RegularExpressionLiteral',
|
|
90
|
+
'RequireKeyword',
|
|
91
|
+
'ReturnKeyword',
|
|
92
|
+
'SemicolonToken',
|
|
93
|
+
'SetKeyword',
|
|
94
|
+
'SlashEqualsToken',
|
|
95
|
+
'SlashToken',
|
|
96
|
+
'StaticKeyword',
|
|
97
|
+
'StringKeyword',
|
|
98
|
+
'StringLiteral',
|
|
99
|
+
'SuperKeyword',
|
|
100
|
+
'SymbolKeyword',
|
|
101
|
+
'TemplateHead',
|
|
102
|
+
'TemplateTail',
|
|
103
|
+
'ThisKeyword',
|
|
104
|
+
'TrueKeyword',
|
|
105
|
+
'TypeKeyword',
|
|
106
|
+
'UndefinedKeyword',
|
|
107
|
+
'UnknownKeyword',
|
|
108
|
+
'VarKeyword',
|
|
109
|
+
'VoidKeyword',
|
|
110
|
+
'WhileKeyword',
|
|
111
|
+
'WithKeyword',
|
|
112
|
+
];
|
|
113
|
+
const codedError = (message) => Object.assign(new Error(message), { code: TYPESCRIPT_UNAVAILABLE });
|
|
114
|
+
const defaultLoad = () => import('typescript/unstable/ast');
|
|
115
|
+
// `typescript/package.json` is on the package's export map, so the version is
|
|
116
|
+
// read from the install itself. A read that fails names no version rather than
|
|
117
|
+
// failing the refusal that wanted it.
|
|
118
|
+
const defaultVersion = async () => {
|
|
119
|
+
try {
|
|
120
|
+
const manifest = (await import('typescript/package.json', {
|
|
121
|
+
with: { type: 'json' },
|
|
122
|
+
}));
|
|
123
|
+
const version = manifest.default?.version;
|
|
124
|
+
return typeof version === 'string' ? version : null;
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
129
|
+
};
|
|
130
|
+
/** The sentence for an absent package, shared so both gates say the same thing. */
|
|
131
|
+
export const absentMessage = (gate) => `the ${gate} gate reads your source with the TypeScript scanner, and the optional peer dependency "typescript" is not installed here. Install it (npm install --save-dev typescript), or drop the "${gate}" section from your configuration to stop invoking this gate. Only the dependency-direction and field-ownership gates need it.`;
|
|
132
|
+
/** Whether `error` carries `code` as a `NodeJS.ErrnoException` would, without assuming `error` is an object at all. */
|
|
133
|
+
export const isCode = (error, code) => error !== null &&
|
|
134
|
+
typeof error === 'object' &&
|
|
135
|
+
error.code === code;
|
|
136
|
+
/**
|
|
137
|
+
* The scanner module, or a refusal carrying `TYPESCRIPT_UNAVAILABLE` that names
|
|
138
|
+
* the gate, the installed version and the repair. Anything that is not one of
|
|
139
|
+
* the three refusals is rethrown unchanged.
|
|
140
|
+
*
|
|
141
|
+
* `load` and `readVersion` are injectable so each refusal has a case that
|
|
142
|
+
* uninstalls nothing.
|
|
143
|
+
*/
|
|
144
|
+
export async function loadTypeScriptScanner(gate, load = defaultLoad, readVersion = defaultVersion) {
|
|
145
|
+
let loaded;
|
|
146
|
+
try {
|
|
147
|
+
loaded = await load();
|
|
148
|
+
}
|
|
149
|
+
catch (error) {
|
|
150
|
+
if (isCode(error, 'ERR_MODULE_NOT_FOUND')) {
|
|
151
|
+
throw codedError(absentMessage(gate));
|
|
152
|
+
}
|
|
153
|
+
if (isCode(error, 'ERR_PACKAGE_PATH_NOT_EXPORTED')) {
|
|
154
|
+
const version = (await readVersion().catch(() => null)) ?? 'an unknown version';
|
|
155
|
+
throw codedError(`the ${gate} gate reads your source with the TypeScript scanner at ${SCANNER_SUBPATH}, and typescript ${version} carries no such subpath; TypeScript ships it from ${SCANNER_SHIPS_FROM}. Install typescript 7 to run this gate, or drop the "${gate}" section from your configuration to stop invoking it.`);
|
|
156
|
+
}
|
|
157
|
+
throw error;
|
|
158
|
+
}
|
|
159
|
+
const module = (loaded ?? {});
|
|
160
|
+
const missing = [];
|
|
161
|
+
if (typeof module.createScanner !== 'function')
|
|
162
|
+
missing.push('createScanner');
|
|
163
|
+
if (typeof module.computeLineStarts !== 'function') {
|
|
164
|
+
missing.push('computeLineStarts');
|
|
165
|
+
}
|
|
166
|
+
const kinds = module.SyntaxKind;
|
|
167
|
+
if (kinds === null || typeof kinds !== 'object') {
|
|
168
|
+
missing.push('SyntaxKind');
|
|
169
|
+
}
|
|
170
|
+
else {
|
|
171
|
+
for (const name of REQUIRED_SYNTAX_KINDS) {
|
|
172
|
+
if (typeof kinds[name] !== 'number')
|
|
173
|
+
missing.push(`SyntaxKind.${name}`);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
if (missing.length > 0) {
|
|
177
|
+
const version = (await readVersion().catch(() => null)) ?? 'an unknown version';
|
|
178
|
+
throw codedError(`the ${gate} gate reads ${missing.join(', ')} from ${SCANNER_SUBPATH}, and typescript ${version} ships it without ${missing.length === 1 ? 'that name' : 'those names'}; a member this gate cannot find would switch a rule off silently, so it refuses instead. This build reads the scanner TypeScript 7 ships.`);
|
|
179
|
+
}
|
|
180
|
+
return module;
|
|
181
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -4,12 +4,14 @@ export type { EvalContract } from './core/schemas/eval-contract.ts';
|
|
|
4
4
|
export { EVAL_CONTRACT_SCHEMA_VERSION } from './core/schemas/eval-contract.ts';
|
|
5
5
|
export type { EvaluatorConfiguration } from './core/schemas/evaluator-configuration.ts';
|
|
6
6
|
export { EVALUATOR_CONFIGURATION_SCHEMA_VERSION } from './core/schemas/evaluator-configuration.ts';
|
|
7
|
-
export type { EvidenceArtifact } from './core/schemas/evidence-artifact.ts';
|
|
7
|
+
export type { CheckResolutionValue, EvidenceArtifact, } from './core/schemas/evidence-artifact.ts';
|
|
8
8
|
export { EVIDENCE_ARTIFACT_SCHEMA_VERSION } from './core/schemas/evidence-artifact.ts';
|
|
9
|
+
export type { Expression, Operand } from './core/schemas/expression.ts';
|
|
9
10
|
export type { IsolationManifest } from './core/schemas/isolation-manifest.ts';
|
|
10
11
|
export { ISOLATION_MANIFEST_SCHEMA_VERSION } from './core/schemas/isolation-manifest.ts';
|
|
11
12
|
export type { PreflightCheck, PreflightVerdict, } from './core/schemas/preflight-verdict.ts';
|
|
12
13
|
export { PREFLIGHT_VERDICT_SCHEMA_VERSION } from './core/schemas/preflight-verdict.ts';
|
|
14
|
+
export type { JsonValue } from './core/schemas/primitives.ts';
|
|
13
15
|
export type { PrivateArtifactManifest } from './core/schemas/private-artifact-manifest.ts';
|
|
14
16
|
export { PRIVATE_ARTIFACT_MANIFEST_SCHEMA_VERSION } from './core/schemas/private-artifact-manifest.ts';
|
|
15
17
|
export type { Probe } from './core/schemas/probe.ts';
|
|
@@ -19,7 +21,7 @@ export type { ScoringPolicy } from './core/schemas/scoring-policy.ts';
|
|
|
19
21
|
export { SCORING_POLICY_SCHEMA_VERSION } from './core/schemas/scoring-policy.ts';
|
|
20
22
|
export type { SealedEvaluatorBrief } from './core/schemas/sealed-evaluator-brief.ts';
|
|
21
23
|
export { SEALED_EVALUATOR_BRIEF_SCHEMA_VERSION } from './core/schemas/sealed-evaluator-brief.ts';
|
|
22
|
-
export type { SealedRunRecord } from './core/schemas/sealed-run-record.ts';
|
|
24
|
+
export type { Observation, SealedRunRecord, } from './core/schemas/sealed-run-record.ts';
|
|
23
25
|
export { SEALED_RUN_RECORD_SCHEMA_VERSION } from './core/schemas/sealed-run-record.ts';
|
|
24
26
|
export type { FixtureReset, ManifestationWitness, SensitivityWitness, SensitivityWitnessLeg, WitnessChannel, WitnessInputs, } from './core/schemas/sensitivity-witness.ts';
|
|
25
|
-
export declare const VERSION = "3.
|
|
27
|
+
export declare const VERSION = "3.4.0";
|
package/dist/index.js
CHANGED
|
@@ -38,4 +38,4 @@ export { PROBE_SCHEMA_VERSION } from './core/schemas/probe.js';
|
|
|
38
38
|
export { SCORING_POLICY_SCHEMA_VERSION } from './core/schemas/scoring-policy.js';
|
|
39
39
|
export { SEALED_EVALUATOR_BRIEF_SCHEMA_VERSION } from './core/schemas/sealed-evaluator-brief.js';
|
|
40
40
|
export { SEALED_RUN_RECORD_SCHEMA_VERSION } from './core/schemas/sealed-run-record.js';
|
|
41
|
-
export const VERSION = '3.
|
|
41
|
+
export const VERSION = '3.4.0';
|
|
@@ -56,12 +56,12 @@ export declare const probeParsers: {
|
|
|
56
56
|
}>;
|
|
57
57
|
pathTemplate: import("zod").ZodString;
|
|
58
58
|
channels: import("zod").ZodObject<{
|
|
59
|
-
path: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../
|
|
60
|
-
query: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../
|
|
59
|
+
path: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>>;
|
|
60
|
+
query: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>>;
|
|
61
61
|
header: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodString>;
|
|
62
62
|
body: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
63
63
|
kind: import("zod").ZodLiteral<"json">;
|
|
64
|
-
value: import("zod").ZodType<import("../
|
|
64
|
+
value: import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>;
|
|
65
65
|
}, import("zod/v4/core").$strict>, import("zod").ZodObject<{
|
|
66
66
|
kind: import("zod").ZodLiteral<"absent">;
|
|
67
67
|
}, import("zod/v4/core").$strict>], "kind">;
|
|
@@ -74,12 +74,12 @@ export declare const probeParsers: {
|
|
|
74
74
|
executable: import("zod").ZodString;
|
|
75
75
|
subcommandPath: import("zod").ZodArray<import("zod").ZodString>;
|
|
76
76
|
channels: import("zod").ZodObject<{
|
|
77
|
-
argument: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../
|
|
78
|
-
option: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../
|
|
77
|
+
argument: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>>;
|
|
78
|
+
option: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>>;
|
|
79
79
|
environment: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodString>;
|
|
80
80
|
stdin: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
81
81
|
kind: import("zod").ZodLiteral<"json">;
|
|
82
|
-
value: import("zod").ZodType<import("../
|
|
82
|
+
value: import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>;
|
|
83
83
|
}, import("zod/v4/core").$strict>, import("zod").ZodObject<{
|
|
84
84
|
kind: import("zod").ZodLiteral<"text">;
|
|
85
85
|
value: import("zod").ZodString;
|
|
@@ -94,7 +94,7 @@ export declare const probeParsers: {
|
|
|
94
94
|
kind: import("zod").ZodLiteral<"mcp">;
|
|
95
95
|
toolName: import("zod").ZodString;
|
|
96
96
|
channels: import("zod").ZodObject<{
|
|
97
|
-
arguments: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../
|
|
97
|
+
arguments: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>>;
|
|
98
98
|
}, import("zod/v4/core").$strict>;
|
|
99
99
|
}, import("zod/v4/core").$strict>], "kind">;
|
|
100
100
|
readonly response: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
@@ -106,7 +106,7 @@ export declare const probeParsers: {
|
|
|
106
106
|
headers: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodString>;
|
|
107
107
|
body: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
108
108
|
kind: import("zod").ZodLiteral<"json">;
|
|
109
|
-
value: import("zod").ZodType<import("../
|
|
109
|
+
value: import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>;
|
|
110
110
|
}, import("zod/v4/core").$strict>, import("zod").ZodObject<{
|
|
111
111
|
kind: import("zod").ZodLiteral<"text">;
|
|
112
112
|
value: import("zod").ZodString;
|
|
@@ -121,7 +121,7 @@ export declare const probeParsers: {
|
|
|
121
121
|
exitCode: import("zod").ZodInt;
|
|
122
122
|
stdout: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
123
123
|
kind: import("zod").ZodLiteral<"json">;
|
|
124
|
-
value: import("zod").ZodType<import("../
|
|
124
|
+
value: import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>;
|
|
125
125
|
}, import("zod/v4/core").$strict>, import("zod").ZodObject<{
|
|
126
126
|
kind: import("zod").ZodLiteral<"text">;
|
|
127
127
|
value: import("zod").ZodString;
|
|
@@ -130,7 +130,7 @@ export declare const probeParsers: {
|
|
|
130
130
|
}, import("zod/v4/core").$strict>], "kind">;
|
|
131
131
|
stderr: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
132
132
|
kind: import("zod").ZodLiteral<"json">;
|
|
133
|
-
value: import("zod").ZodType<import("../
|
|
133
|
+
value: import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>;
|
|
134
134
|
}, import("zod/v4/core").$strict>, import("zod").ZodObject<{
|
|
135
135
|
kind: import("zod").ZodLiteral<"text">;
|
|
136
136
|
value: import("zod").ZodString;
|
|
@@ -139,7 +139,7 @@ export declare const probeParsers: {
|
|
|
139
139
|
}, import("zod/v4/core").$strict>], "kind">;
|
|
140
140
|
artifacts: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
141
141
|
kind: import("zod").ZodLiteral<"json">;
|
|
142
|
-
value: import("zod").ZodType<import("../
|
|
142
|
+
value: import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>;
|
|
143
143
|
}, import("zod/v4/core").$strict>, import("zod").ZodObject<{
|
|
144
144
|
kind: import("zod").ZodLiteral<"text">;
|
|
145
145
|
value: import("zod").ZodString;
|
|
@@ -154,7 +154,7 @@ export declare const probeParsers: {
|
|
|
154
154
|
isError: import("zod").ZodBoolean;
|
|
155
155
|
result: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
156
156
|
kind: import("zod").ZodLiteral<"json">;
|
|
157
|
-
value: import("zod").ZodType<import("../
|
|
157
|
+
value: import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>;
|
|
158
158
|
}, import("zod/v4/core").$strict>, import("zod").ZodObject<{
|
|
159
159
|
kind: import("zod").ZodLiteral<"text">;
|
|
160
160
|
value: import("zod").ZodString;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "eval-quality",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.4.0",
|
|
4
4
|
"description": "Compile disciplined Behavioral Evaluation Contracts and score their ability to catch known defects.",
|
|
5
5
|
"author": "Murat Ozcan",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -76,6 +76,7 @@
|
|
|
76
76
|
"check:doc-invocations": "node scripts/gates-cli.ts doc-invocations",
|
|
77
77
|
"check:doc-counts": "node scripts/gates-cli.ts doc-counts",
|
|
78
78
|
"check:doc-claims": "node scripts/gates-cli.ts doc-claims",
|
|
79
|
+
"hash:doc-claim-subject": "node scripts/hash-doc-claim-subject.ts",
|
|
79
80
|
"lint:spine": "python3 scripts/spine-lint/lint_spine.py --registry-ad 5 --workspace-root . --fail-on high",
|
|
80
81
|
"test:spine-lint": "uv run --with pytest pytest scripts/spine-lint/tests -q",
|
|
81
82
|
"build:shareable": "node scripts/build-shareable.mjs",
|
|
@@ -101,6 +102,8 @@
|
|
|
101
102
|
"check:corpus": "node scripts/check-dev-corpus.ts",
|
|
102
103
|
"generate:worked-example": "node scripts/generate-worked-example.ts",
|
|
103
104
|
"check:worked-example": "node scripts/check-worked-example.ts",
|
|
105
|
+
"generate:tutorials": "node scripts/generate-tutorials.ts",
|
|
106
|
+
"check:tutorials": "node scripts/check-tutorials.ts",
|
|
104
107
|
"generate:version": "node scripts/generate-version.ts",
|
|
105
108
|
"check:version": "node scripts/check-version.ts",
|
|
106
109
|
"generate:lockfile-age-cache": "node scripts/generate-lockfile-age-cache.ts",
|
|
@@ -117,7 +120,7 @@
|
|
|
117
120
|
"release:major": "gh workflow run publish.yml --ref main -f bump=major",
|
|
118
121
|
"release:prepare": "node scripts/release-prepare.mjs",
|
|
119
122
|
"release:publish": "gh workflow run publish.yml --ref main -f bump=none",
|
|
120
|
-
"validate": "npm run build && npm run typecheck && npm run lint && npm run check:docs && npm run check:doc-invocations && npm run check:shareable && npm run lint:spine && npm run check:vectors && npm run check:schemas && npm run check:ad5-registry && npm run check:ad28-registry && npm run check:ad31-table && npm run check:ad33-table && npm run check:ad21-table && npm run check:layers && npm run check:lineage && npm run check:boundary && npm run check:corpus && npm run check:doc-counts && npm run check:doc-claims && npm run check:worked-example && npm run check:version && npm run check:lockfile-age && npm run check:licences && npm run test:coverage",
|
|
123
|
+
"validate": "npm run build && npm run typecheck && npm run lint && npm run check:docs && npm run check:doc-invocations && npm run check:shareable && npm run lint:spine && npm run check:vectors && npm run check:schemas && npm run check:ad5-registry && npm run check:ad28-registry && npm run check:ad31-table && npm run check:ad33-table && npm run check:ad21-table && npm run check:layers && npm run check:lineage && npm run check:boundary && npm run check:corpus && npm run check:doc-counts && npm run check:doc-claims && npm run check:worked-example && npm run check:tutorials && npm run check:version && npm run check:lockfile-age && npm run check:licences && npm run test:coverage",
|
|
121
124
|
"prepack": "npm run clean && npm run build",
|
|
122
125
|
"prepublishOnly": "node scripts/assert-publish-authorized.mjs"
|
|
123
126
|
},
|