eval-quality 3.2.0 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -134,7 +134,7 @@ Node.js 22.20.0 or newer. `zod` is the only production dependency.
134
134
  npm install eval-quality
135
135
  ```
136
136
 
137
- Every command runs through `npx`:
137
+ Every command runs through `npx`. The four below are the grammar rather than a sequence to copy: they name files your own harness produces. [The full walkthrough](https://bmad-code-org.github.io/bmad-eval-quality/how-to/author-behavioral-contracts/) runs the same four end to end over files this repository commits, with no placeholder in the path.
138
138
 
139
139
  ```bash
140
140
  npx eval-quality compile --in contract.json --out ./eval-out
@@ -6,6 +6,11 @@
6
6
  * unamended.
7
7
  */
8
8
  export { digestArtifact, digestBytes, digestComposite, } from '../core/canonical/digest.ts';
9
+ export { makePointerDenotesCollection, makeResolveOperand, referenceSetKeysOf, } from '../core/evaluate/evidence-resolution.ts';
10
+ export type { PointerDenotesCollection, ReferenceSetKeys, ResolveOperand, } from '../core/evaluate/resolution.ts';
11
+ export { resolveCheck } from '../core/evaluate/resolution.ts';
12
+ export type { ResolvedValue } from '../core/evaluate/resolved-value.ts';
13
+ export { ABSENT } from '../core/evaluate/resolved-value.ts';
9
14
  export type { FailureCode } from '../core/failure-codes.ts';
10
15
  export { FAILURE_CODES, StructuralFailure } from '../core/failure-codes.ts';
11
16
  export type { LineageChainReport, LineageFinding, } from '../core/lineage/chain.ts';
@@ -21,6 +26,7 @@ export type { QualificationFailure, QualificationFailureCode, QualificationResul
21
26
  export { QUALIFICATION_FAILURES } from '../core/score/qualification.ts';
22
27
  export type { ComparableResult, DominanceRelationValue, } from '../core/score/strength.ts';
23
28
  export { compareDominance, DOMINANCE_RELATIONS, } from '../core/score/strength.ts';
29
+ export type { PlanIndex } from '../core/seal/plan-index.ts';
24
30
  export { compile } from './compile.ts';
25
31
  export type { Diagnostic, DiagnosticSink } from './diagnostics.ts';
26
32
  export type { PreflightFromObservationsOptions, RunPreflightOptions, } from './preflight.ts';
@@ -6,6 +6,14 @@
6
6
  * unamended.
7
7
  */
8
8
  export { digestArtifact, digestBytes, digestComposite, } from '../core/canonical/digest.js';
9
+ // The evaluator, for a consumer that resolves a check itself: the resolver,
10
+ // the two factories that build its operand and collection predicates from a
11
+ // contract and its observations, and the reference-set keys. `ABSENT` ships
12
+ // beside them because a `ResolveOperand` a consumer writes has to return it,
13
+ // and a sentinel the type names and the barrel withholds cannot be returned.
14
+ export { makePointerDenotesCollection, makeResolveOperand, referenceSetKeysOf, } from '../core/evaluate/evidence-resolution.js';
15
+ export { resolveCheck } from '../core/evaluate/resolution.js';
16
+ export { ABSENT } from '../core/evaluate/resolved-value.js';
9
17
  export { FAILURE_CODES, StructuralFailure } from '../core/failure-codes.js';
10
18
  export { validateLineageChain } from '../core/lineage/chain.js';
11
19
  export { INTERCHANGE_ARTIFACT_KEYS } from '../core/schemas/artifact.js';
@@ -18,6 +18,7 @@
18
18
  // appear in this file or anything it imports, or the gate fails at load.
19
19
  import { z } from 'zod';
20
20
  import { discoverSourceFiles } from './discover-source-files.js';
21
+ import { isCode, loadTypeScriptScanner, TYPESCRIPT_UNAVAILABLE, } from './typescript-scanner.js';
21
22
  /** The gate's key in the configuration file, and the token the binary dispatches on. */
22
23
  export const DEPENDENCY_DIRECTION_GATE = 'dependency-direction';
23
24
  /**
@@ -26,7 +27,7 @@ export const DEPENDENCY_DIRECTION_GATE = 'dependency-direction';
26
27
  * description and asserted by `tests/architecture/dependency-direction.test.ts`,
27
28
  * so the ordering property below is a measured fact rather than a warning.
28
29
  */
29
- export const ORDERING_WITNESS_VIOLATIONS = 78;
30
+ export const ORDERING_WITNESS_VIOLATIONS = 80;
30
31
  /** The optional peer is absent. The consumer repairs it by installing it, so it takes the usage code. */
31
32
  export const TYPESCRIPT_PEER_MISSING = 'EVAL_QUALITY_TYPESCRIPT_PEER_MISSING';
32
33
  /** A declared scan root could not be walked. Also a usage code: nothing was scanned, so nothing was answered. */
@@ -228,25 +229,24 @@ const refuse = (code, message) => ({
228
229
  /**
229
230
  * The scanner needs `typescript/unstable/ast`, and `typescript` is an optional
230
231
  * peer dependency so that a consumer running the other gates installs nothing.
231
- * Probing it by name is what turns a resolver stack trace into a sentence naming
232
- * the dependency and the gate that wanted it.
232
+ * `typescript-scanner.ts` is what turns a resolver stack trace into a sentence
233
+ * naming the dependency, the installed version and the gate that wanted it; a
234
+ * failure of any other shape is a bug in the scanner, not a missing peer, and
235
+ * is rethrown unchanged.
233
236
  *
234
237
  * `load` is injectable so a test can exercise the refusal without uninstalling
235
238
  * the package the test runner itself needs.
236
239
  */
237
- export async function probeTypeScript(load = () => import('typescript/unstable/ast')) {
240
+ export async function probeTypeScript(load = () => loadTypeScriptScanner(DEPENDENCY_DIRECTION_GATE)) {
238
241
  try {
239
242
  await load();
240
243
  return { ok: true };
241
244
  }
242
245
  catch (error) {
243
- if (error.code !== 'ERR_MODULE_NOT_FOUND') {
244
- throw error;
246
+ if (isCode(error, TYPESCRIPT_UNAVAILABLE)) {
247
+ return { ok: false, message: error.message };
245
248
  }
246
- return {
247
- ok: false,
248
- message: `the ${DEPENDENCY_DIRECTION_GATE} gate reads your source with the TypeScript scanner, and the optional peer dependency "typescript" is not installed here. Install it (npm install --save-dev typescript), or drop the "${DEPENDENCY_DIRECTION_GATE}" section from your configuration to stop invoking this gate. No other gate needs it.`,
249
- };
249
+ throw error;
250
250
  }
251
251
  }
252
252
  const orderViolations = (violations) => [...violations].sort((a, b) => a.file === b.file ? a.line - b.line : a.file < b.file ? -1 : 1);
@@ -480,7 +480,21 @@ export async function runDocClaims(root, section) {
480
480
  // this the short form is ambiguous and the check would refuse a citation
481
481
  // a reader resolves without effort.
482
482
  const qualified = new Map();
483
+ // A fenced block is output rather than prose, and a diagnostic a page
484
+ // transcribes carries the file and line of whatever tree the command
485
+ // ran over, which is commonly a fixture the page itself created. Read
486
+ // as a citation that path resolves to nothing here and the class fails
487
+ // on a page that is correct. The invocation check already holds a
488
+ // transcribed block against the bytes the run really wrote, so the
489
+ // claim is held either way.
490
+ let fenced = false;
483
491
  lines.forEach((line, index) => {
492
+ if (line.trimStart().startsWith('```')) {
493
+ fenced = !fenced;
494
+ return;
495
+ }
496
+ if (fenced)
497
+ return;
484
498
  for (const match of line.matchAll(citation)) {
485
499
  const cited = match[1];
486
500
  const first = Number(match[2]);
@@ -1,6 +1,13 @@
1
1
  // A published gate: every fenced command-line invocation in the pages a
2
- // consumer names is run against the binary they name, and the exit code is
3
- // compared with what the page claims.
2
+ // consumer names is run against the binary whose spelling opens the line, and
3
+ // the exit code is compared with what the page claims.
4
+ //
5
+ // A section names one binary, or several. A package that publishes two
6
+ // commands documents both, and each one carries its own built entry, its own
7
+ // spellings and its own installed-path prefix. Every spelling is matched
8
+ // against the same page and the longest one wins, because two published names
9
+ // commonly share a prefix and declaration order says nothing about which of
10
+ // them a line belongs to.
4
11
  //
5
12
  // The check exists because the documentation once described a product this
6
13
  // repository does not contain. A usage exit is what a command line returns when
@@ -14,8 +21,12 @@
14
21
  // reported no problems. So the exit code is judged too, wherever judging it
15
22
  // means anything:
16
23
  //
17
- // * A usage error and a crash always fail, for every invocation. Both are
18
- // about the command line alone, so a stand-in input cannot excuse them.
24
+ // * A crash always fails, for every invocation, and so does a usage error the
25
+ // page did not declare. Both are about the command line alone, so a
26
+ // stand-in input cannot excuse them. A page that declares the usage exit
27
+ // for a faithful invocation is claiming that refusal on purpose, which is
28
+ // what a binary spending that code on a configuration it would not read
29
+ // needs.
19
30
  // * An invocation is FAITHFUL when every input it names resolved to real
20
31
  // bytes: a file the repository ships, a file the same page told the reader
21
32
  // to create, or an artifact an earlier command on the page wrote. A
@@ -35,9 +46,11 @@
35
46
  // The exit code alone is a weak claim, because it is shared. One code commonly
36
47
  // covers a whole family of failures, so a page can name one failure while the
37
48
  // binary reports another and the codes still agree. A page that declares its
38
- // exit code may therefore transcribe the diagnostic beside it, and that block is
39
- // compared line for line against what the run wrote to stderr. Four rules shape
40
- // which block gets compared:
49
+ // exit code may therefore transcribe the output beside it, and that block is
50
+ // compared line for line against what the run wrote: stderr when the run wrote
51
+ // any, and stdout otherwise, since a page documenting a command that worked is
52
+ // quoting the answer rather than a diagnostic. Four rules shape which block gets
53
+ // compared:
41
54
  //
42
55
  // * The block is a `text` fence separated from the command's fence by blank
43
56
  // lines only. Prose between them detaches it, and a fence carrying any
@@ -311,11 +324,15 @@ function describesLine(documented, actual) {
311
324
  * fence or the next unindented line.
312
325
  *
313
326
  * A `text` fence separated from a declared-exit invocation by nothing but blank
314
- * lines is that invocation's transcribed diagnostic, and it travels on the run
315
- * as `expectStderr`. Only a declared-exit invocation collects one: a page that
316
- * shows the output of a command that succeeded is showing stdout, and every
317
- * page that documents a failure prints it on stderr. The fence has to have
327
+ * lines is that invocation's transcribed output, and it travels on the run as
328
+ * `expectStderr`. Only a declared-exit invocation collects one, so a page opts
329
+ * in to the comparison by declaring what the run returns. The fence has to have
318
330
  * pushed exactly one such invocation, since both would carry its declaration.
331
+ *
332
+ * `spellings` is every spelling across every declared binary, already sorted
333
+ * longest first, and each one carries the index of the binary it belongs to.
334
+ * That index travels on the run, so the caller knows which entry to execute and
335
+ * whose installed-path prefix to resolve the arguments against.
319
336
  */
320
337
  function extractActions(file, source, spellings) {
321
338
  const lines = source.split('\n');
@@ -421,10 +438,12 @@ function extractActions(file, source, spellings) {
421
438
  index += 1;
422
439
  text = `${text.slice(0, -1).trim()} ${lines[index].trim()}`;
423
440
  }
424
- const match = spellings.map((pattern) => text.match(pattern)).find(Boolean);
425
- if (!match)
441
+ const matched = spellings
442
+ .map((spelling) => ({ spelling, match: text.match(spelling.pattern) }))
443
+ .find((candidate) => candidate.match !== null);
444
+ if (matched === undefined)
426
445
  continue;
427
- const tail = (match[1] ?? '').trim();
446
+ const tail = (matched.match[1] ?? '').trim();
428
447
  if (tail === '')
429
448
  continue;
430
449
  const first = tokenize(tail)[0];
@@ -436,6 +455,7 @@ function extractActions(file, source, spellings) {
436
455
  line: startLine,
437
456
  invocation: text,
438
457
  tail,
458
+ binary: matched.spelling.binary,
439
459
  expectExit,
440
460
  expectStderr: null,
441
461
  };
@@ -453,12 +473,24 @@ function extractActions(file, source, spellings) {
453
473
  * section names resolves against it.
454
474
  */
455
475
  export function runDocInvocations(root, section) {
456
- const entry = resolve(root, section.binary.entry);
476
+ // One binary is the ordinary case and stays spelled as one object. The list
477
+ // is built once here, so everything below reads the same shape.
478
+ const declared = Array.isArray(section.binary)
479
+ ? section.binary
480
+ : [section.binary];
481
+ const binaries = declared.map((binary) => ({
482
+ entry: resolve(root, binary.entry),
483
+ installedPrefix: binary.installedPrefix,
484
+ }));
457
485
  // A build is a precondition rather than an excuse. Skipping here would let
458
486
  // the gate exit 0 having executed nothing, which is the vacuous pass the
459
- // whole check exists to prevent.
460
- if (!existsSync(entry)) {
461
- throw codedError(DOC_PATH_ERROR, `${entry} does not exist; the "doc-invocations" section names it under binary.entry, so build it before the gate runs`);
487
+ // whole check exists to prevent. The refusal names which binary is missing,
488
+ // since a reader with two of them has two build steps to choose between.
489
+ for (const [index, binary] of binaries.entries()) {
490
+ if (existsSync(binary.entry))
491
+ continue;
492
+ const field = declared.length === 1 ? 'binary.entry' : `binary[${index}].entry`;
493
+ throw codedError(DOC_PATH_ERROR, `${binary.entry} does not exist; the "doc-invocations" section names it under ${field}, so build it before the gate runs`);
462
494
  }
463
495
  const sampleInput = resolve(root, section.sampleInput);
464
496
  if (!existsSync(sampleInput)) {
@@ -486,12 +518,26 @@ export function runDocInvocations(root, section) {
486
518
  throw codedError(DOC_PATH_ERROR, `the "doc-invocations" section names "${page}" under pages, and that encloses the directory the configuration sits in; name a page or a directory inside it, since every fenced command under a whole repository is more than this gate should run`);
487
519
  }
488
520
  }
489
- const spellings = section.binary.spellings.map(spellingPattern);
490
- const context = {
491
- repoRoot: resolve(root),
521
+ // Longest first, and the first match wins. Two published names commonly
522
+ // share a prefix, so matching in declaration order would let a short
523
+ // spelling belonging to one binary claim a line that opens with a longer
524
+ // spelling belonging to another, and the line would then run against the
525
+ // wrong entry and be judged against the wrong installed-path prefix.
526
+ const spellings = declared
527
+ .flatMap((binary, index) => binary.spellings.map((spelling) => ({
528
+ text: spelling.trim(),
529
+ pattern: spellingPattern(spelling),
530
+ binary: index,
531
+ })))
532
+ .sort((a, b) => b.text.length - a.text.length);
533
+ const repoRoot = resolve(root);
534
+ // One context per binary, because `installedPrefix` belongs to the binary a
535
+ // line matched. Two binaries on one page can map different installed paths.
536
+ const contexts = binaries.map((binary) => ({
537
+ repoRoot,
492
538
  sampleInput,
493
- installedPrefix: section.binary.installedPrefix,
494
- };
539
+ installedPrefix: binary.installedPrefix,
540
+ }));
495
541
  const files = section.pages
496
542
  .flatMap((page) => collectMarkdown(resolve(root, page)))
497
543
  .sort();
@@ -506,8 +552,8 @@ export function runDocInvocations(root, section) {
506
552
  let compared = 0;
507
553
  try {
508
554
  for (const [index, absolute] of files.entries()) {
509
- const file = absolute.startsWith(context.repoRoot)
510
- ? absolute.slice(context.repoRoot.length + 1)
555
+ const file = absolute.startsWith(repoRoot)
556
+ ? absolute.slice(repoRoot.length + 1)
511
557
  : absolute;
512
558
  const sandbox = createPageSandbox(workDir, index);
513
559
  for (const action of extractActions(file, readFileSync(absolute, 'utf8'), spellings)) {
@@ -520,11 +566,11 @@ export function runDocInvocations(root, section) {
520
566
  continue;
521
567
  }
522
568
  scanned += 1;
523
- const { tokens, faithful } = realizeArguments(action.tail, sandbox, context);
569
+ const { tokens, faithful } = realizeArguments(action.tail, sandbox, contexts[action.binary]);
524
570
  // The sandbox root is the working directory and stdin is closed: a
525
571
  // relative write lands inside the sandbox, and a command that reads
526
572
  // stdin sees an empty stream and returns at once.
527
- const result = spawnSync(process.execPath, [entry, ...tokens], {
573
+ const result = spawnSync(process.execPath, [binaries[action.binary].entry, ...tokens], {
528
574
  cwd: sandbox.root,
529
575
  encoding: 'utf8',
530
576
  input: '',
@@ -556,7 +602,14 @@ export function runDocInvocations(root, section) {
556
602
  record('the binary crashed');
557
603
  continue;
558
604
  }
559
- if (result.status === section.usageExit) {
605
+ // A usage exit is a mistyped command or a flag that stopped existing,
606
+ // except where the page declares it. One binary can spend the same
607
+ // code on a configuration it refused to read, and a page teaching a
608
+ // reader to recognise that refusal is making a claim about it like
609
+ // any other. The declaration is what separates the two, so an
610
+ // undeclared usage exit still fails every invocation.
611
+ if (result.status === section.usageExit &&
612
+ action.expectExit !== section.usageExit) {
560
613
  record('usage error: the documented command or flag does not exist');
561
614
  continue;
562
615
  }
@@ -580,7 +633,13 @@ export function runDocInvocations(root, section) {
580
633
  if (action.expectStderr === null)
581
634
  continue;
582
635
  compared += 1;
583
- const written = result.stderr.split('\n');
636
+ // A page documenting a rejection quotes stderr, and a page
637
+ // documenting a command that worked quotes stdout. A run that wrote
638
+ // nothing to stderr is the second case, so the block is compared
639
+ // against what the run actually said rather than against an empty
640
+ // stream. Declaring the exit code is still what attaches a block at
641
+ // all, so no page acquires a comparison it did not ask for.
642
+ const written = (result.stderr.trim() === '' ? result.stdout : result.stderr).split('\n');
584
643
  const documented = action.expectStderr;
585
644
  const overElided = documented.findIndex((line) => line.split(ELISION).length - 1 > section.elisionLimit);
586
645
  if (overElided !== -1) {
@@ -612,7 +671,7 @@ export function runDocInvocations(root, section) {
612
671
  // no commands at all. The pages are there and the binary is there, so the one
613
672
  // thing left to name is the spelling list.
614
673
  if (scanned === 0) {
615
- throw codedError(DOC_PATH_ERROR, `no fenced command in ${files.length} page(s) matched any spelling the "doc-invocations" section declares (${section.binary.spellings.join(', ')}); a gate that extracted nothing reports a pass over nothing`);
674
+ throw codedError(DOC_PATH_ERROR, `no fenced command in ${files.length} page(s) matched any spelling the "doc-invocations" section declares (${spellings.map((spelling) => spelling.text).join(', ')}); a gate that extracted nothing reports a pass over nothing`);
616
675
  }
617
676
  return { failures, scanned, judged, compared, pages: files.length };
618
677
  }
@@ -473,6 +473,20 @@ function scanFile(file, source, files, graph, violations) {
473
473
  const openIndex = tokens[i + 1]?.kind === SyntaxKind.QuestionDotToken ? i + 2 : i + 1;
474
474
  if (tokens[openIndex]?.kind !== SyntaxKind.OpenParenToken)
475
475
  continue;
476
+ // The site this arm exists for is the free identifier `require` called
477
+ // with a specifier. Two shapes share its tokens and are ordinary
478
+ // JavaScript: a member call, `sandbox.require('fs')`, where the token
479
+ // before is `.` or `?.`, and a method named require, `require(name) {`
480
+ // in an object literal or a class, where the token after the matching
481
+ // `)` is `{`. A call's own `)` is never followed by `{`, so
482
+ // `if (require('x')) {` still reads as the call it is. A consumer
483
+ // renaming a mock to get past this arm is a gate teaching the wrong lesson.
484
+ const before = tokens[i - 1]?.kind;
485
+ if (before === SyntaxKind.DotToken ||
486
+ before === SyntaxKind.QuestionDotToken ||
487
+ isMethodDefinition(tokens, i, openIndex)) {
488
+ continue;
489
+ }
476
490
  if (graph.commonjs === 'forbid') {
477
491
  violations.push({
478
492
  file,
@@ -545,6 +559,122 @@ function scanFile(file, source, files, graph, violations) {
545
559
  * against `graph` and returns every violation found, in no particular cross-file
546
560
  * order.
547
561
  */
562
+ /**
563
+ * A modifier or generator marker that can sit between a declaration boundary
564
+ * and the member name it modifies: `async`, `static`, `get`, `set`,
565
+ * `readonly`, an access modifier, or `*`. Skipped when walking backward from
566
+ * `require` to find what actually introduces it.
567
+ */
568
+ const MEMBER_MODIFIERS = new Set([
569
+ SyntaxKind.AsyncKeyword,
570
+ SyntaxKind.StaticKeyword,
571
+ SyntaxKind.GetKeyword,
572
+ SyntaxKind.SetKeyword,
573
+ SyntaxKind.ReadonlyKeyword,
574
+ SyntaxKind.PrivateKeyword,
575
+ SyntaxKind.PublicKeyword,
576
+ SyntaxKind.ProtectedKeyword,
577
+ SyntaxKind.AsteriskToken,
578
+ ]);
579
+ /**
580
+ * Whether `require`, immediately followed by a parameter list and then `{`,
581
+ * actually sits where a name is declared rather than where a call's result is
582
+ * followed by an unrelated block statement: `const mod = require('x')` and a
583
+ * stray `{` on the next line tokenize exactly like a method body, and only
584
+ * what precedes `require` tells them apart. Skips the modifiers above, then
585
+ * requires the next token to be the opening brace of the object, class or
586
+ * interface `require` is the first member of, a `,` or `;` separating it from
587
+ * a prior member, a `case`/`default` label, `function` for a function
588
+ * declaration, or the start of the file. Anything else -- `=`, `return`, or
589
+ * any other token an expression puts before a call -- means this is a call.
590
+ *
591
+ * A `;` or `}` immediately before `require` stays undecidable this way: both
592
+ * a prior class member and a prior unrelated statement end on one, and
593
+ * telling them apart needs knowing what kind of block `require` sits in,
594
+ * which a token stream does not carry. Rare enough, and specific enough to
595
+ * write on purpose, that it is left as the one shape this arm still misses.
596
+ */
597
+ function isDeclarationPosition(tokens, requireIndex) {
598
+ let j = requireIndex - 1;
599
+ while (j >= 0 && MEMBER_MODIFIERS.has(tokens[j]?.kind))
600
+ j -= 1;
601
+ if (j < 0)
602
+ return true;
603
+ const kind = tokens[j]?.kind;
604
+ return (kind === SyntaxKind.OpenBraceToken ||
605
+ kind === SyntaxKind.CommaToken ||
606
+ kind === SyntaxKind.SemicolonToken ||
607
+ kind === SyntaxKind.CaseKeyword ||
608
+ kind === SyntaxKind.DefaultKeyword ||
609
+ kind === SyntaxKind.FunctionKeyword);
610
+ }
611
+ /**
612
+ * Whether the parenthesised list opening at `openIndex` is a parameter list,
613
+ * which is to say `require` here is a method or a signature and never a call.
614
+ * A `{` straight after the matching `)` defers to `isDeclarationPosition`. A
615
+ * `:` after it is either a return-type annotation, a ternary's else, or a
616
+ * `case`/`default` label, and the three are told apart by looking back from
617
+ * `require` at bracket depth zero: a ternary has its `?` before the call and
618
+ * inside the same expression, a `case`/`default` label has the keyword
619
+ * immediately before the call and nothing between them, and a member
620
+ * declaration has none of those before its own `{`, `,` or `;` -- or before
621
+ * reaching an enclosing `(`, `[` or `{` with nothing still open inside it,
622
+ * which is the same boundary one level up. An unclosed list reads as a call,
623
+ * which is what this arm already did with a stream it could not place.
624
+ */
625
+ function isMethodDefinition(tokens, requireIndex, openIndex) {
626
+ let depth = 0;
627
+ let closeIndex = -1;
628
+ for (let j = openIndex; j < tokens.length; j++) {
629
+ const kind = tokens[j]?.kind;
630
+ if (kind === SyntaxKind.OpenParenToken)
631
+ depth += 1;
632
+ else if (kind === SyntaxKind.CloseParenToken) {
633
+ depth -= 1;
634
+ if (depth === 0) {
635
+ closeIndex = j;
636
+ break;
637
+ }
638
+ }
639
+ }
640
+ if (closeIndex === -1)
641
+ return false;
642
+ const after = tokens[closeIndex + 1]?.kind;
643
+ if (after === SyntaxKind.OpenBraceToken) {
644
+ return isDeclarationPosition(tokens, requireIndex);
645
+ }
646
+ if (after !== SyntaxKind.ColonToken)
647
+ return false;
648
+ depth = 0;
649
+ for (let j = requireIndex - 1; j >= 0; j--) {
650
+ const kind = tokens[j]?.kind;
651
+ if (kind === SyntaxKind.CloseParenToken ||
652
+ kind === SyntaxKind.CloseBracketToken ||
653
+ kind === SyntaxKind.CloseBraceToken) {
654
+ depth += 1;
655
+ continue;
656
+ }
657
+ if (kind === SyntaxKind.OpenParenToken ||
658
+ kind === SyntaxKind.OpenBracketToken ||
659
+ kind === SyntaxKind.OpenBraceToken) {
660
+ if (depth === 0)
661
+ return true;
662
+ depth -= 1;
663
+ continue;
664
+ }
665
+ if (depth > 0)
666
+ continue;
667
+ if (kind === SyntaxKind.QuestionToken)
668
+ return false;
669
+ if (kind === SyntaxKind.CaseKeyword || kind === SyntaxKind.DefaultKeyword) {
670
+ return false;
671
+ }
672
+ if (kind === SyntaxKind.SemicolonToken || kind === SyntaxKind.CommaToken) {
673
+ return true;
674
+ }
675
+ }
676
+ return true;
677
+ }
548
678
  export function scanSources(files, graph) {
549
679
  const fileSet = new Set(files.keys());
550
680
  const violations = [];
@@ -279,6 +279,16 @@ const LicencesSection = z
279
279
  * `node_modules`, which is what keeps the pre-install path open for the two
280
280
  * gates that run before `npm ci`.
281
281
  */
282
+ const DocumentedBinary = z
283
+ .strictObject({
284
+ entry: RelativePath.describe('The built entry point every documented command is run against. It is a precondition: a gate that skipped when it was absent would exit 0 having executed nothing.'),
285
+ spellings: z
286
+ .array(NonEmpty)
287
+ .min(1)
288
+ .describe('How your documentation writes the command, as the literal text a reader types. Each is matched at the start of a fenced line, with whitespace or end of line after it, so a sample of your own diagnostic output is left alone.'),
289
+ installedPrefix: NonEmpty.optional().describe('The path prefix a page uses for a file inside the installed package, such as "node_modules/your-package/". Mapping it away is what lets those examples be checked against real bytes.'),
290
+ })
291
+ .describe('One binary a documented command line runs.');
282
292
  const DocInvocationsSection = z
283
293
  .strictObject({
284
294
  pages: z
@@ -286,15 +296,8 @@ const DocInvocationsSection = z
286
296
  .min(1)
287
297
  .describe("The pages whose fenced commands are run, as files or directories to walk. Naming the configuration's own directory is refused: every fenced command in a whole repository is more than this gate should run."),
288
298
  binary: z
289
- .strictObject({
290
- entry: RelativePath.describe('The built entry point every documented command is run against. It is a precondition: a gate that skipped when it was absent would exit 0 having executed nothing.'),
291
- spellings: z
292
- .array(NonEmpty)
293
- .min(1)
294
- .describe('How your documentation writes the command, as the literal text a reader types. Each is matched at the start of a fenced line, with whitespace or end of line after it, so a sample of your own diagnostic output is left alone.'),
295
- installedPrefix: NonEmpty.optional().describe('The path prefix a page uses for a file inside the installed package, such as "node_modules/your-package/". Mapping it away is what lets those examples be checked against real bytes.'),
296
- })
297
- .describe('What a documented command line runs.'),
299
+ .union([DocumentedBinary, z.array(DocumentedBinary).min(1)])
300
+ .describe("What a documented command line runs. One object for a package that publishes one binary, or an array for a package that publishes several, each carrying its own entry, spellings and installedPrefix. Every spelling across every binary is matched against the same page and the longest one wins, so a short spelling belonging to one binary never claims a line that opens with a longer spelling belonging to another. Each binary's entry is its own precondition and the refusal names which one is missing. An empty array is refused, since a gate with no spelling to match extracts nothing and reports a pass over nothing."),
298
301
  sampleInput: RelativePath.describe('A file that stands in for an input only a reader has, such as `<path>`. A run that needed one is judged for usage errors and crashes and no more.'),
299
302
  usageExit: z
300
303
  .int()
@@ -32,39 +32,36 @@
32
32
  // appear in this file or anything it imports.
33
33
  import { z } from 'zod';
34
34
  import { discoverEntries, RelativePath, RelativePrefix, ScannedPathList, } from './scanned-paths.js';
35
+ import { loadTypeScriptScanner, TYPESCRIPT_UNAVAILABLE, } from './typescript-scanner.js';
35
36
  /** The gate needs `typescript` and could not resolve it. */
36
- export const TYPESCRIPT_UNAVAILABLE = 'EVAL_QUALITY_TYPESCRIPT_UNAVAILABLE';
37
- const codedError = (code, message) => Object.assign(new Error(message), { code });
38
- const importTokenScanner = async () => {
37
+ export { TYPESCRIPT_UNAVAILABLE };
38
+ const importTokenScanner = (gate) => async () => {
39
39
  // `token-scan.ts` imports `typescript/unstable/ast` at its own top level, so
40
- // this is the one place the dependency is reached and the one place its
41
- // absence can be turned into a sentence.
42
- const [ast, scan] = await Promise.all([
43
- import('typescript/unstable/ast'),
44
- import('./token-scan.js'),
45
- ]);
40
+ // the loader runs first and its refusal is the one a consumer reads; the
41
+ // import of `token-scan.ts` follows only once the scanner is known to be there.
42
+ const ast = await loadTypeScriptScanner(gate);
43
+ const scan = await import('./token-scan.js');
46
44
  return {
47
45
  scanTokens: scan.scanTokens,
48
46
  computeLineStarts: scan.computeLineStarts,
49
47
  lineOf: scan.lineOf,
48
+ // `loadTypeScriptScanner` types `SyntaxKind` as a generic
49
+ // `Readonly<Record<string, number>>`, since its own shape check indexes
50
+ // it by whichever member name each of the three scanner modules reads.
51
+ // That check has already run and passed by the time this line executes,
52
+ // so every member this file reads off `Syntax` is verified present; the
53
+ // double cast is regaining the precise type the runtime check earned.
50
54
  syntax: ast.SyntaxKind,
51
55
  };
52
56
  };
53
57
  /**
54
- * The tokenizer, or a refusal naming the dependency and the gate that needs it.
55
- * `load` is injectable so the refusal has a test that does not require
56
- * uninstalling anything.
58
+ * The tokenizer, or the refusal `loadTypeScriptScanner` throws naming the
59
+ * dependency and the gate that needs it. `load` is injectable so a test can
60
+ * exercise that refusal without uninstalling anything the test runner itself
61
+ * needs.
57
62
  */
58
- export async function loadTokenScanner(gate, load = importTokenScanner) {
59
- try {
60
- return await load();
61
- }
62
- catch (error) {
63
- if (error.code !== 'ERR_MODULE_NOT_FOUND') {
64
- throw error;
65
- }
66
- throw codedError(TYPESCRIPT_UNAVAILABLE, `the ${gate} gate reads your source through the typescript package's own scanner, and typescript did not resolve. Install typescript to run this gate; every other gate needs nothing beyond this package.`);
67
- }
63
+ export async function loadTokenScanner(gate, load = importTokenScanner(gate)) {
64
+ return load();
68
65
  }
69
66
  const Identifier = z
70
67
  .string()
@@ -0,0 +1,181 @@
1
+ // The one place the two source-scanning gates reach TypeScript.
2
+ //
3
+ // The scanner lives at `typescript/unstable/ast`, a subpath whose own name says
4
+ // it may move, and it exists from TypeScript 7.0: the 5.x line has no such
5
+ // subpath and the 7.x main entry exports no scanner. The optional peer range
6
+ // says `>=5.7.0` and stays that wide on purpose. npm resolves an optional peer
7
+ // that is present, so a range of `>=7` would turn `npm install` red for every
8
+ // consumer with TypeScript 5 in its tree and no interest in these two gates.
9
+ // The version fact lives here instead, spoken at the one moment it matters:
10
+ // when a consumer runs one of the two gates.
11
+ //
12
+ // Three refusals, each its own because the repair is different. The package
13
+ // is absent; the package is a version with no such subpath; the subpath is
14
+ // there and lacks a member this build reads. The third is the quiet one: a
15
+ // renamed enum member reads as `undefined`, `token.kind === undefined` never
16
+ // matches, and the rule it guarded switches off with every gate green. So every
17
+ // `SyntaxKind` member the three scanner modules read is listed below, and
18
+ // `tests/architecture/typescript-scanner.test.ts` derives the same list from
19
+ // their sources, so the list is not a copy a hand maintains.
20
+ //
21
+ // Run by `node` directly: Node's type stripping erases types only, so no
22
+ // TypeScript enum, namespace, parameter property, or non-type re-export may
23
+ // appear in this file or anything it imports, or the gate fails at load.
24
+ /** `error.code` on every refusal here; the binary maps it to the usage exit. */
25
+ export const TYPESCRIPT_UNAVAILABLE = 'EVAL_QUALITY_TYPESCRIPT_UNAVAILABLE';
26
+ /** Where the scanner is read from, and the first TypeScript that ships it. */
27
+ export const SCANNER_SUBPATH = 'typescript/unstable/ast';
28
+ export const SCANNER_SHIPS_FROM = '7.0.0';
29
+ /** Every `SyntaxKind` member `token-scan.ts`, `dependency-direction.ts` and `lineage-ownership.ts` read. */
30
+ export const REQUIRED_SYNTAX_KINDS = [
31
+ 'AmpersandToken',
32
+ 'AnyKeyword',
33
+ 'AsKeyword',
34
+ 'AsteriskToken',
35
+ 'AsyncKeyword',
36
+ 'AwaitKeyword',
37
+ 'BarToken',
38
+ 'BigIntKeyword',
39
+ 'BigIntLiteral',
40
+ 'BooleanKeyword',
41
+ 'CaseKeyword',
42
+ 'CatchKeyword',
43
+ 'ClassKeyword',
44
+ 'CloseBraceToken',
45
+ 'CloseBracketToken',
46
+ 'CloseParenToken',
47
+ 'ColonToken',
48
+ 'CommaToken',
49
+ 'ConstKeyword',
50
+ 'DefaultKeyword',
51
+ 'DotToken',
52
+ 'EndOfFile',
53
+ 'EqualsGreaterThanToken',
54
+ 'EqualsToken',
55
+ 'ExclamationToken',
56
+ 'ExportKeyword',
57
+ 'ExtendsKeyword',
58
+ 'FalseKeyword',
59
+ 'FirstAssignment',
60
+ 'ForKeyword',
61
+ 'FromKeyword',
62
+ 'FunctionKeyword',
63
+ 'GetKeyword',
64
+ 'Identifier',
65
+ 'IfKeyword',
66
+ 'ImportKeyword',
67
+ 'InterfaceKeyword',
68
+ 'LastAssignment',
69
+ 'LessThanToken',
70
+ 'LetKeyword',
71
+ 'MinusMinusToken',
72
+ 'NeverKeyword',
73
+ 'NewKeyword',
74
+ 'NoSubstitutionTemplateLiteral',
75
+ 'NullKeyword',
76
+ 'NumberKeyword',
77
+ 'NumericLiteral',
78
+ 'ObjectKeyword',
79
+ 'OpenBraceToken',
80
+ 'OpenBracketToken',
81
+ 'OpenParenToken',
82
+ 'PlusPlusToken',
83
+ 'PrivateKeyword',
84
+ 'ProtectedKeyword',
85
+ 'PublicKeyword',
86
+ 'QuestionDotToken',
87
+ 'QuestionToken',
88
+ 'ReadonlyKeyword',
89
+ 'RegularExpressionLiteral',
90
+ 'RequireKeyword',
91
+ 'ReturnKeyword',
92
+ 'SemicolonToken',
93
+ 'SetKeyword',
94
+ 'SlashEqualsToken',
95
+ 'SlashToken',
96
+ 'StaticKeyword',
97
+ 'StringKeyword',
98
+ 'StringLiteral',
99
+ 'SuperKeyword',
100
+ 'SymbolKeyword',
101
+ 'TemplateHead',
102
+ 'TemplateTail',
103
+ 'ThisKeyword',
104
+ 'TrueKeyword',
105
+ 'TypeKeyword',
106
+ 'UndefinedKeyword',
107
+ 'UnknownKeyword',
108
+ 'VarKeyword',
109
+ 'VoidKeyword',
110
+ 'WhileKeyword',
111
+ 'WithKeyword',
112
+ ];
113
+ const codedError = (message) => Object.assign(new Error(message), { code: TYPESCRIPT_UNAVAILABLE });
114
+ const defaultLoad = () => import('typescript/unstable/ast');
115
+ // `typescript/package.json` is on the package's export map, so the version is
116
+ // read from the install itself. A read that fails names no version rather than
117
+ // failing the refusal that wanted it.
118
+ const defaultVersion = async () => {
119
+ try {
120
+ const manifest = (await import('typescript/package.json', {
121
+ with: { type: 'json' },
122
+ }));
123
+ const version = manifest.default?.version;
124
+ return typeof version === 'string' ? version : null;
125
+ }
126
+ catch {
127
+ return null;
128
+ }
129
+ };
130
+ /** The sentence for an absent package, shared so both gates say the same thing. */
131
+ export const absentMessage = (gate) => `the ${gate} gate reads your source with the TypeScript scanner, and the optional peer dependency "typescript" is not installed here. Install it (npm install --save-dev typescript), or drop the "${gate}" section from your configuration to stop invoking this gate. Only the dependency-direction and field-ownership gates need it.`;
132
+ /** Whether `error` carries `code` as a `NodeJS.ErrnoException` would, without assuming `error` is an object at all. */
133
+ export const isCode = (error, code) => error !== null &&
134
+ typeof error === 'object' &&
135
+ error.code === code;
136
+ /**
137
+ * The scanner module, or a refusal carrying `TYPESCRIPT_UNAVAILABLE` that names
138
+ * the gate, the installed version and the repair. Anything that is not one of
139
+ * the three refusals is rethrown unchanged.
140
+ *
141
+ * `load` and `readVersion` are injectable so each refusal has a case that
142
+ * uninstalls nothing.
143
+ */
144
+ export async function loadTypeScriptScanner(gate, load = defaultLoad, readVersion = defaultVersion) {
145
+ let loaded;
146
+ try {
147
+ loaded = await load();
148
+ }
149
+ catch (error) {
150
+ if (isCode(error, 'ERR_MODULE_NOT_FOUND')) {
151
+ throw codedError(absentMessage(gate));
152
+ }
153
+ if (isCode(error, 'ERR_PACKAGE_PATH_NOT_EXPORTED')) {
154
+ const version = (await readVersion().catch(() => null)) ?? 'an unknown version';
155
+ throw codedError(`the ${gate} gate reads your source with the TypeScript scanner at ${SCANNER_SUBPATH}, and typescript ${version} carries no such subpath; TypeScript ships it from ${SCANNER_SHIPS_FROM}. Install typescript 7 to run this gate, or drop the "${gate}" section from your configuration to stop invoking it.`);
156
+ }
157
+ throw error;
158
+ }
159
+ const module = (loaded ?? {});
160
+ const missing = [];
161
+ if (typeof module.createScanner !== 'function')
162
+ missing.push('createScanner');
163
+ if (typeof module.computeLineStarts !== 'function') {
164
+ missing.push('computeLineStarts');
165
+ }
166
+ const kinds = module.SyntaxKind;
167
+ if (kinds === null || typeof kinds !== 'object') {
168
+ missing.push('SyntaxKind');
169
+ }
170
+ else {
171
+ for (const name of REQUIRED_SYNTAX_KINDS) {
172
+ if (typeof kinds[name] !== 'number')
173
+ missing.push(`SyntaxKind.${name}`);
174
+ }
175
+ }
176
+ if (missing.length > 0) {
177
+ const version = (await readVersion().catch(() => null)) ?? 'an unknown version';
178
+ throw codedError(`the ${gate} gate reads ${missing.join(', ')} from ${SCANNER_SUBPATH}, and typescript ${version} ships it without ${missing.length === 1 ? 'that name' : 'those names'}; a member this gate cannot find would switch a rule off silently, so it refuses instead. This build reads the scanner TypeScript 7 ships.`);
179
+ }
180
+ return module;
181
+ }
package/dist/index.d.ts CHANGED
@@ -4,12 +4,14 @@ export type { EvalContract } from './core/schemas/eval-contract.ts';
4
4
  export { EVAL_CONTRACT_SCHEMA_VERSION } from './core/schemas/eval-contract.ts';
5
5
  export type { EvaluatorConfiguration } from './core/schemas/evaluator-configuration.ts';
6
6
  export { EVALUATOR_CONFIGURATION_SCHEMA_VERSION } from './core/schemas/evaluator-configuration.ts';
7
- export type { EvidenceArtifact } from './core/schemas/evidence-artifact.ts';
7
+ export type { CheckResolutionValue, EvidenceArtifact, } from './core/schemas/evidence-artifact.ts';
8
8
  export { EVIDENCE_ARTIFACT_SCHEMA_VERSION } from './core/schemas/evidence-artifact.ts';
9
+ export type { Expression, Operand } from './core/schemas/expression.ts';
9
10
  export type { IsolationManifest } from './core/schemas/isolation-manifest.ts';
10
11
  export { ISOLATION_MANIFEST_SCHEMA_VERSION } from './core/schemas/isolation-manifest.ts';
11
12
  export type { PreflightCheck, PreflightVerdict, } from './core/schemas/preflight-verdict.ts';
12
13
  export { PREFLIGHT_VERDICT_SCHEMA_VERSION } from './core/schemas/preflight-verdict.ts';
14
+ export type { JsonValue } from './core/schemas/primitives.ts';
13
15
  export type { PrivateArtifactManifest } from './core/schemas/private-artifact-manifest.ts';
14
16
  export { PRIVATE_ARTIFACT_MANIFEST_SCHEMA_VERSION } from './core/schemas/private-artifact-manifest.ts';
15
17
  export type { Probe } from './core/schemas/probe.ts';
@@ -19,7 +21,7 @@ export type { ScoringPolicy } from './core/schemas/scoring-policy.ts';
19
21
  export { SCORING_POLICY_SCHEMA_VERSION } from './core/schemas/scoring-policy.ts';
20
22
  export type { SealedEvaluatorBrief } from './core/schemas/sealed-evaluator-brief.ts';
21
23
  export { SEALED_EVALUATOR_BRIEF_SCHEMA_VERSION } from './core/schemas/sealed-evaluator-brief.ts';
22
- export type { SealedRunRecord } from './core/schemas/sealed-run-record.ts';
24
+ export type { Observation, SealedRunRecord, } from './core/schemas/sealed-run-record.ts';
23
25
  export { SEALED_RUN_RECORD_SCHEMA_VERSION } from './core/schemas/sealed-run-record.ts';
24
26
  export type { FixtureReset, ManifestationWitness, SensitivityWitness, SensitivityWitnessLeg, WitnessChannel, WitnessInputs, } from './core/schemas/sensitivity-witness.ts';
25
- export declare const VERSION = "3.2.0";
27
+ export declare const VERSION = "3.3.0";
package/dist/index.js CHANGED
@@ -38,4 +38,4 @@ export { PROBE_SCHEMA_VERSION } from './core/schemas/probe.js';
38
38
  export { SCORING_POLICY_SCHEMA_VERSION } from './core/schemas/scoring-policy.js';
39
39
  export { SEALED_EVALUATOR_BRIEF_SCHEMA_VERSION } from './core/schemas/sealed-evaluator-brief.js';
40
40
  export { SEALED_RUN_RECORD_SCHEMA_VERSION } from './core/schemas/sealed-run-record.js';
41
- export const VERSION = '3.2.0';
41
+ export const VERSION = '3.3.0';
@@ -56,12 +56,12 @@ export declare const probeParsers: {
56
56
  }>;
57
57
  pathTemplate: import("zod").ZodString;
58
58
  channels: import("zod").ZodObject<{
59
- path: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>>;
60
- query: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>>;
59
+ path: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>>;
60
+ query: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>>;
61
61
  header: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodString>;
62
62
  body: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
63
63
  kind: import("zod").ZodLiteral<"json">;
64
- value: import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>;
64
+ value: import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>;
65
65
  }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
66
66
  kind: import("zod").ZodLiteral<"absent">;
67
67
  }, import("zod/v4/core").$strict>], "kind">;
@@ -74,12 +74,12 @@ export declare const probeParsers: {
74
74
  executable: import("zod").ZodString;
75
75
  subcommandPath: import("zod").ZodArray<import("zod").ZodString>;
76
76
  channels: import("zod").ZodObject<{
77
- argument: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>>;
78
- option: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>>;
77
+ argument: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>>;
78
+ option: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>>;
79
79
  environment: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodString>;
80
80
  stdin: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
81
81
  kind: import("zod").ZodLiteral<"json">;
82
- value: import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>;
82
+ value: import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>;
83
83
  }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
84
84
  kind: import("zod").ZodLiteral<"text">;
85
85
  value: import("zod").ZodString;
@@ -94,7 +94,7 @@ export declare const probeParsers: {
94
94
  kind: import("zod").ZodLiteral<"mcp">;
95
95
  toolName: import("zod").ZodString;
96
96
  channels: import("zod").ZodObject<{
97
- arguments: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>>;
97
+ arguments: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>>;
98
98
  }, import("zod/v4/core").$strict>;
99
99
  }, import("zod/v4/core").$strict>], "kind">;
100
100
  readonly response: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
@@ -106,7 +106,7 @@ export declare const probeParsers: {
106
106
  headers: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodString>;
107
107
  body: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
108
108
  kind: import("zod").ZodLiteral<"json">;
109
- value: import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>;
109
+ value: import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>;
110
110
  }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
111
111
  kind: import("zod").ZodLiteral<"text">;
112
112
  value: import("zod").ZodString;
@@ -121,7 +121,7 @@ export declare const probeParsers: {
121
121
  exitCode: import("zod").ZodInt;
122
122
  stdout: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
123
123
  kind: import("zod").ZodLiteral<"json">;
124
- value: import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>;
124
+ value: import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>;
125
125
  }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
126
126
  kind: import("zod").ZodLiteral<"text">;
127
127
  value: import("zod").ZodString;
@@ -130,7 +130,7 @@ export declare const probeParsers: {
130
130
  }, import("zod/v4/core").$strict>], "kind">;
131
131
  stderr: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
132
132
  kind: import("zod").ZodLiteral<"json">;
133
- value: import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>;
133
+ value: import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>;
134
134
  }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
135
135
  kind: import("zod").ZodLiteral<"text">;
136
136
  value: import("zod").ZodString;
@@ -139,7 +139,7 @@ export declare const probeParsers: {
139
139
  }, import("zod/v4/core").$strict>], "kind">;
140
140
  artifacts: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
141
141
  kind: import("zod").ZodLiteral<"json">;
142
- value: import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>;
142
+ value: import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>;
143
143
  }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
144
144
  kind: import("zod").ZodLiteral<"text">;
145
145
  value: import("zod").ZodString;
@@ -154,7 +154,7 @@ export declare const probeParsers: {
154
154
  isError: import("zod").ZodBoolean;
155
155
  result: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
156
156
  kind: import("zod").ZodLiteral<"json">;
157
- value: import("zod").ZodType<import("../core/schemas/primitives.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../core/schemas/primitives.ts").JsonValue, unknown>>;
157
+ value: import("zod").ZodType<import("../index.ts").JsonValue, unknown, import("zod/v4/core").$ZodTypeInternals<import("../index.ts").JsonValue, unknown>>;
158
158
  }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
159
159
  kind: import("zod").ZodLiteral<"text">;
160
160
  value: import("zod").ZodString;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "eval-quality",
3
- "version": "3.2.0",
3
+ "version": "3.3.0",
4
4
  "description": "Compile disciplined Behavioral Evaluation Contracts and score their ability to catch known defects.",
5
5
  "author": "Murat Ozcan",
6
6
  "license": "Apache-2.0",
@@ -101,6 +101,8 @@
101
101
  "check:corpus": "node scripts/check-dev-corpus.ts",
102
102
  "generate:worked-example": "node scripts/generate-worked-example.ts",
103
103
  "check:worked-example": "node scripts/check-worked-example.ts",
104
+ "generate:tutorials": "node scripts/generate-tutorials.ts",
105
+ "check:tutorials": "node scripts/check-tutorials.ts",
104
106
  "generate:version": "node scripts/generate-version.ts",
105
107
  "check:version": "node scripts/check-version.ts",
106
108
  "generate:lockfile-age-cache": "node scripts/generate-lockfile-age-cache.ts",
@@ -117,7 +119,7 @@
117
119
  "release:major": "gh workflow run publish.yml --ref main -f bump=major",
118
120
  "release:prepare": "node scripts/release-prepare.mjs",
119
121
  "release:publish": "gh workflow run publish.yml --ref main -f bump=none",
120
- "validate": "npm run build && npm run typecheck && npm run lint && npm run check:docs && npm run check:doc-invocations && npm run check:shareable && npm run lint:spine && npm run check:vectors && npm run check:schemas && npm run check:ad5-registry && npm run check:ad28-registry && npm run check:ad31-table && npm run check:ad33-table && npm run check:ad21-table && npm run check:layers && npm run check:lineage && npm run check:boundary && npm run check:corpus && npm run check:doc-counts && npm run check:doc-claims && npm run check:worked-example && npm run check:version && npm run check:lockfile-age && npm run check:licences && npm run test:coverage",
122
+ "validate": "npm run build && npm run typecheck && npm run lint && npm run check:docs && npm run check:doc-invocations && npm run check:shareable && npm run lint:spine && npm run check:vectors && npm run check:schemas && npm run check:ad5-registry && npm run check:ad28-registry && npm run check:ad31-table && npm run check:ad33-table && npm run check:ad21-table && npm run check:layers && npm run check:lineage && npm run check:boundary && npm run check:corpus && npm run check:doc-counts && npm run check:doc-claims && npm run check:worked-example && npm run check:tutorials && npm run check:version && npm run check:lockfile-age && npm run check:licences && npm run test:coverage",
121
123
  "prepack": "npm run clean && npm run build",
122
124
  "prepublishOnly": "node scripts/assert-publish-authorized.mjs"
123
125
  },