eval-quality 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. package/README.md +142 -67
  2. package/corpus/dev/README.md +17 -11
  3. package/corpus/dev/compile-seal-example/brief.json +1 -1
  4. package/corpus/dev/compile-seal-example/contract.json +1 -1
  5. package/corpus/dev/contracts/absent-collection-locations.json +1 -1
  6. package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
  7. package/corpus/dev/contracts/absent-success-indicator.json +1 -1
  8. package/corpus/dev/contracts/empty-channel-roles.json +1 -1
  9. package/corpus/dev/contracts/empty-collection-locations.json +1 -1
  10. package/corpus/dev/contracts/empty-request-shapes.json +1 -1
  11. package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
  12. package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
  13. package/corpus/dev/contracts/no-operation-inventory.json +1 -1
  14. package/corpus/dev/contracts/no-read-back-relation.json +1 -1
  15. package/corpus/dev/contracts/no-state-change-marker.json +1 -1
  16. package/corpus/dev/contracts/no-type-violating-step.json +1 -1
  17. package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
  18. package/corpus/dev/contracts/satisfied-declarations.json +1 -1
  19. package/corpus/dev/contracts/single-required-response-key.json +1 -1
  20. package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
  21. package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
  22. package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
  23. package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
  24. package/corpus/dev/index.json +1 -1
  25. package/dist/application/index.d.ts +2 -0
  26. package/dist/application/index.js +1 -0
  27. package/dist/application/score.d.ts +48 -0
  28. package/dist/application/score.js +186 -0
  29. package/dist/cli/arguments.d.ts +7 -3
  30. package/dist/cli/arguments.js +73 -5
  31. package/dist/cli/exit-codes.d.ts +6 -2
  32. package/dist/cli/exit-codes.js +8 -9
  33. package/dist/cli/main.js +9 -0
  34. package/dist/cli/render.d.ts +1 -1
  35. package/dist/cli/render.js +6 -4
  36. package/dist/cli/run.d.ts +13 -3
  37. package/dist/cli/run.js +136 -16
  38. package/dist/core/canonical/canonicalize.d.ts +12 -0
  39. package/dist/core/canonical/canonicalize.js +18 -4
  40. package/dist/core/compile/bindings.d.ts +70 -0
  41. package/dist/core/compile/bindings.js +317 -0
  42. package/dist/core/compile/compile.d.ts +5 -0
  43. package/dist/core/compile/compile.js +9 -0
  44. package/dist/core/compile/expression-legality.d.ts +49 -0
  45. package/dist/core/compile/expression-legality.js +105 -31
  46. package/dist/core/compile/forbidden-inputs.d.ts +8 -1
  47. package/dist/core/compile/forbidden-inputs.js +16 -3
  48. package/dist/core/compile/interface-inventory.d.ts +23 -1
  49. package/dist/core/compile/interface-inventory.js +32 -3
  50. package/dist/core/compile/reachability.d.ts +18 -1
  51. package/dist/core/compile/reachability.js +46 -1
  52. package/dist/core/compile/sensitivity-witness.d.ts +1 -1
  53. package/dist/core/compile/sensitivity-witness.js +1 -1
  54. package/dist/core/emit/emit.d.ts +9 -0
  55. package/dist/core/emit/emit.js +214 -0
  56. package/dist/core/emit/private-artifact-digest.d.ts +11 -0
  57. package/dist/core/emit/private-artifact-digest.js +31 -0
  58. package/dist/core/evaluate/evidence-resolution.d.ts +11 -1
  59. package/dist/core/evaluate/evidence-resolution.js +4 -1
  60. package/dist/core/failure-codes.d.ts +2 -2
  61. package/dist/core/failure-codes.js +4 -2
  62. package/dist/core/ingest/conditions.d.ts +201 -0
  63. package/dist/core/ingest/conditions.js +48 -0
  64. package/dist/core/ingest/index.d.ts +13 -0
  65. package/dist/core/ingest/index.js +11 -0
  66. package/dist/core/ingest/ingest.d.ts +68 -0
  67. package/dist/core/ingest/ingest.js +310 -0
  68. package/dist/core/lineage/stage-table.d.ts +21 -4
  69. package/dist/core/lineage/stage-table.js +35 -3
  70. package/dist/core/preflight/witness-evidence.js +10 -0
  71. package/dist/core/schemas/artifact.d.ts +385 -14
  72. package/dist/core/schemas/constraint-ledger.js +40 -0
  73. package/dist/core/schemas/defect-signature.d.ts +273 -0
  74. package/dist/core/schemas/defect-signature.js +127 -0
  75. package/dist/core/schemas/eval-contract.d.ts +33 -0
  76. package/dist/core/schemas/eval-contract.js +14 -0
  77. package/dist/core/schemas/evidence-artifact.d.ts +99 -19
  78. package/dist/core/schemas/evidence-artifact.js +64 -8
  79. package/dist/core/schemas/plan.d.ts +112 -0
  80. package/dist/core/schemas/plan.js +67 -2
  81. package/dist/core/schemas/probe-qualification.d.ts +168 -0
  82. package/dist/core/schemas/probe-qualification.js +106 -0
  83. package/dist/core/schemas/probe.d.ts +312 -0
  84. package/dist/core/schemas/probe.js +6 -2
  85. package/dist/core/schemas/scoring-policy.d.ts +1 -0
  86. package/dist/core/schemas/scoring-policy.js +6 -1
  87. package/dist/core/schemas/sealed-evaluator-brief.d.ts +1 -0
  88. package/dist/core/schemas/sealed-evaluator-brief.js +4 -1
  89. package/dist/core/schemas/sealed-run-record.d.ts +30 -3
  90. package/dist/core/schemas/sealed-run-record.js +37 -5
  91. package/dist/core/score/binding-order.d.ts +31 -0
  92. package/dist/core/score/binding-order.js +88 -0
  93. package/dist/core/score/bindings.d.ts +93 -0
  94. package/dist/core/score/bindings.js +297 -0
  95. package/dist/core/score/ladder-table.d.ts +23 -0
  96. package/dist/core/score/ladder-table.js +109 -0
  97. package/dist/core/score/ladder.d.ts +229 -0
  98. package/dist/core/score/ladder.js +464 -0
  99. package/dist/core/score/mode-agreement.d.ts +30 -0
  100. package/dist/core/score/mode-agreement.js +16 -0
  101. package/dist/core/score/outcome-table.d.ts +17 -0
  102. package/dist/core/score/outcome-table.js +172 -0
  103. package/dist/core/score/outcome.d.ts +466 -0
  104. package/dist/core/score/outcome.js +562 -0
  105. package/dist/core/score/qualification.d.ts +83 -0
  106. package/dist/core/score/qualification.js +615 -0
  107. package/dist/core/score/quotation.d.ts +79 -0
  108. package/dist/core/score/quotation.js +201 -0
  109. package/dist/core/score/reduce-trials.d.ts +80 -0
  110. package/dist/core/score/reduce-trials.js +90 -0
  111. package/dist/core/score/score.d.ts +72 -0
  112. package/dist/core/score/score.js +581 -0
  113. package/dist/core/score/selection.d.ts +80 -0
  114. package/dist/core/score/selection.js +98 -0
  115. package/dist/core/score/strength.d.ts +49 -0
  116. package/dist/core/score/strength.js +193 -0
  117. package/dist/core/score/witness.d.ts +117 -0
  118. package/dist/core/score/witness.js +315 -0
  119. package/dist/core/seal/derived-reference.d.ts +1 -1
  120. package/dist/core/seal/derived-reference.js +236 -28
  121. package/dist/core/seal/seal.js +8 -3
  122. package/dist/core/stage-contracts.d.ts +62 -2
  123. package/dist/index.d.ts +1 -1
  124. package/dist/index.js +1 -1
  125. package/dist/testing/index.d.ts +2 -0
  126. package/dist/testing/index.js +1 -0
  127. package/package.json +12 -3
  128. package/schemas/eval-contract.schema.json +100 -2
  129. package/schemas/evidence-artifact.schema.json +96 -9
  130. package/schemas/probe.schema.json +508 -3
  131. package/schemas/scoring-policy.schema.json +8 -1
  132. package/schemas/sealed-evaluator-brief.schema.json +11 -1
  133. package/schemas/sealed-run-record.schema.json +19 -2
@@ -4,24 +4,68 @@
4
4
  * because every command is non-interactive. Paths are carried through exactly
5
5
  * as given and resolved by the caller.
6
6
  */
7
- export const COMMANDS = ['compile', 'seal', 'preflight'];
7
+ export const COMMANDS = [
8
+ 'compile',
9
+ 'seal',
10
+ 'preflight',
11
+ 'score',
12
+ ];
8
13
  /** The input flags each command takes, and which of them it requires. */
9
14
  const INPUT_KEYS = {
10
15
  compile: ['in'],
11
16
  seal: ['in'],
12
17
  preflight: ['contract', 'probes', 'observations'],
18
+ score: [
19
+ 'record',
20
+ 'isolation-manifest',
21
+ 'evaluator-configuration',
22
+ 'contract',
23
+ 'probe',
24
+ 'preflight-verdict',
25
+ 'policy',
26
+ 'private-manifest',
27
+ ],
13
28
  };
14
- /** `--strict-inputs` is AD-4's compile mode; `preflight` has no compile step. */
29
+ /**
30
+ * Keys whose absence is legal even though the command declares them: `in`
31
+ * falls back to stdin, and `score`'s three name artifacts that are genuinely
32
+ * optional at the domain level (AD-16's absent isolation manifest, an absent
33
+ * evaluator configuration, and an optional private-manifest no stage row
34
+ * requires).
35
+ */
36
+ const OPTIONAL_INPUT_KEYS = new Set([
37
+ 'in',
38
+ 'isolation-manifest',
39
+ 'evaluator-configuration',
40
+ 'private-manifest',
41
+ ]);
42
+ /** `--strict-inputs` is AD-4's compile mode; `preflight` and `score` have no compile step. */
15
43
  const TAKES_STRICT_INPUTS = {
16
44
  compile: true,
17
45
  seal: true,
18
46
  preflight: false,
47
+ score: false,
19
48
  };
20
- /** `--run-id` names the run a verdict is minted for, which only `preflight` does. */
49
+ /** `--run-id` names the run a verdict is minted for, which only `preflight` does: `score` reads its run identifier off `--record` instead. */
21
50
  const TAKES_RUN_ID = {
22
51
  compile: false,
23
52
  seal: false,
24
53
  preflight: true,
54
+ score: false,
55
+ };
56
+ /** `--corpus-digest` supplies `score`'s one caller-attested scoring-version input with no artifact source; required whenever it applies, the same posture `--run-id` takes for `preflight`. */
57
+ const TAKES_CORPUS_DIGEST = {
58
+ compile: false,
59
+ seal: false,
60
+ preflight: false,
61
+ score: true,
62
+ };
63
+ /** `--corpus-root` names the directory `score`'s corpus-port adapter resolves a private reference under; always optional at this layer, since whether one is actually needed depends on the artifacts' own content, not the argument grammar. */
64
+ const TAKES_CORPUS_ROOT = {
65
+ compile: false,
66
+ seal: false,
67
+ preflight: false,
68
+ score: true,
25
69
  };
26
70
  const STDIN = '-';
27
71
  const isCommand = (token) => COMMANDS.includes(token);
@@ -63,10 +107,17 @@ function parseCommand(command, rest) {
63
107
  valueFlags.set('--out', 'out');
64
108
  if (TAKES_RUN_ID[command])
65
109
  valueFlags.set('--run-id', 'run-id');
110
+ if (TAKES_CORPUS_DIGEST[command]) {
111
+ valueFlags.set('--corpus-digest', 'corpus-digest');
112
+ }
113
+ if (TAKES_CORPUS_ROOT[command])
114
+ valueFlags.set('--corpus-root', 'corpus-root');
66
115
  const inputs = {};
67
116
  const seen = new Map();
68
117
  let out = null;
69
118
  let runId = null;
119
+ let corpusDigest = null;
120
+ let corpusRoot = null;
70
121
  let strictInputs = true;
71
122
  let strict = false;
72
123
  const takeValue = (flag, inline, index) => {
@@ -114,6 +165,10 @@ function parseCommand(command, rest) {
114
165
  out = taken.value;
115
166
  else if (target === 'run-id')
116
167
  runId = taken.value;
168
+ else if (target === 'corpus-digest')
169
+ corpusDigest = taken.value;
170
+ else if (target === 'corpus-root')
171
+ corpusRoot = taken.value;
117
172
  else
118
173
  inputs[target] = taken.value;
119
174
  index = taken.next;
@@ -140,10 +195,13 @@ function parseCommand(command, rest) {
140
195
  return usageError(`unknown flag "${token}" for ${command}`);
141
196
  }
142
197
  const missing = INPUT_KEYS[command]
143
- .filter((key) => key !== 'in' && inputs[key] === undefined)
198
+ .filter((key) => !OPTIONAL_INPUT_KEYS.has(key) && inputs[key] === undefined)
144
199
  .map((key) => `--${key}`);
145
200
  if (TAKES_RUN_ID[command] && runId === null)
146
201
  missing.push('--run-id');
202
+ if (TAKES_CORPUS_DIGEST[command] && corpusDigest === null) {
203
+ missing.push('--corpus-digest');
204
+ }
147
205
  // `PreflightVerdict.runId` is `z.string().min(1)`, which a run of spaces
148
206
  // satisfies. It then renders as blank in every diagnostic line and names
149
207
  // nothing a reader can correlate, so the argument surface refuses it here
@@ -163,5 +221,15 @@ function parseCommand(command, rest) {
163
221
  : `${flags.join(', ')} all name`;
164
222
  return usageError(`only one input may read stdin, but ${named} "-"`);
165
223
  }
166
- return { kind: 'run', command, inputs, out, runId, strictInputs, strict };
224
+ return {
225
+ kind: 'run',
226
+ command,
227
+ inputs,
228
+ out,
229
+ runId,
230
+ corpusDigest,
231
+ corpusRoot,
232
+ strictInputs,
233
+ strict,
234
+ };
167
235
  }
@@ -22,8 +22,12 @@ export type CommandOutcome = {
22
22
  readonly passed: boolean;
23
23
  } | {
24
24
  readonly kind: 'verdict';
25
- readonly verdict: Verdict;
26
- readonly evidenceConditionsOnly: boolean;
25
+ /** `null` is AD-21's Invalid rung: `LadderResolution.verdict` on that rung, never a `Verdict`. */
26
+ readonly verdict: Verdict | null;
27
+ /** `LadderResolution.exitCode`, read directly rather than recomputed from `verdict`. */
28
+ readonly exitCode: number;
29
+ /** `LadderResolution.strictPromotable`, carried through unchanged: never inverted into a locally-invented `evidenceConditionsOnly`. */
30
+ readonly strictPromotable: boolean;
27
31
  } | {
28
32
  readonly kind: 'structural-failure';
29
33
  } | {
@@ -7,17 +7,16 @@ export const EXIT_FAULT = 5;
7
7
  /** sysexits.h EX_USAGE. Outside the verdict range and outside AD-21's codes. */
8
8
  export const EXIT_USAGE = 64;
9
9
  function verdictExit(outcome, strict) {
10
- if (outcome.verdict === 'FAIL')
11
- return EXIT_FAIL;
12
- // `--strict` promotes CONCERNS to one, "except a CONCERNS whose only firing
13
- // conditions are evidence conditions, which `--strict` never promotes"
14
- // (AD-21). A thinner measurement is not a claim about the system.
15
- if (outcome.verdict === 'CONCERNS' &&
16
- strict &&
17
- !outcome.evidenceConditionsOnly) {
10
+ // `--strict` promotes a CONCERNS to one, "except a CONCERNS whose only
11
+ // firing conditions are evidence conditions, which `--strict` never
12
+ // promotes" (AD-21). `LADDER_EXIT_CODES.CONCERNS` is already zero, so this
13
+ // promotion cannot be read out of `outcome.exitCode` itself: it is applied
14
+ // on top of it. Every other rung -- PASS, WAIVED, FAIL, and Invalid's
15
+ // `null` -- returns its own `exitCode` unchanged.
16
+ if (outcome.verdict === 'CONCERNS' && strict && outcome.strictPromotable) {
18
17
  return EXIT_CONCERNS_PROMOTED;
19
18
  }
20
- return EXIT_OK;
19
+ return outcome.exitCode;
21
20
  }
22
21
  export function exitCodeFor(outcome, options) {
23
22
  switch (outcome.kind) {
package/dist/cli/main.js CHANGED
@@ -11,6 +11,7 @@
11
11
  import { mkdir, readFile, stat, writeFile } from 'node:fs/promises';
12
12
  import { dirname, join, resolve } from 'node:path';
13
13
  import process from 'node:process';
14
+ import { createLocalCorpusAdapter } from '../adapters/local-corpus-adapter.js';
14
15
  import { parseArguments } from './arguments.js';
15
16
  import { exitCodeFor } from './exit-codes.js';
16
17
  import { renderError, renderUsage } from './render.js';
@@ -88,6 +89,14 @@ function environmentOf(version) {
88
89
  return false;
89
90
  }
90
91
  },
92
+ // `score`'s corpus-port wiring: the one reference adapter, over a root
93
+ // known only once `--corpus-root` is parsed, which is why this is a
94
+ // factory rather than a constructed port.
95
+ corpusPort: (root) => createLocalCorpusAdapter({ root }),
96
+ // One controller per process invocation, never aborted here: no flag
97
+ // this binary parses requests cancellation, so the signal only ever
98
+ // answers "was this already aborted before the call" as false.
99
+ signal: new AbortController().signal,
91
100
  version,
92
101
  };
93
102
  }
@@ -21,6 +21,6 @@ export declare function renderError(error: unknown): string;
21
21
  * AD-21's seven exit codes, one line each. The `--help` output and the README
22
22
  * table are this text, so the two cannot drift.
23
23
  */
24
- export declare const EXIT_CODE_TABLE = "Exit codes (AD-21):\n 0 success, and every verdict other than FAIL or a promoted CONCERNS\n 1 CONCERNS promoted by --strict\n 2 FAIL\n 3 invalid: a pre-flight verdict that did not pass\n 4 structural failure\n 5 runtime fault\n 64 usage error\n\n 1 and 2 report a scored verdict. Scoring ships in a later release, so no\n command here reaches either yet, and --strict changes no code this binary\n produces.";
24
+ export declare const EXIT_CODE_TABLE = "Exit codes (AD-21):\n 0 success, and every verdict other than FAIL or a promoted CONCERNS\n 1 CONCERNS promoted by --strict\n 2 FAIL\n 3 invalid: a failed pre-flight, or any other AD-21 invalidating condition\n 4 structural failure\n 5 runtime fault\n 64 usage error\n\n --strict never promotes a CONCERNS whose firing conditions are all evidence\n conditions: those conditions report that the measurement fell short of the\n policy. 1 and 2 report a verdict the score command's ladder resolved; every\n other invalidating condition behind 3 is reachable there too, alongside the\n failed pre-flight the preflight command itself reports.";
25
25
  /** `eval-quality: usage: <message>` */
26
26
  export declare function renderUsage(message: string): string;
@@ -39,14 +39,16 @@ export const EXIT_CODE_TABLE = `Exit codes (AD-21):
39
39
  0 success, and every verdict other than FAIL or a promoted CONCERNS
40
40
  1 CONCERNS promoted by --strict
41
41
  2 FAIL
42
- 3 invalid: a pre-flight verdict that did not pass
42
+ 3 invalid: a failed pre-flight, or any other AD-21 invalidating condition
43
43
  4 structural failure
44
44
  5 runtime fault
45
45
  64 usage error
46
46
 
47
- 1 and 2 report a scored verdict. Scoring ships in a later release, so no
48
- command here reaches either yet, and --strict changes no code this binary
49
- produces.`;
47
+ --strict never promotes a CONCERNS whose firing conditions are all evidence
48
+ conditions: those conditions report that the measurement fell short of the
49
+ policy. 1 and 2 report a verdict the score command's ladder resolved; every
50
+ other invalidating condition behind 3 is reachable there too, alongside the
51
+ failed pre-flight the preflight command itself reports.`;
50
52
  /** `eval-quality: usage: <message>` */
51
53
  export function renderUsage(message) {
52
54
  return `${PREFIX}: usage: ${message}`;
package/dist/cli/run.d.ts CHANGED
@@ -1,11 +1,11 @@
1
1
  /**
2
- * The three commands. Each one reads its inputs, makes exactly one call into
2
+ * The four commands. Each one reads its inputs, makes exactly one call into
3
3
  * `application/`, serializes what came back, and returns the outcome the exit
4
4
  * code is derived from. Every effect is a member of `RunEnvironment`, so this
5
5
  * module touches no stream, no `process`, and no Node builtin, and its tests
6
6
  * run in memory.
7
7
  */
8
- import { compile, preflightFromObservations, seal } from '../application/index.ts';
8
+ import { compile, preflightFromObservations, type RunScoreOptions, runScore, seal } from '../application/index.ts';
9
9
  import type { Command, ParsedInvocation } from './arguments.ts';
10
10
  import type { CommandOutcome } from './exit-codes.ts';
11
11
  export type RunEnvironment = {
@@ -21,10 +21,19 @@ export type RunEnvironment = {
21
21
  * spellings that no amount of folding brings together.
22
22
  */
23
23
  readonly sameFile: (left: string, right: string) => Promise<boolean>;
24
+ /**
25
+ * `score`'s corpus-port factory over a caller-named root directory: a
26
+ * constructor, not a constructed value, since the root is only known once
27
+ * `--corpus-root` is parsed. `cli/` may not import `ports/` directly, so
28
+ * the port's own type is read off `RunScoreOptions` instead.
29
+ */
30
+ readonly corpusPort: (root: string) => NonNullable<RunScoreOptions['port']>;
31
+ /** One per process invocation, the same shape `RunPreflightOptions.signal` already declares. */
32
+ readonly signal: AbortSignal;
24
33
  readonly version: string;
25
34
  };
26
35
  /**
27
- * The three orchestration calls, behind one object so a test can count them.
36
+ * The four orchestration calls, behind one object so a test can count them.
28
37
  * An input/output count says nothing about how many calls into `application/`
29
38
  * happened, which is the property AD-14 constrains.
30
39
  */
@@ -32,6 +41,7 @@ export type ApplicationFacade = {
32
41
  readonly compile: typeof compile;
33
42
  readonly seal: typeof seal;
34
43
  readonly preflightFromObservations: typeof preflightFromObservations;
44
+ readonly runScore: typeof runScore;
35
45
  };
36
46
  export declare const APPLICATION: ApplicationFacade;
37
47
  export type RunResult = {
package/dist/cli/run.js CHANGED
@@ -1,16 +1,17 @@
1
1
  /**
2
- * The three commands. Each one reads its inputs, makes exactly one call into
2
+ * The four commands. Each one reads its inputs, makes exactly one call into
3
3
  * `application/`, serializes what came back, and returns the outcome the exit
4
4
  * code is derived from. Every effect is a member of `RunEnvironment`, so this
5
5
  * module touches no stream, no `process`, and no Node builtin, and its tests
6
6
  * run in memory.
7
7
  */
8
- import { compile, preflightFromObservations, RuntimeFault, StructuralFailure, seal, } from '../application/index.js';
8
+ import { compile, preflightFromObservations, RuntimeFault, runScore, StructuralFailure, seal, } from '../application/index.js';
9
9
  import { EXIT_CODE_TABLE, renderArtifact, renderDiagnostic, renderError, renderUsage, } from './render.js';
10
10
  export const APPLICATION = {
11
11
  compile,
12
12
  seal,
13
13
  preflightFromObservations,
14
+ runScore,
14
15
  };
15
16
  /** The artifact each command emits: its schema name and its file name. */
16
17
  const EMITTED = {
@@ -20,6 +21,7 @@ const EMITTED = {
20
21
  kind: 'sealed-evaluator-brief',
21
22
  },
22
23
  preflight: { artifactPath: 'PreflightVerdict', kind: 'preflight-verdict' },
24
+ score: { artifactPath: 'EvidenceArtifact', kind: 'evidence-artifact' },
23
25
  };
24
26
  /** The schema an input key deserializes into, for the parse fault's path. */
25
27
  const INPUT_ARTIFACT_PATH = {
@@ -27,6 +29,13 @@ const INPUT_ARTIFACT_PATH = {
27
29
  contract: 'EvalContract',
28
30
  probes: 'Probe',
29
31
  observations: 'ProbeObservation',
32
+ record: 'SealedRunRecord',
33
+ 'isolation-manifest': 'IsolationManifest',
34
+ 'evaluator-configuration': 'EvaluatorConfiguration',
35
+ probe: 'Probe',
36
+ 'preflight-verdict': 'PreflightVerdict',
37
+ policy: 'ScoringPolicy',
38
+ 'private-manifest': 'PrivateArtifactManifest',
30
39
  };
31
40
  const USAGE = `Usage:
32
41
  eval-quality compile [--in <path>] [--out <target>]
@@ -35,6 +44,12 @@ const USAGE = `Usage:
35
44
  [--strict-inputs | --no-strict-inputs] [--strict]
36
45
  eval-quality preflight --contract <path> --probes <path> --observations <path>
37
46
  --run-id <id> [--out <target>] [--strict]
47
+ eval-quality score --record <path> --contract <path> --probe <path>
48
+ --preflight-verdict <path> --policy <path>
49
+ --corpus-digest <digest>
50
+ [--isolation-manifest <path>] [--evaluator-configuration <path>]
51
+ [--private-manifest <path>] [--corpus-root <dir>]
52
+ [--out <target>] [--strict]
38
53
  eval-quality --help | -h | help [<command>]
39
54
  eval-quality --version | -V`;
40
55
  const COMMAND_USAGE = {
@@ -66,10 +81,36 @@ const COMMAND_USAGE = {
66
81
  --run-id <id> the run identifier the verdict is minted for
67
82
  --out <target> a .json file path, or a directory taking preflight-verdict.json
68
83
  --strict promote CONCERNS to exit 1`,
84
+ score: `Usage:
85
+ eval-quality score --record <path> --contract <path> --probe <path>
86
+ --preflight-verdict <path> --policy <path>
87
+ --corpus-digest <digest>
88
+ [--isolation-manifest <path>] [--evaluator-configuration <path>]
89
+ [--private-manifest <path>] [--corpus-root <dir>]
90
+ [--out <target>] [--strict]
91
+
92
+ --record <path> the sealed run record to ingest
93
+ --contract <path> the compiled contract to score against
94
+ --probe <path> the probe the record was run against
95
+ --preflight-verdict <path> the pre-flight verdict, also the source of the AD-11 fixture digest
96
+ --policy <path> the scoring policy
97
+ --corpus-digest <digest> AD-11's caller-attested corpus digest; no artifact carries it
98
+ --isolation-manifest <path> the isolation manifest; absent invalidates the run under AD-16
99
+ --evaluator-configuration <path> the evaluator configuration; absent invalidates the run
100
+ --private-manifest <path> each entry's digest is checked against its resolved bytes
101
+ --corpus-root <dir> the directory a private reference resolves under; required only
102
+ when --private-manifest or a private-storage isolation-manifest
103
+ reference is present
104
+ --out <target> a .json file path, or a directory taking evidence-artifact.json
105
+ --strict promote CONCERNS to exit 1`,
69
106
  };
70
107
  const IO_RULES = `Inputs and outputs:
71
- An input flag left out reads stdin, and "-" names stdin explicitly; at most
72
- one input may be "-". Without --out the artifact goes to stdout. An --out
108
+ --in is the only input that falls back to stdin: compile and seal read it
109
+ when --in is left out. "-" names stdin explicitly on any input, and at most
110
+ one input may be "-" per invocation. compile and seal each take one input;
111
+ preflight takes three, all required; score takes eight, three of them
112
+ optional (--isolation-manifest, --evaluator-configuration, and
113
+ --private-manifest). Without --out the artifact goes to stdout. An --out
73
114
  ending in .json is a file path; anything else is a directory taking
74
115
  <target>/<kind>.json. Diagnostics and errors go to stderr.`;
75
116
  export function helpText(command) {
@@ -155,19 +196,32 @@ async function runCommand(invocation, environment, application) {
155
196
  }
156
197
  }
157
198
  try {
158
- if (command === 'preflight') {
159
- const verdict = await runPreflightCommand(invocation, environment, application, target);
160
- return { outcome: { kind: 'preflight', passed: verdict.passed } };
199
+ // Exhaustive over `Command`, not a fallthrough: a fifth command with no
200
+ // case here is a compile error ("not all code paths return a value"),
201
+ // never a silent seal. `compile` and `seal` share one case body, and the
202
+ // ternary inside it is itself exhaustive over the narrowed two-member
203
+ // union the shared case leaves `command` as.
204
+ switch (command) {
205
+ case 'preflight': {
206
+ const verdict = await runPreflightCommand(invocation, environment, application, target);
207
+ return { outcome: { kind: 'preflight', passed: verdict.passed } };
208
+ }
209
+ case 'score':
210
+ return await runScoreCommand(invocation, environment, application, target);
211
+ case 'compile':
212
+ case 'seal': {
213
+ const input = await readJson(environment, 'in', inputs.in);
214
+ const options = { strict: strictInputs };
215
+ const artifact = command === 'compile'
216
+ ? application.compile(input, options)
217
+ : application.seal(input, options);
218
+ // No diagnostic on success: neither command carries a run
219
+ // identifier, so any line either wrote would name no run and no
220
+ // stage.
221
+ await emitArtifact(environment, artifact, command, target);
222
+ return { outcome: { kind: 'artifact' } };
223
+ }
161
224
  }
162
- const input = await readJson(environment, 'in', inputs.in);
163
- const options = { strict: strictInputs };
164
- const artifact = command === 'compile'
165
- ? application.compile(input, options)
166
- : application.seal(input, options);
167
- // No diagnostic on success: neither command carries a run identifier,
168
- // so any line either wrote would name no run and no stage.
169
- await emitArtifact(environment, artifact, command, target);
170
- return { outcome: { kind: 'artifact' } };
171
225
  }
172
226
  catch (error) {
173
227
  if (error instanceof StructuralFailure) {
@@ -201,3 +255,69 @@ async function runPreflightCommand(invocation, environment, application, target)
201
255
  await emitArtifact(environment, verdict, 'preflight', target);
202
256
  return verdict;
203
257
  }
258
+ /** `undefined` means the optional flag was not given; unlike `'in'`, none of `score`'s three optional inputs falls back to stdin. */
259
+ async function readOptionalJson(environment, key, value) {
260
+ return value === undefined ? null : await readJson(environment, key, value);
261
+ }
262
+ /**
263
+ * Whether `score` needs a `CorpusPort` at all: a `--private-manifest` with at
264
+ * least one entry, or a private-storage `isolationManifestArtifact`. Read off
265
+ * unvalidated JSON with optional chaining throughout -- a malformed shape
266
+ * here simply reads `undefined` and falls through to `application.runScore`'s
267
+ * own real parse, which raises the accurate `schema-parse-failure` rather
268
+ * than this pre-check inventing one.
269
+ */
270
+ function needsCorpusPort(record, privateManifest) {
271
+ const manifest = privateManifest;
272
+ if (manifest !== null && (manifest.entries?.length ?? 0) > 0)
273
+ return true;
274
+ const typedRecord = record;
275
+ return typedRecord?.isolationManifestArtifact?.storage === 'private';
276
+ }
277
+ /** `CommandOutcome`'s `'verdict'` kind, read straight off `LadderResolution`: no inversion, no recomputation. */
278
+ function scoreOutcomeOf(ladder) {
279
+ return {
280
+ kind: 'verdict',
281
+ verdict: ladder.verdict,
282
+ exitCode: ladder.exitCode,
283
+ strictPromotable: ladder.strictPromotable,
284
+ };
285
+ }
286
+ async function runScoreCommand(invocation, environment, application, target) {
287
+ const { inputs, corpusDigest, corpusRoot } = invocation;
288
+ const record = (await readJson(environment, 'record', inputs.record));
289
+ const manifest = (await readOptionalJson(environment, 'isolation-manifest', inputs['isolation-manifest']));
290
+ const configuration = (await readOptionalJson(environment, 'evaluator-configuration', inputs['evaluator-configuration']));
291
+ const contract = (await readJson(environment, 'contract', inputs.contract));
292
+ const probe = (await readJson(environment, 'probe', inputs.probe));
293
+ const preflightVerdict = (await readJson(environment, 'preflight-verdict', inputs['preflight-verdict']));
294
+ const policy = (await readJson(environment, 'policy', inputs.policy));
295
+ const privateManifest = (await readOptionalJson(environment, 'private-manifest', inputs['private-manifest']));
296
+ // A private reference with no `--corpus-root` to resolve it under is a
297
+ // usage error naming the missing flag, not a silently skipped check.
298
+ // `application.runScore` never returns `usage-error` itself (that
299
+ // vocabulary is `cli/`'s alone), so this is checked here, before the call.
300
+ if (corpusRoot === null && needsCorpusPort(record, privateManifest)) {
301
+ environment.writeDiagnostic(renderUsage('--corpus-root is required to resolve a --private-manifest entry or a private-storage isolationManifestArtifact reference'));
302
+ return { outcome: { kind: 'usage-error' } };
303
+ }
304
+ const result = await application.runScore({
305
+ record,
306
+ manifest,
307
+ configuration,
308
+ contract,
309
+ probe,
310
+ preflightVerdict,
311
+ policy,
312
+ privateManifest,
313
+ // The parser requires `--corpus-digest` on this command, so it is
314
+ // never null.
315
+ corpusDigest: corpusDigest ?? '',
316
+ port: corpusRoot === null ? undefined : environment.corpusPort(corpusRoot),
317
+ signal: environment.signal,
318
+ });
319
+ if (result.artifact !== null) {
320
+ await emitArtifact(environment, result.artifact, 'score', target);
321
+ }
322
+ return { outcome: scoreOutcomeOf(result.ladder) };
323
+ }
@@ -1 +1,13 @@
1
1
  export declare function canonicalize(value: unknown, artifactPath: string): Uint8Array;
2
+ /**
3
+ * The RFC 8785 string form, exported because AD-40's quotation audit projects a
4
+ * structured evidence channel to text and asks for the quoted evidence as an
5
+ * exact substring of it. `canonicalize` returns a `Uint8Array` and `core/` may
6
+ * import no Node builtin, so decoding those bytes back is not available; the
7
+ * string is what already exists one call in.
8
+ *
9
+ * Two parameters, the same two `canonicalize` takes. The traversal state is
10
+ * seeded here and reachable nowhere else, so no caller can hand in a primed
11
+ * ancestor set or a depth that starts the AD-36 nesting guard part-spent.
12
+ */
13
+ export declare function serialize(value: unknown, artifactPath: string): string;
@@ -11,12 +11,26 @@ import { assertDomainNumber, assertDomainString, MAX_NESTING_DEPTH, } from './va
11
11
  // lying Proxy could answer the emit-time read differently than the
12
12
  // validation read.
13
13
  export function canonicalize(value, artifactPath) {
14
- return new TextEncoder().encode(serialize(value, artifactPath, '$', new Set(), 0));
14
+ return new TextEncoder().encode(serialize(value, artifactPath));
15
15
  }
16
16
  const fault = (artifactPath, detail) => {
17
17
  throw new RuntimeFault('non-canonicalizable-value', artifactPath, detail);
18
18
  };
19
- function serialize(value, artifactPath, location, ancestors, depth) {
19
+ /**
20
+ * The RFC 8785 string form, exported because AD-40's quotation audit projects a
21
+ * structured evidence channel to text and asks for the quoted evidence as an
22
+ * exact substring of it. `canonicalize` returns a `Uint8Array` and `core/` may
23
+ * import no Node builtin, so decoding those bytes back is not available; the
24
+ * string is what already exists one call in.
25
+ *
26
+ * Two parameters, the same two `canonicalize` takes. The traversal state is
27
+ * seeded here and reachable nowhere else, so no caller can hand in a primed
28
+ * ancestor set or a depth that starts the AD-36 nesting guard part-spent.
29
+ */
30
+ export function serialize(value, artifactPath) {
31
+ return serializeNode(value, artifactPath, '$', new Set(), 0);
32
+ }
33
+ function serializeNode(value, artifactPath, location, ancestors, depth) {
20
34
  if (value === null)
21
35
  return 'null';
22
36
  switch (typeof value) {
@@ -75,7 +89,7 @@ function serialize(value, artifactPath, location, ancestors, depth) {
75
89
  fault(artifactPath, `array with holes or non-index properties at ${location}`);
76
90
  }
77
91
  rendered = `[${elements
78
- .map((element, index) => serialize(element, artifactPath, `${location}[${index}]`, ancestors, depth + 1))
92
+ .map((element, index) => serializeNode(element, artifactPath, `${location}[${index}]`, ancestors, depth + 1))
79
93
  .join(',')}]`;
80
94
  }
81
95
  else {
@@ -105,7 +119,7 @@ function serialize(value, artifactPath, location, ancestors, depth) {
105
119
  // never localeCompare, never Intl, never .normalize().
106
120
  properties.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
107
121
  rendered = `{${properties
108
- .map(([key, entry]) => `${JSON.stringify(key)}:${serialize(entry, artifactPath, `${location}.${key}`, ancestors, depth + 1)}`)
122
+ .map(([key, entry]) => `${JSON.stringify(key)}:${serializeNode(entry, artifactPath, `${location}.${key}`, ancestors, depth + 1)}`)
109
123
  .join(',')}}`;
110
124
  }
111
125
  ancestors.delete(object);
@@ -0,0 +1,70 @@
1
+ import type { EvalContract } from '../schemas/eval-contract.ts';
2
+ import type { InteractionStep } from '../schemas/plan.ts';
3
+ import { type TransportChannelName } from '../schemas/pointer.ts';
4
+ import { type EvidenceTarget } from '../seal/plan-index.ts';
5
+ /** One `{ captured }` binding, resolved to the pointer target it addresses. */
6
+ export type CapturedBinding = {
7
+ readonly transportChannel: TransportChannelName;
8
+ readonly key: string;
9
+ readonly pointer: string;
10
+ readonly target: EvidenceTarget;
11
+ };
12
+ /**
13
+ * Every captured binding one step declares, in fixed transport-channel order
14
+ * then by key name, so which binding a check reports never depends on a
15
+ * caller-keyed map's insertion order. Exported because `score/binding-order.ts`
16
+ * and `score/bindings.ts` need the same reading of which bindings are captures.
17
+ */
18
+ export declare function capturedBindings(step: InteractionStep): readonly CapturedBinding[];
19
+ /**
20
+ * `binding-cycle`: a cycle over the union of the capture edges and AD-39's
21
+ * `after` edges that contains at least one capture edge.
22
+ *
23
+ * Both edge kinds assert that this step's observation comes after that one, so
24
+ * a capture B to A together with `A.after = B` is an unsatisfiable ordering
25
+ * while each graph alone stays acyclic. A cycle made only of `after` edges is
26
+ * left to `checkNestedTemporalClause`, which already catches every one of them,
27
+ * so requiring a capture edge in the cycle keeps the two codes disjoint.
28
+ *
29
+ * Decided by strongly connected components rather than by a depth-first walk
30
+ * that inspects each back edge. A cycle lies entirely inside one component, and
31
+ * a capture edge whose endpoints share a component is closed by a path back
32
+ * through that component, so "some cycle contains a capture edge" is exactly
33
+ * "some capture edge has both endpoints in one component". A self-capture is
34
+ * the singleton case and needs no rule of its own. The depth-first form was
35
+ * written first and rejected: settling a node after rejecting a pure-`after`
36
+ * back edge can hide a mixed cycle a later root would reach, and whether it
37
+ * does depends on the order edges happen to be enumerated in, which is an
38
+ * invariant nothing states.
39
+ *
40
+ * Nodes are the plan's declared step ids; an edge whose target no step declares
41
+ * is dropped, since a dangling reference is not a cycle and AD-39 makes a
42
+ * dangling `after` permissive besides. The binding reported is the first, in
43
+ * the plan's own declaration order, whose endpoints share a component.
44
+ */
45
+ export declare function checkBindingCycle(contract: EvalContract): void;
46
+ /**
47
+ * `captured-channel-undeclared`: a captured pointer naming any AD-26 channel
48
+ * but `response-body`. `call-inputs` addresses a step's own request,
49
+ * `stdout`, `stderr`, and `exit-code` are process channels no operation
50
+ * surviving `unsupported-interface-kind` produces, and `response-headers` and
51
+ * `response-status` have no declared structure to give a captured value a
52
+ * type.
53
+ */
54
+ export declare function checkCapturedChannel(contract: EvalContract): void;
55
+ /**
56
+ * `unreachable-check-evidence` over the captured-pointer residue: an
57
+ * unresolvable step or operation, an undeclared body key, a tail that is not
58
+ * exactly one segment, an array index, an absent, indeterminate, or non-scalar
59
+ * declared type, and a declared type that does not equal the bound
60
+ * parameter's.
61
+ *
62
+ * A capture naming another channel is reported here only for the two conditions
63
+ * that are decidable without a response descriptor: an unresolvable step and an
64
+ * unresolvable operation. Everything else about such a pointer is
65
+ * `checkCapturedChannel`'s. That split is what lets this check run at the
66
+ * registry's third rung without shadowing the fifteenth, and it keeps the
67
+ * higher-ranked code first on a pointer that is unresolvable and off-body at
68
+ * once.
69
+ */
70
+ export declare function checkCapturedReachability(contract: EvalContract): void;