eval-quality 1.4.1 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/README.md +5 -5
  2. package/corpus/dev/README.md +24 -13
  3. package/corpus/dev/compile-seal-example/brief.json +1 -1
  4. package/corpus/dev/compile-seal-example/contract.json +1 -1
  5. package/corpus/dev/contracts/absent-collection-locations.json +1 -1
  6. package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
  7. package/corpus/dev/contracts/absent-success-indicator.json +1 -1
  8. package/corpus/dev/contracts/captured-read-back.json +1 -0
  9. package/corpus/dev/contracts/checklist-selection.json +1 -0
  10. package/corpus/dev/contracts/empty-channel-roles.json +1 -1
  11. package/corpus/dev/contracts/empty-collection-locations.json +1 -1
  12. package/corpus/dev/contracts/empty-request-shapes.json +1 -1
  13. package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
  14. package/corpus/dev/contracts/fragment-selection.json +1 -1
  15. package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
  16. package/corpus/dev/contracts/no-operation-inventory.json +1 -1
  17. package/corpus/dev/contracts/no-read-back-relation.json +1 -1
  18. package/corpus/dev/contracts/no-state-change-marker.json +1 -1
  19. package/corpus/dev/contracts/no-type-violating-step.json +1 -1
  20. package/corpus/dev/contracts/notes-tool-server.json +1 -0
  21. package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
  22. package/corpus/dev/contracts/review-corpus.json +1 -1
  23. package/corpus/dev/contracts/satisfied-declarations.json +1 -1
  24. package/corpus/dev/contracts/single-required-response-key.json +1 -1
  25. package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
  26. package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
  27. package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
  28. package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
  29. package/corpus/dev/index.json +1 -1
  30. package/dist/adapters/command-line-adapter.js +44 -6
  31. package/dist/adapters/index.d.ts +2 -0
  32. package/dist/adapters/index.js +1 -0
  33. package/dist/adapters/mcp-adapter.d.ts +32 -0
  34. package/dist/adapters/mcp-adapter.js +357 -0
  35. package/dist/adapters/mcp-target-policy.d.ts +39 -0
  36. package/dist/adapters/mcp-target-policy.js +30 -0
  37. package/dist/cli/render.d.ts +8 -3
  38. package/dist/cli/render.js +13 -6
  39. package/dist/cli/run.js +2 -1
  40. package/dist/core/compile/bindings.d.ts +8 -18
  41. package/dist/core/compile/bindings.js +13 -10
  42. package/dist/core/compile/interface-inventory.d.ts +81 -3
  43. package/dist/core/compile/interface-inventory.js +134 -23
  44. package/dist/core/compile/reachability.d.ts +73 -1
  45. package/dist/core/compile/reachability.js +134 -19
  46. package/dist/core/compile/sensitivity-witness.d.ts +22 -10
  47. package/dist/core/compile/sensitivity-witness.js +101 -13
  48. package/dist/core/coverage/operations.d.ts +1 -1
  49. package/dist/core/coverage/operations.js +1 -1
  50. package/dist/core/coverage/relevance.d.ts +3 -3
  51. package/dist/core/coverage/relevance.js +3 -3
  52. package/dist/core/declared-inputs.d.ts +21 -9
  53. package/dist/core/declared-inputs.js +51 -16
  54. package/dist/core/evaluate/evidence-resolution.d.ts +8 -7
  55. package/dist/core/evaluate/evidence-resolution.js +25 -18
  56. package/dist/core/preflight/plan.js +28 -6
  57. package/dist/core/preflight/projection.d.ts +10 -1
  58. package/dist/core/preflight/projection.js +9 -5
  59. package/dist/core/preflight/reduce.js +6 -2
  60. package/dist/core/preflight/witness-evidence.js +31 -9
  61. package/dist/core/schemas/artifact.d.ts +145 -49
  62. package/dist/core/schemas/defect-signature.d.ts +280 -23
  63. package/dist/core/schemas/defect-signature.js +65 -37
  64. package/dist/core/schemas/eval-contract.d.ts +33 -48
  65. package/dist/core/schemas/eval-contract.js +3 -3
  66. package/dist/core/schemas/interface.d.ts +193 -55
  67. package/dist/core/schemas/interface.js +82 -15
  68. package/dist/core/schemas/plan.d.ts +45 -1
  69. package/dist/core/schemas/plan.js +13 -2
  70. package/dist/core/schemas/pointer.d.ts +23 -9
  71. package/dist/core/schemas/pointer.js +25 -11
  72. package/dist/core/schemas/port-messages.d.ts +81 -0
  73. package/dist/core/schemas/port-messages.js +50 -3
  74. package/dist/core/schemas/primitives.d.ts +18 -0
  75. package/dist/core/schemas/primitives.js +29 -0
  76. package/dist/core/schemas/probe-policy.d.ts +41 -0
  77. package/dist/core/schemas/probe-policy.js +61 -1
  78. package/dist/core/schemas/probe-qualification.js +4 -1
  79. package/dist/core/schemas/probe.d.ts +115 -2
  80. package/dist/core/schemas/probe.js +2 -2
  81. package/dist/core/schemas/sealed-run-record.d.ts +11 -5
  82. package/dist/core/schemas/sealed-run-record.js +21 -9
  83. package/dist/core/schemas/sensitivity-witness.d.ts +31 -7
  84. package/dist/core/schemas/sensitivity-witness.js +32 -9
  85. package/dist/core/score/bindings.d.ts +1 -1
  86. package/dist/core/score/bindings.js +4 -4
  87. package/dist/core/score/qualification.d.ts +7 -5
  88. package/dist/core/score/qualification.js +92 -23
  89. package/dist/core/score/score.js +1 -1
  90. package/dist/core/seal/derived-reference.js +20 -17
  91. package/dist/core/seal/plan-index.d.ts +18 -9
  92. package/dist/core/seal/plan-index.js +79 -37
  93. package/dist/index.d.ts +1 -1
  94. package/dist/index.js +1 -1
  95. package/dist/ports/environment-probe-port.d.ts +50 -17
  96. package/dist/ports/environment-probe-port.js +26 -17
  97. package/dist/testing/conformance.d.ts +3 -2
  98. package/dist/testing/conformance.js +2 -1
  99. package/dist/testing/index.d.ts +3 -3
  100. package/dist/testing/index.js +1 -1
  101. package/dist/testing/probe-conformance.d.ts +90 -13
  102. package/dist/testing/probe-conformance.js +375 -160
  103. package/package.json +4 -2
  104. package/schemas/eval-contract.schema.json +571 -18
  105. package/schemas/probe.schema.json +156 -14
  106. package/schemas/rubric.schema.json +1 -1
  107. package/schemas/sealed-run-record.schema.json +24 -17
@@ -1,5 +1,38 @@
1
1
  import type { EvalContract } from '../schemas/eval-contract.ts';
2
- import type { AnyOperation } from '../schemas/interface.ts';
2
+ import type { AnyOperation, InterfaceKindName } from '../schemas/interface.ts';
3
+ /**
4
+ * The kinds whose probe semantics AD-10 requires before it opens one. `cli` and
5
+ * `mcp` declare theirs, so both run; `web` declares none and fails here, which
6
+ * keeps this code fireable and AD-10's sentence true of it.
7
+ *
8
+ * Each tuple is spelled out and typed against `INTERFACE_KINDS`, on the
9
+ * reasoning `pointer.ts` records for `NON_IDENTIFIER_ROOTED_CHANNELS`: adding a
10
+ * fifth kind is then a decision about which side of this line it falls on, and
11
+ * a test asserts the partition. Exported because two other gates assert the
12
+ * same fact: `preflight/plan.ts`, which a caller can reach by assembling a plan
13
+ * by hand, and `score/qualification.ts`, which reads a probe's declared kind
14
+ * where these two read a contract's.
15
+ */
16
+ export declare const SUPPORTED_INTERFACE_KINDS: readonly ["api", "cli", "mcp"];
17
+ export declare const UNSUPPORTED_INTERFACE_KINDS: readonly ["web"];
18
+ /**
19
+ * Whether `compile`, the pre-flight plan, and the probe qualification gate
20
+ * admit a declared kind. All three call this, so what a contract may declare
21
+ * and what a defect signature may declare against cannot disagree.
22
+ */
23
+ export declare const isSupportedInterfaceKind: (kind: InterfaceKindName) => boolean;
24
+ /**
25
+ * A quoted kind list with its own verb, for a diagnostic that names what is
26
+ * supported. A function so the one-element form has a test: closing a kind is a
27
+ * move the tuple below is built to allow, and the general form would render one
28
+ * element as a leading " and ".
29
+ */
30
+ export declare const kindsClause: (kinds: readonly string[]) => string;
31
+ /**
32
+ * The supported list as both throwers spell it, rendered from the tuple so a
33
+ * message cannot claim a set the check does not enforce.
34
+ */
35
+ export declare const SUPPORTED_KINDS_CLAUSE: string;
3
36
  export declare function checkInterfaceKind(contract: EvalContract): void;
4
37
  /**
5
38
  * The transport identity AD-40 resolves a defect signature against: the method
@@ -32,9 +65,45 @@ export declare function commandSignature(operation: {
32
65
  readonly subcommandPath: readonly string[];
33
66
  };
34
67
  }): string;
35
- /** The transport identity of an operation of either kind. */
68
+ /**
69
+ * The transport identity of a tool call: the published tool name, alone and
70
+ * compared literally.
71
+ *
72
+ * There is nothing to erase and nothing to join. Every MCP call shares the one
73
+ * transport identity `tools/call`, so the tool name is the whole of what tells
74
+ * two calls apart, and a method and a path template would render one signature
75
+ * for every tool a server publishes. Takes the field rather than an
76
+ * `McpOperation`, on `operationSignature`'s own terms: AD-40's corpus-side
77
+ * signature declares the same name and must produce the same string from it.
78
+ */
79
+ export declare function mcpSignature(operation: {
80
+ readonly toolName: string;
81
+ }): string;
82
+ /** The transport identity of an operation of any kind. */
36
83
  export declare const anyOperationSignature: (operation: AnyOperation) => string;
37
- /** Finds duplicate method and path signatures across the full inventory. */
84
+ export type SignatureFamily = 'api' | 'cli' | 'mcp';
85
+ /**
86
+ * The shape family an interface kind's transport identity is rendered in.
87
+ * Three renderings, and `api` and `web` share one because they share an
88
+ * operation shape.
89
+ *
90
+ * Typed against the kind vocabulary rather than `string`, so a fifth
91
+ * `INTERFACE_KINDS` member fails the typecheck here instead of landing in the
92
+ * `api` family and rendering a method and a path template it does not have.
93
+ */
94
+ export declare const signatureFamilyOf: (kind: InterfaceKindName) => SignatureFamily;
95
+ /**
96
+ * Finds duplicate transport identities across the full inventory.
97
+ *
98
+ * Keyed on the declaring kind's shape family beside the rendered string,
99
+ * because the three renderings draw from different namespaces: a tool named
100
+ * `notes` and an executable named `notes` both render `notes` while naming
101
+ * different things on different machines, and refusing that pair would be a
102
+ * collision the author cannot fix. `api` and `web` share one family, so two
103
+ * operations sharing a method and a path template across those two kinds still
104
+ * collide. `resolveHomeOperation` compares within a family for the same reason,
105
+ * so the two agree about what a collision is.
106
+ */
38
107
  export declare function checkDuplicateOperationSignature(contract: EvalContract): void;
39
108
  /**
40
109
  * `unresolved-artifact-reference`: an artifact identifier nothing declares.
@@ -45,6 +114,15 @@ export declare function checkDuplicateOperationSignature(contract: EvalContract)
45
114
  * resolving `absent`, on AD-26's own precedent for a dangling reference-set
46
115
  * identifier: `absent` is defined over pointers that do not resolve against
47
116
  * observed evidence, and a dangling declaration is neither.
117
+ *
118
+ * Resolving the pointer's step segment against the interaction plan is the
119
+ * route for an oracle check, an oracle direction's evidence target and a rubric
120
+ * criterion. A sensitivity-witness relation takes the other route, because a leg
121
+ * identifier shares the step namespace without being a step and the plan lookup
122
+ * cannot answer for one. The witness scope is what decides it, and preferring it
123
+ * over the plan matters for ordering: this check runs at `compile.ts` ahead of
124
+ * `checkWitnessLegIdentifiers`, so a leg id colliding with a step id is still
125
+ * present here and the plan route would answer it against the wrong operation.
48
126
  */
49
127
  export declare function checkArtifactReferences(contract: EvalContract): void;
50
128
  /**
@@ -2,24 +2,59 @@
2
2
  * Checks interface kinds, inventory-wide operation signatures, and step input
3
3
  * bindings against each operation's request shape.
4
4
  */
5
- import { boundChannelsOf, declaredArtifactsOf, descriptorArtifactOf, isCommandOperation, requestShapeOf, } from '../declared-inputs.js';
5
+ import { boundChannelsOf, declaredArtifactsOf, descriptorArtifactOf, isCommandOperation, isMcpOperation, requestShapeOf, } from '../declared-inputs.js';
6
6
  import { StructuralFailure } from '../failure-codes.js';
7
7
  import { operationsOf } from '../schemas/interface.js';
8
8
  import { anyOperationOf, buildPlanIndex, parseEvidenceTarget, } from '../seal/plan-index.js';
9
9
  import { forEachArtifactPointer } from './reachability.js';
10
10
  /**
11
- * Rejects permitted interface kinds whose probe semantics are undeclared.
11
+ * The kinds whose probe semantics AD-10 requires before it opens one. `cli` and
12
+ * `mcp` declare theirs, so both run; `web` declares none and fails here, which
13
+ * keeps this code fireable and AD-10's sentence true of it.
12
14
  *
13
- * AD-10 closed all three non-api kinds and named the condition for opening
14
- * one: the semantics have to be declared. They are declared for `cli`, so it
15
- * runs; `web` and `mcp` are still undeclared and still fail here, which is
16
- * what keeps this code fireable and keeps AD-10's sentence true of them.
15
+ * Each tuple is spelled out and typed against `INTERFACE_KINDS`, on the
16
+ * reasoning `pointer.ts` records for `NON_IDENTIFIER_ROOTED_CHANNELS`: adding a
17
+ * fifth kind is then a decision about which side of this line it falls on, and
18
+ * a test asserts the partition. Exported because two other gates assert the
19
+ * same fact: `preflight/plan.ts`, which a caller can reach by assembling a plan
20
+ * by hand, and `score/qualification.ts`, which reads a probe's declared kind
21
+ * where these two read a contract's.
17
22
  */
18
- const SUPPORTED_INTERFACE_KINDS = ['api', 'cli'];
23
+ export const SUPPORTED_INTERFACE_KINDS = [
24
+ 'api',
25
+ 'cli',
26
+ 'mcp',
27
+ ];
28
+ export const UNSUPPORTED_INTERFACE_KINDS = [
29
+ 'web',
30
+ ];
31
+ /**
32
+ * Whether `compile`, the pre-flight plan, and the probe qualification gate
33
+ * admit a declared kind. All three call this, so what a contract may declare
34
+ * and what a defect signature may declare against cannot disagree.
35
+ */
36
+ export const isSupportedInterfaceKind = (kind) => SUPPORTED_INTERFACE_KINDS.includes(kind);
37
+ /**
38
+ * A quoted kind list with its own verb, for a diagnostic that names what is
39
+ * supported. A function so the one-element form has a test: closing a kind is a
40
+ * move the tuple below is built to allow, and the general form would render one
41
+ * element as a leading " and ".
42
+ */
43
+ export const kindsClause = (kinds) => {
44
+ const quoted = kinds.map((kind) => `"${kind}"`);
45
+ return quoted.length === 1
46
+ ? `${quoted.join('')} is`
47
+ : `${quoted.slice(0, -1).join(', ')} and ${quoted.slice(-1).join('')} are`;
48
+ };
49
+ /**
50
+ * The supported list as both throwers spell it, rendered from the tuple so a
51
+ * message cannot claim a set the check does not enforce.
52
+ */
53
+ export const SUPPORTED_KINDS_CLAUSE = kindsClause(SUPPORTED_INTERFACE_KINDS);
19
54
  export function checkInterfaceKind(contract) {
20
55
  for (const iface of contract.permittedInterfaces) {
21
- if (!SUPPORTED_INTERFACE_KINDS.includes(iface.kind)) {
22
- throw new StructuralFailure('unsupported-interface-kind', `EvalContract.permittedInterfaces[logicalId=${iface.logicalId}].kind`, `"${iface.kind}" is not supported; "api" and "cli" are (AD-10)`);
56
+ if (!isSupportedInterfaceKind(iface.kind)) {
57
+ throw new StructuralFailure('unsupported-interface-kind', `EvalContract.permittedInterfaces[logicalId=${iface.logicalId}].kind`, `"${iface.kind}" is not supported; ${SUPPORTED_KINDS_CLAUSE} (AD-10)`);
23
58
  }
24
59
  }
25
60
  }
@@ -56,21 +91,79 @@ export function commandSignature(operation) {
56
91
  ...operation.invocation.subcommandPath,
57
92
  ].join(COMMAND_SIGNATURE_SEPARATOR);
58
93
  }
59
- /** The transport identity of an operation of either kind. */
60
- export const anyOperationSignature = (operation) => isCommandOperation(operation)
61
- ? commandSignature(operation)
62
- : operationSignature(operation);
63
- /** Finds duplicate method and path signatures across the full inventory. */
94
+ /**
95
+ * The transport identity of a tool call: the published tool name, alone and
96
+ * compared literally.
97
+ *
98
+ * There is nothing to erase and nothing to join. Every MCP call shares the one
99
+ * transport identity `tools/call`, so the tool name is the whole of what tells
100
+ * two calls apart, and a method and a path template would render one signature
101
+ * for every tool a server publishes. Takes the field rather than an
102
+ * `McpOperation`, on `operationSignature`'s own terms: AD-40's corpus-side
103
+ * signature declares the same name and must produce the same string from it.
104
+ */
105
+ export function mcpSignature(operation) {
106
+ return operation.toolName;
107
+ }
108
+ /** The transport identity of an operation of any kind. */
109
+ export const anyOperationSignature = (operation) => {
110
+ if (isCommandOperation(operation))
111
+ return commandSignature(operation);
112
+ if (isMcpOperation(operation))
113
+ return mcpSignature(operation);
114
+ return operationSignature(operation);
115
+ };
116
+ /**
117
+ * The shape family an interface kind's transport identity is rendered in.
118
+ * Three renderings, and `api` and `web` share one because they share an
119
+ * operation shape.
120
+ *
121
+ * Typed against the kind vocabulary rather than `string`, so a fifth
122
+ * `INTERFACE_KINDS` member fails the typecheck here instead of landing in the
123
+ * `api` family and rendering a method and a path template it does not have.
124
+ */
125
+ export const signatureFamilyOf = (kind) => {
126
+ switch (kind) {
127
+ case 'cli':
128
+ return 'cli';
129
+ case 'mcp':
130
+ return 'mcp';
131
+ case 'api':
132
+ case 'web':
133
+ return 'api';
134
+ }
135
+ };
136
+ /**
137
+ * Finds duplicate transport identities across the full inventory.
138
+ *
139
+ * Keyed on the declaring kind's shape family beside the rendered string,
140
+ * because the three renderings draw from different namespaces: a tool named
141
+ * `notes` and an executable named `notes` both render `notes` while naming
142
+ * different things on different machines, and refusing that pair would be a
143
+ * collision the author cannot fix. `api` and `web` share one family, so two
144
+ * operations sharing a method and a path template across those two kinds still
145
+ * collide. `resolveHomeOperation` compares within a family for the same reason,
146
+ * so the two agree about what a collision is.
147
+ */
64
148
  export function checkDuplicateOperationSignature(contract) {
65
149
  const seen = new Map();
66
150
  for (const iface of contract.permittedInterfaces) {
67
151
  for (const operation of operationsOf(iface)) {
68
152
  const signature = anyOperationSignature(operation);
69
- const collision = seen.get(signature);
153
+ const family = signatureFamilyOf(iface.kind);
154
+ const key = `${family} ${signature}`;
155
+ const collision = seen.get(key);
70
156
  if (collision !== undefined) {
71
- throw new StructuralFailure('duplicate-operation-signature', `EvalContract.permittedInterfaces[logicalId=${iface.logicalId}].operations[operationId=${operation.operationId}]`, `collides with permittedInterfaces[logicalId=${collision.logicalId}].operations[operationId=${collision.operation.operationId}] after parameter-name erasure ("${signature}") (AD-19, AD-40)`);
157
+ // The message names the family, because the identity alone no
158
+ // longer says which namespace it was compared in, and the
159
+ // erasure clause is scoped to the family it applies to: a tool
160
+ // name and an executable path carry no parameters to erase.
161
+ const how = family === 'api'
162
+ ? 'after parameter-name erasure'
163
+ : 'on the identity it renders';
164
+ throw new StructuralFailure('duplicate-operation-signature', `EvalContract.permittedInterfaces[logicalId=${iface.logicalId}].operations[operationId=${operation.operationId}]`, `collides with permittedInterfaces[logicalId=${collision.logicalId}].operations[operationId=${collision.operation.operationId}] ${how} ("${signature}") among ${family}-shaped operations (AD-19, AD-40)`);
72
165
  }
73
- seen.set(signature, { logicalId: iface.logicalId, operation });
166
+ seen.set(key, { logicalId: iface.logicalId, operation });
74
167
  }
75
168
  }
76
169
  }
@@ -83,6 +176,15 @@ export function checkDuplicateOperationSignature(contract) {
83
176
  * resolving `absent`, on AD-26's own precedent for a dangling reference-set
84
177
  * identifier: `absent` is defined over pointers that do not resolve against
85
178
  * observed evidence, and a dangling declaration is neither.
179
+ *
180
+ * Resolving the pointer's step segment against the interaction plan is the
181
+ * route for an oracle check, an oracle direction's evidence target and a rubric
182
+ * criterion. A sensitivity-witness relation takes the other route, because a leg
183
+ * identifier shares the step namespace without being a step and the plan lookup
184
+ * cannot answer for one. The witness scope is what decides it, and preferring it
185
+ * over the plan matters for ordering: this check runs at `compile.ts` ahead of
186
+ * `checkWitnessLegIdentifiers`, so a leg id colliding with a step id is still
187
+ * present here and the plan route would answer it against the wrong operation.
86
188
  */
87
189
  export function checkArtifactReferences(contract) {
88
190
  for (const iface of contract.permittedInterfaces) {
@@ -95,14 +197,23 @@ export function checkArtifactReferences(contract) {
95
197
  }
96
198
  }
97
199
  const index = buildPlanIndex(contract.interactionPlan, contract.permittedInterfaces, { duplicateIds: 'unresolved' });
98
- forEachArtifactPointer(contract, (pointer, path) => {
200
+ const operationFor = (witnessScope, stepId) => {
201
+ // A relation pointer rooted at anything but its own leg is
202
+ // `checkWitnessLegality`'s, which names the witness and the stray root.
203
+ if (witnessScope !== null)
204
+ return witnessScope.legIds.includes(stepId)
205
+ ? witnessScope.operation
206
+ : undefined;
207
+ const step = index.stepOf(stepId);
208
+ if (step === undefined)
209
+ return undefined;
210
+ return anyOperationOf(index, step.operationId);
211
+ };
212
+ forEachArtifactPointer(contract, (pointer, path, witnessScope) => {
99
213
  const target = parseEvidenceTarget(pointer);
100
214
  if (target.artifactId === null)
101
215
  return;
102
- const step = index.stepOf(target.stepId);
103
- if (step === undefined)
104
- return;
105
- const operation = anyOperationOf(index, step.operationId);
216
+ const operation = operationFor(witnessScope, target.stepId);
106
217
  // An unresolvable step or operation is `unreachable-check-evidence`'s,
107
218
  // at a higher rung; this check has nothing to compare against.
108
219
  if (operation === undefined)
@@ -138,7 +249,7 @@ export function checkUndeclaredMandatoryInput(contract) {
138
249
  if (binding === null)
139
250
  continue;
140
251
  const shape = requestShapeOf(operation, channel);
141
- // A step binding a channel of the other kind has no declared shape
252
+ // A step binding a channel of another kind has no declared shape
142
253
  // to answer to. Reporting it as an undeclared input is true as far
143
254
  // as it goes: the operation declares no such channel, and so
144
255
  // declares no such key on it.
@@ -2,6 +2,21 @@ import type { EvalContract } from '../schemas/eval-contract.ts';
2
2
  import type { Expression } from '../schemas/expression.ts';
3
3
  import type { AnyOperation } from '../schemas/interface.ts';
4
4
  import { type PlanIndex } from '../seal/plan-index.ts';
5
+ /**
6
+ * A sensitivity witness's own resolution context: the operation it is declared
7
+ * on, and the leg identifiers its relation's pointers may root at. Both halves
8
+ * are needed. The operation is the only one a leg can be issued against, and a
9
+ * pointer rooted at anything else is `checkWitnessLegality`'s to report, so a
10
+ * consumer answers for a leg and abstains on the rest.
11
+ *
12
+ * `ExpressionSite.witnessScope` in `expression-legality.ts` carries the same
13
+ * pair for the same reason. Two spellings of one idea, kept apart while each
14
+ * has one consumer.
15
+ */
16
+ export type WitnessScope = {
17
+ readonly operation: AnyOperation;
18
+ readonly legIds: readonly string[];
19
+ };
5
20
  /**
6
21
  * Every interaction-rooted pointer the contract writes down, wherever it sits:
7
22
  * an oracle's check and its direction's evidence targets, a rubric criterion's
@@ -10,8 +25,17 @@ import { type PlanIndex } from '../seal/plan-index.ts';
10
25
  * Broader than `forEachCheckPointer`, which walks oracle checks alone, because
11
26
  * an artifact identifier is an authoring fault at every site that names one and
12
27
  * a check that walked only the checks would report half of them.
28
+ *
29
+ * The third argument is the witness scope, supplied at a sensitivity-witness
30
+ * relation and `null` everywhere else. A witness leg carries no operation of its
31
+ * own: it probes the operation declaring the witness, and its `legId` roots the
32
+ * relation's pointers in the same namespace as interaction-plan step ids
33
+ * without being a step. So a caller resolving the pointer's own step segment
34
+ * against the plan finds nothing at a witness site and has to be handed the
35
+ * scope instead. Everywhere else the pointer's step segment is the only thing
36
+ * that names an operation.
13
37
  */
14
- export declare function forEachArtifactPointer(contract: EvalContract, visit: (pointer: string, artifactPath: string) => void): void;
38
+ export declare function forEachArtifactPointer(contract: EvalContract, visit: (pointer: string, artifactPath: string, witnessScope: WitnessScope | null) => void): void;
15
39
  export declare function checkBoundElementScope(contract: EvalContract): void;
16
40
  /** `checkBoundElementScope` over one bare `Expression`. */
17
41
  export declare function checkExpressionBoundElementScope(expression: Expression, artifactPath: string): void;
@@ -24,6 +48,54 @@ export declare function checkExpressionBoundElementScope(expression: Expression,
24
48
  export declare function forEachExpressionPointer(expression: Expression, visit: (pointer: string, path: string) => void): void;
25
49
  /** `checkEvidenceReachability` over one bare `Expression` and one operation. */
26
50
  export declare function checkExpressionEvidenceReachability(expression: Expression, artifactPath: string, operation: AnyOperation): void;
51
+ /**
52
+ * `unreachable-check-evidence` for a pointer at a channel a witness leg never
53
+ * carries.
54
+ *
55
+ * `evaluateReachabilityAgainstOperation` answers for a sealed run record, where
56
+ * every channel the interface produces is observed. A witness leg is narrower.
57
+ * `evidenceOf` builds one from a single probe observation and writes every
58
+ * channel that observation does not describe blank, so a bare pointer at one
59
+ * resolves absent on both legs however reachable it looks against the
60
+ * declaration. `deep-equality` over an absent side is `false` and the enclosing
61
+ * `not` then certifies the operation sensitive on every run.
62
+ *
63
+ * What a leg carries is the operation's own described channel, its transport
64
+ * inputs, and the channels its transport produces with no descriptor of their
65
+ * own: the status and the headers off an interface that speaks HTTP, the exit
66
+ * code off a command, the error flag alone off a tool call. `response-headers`
67
+ * stays legal for HTTP deliberately, matching the note on `projectObservation`
68
+ * that the projection does not carry headers and the relation reads them raw;
69
+ * a tool call has none to read.
70
+ *
71
+ * The artifact channel narrows once more. A leg carries the one artifact the
72
+ * descriptor nominates, so a pointer at any other declared artifact is blank
73
+ * even though the contract declares the operation writes it.
74
+ */
75
+ export declare function checkExpressionLegChannel(expression: Expression, artifactPath: string, operation: AnyOperation): void;
76
+ /**
77
+ * `unreachable-check-evidence` for a pointer at a field the operation declares
78
+ * volatile. Reachability answers the descriptor question and this answers the
79
+ * projection one, so a caller that runs both gets the whole answer.
80
+ *
81
+ * Only a sensitivity-witness relation asks. `projectObservation` prunes every
82
+ * volatile pointer from the described channel before `evidenceOf` builds the
83
+ * leg the relation reads, so a relation addressing one resolves absent on both
84
+ * legs. `deep-equality` over an absent side is `false`, and the enclosing `not`
85
+ * then reports the operation sensitive on every run from a pair of pointers that
86
+ * never resolved. It is one of two ways a leg comes back blank, and
87
+ * `checkExpressionLegChannel` above covers the other. An oracle
88
+ * is scored against a sealed run record, which carries no projection, so the
89
+ * same pointer is legitimate there.
90
+ *
91
+ * The empty pointer is RFC 6901's whole document, and `pruneVolatile` reads it
92
+ * that way, so an operation declaring it makes every pointer at the described
93
+ * channel volatile. `pruneVolatile` prunes a `json` body alone, so a body
94
+ * arriving as text is never pruned and a differential over it would work.
95
+ * Refusing it anyway is the right default: a descriptor declaring `requiredKeys`
96
+ * and `types` says the body is json, and compile cannot see what arrives.
97
+ */
98
+ export declare function checkExpressionVolatility(expression: Expression, artifactPath: string, operation: AnyOperation): void;
27
99
  type ReachabilityResult = {
28
100
  readonly reachable: true;
29
101
  } | {
@@ -7,12 +7,12 @@
7
7
  * `evaluatePointerReachability` is exported separately as the non-throwing
8
8
  * per-pointer core, for reuse and direct testing.
9
9
  */
10
- import { declaredArtifactsOf, descriptorArtifactOf, descriptorChannelOf, isCommandOperation, requestShapeOf, } from '../declared-inputs.js';
10
+ import { declaredArtifactsOf, descriptorArtifactOf, descriptorChannelOf, isCommandOperation, isMcpOperation, requestShapeOf, } from '../declared-inputs.js';
11
11
  import { ARRAY_INDEX_PATTERN } from '../evaluate/evidence-resolution.js';
12
12
  import { StructuralFailure } from '../failure-codes.js';
13
13
  import { operationsOf } from '../schemas/interface.js';
14
14
  import { JsonTypeName } from '../schemas/primitives.js';
15
- import { anyOperationOf, buildPlanIndex, parseEvidenceTarget, } from '../seal/plan-index.js';
15
+ import { anyOperationOf, buildPlanIndex, decodeTail, parseEvidenceTarget, } from '../seal/plan-index.js';
16
16
  function visitOperand(operand, path, insideQuantifier, visit) {
17
17
  if ('pointer' in operand)
18
18
  visit({ pointer: operand.pointer, path, insideQuantifier });
@@ -62,23 +62,32 @@ function forEachCheckPointer(contract, visit) {
62
62
  * Broader than `forEachCheckPointer`, which walks oracle checks alone, because
63
63
  * an artifact identifier is an authoring fault at every site that names one and
64
64
  * a check that walked only the checks would report half of them.
65
+ *
66
+ * The third argument is the witness scope, supplied at a sensitivity-witness
67
+ * relation and `null` everywhere else. A witness leg carries no operation of its
68
+ * own: it probes the operation declaring the witness, and its `legId` roots the
69
+ * relation's pointers in the same namespace as interaction-plan step ids
70
+ * without being a step. So a caller resolving the pointer's own step segment
71
+ * against the plan finds nothing at a witness site and has to be handed the
72
+ * scope instead. Everywhere else the pointer's step segment is the only thing
73
+ * that names an operation.
65
74
  */
66
75
  export function forEachArtifactPointer(contract, visit) {
67
- const seen = (pointer, artifactPath) => {
76
+ const seen = (pointer, artifactPath, witnessScope) => {
68
77
  if (pointer.startsWith('@'))
69
78
  return;
70
- visit(pointer, artifactPath);
79
+ visit(pointer, artifactPath, witnessScope);
71
80
  };
72
81
  contract.oracles.forEach((oracle) => {
73
82
  if (oracle.check !== null)
74
- visitExpression(oracle.check, 'check', false, (site) => seen(site.pointer, `EvalContract.oracles[id=${oracle.id}].${site.path}`));
83
+ visitExpression(oracle.check, 'check', false, (site) => seen(site.pointer, `EvalContract.oracles[id=${oracle.id}].${site.path}`, null));
75
84
  oracle.direction?.evidenceTargets.forEach((target, index) => {
76
- seen(target, `EvalContract.oracles[id=${oracle.id}].direction.evidenceTargets[${index}]`);
85
+ seen(target, `EvalContract.oracles[id=${oracle.id}].direction.evidenceTargets[${index}]`, null);
77
86
  });
78
87
  });
79
88
  contract.rubrics.forEach((rubric) => {
80
89
  rubric.criteria.forEach((criterion) => {
81
- seen(criterion.evidence, `EvalContract.rubrics[id=${rubric.id}].criteria[id=${criterion.id}].evidence`);
90
+ seen(criterion.evidence, `EvalContract.rubrics[id=${rubric.id}].criteria[id=${criterion.id}].evidence`, null);
82
91
  });
83
92
  });
84
93
  contract.permittedInterfaces.forEach((iface, interfaceIndex) => {
@@ -86,7 +95,11 @@ export function forEachArtifactPointer(contract, visit) {
86
95
  const witness = operation.sensitivityWitness;
87
96
  if (witness === null)
88
97
  return;
89
- visitExpression(witness.relation, 'relation', false, (site) => seen(site.pointer, `EvalContract.permittedInterfaces[${interfaceIndex}].operations[${operationIndex}].sensitivityWitness.${site.path}`));
98
+ const scope = {
99
+ operation,
100
+ legIds: witness.legs.map((leg) => leg.legId),
101
+ };
102
+ visitExpression(witness.relation, 'relation', false, (site) => seen(site.pointer, `EvalContract.permittedInterfaces[${interfaceIndex}].operations[${operationIndex}].sensitivityWitness.${site.path}`, scope));
90
103
  });
91
104
  });
92
105
  }
@@ -126,6 +139,93 @@ export function checkExpressionEvidenceReachability(expression, artifactPath, op
126
139
  }
127
140
  });
128
141
  }
142
+ /**
143
+ * `unreachable-check-evidence` for a pointer at a channel a witness leg never
144
+ * carries.
145
+ *
146
+ * `evaluateReachabilityAgainstOperation` answers for a sealed run record, where
147
+ * every channel the interface produces is observed. A witness leg is narrower.
148
+ * `evidenceOf` builds one from a single probe observation and writes every
149
+ * channel that observation does not describe blank, so a bare pointer at one
150
+ * resolves absent on both legs however reachable it looks against the
151
+ * declaration. `deep-equality` over an absent side is `false` and the enclosing
152
+ * `not` then certifies the operation sensitive on every run.
153
+ *
154
+ * What a leg carries is the operation's own described channel, its transport
155
+ * inputs, and the channels its transport produces with no descriptor of their
156
+ * own: the status and the headers off an interface that speaks HTTP, the exit
157
+ * code off a command, the error flag alone off a tool call. `response-headers`
158
+ * stays legal for HTTP deliberately, matching the note on `projectObservation`
159
+ * that the projection does not carry headers and the relation reads them raw;
160
+ * a tool call has none to read.
161
+ *
162
+ * The artifact channel narrows once more. A leg carries the one artifact the
163
+ * descriptor nominates, so a pointer at any other declared artifact is blank
164
+ * even though the contract declares the operation writes it.
165
+ */
166
+ export function checkExpressionLegChannel(expression, artifactPath, operation) {
167
+ const described = descriptorChannelOf(operation);
168
+ const describedArtifact = descriptorArtifactOf(operation);
169
+ const carried = isCommandOperation(operation)
170
+ ? [described, 'exit-code', 'call-inputs']
171
+ : isMcpOperation(operation)
172
+ ? [described, 'response-status', 'call-inputs']
173
+ : [described, 'response-headers', 'response-status', 'call-inputs'];
174
+ visitExpression(expression, '', false, (site) => {
175
+ if (site.pointer.startsWith('@'))
176
+ return;
177
+ const target = parseEvidenceTarget(site.pointer);
178
+ const refuse = (reason) => {
179
+ throw new StructuralFailure('unreachable-check-evidence', `${artifactPath}${site.path}`, `"${site.pointer}" ${reason}`);
180
+ };
181
+ if (!carried.includes(target.channel))
182
+ refuse(`addresses ${target.channel}, which a witness leg of operation "${operation.operationId}" does not carry; pre-flight builds each leg from ${carried.join(', ')} alone`);
183
+ if (target.artifactId !== null && target.artifactId !== describedArtifact)
184
+ refuse(`addresses the "${target.artifactId}" artifact, which operation "${operation.operationId}" declares it writes and its descriptor does not nominate, so a witness leg carries nothing for it`);
185
+ });
186
+ }
187
+ /**
188
+ * `unreachable-check-evidence` for a pointer at a field the operation declares
189
+ * volatile. Reachability answers the descriptor question and this answers the
190
+ * projection one, so a caller that runs both gets the whole answer.
191
+ *
192
+ * Only a sensitivity-witness relation asks. `projectObservation` prunes every
193
+ * volatile pointer from the described channel before `evidenceOf` builds the
194
+ * leg the relation reads, so a relation addressing one resolves absent on both
195
+ * legs. `deep-equality` over an absent side is `false`, and the enclosing `not`
196
+ * then reports the operation sensitive on every run from a pair of pointers that
197
+ * never resolved. It is one of two ways a leg comes back blank, and
198
+ * `checkExpressionLegChannel` above covers the other. An oracle
199
+ * is scored against a sealed run record, which carries no projection, so the
200
+ * same pointer is legitimate there.
201
+ *
202
+ * The empty pointer is RFC 6901's whole document, and `pruneVolatile` reads it
203
+ * that way, so an operation declaring it makes every pointer at the described
204
+ * channel volatile. `pruneVolatile` prunes a `json` body alone, so a body
205
+ * arriving as text is never pruned and a differential over it would work.
206
+ * Refusing it anyway is the right default: a descriptor declaring `requiredKeys`
207
+ * and `types` says the body is json, and compile cannot see what arrives.
208
+ */
209
+ export function checkExpressionVolatility(expression, artifactPath, operation) {
210
+ if (operation.volatilePointers.length === 0)
211
+ return;
212
+ const channel = descriptorChannelOf(operation);
213
+ const describedArtifact = descriptorArtifactOf(operation);
214
+ const volatileTails = operation.volatilePointers.map(decodeTail);
215
+ visitExpression(expression, '', false, (site) => {
216
+ if (site.pointer.startsWith('@'))
217
+ return;
218
+ const target = parseEvidenceTarget(site.pointer);
219
+ if (target.channel !== channel)
220
+ return;
221
+ if (target.artifactId !== null && target.artifactId !== describedArtifact)
222
+ return;
223
+ const pruned = volatileTails.find((tokens) => tokens.every((token, index) => target.tail[index] === token));
224
+ if (pruned === undefined)
225
+ return;
226
+ throw new StructuralFailure('unreachable-check-evidence', `${artifactPath}${site.path}`, `"${site.pointer}" addresses "${`/${pruned.join('/')}`}", which operation "${operation.operationId}" declares volatile, so the projection the relation reads has already removed it`);
227
+ });
228
+ }
129
229
  const reachable = () => ({ reachable: true });
130
230
  const unreachable = (reason) => ({
131
231
  reachable: false,
@@ -251,6 +351,16 @@ function evaluateReachabilityAgainstOperation(pointer, operation) {
251
351
  if (target.channel === descriptorChannel) {
252
352
  return descendThroughDescriptor(operation.responseDescriptor, target, operation.operationId, target.channel);
253
353
  }
354
+ // A tool call produces its structured result on the descriptor's channel and
355
+ // its error flag on `response-status`, and nothing else: no HTTP headers, no
356
+ // process exit code, no stream, and no written file. Placed ahead of the
357
+ // residue below, which answers reachable for three channels a tool call
358
+ // never fills.
359
+ if (isMcpOperation(operation) &&
360
+ target.channel !== 'response-status' &&
361
+ target.channel !== 'call-inputs') {
362
+ return unreachable(`addresses ${target.channel} on operation "${operation.operationId}", which is a tool call and carries its result on ${descriptorChannel} and its error flag on response-status alone`);
363
+ }
254
364
  if (target.channel === 'stdout' || target.channel === 'stderr') {
255
365
  // The stream this operation's descriptor does not describe carries no
256
366
  // declared structure, so a non-empty tail proves the pointer
@@ -267,28 +377,33 @@ function evaluateReachabilityAgainstOperation(pointer, operation) {
267
377
  return unreachable(`addresses ${target.channel} on operation "${operation.operationId}", which runs behind a command and produces no HTTP response`);
268
378
  }
269
379
  if (target.channel === 'call-inputs') {
270
- if (target.tail.length === 0)
271
- return reachable();
272
- const { transportChannel } = target;
273
- if (transportChannel === null) {
380
+ const { inputChannel } = target;
381
+ if (inputChannel === null) {
274
382
  // Unreachable: parseEvidenceTarget's own guarantee.
275
- throw new TypeError('call-inputs evidence target carries no transport channel');
383
+ throw new TypeError('call-inputs evidence target carries no input channel');
384
+ }
385
+ // The channel test runs ahead of the tail test, because a tail-less
386
+ // pointer at a channel the operation does not accept is unreachable
387
+ // too: the recorded call inputs carry no key for it, so the pointer
388
+ // resolves absent on every run and no run can change that. Asking about
389
+ // the tail first admitted every such pointer, on all nine channels.
390
+ const shape = requestShapeOf(operation, inputChannel);
391
+ if (shape === undefined) {
392
+ return unreachable(`addresses call-inputs ${inputChannel}, a channel operation "${operation.operationId}" does not accept input on`);
276
393
  }
394
+ if (target.tail.length === 0)
395
+ return reachable();
277
396
  const firstToken = target.tail[0];
278
397
  if (firstToken === undefined) {
279
398
  throw new TypeError('evidence-target tail is non-empty but has no first token');
280
399
  }
281
- const shape = requestShapeOf(operation, transportChannel);
282
- if (shape === undefined) {
283
- return unreachable(`addresses call-inputs ${transportChannel}, a channel operation "${operation.operationId}" does not accept input on`);
284
- }
285
400
  const { requiredKeys, permittedKeys, types } = shape;
286
401
  if (!requiredKeys.includes(firstToken) &&
287
402
  !permittedKeys.includes(firstToken)) {
288
- return unreachable(`addresses call-inputs ${transportChannel} field "${firstToken}", which operation "${operation.operationId}" declares in neither requiredKeys nor permittedKeys`);
403
+ return unreachable(`addresses call-inputs ${inputChannel} field "${firstToken}", which operation "${operation.operationId}" declares in neither requiredKeys nor permittedKeys`);
289
404
  }
290
405
  if (descendsIntoDeclaredScalar(types, target.tail, firstToken)) {
291
- return unreachable(`descends into call-inputs ${transportChannel} field "${firstToken}", which operation "${operation.operationId}" declares a scalar with no further structure`);
406
+ return unreachable(`descends into call-inputs ${inputChannel} field "${firstToken}", which operation "${operation.operationId}" declares a scalar with no further structure`);
292
407
  }
293
408
  return reachable();
294
409
  }