eval-quality 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. package/README.md +142 -67
  2. package/corpus/dev/README.md +20 -13
  3. package/corpus/dev/compile-seal-example/brief.json +1 -1
  4. package/corpus/dev/compile-seal-example/contract.json +1 -1
  5. package/corpus/dev/contracts/absent-collection-locations.json +1 -1
  6. package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
  7. package/corpus/dev/contracts/absent-success-indicator.json +1 -1
  8. package/corpus/dev/contracts/empty-channel-roles.json +1 -1
  9. package/corpus/dev/contracts/empty-collection-locations.json +1 -1
  10. package/corpus/dev/contracts/empty-request-shapes.json +1 -1
  11. package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
  12. package/corpus/dev/contracts/fragment-selection.json +1 -0
  13. package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
  14. package/corpus/dev/contracts/no-operation-inventory.json +1 -1
  15. package/corpus/dev/contracts/no-read-back-relation.json +1 -1
  16. package/corpus/dev/contracts/no-state-change-marker.json +1 -1
  17. package/corpus/dev/contracts/no-type-violating-step.json +1 -1
  18. package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
  19. package/corpus/dev/contracts/review-corpus.json +1 -0
  20. package/corpus/dev/contracts/satisfied-declarations.json +1 -1
  21. package/corpus/dev/contracts/single-required-response-key.json +1 -1
  22. package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
  23. package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
  24. package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
  25. package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
  26. package/corpus/dev/index.json +1 -1
  27. package/dist/application/index.d.ts +2 -0
  28. package/dist/application/index.js +1 -0
  29. package/dist/application/score.d.ts +48 -0
  30. package/dist/application/score.js +186 -0
  31. package/dist/cli/arguments.d.ts +7 -3
  32. package/dist/cli/arguments.js +73 -5
  33. package/dist/cli/exit-codes.d.ts +6 -2
  34. package/dist/cli/exit-codes.js +8 -9
  35. package/dist/cli/main.js +9 -0
  36. package/dist/cli/render.d.ts +3 -6
  37. package/dist/cli/render.js +57 -5
  38. package/dist/cli/run.d.ts +13 -3
  39. package/dist/cli/run.js +136 -16
  40. package/dist/core/canonical/canonicalize.d.ts +12 -0
  41. package/dist/core/canonical/canonicalize.js +18 -4
  42. package/dist/core/compile/bindings.d.ts +89 -0
  43. package/dist/core/compile/bindings.js +334 -0
  44. package/dist/core/compile/compile.d.ts +17 -0
  45. package/dist/core/compile/compile.js +37 -1
  46. package/dist/core/compile/excluded-content.d.ts +11 -0
  47. package/dist/core/compile/excluded-content.js +42 -0
  48. package/dist/core/compile/expression-legality.d.ts +49 -0
  49. package/dist/core/compile/expression-legality.js +158 -38
  50. package/dist/core/compile/forbidden-inputs.d.ts +8 -1
  51. package/dist/core/compile/forbidden-inputs.js +16 -3
  52. package/dist/core/compile/interface-inventory.d.ts +57 -2
  53. package/dist/core/compile/interface-inventory.js +119 -13
  54. package/dist/core/compile/reachability.d.ts +28 -1
  55. package/dist/core/compile/reachability.js +167 -35
  56. package/dist/core/compile/schema-version.d.ts +2 -0
  57. package/dist/core/compile/schema-version.js +25 -0
  58. package/dist/core/compile/sensitivity-witness.d.ts +31 -12
  59. package/dist/core/compile/sensitivity-witness.js +110 -23
  60. package/dist/core/compile/step-reference.d.ts +2 -0
  61. package/dist/core/compile/step-reference.js +49 -0
  62. package/dist/core/coverage/operations.d.ts +62 -0
  63. package/dist/core/coverage/operations.js +57 -0
  64. package/dist/core/coverage/relevance.d.ts +4 -2
  65. package/dist/core/coverage/relevance.js +22 -23
  66. package/dist/core/coverage/satisfaction.d.ts +2 -2
  67. package/dist/core/coverage/satisfaction.js +73 -48
  68. package/dist/core/declared-inputs.d.ts +83 -4
  69. package/dist/core/declared-inputs.js +105 -8
  70. package/dist/core/emit/emit.d.ts +9 -0
  71. package/dist/core/emit/emit.js +214 -0
  72. package/dist/core/emit/private-artifact-digest.d.ts +11 -0
  73. package/dist/core/emit/private-artifact-digest.js +31 -0
  74. package/dist/core/evaluate/evidence-resolution.d.ts +10 -3
  75. package/dist/core/evaluate/evidence-resolution.js +74 -11
  76. package/dist/core/evaluate/operators.d.ts +22 -0
  77. package/dist/core/evaluate/operators.js +17 -1
  78. package/dist/core/evaluate/resolution.d.ts +14 -3
  79. package/dist/core/evaluate/resolution.js +63 -3
  80. package/dist/core/excluded-content.d.ts +65 -0
  81. package/dist/core/excluded-content.js +113 -0
  82. package/dist/core/failure-codes.d.ts +2 -2
  83. package/dist/core/failure-codes.js +7 -2
  84. package/dist/core/ingest/conditions.d.ts +201 -0
  85. package/dist/core/ingest/conditions.js +48 -0
  86. package/dist/core/ingest/index.d.ts +13 -0
  87. package/dist/core/ingest/index.js +11 -0
  88. package/dist/core/ingest/ingest.d.ts +68 -0
  89. package/dist/core/ingest/ingest.js +318 -0
  90. package/dist/core/lineage/stage-table.d.ts +21 -4
  91. package/dist/core/lineage/stage-table.js +35 -3
  92. package/dist/core/preflight/plan.d.ts +7 -5
  93. package/dist/core/preflight/plan.js +75 -32
  94. package/dist/core/preflight/projection.d.ts +6 -3
  95. package/dist/core/preflight/projection.js +22 -2
  96. package/dist/core/preflight/reduce.js +47 -7
  97. package/dist/core/preflight/witness-evidence.d.ts +5 -5
  98. package/dist/core/preflight/witness-evidence.js +73 -20
  99. package/dist/core/schemas/artifact.d.ts +1049 -30
  100. package/dist/core/schemas/constraint-ledger.js +51 -0
  101. package/dist/core/schemas/defect-signature.d.ts +695 -0
  102. package/dist/core/schemas/defect-signature.js +175 -0
  103. package/dist/core/schemas/eval-contract.d.ts +550 -12
  104. package/dist/core/schemas/eval-contract.js +32 -2
  105. package/dist/core/schemas/evidence-artifact.d.ts +109 -19
  106. package/dist/core/schemas/evidence-artifact.js +64 -8
  107. package/dist/core/schemas/interface.d.ts +741 -13
  108. package/dist/core/schemas/interface.js +120 -7
  109. package/dist/core/schemas/isolation-manifest.js +16 -9
  110. package/dist/core/schemas/plan.d.ts +329 -3
  111. package/dist/core/schemas/plan.js +89 -3
  112. package/dist/core/schemas/pointer.d.ts +47 -1
  113. package/dist/core/schemas/pointer.js +89 -8
  114. package/dist/core/schemas/port-messages.d.ts +194 -10
  115. package/dist/core/schemas/port-messages.js +73 -6
  116. package/dist/core/schemas/probe-body.d.ts +18 -0
  117. package/dist/core/schemas/probe-body.js +13 -0
  118. package/dist/core/schemas/probe-qualification.d.ts +168 -0
  119. package/dist/core/schemas/probe-qualification.js +106 -0
  120. package/dist/core/schemas/probe.d.ts +431 -0
  121. package/dist/core/schemas/probe.js +6 -2
  122. package/dist/core/schemas/scoring-policy.d.ts +1 -0
  123. package/dist/core/schemas/scoring-policy.js +6 -1
  124. package/dist/core/schemas/sealed-evaluator-brief.d.ts +1 -0
  125. package/dist/core/schemas/sealed-evaluator-brief.js +4 -1
  126. package/dist/core/schemas/sealed-run-record.d.ts +161 -12
  127. package/dist/core/schemas/sealed-run-record.js +92 -15
  128. package/dist/core/schemas/sensitivity-witness.d.ts +108 -6
  129. package/dist/core/schemas/sensitivity-witness.js +61 -5
  130. package/dist/core/score/binding-order.d.ts +31 -0
  131. package/dist/core/score/binding-order.js +88 -0
  132. package/dist/core/score/bindings.d.ts +93 -0
  133. package/dist/core/score/bindings.js +309 -0
  134. package/dist/core/score/ladder-table.d.ts +23 -0
  135. package/dist/core/score/ladder-table.js +109 -0
  136. package/dist/core/score/ladder.d.ts +229 -0
  137. package/dist/core/score/ladder.js +464 -0
  138. package/dist/core/score/mode-agreement.d.ts +30 -0
  139. package/dist/core/score/mode-agreement.js +16 -0
  140. package/dist/core/score/outcome-table.d.ts +17 -0
  141. package/dist/core/score/outcome-table.js +172 -0
  142. package/dist/core/score/outcome.d.ts +466 -0
  143. package/dist/core/score/outcome.js +562 -0
  144. package/dist/core/score/qualification.d.ts +83 -0
  145. package/dist/core/score/qualification.js +645 -0
  146. package/dist/core/score/quotation.d.ts +56 -0
  147. package/dist/core/score/quotation.js +226 -0
  148. package/dist/core/score/reduce-trials.d.ts +80 -0
  149. package/dist/core/score/reduce-trials.js +90 -0
  150. package/dist/core/score/score.d.ts +72 -0
  151. package/dist/core/score/score.js +607 -0
  152. package/dist/core/score/selection.d.ts +80 -0
  153. package/dist/core/score/selection.js +98 -0
  154. package/dist/core/score/strength.d.ts +58 -0
  155. package/dist/core/score/strength.js +227 -0
  156. package/dist/core/score/witness.d.ts +138 -0
  157. package/dist/core/score/witness.js +320 -0
  158. package/dist/core/seal/derived-reference.d.ts +2 -9
  159. package/dist/core/seal/derived-reference.js +293 -46
  160. package/dist/core/seal/plan-index.d.ts +18 -3
  161. package/dist/core/seal/plan-index.js +44 -8
  162. package/dist/core/seal/seal.js +8 -3
  163. package/dist/core/stage-contracts.d.ts +62 -2
  164. package/dist/index.d.ts +1 -1
  165. package/dist/index.js +1 -1
  166. package/dist/ports/environment-probe-port.d.ts +60 -4
  167. package/dist/testing/index.d.ts +2 -0
  168. package/dist/testing/index.js +1 -0
  169. package/dist/testing/probe-conformance.d.ts +29 -0
  170. package/dist/testing/probe-conformance.js +34 -5
  171. package/package.json +12 -3
  172. package/schemas/eval-contract.schema.json +1787 -797
  173. package/schemas/evidence-artifact.schema.json +126 -9
  174. package/schemas/isolation-manifest.schema.json +17 -10
  175. package/schemas/probe.schema.json +636 -5
  176. package/schemas/rubric.schema.json +1 -1
  177. package/schemas/scoring-policy.schema.json +8 -1
  178. package/schemas/sealed-evaluator-brief.schema.json +11 -1
  179. package/schemas/sealed-run-record.schema.json +314 -40
@@ -8,25 +8,53 @@
8
8
  * the operation and the witness themselves: there is nothing to look an
9
9
  * identifier up in at reduce time.
10
10
  */
11
- import { checkInputsAgainstShape } from '../compile/sensitivity-witness.js';
12
- import { declaresNoRequiredKeys } from '../declared-inputs.js';
11
+ import { checkInputsAgainstShape, isApiWitnessInputs, } from '../compile/sensitivity-witness.js';
12
+ import { declaresNoRequiredKeys, isCommandOperation, } from '../declared-inputs.js';
13
+ import { referenceSetKeysOf } from '../evaluate/evidence-resolution.js';
13
14
  import { StructuralFailure } from '../failure-codes.js';
15
+ import { operationsOf } from '../schemas/interface.js';
14
16
  import { referenceSetMembers } from './witness-evidence.js';
15
- const requestOf = (legId, interfaceId, operation, inputs) => ({
16
- // NFR9's correlation by identifier: the port echoes this back, and it is
17
- // the only thing that tells two legs of one operation apart.
18
- probeId: legId,
19
- interfaceId,
20
- operationId: operation.operationId,
21
- method: operation.method,
22
- pathTemplate: operation.pathTemplate,
23
- channels: {
24
- path: inputs.path,
25
- query: inputs.query,
26
- header: inputs.header,
27
- body: inputs.body,
28
- },
29
- });
17
+ const requestOf = (legId, interfaceId, operation, inputs) => {
18
+ // NFR9's correlation by identifier: the port echoes these back, and they
19
+ // are the only thing that tells two legs of one operation apart.
20
+ const correlation = {
21
+ probeId: legId,
22
+ interfaceId,
23
+ operationId: operation.operationId,
24
+ };
25
+ if (isCommandOperation(operation)) {
26
+ if (isApiWitnessInputs(inputs)) {
27
+ throw new StructuralFailure('undeclared-mandatory-input', `EvalContract.permittedInterfaces[logicalId=${interfaceId}].operations[operationId=${operation.operationId}]`, `leg "${legId}" supplies transport channels to an operation that runs behind a command (AD-10, AD-19)`);
28
+ }
29
+ return {
30
+ ...correlation,
31
+ kind: 'cli',
32
+ executable: operation.invocation.executable,
33
+ subcommandPath: operation.invocation.subcommandPath,
34
+ channels: {
35
+ argument: inputs.argument,
36
+ option: inputs.option,
37
+ environment: inputs.environment,
38
+ stdin: inputs.stdin,
39
+ },
40
+ };
41
+ }
42
+ if (!isApiWitnessInputs(inputs)) {
43
+ throw new StructuralFailure('undeclared-mandatory-input', `EvalContract.permittedInterfaces[logicalId=${interfaceId}].operations[operationId=${operation.operationId}]`, `leg "${legId}" supplies command channels to an operation that speaks HTTP (AD-10, AD-19)`);
44
+ }
45
+ return {
46
+ ...correlation,
47
+ kind: 'api',
48
+ method: operation.method,
49
+ pathTemplate: operation.pathTemplate,
50
+ channels: {
51
+ path: inputs.path,
52
+ query: inputs.query,
53
+ header: inputs.header,
54
+ body: inputs.body,
55
+ },
56
+ };
57
+ };
30
58
  /**
31
59
  * The contract declares no identifier for a control leg, so one is minted, and it
32
60
  * has to avoid every identifier the contract already spends. The same input
@@ -57,13 +85,19 @@ const declaredLegIds = (contract, probes) => {
57
85
  taken.add(legId);
58
86
  }
59
87
  for (const iface of contract.permittedInterfaces)
60
- for (const operation of iface.operations)
88
+ for (const operation of operationsOf(iface))
61
89
  for (const leg of operation.sensitivityWitness?.legs ?? [])
62
90
  taken.add(leg.legId);
63
91
  if (contract.fixtureReset !== null)
64
92
  taken.add(contract.fixtureReset.legId);
65
93
  return taken;
66
94
  };
95
+ const EMPTY_COMMAND_INPUTS = {
96
+ argument: {},
97
+ option: {},
98
+ environment: {},
99
+ stdin: { kind: 'absent' },
100
+ };
67
101
  const EMPTY_INPUTS = {
68
102
  path: {},
69
103
  query: {},
@@ -85,7 +119,12 @@ const controlInputs = (operation) => {
85
119
  const declared = operation.sensitivityWitness?.legs[0]?.inputs;
86
120
  if (declared !== undefined)
87
121
  return declared;
88
- return declaresNoRequiredKeys(operation) ? EMPTY_INPUTS : null;
122
+ // AD-10 exempts an operation with no required key, so an empty leg is a
123
+ // legal request for it. Which empty leg depends on the kind, because the
124
+ // port sends what the operation's own channels name.
125
+ if (!declaresNoRequiredKeys(operation))
126
+ return null;
127
+ return isCommandOperation(operation) ? EMPTY_COMMAND_INPUTS : EMPTY_INPUTS;
89
128
  };
90
129
  const targetOf = (iface, operation) => {
91
130
  const inputs = controlInputs(operation);
@@ -98,10 +137,10 @@ const targetOf = (iface, operation) => {
98
137
  * marker-true operation on the interface the fixture reset names, resolved
99
138
  * independently of where the observed read came from.
100
139
  */
101
- const selectControl = (contract) => {
140
+ const selectControl = (contract, permittedInterfaces) => {
102
141
  let observed = null;
103
- for (const iface of contract.permittedInterfaces) {
104
- for (const operation of iface.operations) {
142
+ for (const iface of permittedInterfaces) {
143
+ for (const operation of operationsOf(iface)) {
105
144
  if (operation.stateChangeMarker)
106
145
  continue;
107
146
  observed = targetOf(iface, operation);
@@ -116,11 +155,13 @@ const selectControl = (contract) => {
116
155
  const reset = contract.fixtureReset;
117
156
  if (reset === null)
118
157
  return { observed, mutating: null };
119
- const resetInterface = contract.permittedInterfaces.find((candidate) => candidate.logicalId === reset.interfaceId);
158
+ const resetInterface = permittedInterfaces.find((candidate) => candidate.logicalId === reset.interfaceId);
120
159
  let mutating = null;
121
160
  for (const operation of resetInterface?.operations ?? []) {
122
161
  if (!operation.stateChangeMarker)
123
162
  continue;
163
+ if (resetInterface === undefined)
164
+ break;
124
165
  mutating = targetOf(resetInterface, operation);
125
166
  if (mutating !== null)
126
167
  break;
@@ -141,12 +182,13 @@ const selectControl = (contract) => {
141
182
  export const planPreflight = (input) => {
142
183
  const { contract, probes, runId } = input;
143
184
  for (const iface of contract.permittedInterfaces) {
144
- // Already thrown at compile; asserted again because the plan is reachable
145
- // from a caller who assembled a contract by hand.
146
- if (iface.kind !== 'api') {
147
- throw new StructuralFailure('unsupported-interface-kind', `EvalContract.permittedInterfaces[logicalId=${iface.logicalId}].kind`, `"${iface.kind}" is not supported in v0; only "api" is (AD-10)`);
185
+ // Already thrown at compile; asserted again because the plan is
186
+ // reachable from a caller who assembled a contract by hand.
187
+ if (iface.kind !== 'api' && iface.kind !== 'cli') {
188
+ throw new StructuralFailure('unsupported-interface-kind', `EvalContract.permittedInterfaces[logicalId=${iface.logicalId}].kind`, `"${iface.kind}" is not supported; "api" and "cli" are (AD-10)`);
148
189
  }
149
190
  }
191
+ const permittedInterfaces = contract.permittedInterfaces;
150
192
  const origins = [];
151
193
  const checks = [];
152
194
  // Keyed by interface AND operation: `Operation.operationId` is scoped to a
@@ -175,8 +217,8 @@ export const planPreflight = (input) => {
175
217
  group.push(legId);
176
218
  };
177
219
  // 1. the sensitivity legs and their checks
178
- contract.permittedInterfaces.forEach((iface, interfaceIndex) => {
179
- iface.operations.forEach((operation, operationIndex) => {
220
+ permittedInterfaces.forEach((iface, interfaceIndex) => {
221
+ operationsOf(iface).forEach((operation, operationIndex) => {
180
222
  const witness = operation.sensitivityWitness;
181
223
  const path = `EvalContract.permittedInterfaces[${interfaceIndex}].operations[${operationIndex}]`;
182
224
  if (witness !== null)
@@ -193,7 +235,7 @@ export const planPreflight = (input) => {
193
235
  });
194
236
  });
195
237
  // 3. the control legs, once per contract
196
- const control = selectControl(contract);
238
+ const control = selectControl(contract, permittedInterfaces);
197
239
  const controlLegIds = [];
198
240
  if (control !== null) {
199
241
  const taken = declaredLegIds(contract, probes);
@@ -215,7 +257,7 @@ export const planPreflight = (input) => {
215
257
  const mutate = mintControl('preflight-control-mutate');
216
258
  addLeg(mutate, 'control-mutate', mutating.iface.logicalId, mutating.operation, mutating.inputs, `the minted control-mutate leg against "${mutating.operation.operationId}"`);
217
259
  controlLegIds.push(mutate);
218
- const resetOperation = mutating.iface.operations.find((candidate) => candidate.operationId === reset.operationId);
260
+ const resetOperation = operationsOf(mutating.iface).find((candidate) => candidate.operationId === reset.operationId);
219
261
  if (resetOperation === undefined) {
220
262
  throw new StructuralFailure('unreachable-check-evidence', 'EvalContract.fixtureReset', `the fixture reset names operation "${reset.operationId}" on interface "${reset.interfaceId}", which the contract does not declare (AD-10)`);
221
263
  }
@@ -246,7 +288,7 @@ export const planPreflight = (input) => {
246
288
  });
247
289
  continue;
248
290
  }
249
- const iface = contract.permittedInterfaces.find((candidate) => candidate.logicalId === witness.interfaceId);
291
+ const iface = permittedInterfaces.find((candidate) => candidate.logicalId === witness.interfaceId);
250
292
  const operation = iface?.operations.find((candidate) => candidate.operationId === witness.operationId);
251
293
  if (operation === undefined) {
252
294
  throw new StructuralFailure('unreachable-check-evidence', path, `the manifestation witness names operation "${witness.operationId}" on interface "${witness.interfaceId}", which the contract does not declare (AD-10)`);
@@ -309,5 +351,6 @@ export const planPreflight = (input) => {
309
351
  legs: origins.map((origin) => origin.leg),
310
352
  checks: [...presence, ...checks],
311
353
  referenceSets: referenceSetMembers(contract),
354
+ referenceSetKeys: referenceSetKeysOf(contract),
312
355
  };
313
356
  };
@@ -1,4 +1,4 @@
1
- import type { Operation } from '../schemas/interface.ts';
1
+ import type { AnyOperation } from '../schemas/interface.ts';
2
2
  import type { ProbeObservation, ProbeObservedBody } from '../schemas/port-messages.ts';
3
3
  export declare const PREFLIGHT_ARTIFACT_PATH = "PreflightVerdict";
4
4
  /**
@@ -17,7 +17,10 @@ export type ProjectedObservation = {
17
17
  readonly legId: string;
18
18
  readonly interfaceId: string;
19
19
  readonly operationId: string;
20
- readonly status: number;
20
+ /** the transport status, or `null` off an interface that speaks HTTP. */
21
+ readonly status: number | null;
22
+ /** the process exit code, or `null` on an interface that does. */
23
+ readonly exitCode: number | null;
21
24
  readonly body: ProbeObservedBody;
22
25
  };
23
26
  /**
@@ -27,7 +30,7 @@ export type ProjectedObservation = {
27
30
  */
28
31
  export declare function pruneVolatile(body: ProbeObservedBody, volatilePointers: readonly string[], artifactPath: string): ProbeObservedBody;
29
32
  /** One observation reduced to the projection, with its volatile pointers gone. */
30
- export declare function projectObservation(observation: ProbeObservation, operation: Operation, artifactPath: string): ProjectedObservation;
33
+ export declare function projectObservation(observation: ProbeObservation, operation: AnyOperation, artifactPath: string): ProjectedObservation;
31
34
  /**
32
35
  * AD-11's fixture digest. Sorted by leg id first, because NFR9 forbids any
33
36
  * stage from reading array position: two runs whose observations arrived in a
@@ -4,6 +4,7 @@
4
4
  * scoring version: two runs whose projections agree describe the same fixture.
5
5
  */
6
6
  import { digestComposite } from '../canonical/digest.js';
7
+ import { descriptorArtifactOf, descriptorChannelOf, } from '../declared-inputs.js';
7
8
  import { DESCRIPTOR_POINTER_PATTERN } from '../schemas/pointer.js';
8
9
  import { decodeTail } from '../seal/plan-index.js';
9
10
  export const PREFLIGHT_ARTIFACT_PATH = 'PreflightVerdict';
@@ -70,14 +71,33 @@ export function pruneVolatile(body, volatilePointers, artifactPath) {
70
71
  }
71
72
  /** One observation reduced to the projection, with its volatile pointers gone. */
72
73
  export function projectObservation(observation, operation, artifactPath) {
74
+ // The channel the operation's descriptor describes is the one AD-10's
75
+ // relation reads, so it is the one the volatile pointers are pruned from.
76
+ // Off an interface that speaks HTTP that is the response body; off a
77
+ // command it is the stream or the file the operation nominates.
78
+ const observed = observation.kind === 'api'
79
+ ? observation.body
80
+ : describedChannelOf(observation, operation);
73
81
  return {
74
82
  legId: observation.probeId,
75
83
  interfaceId: observation.interfaceId,
76
84
  operationId: observation.operationId,
77
- status: observation.status,
78
- body: pruneVolatile(observation.body, operation.volatilePointers, artifactPath),
85
+ // A command has no transport status. `null` is what a reader of this
86
+ // field asks about, and every such reader already handles it.
87
+ status: observation.kind === 'api' ? observation.status : null,
88
+ exitCode: observation.kind === 'api' ? null : observation.exitCode,
89
+ body: pruneVolatile(observed, operation.volatilePointers, artifactPath),
79
90
  };
80
91
  }
92
+ /** The observed value of whichever channel this operation's descriptor describes. */
93
+ function describedChannelOf(observation, operation) {
94
+ const artifactId = descriptorArtifactOf(operation);
95
+ if (artifactId !== null)
96
+ return observation.artifacts[artifactId] ?? { kind: 'absent' };
97
+ return descriptorChannelOf(operation) === 'stderr'
98
+ ? observation.stderr
99
+ : observation.stdout;
100
+ }
81
101
  /**
82
102
  * AD-11's fixture digest. Sorted by leg id first, because NFR9 forbids any
83
103
  * stage from reading array position: two runs whose observations arrived in a
@@ -16,10 +16,39 @@ import { evidenceOf, resolveWitnessRelation } from './witness-evidence.js';
16
16
  /**
17
17
  * AD-10 names no threshold, and this is the one the repository already speaks:
18
18
  * the published conformance suite ships `probe/observe-anomalous-status`, and
19
- * `ProbeObservation.status` is bounded to 100-599 at the port, so HTTP is
20
- * already assumed at that boundary.
19
+ * an api observation's `status` is bounded to 100-599 at the port.
21
20
  */
22
21
  const ANOMALOUS_STATUS = 400;
22
+ /**
23
+ * What makes a control leg anomalous, in the vocabulary of the kind it ran
24
+ * against, or `null` when nothing does. A command's analogue of a 4xx is a
25
+ * non-zero exit: both are the system saying the call did not go through, and
26
+ * AD-10's clean-control check is about exactly that.
27
+ */
28
+ function anomalyOf(observation) {
29
+ if (observation.kind === 'api') {
30
+ return observation.status >= ANOMALOUS_STATUS
31
+ ? `status ${observation.status}`
32
+ : null;
33
+ }
34
+ return observation.exitCode === 0 ? null : `exit code ${observation.exitCode}`;
35
+ }
36
+ /**
37
+ * Whether the observation answers the mechanism the leg asked about.
38
+ *
39
+ * `kind` alone, and deliberately not `interfaceId` or `operationId`: those two
40
+ * are already read by the `interface-present` check, which reports a mismatch
41
+ * as a failed verdict rather than a fault, and reclassifying them here would
42
+ * turn a shipped verdict into a thrown fault. `kind` has no such reader. An
43
+ * observation of the wrong mechanism is not a weaker answer to the question, it
44
+ * is an answer to a different one, and every check below would read it as the
45
+ * leg's own result.
46
+ */
47
+ function kindMismatch(leg, observation) {
48
+ if (leg.request.kind === observation.kind)
49
+ return undefined;
50
+ return `leg "${leg.legId}" asked for a "${leg.request.kind}" probe and was answered with a "${observation.kind}" observation, so the port answered a question nobody asked`;
51
+ }
23
52
  const check = (kind, operationId, outcome, note) => ({ kind, operationId, outcome, note });
24
53
  /**
25
54
  * Two projections describe the same fixture state. `legId` is excluded because
@@ -48,7 +77,7 @@ const resolveAgainst = (witness, state, plan, artifactPath) => {
48
77
  // Keyed by the witness's own leg id, since the relation addresses
49
78
  // `/interactions/{witness.legId}/…`. The scoped row asks whether that same
50
79
  // relation would fire on a clean leg's observation.
51
- return resolveWitnessRelation(witness.relation, { [witness.legId]: state.evidence }, state.leg.operation, plan.referenceSets, artifactPath).resolution;
80
+ return resolveWitnessRelation(witness.relation, { [witness.legId]: state.evidence }, state.leg.operation, plan.referenceSets, plan.referenceSetKeys, artifactPath).resolution;
52
81
  };
53
82
  export const reducePreflight = (plan, { observations }) => {
54
83
  const byProbeId = new Map();
@@ -68,12 +97,22 @@ export const reducePreflight = (plan, { observations }) => {
68
97
  const observation = byProbeId.get(leg.legId);
69
98
  if (observation === undefined)
70
99
  continue;
100
+ // The echo is checked, not assumed. Both port messages are unions, so a
101
+ // port can answer a command leg with a schema-valid HTTP observation and
102
+ // every check below reads it as the leg's own result: a command contract
103
+ // then passes pre-flight with four checks satisfied and no command ever
104
+ // run. `probeId` alone binds the two together and says nothing about
105
+ // whether the answer is to this question.
106
+ const mismatch = kindMismatch(leg, observation);
107
+ if (mismatch !== undefined) {
108
+ throw new RuntimeFault('port-contract-violation', PREFLIGHT_ARTIFACT_PATH, mismatch);
109
+ }
71
110
  const projected = projectObservation(observation, leg.operation, PREFLIGHT_ARTIFACT_PATH);
72
111
  states.set(leg.legId, {
73
112
  leg,
74
113
  observation,
75
114
  projected,
76
- evidence: evidenceOf(projected, observation, leg.inputs),
115
+ evidence: evidenceOf(projected, observation, leg.inputs, leg.operation),
77
116
  });
78
117
  }
79
118
  const reduceCheck = (planned) => {
@@ -105,7 +144,7 @@ export const reducePreflight = (plan, { observations }) => {
105
144
  return check(planned.kind, planned.operationId, 'failed', `witness leg "${leg.legId}" produced no observation`);
106
145
  evidence[leg.legId] = state.evidence;
107
146
  }
108
- const { resolution } = resolveWitnessRelation(witness.relation, evidence, operation, plan.referenceSets, PREFLIGHT_ARTIFACT_PATH);
147
+ const { resolution } = resolveWitnessRelation(witness.relation, evidence, operation, plan.referenceSets, plan.referenceSetKeys, PREFLIGHT_ARTIFACT_PATH);
109
148
  // AD-10's own sentence, and the most load-bearing line here: a
110
149
  // sensitivity check that examined nothing has established nothing,
111
150
  // so `insufficient-evidence` fails.
@@ -128,8 +167,9 @@ export const reducePreflight = (plan, { observations }) => {
128
167
  const state = states.get(legId);
129
168
  if (state === undefined)
130
169
  return check(planned.kind, null, 'failed', `control leg "${legId}" produced no observation`);
131
- if (state.observation.status >= ANOMALOUS_STATUS)
132
- return check(planned.kind, null, 'failed', `control leg "${legId}" observed status ${state.observation.status}`);
170
+ const anomaly = anomalyOf(state.observation);
171
+ if (anomaly !== null)
172
+ return check(planned.kind, null, 'failed', `control leg "${legId}" observed ${anomaly}`);
133
173
  }
134
174
  return check(planned.kind, null, 'satisfied', null);
135
175
  }
@@ -4,11 +4,11 @@
4
4
  * unchanged. This module supplies the `Observation` shape they read and a
5
5
  * collection predicate scoped to one operation.
6
6
  */
7
- import { type PointerDenotesCollection } from '../evaluate/resolution.ts';
7
+ import { type PointerDenotesCollection, type ReferenceSetKeys } from '../evaluate/resolution.ts';
8
8
  import type { EvalContract } from '../schemas/eval-contract.ts';
9
9
  import type { CheckResolutionValue } from '../schemas/evidence-artifact.ts';
10
10
  import type { Expression } from '../schemas/expression.ts';
11
- import type { Operation } from '../schemas/interface.ts';
11
+ import type { AnyOperation } from '../schemas/interface.ts';
12
12
  import type { ProbeObservation } from '../schemas/port-messages.ts';
13
13
  import type { JsonValue } from '../schemas/primitives.ts';
14
14
  import type { Observation } from '../schemas/sealed-run-record.ts';
@@ -27,7 +27,7 @@ export declare const PREFLIGHT_REGEX_MATCH_STEP_BUDGET = 1000000;
27
27
  * the projection does not carry. `provenance` is `baseline`: a pre-flight leg is
28
28
  * pre-canned by definition.
29
29
  */
30
- export declare function evidenceOf(projected: ProjectedObservation, observation: ProbeObservation, inputs: WitnessInputs): Observation;
30
+ export declare function evidenceOf(projected: ProjectedObservation, observation: ProbeObservation, inputs: WitnessInputs, operation: AnyOperation): Observation;
31
31
  /** the contract's declared reference sets in the shape the resolver wants. */
32
32
  export declare function referenceSetMembers(contract: EvalContract): Readonly<Record<string, JsonValue[]>>;
33
33
  /**
@@ -37,6 +37,6 @@ export declare function referenceSetMembers(contract: EvalContract): Readonly<Re
37
37
  * never appears, and fabricating step objects for a one-operation lookup is more
38
38
  * machinery than the predicate.
39
39
  */
40
- export declare function makeWitnessPointerDenotesCollection(operation: Operation): PointerDenotesCollection;
40
+ export declare function makeWitnessPointerDenotesCollection(operation: AnyOperation): PointerDenotesCollection;
41
41
  /** Resolves one witness relation over the legs it addresses. */
42
- export declare function resolveWitnessRelation(relation: Expression, legEvidence: Readonly<Record<string, Observation>>, operation: Operation, referenceSets: Readonly<Record<string, JsonValue[]>>, artifactPath: string): CheckResolutionValue;
42
+ export declare function resolveWitnessRelation(relation: Expression, legEvidence: Readonly<Record<string, Observation>>, operation: AnyOperation, referenceSets: Readonly<Record<string, JsonValue[]>>, referenceSetKeys: ReferenceSetKeys, artifactPath: string): CheckResolutionValue;
@@ -4,6 +4,7 @@
4
4
  * unchanged. This module supplies the `Observation` shape they read and a
5
5
  * collection predicate scoped to one operation.
6
6
  */
7
+ import { descriptorArtifactOf, descriptorChannelOf, } from '../declared-inputs.js';
7
8
  import { makeResolveOperand } from '../evaluate/evidence-resolution.js';
8
9
  import { resolveCheck, } from '../evaluate/resolution.js';
9
10
  import { decodeTail, parseEvidenceTarget } from '../seal/plan-index.js';
@@ -17,17 +18,46 @@ const asJsonObject = (value) => value !== null && typeof value === 'object' && !
17
18
  ? value
18
19
  : null;
19
20
  /**
20
- * `ObservedCallInputs` is narrower than `WitnessInputs`, so `body` loses
21
+ * `ObservedCallInputs` is narrower than `ApiWitnessInputs`, so `body` loses
21
22
  * information here: an absent body and a non-object JSON body both map to
22
23
  * `null`, and a relation addressing `/interactions/{legId}/call-inputs/body` on
23
24
  * such a leg resolves `ABSENT`.
24
25
  */
25
- const callInputsOf = (inputs) => ({
26
- path: inputs.path,
27
- query: inputs.query,
28
- header: inputs.header,
29
- body: inputs.body.kind === 'json' ? asJsonObject(inputs.body.value) : null,
30
- });
26
+ const ABSENT_CHANNEL = { kind: 'absent' };
27
+ const bodyValue = (body) => body.kind === 'absent' ? null : body.value;
28
+ /**
29
+ * The leg's supplied inputs as the record spells them: every channel the leg
30
+ * did not use is `null` rather than absent, which is the observation's own
31
+ * convention for an unused channel.
32
+ */
33
+ const callInputsOf = (inputs) => {
34
+ const empty = {
35
+ path: null,
36
+ query: null,
37
+ header: null,
38
+ body: null,
39
+ argument: null,
40
+ option: null,
41
+ environment: null,
42
+ stdin: null,
43
+ };
44
+ if ('body' in inputs) {
45
+ return {
46
+ ...empty,
47
+ path: inputs.path,
48
+ query: inputs.query,
49
+ header: inputs.header,
50
+ body: inputs.body.kind === 'json' ? asJsonObject(inputs.body.value) : null,
51
+ };
52
+ }
53
+ return {
54
+ ...empty,
55
+ argument: inputs.argument,
56
+ option: inputs.option,
57
+ environment: inputs.environment,
58
+ stdin: inputs.stdin.kind === 'json' ? asJsonObject(inputs.stdin.value) : null,
59
+ };
60
+ };
31
61
  /**
32
62
  * One leg as the `Observation` `makeResolveOperand` takes. The relation reads the
33
63
  * **projected** body, which is AD-10's "evaluated over that operation's response
@@ -35,23 +65,46 @@ const callInputsOf = (inputs) => ({
35
65
  * the projection does not carry. `provenance` is `baseline`: a pre-flight leg is
36
66
  * pre-canned by definition.
37
67
  */
38
- export function evidenceOf(projected, observation, inputs) {
68
+ export function evidenceOf(projected, observation, inputs, operation) {
39
69
  const { body } = projected;
70
+ const descriptorChannel = descriptorChannelOf(operation);
71
+ const describedArtifact = descriptorArtifactOf(operation);
40
72
  return {
41
73
  observationId: projected.legId,
74
+ // A synthetic, single-observation shape built fresh per leg and never
75
+ // collected alongside a sibling: pre-flight resolves one leg's witness
76
+ // relation at a time, so the schema's ordering and uniqueness concerns
77
+ // (owed item 2) have nothing to apply to here. Constant, since no reader
78
+ // of this value cares which leg it was. This stops being safe if
79
+ // pre-flight ever needs to assemble multiple legs' observations
80
+ // together: AD-40 already names such a future need, pair-defect signing
81
+ // across the monotonic sequence of owed item 2, so revisit this
82
+ // constant then.
83
+ sequence: 1,
42
84
  operationId: projected.operationId,
43
85
  provenance: 'baseline',
86
+ // A pre-flight leg is issued by this package rather than by a harness
87
+ // acting as a declared account, so it names no principal.
88
+ principal: null,
44
89
  callInputs: callInputsOf(inputs),
45
- responseBody: body.kind === 'json'
46
- ? body.value
47
- : body.kind === 'text'
48
- ? body.value
49
- : null,
50
- responseHeaders: observation.headers,
51
- responseStatus: observation.status,
52
- stdout: null,
53
- stderr: null,
54
- exitCode: null,
90
+ // The projected body is whichever channel the operation's descriptor
91
+ // describes, so it lands on the channel a relation addresses. Both are
92
+ // filled from the same projection rather than one being derived from
93
+ // the other, and every channel the leg did not observe is written down
94
+ // as unobserved rather than left to a default.
95
+ responseBody: observation.kind === 'api' ? bodyValue(body) : null,
96
+ responseHeaders: observation.kind === 'api' ? observation.headers : null,
97
+ responseStatus: observation.kind === 'api' ? observation.status : null,
98
+ stdout: observation.kind === 'api' || descriptorChannel !== 'stdout'
99
+ ? ABSENT_CHANNEL
100
+ : body,
101
+ stderr: observation.kind === 'api' || descriptorChannel !== 'stderr'
102
+ ? ABSENT_CHANNEL
103
+ : body,
104
+ exitCode: observation.kind === 'api' ? null : observation.exitCode,
105
+ artifacts: observation.kind === 'api' || describedArtifact === null
106
+ ? {}
107
+ : { [describedArtifact]: body },
55
108
  };
56
109
  }
57
110
  /** the contract's declared reference sets in the shape the resolver wants. */
@@ -83,6 +136,6 @@ export function makeWitnessPointerDenotesCollection(operation) {
83
136
  };
84
137
  }
85
138
  /** Resolves one witness relation over the legs it addresses. */
86
- export function resolveWitnessRelation(relation, legEvidence, operation, referenceSets, artifactPath) {
87
- return resolveCheck(relation, makeResolveOperand(legEvidence, referenceSets), makeWitnessPointerDenotesCollection(operation), PREFLIGHT_REGEX_MATCH_STEP_BUDGET, artifactPath);
139
+ export function resolveWitnessRelation(relation, legEvidence, operation, referenceSets, referenceSetKeys, artifactPath) {
140
+ return resolveCheck(relation, makeResolveOperand(legEvidence, referenceSets), makeWitnessPointerDenotesCollection(operation), referenceSetKeys, PREFLIGHT_REGEX_MATCH_STEP_BUDGET, artifactPath);
88
141
  }