eval-quality 0.2.0 → 0.5.0

package/dist/core/excluded-content.js

@@ -0,0 +1,113 @@
+/**
+ * AD-18's excluded categories, as patterns that read a value rather than a name.
+ *
+ * AD-18 excludes "credentials, tokens, real names, email addresses, account
+ * identifiers, and transaction content" from every artifact this package
+ * produces or publishes, and binds "published examples and test fixtures as
+ * strictly as real runs". Until this module the whole prohibition was carried by
+ * one test that scans `corpus/dev`, so a contract carrying a live-shaped token
+ * compiled clean and the token travelled into the sealed brief.
+ *
+ * Only the value-shaped half of that scan ships here, and the split is the point
+ * rather than an economy. AD-18's own reading of the rule for the header channel
+ * is that a declaration "names the header and its type and never carries a
+ * credential value", so a name is what an author is supposed to write: an
+ * operation declaring a `password` body key or an `Authorization` header is a
+ * contract for an authentication API, and a gate matching `\bpassword\b` would
+ * reject exactly the systems this package exists to evaluate. The patterns below
+ * therefore match only strings whose shape is decidable without knowing what
+ * field they sit in: a PEM header, a token with an issuer prefix, an address, a
+ * checkable account number, a card-shaped digit run. The name-shaped patterns
+ * stay in `tests/architecture/dev-corpus.test.ts`, where review rather than a
+ * compiler decides.
+ *
+ * Each pattern carries a string it must fire on, and a test asserts every one
+ * still does. A pattern that has rotted into a regex matching nothing is worse
+ * than no pattern: it reports clean forever.
+ */
+/** AD-18's six categories, in the rule's own order. */
+export const EXCLUDED_CATEGORIES = [
+    'credentials',
+    'tokens',
+    'real names',
+    'email addresses',
+    'account identifiers',
+    'transaction content',
+];
+/**
+ * `real names` has no member here and never will: a personal name has no surface
+ * form, so every pattern that reaches one reads the field that carries it, which
+ * is a name-shaped test. The category stays in the tuple above because AD-18
+ * names six and dropping one from the vocabulary would make the gate look total.
+ */
+export const EXCLUDED_VALUE_PATTERNS = [
+    {
+        category: 'credentials',
+        regex: /-----BEGIN [A-Z ]*PRIVATE KEY-----/,
+        fires: '-----BEGIN RSA PRIVATE KEY-----',
+    },
+    {
+        category: 'tokens',
+        regex: /(?<![A-Za-z])(?:[Bb]earer\s+[\w.~+/-]{16,}|eyJ[\w-]{8,}\.[\w-]{8,}|sk-[A-Za-z0-9]{16,}|gh[pousr]_[A-Za-z0-9]{16,}|AKIA[0-9A-Z]{16})/,
+        fires: 'Authorization: Bearer abcdefghijklmnopqrstuv',
+    },
+    {
+        category: 'email addresses',
+        regex: /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/,
+        fires: '{"contact":"ada@example.com"}',
+    },
+    {
+        // Case-sensitive: an IBAN is upper case, and a case-insensitive form
+        // matches a run inside a lower-case hex digest.
+        category: 'account identifiers',
+        regex: /\b[A-Z]{2}\d{2}[A-Z0-9]{11,30}\b|\b\d{3}-\d{2}-\d{4}\b/,
+        fires: 'GB29NWBK60161331926819',
+    },
+    {
+        category: 'transaction content',
+        regex: /(?<![0-9A-Za-z])\d{4}(?:[ -]\d{4}){3}(?![0-9A-Za-z])/,
+        fires: 'charged 4111 1111 1111 1111',
+    },
+];
+const step = (path, key) => typeof key === 'number' ? `${path}[${key}]` : `${path}.${key}`;
+/**
+ * Every string anywhere inside one value, against every value-shaped pattern.
+ *
+ * Object keys are walked as paths and never scanned as content: a key is a name
+ * by construction, which is the same line the pattern set is drawn on. The walk
+ * is total over JSON and stops at nothing, so a token nested under a body
+ * example twelve levels down is found; the first matching category per string
+ * wins, since one string reported six times says nothing more than once.
+ *
+ * Order is the walk's order, which for an object is its own key order. A caller
+ * that needs a stable report over two spellings of the same object sorts what
+ * comes back; the compile gate below reads only the first hit and never the
+ * order.
+ */
+export function scanExcludedContent(value, path) {
+    const hits = [];
+    const visit = (node, at) => {
+        if (typeof node === 'string') {
+            for (const { category, regex } of EXCLUDED_VALUE_PATTERNS) {
+                const match = regex.exec(node);
+                if (match === null)
+                    continue;
+                hits.push({ category, path: at, match: match[0] });
+                return;
+            }
+            return;
+        }
+        if (Array.isArray(node)) {
+            node.forEach((element, index) => {
+                visit(element, step(at, index));
+            });
+            return;
+        }
+        if (node === null || typeof node !== 'object')
+            return;
+        for (const [key, child] of Object.entries(node))
+            visit(child, step(at, key));
+    };
+    visit(value, path);
+    return hits;
+}

package/dist/core/failure-codes.d.ts

@@ -1,5 +1,5 @@
-/** AD-5's twenty-three compile-time failure codes, as data, in the table's order. */
-export declare const FAILURE_CODES: readonly ['missing-requirement-linkage', 'no-observable-success-criterion', 'unreachable-check-evidence', 'malformed-operator-expression', 'quantifier-over-non-collection', 'quantifier-nesting-exceeded', 'unresolved-reference-set', 'duplicate-operation-signature', 'undeclared-mandatory-input', 'oracle-missing-channel', 'direction-check-misaligned', 'unsupported-interface-kind', 'nested-temporal-clause', 'plan-exceeds-scripting-bound', 'binding-cycle', 'captured-channel-undeclared', 'rubric-scores-reasoning-prose', 'rubric-unanchored', 'rubric-evidence-unreachable', 'forbidden-input-floor-incomplete', 'scoped-reference-resolves-forbidden', 'waiver-incomplete', 'brief-exceeds-scripting-bound'];
+/** AD-5's twenty-six compile-time failure codes, as data, in the table's order. */
+export declare const FAILURE_CODES: readonly ['missing-requirement-linkage', 'no-observable-success-criterion', 'unreachable-check-evidence', 'malformed-operator-expression', 'quantifier-over-non-collection', 'quantifier-nesting-exceeded', 'unresolved-reference-set', 'duplicate-operation-signature', 'undeclared-mandatory-input', 'oracle-missing-channel', 'direction-check-misaligned', 'unsupported-interface-kind', 'nested-temporal-clause', 'plan-exceeds-scripting-bound', 'binding-cycle', 'captured-channel-undeclared', 'rubric-scores-reasoning-prose', 'rubric-unanchored', 'rubric-evidence-unreachable', 'forbidden-input-floor-incomplete', 'scoped-reference-resolves-forbidden', 'waiver-incomplete', 'brief-exceeds-scripting-bound', 'unresolved-artifact-reference', 'irreducible-step-reference', 'excluded-content-in-declaration'];
 export type FailureCode = (typeof FAILURE_CODES)[number];
 /**
  * AD-5's compile-time failure registry: thrown when a structural error fails

package/dist/core/failure-codes.js

@@ -1,11 +1,11 @@
-/** AD-5's twenty-three compile-time failure codes, as data, in the table's order. */
+/** AD-5's twenty-six compile-time failure codes, as data, in the table's order. */
 // This lives in `core/`, since it's a plain data tuple and `core/schemas/`
 // is reserved for Zod definitions. The compiler that emits these codes lives
 // in `core/compile/`, one directory over.
 //
 // `scripts/check-ad5-registry.ts` checks this tuple against AD-5's own
 // registry table under `npm run validate`; the tuple's own invariants
-// (twenty-three members, unique, kebab-case) are locked in
+// (twenty-six members, unique, kebab-case) are locked in
 // tests/schemas/failure-codes.test.ts.
 export const FAILURE_CODES = [
     'missing-requirement-linkage',
@@ -31,6 +31,9 @@ export const FAILURE_CODES = [
     'scoped-reference-resolves-forbidden',
     'waiver-incomplete',
     'brief-exceeds-scripting-bound',
+    'unresolved-artifact-reference',
+    'irreducible-step-reference',
+    'excluded-content-in-declaration',
 ];
 /**
  * AD-5's compile-time failure registry: thrown when a structural error fails

package/dist/core/ingest/conditions.d.ts

@@ -67,7 +67,7 @@ export type AgreementField = (typeof AGREEMENT_FIELDS)[number];
  */
 export type DuplicateRecordIdentifier = {
     readonly kind: Extract<IngestConditionKind, 'duplicate-record-identifier'>;
-    readonly subject: 'observation' | 'finding' | 'oracle-disposition';
+    readonly subject: 'observation' | 'finding' | 'oracle-disposition' | 'judge-result';
     readonly identifier: string;
     readonly occurrences: number;
 };

package/dist/core/ingest/ingest.js

@@ -106,6 +106,14 @@ export const ingest = (record, manifest, configuration) => {
             'oracle-disposition',
             record.oracleDispositions.map((disposition) => disposition.oracleId),
         ],
+        // AD-17's "one judge call scoring all named criteria", read as far as
+        // one record can be read: a criterion scored twice is not one call's
+        // product whatever the rest of the record says. Keyed by the pair,
+        // since two rubrics may each declare a criterion of the same name.
+        [
+            'judge-result',
+            record.judgeResults.map((result) => `${result.rubricId}/${result.criterionId}`),
+        ],
     ]) {
         const counts = new Map();
         for (const identifier of identifiers) {

package/dist/core/preflight/plan.d.ts

@@ -1,5 +1,6 @@
+import type { ReferenceSetKeys } from '../evaluate/resolution.ts';
 import type { EvalContract } from '../schemas/eval-contract.ts';
-import type { Operation } from '../schemas/interface.ts';
+import type { AnyOperation } from '../schemas/interface.ts';
 import type { ProbeRequest } from '../schemas/port-messages.ts';
 import type { JsonValue } from '../schemas/primitives.ts';
 import type { Probe } from '../schemas/probe.ts';
@@ -15,7 +16,7 @@ export type PlannedLeg = {
     readonly legId: string;
     readonly purpose: PlannedLegPurpose;
     readonly request: ProbeRequest;
-    readonly operation: Operation;
+    readonly operation: AnyOperation;
     readonly inputs: WitnessInputs;
 };
 export type PlannedCheck = {
@@ -28,7 +29,7 @@ export type PlannedCheck = {
     readonly interfaceId: string;
     readonly operationId: string;
     readonly witness: SensitivityWitness | null;
-    readonly operation: Operation;
+    readonly operation: AnyOperation;
 } | {
     readonly kind: 'state-reset';
     readonly legIds: readonly [string, string];
@@ -39,19 +40,20 @@ export type PlannedCheck = {
     readonly kind: 'seeded-faults-scoped';
     readonly defectId: string;
     readonly witness: ManifestationWitness;
-    readonly operation: Operation;
+    readonly operation: AnyOperation;
     readonly cleanLegIds: readonly string[];
 } | {
     readonly kind: 'seeded-fault-fired';
     readonly defectId: string;
     readonly witness: ManifestationWitness | null;
-    readonly operation: Operation | null;
+    readonly operation: AnyOperation | null;
 };
 export type PreflightPlan = {
     readonly runId: string;
     readonly legs: readonly PlannedLeg[];
     readonly checks: readonly PlannedCheck[];
     readonly referenceSets: Readonly<Record<string, JsonValue[]>>;
+    readonly referenceSetKeys: ReferenceSetKeys;
 };
 /**
  * AD-10's plan, derived from the interfaces the contract's probes exercise.

package/dist/core/preflight/plan.js

@@ -8,25 +8,53 @@
  * the operation and the witness themselves: there is nothing to look an
  * identifier up in at reduce time.
  */
-import { checkInputsAgainstShape } from '../compile/sensitivity-witness.js';
-import { declaresNoRequiredKeys } from '../declared-inputs.js';
+import { checkInputsAgainstShape, isApiWitnessInputs, } from '../compile/sensitivity-witness.js';
+import { declaresNoRequiredKeys, isCommandOperation, } from '../declared-inputs.js';
+import { referenceSetKeysOf } from '../evaluate/evidence-resolution.js';
 import { StructuralFailure } from '../failure-codes.js';
+import { operationsOf } from '../schemas/interface.js';
 import { referenceSetMembers } from './witness-evidence.js';
-const requestOf = (legId, interfaceId, operation, inputs) => ({
-    // NFR9's correlation by identifier: the port echoes this back, and it is
-    // the only thing that tells two legs of one operation apart.
-    probeId: legId,
-    interfaceId,
-    operationId: operation.operationId,
-    method: operation.method,
-    pathTemplate: operation.pathTemplate,
-    channels: {
-        path: inputs.path,
-        query: inputs.query,
-        header: inputs.header,
-        body: inputs.body,
-    },
-});
+const requestOf = (legId, interfaceId, operation, inputs) => {
+    // NFR9's correlation by identifier: the port echoes these back, and they
+    // are the only thing that tells two legs of one operation apart.
+    const correlation = {
+        probeId: legId,
+        interfaceId,
+        operationId: operation.operationId,
+    };
+    if (isCommandOperation(operation)) {
+        if (isApiWitnessInputs(inputs)) {
+            throw new StructuralFailure('undeclared-mandatory-input', `EvalContract.permittedInterfaces[logicalId=${interfaceId}].operations[operationId=${operation.operationId}]`, `leg "${legId}" supplies transport channels to an operation that runs behind a command (AD-10, AD-19)`);
+        }
+        return {
+            ...correlation,
+            kind: 'cli',
+            executable: operation.invocation.executable,
+            subcommandPath: operation.invocation.subcommandPath,
+            channels: {
+                argument: inputs.argument,
+                option: inputs.option,
+                environment: inputs.environment,
+                stdin: inputs.stdin,
+            },
+        };
+    }
+    if (!isApiWitnessInputs(inputs)) {
+        throw new StructuralFailure('undeclared-mandatory-input', `EvalContract.permittedInterfaces[logicalId=${interfaceId}].operations[operationId=${operation.operationId}]`, `leg "${legId}" supplies command channels to an operation that speaks HTTP (AD-10, AD-19)`);
+    }
+    return {
+        ...correlation,
+        kind: 'api',
+        method: operation.method,
+        pathTemplate: operation.pathTemplate,
+        channels: {
+            path: inputs.path,
+            query: inputs.query,
+            header: inputs.header,
+            body: inputs.body,
+        },
+    };
+};
 /**
  * The contract declares no identifier for a control leg, so one is minted, and it
  * has to avoid every identifier the contract already spends. The same input
@@ -57,13 +85,19 @@ const declaredLegIds = (contract, probes) => {
                     taken.add(legId);
             }
     for (const iface of contract.permittedInterfaces)
-        for (const operation of iface.operations)
+        for (const operation of operationsOf(iface))
             for (const leg of operation.sensitivityWitness?.legs ?? [])
                 taken.add(leg.legId);
     if (contract.fixtureReset !== null)
         taken.add(contract.fixtureReset.legId);
     return taken;
 };
+const EMPTY_COMMAND_INPUTS = {
+    argument: {},
+    option: {},
+    environment: {},
+    stdin: { kind: 'absent' },
+};
 const EMPTY_INPUTS = {
     path: {},
     query: {},
@@ -85,7 +119,12 @@ const controlInputs = (operation) => {
     const declared = operation.sensitivityWitness?.legs[0]?.inputs;
     if (declared !== undefined)
         return declared;
-    return declaresNoRequiredKeys(operation) ? EMPTY_INPUTS : null;
+    // AD-10 exempts an operation with no required key, so an empty leg is a
+    // legal request for it. Which empty leg depends on the kind, because the
+    // port sends what the operation's own channels name.
+    if (!declaresNoRequiredKeys(operation))
+        return null;
+    return isCommandOperation(operation) ? EMPTY_COMMAND_INPUTS : EMPTY_INPUTS;
 };
 const targetOf = (iface, operation) => {
     const inputs = controlInputs(operation);
@@ -98,10 +137,10 @@ const targetOf = (iface, operation) => {
  * marker-true operation on the interface the fixture reset names, resolved
  * independently of where the observed read came from.
  */
-const selectControl = (contract) => {
+const selectControl = (contract, permittedInterfaces) => {
     let observed = null;
-    for (const iface of contract.permittedInterfaces) {
-        for (const operation of iface.operations) {
+    for (const iface of permittedInterfaces) {
+        for (const operation of operationsOf(iface)) {
             if (operation.stateChangeMarker)
                 continue;
             observed = targetOf(iface, operation);
@@ -116,11 +155,13 @@ const selectControl = (contract) => {
     const reset = contract.fixtureReset;
     if (reset === null)
         return { observed, mutating: null };
-    const resetInterface = contract.permittedInterfaces.find((candidate) => candidate.logicalId === reset.interfaceId);
+    const resetInterface = permittedInterfaces.find((candidate) => candidate.logicalId === reset.interfaceId);
     let mutating = null;
     for (const operation of resetInterface?.operations ?? []) {
         if (!operation.stateChangeMarker)
             continue;
+        if (resetInterface === undefined)
+            break;
         mutating = targetOf(resetInterface, operation);
         if (mutating !== null)
             break;
@@ -141,12 +182,13 @@ const selectControl = (contract) => {
 export const planPreflight = (input) => {
     const { contract, probes, runId } = input;
     for (const iface of contract.permittedInterfaces) {
-        // Already thrown at compile; asserted again because the plan is reachable
-        // from a caller who assembled a contract by hand.
-        if (iface.kind !== 'api') {
-            throw new StructuralFailure('unsupported-interface-kind', `EvalContract.permittedInterfaces[logicalId=${iface.logicalId}].kind`, `"${iface.kind}" is not supported in v0; only "api" is (AD-10)`);
+        // Already thrown at compile; asserted again because the plan is
+        // reachable from a caller who assembled a contract by hand.
+        if (iface.kind !== 'api' && iface.kind !== 'cli') {
+            throw new StructuralFailure('unsupported-interface-kind', `EvalContract.permittedInterfaces[logicalId=${iface.logicalId}].kind`, `"${iface.kind}" is not supported; "api" and "cli" are (AD-10)`);
         }
     }
+    const permittedInterfaces = contract.permittedInterfaces;
     const origins = [];
     const checks = [];
     // Keyed by interface AND operation: `Operation.operationId` is scoped to a
@@ -175,8 +217,8 @@ export const planPreflight = (input) => {
             group.push(legId);
     };
     // 1. the sensitivity legs and their checks
-    contract.permittedInterfaces.forEach((iface, interfaceIndex) => {
-        iface.operations.forEach((operation, operationIndex) => {
+    permittedInterfaces.forEach((iface, interfaceIndex) => {
+        operationsOf(iface).forEach((operation, operationIndex) => {
             const witness = operation.sensitivityWitness;
             const path = `EvalContract.permittedInterfaces[${interfaceIndex}].operations[${operationIndex}]`;
             if (witness !== null)
@@ -193,7 +235,7 @@ export const planPreflight = (input) => {
         });
     });
     // 3. the control legs, once per contract
-    const control = selectControl(contract);
+    const control = selectControl(contract, permittedInterfaces);
     const controlLegIds = [];
     if (control !== null) {
         const taken = declaredLegIds(contract, probes);
@@ -215,7 +257,7 @@ export const planPreflight = (input) => {
             const mutate = mintControl('preflight-control-mutate');
             addLeg(mutate, 'control-mutate', mutating.iface.logicalId, mutating.operation, mutating.inputs, `the minted control-mutate leg against "${mutating.operation.operationId}"`);
             controlLegIds.push(mutate);
-            const resetOperation = mutating.iface.operations.find((candidate) => candidate.operationId === reset.operationId);
+            const resetOperation = operationsOf(mutating.iface).find((candidate) => candidate.operationId === reset.operationId);
             if (resetOperation === undefined) {
                 throw new StructuralFailure('unreachable-check-evidence', 'EvalContract.fixtureReset', `the fixture reset names operation "${reset.operationId}" on interface "${reset.interfaceId}", which the contract does not declare (AD-10)`);
             }
@@ -246,7 +288,7 @@ export const planPreflight = (input) => {
                 });
                 continue;
             }
-            const iface = contract.permittedInterfaces.find((candidate) => candidate.logicalId === witness.interfaceId);
+            const iface = permittedInterfaces.find((candidate) => candidate.logicalId === witness.interfaceId);
             const operation = iface?.operations.find((candidate) => candidate.operationId === witness.operationId);
             if (operation === undefined) {
                 throw new StructuralFailure('unreachable-check-evidence', path, `the manifestation witness names operation "${witness.operationId}" on interface "${witness.interfaceId}", which the contract does not declare (AD-10)`);
@@ -309,5 +351,6 @@ export const planPreflight = (input) => {
         legs: origins.map((origin) => origin.leg),
         checks: [...presence, ...checks],
         referenceSets: referenceSetMembers(contract),
+        referenceSetKeys: referenceSetKeysOf(contract),
     };
 };

package/dist/core/preflight/projection.d.ts

@@ -1,4 +1,4 @@
-import type { Operation } from '../schemas/interface.ts';
+import type { AnyOperation } from '../schemas/interface.ts';
 import type { ProbeObservation, ProbeObservedBody } from '../schemas/port-messages.ts';
 export declare const PREFLIGHT_ARTIFACT_PATH = "PreflightVerdict";
 /**
@@ -17,7 +17,10 @@ export type ProjectedObservation = {
     readonly legId: string;
     readonly interfaceId: string;
     readonly operationId: string;
-    readonly status: number;
+    /** the transport status, or `null` off an interface that speaks HTTP. */
+    readonly status: number | null;
+    /** the process exit code, or `null` on an interface that does. */
+    readonly exitCode: number | null;
     readonly body: ProbeObservedBody;
 };
 /**
@@ -27,7 +30,7 @@ export type ProjectedObservation = {
  */
 export declare function pruneVolatile(body: ProbeObservedBody, volatilePointers: readonly string[], artifactPath: string): ProbeObservedBody;
 /** One observation reduced to the projection, with its volatile pointers gone. */
-export declare function projectObservation(observation: ProbeObservation, operation: Operation, artifactPath: string): ProjectedObservation;
+export declare function projectObservation(observation: ProbeObservation, operation: AnyOperation, artifactPath: string): ProjectedObservation;
 /**
  * AD-11's fixture digest. Sorted by leg id first, because NFR9 forbids any
  * stage from reading array position: two runs whose observations arrived in a

package/dist/core/preflight/projection.js

@@ -4,6 +4,7 @@
  * scoring version: two runs whose projections agree describe the same fixture.
  */
 import { digestComposite } from '../canonical/digest.js';
+import { descriptorArtifactOf, descriptorChannelOf, } from '../declared-inputs.js';
 import { DESCRIPTOR_POINTER_PATTERN } from '../schemas/pointer.js';
 import { decodeTail } from '../seal/plan-index.js';
 export const PREFLIGHT_ARTIFACT_PATH = 'PreflightVerdict';
@@ -70,14 +71,33 @@ export function pruneVolatile(body, volatilePointers, artifactPath) {
 }
 /** One observation reduced to the projection, with its volatile pointers gone. */
 export function projectObservation(observation, operation, artifactPath) {
+    // The channel the operation's descriptor describes is the one AD-10's
+    // relation reads, so it is the one the volatile pointers are pruned from.
+    // Off an interface that speaks HTTP that is the response body; off a
+    // command it is the stream or the file the operation nominates.
+    const observed = observation.kind === 'api'
+        ? observation.body
+        : describedChannelOf(observation, operation);
     return {
         legId: observation.probeId,
         interfaceId: observation.interfaceId,
         operationId: observation.operationId,
-        status: observation.status,
-        body: pruneVolatile(observation.body, operation.volatilePointers, artifactPath),
+        // A command has no transport status. `null` is what a reader of this
+        // field asks about, and every such reader already handles it.
+        status: observation.kind === 'api' ? observation.status : null,
+        exitCode: observation.kind === 'api' ? null : observation.exitCode,
+        body: pruneVolatile(observed, operation.volatilePointers, artifactPath),
     };
 }
+/** The observed value of whichever channel this operation's descriptor describes. */
+function describedChannelOf(observation, operation) {
+    const artifactId = descriptorArtifactOf(operation);
+    if (artifactId !== null)
+        return observation.artifacts[artifactId] ?? { kind: 'absent' };
+    return descriptorChannelOf(operation) === 'stderr'
+        ? observation.stderr
+        : observation.stdout;
+}
 /**
  * AD-11's fixture digest. Sorted by leg id first, because NFR9 forbids any
  * stage from reading array position: two runs whose observations arrived in a

package/dist/core/preflight/reduce.js

@@ -16,10 +16,39 @@ import { evidenceOf, resolveWitnessRelation } from './witness-evidence.js';
 /**
  * AD-10 names no threshold, and this is the one the repository already speaks:
  * the published conformance suite ships `probe/observe-anomalous-status`, and
- * `ProbeObservation.status` is bounded to 100-599 at the port, so HTTP is
- * already assumed at that boundary.
+ * an api observation's `status` is bounded to 100-599 at the port.
  */
 const ANOMALOUS_STATUS = 400;
+/**
+ * What makes a control leg anomalous, in the vocabulary of the kind it ran
+ * against, or `null` when nothing does. A command's analogue of a 4xx is a
+ * non-zero exit: both are the system saying the call did not go through, and
+ * AD-10's clean-control check is about exactly that.
+ */
+function anomalyOf(observation) {
+    if (observation.kind === 'api') {
+        return observation.status >= ANOMALOUS_STATUS
+            ? `status ${observation.status}`
+            : null;
+    }
+    return observation.exitCode === 0 ? null : `exit code ${observation.exitCode}`;
+}
+/**
+ * Whether the observation answers the mechanism the leg asked about.
+ *
+ * `kind` alone, and deliberately not `interfaceId` or `operationId`: those two
+ * are already read by the `interface-present` check, which reports a mismatch
+ * as a failed verdict rather than a fault, and reclassifying them here would
+ * turn a shipped verdict into a thrown fault. `kind` has no such reader. An
+ * observation of the wrong mechanism is not a weaker answer to the question, it
+ * is an answer to a different one, and every check below would read it as the
+ * leg's own result.
+ */
+function kindMismatch(leg, observation) {
+    if (leg.request.kind === observation.kind)
+        return undefined;
+    return `leg "${leg.legId}" asked for a "${leg.request.kind}" probe and was answered with a "${observation.kind}" observation, so the port answered a question nobody asked`;
+}
 const check = (kind, operationId, outcome, note) => ({ kind, operationId, outcome, note });
 /**
  * Two projections describe the same fixture state. `legId` is excluded because
@@ -48,7 +77,7 @@ const resolveAgainst = (witness, state, plan, artifactPath) => {
     // Keyed by the witness's own leg id, since the relation addresses
     // `/interactions/{witness.legId}/…`. The scoped row asks whether that same
     // relation would fire on a clean leg's observation.
-    return resolveWitnessRelation(witness.relation, { [witness.legId]: state.evidence }, state.leg.operation, plan.referenceSets, artifactPath).resolution;
+    return resolveWitnessRelation(witness.relation, { [witness.legId]: state.evidence }, state.leg.operation, plan.referenceSets, plan.referenceSetKeys, artifactPath).resolution;
 };
 export const reducePreflight = (plan, { observations }) => {
     const byProbeId = new Map();
@@ -68,12 +97,22 @@ export const reducePreflight = (plan, { observations }) => {
         const observation = byProbeId.get(leg.legId);
         if (observation === undefined)
             continue;
+        // The echo is checked, not assumed. Both port messages are unions, so a
+        // port can answer a command leg with a schema-valid HTTP observation and
+        // every check below reads it as the leg's own result: a command contract
+        // then passes pre-flight with four checks satisfied and no command ever
+        // run. `probeId` alone binds the two together and says nothing about
+        // whether the answer is to this question.
+        const mismatch = kindMismatch(leg, observation);
+        if (mismatch !== undefined) {
+            throw new RuntimeFault('port-contract-violation', PREFLIGHT_ARTIFACT_PATH, mismatch);
+        }
         const projected = projectObservation(observation, leg.operation, PREFLIGHT_ARTIFACT_PATH);
         states.set(leg.legId, {
             leg,
             observation,
             projected,
-            evidence: evidenceOf(projected, observation, leg.inputs),
+            evidence: evidenceOf(projected, observation, leg.inputs, leg.operation),
         });
     }
     const reduceCheck = (planned) => {
@@ -105,7 +144,7 @@ export const reducePreflight = (plan, { observations }) => {
                         return check(planned.kind, planned.operationId, 'failed', `witness leg "${leg.legId}" produced no observation`);
                     evidence[leg.legId] = state.evidence;
                 }
-                const { resolution } = resolveWitnessRelation(witness.relation, evidence, operation, plan.referenceSets, PREFLIGHT_ARTIFACT_PATH);
+                const { resolution } = resolveWitnessRelation(witness.relation, evidence, operation, plan.referenceSets, plan.referenceSetKeys, PREFLIGHT_ARTIFACT_PATH);
                 // AD-10's own sentence, and the most load-bearing line here: a
                 // sensitivity check that examined nothing has established nothing,
                 // so `insufficient-evidence` fails.
@@ -128,8 +167,9 @@ export const reducePreflight = (plan, { observations }) => {
                     const state = states.get(legId);
                     if (state === undefined)
                         return check(planned.kind, null, 'failed', `control leg "${legId}" produced no observation`);
-                    if (state.observation.status >= ANOMALOUS_STATUS)
-                        return check(planned.kind, null, 'failed', `control leg "${legId}" observed status ${state.observation.status}`);
+                    const anomaly = anomalyOf(state.observation);
+                    if (anomaly !== null)
+                        return check(planned.kind, null, 'failed', `control leg "${legId}" observed ${anomaly}`);
                 }
                 return check(planned.kind, null, 'satisfied', null);
             }

package/dist/core/preflight/witness-evidence.d.ts

@@ -4,11 +4,11 @@
  * unchanged. This module supplies the `Observation` shape they read and a
  * collection predicate scoped to one operation.
  */
-import { type PointerDenotesCollection } from '../evaluate/resolution.ts';
+import { type PointerDenotesCollection, type ReferenceSetKeys } from '../evaluate/resolution.ts';
 import type { EvalContract } from '../schemas/eval-contract.ts';
 import type { CheckResolutionValue } from '../schemas/evidence-artifact.ts';
 import type { Expression } from '../schemas/expression.ts';
-import type { Operation } from '../schemas/interface.ts';
+import type { AnyOperation } from '../schemas/interface.ts';
 import type { ProbeObservation } from '../schemas/port-messages.ts';
 import type { JsonValue } from '../schemas/primitives.ts';
 import type { Observation } from '../schemas/sealed-run-record.ts';
@@ -27,7 +27,7 @@ export declare const PREFLIGHT_REGEX_MATCH_STEP_BUDGET = 1000000;
  * the projection does not carry. `provenance` is `baseline`: a pre-flight leg is
  * pre-canned by definition.
  */
-export declare function evidenceOf(projected: ProjectedObservation, observation: ProbeObservation, inputs: WitnessInputs): Observation;
+export declare function evidenceOf(projected: ProjectedObservation, observation: ProbeObservation, inputs: WitnessInputs, operation: AnyOperation): Observation;
 /** the contract's declared reference sets in the shape the resolver wants. */
 export declare function referenceSetMembers(contract: EvalContract): Readonly<Record<string, JsonValue[]>>;
 /**
@@ -37,6 +37,6 @@ export declare function referenceSetMembers(contract: EvalContract): Readonly<Re
  * never appears, and fabricating step objects for a one-operation lookup is more
  * machinery than the predicate.
  */
-export declare function makeWitnessPointerDenotesCollection(operation: Operation): PointerDenotesCollection;
+export declare function makeWitnessPointerDenotesCollection(operation: AnyOperation): PointerDenotesCollection;
 /** Resolves one witness relation over the legs it addresses. */
-export declare function resolveWitnessRelation(relation: Expression, legEvidence: Readonly<Record<string, Observation>>, operation: Operation, referenceSets: Readonly<Record<string, JsonValue[]>>, artifactPath: string): CheckResolutionValue;
+export declare function resolveWitnessRelation(relation: Expression, legEvidence: Readonly<Record<string, Observation>>, operation: AnyOperation, referenceSets: Readonly<Record<string, JsonValue[]>>, referenceSetKeys: ReferenceSetKeys, artifactPath: string): CheckResolutionValue;