eval-quality 1.4.2 → 3.0.0

package/dist/testing/probe-conformance.js

@@ -1,10 +1,115 @@
 import { probeParsers } from '../ports/environment-probe-port.js';
 import { buildScenario, countCalls, describeThrown, disposeScenario, faultView, reportOf, runSharedAssertions, settle, titledOutcome, withDispose, } from './conformance.js';
+const DENIED = 'forbidden-target';
+const CAPPED = 'budget-exhausted';
+/**
+ * Which of the four echoed fields the observation failed to return unchanged,
+ * or `undefined` when it answered the question it was asked.
+ */
+function echoMismatch(request, observation) {
+    const echoed = [
+        ['kind', request.kind, observation.kind],
+        ['probeId', request.probeId, observation.probeId],
+        ['interfaceId', request.interfaceId, observation.interfaceId],
+        ['operationId', request.operationId, observation.operationId],
+    ];
+    for (const [field, asked, answered] of echoed) {
+        if (asked === answered)
+            continue;
+        return `observed ${field} "${answered}" for a request carrying "${asked}", so the answer does not correlate with the question`;
+    }
+    return undefined;
+}
+function checkRejected(assertion, expectedCode, error) {
+    const view = faultView(error);
+    if (view === undefined) {
+        return titledOutcome(assertion.id, assertion.title, false, `rejected with ${describeThrown(error)}, which carries no declared AD-28 code`);
+    }
+    return titledOutcome(assertion.id, assertion.title, view.code === expectedCode, `rejected with code "${view.code}", expected "${expectedCode}"`);
+}
+/**
+ * Schema, then correlation, then the assertion's own predicate. Generic over
+ * the subject rather than written once per arm: a function parameter is
+ * checked contravariantly, so an arm's `check` cannot be widened across
+ * subject types, and inferring the subject here costs the assertion records no
+ * type parameter of their own.
+ */
+function checkResolvedFor(assertion, check, value, request, subject) {
+    const parsed = probeParsers.response.safeParse(value);
+    if (!parsed.success) {
+        return titledOutcome(assertion.id, assertion.title, false, 'the resolved value is not a schema-valid ProbeObservation');
+    }
+    // Schema validity is not correlation. Both messages are unions, so an
+    // adapter can answer a command request or a tool call with a schema-valid
+    // HTTP observation and satisfy every assertion below without running a
+    // command, opening a session, or consulting a policy. The four echoed
+    // fields are what tie one answer to one question, and `ProbeRequest.probeId`'s
+    // own description already says the port returns them unchanged.
+    const mismatch = echoMismatch(request, parsed.data);
+    if (mismatch !== undefined) {
+        return titledOutcome(assertion.id, assertion.title, false, mismatch);
+    }
+    const complaint = check?.(parsed.data, subject);
+    return titledOutcome(assertion.id, assertion.title, complaint === undefined, complaint ?? '');
+}
+/** Generic for `checkResolvedFor`'s reason: `expectedCalls` names its own arm's subject type. */
+function checkCallCount(expectedCalls, subject, built, base) {
+    if (expectedCalls === undefined)
+        return base;
+    // Every branch keeps whatever the assertion's own check already said. An
+    // outcome that failed its predicate AND hit a subject-side count problem
+    // used to report only the second, which hid the reason the assertion was
+    // looked at in the first place.
+    const also = (complaint) => ({
+        ...base,
+        passed: false,
+        detail: base.passed ? complaint : `${base.detail}; ${complaint}`,
+    });
+    const expected = expectedCalls(subject);
+    if (typeof expected === 'string')
+        return also(expected);
+    const actual = countCalls(built);
+    if (typeof actual === 'string')
+        return also(actual);
+    if (actual === expected)
+        return base;
+    return also(`underlyingCalls() was ${actual}, expected ${expected}`);
+}
+async function runArmAssertion(assertion, subject) {
+    const run = await buildScenario(subject, 'resolves');
+    if (!run.ok) {
+        return titledOutcome(assertion.id, assertion.title, false, run.detail);
+    }
+    const request = assertion.request(subject);
+    const settled = await settle(run.built.port(request, new AbortController().signal));
+    let result;
+    if (assertion.expectation.kind === 'resolves') {
+        result =
+            settled.kind === 'resolved'
+                ? checkResolvedFor(assertion, assertion.expectation.check, settled.value, request, subject)
+                : titledOutcome(assertion.id, assertion.title, false, `rejected with ${describeThrown(settled.error)} instead of returning an observation`);
+    }
+    else {
+        result =
+            settled.kind === 'rejected'
+                ? checkRejected(assertion, assertion.expectation.code, settled.error)
+                : titledOutcome(assertion.id, assertion.title, false, `resolved instead of rejecting with "${assertion.expectation.code}"`);
+    }
+    result = checkCallCount(assertion.expectedCalls, subject, run.built, result);
+    return withDispose(result, await disposeScenario(run.built));
+}
+/** The six shared assertions, then the arm's own, in list order. */
+async function runArm(subject, port, assertions) {
+    const shared = await runSharedAssertions('probe', subject, probeParsers.response);
+    const additional = [];
+    for (const assertion of assertions) {
+        additional.push(await runArmAssertion(assertion, subject));
+    }
+    return reportOf(subject.name, port, [...shared, ...additional]);
+}
 function authorizationFor(subject, request) {
     return subject.policy.authorizations.find((each) => each.interfaceId === request.interfaceId);
 }
-const DENIED = 'forbidden-target';
-const CAPPED = 'budget-exhausted';
 const PROBE_ASSERTIONS = [
     {
         // Without this one, a subject that denies everything scores twelve of
@@ -18,6 +123,13 @@ const PROBE_ASSERTIONS = [
     {
         // A subject that throws on 5xx fails here and nowhere else, and it makes
         // AD-10's seeded-fault check unimplementable.
+        //
+        // The non-`api` arm of the detail below names the kind it observed. It
+        // is reached when the subject declares a `faultingRequest` of another
+        // kind and answers it correlated, which `ProbeSubject` admits because
+        // the field is typed over the whole request union. Substituting only
+        // the observation's kind does not reach it: `echoMismatch` compares
+        // `kind` among the four echoed fields and short-circuits first.
         id: 'probe/observe-anomalous-status',
         title: 'a 500 from an authorized target is an observation, not a fault',
         request: (subject) => subject.faultingRequest,
@@ -25,7 +137,7 @@ const PROBE_ASSERTIONS = [
             kind: 'resolves',
             check: (observation) => observation.kind === 'api' && observation.status === 500
                 ? undefined
-                : `observed ${observation.kind === 'api' ? `status ${observation.status}` : 'a command observation'}, expected status 500`,
+                : `observed ${observation.kind === 'api' ? `status ${observation.status}` : `an observation of kind "${observation.kind}"`}, expected status 500`,
         },
     },
     {
@@ -114,92 +226,6 @@ const PROBE_ASSERTIONS = [
         expectation: { kind: 'rejects', code: CAPPED },
     },
 ];
-function checkResolved(assertion, expectation, value, request) {
-    const parsed = probeParsers.response.safeParse(value);
-    if (!parsed.success) {
-        return titledOutcome(assertion.id, assertion.title, false, 'the resolved value is not a schema-valid ProbeObservation');
-    }
-    // Schema validity is not correlation. Both messages are unions now, so an
-    // adapter can answer a command request with a schema-valid HTTP observation
-    // and satisfy every assertion below without running a command or consulting
-    // a policy. The four echoed fields are what tie one answer to one question,
-    // and `ProbeRequest.probeId`'s own description already says the port returns
-    // them unchanged.
-    const mismatch = echoMismatch(request, parsed.data);
-    if (mismatch !== undefined) {
-        return titledOutcome(assertion.id, assertion.title, false, mismatch);
-    }
-    const complaint = expectation.check?.(parsed.data);
-    return titledOutcome(assertion.id, assertion.title, complaint === undefined, complaint ?? '');
-}
-/**
- * Which of the four echoed fields the observation failed to return unchanged,
- * or `undefined` when it answered the question it was asked.
- */
-function echoMismatch(request, observation) {
-    const echoed = [
-        ['kind', request.kind, observation.kind],
-        ['probeId', request.probeId, observation.probeId],
-        ['interfaceId', request.interfaceId, observation.interfaceId],
-        ['operationId', request.operationId, observation.operationId],
-    ];
-    for (const [field, asked, answered] of echoed) {
-        if (asked === answered)
-            continue;
-        return `observed ${field} "${answered}" for a request carrying "${asked}", so the answer does not correlate with the question`;
-    }
-    return undefined;
-}
-/** Reads only `id` and `title`, so both arms' assertion shapes satisfy it structurally. */
-function checkRejected(assertion, expectedCode, error) {
-    const view = faultView(error);
-    if (view === undefined) {
-        return titledOutcome(assertion.id, assertion.title, false, `rejected with ${describeThrown(error)}, which carries no declared AD-28 code`);
-    }
-    return titledOutcome(assertion.id, assertion.title, view.code === expectedCode, `rejected with code "${view.code}", expected "${expectedCode}"`);
-}
-function checkCalls(assertion, subject, built, base) {
-    if (assertion.expectedCalls === undefined)
-        return base;
-    const expected = assertion.expectedCalls(subject);
-    if (typeof expected === 'string') {
-        return { ...base, passed: false, detail: expected };
-    }
-    const actual = countCalls(built);
-    if (typeof actual === 'string') {
-        return { ...base, passed: false, detail: actual };
-    }
-    if (actual === expected)
-        return base;
-    return {
-        ...base,
-        passed: false,
-        detail: `${base.passed ? '' : `${base.detail}; `}underlyingCalls() was ${actual}, expected ${expected}`,
-    };
-}
-async function runProbeAssertion(assertion, subject) {
-    const run = await buildScenario(subject, 'resolves');
-    if (!run.ok) {
-        return titledOutcome(assertion.id, assertion.title, false, run.detail);
-    }
-    const request = assertion.request(subject);
-    const settled = await settle(run.built.port(request, new AbortController().signal));
-    let result;
-    if (assertion.expectation.kind === 'resolves') {
-        result =
-            settled.kind === 'resolved'
-                ? checkResolved(assertion, assertion.expectation, settled.value, request)
-                : titledOutcome(assertion.id, assertion.title, false, `rejected with ${describeThrown(settled.error)} instead of returning an observation`);
-    }
-    else {
-        result =
-            settled.kind === 'rejected'
-                ? checkRejected(assertion, assertion.expectation.code, settled.error)
-                : titledOutcome(assertion.id, assertion.title, false, `resolved instead of rejecting with "${assertion.expectation.code}"`);
-    }
-    result = checkCalls(assertion, subject, run.built, result);
-    return withDispose(result, await disposeScenario(run.built));
-}
 /**
  * Nineteen outcomes: the six shared assertions plus AD-35's thirteen.
  * `maxRequestBytes` is the one cap with no assertion; the request shape is the
@@ -207,12 +233,24 @@ async function runProbeAssertion(assertion, subject) {
  * stays declared and adapter-enforced.
  */
 export async function runEnvironmentProbePortConformance(subject) {
-    const shared = await runSharedAssertions('probe', subject, probeParsers.response);
-    const additional = [];
-    for (const assertion of PROBE_ASSERTIONS) {
-        additional.push(await runProbeAssertion(assertion, subject));
-    }
-    return reportOf(subject.name, 'environment-probe', [...shared, ...additional]);
+    return runArm(subject, 'environment-probe', PROBE_ASSERTIONS);
+}
+/**
+ * The authorization the subject's own command policy resolves this request to.
+ * Keyed by the pair, since `CommandTargetAuthorization` is.
+ *
+ * The two denial assertions that call it read it for the reason
+ * `mcpAuthorizationFor` exists: a subject whose "unmapped" request is in fact
+ * mapped passes its own denial for the wrong reason.
+ *
+ * It takes the narrowed request, so each caller states what it does about a
+ * subject that declared another kind there rather than inheriting a silent
+ * `undefined` from here. `mcpAuthorizationFor` needs no such narrowing because
+ * it reads `interfaceId`, which every member of the request union carries.
+ */
+function commandAuthorizationFor(subject, request) {
+    return subject.policy.authorizations.find((each) => each.interfaceId === request.interfaceId &&
+        each.executable === request.executable);
 }
 const COMMAND_ASSERTIONS = [
     {
@@ -222,6 +260,11 @@ const COMMAND_ASSERTIONS = [
         expectation: { kind: 'resolves' },
     },
     {
+        // The non-`cli` arm of the detail below is reached the same way the
+        // `api` arm's is: `nonZeroExitRequest` is typed over the whole request
+        // union, so a subject may declare another kind there and answer it
+        // correlated. It names the observed kind; the binary ternary it
+        // replaced called every one of them "an api observation".
         id: 'command/observe-nonzero-exit',
         title: 'a non-zero exit from an authorized command is an observation, not a fault',
         request: (subject) => subject.nonZeroExitRequest,
@@ -229,7 +272,7 @@ const COMMAND_ASSERTIONS = [
             kind: 'resolves',
             check: (observation) => observation.kind === 'cli' && observation.exitCode !== 0
                 ? undefined
-                : `observed ${observation.kind === 'cli' ? `exit code ${observation.exitCode}` : 'an api observation'}, expected a non-zero exit`,
+                : `observed ${observation.kind === 'cli' ? `exit code ${observation.exitCode}` : `an observation of kind "${observation.kind}"`}, expected a non-zero exit`,
         },
     },
     {
@@ -237,21 +280,71 @@ const COMMAND_ASSERTIONS = [
         title: 'an interface no authorization names is refused before a process spawns',
         request: (subject) => subject.unmappedInterfaceRequest,
         expectation: { kind: 'rejects', code: DENIED },
-        expectedCalls: () => 0,
+        expectedCalls: (subject) => subject.policy.authorizations.some((each) => each.interfaceId === subject.unmappedInterfaceRequest.interfaceId)
+            ? "the subject's policy names an authorization for unmappedInterfaceRequest.interfaceId, so the request it presents as unmapped is mapped"
+            : 0,
     },
     {
         id: 'command/deny-unmapped-executable',
         title: 'an executable the interface is never paired with is refused before a process spawns',
         request: (subject) => subject.unmappedExecutableRequest,
         expectation: { kind: 'rejects', code: DENIED },
-        expectedCalls: () => 0,
+        expectedCalls: (subject) => {
+            const request = subject.unmappedExecutableRequest;
+            if (request.kind !== 'cli') {
+                return `unmappedExecutableRequest declares a "${request.kind}" request, which names no executable for the policy to refuse`;
+            }
+            return commandAuthorizationFor(subject, request) === undefined
+                ? 0
+                : "the subject's policy pairs unmappedExecutableRequest's interface with its executable, so the pair it presents as unmapped is mapped";
+        },
     },
     {
         id: 'command/deny-unauthorized-subcommand',
         title: 'a subcommand path outside the authorized set is refused before a process spawns',
         request: (subject) => subject.unauthorizedSubcommandRequest,
         expectation: { kind: 'rejects', code: DENIED },
-        expectedCalls: () => 0,
+        expectedCalls: (subject) => {
+            const request = subject.unauthorizedSubcommandRequest;
+            if (request.kind !== 'cli') {
+                return `unauthorizedSubcommandRequest declares a "${request.kind}" request, which names no subcommand path for the policy to refuse`;
+            }
+            const authorization = commandAuthorizationFor(subject, request);
+            if (authorization === undefined) {
+                return "the subject's policy names no authorization for unauthorizedSubcommandRequest's interface and executable, so this case cannot tell an unmapped pair from an unauthorized subcommand";
+            }
+            return authorization.permittedSubcommandPaths.some((permitted) => permitted.length === request.subcommandPath.length &&
+                permitted.every((segment, at) => segment === request.subcommandPath[at]))
+                ? `the subject's policy permits subcommand path ${JSON.stringify(request.subcommandPath)}, so the request it presents as unauthorized is authorized`
+                : 0;
+        },
+    },
+    {
+        // The environment channel is the one a contract author declares and an
+        // operator has to bound. This is the assertion that holds an adapter to
+        // bounding it: without `permittedEnvironmentKeys` enforced, any key the
+        // contract names reaches the process.
+        id: 'command/deny-unauthorized-environment-key',
+        title: 'an environment key the authorization does not permit is refused before a process spawns',
+        request: (subject) => subject.unauthorizedEnvironmentKeyRequest,
+        expectation: { kind: 'rejects', code: DENIED },
+        expectedCalls: (subject) => {
+            const request = subject.unauthorizedEnvironmentKeyRequest;
+            if (request.kind !== 'cli') {
+                return `unauthorizedEnvironmentKeyRequest declares a "${request.kind}" request, which carries no environment channel for the policy to refuse`;
+            }
+            const authorization = commandAuthorizationFor(subject, request);
+            if (authorization === undefined) {
+                return "the subject's policy names no authorization for unauthorizedEnvironmentKeyRequest's interface and executable, so this case cannot tell an unmapped pair from an unauthorized environment key";
+            }
+            const declared = Object.keys(request.channels.environment);
+            if (declared.length === 0) {
+                return 'unauthorizedEnvironmentKeyRequest declares no environment key, so nothing about the allowlist is exercised';
+            }
+            return declared.every((key) => authorization.permittedEnvironmentKeys.includes(key))
+                ? `the subject's policy permits every environment key unauthorizedEnvironmentKeyRequest declares (${JSON.stringify(declared)}), so the request it presents as unauthorized is authorized`
+                : 0;
+        },
     },
     {
         // The strongest available proof that no shell ever reads a channel value:
@@ -306,72 +399,194 @@ const COMMAND_ASSERTIONS = [
         expectation: { kind: 'rejects', code: CAPPED },
     },
 ];
-function checkCommandResolved(assertion, expectation, value, request, subject) {
-    const parsed = probeParsers.response.safeParse(value);
-    if (!parsed.success) {
-        return titledOutcome(assertion.id, assertion.title, false, 'the resolved value is not a schema-valid ProbeObservation');
-    }
-    const mismatch = echoMismatch(request, parsed.data);
-    if (mismatch !== undefined) {
-        return titledOutcome(assertion.id, assertion.title, false, mismatch);
-    }
-    const complaint = expectation.check?.(parsed.data, subject);
-    return titledOutcome(assertion.id, assertion.title, complaint === undefined, complaint ?? '');
-}
-/** `checkCalls`'s own logic, typed against `CommandAssertion`/`CommandProbeSubject` rather than the `api` arm's pair: a function parameter type is checked contravariantly, so the two assertion shapes cannot share one checker. */
-function checkCommandCalls(assertion, subject, built, base) {
-    if (assertion.expectedCalls === undefined)
-        return base;
-    const expected = assertion.expectedCalls(subject);
-    if (typeof expected === 'string') {
-        return { ...base, passed: false, detail: expected };
-    }
-    const actual = countCalls(built);
-    if (typeof actual === 'string') {
-        return { ...base, passed: false, detail: actual };
-    }
-    if (actual === expected)
-        return base;
-    return {
-        ...base,
-        passed: false,
-        detail: `${base.passed ? '' : `${base.detail}; `}underlyingCalls() was ${actual}, expected ${expected}`,
-    };
-}
-async function runCommandAssertion(assertion, subject) {
-    const run = await buildScenario(subject, 'resolves');
-    if (!run.ok) {
-        return titledOutcome(assertion.id, assertion.title, false, run.detail);
-    }
-    const request = assertion.request(subject);
-    const settled = await settle(run.built.port(request, new AbortController().signal));
-    let result;
-    if (assertion.expectation.kind === 'resolves') {
-        result =
-            settled.kind === 'resolved'
-                ? checkCommandResolved(assertion, assertion.expectation, settled.value, request, subject)
-                : titledOutcome(assertion.id, assertion.title, false, `rejected with ${describeThrown(settled.error)} instead of returning an observation`);
-    }
-    else {
-        result =
-            settled.kind === 'rejected'
-                ? checkRejected(assertion, assertion.expectation.code, settled.error)
-                : titledOutcome(assertion.id, assertion.title, false, `resolved instead of rejecting with "${assertion.expectation.code}"`);
-    }
-    result = checkCommandCalls(assertion, subject, run.built, result);
-    return withDispose(result, await disposeScenario(run.built));
-}
 /**
- * Fifteen outcomes: the six shared assertions plus the nine above. Unlike the
+ * Sixteen outcomes: the six shared assertions plus the ten above. Unlike the
  * `api` arm, every cap here has an assertion: a command subject's fixture
  * script can be told to overrun a byte cap directly, with no oversize-request
  * problem to work around.
  */
 export async function runCommandLineProbeConformance(subject) {
-    const shared = await runSharedAssertions('probe', subject, probeParsers.response);
-    const additional = [];
-    for (const assertion of COMMAND_ASSERTIONS) {
-        additional.push(await runCommandAssertion(assertion, subject));
+    return runArm(subject, 'command-probe', COMMAND_ASSERTIONS);
+}
+/**
+ * The authorization the subject's own policy resolves this request to.
+ *
+ * The two denial assertions read it to check the subject is internally
+ * consistent: a subject whose "unmapped" request is in fact mapped, or whose
+ * "unauthorized" tool is in fact on the allowlist, would pass its own denial
+ * for the wrong reason and certify an adapter that never refused anything.
+ */
+function mcpAuthorizationFor(subject, request) {
+    return subject.policy.authorizations.find((each) => each.interfaceId === request.interfaceId);
+}
+/** The tool's structured result as a key map, or `undefined` when the observation carries no JSON object there. */
+function structuredResultOf(observation) {
+    if (observation.kind !== 'mcp' || observation.result.kind !== 'json') {
+        return undefined;
     }
-    return reportOf(subject.name, 'command-probe', [...shared, ...additional]);
+    const { value } = observation.result;
+    return typeof value === 'object' && value !== null && !Array.isArray(value)
+        ? value
+        : undefined;
+}
+const MCP_ASSERTIONS = [
+    {
+        id: 'mcp/allow-authorized-tool-call',
+        title: 'an explicitly authorized tool call reaches its server and is observed',
+        request: (subject) => subject.authorizedRequest,
+        expectation: { kind: 'resolves' },
+    },
+    {
+        // The rule a tool-use adapter breaks first. A tool's own error result is
+        // the payload AD-10's seeded-fault check reads, so an adapter that
+        // throws on it makes every probe written to catch that refusal invisible.
+        //
+        // The non-`mcp` arm of the detail is reachable the way the other two
+        // arms' are: `errorResultRequest` is typed over the whole request union,
+        // so a subject may declare another kind there and answer it correlated,
+        // which is what gets past `echoMismatch` to this check.
+        id: 'mcp/observe-error-result',
+        title: 'a tool-reported error from an authorized server is an observation, not a fault',
+        request: (subject) => subject.errorResultRequest,
+        expectation: {
+            kind: 'resolves',
+            check: (observation) => {
+                if (observation.kind !== 'mcp') {
+                    return `observed an observation of kind "${observation.kind}", expected the envelope's error flag set`;
+                }
+                if (!observation.isError) {
+                    return "observed a tool call reporting no error, expected the envelope's error flag set";
+                }
+                // The flag alone is half the rule. An adapter that reads the
+                // envelope and drops what the tool said about the failure passes a
+                // flag-only check, and the seeded-fault oracle downstream then has
+                // the error announced and nothing to assert on.
+                //
+                // The detail names the subject's obligation rather than accusing
+                // the adapter, because the two are indistinguishable from here: a
+                // tool reporting a failure through a prose `content` array and no
+                // structured content produces the same absent channel, and the
+                // kind's first version describes structured results only.
+                return observation.result.kind === 'absent'
+                    ? 'the error came back on an absent result channel; errorResultRequest has to name a tool whose failure carries structured content, since a tool that publishes none looks the same here as an adapter that dropped it'
+                    : undefined;
+            },
+        },
+    },
+    {
+        id: 'mcp/deny-unmapped-interface',
+        title: 'an interface no authorization names is refused before a server process starts',
+        request: (subject) => subject.unmappedInterfaceRequest,
+        expectation: { kind: 'rejects', code: DENIED },
+        expectedCalls: (subject) => mcpAuthorizationFor(subject, subject.unmappedInterfaceRequest) ===
+            undefined
+            ? 0
+            : "the subject's policy names an authorization for unmappedInterfaceRequest.interfaceId, so the request it presents as unmapped is mapped",
+    },
+    {
+        // The tool allowlist is the second and last authorization-scoped field,
+        // and it draws AD-35's disclosure boundary inside the server: a tool the
+        // list omits is unreachable even when the server publishes it.
+        id: 'mcp/deny-unauthorized-tool',
+        title: 'a tool outside the authorized list is refused before a server process starts',
+        request: (subject) => subject.unauthorizedToolRequest,
+        expectation: { kind: 'rejects', code: DENIED },
+        expectedCalls: (subject) => {
+            const request = subject.unauthorizedToolRequest;
+            if (request.kind !== 'mcp') {
+                return `unauthorizedToolRequest declares a "${request.kind}" request, which names no tool for the policy to refuse`;
+            }
+            const authorization = mcpAuthorizationFor(subject, request);
+            if (authorization === undefined) {
+                return "the subject's policy names no authorization for unauthorizedToolRequest.interfaceId, so this case cannot tell an unmapped interface from an unauthorized tool";
+            }
+            return authorization.tools.includes(request.toolName)
+                ? `the subject's policy permits tool "${request.toolName}" on interface "${request.interfaceId}", so the request it presents as unauthorized is authorized`
+                : 0;
+        },
+    },
+    {
+        // The tool-call twin of the command arm's literal-argument proof. A
+        // declared value crosses a JSON-RPC frame here rather than a command
+        // line, so what this catches is a value re-encoded, coerced, or
+        // truncated in framing, and the metacharacters are what make a
+        // re-encoding visible in the echo.
+        id: 'mcp/arguments-passed-as-declared',
+        title: 'a declared argument value reaches the tool byte for byte',
+        request: (subject) => subject.argumentEchoRequest,
+        expectation: {
+            kind: 'resolves',
+            check: (observation, subject) => {
+                if (observation.kind !== 'mcp') {
+                    return `observed an observation of kind "${observation.kind}", expected a tool call`;
+                }
+                const result = structuredResultOf(observation);
+                if (result === undefined) {
+                    return 'the observation carries no structured result to read the received argument from';
+                }
+                const received = result[subject.argumentEchoResultKey];
+                return received === subject.argumentEchoValue
+                    ? undefined
+                    : `the tool reported receiving ${JSON.stringify(received)} on "${subject.argumentEchoResultKey}", expected the declared literal`;
+            },
+        },
+    },
+    {
+        // An adapter that reads the envelope and drops `structuredContent`
+        // still resolves and still correlates, and every oracle over the result
+        // then resolves absent. That reads as a contract nothing satisfies
+        // rather than as an adapter that answered nowhere, which is the failure
+        // this assertion exists to name.
+        id: 'mcp/observe-declared-result-channel',
+        title: "the tool's structured result is carried on the channel its descriptor describes",
+        request: (subject) => subject.structuredResultRequest,
+        expectation: {
+            kind: 'resolves',
+            check: (observation, subject) => {
+                if (observation.kind !== 'mcp') {
+                    return `observed an observation of kind "${observation.kind}", expected a tool call`;
+                }
+                const result = structuredResultOf(observation);
+                if (result === undefined) {
+                    return `the result channel carried ${JSON.stringify(observation.result)}, expected the structured result carrying ${JSON.stringify(subject.structuredResultKeys)}`;
+                }
+                const missing = subject.structuredResultKeys.filter((key) => !(key in result));
+                return missing.length === 0
+                    ? undefined
+                    : `the result channel is missing ${JSON.stringify(missing)}`;
+            },
+        },
+    },
+    {
+        id: 'mcp/cap-elapsed',
+        title: 'a session past maxElapsedMs is capped, not left running for the caller',
+        request: (subject) => subject.overElapsedRequest,
+        expectation: { kind: 'rejects', code: CAPPED },
+    },
+    {
+        id: 'mcp/cap-result-bytes',
+        title: 'a result past maxOutputBytes is capped, not returned in part',
+        request: (subject) => subject.overResultBytesRequest,
+        expectation: { kind: 'rejects', code: CAPPED },
+    },
+];
+/**
+ * Fourteen outcomes: the six shared assertions plus the eight above. Fewer
+ * than either other arm because a tool-server authorization scopes two fields:
+ * one session is opened against one server and every tool it offers belongs to
+ * that server, so the interface identifier is the server identity and the tool
+ * allowlist is the only thing left to deny on.
+ *
+ * Two rules the reference adapter follows are outside what a green run
+ * certifies, and an author reading this should know which. A JSON-RPC error
+ * answering `tools/call` is an observation on the same terms a tool-reported
+ * error is, and the byte cap applies to the server's own stderr as well as its
+ * stdout. Neither has an assertion here, because the derivation above gives
+ * each arm one denial per authorization-scoped field and one assertion per cap
+ * the suite can make a subject exceed, and both of these are a second spelling
+ * of an assertion the arm already carries. The reference adapter's own tests
+ * cover both.
+ */
+export async function runMcpProbeConformance(subject) {
+    return runArm(subject, 'mcp-probe', MCP_ASSERTIONS);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eval-quality",
-  "version": "1.4.2",
+  "version": "3.0.0",
   "description": "Compile disciplined Behavioral Evaluation Contracts and score their ability to catch known defects.",
   "author": "Murat Ozcan",
   "license": "Apache-2.0",
@@ -73,6 +73,8 @@
     "format": "biome format --write .",
     "check:docs": "node scripts/check-docs.mjs",
     "check:doc-invocations": "node scripts/check-doc-invocations.mjs",
+    "check:doc-counts": "node scripts/check-doc-counts.ts",
+    "check:doc-claims": "node scripts/check-doc-claims.ts",
     "lint:spine": "python3 scripts/spine-lint/lint_spine.py --registry-ad 5 --workspace-root . --fail-on high",
     "test:spine-lint": "uv run --with pytest pytest scripts/spine-lint/tests -q",
     "build:shareable": "node scripts/build-shareable.mjs",
@@ -110,7 +112,7 @@
     "release:major": "gh workflow run publish.yml --ref main -f bump=major",
     "release:prepare": "node scripts/release-prepare.mjs",
     "release:publish": "gh workflow run publish.yml --ref main -f bump=none",
-    "validate": "npm run build && npm run typecheck && npm run lint && npm run check:docs && npm run check:doc-invocations && npm run check:shareable && npm run lint:spine && npm run check:vectors && npm run check:schemas && npm run check:ad5-registry && npm run check:ad28-registry && npm run check:ad31-table && npm run check:ad33-table && npm run check:ad21-table && npm run check:layers && npm run check:lineage && npm run check:boundary && npm run check:corpus && npm run check:worked-example && npm run check:website-deps && npm run test:coverage",
+    "validate": "npm run build && npm run typecheck && npm run lint && npm run check:docs && npm run check:doc-invocations && npm run check:shareable && npm run lint:spine && npm run check:vectors && npm run check:schemas && npm run check:ad5-registry && npm run check:ad28-registry && npm run check:ad31-table && npm run check:ad33-table && npm run check:ad21-table && npm run check:layers && npm run check:lineage && npm run check:boundary && npm run check:corpus && npm run check:doc-counts && npm run check:doc-claims && npm run check:worked-example && npm run check:website-deps && npm run test:coverage",
     "prepack": "npm run clean && npm run build",
     "prepublishOnly": "node scripts/assert-publish-authorized.mjs"
   },