@emseepea/testing 0.21.7 → 0.23.0

package/README.md

@@ -29,6 +29,39 @@ sequence in the listed order and count. The evidence retains all alternatives
 and the matched sequence; arguments, meaning, confirmation and feedback still
 need their own assertions.
 
+Use `assertToolSelection` when valid tool order or recovery paths can vary, but
+the permitted tools and call counts are bounded. Its tool-selection contract
+lists required, allowed, and explicitly forbidden tools. Any advertised tool
+that is not required or allowed is rejected.
+
+```js
+assertToolSelection(turn, {
+  required: [
+    { name: "list-candidates", minCalls: 1, maxCalls: 1 },
+    { name: "commit-selection", minCalls: 1, maxCalls: 1 },
+  ],
+  allowed: [{ name: "lookup-candidate", maxCalls: 2 }],
+  forbidden: [],
+  totalCalls: { minCalls: 2, maxCalls: 4 },
+});
+```
+
+The finite maximums are checked after the turn. They do not stop tool calls or
+truncate evidence.
+
+A passing selection assertion proves only that every
+recorded call satisfied the declared tool and count rules. It does not prove
+argument correctness, persisted effects, response meaning, or recovery from an
+error. Keep an exact assertion when order, count, and arguments are
+deterministic, and add the separate assertions your application outcome needs.
+
+Each turn also exposes `toolOutcomes`, aligned by trial and call order with
+`toolCalls`. Each outcome is a read-only snapshot of what the model saw. It
+contains only `name`, `arguments`, the model-visible `result`, and `isError`.
+Use it to check an expected error and the later calls that followed the error.
+A later non-error result does not by itself prove recovery or a persisted
+effect.
+
 For a successful turn that advertises feedback, use
 `assertToolCallsWithOptionalFeedback`. It requires the exact ordered primary
 calls and arguments, then allows no feedback call or one trailing
@@ -57,6 +90,12 @@ advertised-tool text, an answer wrapper, or prepared MCP material. Exact tool
 assertions come from the provider's native MCP events. Follow-up messages use
 the same conversation.
 
+Native conversations have no framework tool-call or native provider-turn count
+limit. Calls must still use advertised MCP tools. The per-send timeout,
+cancellation, output checks, model checks, and isolated test environment remain
+effective. Application assertions decide which calls and results are
+acceptable. Tool-free judge invocations retain their existing limits.
+
 Call `await conversation.fresh()` to discard provider transcript history while
 keeping the same running test application and its saved data. The next `send`
 starts a new provider-native session in each trial.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@emseepea/testing",
-  "version": "0.21.7",
+  "version": "0.23.0",
   "description": "MCP integration and semantic testing helpers",
   "license": "MIT",
   "repository": {
@@ -43,10 +43,11 @@
     "README.md"
   ],
   "scripts": {
-    "build": "tsc -p tsconfig.json",
+    "build": "tsc -p tsconfig.json && tsc -p tsconfig.type-tests.json",
     "prepack": "npm run build",
     "test": "npm run build && npm run test:built",
-    "test:built": "node --test test/*.test.mjs"
+    "test:built": "node --test test/*.test.mjs",
+    "test:llm:built": "emseepea-test eval"
   },
   "dependencies": {
     "@emseepea/server": "0.23.0",

package/semantic/cli.mjs

@@ -1,23 +1,12 @@
 #!/usr/bin/env node
 import { spawn } from "node:child_process";
-import { createHash } from "node:crypto";
 import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { dirname, join, relative, resolve } from "node:path";
+import { validRecord } from "./evidence.mjs";
 import { discoverTests } from "./discover.mjs";
 import { modelVersion, providerModel, providerSettings } from "./provider.mjs";
 
-const negativeFeedbackObservations = new Set([
-  "error",
-  "friction",
-  "annoyance",
-  "unnecessary_difficulty",
-  "confusion",
-  "repetition",
-  "unexpected_bad_result",
-  "capability_mismatch",
-]);
-
 const paths = [];
 let provider = process.env.EMSEEPEA_EVAL_PROVIDER ?? "claude-local";
 let output = "artifacts/llm-eval/evidence.json";
@@ -101,7 +90,7 @@ try {
   for (const file of files) {
     const displayFile = relative(process.cwd(), file);
     const cases = Object.values(evidence.cases).filter((record) => record.file === displayFile);
-    if (!cases.length || cases.some((record) => !validRecord(record, evidence.authoritative, smoke))) {
+    if (!cases.length || cases.some((record) => !validRecord(record, evidence))) {
       evidence.errors.push(`Missing or failed qualification: ${displayFile}`);
     }
   }
@@ -114,138 +103,3 @@ try {
 }
 if (evidence.status !== "passed") process.exitCode = 1;
 console.log(`Semantic checks ${evidence.status}; evidence: ${output}`);
-
-function validRecord(record, authoritative, smoke) {
-  const isHash = (value) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
-  if (record.status !== "passed" || record.authoritative !== authoritative || record.smoke !== smoke
-    || record.provider !== evidence.provider || record.configuredModel !== evidence.configuredModel
-    || record.modelEvidence !== evidence.modelEvidence
-    || JSON.stringify(record.settings) !== JSON.stringify(evidence.settings)
-    || record.mode !== "conversation" || record.answerTrials?.length !== 3
-    || !Number.isInteger(record.judgeVerdicts?.length) || record.judgeVerdicts.length < 9
-    || record.judgeVerdicts.length % 3 !== 0
-    || !record.judgeVerdicts.every((judgment) => isHash(judgment.expectationSha256)
-      && isHash(judgment.requestSha256) && isHash(judgment.responseSha256)
-      && typeof judgment.expectedMeaning === "string" && judgment.expectedMeaning.length > 0
-      && judgment.verdict?.pass === true && judgment.verdict.score === 1
-      && typeof judgment.verdict.reason === "string" && judgment.verdict.reason.length > 0)) return false;
-  return record.answerTrials.every((trial) => Array.isArray(trial.turns) && trial.turns.length > 0
-    && trial.turns.every((turn) => Number.isInteger(turn.advertisedToolCount)
-      && turn.advertisedToolCount >= 0
-      && turn.interactionMode === "native-mcp"
-      && turn.answerTurnCount === 1
-      && turn.answerProviderToolCount === turn.toolCallCount
-      && turn.answerProviderTurnCount === turn.toolCallCount + 1
-      && Number.isInteger(turn.toolCallCount) && turn.toolCallCount >= 0 && turn.toolCallCount <= 3
-      && typeof turn.prompt === "string" && turn.prompt.length > 0
-      && typeof turn.response === "string"
-      && isHash(turn.promptSha256) && isHash(turn.answerSha256)
-      && isHash(turn.advertisedToolsSha256) && isHash(turn.selectedCallsSha256)
-      && Array.isArray(turn.toolCalls)
-      && JSON.stringify(turn.selectedTools)
-        === JSON.stringify(turn.toolCalls.map(({ name }) => name))
-      && validToolAssertions(turn, isHash)
-      && validNegativeFeedbackAssertion(turn)
-      && validFeedbackDisclosure(turn, record.judgeVerdicts, trial.trial, isHash)
-      && turn.toolCalls.every((call) => Object.hasOwn(call, "result") && typeof call.isError === "boolean")
-      && Array.isArray(turn.pathEvidence) && turn.pathEvidence.length === turn.toolCallCount
-      && turn.pathEvidence.every(({ method, target, requestSha256, responseSha256 }) =>
-        method === "tools/call" && turn.selectedTools.includes(target)
-          && isHash(requestSha256) && isHash(responseSha256))));
-}
-
-function validToolAssertions(turn, isHash) {
-  if (turn.expectedOptionalFeedback === true) {
-    if (!Array.isArray(turn.expectedCalls)) return false;
-    const expectedHash = createHash("sha256").update(JSON.stringify({
-      calls: turn.expectedCalls,
-      optionalFeedback: true,
-    })).digest("hex");
-    const calls = turn.toolCalls.map(({ name, arguments: args }) => ({ name, arguments: args }));
-    const primaryCalls = calls.slice(0, turn.expectedCalls?.length);
-    const trailingCalls = calls.slice(turn.expectedCalls?.length);
-    return expectedHash === turn.expectedSelectionSha256
-      && JSON.stringify(primaryCalls) === JSON.stringify(turn.expectedCalls)
-      && (trailingCalls.length === 0
-        || (trailingCalls.length === 1 && trailingCalls[0].name === "submit-feedback"));
-  }
-  if (typeof turn.expectedOptionalTool === "string" && turn.expectedOptionalTool.trim()) {
-    const expectedHash = createHash("sha256").update(JSON.stringify({
-      optionalTool: turn.expectedOptionalTool,
-    })).digest("hex");
-    return expectedHash === turn.expectedSelectionSha256
-      && (turn.toolCalls.length === 0
-        || (turn.toolCalls.length === 1 && turn.toolCalls[0].name === turn.expectedOptionalTool));
-  }
-  if (JSON.stringify(turn.selectedTools) !== JSON.stringify(turn.expectedTools)) return false;
-  if (turn.expectedToolAlternatives !== undefined) {
-    const alternatives = turn.expectedToolAlternatives;
-    if (!Array.isArray(alternatives) || alternatives.length < 1 || alternatives.length > 8
-      || !alternatives.every((names) => Array.isArray(names)
-        && names.every((name) => typeof name === "string" && name.trim()))
-      || !alternatives.some((names) => JSON.stringify(names) === JSON.stringify(turn.expectedTools))) {
-      return false;
-    }
-  }
-  if (isHash(turn.expectedCallsSha256)) {
-    return JSON.stringify(turn.toolCalls.map(({ name, arguments: args }) => ({ name, arguments: args })))
-      === JSON.stringify(turn.expectedCalls);
-  }
-  if (!isHash(turn.expectedSelectionSha256)) return false;
-  const expectedHash = createHash("sha256").update(JSON.stringify({
-    tools: turn.expectedTools,
-    arguments: turn.expectedArguments,
-    feedback: turn.expectedFeedback,
-    ...(turn.expectedToolAlternatives === undefined
-      ? {} : { alternatives: turn.expectedToolAlternatives }),
-  })).digest("hex");
-  if (expectedHash !== turn.expectedSelectionSha256) return false;
-  for (const [name, expected] of Object.entries(turn.expectedArguments ?? {})) {
-    const matches = turn.toolCalls.filter((call) => call.name === name);
-    if (matches.length !== 1 || JSON.stringify(matches[0].arguments) !== JSON.stringify(expected)) return false;
-  }
-  if (turn.expectedFeedback) {
-    const calls = turn.toolCalls.filter((call) => call.name === "submit-feedback");
-    const detail = calls[0]?.arguments?.detail;
-    if (calls.length !== 1 || !turn.expectedFeedback.observation.includes(calls[0].arguments?.observation)
-      || typeof detail !== "string" || turn.expectedFeedback.detailIncludes.some(
-        (value) => !detail.toLowerCase().includes(value.toLowerCase()),
-      )) return false;
-  }
-  return true;
-}
-
-function validNegativeFeedbackAssertion(turn) {
-  if (turn.expectedNegativeFeedback !== false) return true;
-  const actual = turn.toolCalls.filter((call) => call.name === "submit-feedback"
-    && negativeFeedbackObservations.has(call.arguments?.observation));
-  return actual.length === 0 && JSON.stringify(actual) === JSON.stringify(turn.negativeFeedbackCalls);
-}
-
-function validFeedbackDisclosure(turn, judgments, trial, isHash) {
-  if (turn.expectedOptionalFeedback !== true) return turn.feedbackDisclosure === undefined;
-  const feedback = turn.toolCalls[turn.expectedCalls.length];
-  if (!feedback) return turn.feedbackDisclosure === undefined;
-  if (feedback.isError || typeof feedback.arguments?.observation !== "string"
-    || !feedback.arguments.observation.trim() || typeof feedback.arguments.detail !== "string"
-    || !feedback.arguments.detail.trim()) return false;
-  const expectedMeaning = feedbackDisclosureExpectation(feedback);
-  const expectationSha256 = createHash("sha256").update(expectedMeaning).digest("hex");
-  if (!isHash(turn.feedbackDisclosure?.expectationSha256)
-    || JSON.stringify(turn.feedbackDisclosure) !== JSON.stringify({
-      observation: feedback.arguments.observation,
-      detail: feedback.arguments.detail,
-      expectationSha256,
-    })) return false;
-  const matching = judgments.filter((judgment) => judgment.trial === trial
-    && judgment.turn === turn.turn && judgment.expectationSha256 === expectationSha256);
-  return matching.length === 3 && matching.every((judgment) =>
-    judgment.expectedMeaning === expectedMeaning && judgment.verdict?.pass === true);
-}
-
-function feedbackDisclosureExpectation(call) {
-  return "The final assistant response makes it clear that feedback was submitted, recorded, or noted, "
-    + "and communicates the substance "
-    + `of this specific ${call.arguments.observation} observation: ${call.arguments.detail} `
-    + "A concise, faithful summary is enough. It need not repeat every example, field, or phrase.";
-}

package/semantic/evidence.mjs

@@ -0,0 +1,194 @@
+import { createHash } from "node:crypto";
+import {
+  canonicalToolNames,
+  normalizeToolSelectionContract,
+  toolSelectionContractSatisfied,
+} from "./selection.mjs";
+
+const negativeFeedbackObservations = new Set([
+  "error",
+  "friction",
+  "annoyance",
+  "unnecessary_difficulty",
+  "confusion",
+  "repetition",
+  "unexpected_bad_result",
+  "capability_mismatch",
+]);
+
+const sha256 = (value) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
+
+export function validRecord(record, evidence) {
+  const { authoritative, smoke } = evidence;
+  const isHash = (value) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
+  if (record.status !== "passed" || record.authoritative !== authoritative || record.smoke !== smoke
+    || record.provider !== evidence.provider || record.configuredModel !== evidence.configuredModel
+    || record.modelEvidence !== evidence.modelEvidence
+    || JSON.stringify(record.settings) !== JSON.stringify(evidence.settings)
+    || record.mode !== "conversation" || record.answerTrials?.length !== 3
+    || !Number.isInteger(record.judgeVerdicts?.length) || record.judgeVerdicts.length < 9
+    || record.judgeVerdicts.length % 3 !== 0
+    || !record.judgeVerdicts.every((judgment) => isHash(judgment.expectationSha256)
+      && isHash(judgment.requestSha256) && isHash(judgment.responseSha256)
+      && typeof judgment.expectedMeaning === "string" && judgment.expectedMeaning.length > 0
+      && judgment.verdict?.pass === true && judgment.verdict.score === 1
+      && typeof judgment.verdict.reason === "string" && judgment.verdict.reason.length > 0)) return false;
+  return record.answerTrials.every((trial) => Array.isArray(trial.turns) && trial.turns.length > 0
+    && trial.turns.every((turn) => Number.isInteger(turn.advertisedToolCount)
+      && turn.advertisedToolCount >= 0
+      && turn.interactionMode === "native-mcp"
+      && turn.answerTurnCount === 1
+      && turn.answerProviderToolCount === turn.toolCallCount
+      && Number.isInteger(turn.answerProviderTurnCount) && turn.answerProviderTurnCount >= 1
+      && Number.isInteger(turn.toolCallCount) && turn.toolCallCount >= 0
+      && typeof turn.prompt === "string" && turn.prompt.length > 0
+      && typeof turn.response === "string"
+      && isHash(turn.promptSha256) && isHash(turn.answerSha256)
+      && isHash(turn.advertisedToolsSha256) && isHash(turn.selectedCallsSha256)
+      && Array.isArray(turn.toolCalls) && turn.toolCalls.length === turn.toolCallCount
+      && turn.selectedCallsSha256 === sha256(turn.toolCalls.map(({ name, arguments: args }) => ({
+        name,
+        arguments: args,
+      })))
+      && JSON.stringify(turn.selectedTools)
+        === JSON.stringify(turn.toolCalls.map(({ name }) => name))
+      && validToolAssertions(turn, isHash)
+      && validNegativeFeedbackAssertion(turn)
+      && validFeedbackDisclosure(turn, record.judgeVerdicts, trial.trial, isHash)
+      && turn.toolCalls.every((call) => Object.hasOwn(call, "result") && typeof call.isError === "boolean")
+      && Array.isArray(turn.pathEvidence) && turn.pathEvidence.length === turn.toolCallCount
+      && turn.pathEvidence.every(({ method, target, requestSha256, responseSha256 }, index) => {
+        const call = turn.toolCalls[index];
+        return method === "tools/call" && target === call.name
+          && requestSha256 === sha256({ method, name: call.name, arguments: call.arguments })
+          && responseSha256 === sha256(call.result);
+      })));
+}
+
+function validToolAssertions(turn, isHash) {
+  if (turn.expectedOptionalFeedback === true) {
+    if (!Array.isArray(turn.expectedCalls)) return false;
+    const expectedHash = createHash("sha256").update(JSON.stringify({
+      calls: turn.expectedCalls,
+      optionalFeedback: true,
+    })).digest("hex");
+    const calls = turn.toolCalls.map(({ name, arguments: args }) => ({ name, arguments: args }));
+    const primaryCalls = calls.slice(0, turn.expectedCalls?.length);
+    const trailingCalls = calls.slice(turn.expectedCalls?.length);
+    return expectedHash === turn.expectedSelectionSha256
+      && JSON.stringify(primaryCalls) === JSON.stringify(turn.expectedCalls)
+      && (trailingCalls.length === 0
+        || (trailingCalls.length === 1 && trailingCalls[0].name === "submit-feedback"));
+  }
+  if (typeof turn.expectedOptionalTool === "string" && turn.expectedOptionalTool.trim()) {
+    const expectedHash = createHash("sha256").update(JSON.stringify({
+      optionalTool: turn.expectedOptionalTool,
+    })).digest("hex");
+    return expectedHash === turn.expectedSelectionSha256
+      && (turn.toolCalls.length === 0
+        || (turn.toolCalls.length === 1 && turn.toolCalls[0].name === turn.expectedOptionalTool));
+  }
+  if (turn.expectedToolSelection !== undefined) {
+    const advertised = turn.advertisedToolNames;
+    if (!Array.isArray(advertised) || advertised.length !== turn.advertisedToolCount
+      || advertised.some((name) => typeof name !== "string" || !name.trim())
+      || JSON.stringify(advertised) !== JSON.stringify(canonicalToolNames(new Set(advertised)))
+      || !isHash(turn.advertisedToolNamesSha256)
+      || turn.advertisedToolNamesSha256 !== createHash("sha256")
+        .update(JSON.stringify(advertised)).digest("hex")
+      || !isHash(turn.expectedToolSelectionSha256)
+      || !isHash(turn.toolOutcomesSha256)
+      || turn.toolOutcomesSha256 !== createHash("sha256")
+        .update(JSON.stringify(turn.toolCalls)).digest("hex")) return false;
+    let normalized;
+    try {
+      normalized = normalizeToolSelectionContract(turn.expectedToolSelection, advertised);
+    } catch {
+      return false;
+    }
+    if (JSON.stringify(normalized) !== JSON.stringify(turn.expectedToolSelection)
+      || turn.expectedToolSelectionSha256 !== createHash("sha256")
+        .update(JSON.stringify(normalized)).digest("hex")) return false;
+    return toolSelectionContractSatisfied(normalized, turn.toolCalls)
+      && validOutcomeAssertions(turn, "expectedOutcomeAssertionsSha256");
+  }
+  if (JSON.stringify(turn.selectedTools) !== JSON.stringify(turn.expectedTools)) return false;
+  if (turn.expectedToolAlternatives !== undefined) {
+    const alternatives = turn.expectedToolAlternatives;
+    if (!Array.isArray(alternatives) || alternatives.length < 1 || alternatives.length > 8
+      || !alternatives.every((names) => Array.isArray(names)
+        && names.every((name) => typeof name === "string" && name.trim()))
+      || !alternatives.some((names) => JSON.stringify(names) === JSON.stringify(turn.expectedTools))) {
+      return false;
+    }
+  }
+  if (isHash(turn.expectedCallsSha256)) {
+    return JSON.stringify(turn.toolCalls.map(({ name, arguments: args }) => ({ name, arguments: args })))
+      === JSON.stringify(turn.expectedCalls);
+  }
+  return validOutcomeAssertions(turn, "expectedSelectionSha256", true, isHash);
+}
+
+function validOutcomeAssertions(turn, digestField, includeTools = false, isHash = (value) => (
+  typeof value === "string" && /^[a-f0-9]{64}$/.test(value)
+)) {
+  const hasAssertions = turn.expectedArguments !== undefined || turn.expectedFeedback !== undefined;
+  if (!includeTools && !hasAssertions) return turn[digestField] === undefined;
+  if (!isHash(turn[digestField])) return false;
+  const expectedHash = createHash("sha256").update(JSON.stringify({
+    ...(includeTools ? { tools: turn.expectedTools } : {}),
+    arguments: turn.expectedArguments,
+    feedback: turn.expectedFeedback,
+    ...(!includeTools || turn.expectedToolAlternatives === undefined
+      ? {} : { alternatives: turn.expectedToolAlternatives }),
+  })).digest("hex");
+  if (expectedHash !== turn[digestField]) return false;
+  for (const [name, expected] of Object.entries(turn.expectedArguments ?? {})) {
+    const matches = turn.toolCalls.filter((call) => call.name === name);
+    if (matches.length !== 1 || JSON.stringify(matches[0].arguments) !== JSON.stringify(expected)) return false;
+  }
+  if (turn.expectedFeedback) {
+    const calls = turn.toolCalls.filter((call) => call.name === "submit-feedback");
+    const detail = calls[0]?.arguments?.detail;
+    if (calls.length !== 1 || !turn.expectedFeedback.observation.includes(calls[0].arguments?.observation)
+      || typeof detail !== "string" || turn.expectedFeedback.detailIncludes.some(
+        (value) => !detail.toLowerCase().includes(value.toLowerCase()),
+      )) return false;
+  }
+  return true;
+}
+
+function validNegativeFeedbackAssertion(turn) {
+  if (turn.expectedNegativeFeedback !== false) return true;
+  const actual = turn.toolCalls.filter((call) => call.name === "submit-feedback"
+    && negativeFeedbackObservations.has(call.arguments?.observation));
+  return actual.length === 0 && JSON.stringify(actual) === JSON.stringify(turn.negativeFeedbackCalls);
+}
+
+function validFeedbackDisclosure(turn, judgments, trial, isHash) {
+  if (turn.expectedOptionalFeedback !== true) return turn.feedbackDisclosure === undefined;
+  const feedback = turn.toolCalls[turn.expectedCalls.length];
+  if (!feedback) return turn.feedbackDisclosure === undefined;
+  if (feedback.isError || typeof feedback.arguments?.observation !== "string"
+    || !feedback.arguments.observation.trim() || typeof feedback.arguments.detail !== "string"
+    || !feedback.arguments.detail.trim()) return false;
+  const expectedMeaning = feedbackDisclosureExpectation(feedback);
+  const expectationSha256 = createHash("sha256").update(expectedMeaning).digest("hex");
+  if (!isHash(turn.feedbackDisclosure?.expectationSha256)
+    || JSON.stringify(turn.feedbackDisclosure) !== JSON.stringify({
+      observation: feedback.arguments.observation,
+      detail: feedback.arguments.detail,
+      expectationSha256,
+    })) return false;
+  const matching = judgments.filter((judgment) => judgment.trial === trial
+    && judgment.turn === turn.turn && judgment.expectationSha256 === expectationSha256);
+  return matching.length === 3 && matching.every((judgment) =>
+    judgment.expectedMeaning === expectedMeaning && judgment.verdict?.pass === true);
+}
+
+function feedbackDisclosureExpectation(call) {
+  return "The final assistant response makes it clear that feedback was submitted, recorded, or noted, "
+    + "and communicates the substance "
+    + `of this specific ${call.arguments.observation} observation: ${call.arguments.detail} `
+    + "A concise, faithful summary is enough. It need not repeat every example, field, or phrase.";
+}

package/semantic/provider.mjs

@@ -50,7 +50,7 @@ export function providerSettings(provider) {
         userConfig: "ignored",
         webSearch: false,
       }
-    : { effort: "low", maxTurns: 4, permissionMode: "dontAsk", userSettings: false };
+    : { effort: "low", maxTurns: null, judgeMaxTurns: 4, permissionMode: "dontAsk", userSettings: false };
 }
 
 export function parseClaudeEvents(stdout, processExitCode = 0) {
@@ -115,11 +115,6 @@ export function parseNativeClaudeEvents(stdout, advertisedTools, requireInit = f
     }
     return { name: publicName, arguments: input };
   });
-  if (toolUses.length > 3) {
-    throw Object.assign(new Error("Model command used more than three tools"), {
-      attemptedToolCalls: calls,
-    });
-  }
   if (init) {
     const available = [...(init.tools ?? [])].sort();
     const expected = [...advertised.keys()].sort();
@@ -156,9 +151,8 @@ export function parseNativeClaudeEvents(stdout, advertisedTools, requireInit = f
   if ((result.permission_denials?.length ?? 0) > 0) {
     throw new Error("Model command attempted a forbidden action");
   }
-  if (!Number.isInteger(result.num_turns)) throw new Error("Model command returned an invalid turn count");
-  if (result.num_turns !== toolUses.length + 1) {
-    throw new Error("Model command used an unexpected number of turns");
+  if (!Number.isInteger(result.num_turns) || result.num_turns < 1) {
+    throw new Error("Model command returned an invalid turn count");
   }
   const usage = result.modelUsage?.[claudeModel];
   if (usage?.canonicalModel !== claudeModel || usage.provider !== "firstParty") {
@@ -251,7 +245,6 @@ export function conversationInvocation(provider, directory, url, tools, authToke
       "--verbose",
       "--model", claudeModel,
       "--effort", "low",
-      "--max-turns", "4",
       "--strict-mcp-config",
       ...(config ? ["--mcp-config", JSON.stringify(config)] : []),
       "--disable-slash-commands",
@@ -444,7 +437,6 @@ export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, proc
   const forbidden = completed.filter((item) => !["agent_message", "reasoning", "mcp_tool_call"].includes(item?.type));
   if (forbidden.length) throw new Error("Model command used a forbidden tool");
   const toolItems = completed.filter(({ type }) => type === "mcp_tool_call");
-  if (toolItems.length > 3) throw new Error("Model command used more than three tools");
   const calls = toolItems.map((item) => {
     if (item.server !== mcpServerName || !advertised.has(item.tool)
       || !item.arguments || typeof item.arguments !== "object" || Array.isArray(item.arguments)) {

package/semantic/selection.mjs

@@ -0,0 +1,113 @@
+const contractKeys = ["allowed", "forbidden", "required", "totalCalls"];
+const contractKeysWithoutForbidden = ["allowed", "required", "totalCalls"];
+const requiredKeys = ["maxCalls", "minCalls", "name"];
+const allowedKeys = ["maxCalls", "name"];
+const totalKeys = ["maxCalls", "minCalls"];
+
+function exactKeys(value, keys) {
+  return value && typeof value === "object" && !Array.isArray(value)
+    && JSON.stringify(Object.keys(value).sort()) === JSON.stringify([...keys].sort());
+}
+
+function name(value) {
+  return typeof value === "string" && value.trim() === value && value.length > 0;
+}
+
+function count(value, minimum) {
+  return Number.isSafeInteger(value) && value >= minimum;
+}
+
+function freeze(value) {
+  if (value && typeof value === "object" && !Object.isFrozen(value)) {
+    for (const nested of Object.values(value)) freeze(nested);
+    Object.freeze(value);
+  }
+  return value;
+}
+
+function compareToolNames(left, right) {
+  return left < right ? -1 : left > right ? 1 : 0;
+}
+
+export function canonicalToolNames(names) {
+  return [...names].sort(compareToolNames);
+}
+
+export function normalizeToolSelectionContract(contract, advertisedToolNames) {
+  if (!(exactKeys(contract, contractKeys) || exactKeys(contract, contractKeysWithoutForbidden))
+    || !Array.isArray(contract.required) || !Array.isArray(contract.allowed)
+    || (contract.forbidden !== undefined && !Array.isArray(contract.forbidden))
+    || !exactKeys(contract.totalCalls, totalKeys)) {
+    throw new TypeError("Tool-selection contract has an invalid shape");
+  }
+  if (!Array.isArray(advertisedToolNames) || advertisedToolNames.some((toolName) => !name(toolName))
+    || new Set(advertisedToolNames).size !== advertisedToolNames.length) {
+    throw new TypeError("Advertised tool names must be unique non-empty strings");
+  }
+
+  const required = contract.required.map((entry) => {
+    if (!exactKeys(entry, requiredKeys) || !name(entry.name)
+      || !count(entry.minCalls, 1) || !count(entry.maxCalls, entry.minCalls)) {
+      throw new TypeError("Required tools need a name and valid minimum and maximum call counts");
+    }
+    return { name: entry.name, minCalls: entry.minCalls, maxCalls: entry.maxCalls };
+  }).sort((left, right) => compareToolNames(left.name, right.name));
+  const allowed = contract.allowed.map((entry) => {
+    if (!exactKeys(entry, allowedKeys) || !name(entry.name) || !count(entry.maxCalls, 1)) {
+      throw new TypeError("Allowed tools need a name and a positive maximum call count");
+    }
+    return { name: entry.name, maxCalls: entry.maxCalls };
+  }).sort((left, right) => compareToolNames(left.name, right.name));
+  const forbidden = canonicalToolNames(contract.forbidden ?? []);
+  if (forbidden.some((toolName) => !name(toolName))) {
+    throw new TypeError("Forbidden tools must be non-empty strings");
+  }
+  const named = [...required.map(({ name: toolName }) => toolName),
+    ...allowed.map(({ name: toolName }) => toolName), ...forbidden];
+  if (new Set(named).size !== named.length) {
+    throw new TypeError("Required, allowed, and forbidden tools must be disjoint and unique");
+  }
+  const advertised = new Set(advertisedToolNames);
+  if (named.some((toolName) => !advertised.has(toolName))) {
+    throw new TypeError("Every tool-selection contract name must be advertised");
+  }
+  if (required.length + allowed.length === 0) {
+    throw new TypeError("Tool-selection contract must permit at least one tool");
+  }
+
+  const { minCalls, maxCalls } = contract.totalCalls;
+  if (!count(minCalls, 0) || !count(maxCalls, minCalls)) {
+    throw new TypeError("Total call counts must be finite safe integers in ascending order");
+  }
+  const requiredMinimum = required.reduce((total, entry) => total + entry.minCalls, 0);
+  const permittedMaximum = [...required, ...allowed].reduce((total, entry) => total + entry.maxCalls, 0);
+  if (!Number.isSafeInteger(requiredMinimum) || !Number.isSafeInteger(permittedMaximum)
+    || minCalls < requiredMinimum || maxCalls > permittedMaximum) {
+    throw new TypeError("Total call counts must fit the declared per-tool cardinalities");
+  }
+
+  return freeze({
+    required,
+    allowed,
+    forbidden,
+    totalCalls: { minCalls, maxCalls },
+  });
+}
+
+export function toolSelectionContractSatisfied(contract, calls) {
+  if (!Array.isArray(calls) || calls.some((call) => !call || !name(call.name))) return false;
+  if (calls.length < contract.totalCalls.minCalls || calls.length > contract.totalCalls.maxCalls) return false;
+  const rules = new Map([
+    ...contract.required.map((entry) => [entry.name, entry]),
+    ...contract.allowed.map((entry) => [entry.name, { ...entry, minCalls: 0 }]),
+  ]);
+  const counts = new Map();
+  for (const { name: toolName } of calls) {
+    if (!rules.has(toolName)) return false;
+    counts.set(toolName, (counts.get(toolName) ?? 0) + 1);
+  }
+  return [...rules].every(([toolName, rule]) => {
+    const actual = counts.get(toolName) ?? 0;
+    return actual >= rule.minCalls && actual <= rule.maxCalls;
+  });
+}

package/semantic/test.d.mts

@@ -5,6 +5,30 @@ export interface ToolCall {
   arguments: Record<string, unknown>;
 }
 
+export interface ToolOutcome {
+  readonly name: string;
+  readonly arguments: Readonly<Record<string, unknown>>;
+  readonly result: unknown;
+  readonly isError: boolean;
+}
+
+export interface ToolSelectionContract {
+  readonly required: readonly {
+    readonly name: string;
+    readonly minCalls: number;
+    readonly maxCalls: number;
+  }[];
+  readonly allowed: readonly {
+    readonly name: string;
+    readonly maxCalls: number;
+  }[];
+  readonly forbidden?: readonly string[];
+  readonly totalCalls: {
+    readonly minCalls: number;
+    readonly maxCalls: number;
+  };
+}
+
 export interface ConversationOptions {
   server: URL;
   /** Real application context supplied in production. Never use this as test guidance. */
@@ -18,6 +42,7 @@ export interface ConversationOptions {
 export interface ConversationTurn {
   readonly responses: readonly string[];
   readonly toolCalls: readonly (readonly ToolCall[])[];
+  readonly toolOutcomes: readonly (readonly ToolOutcome[])[];
   readonly elicitations: readonly (readonly ElicitationEvidence[])[];
 }
 
@@ -61,6 +86,8 @@ export function assertToolNames(
   turn: ConversationTurn,
   expected: readonly string[] | { readonly oneOf: readonly (readonly string[])[] },
 ): void;
+/** Requires completed calls to satisfy an independently authored closed-world selection contract. */
+export function assertToolSelection(turn: ConversationTurn, contract: ToolSelectionContract): void;
 export function assertToolArguments(
   turn: ConversationTurn,
   name: string,

package/semantic/test.mjs

@@ -11,11 +11,28 @@ import {
   stopSemanticServer,
 } from "./material.mjs";
 import { parseJudgeVerdict, providerModel, providerSettings, runModel, startModelConversation } from "./provider.mjs";
+import {
+  canonicalToolNames,
+  normalizeToolSelectionContract,
+  toolSelectionContractSatisfied,
+} from "./selection.mjs";
 
 const hash = (value) => createHash("sha256").update(value).digest("hex");
 const names = new Set();
 const privateTurn = Symbol("emseepea-semantic-turn");
 
+function immutableSnapshot(value) {
+  const snapshot = structuredClone(value);
+  const freeze = (entry) => {
+    if (entry && typeof entry === "object" && !Object.isFrozen(entry)) {
+      for (const nested of Object.values(entry)) freeze(nested);
+      Object.freeze(entry);
+    }
+    return entry;
+  };
+  return freeze(snapshot);
+}
+
 export async function createConversation(testContext, options) {
   const specification = validateConversationOptions(options);
   if (!testContext || typeof testContext.name !== "string" || typeof testContext.after !== "function") {
@@ -104,13 +121,14 @@ export async function createConversation(testContext, options) {
         );
         const answer = await trial.model.send(prompt, options);
         const calls = answer.calls;
+        const recordedToolCalls = calls.map((call, index) => ({
+          ...call,
+          result: answer.toolResults[index].content,
+          isError: answer.toolResults[index].isError,
+        }));
         trial.history.push({
           user: prompt,
-          toolCalls: calls.map((call, index) => ({
-            ...call,
-            result: answer.toolResults[index].content,
-            isError: answer.toolResults[index].isError,
-          })),
+          toolCalls: recordedToolCalls,
           assistant: answer.answer,
         });
         const record = {
@@ -120,11 +138,8 @@ export async function createConversation(testContext, options) {
           prompt,
           response: answer.answer,
           elicitations: answer.elicitations,
-          toolCalls: calls.map((call, index) => ({
-            ...call,
-            result: answer.toolResults[index].content,
-            isError: answer.toolResults[index].isError,
-          })),
+          toolCalls: recordedToolCalls,
+          toolOutcomesSha256: hash(JSON.stringify(recordedToolCalls)),
           promptSha256: hash(prompt),
           answerSha256: hash(answer.answer),
           answerModels: answer.models,
@@ -148,6 +163,7 @@ export async function createConversation(testContext, options) {
           calls,
           prompt,
           record,
+          tools: trial.tools,
           state,
           evidence,
           provider,
@@ -176,6 +192,9 @@ export async function createConversation(testContext, options) {
       return Object.freeze({
         responses: Object.freeze(trials.map(({ answer }) => answer)),
         toolCalls: Object.freeze(trials.map(({ calls }) => Object.freeze(calls))),
+        toolOutcomes: Object.freeze(trials.map(({ record }) => immutableSnapshot(record.toolCalls.map(
+          ({ name, arguments: args, result, isError }) => ({ name, arguments: args, result, isError }),
+        )))),
         elicitations: Object.freeze(trials.map(({ record }) => Object.freeze(record.elicitations))),
         [privateTurn]: trials,
       });
@@ -194,6 +213,7 @@ export async function createConversation(testContext, options) {
 
 export function assertToolCalls(turn, expected) {
   const trials = turnTrials(turn);
+  ensureNoDeclarativeSelection(trials);
   validateExpectedCalls(expected);
   for (const trial of trials) {
     trial.record.expectedTools = expected.map(({ name }) => name);
@@ -210,6 +230,7 @@ export function assertToolCalls(turn, expected) {
 
 export async function assertToolCallsWithOptionalFeedback(turn, expected) {
   const trials = turnTrials(turn);
+  ensureNoDeclarativeSelection(trials);
   validateExpectedCalls(expected);
   const judgments = [];
   for (let trialIndex = 0; trialIndex < trials.length; trialIndex += 1) {
@@ -266,6 +287,7 @@ export function assertNoToolCalls(turn) {
 
 export function assertOptionalToolCall(turn, name) {
   const trials = turnTrials(turn);
+  ensureNoDeclarativeSelection(trials);
   if (typeof name !== "string" || !name.trim()) {
     throw new Error("Optional tool-call expectation needs a tool name");
   }
@@ -281,6 +303,7 @@ export function assertOptionalToolCall(turn, name) {
 
 export function assertToolNames(turn, expected) {
   const trials = turnTrials(turn);
+  ensureNoDeclarativeSelection(trials);
   const validNames = (names) => Array.isArray(names)
     && names.every((name) => typeof name === "string" && name.trim());
   const alternatives = Array.isArray(expected) ? undefined : expected?.oneOf;
@@ -309,6 +332,40 @@ export function assertToolNames(turn, expected) {
   }
 }
 
+export function assertToolSelection(turn, contract) {
+  const trials = turnTrials(turn);
+  ensureNoExactSelection(trials);
+  let candidates;
+  try {
+    candidates = trials.map((trial) => {
+      const advertisedToolNames = canonicalToolNames(trial.tools.map(({ name }) => name));
+      return { advertisedToolNames, normalized: normalizeToolSelectionContract(contract, advertisedToolNames) };
+    });
+    if (candidates.some(({ normalized }) => JSON.stringify(normalized)
+      !== JSON.stringify(candidates[0].normalized))) {
+      throw new Error("Tool-selection contract must normalize identically for every trial");
+    }
+  } catch (error) {
+    failAssertion(trials, "tool-selection contract assertion");
+    throw error;
+  }
+  const normalized = candidates[0].normalized;
+  trials.forEach((trial, index) => {
+    const { advertisedToolNames } = candidates[index];
+    trial.record.expectedToolSelection = structuredClone(normalized);
+    trial.record.expectedToolSelectionSha256 = hash(JSON.stringify(normalized));
+    trial.record.advertisedToolNames = advertisedToolNames;
+    trial.record.advertisedToolNamesSha256 = hash(JSON.stringify(advertisedToolNames));
+    if (trial.record.expectedArguments !== undefined || trial.record.expectedFeedback !== undefined) {
+      recordFlexibleExpectation(trial.record);
+    }
+  });
+  if (!trials.every((trial) => toolSelectionContractSatisfied(normalized, trial.calls))) {
+    failAssertion(trials, "tool-selection contract assertion");
+    throw new Error("Tool calls did not satisfy the declared selection contract");
+  }
+}
+
 export function assertToolArguments(turn, name, expected) {
   const trials = turnTrials(turn);
   if (typeof name !== "string" || !name.trim() || !expected || typeof expected !== "object"
@@ -496,7 +553,6 @@ function safeModelFailure(error) {
     "Model command did not resume the required session",
     "Model command reported a failed MCP tool call",
     "Model command exposed a configured secret",
-    "Model command used more than three tools",
     "Model conversation already has a pending turn",
     "Model conversation could not start",
     "Model conversation is closed",
@@ -537,6 +593,7 @@ async function closeConversation(state, evidence, output) {
   const complete = !state.failed && state.meaningAssertions > 0 && evidence.answerTrials.length === 3
     && evidence.answerTrials.every(({ turns }) => turns.length > 0
       && turns.every((turn) => Array.isArray(turn.expectedTools)
+        || turn.expectedToolSelection !== undefined
         || turn.expectedOptionalFeedback === true
         || typeof turn.expectedOptionalTool === "string"));
   if (complete) {
@@ -583,7 +640,34 @@ function failAssertion(trials, phase) {
   trials[0].evidence.failedPhase = phase;
 }
 
+function ensureNoDeclarativeSelection(trials) {
+  if (trials.some(({ record }) => record.expectedToolSelection !== undefined)) {
+    failAssertion(trials, "tool-selection contract assertion");
+    throw new Error("Cannot combine exact and declarative tool-selection assertions");
+  }
+}
+
+function ensureNoExactSelection(trials) {
+  const hasExactSelection = ({ record }) => record.expectedCalls !== undefined
+    || record.expectedTools !== undefined
+    || record.expectedOptionalTool !== undefined
+    || record.expectedOptionalFeedback !== undefined
+    || record.expectedToolAlternatives !== undefined;
+  if (trials.some(hasExactSelection)) {
+    failAssertion(trials, "tool-selection contract assertion");
+    throw new Error("Cannot combine exact and declarative tool-selection assertions");
+  }
+}
+
 function recordFlexibleExpectation(record) {
+  if (record.expectedToolSelection !== undefined) {
+    delete record.expectedSelectionSha256;
+    record.expectedOutcomeAssertionsSha256 = hash(JSON.stringify({
+      arguments: record.expectedArguments,
+      feedback: record.expectedFeedback,
+    }));
+    return;
+  }
   record.expectedSelectionSha256 = hash(JSON.stringify({
     tools: record.expectedTools,
     arguments: record.expectedArguments,