@emseepea/testing 0.22.0 → 0.23.1

package/README.md

@@ -29,6 +29,39 @@ sequence in the listed order and count. The evidence retains all alternatives
 and the matched sequence; arguments, meaning, confirmation and feedback still
 need their own assertions.
 
+Use `assertToolSelection` when valid tool order or recovery paths can vary, but
+the permitted tools and call counts are bounded. Its tool-selection contract
+lists required, allowed, and explicitly forbidden tools. Any advertised tool
+that is not required or allowed is rejected.
+
+```js
+assertToolSelection(turn, {
+  required: [
+    { name: "list-candidates", minCalls: 1, maxCalls: 1 },
+    { name: "commit-selection", minCalls: 1, maxCalls: 1 },
+  ],
+  allowed: [{ name: "lookup-candidate", maxCalls: 2 }],
+  forbidden: [],
+  totalCalls: { minCalls: 2, maxCalls: 4 },
+});
+```
+
+The finite maximums are checked after the turn. They do not stop tool calls or
+truncate evidence.
+
+A passing selection assertion proves only that every
+recorded call satisfied the declared tool and count rules. It does not prove
+argument correctness, persisted effects, response meaning, or recovery from an
+error. Keep an exact assertion when order, count, and arguments are
+deterministic, and add the separate assertions your application outcome needs.
+
+Each turn also exposes `toolOutcomes`, aligned by trial and call order with
+`toolCalls`. Each outcome is a read-only snapshot of what the model saw. It
+contains only `name`, `arguments`, the model-visible `result`, and `isError`.
+Use it to check an expected error and the later calls that followed the error.
+A later non-error result does not by itself prove recovery or a persisted
+effect.
+
 For a successful turn that advertises feedback, use
 `assertToolCallsWithOptionalFeedback`. It requires the exact ordered primary
 calls and arguments, then allows no feedback call or one trailing
@@ -60,7 +93,7 @@ the same conversation.
 Native conversations have no framework tool-call or native provider-turn count
 limit. Calls must still use advertised MCP tools. The per-send timeout,
 cancellation, output checks, model checks, and isolated test environment remain
-effective. Exact application assertions decide which calls and results are
+effective. Application assertions decide which calls and results are
 acceptable. Tool-free judge invocations retain their existing limits.
 
 Call `await conversation.fresh()` to discard provider transcript history while

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@emseepea/testing",
-  "version": "0.22.0",
+  "version": "0.23.1",
   "description": "MCP integration and semantic testing helpers",
   "license": "MIT",
   "repository": {
@@ -43,7 +43,7 @@
     "README.md"
   ],
   "scripts": {
-    "build": "tsc -p tsconfig.json",
+    "build": "tsc -p tsconfig.json && tsc -p tsconfig.type-tests.json",
     "prepack": "npm run build",
     "test": "npm run build && npm run test:built",
     "test:built": "node --test test/*.test.mjs",

package/semantic/evidence.mjs

@@ -1,4 +1,9 @@
 import { createHash } from "node:crypto";
+import {
+  canonicalToolNames,
+  normalizeToolSelectionContract,
+  toolSelectionContractSatisfied,
+} from "./selection.mjs";
 
 const negativeFeedbackObservations = new Set([
   "error",
@@ -11,6 +16,8 @@ const negativeFeedbackObservations = new Set([
   "capability_mismatch",
 ]);
 
+const sha256 = (value) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
+
 export function validRecord(record, evidence) {
   const { authoritative, smoke } = evidence;
   const isHash = (value) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
@@ -39,6 +46,10 @@ export function validRecord(record, evidence) {
       && isHash(turn.promptSha256) && isHash(turn.answerSha256)
       && isHash(turn.advertisedToolsSha256) && isHash(turn.selectedCallsSha256)
       && Array.isArray(turn.toolCalls) && turn.toolCalls.length === turn.toolCallCount
+      && turn.selectedCallsSha256 === sha256(turn.toolCalls.map(({ name, arguments: args }) => ({
+        name,
+        arguments: args,
+      })))
       && JSON.stringify(turn.selectedTools)
         === JSON.stringify(turn.toolCalls.map(({ name }) => name))
       && validToolAssertions(turn, isHash)
@@ -46,9 +57,12 @@ export function validRecord(record, evidence) {
       && validFeedbackDisclosure(turn, record.judgeVerdicts, trial.trial, isHash)
       && turn.toolCalls.every((call) => Object.hasOwn(call, "result") && typeof call.isError === "boolean")
       && Array.isArray(turn.pathEvidence) && turn.pathEvidence.length === turn.toolCallCount
-      && turn.pathEvidence.every(({ method, target, requestSha256, responseSha256 }) =>
-        method === "tools/call" && turn.selectedTools.includes(target)
-          && isHash(requestSha256) && isHash(responseSha256))));
+      && turn.pathEvidence.every(({ method, target, requestSha256, responseSha256 }, index) => {
+        const call = turn.toolCalls[index];
+        return method === "tools/call" && target === call.name
+          && requestSha256 === sha256({ method, name: call.name, arguments: call.arguments })
+          && responseSha256 === sha256(call.result);
+      })));
 }
 
 function validToolAssertions(turn, isHash) {
@@ -74,6 +88,30 @@ function validToolAssertions(turn, isHash) {
       && (turn.toolCalls.length === 0
         || (turn.toolCalls.length === 1 && turn.toolCalls[0].name === turn.expectedOptionalTool));
   }
+  if (turn.expectedToolSelection !== undefined) {
+    const advertised = turn.advertisedToolNames;
+    if (!Array.isArray(advertised) || advertised.length !== turn.advertisedToolCount
+      || advertised.some((name) => typeof name !== "string" || !name.trim())
+      || JSON.stringify(advertised) !== JSON.stringify(canonicalToolNames(new Set(advertised)))
+      || !isHash(turn.advertisedToolNamesSha256)
+      || turn.advertisedToolNamesSha256 !== createHash("sha256")
+        .update(JSON.stringify(advertised)).digest("hex")
+      || !isHash(turn.expectedToolSelectionSha256)
+      || !isHash(turn.toolOutcomesSha256)
+      || turn.toolOutcomesSha256 !== createHash("sha256")
+        .update(JSON.stringify(turn.toolCalls)).digest("hex")) return false;
+    let normalized;
+    try {
+      normalized = normalizeToolSelectionContract(turn.expectedToolSelection, advertised);
+    } catch {
+      return false;
+    }
+    if (JSON.stringify(normalized) !== JSON.stringify(turn.expectedToolSelection)
+      || turn.expectedToolSelectionSha256 !== createHash("sha256")
+        .update(JSON.stringify(normalized)).digest("hex")) return false;
+    return toolSelectionContractSatisfied(normalized, turn.toolCalls)
+      && validOutcomeAssertions(turn, "expectedOutcomeAssertionsSha256");
+  }
   if (JSON.stringify(turn.selectedTools) !== JSON.stringify(turn.expectedTools)) return false;
   if (turn.expectedToolAlternatives !== undefined) {
     const alternatives = turn.expectedToolAlternatives;
@@ -88,15 +126,23 @@ function validToolAssertions(turn, isHash) {
     return JSON.stringify(turn.toolCalls.map(({ name, arguments: args }) => ({ name, arguments: args })))
       === JSON.stringify(turn.expectedCalls);
   }
-  if (!isHash(turn.expectedSelectionSha256)) return false;
+  return validOutcomeAssertions(turn, "expectedSelectionSha256", true, isHash);
+}
+
+function validOutcomeAssertions(turn, digestField, includeTools = false, isHash = (value) => (
+  typeof value === "string" && /^[a-f0-9]{64}$/.test(value)
+)) {
+  const hasAssertions = turn.expectedArguments !== undefined || turn.expectedFeedback !== undefined;
+  if (!includeTools && !hasAssertions) return turn[digestField] === undefined;
+  if (!isHash(turn[digestField])) return false;
   const expectedHash = createHash("sha256").update(JSON.stringify({
-    tools: turn.expectedTools,
+    ...(includeTools ? { tools: turn.expectedTools } : {}),
     arguments: turn.expectedArguments,
     feedback: turn.expectedFeedback,
-    ...(turn.expectedToolAlternatives === undefined
+    ...(!includeTools || turn.expectedToolAlternatives === undefined
       ? {} : { alternatives: turn.expectedToolAlternatives }),
   })).digest("hex");
-  if (expectedHash !== turn.expectedSelectionSha256) return false;
+  if (expectedHash !== turn[digestField]) return false;
   for (const [name, expected] of Object.entries(turn.expectedArguments ?? {})) {
     const matches = turn.toolCalls.filter((call) => call.name === name);
     if (matches.length !== 1 || JSON.stringify(matches[0].arguments) !== JSON.stringify(expected)) return false;

package/semantic/provider.mjs

@@ -529,6 +529,7 @@ function codexInvocation(provider, prompt, directory, conversation) {
     "-c", 'shell_environment_policy.inherit="none"',
     "-c", "allow_login_shell=false",
     "-c", "features.multi_agent=false",
+    "-c", "features.shell_tool=false",
     "-c", "features.unified_exec=false",
   ];
   if (conversation?.context) args.push("-c", `developer_instructions=${JSON.stringify(conversation.context)}`);

package/semantic/selection.mjs

@@ -0,0 +1,113 @@
+const contractKeys = ["allowed", "forbidden", "required", "totalCalls"];
+const contractKeysWithoutForbidden = ["allowed", "required", "totalCalls"];
+const requiredKeys = ["maxCalls", "minCalls", "name"];
+const allowedKeys = ["maxCalls", "name"];
+const totalKeys = ["maxCalls", "minCalls"];
+
+function exactKeys(value, keys) {
+  return value && typeof value === "object" && !Array.isArray(value)
+    && JSON.stringify(Object.keys(value).sort()) === JSON.stringify([...keys].sort());
+}
+
+function name(value) {
+  return typeof value === "string" && value.trim() === value && value.length > 0;
+}
+
+function count(value, minimum) {
+  return Number.isSafeInteger(value) && value >= minimum;
+}
+
+function freeze(value) {
+  if (value && typeof value === "object" && !Object.isFrozen(value)) {
+    for (const nested of Object.values(value)) freeze(nested);
+    Object.freeze(value);
+  }
+  return value;
+}
+
+function compareToolNames(left, right) {
+  return left < right ? -1 : left > right ? 1 : 0;
+}
+
+export function canonicalToolNames(names) {
+  return [...names].sort(compareToolNames);
+}
+
+export function normalizeToolSelectionContract(contract, advertisedToolNames) {
+  if (!(exactKeys(contract, contractKeys) || exactKeys(contract, contractKeysWithoutForbidden))
+    || !Array.isArray(contract.required) || !Array.isArray(contract.allowed)
+    || (contract.forbidden !== undefined && !Array.isArray(contract.forbidden))
+    || !exactKeys(contract.totalCalls, totalKeys)) {
+    throw new TypeError("Tool-selection contract has an invalid shape");
+  }
+  if (!Array.isArray(advertisedToolNames) || advertisedToolNames.some((toolName) => !name(toolName))
+    || new Set(advertisedToolNames).size !== advertisedToolNames.length) {
+    throw new TypeError("Advertised tool names must be unique non-empty strings");
+  }
+
+  const required = contract.required.map((entry) => {
+    if (!exactKeys(entry, requiredKeys) || !name(entry.name)
+      || !count(entry.minCalls, 1) || !count(entry.maxCalls, entry.minCalls)) {
+      throw new TypeError("Required tools need a name and valid minimum and maximum call counts");
+    }
+    return { name: entry.name, minCalls: entry.minCalls, maxCalls: entry.maxCalls };
+  }).sort((left, right) => compareToolNames(left.name, right.name));
+  const allowed = contract.allowed.map((entry) => {
+    if (!exactKeys(entry, allowedKeys) || !name(entry.name) || !count(entry.maxCalls, 1)) {
+      throw new TypeError("Allowed tools need a name and a positive maximum call count");
+    }
+    return { name: entry.name, maxCalls: entry.maxCalls };
+  }).sort((left, right) => compareToolNames(left.name, right.name));
+  const forbidden = canonicalToolNames(contract.forbidden ?? []);
+  if (forbidden.some((toolName) => !name(toolName))) {
+    throw new TypeError("Forbidden tools must be non-empty strings");
+  }
+  const named = [...required.map(({ name: toolName }) => toolName),
+    ...allowed.map(({ name: toolName }) => toolName), ...forbidden];
+  if (new Set(named).size !== named.length) {
+    throw new TypeError("Required, allowed, and forbidden tools must be disjoint and unique");
+  }
+  const advertised = new Set(advertisedToolNames);
+  if (named.some((toolName) => !advertised.has(toolName))) {
+    throw new TypeError("Every tool-selection contract name must be advertised");
+  }
+  if (required.length + allowed.length === 0) {
+    throw new TypeError("Tool-selection contract must permit at least one tool");
+  }
+
+  const { minCalls, maxCalls } = contract.totalCalls;
+  if (!count(minCalls, 0) || !count(maxCalls, minCalls)) {
+    throw new TypeError("Total call counts must be finite safe integers in ascending order");
+  }
+  const requiredMinimum = required.reduce((total, entry) => total + entry.minCalls, 0);
+  const permittedMaximum = [...required, ...allowed].reduce((total, entry) => total + entry.maxCalls, 0);
+  if (!Number.isSafeInteger(requiredMinimum) || !Number.isSafeInteger(permittedMaximum)
+    || minCalls < requiredMinimum || maxCalls > permittedMaximum) {
+    throw new TypeError("Total call counts must fit the declared per-tool cardinalities");
+  }
+
+  return freeze({
+    required,
+    allowed,
+    forbidden,
+    totalCalls: { minCalls, maxCalls },
+  });
+}
+
+export function toolSelectionContractSatisfied(contract, calls) {
+  if (!Array.isArray(calls) || calls.some((call) => !call || !name(call.name))) return false;
+  if (calls.length < contract.totalCalls.minCalls || calls.length > contract.totalCalls.maxCalls) return false;
+  const rules = new Map([
+    ...contract.required.map((entry) => [entry.name, entry]),
+    ...contract.allowed.map((entry) => [entry.name, { ...entry, minCalls: 0 }]),
+  ]);
+  const counts = new Map();
+  for (const { name: toolName } of calls) {
+    if (!rules.has(toolName)) return false;
+    counts.set(toolName, (counts.get(toolName) ?? 0) + 1);
+  }
+  return [...rules].every(([toolName, rule]) => {
+    const actual = counts.get(toolName) ?? 0;
+    return actual >= rule.minCalls && actual <= rule.maxCalls;
+  });
+}

package/semantic/test.d.mts

@@ -5,6 +5,30 @@ export interface ToolCall {
   arguments: Record<string, unknown>;
 }
 
+export interface ToolOutcome {
+  readonly name: string;
+  readonly arguments: Readonly<Record<string, unknown>>;
+  readonly result: unknown;
+  readonly isError: boolean;
+}
+
+export interface ToolSelectionContract {
+  readonly required: readonly {
+    readonly name: string;
+    readonly minCalls: number;
+    readonly maxCalls: number;
+  }[];
+  readonly allowed: readonly {
+    readonly name: string;
+    readonly maxCalls: number;
+  }[];
+  readonly forbidden?: readonly string[];
+  readonly totalCalls: {
+    readonly minCalls: number;
+    readonly maxCalls: number;
+  };
+}
+
 export interface ConversationOptions {
   server: URL;
   /** Real application context supplied in production. Never use this as test guidance. */
@@ -18,6 +42,7 @@ export interface ConversationOptions {
 export interface ConversationTurn {
   readonly responses: readonly string[];
   readonly toolCalls: readonly (readonly ToolCall[])[];
+  readonly toolOutcomes: readonly (readonly ToolOutcome[])[];
   readonly elicitations: readonly (readonly ElicitationEvidence[])[];
 }
 
@@ -61,6 +86,8 @@ export function assertToolNames(
   turn: ConversationTurn,
   expected: readonly string[] | { readonly oneOf: readonly (readonly string[])[] },
 ): void;
+/** Requires completed calls to satisfy an independently authored closed-world selection contract. */
+export function assertToolSelection(turn: ConversationTurn, contract: ToolSelectionContract): void;
 export function assertToolArguments(
   turn: ConversationTurn,
   name: string,

package/semantic/test.mjs

@@ -11,11 +11,28 @@ import {
   stopSemanticServer,
 } from "./material.mjs";
 import { parseJudgeVerdict, providerModel, providerSettings, runModel, startModelConversation } from "./provider.mjs";
+import {
+  canonicalToolNames,
+  normalizeToolSelectionContract,
+  toolSelectionContractSatisfied,
+} from "./selection.mjs";
 
 const hash = (value) => createHash("sha256").update(value).digest("hex");
 const names = new Set();
 const privateTurn = Symbol("emseepea-semantic-turn");
 
+function immutableSnapshot(value) {
+  const snapshot = structuredClone(value);
+  const freeze = (entry) => {
+    if (entry && typeof entry === "object" && !Object.isFrozen(entry)) {
+      for (const nested of Object.values(entry)) freeze(nested);
+      Object.freeze(entry);
+    }
+    return entry;
+  };
+  return freeze(snapshot);
+}
+
 export async function createConversation(testContext, options) {
   const specification = validateConversationOptions(options);
   if (!testContext || typeof testContext.name !== "string" || typeof testContext.after !== "function") {
@@ -104,13 +121,14 @@ export async function createConversation(testContext, options) {
         );
         const answer = await trial.model.send(prompt, options);
         const calls = answer.calls;
+        const recordedToolCalls = calls.map((call, index) => ({
+          ...call,
+          result: answer.toolResults[index].content,
+          isError: answer.toolResults[index].isError,
+        }));
         trial.history.push({
           user: prompt,
-          toolCalls: calls.map((call, index) => ({
-            ...call,
-            result: answer.toolResults[index].content,
-            isError: answer.toolResults[index].isError,
-          })),
+          toolCalls: recordedToolCalls,
           assistant: answer.answer,
         });
         const record = {
@@ -120,11 +138,8 @@ export async function createConversation(testContext, options) {
           prompt,
           response: answer.answer,
           elicitations: answer.elicitations,
-          toolCalls: calls.map((call, index) => ({
-            ...call,
-            result: answer.toolResults[index].content,
-            isError: answer.toolResults[index].isError,
-          })),
+          toolCalls: recordedToolCalls,
+          toolOutcomesSha256: hash(JSON.stringify(recordedToolCalls)),
           promptSha256: hash(prompt),
           answerSha256: hash(answer.answer),
           answerModels: answer.models,
@@ -148,6 +163,7 @@ export async function createConversation(testContext, options) {
           calls,
           prompt,
           record,
+          tools: trial.tools,
           state,
           evidence,
           provider,
@@ -176,6 +192,9 @@ export async function createConversation(testContext, options) {
       return Object.freeze({
         responses: Object.freeze(trials.map(({ answer }) => answer)),
         toolCalls: Object.freeze(trials.map(({ calls }) => Object.freeze(calls))),
+        toolOutcomes: Object.freeze(trials.map(({ record }) => immutableSnapshot(record.toolCalls.map(
+          ({ name, arguments: args, result, isError }) => ({ name, arguments: args, result, isError }),
+        )))),
         elicitations: Object.freeze(trials.map(({ record }) => Object.freeze(record.elicitations))),
         [privateTurn]: trials,
       });
@@ -194,6 +213,7 @@ export async function createConversation(testContext, options) {
 
 export function assertToolCalls(turn, expected) {
   const trials = turnTrials(turn);
+  ensureNoDeclarativeSelection(trials);
   validateExpectedCalls(expected);
   for (const trial of trials) {
     trial.record.expectedTools = expected.map(({ name }) => name);
@@ -210,6 +230,7 @@ export function assertToolCalls(turn, expected) {
 
 export async function assertToolCallsWithOptionalFeedback(turn, expected) {
   const trials = turnTrials(turn);
+  ensureNoDeclarativeSelection(trials);
   validateExpectedCalls(expected);
   const judgments = [];
   for (let trialIndex = 0; trialIndex < trials.length; trialIndex += 1) {
@@ -266,6 +287,7 @@ export function assertNoToolCalls(turn) {
 
 export function assertOptionalToolCall(turn, name) {
   const trials = turnTrials(turn);
+  ensureNoDeclarativeSelection(trials);
   if (typeof name !== "string" || !name.trim()) {
     throw new Error("Optional tool-call expectation needs a tool name");
   }
@@ -281,6 +303,7 @@ export function assertOptionalToolCall(turn, name) {
 
 export function assertToolNames(turn, expected) {
   const trials = turnTrials(turn);
+  ensureNoDeclarativeSelection(trials);
   const validNames = (names) => Array.isArray(names)
     && names.every((name) => typeof name === "string" && name.trim());
   const alternatives = Array.isArray(expected) ? undefined : expected?.oneOf;
@@ -309,6 +332,40 @@ export function assertToolNames(turn, expected) {
   }
 }
 
+export function assertToolSelection(turn, contract) {
+  const trials = turnTrials(turn);
+  ensureNoExactSelection(trials);
+  let candidates;
+  try {
+    candidates = trials.map((trial) => {
+      const advertisedToolNames = canonicalToolNames(trial.tools.map(({ name }) => name));
+      return { advertisedToolNames, normalized: normalizeToolSelectionContract(contract, advertisedToolNames) };
+    });
+    if (candidates.some(({ normalized }) => JSON.stringify(normalized)
+      !== JSON.stringify(candidates[0].normalized))) {
+      throw new Error("Tool-selection contract must normalize identically for every trial");
+    }
+  } catch (error) {
+    failAssertion(trials, "tool-selection contract assertion");
+    throw error;
+  }
+  const normalized = candidates[0].normalized;
+  trials.forEach((trial, index) => {
+    const { advertisedToolNames } = candidates[index];
+    trial.record.expectedToolSelection = structuredClone(normalized);
+    trial.record.expectedToolSelectionSha256 = hash(JSON.stringify(normalized));
+    trial.record.advertisedToolNames = advertisedToolNames;
+    trial.record.advertisedToolNamesSha256 = hash(JSON.stringify(advertisedToolNames));
+    if (trial.record.expectedArguments !== undefined || trial.record.expectedFeedback !== undefined) {
+      recordFlexibleExpectation(trial.record);
+    }
+  });
+  if (!trials.every((trial) => toolSelectionContractSatisfied(normalized, trial.calls))) {
+    failAssertion(trials, "tool-selection contract assertion");
+    throw new Error("Tool calls did not satisfy the declared selection contract");
+  }
+}
+
 export function assertToolArguments(turn, name, expected) {
   const trials = turnTrials(turn);
   if (typeof name !== "string" || !name.trim() || !expected || typeof expected !== "object"
@@ -536,6 +593,7 @@ async function closeConversation(state, evidence, output) {
   const complete = !state.failed && state.meaningAssertions > 0 && evidence.answerTrials.length === 3
     && evidence.answerTrials.every(({ turns }) => turns.length > 0
       && turns.every((turn) => Array.isArray(turn.expectedTools)
+        || turn.expectedToolSelection !== undefined
         || turn.expectedOptionalFeedback === true
         || typeof turn.expectedOptionalTool === "string"));
   if (complete) {
@@ -582,7 +640,34 @@ function failAssertion(trials, phase) {
   trials[0].evidence.failedPhase = phase;
 }
 
+function ensureNoDeclarativeSelection(trials) {
+  if (trials.some(({ record }) => record.expectedToolSelection !== undefined)) {
+    failAssertion(trials, "tool-selection contract assertion");
+    throw new Error("Cannot combine exact and declarative tool-selection assertions");
+  }
+}
+
+function ensureNoExactSelection(trials) {
+  const hasExactSelection = ({ record }) => record.expectedCalls !== undefined
+    || record.expectedTools !== undefined
+    || record.expectedOptionalTool !== undefined
+    || record.expectedOptionalFeedback !== undefined
+    || record.expectedToolAlternatives !== undefined;
+  if (trials.some(hasExactSelection)) {
+    failAssertion(trials, "tool-selection contract assertion");
+    throw new Error("Cannot combine exact and declarative tool-selection assertions");
+  }
+}
+
 function recordFlexibleExpectation(record) {
+  if (record.expectedToolSelection !== undefined) {
+    delete record.expectedSelectionSha256;
+    record.expectedOutcomeAssertionsSha256 = hash(JSON.stringify({
+      arguments: record.expectedArguments,
+      feedback: record.expectedFeedback,
+    }));
+    return;
+  }
   record.expectedSelectionSha256 = hash(JSON.stringify({
     tools: record.expectedTools,
     arguments: record.expectedArguments,