@emseepea/testing 0.21.7 → 0.22.0

package/README.md

@@ -57,6 +57,12 @@ advertised-tool text, an answer wrapper, or prepared MCP material. Exact tool
 assertions come from the provider's native MCP events. Follow-up messages use
 the same conversation.
 
+Native conversations have no framework tool-call or native provider-turn count
+limit. Calls must still use advertised MCP tools. The per-send timeout,
+cancellation, output checks, model checks, and isolated test environment remain
+effective. Exact application assertions decide which calls and results are
+acceptable. Tool-free judge invocations retain their existing limits.
+
 Call `await conversation.fresh()` to discard provider transcript history while
 keeping the same running test application and its saved data. The next `send`
 starts a new provider-native session in each trial.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@emseepea/testing",
-  "version": "0.21.7",
+  "version": "0.22.0",
   "description": "MCP integration and semantic testing helpers",
   "license": "MIT",
   "repository": {
@@ -46,7 +46,8 @@
     "build": "tsc -p tsconfig.json",
     "prepack": "npm run build",
     "test": "npm run build && npm run test:built",
-    "test:built": "node --test test/*.test.mjs"
+    "test:built": "node --test test/*.test.mjs",
+    "test:llm:built": "emseepea-test eval"
   },
   "dependencies": {
     "@emseepea/server": "0.23.0",

package/semantic/cli.mjs

@@ -1,23 +1,12 @@
 #!/usr/bin/env node
 import { spawn } from "node:child_process";
-import { createHash } from "node:crypto";
 import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { dirname, join, relative, resolve } from "node:path";
+import { validRecord } from "./evidence.mjs";
 import { discoverTests } from "./discover.mjs";
 import { modelVersion, providerModel, providerSettings } from "./provider.mjs";
 
-const negativeFeedbackObservations = new Set([
-  "error",
-  "friction",
-  "annoyance",
-  "unnecessary_difficulty",
-  "confusion",
-  "repetition",
-  "unexpected_bad_result",
-  "capability_mismatch",
-]);
-
 const paths = [];
 let provider = process.env.EMSEEPEA_EVAL_PROVIDER ?? "claude-local";
 let output = "artifacts/llm-eval/evidence.json";
@@ -101,7 +90,7 @@ try {
   for (const file of files) {
     const displayFile = relative(process.cwd(), file);
     const cases = Object.values(evidence.cases).filter((record) => record.file === displayFile);
-    if (!cases.length || cases.some((record) => !validRecord(record, evidence.authoritative, smoke))) {
+    if (!cases.length || cases.some((record) => !validRecord(record, evidence))) {
       evidence.errors.push(`Missing or failed qualification: ${displayFile}`);
     }
   }
@@ -114,138 +103,3 @@ try {
 }
 if (evidence.status !== "passed") process.exitCode = 1;
 console.log(`Semantic checks ${evidence.status}; evidence: ${output}`);
-
-function validRecord(record, authoritative, smoke) {
-  const isHash = (value) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
-  if (record.status !== "passed" || record.authoritative !== authoritative || record.smoke !== smoke
-    || record.provider !== evidence.provider || record.configuredModel !== evidence.configuredModel
-    || record.modelEvidence !== evidence.modelEvidence
-    || JSON.stringify(record.settings) !== JSON.stringify(evidence.settings)
-    || record.mode !== "conversation" || record.answerTrials?.length !== 3
-    || !Number.isInteger(record.judgeVerdicts?.length) || record.judgeVerdicts.length < 9
-    || record.judgeVerdicts.length % 3 !== 0
-    || !record.judgeVerdicts.every((judgment) => isHash(judgment.expectationSha256)
-      && isHash(judgment.requestSha256) && isHash(judgment.responseSha256)
-      && typeof judgment.expectedMeaning === "string" && judgment.expectedMeaning.length > 0
-      && judgment.verdict?.pass === true && judgment.verdict.score === 1
-      && typeof judgment.verdict.reason === "string" && judgment.verdict.reason.length > 0)) return false;
-  return record.answerTrials.every((trial) => Array.isArray(trial.turns) && trial.turns.length > 0
-    && trial.turns.every((turn) => Number.isInteger(turn.advertisedToolCount)
-      && turn.advertisedToolCount >= 0
-      && turn.interactionMode === "native-mcp"
-      && turn.answerTurnCount === 1
-      && turn.answerProviderToolCount === turn.toolCallCount
-      && turn.answerProviderTurnCount === turn.toolCallCount + 1
-      && Number.isInteger(turn.toolCallCount) && turn.toolCallCount >= 0 && turn.toolCallCount <= 3
-      && typeof turn.prompt === "string" && turn.prompt.length > 0
-      && typeof turn.response === "string"
-      && isHash(turn.promptSha256) && isHash(turn.answerSha256)
-      && isHash(turn.advertisedToolsSha256) && isHash(turn.selectedCallsSha256)
-      && Array.isArray(turn.toolCalls)
-      && JSON.stringify(turn.selectedTools)
-        === JSON.stringify(turn.toolCalls.map(({ name }) => name))
-      && validToolAssertions(turn, isHash)
-      && validNegativeFeedbackAssertion(turn)
-      && validFeedbackDisclosure(turn, record.judgeVerdicts, trial.trial, isHash)
-      && turn.toolCalls.every((call) => Object.hasOwn(call, "result") && typeof call.isError === "boolean")
-      && Array.isArray(turn.pathEvidence) && turn.pathEvidence.length === turn.toolCallCount
-      && turn.pathEvidence.every(({ method, target, requestSha256, responseSha256 }) =>
-        method === "tools/call" && turn.selectedTools.includes(target)
-          && isHash(requestSha256) && isHash(responseSha256))));
-}
-
-function validToolAssertions(turn, isHash) {
-  if (turn.expectedOptionalFeedback === true) {
-    if (!Array.isArray(turn.expectedCalls)) return false;
-    const expectedHash = createHash("sha256").update(JSON.stringify({
-      calls: turn.expectedCalls,
-      optionalFeedback: true,
-    })).digest("hex");
-    const calls = turn.toolCalls.map(({ name, arguments: args }) => ({ name, arguments: args }));
-    const primaryCalls = calls.slice(0, turn.expectedCalls?.length);
-    const trailingCalls = calls.slice(turn.expectedCalls?.length);
-    return expectedHash === turn.expectedSelectionSha256
-      && JSON.stringify(primaryCalls) === JSON.stringify(turn.expectedCalls)
-      && (trailingCalls.length === 0
-        || (trailingCalls.length === 1 && trailingCalls[0].name === "submit-feedback"));
-  }
-  if (typeof turn.expectedOptionalTool === "string" && turn.expectedOptionalTool.trim()) {
-    const expectedHash = createHash("sha256").update(JSON.stringify({
-      optionalTool: turn.expectedOptionalTool,
-    })).digest("hex");
-    return expectedHash === turn.expectedSelectionSha256
-      && (turn.toolCalls.length === 0
-        || (turn.toolCalls.length === 1 && turn.toolCalls[0].name === turn.expectedOptionalTool));
-  }
-  if (JSON.stringify(turn.selectedTools) !== JSON.stringify(turn.expectedTools)) return false;
-  if (turn.expectedToolAlternatives !== undefined) {
-    const alternatives = turn.expectedToolAlternatives;
-    if (!Array.isArray(alternatives) || alternatives.length < 1 || alternatives.length > 8
-      || !alternatives.every((names) => Array.isArray(names)
-        && names.every((name) => typeof name === "string" && name.trim()))
-      || !alternatives.some((names) => JSON.stringify(names) === JSON.stringify(turn.expectedTools))) {
-      return false;
-    }
-  }
-  if (isHash(turn.expectedCallsSha256)) {
-    return JSON.stringify(turn.toolCalls.map(({ name, arguments: args }) => ({ name, arguments: args })))
-      === JSON.stringify(turn.expectedCalls);
-  }
-  if (!isHash(turn.expectedSelectionSha256)) return false;
-  const expectedHash = createHash("sha256").update(JSON.stringify({
-    tools: turn.expectedTools,
-    arguments: turn.expectedArguments,
-    feedback: turn.expectedFeedback,
-    ...(turn.expectedToolAlternatives === undefined
-      ? {} : { alternatives: turn.expectedToolAlternatives }),
-  })).digest("hex");
-  if (expectedHash !== turn.expectedSelectionSha256) return false;
-  for (const [name, expected] of Object.entries(turn.expectedArguments ?? {})) {
-    const matches = turn.toolCalls.filter((call) => call.name === name);
-    if (matches.length !== 1 || JSON.stringify(matches[0].arguments) !== JSON.stringify(expected)) return false;
-  }
-  if (turn.expectedFeedback) {
-    const calls = turn.toolCalls.filter((call) => call.name === "submit-feedback");
-    const detail = calls[0]?.arguments?.detail;
-    if (calls.length !== 1 || !turn.expectedFeedback.observation.includes(calls[0].arguments?.observation)
-      || typeof detail !== "string" || turn.expectedFeedback.detailIncludes.some(
-        (value) => !detail.toLowerCase().includes(value.toLowerCase()),
-      )) return false;
-  }
-  return true;
-}
-
-function validNegativeFeedbackAssertion(turn) {
-  if (turn.expectedNegativeFeedback !== false) return true;
-  const actual = turn.toolCalls.filter((call) => call.name === "submit-feedback"
-    && negativeFeedbackObservations.has(call.arguments?.observation));
-  return actual.length === 0 && JSON.stringify(actual) === JSON.stringify(turn.negativeFeedbackCalls);
-}
-
-function validFeedbackDisclosure(turn, judgments, trial, isHash) {
-  if (turn.expectedOptionalFeedback !== true) return turn.feedbackDisclosure === undefined;
-  const feedback = turn.toolCalls[turn.expectedCalls.length];
-  if (!feedback) return turn.feedbackDisclosure === undefined;
-  if (feedback.isError || typeof feedback.arguments?.observation !== "string"
-    || !feedback.arguments.observation.trim() || typeof feedback.arguments.detail !== "string"
-    || !feedback.arguments.detail.trim()) return false;
-  const expectedMeaning = feedbackDisclosureExpectation(feedback);
-  const expectationSha256 = createHash("sha256").update(expectedMeaning).digest("hex");
-  if (!isHash(turn.feedbackDisclosure?.expectationSha256)
-    || JSON.stringify(turn.feedbackDisclosure) !== JSON.stringify({
-      observation: feedback.arguments.observation,
-      detail: feedback.arguments.detail,
-      expectationSha256,
-    })) return false;
-  const matching = judgments.filter((judgment) => judgment.trial === trial
-    && judgment.turn === turn.turn && judgment.expectationSha256 === expectationSha256);
-  return matching.length === 3 && matching.every((judgment) =>
-    judgment.expectedMeaning === expectedMeaning && judgment.verdict?.pass === true);
-}
-
-function feedbackDisclosureExpectation(call) {
-  return "The final assistant response makes it clear that feedback was submitted, recorded, or noted, "
-    + "and communicates the substance "
-    + `of this specific ${call.arguments.observation} observation: ${call.arguments.detail} `
-    + "A concise, faithful summary is enough. It need not repeat every example, field, or phrase.";
-}

package/semantic/evidence.mjs

@@ -0,0 +1,148 @@
+import { createHash } from "node:crypto";
+
+const negativeFeedbackObservations = new Set([
+  "error",
+  "friction",
+  "annoyance",
+  "unnecessary_difficulty",
+  "confusion",
+  "repetition",
+  "unexpected_bad_result",
+  "capability_mismatch",
+]);
+
+export function validRecord(record, evidence) {
+  const { authoritative, smoke } = evidence;
+  const isHash = (value) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
+  if (record.status !== "passed" || record.authoritative !== authoritative || record.smoke !== smoke
+    || record.provider !== evidence.provider || record.configuredModel !== evidence.configuredModel
+    || record.modelEvidence !== evidence.modelEvidence
+    || JSON.stringify(record.settings) !== JSON.stringify(evidence.settings)
+    || record.mode !== "conversation" || record.answerTrials?.length !== 3
+    || !Number.isInteger(record.judgeVerdicts?.length) || record.judgeVerdicts.length < 9
+    || record.judgeVerdicts.length % 3 !== 0
+    || !record.judgeVerdicts.every((judgment) => isHash(judgment.expectationSha256)
+      && isHash(judgment.requestSha256) && isHash(judgment.responseSha256)
+      && typeof judgment.expectedMeaning === "string" && judgment.expectedMeaning.length > 0
+      && judgment.verdict?.pass === true && judgment.verdict.score === 1
+      && typeof judgment.verdict.reason === "string" && judgment.verdict.reason.length > 0)) return false;
+  return record.answerTrials.every((trial) => Array.isArray(trial.turns) && trial.turns.length > 0
+    && trial.turns.every((turn) => Number.isInteger(turn.advertisedToolCount)
+      && turn.advertisedToolCount >= 0
+      && turn.interactionMode === "native-mcp"
+      && turn.answerTurnCount === 1
+      && turn.answerProviderToolCount === turn.toolCallCount
+      && Number.isInteger(turn.answerProviderTurnCount) && turn.answerProviderTurnCount >= 1
+      && Number.isInteger(turn.toolCallCount) && turn.toolCallCount >= 0
+      && typeof turn.prompt === "string" && turn.prompt.length > 0
+      && typeof turn.response === "string"
+      && isHash(turn.promptSha256) && isHash(turn.answerSha256)
+      && isHash(turn.advertisedToolsSha256) && isHash(turn.selectedCallsSha256)
+      && Array.isArray(turn.toolCalls) && turn.toolCalls.length === turn.toolCallCount
+      && JSON.stringify(turn.selectedTools)
+        === JSON.stringify(turn.toolCalls.map(({ name }) => name))
+      && validToolAssertions(turn, isHash)
+      && validNegativeFeedbackAssertion(turn)
+      && validFeedbackDisclosure(turn, record.judgeVerdicts, trial.trial, isHash)
+      && turn.toolCalls.every((call) => Object.hasOwn(call, "result") && typeof call.isError === "boolean")
+      && Array.isArray(turn.pathEvidence) && turn.pathEvidence.length === turn.toolCallCount
+      && turn.pathEvidence.every(({ method, target, requestSha256, responseSha256 }) =>
+        method === "tools/call" && turn.selectedTools.includes(target)
+          && isHash(requestSha256) && isHash(responseSha256))));
+}
+
+function validToolAssertions(turn, isHash) {
+  if (turn.expectedOptionalFeedback === true) {
+    if (!Array.isArray(turn.expectedCalls)) return false;
+    const expectedHash = createHash("sha256").update(JSON.stringify({
+      calls: turn.expectedCalls,
+      optionalFeedback: true,
+    })).digest("hex");
+    const calls = turn.toolCalls.map(({ name, arguments: args }) => ({ name, arguments: args }));
+    const primaryCalls = calls.slice(0, turn.expectedCalls?.length);
+    const trailingCalls = calls.slice(turn.expectedCalls?.length);
+    return expectedHash === turn.expectedSelectionSha256
+      && JSON.stringify(primaryCalls) === JSON.stringify(turn.expectedCalls)
+      && (trailingCalls.length === 0
+        || (trailingCalls.length === 1 && trailingCalls[0].name === "submit-feedback"));
+  }
+  if (typeof turn.expectedOptionalTool === "string" && turn.expectedOptionalTool.trim()) {
+    const expectedHash = createHash("sha256").update(JSON.stringify({
+      optionalTool: turn.expectedOptionalTool,
+    })).digest("hex");
+    return expectedHash === turn.expectedSelectionSha256
+      && (turn.toolCalls.length === 0
+        || (turn.toolCalls.length === 1 && turn.toolCalls[0].name === turn.expectedOptionalTool));
+  }
+  if (JSON.stringify(turn.selectedTools) !== JSON.stringify(turn.expectedTools)) return false;
+  if (turn.expectedToolAlternatives !== undefined) {
+    const alternatives = turn.expectedToolAlternatives;
+    if (!Array.isArray(alternatives) || alternatives.length < 1 || alternatives.length > 8
+      || !alternatives.every((names) => Array.isArray(names)
+        && names.every((name) => typeof name === "string" && name.trim()))
+      || !alternatives.some((names) => JSON.stringify(names) === JSON.stringify(turn.expectedTools))) {
+      return false;
+    }
+  }
+  if (isHash(turn.expectedCallsSha256)) {
+    return JSON.stringify(turn.toolCalls.map(({ name, arguments: args }) => ({ name, arguments: args })))
+      === JSON.stringify(turn.expectedCalls);
+  }
+  if (!isHash(turn.expectedSelectionSha256)) return false;
+  const expectedHash = createHash("sha256").update(JSON.stringify({
+    tools: turn.expectedTools,
+    arguments: turn.expectedArguments,
+    feedback: turn.expectedFeedback,
+    ...(turn.expectedToolAlternatives === undefined
+      ? {} : { alternatives: turn.expectedToolAlternatives }),
+  })).digest("hex");
+  if (expectedHash !== turn.expectedSelectionSha256) return false;
+  for (const [name, expected] of Object.entries(turn.expectedArguments ?? {})) {
+    const matches = turn.toolCalls.filter((call) => call.name === name);
+    if (matches.length !== 1 || JSON.stringify(matches[0].arguments) !== JSON.stringify(expected)) return false;
+  }
+  if (turn.expectedFeedback) {
+    const calls = turn.toolCalls.filter((call) => call.name === "submit-feedback");
+    const detail = calls[0]?.arguments?.detail;
+    if (calls.length !== 1 || !turn.expectedFeedback.observation.includes(calls[0].arguments?.observation)
+      || typeof detail !== "string" || turn.expectedFeedback.detailIncludes.some(
+        (value) => !detail.toLowerCase().includes(value.toLowerCase()),
+      )) return false;
+  }
+  return true;
+}
+
+function validNegativeFeedbackAssertion(turn) {
+  if (turn.expectedNegativeFeedback !== false) return true;
+  const actual = turn.toolCalls.filter((call) => call.name === "submit-feedback"
+    && negativeFeedbackObservations.has(call.arguments?.observation));
+  return actual.length === 0 && JSON.stringify(actual) === JSON.stringify(turn.negativeFeedbackCalls);
+}
+
+function validFeedbackDisclosure(turn, judgments, trial, isHash) {
+  if (turn.expectedOptionalFeedback !== true) return turn.feedbackDisclosure === undefined;
+  const feedback = turn.toolCalls[turn.expectedCalls.length];
+  if (!feedback) return turn.feedbackDisclosure === undefined;
+  if (feedback.isError || typeof feedback.arguments?.observation !== "string"
+    || !feedback.arguments.observation.trim() || typeof feedback.arguments.detail !== "string"
+    || !feedback.arguments.detail.trim()) return false;
+  const expectedMeaning = feedbackDisclosureExpectation(feedback);
+  const expectationSha256 = createHash("sha256").update(expectedMeaning).digest("hex");
+  if (!isHash(turn.feedbackDisclosure?.expectationSha256)
+    || JSON.stringify(turn.feedbackDisclosure) !== JSON.stringify({
+      observation: feedback.arguments.observation,
+      detail: feedback.arguments.detail,
+      expectationSha256,
+    })) return false;
+  const matching = judgments.filter((judgment) => judgment.trial === trial
+    && judgment.turn === turn.turn && judgment.expectationSha256 === expectationSha256);
+  return matching.length === 3 && matching.every((judgment) =>
+    judgment.expectedMeaning === expectedMeaning && judgment.verdict?.pass === true);
+}
+
+function feedbackDisclosureExpectation(call) {
+  return "The final assistant response makes it clear that feedback was submitted, recorded, or noted, "
+    + "and communicates the substance "
+    + `of this specific ${call.arguments.observation} observation: ${call.arguments.detail} `
+    + "A concise, faithful summary is enough. It need not repeat every example, field, or phrase.";
+}

package/semantic/provider.mjs

@@ -50,7 +50,7 @@ export function providerSettings(provider) {
         userConfig: "ignored",
         webSearch: false,
       }
-    : { effort: "low", maxTurns: 4, permissionMode: "dontAsk", userSettings: false };
+    : { effort: "low", maxTurns: null, judgeMaxTurns: 4, permissionMode: "dontAsk", userSettings: false };
 }
 
 export function parseClaudeEvents(stdout, processExitCode = 0) {
@@ -115,11 +115,6 @@ export function parseNativeClaudeEvents(stdout, advertisedTools, requireInit = f
     }
     return { name: publicName, arguments: input };
   });
-  if (toolUses.length > 3) {
-    throw Object.assign(new Error("Model command used more than three tools"), {
-      attemptedToolCalls: calls,
-    });
-  }
   if (init) {
     const available = [...(init.tools ?? [])].sort();
     const expected = [...advertised.keys()].sort();
@@ -156,9 +151,8 @@ export function parseNativeClaudeEvents(stdout, advertisedTools, requireInit = f
   if ((result.permission_denials?.length ?? 0) > 0) {
     throw new Error("Model command attempted a forbidden action");
   }
-  if (!Number.isInteger(result.num_turns)) throw new Error("Model command returned an invalid turn count");
-  if (result.num_turns !== toolUses.length + 1) {
-    throw new Error("Model command used an unexpected number of turns");
+  if (!Number.isInteger(result.num_turns) || result.num_turns < 1) {
+    throw new Error("Model command returned an invalid turn count");
   }
   const usage = result.modelUsage?.[claudeModel];
   if (usage?.canonicalModel !== claudeModel || usage.provider !== "firstParty") {
@@ -251,7 +245,6 @@ export function conversationInvocation(provider, directory, url, tools, authToke
       "--verbose",
       "--model", claudeModel,
       "--effort", "low",
-      "--max-turns", "4",
       "--strict-mcp-config",
       ...(config ? ["--mcp-config", JSON.stringify(config)] : []),
       "--disable-slash-commands",
@@ -444,7 +437,6 @@ export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, proc
   const forbidden = completed.filter((item) => !["agent_message", "reasoning", "mcp_tool_call"].includes(item?.type));
   if (forbidden.length) throw new Error("Model command used a forbidden tool");
   const toolItems = completed.filter(({ type }) => type === "mcp_tool_call");
-  if (toolItems.length > 3) throw new Error("Model command used more than three tools");
   const calls = toolItems.map((item) => {
     if (item.server !== mcpServerName || !advertised.has(item.tool)
       || !item.arguments || typeof item.arguments !== "object" || Array.isArray(item.arguments)) {

package/semantic/test.mjs

@@ -496,7 +496,6 @@ function safeModelFailure(error) {
     "Model command did not resume the required session",
     "Model command reported a failed MCP tool call",
     "Model command exposed a configured secret",
-    "Model command used more than three tools",
     "Model conversation already has a pending turn",
     "Model conversation could not start",
     "Model conversation is closed",