@emseepea/testing 0.26.13 → 0.26.15

package/README.md

@@ -109,6 +109,14 @@ argument correctness, persisted effects, response meaning, or recovery from an
 error. Keep an exact assertion when order, count, and arguments are
 deterministic, and add the separate assertions your application outcome needs.
 
+Each turn exposes `assistantMessages`, an immutable array of visible native
+Claude messages for each trial, in event order. Other providers currently return
+empty arrays. These messages may include the final message; `responses` and
+response assertions continue to use the separate terminal result.
+
+Saved evidence and meaning-judge history retain these messages without hidden
+reasoning. Earlier text does not satisfy an assertion about the final response.
+
 Each turn also exposes `toolOutcomes`, aligned by trial and call order with
 `toolCalls`. Each outcome is a read-only snapshot of what the model saw. It
 contains only `name`, `arguments`, the model-visible `result`, and `isError`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@emseepea/testing",
-  "version": "0.26.13",
+  "version": "0.26.15",
   "description": "MCP integration and semantic testing helpers",
   "license": "MIT",
   "repository": {
@@ -57,7 +57,7 @@
     "test:llm:built": "emseepea-test eval"
   },
   "dependencies": {
-    "@emseepea/server": "0.25.10",
+    "@emseepea/server": "0.26.0",
     "@modelcontextprotocol/client": "2.3.1",
     "@modelcontextprotocol/server": "2.3.1",
     "@modelcontextprotocol/core": "2.3.1"

package/semantic/provider.mjs

@@ -190,6 +190,12 @@ export function parseNativeClaudeEvents(stdout, advertisedTools, requireInit = f
   }
   return {
     answer: result.result,
+    assistantMessages: events.flatMap(({ type, message }) => {
+      if (type !== "assistant" || !Array.isArray(message?.content)
+        || (message.role !== undefined && message.role !== "assistant")) return [];
+      const text = message.content.filter((block) => block.type === "text" && typeof block.text === "string");
+      return text.length ? [text.map((block) => block.text).join("")] : [];
+    }),
     calls,
     toolResults: toolUses.map(({ id }) => {
       const toolResult = toolResults.get(id);
@@ -448,9 +454,6 @@ export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, proc
   if (authentication.test(errorText)) {
     throw codexEvidenceError("Model command is not signed in", "authentication");
   }
-  const failedTool = events.some(({ item }) => item?.type === "mcp_tool_call"
-    && (item.status === "failed" || item.result?.isError === true || item.result?.is_error === true));
-  if (failedTool) throw new Error("Model command reported a failed MCP tool call");
   if (processExitCode !== 0) throw codexProcessError(processExitCode, providerFailureCategory);
   if (events.some(({ type }) => type === "turn.failed" || type === "error")) {
     throw new Error("Model command reported an error");
@@ -473,6 +476,20 @@ export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, proc
   const forbidden = completed.filter((item) => !["agent_message", "reasoning", "mcp_tool_call"].includes(item?.type));
   if (forbidden.length) throw forbiddenItemError(forbidden);
   const toolItems = completed.filter(({ type }) => type === "mcp_tool_call");
+  const terminalAnswerIndex = completed.findLastIndex(({ type }) => type === "agent_message");
+  const codexApplicationError = (item) => item.status === "failed"
+    && item.server === mcpServerName && advertised.has(item.tool)
+    && item.arguments && typeof item.arguments === "object" && !Array.isArray(item.arguments)
+    && item.result && typeof item.result === "object" && !Array.isArray(item.result)
+    && Object.keys(item.result).sort().join(",") === "content,structured_content"
+    && item.result.structured_content === null
+    && Array.isArray(item.result.content) && item.result.content.length > 0
+    && item.result.content.every((block) => block && block.type === "text"
+      && typeof block.text === "string" && block.text.length > 0)
+    && completed.indexOf(item) < terminalAnswerIndex;
+  if (toolItems.some((item) => item.status === "failed" && !codexApplicationError(item))) {
+    throw new Error("Model command reported a failed MCP tool call");
+  }
   const resourceItems = resourceCatalogue ? toolItems.flatMap((item) => {
     const operation = resourceHelperOperation(item, resourceCatalogue);
     return operation ? [{ item, operation }] : [];
@@ -489,8 +506,10 @@ export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, proc
       || !item.arguments || typeof item.arguments !== "object" || Array.isArray(item.arguments)) {
       throw forbiddenItemError([item]);
     }
-    if (!item.result || typeof item.result !== "object" || item.status === "failed"
-      || item.result.isError === true || item.result.is_error === true) {
+    const applicationError = codexApplicationError(item)
+      || item.result?.isError === true || item.result?.is_error === true;
+    if (!item.result || typeof item.result !== "object"
+      || (item.status !== "completed" && !(item.status === "failed" && applicationError))) {
       throw new Error("Model command reported a failed MCP tool call");
     }
     return { name: item.tool, arguments: item.arguments };
@@ -498,7 +517,11 @@ export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, proc
   const messages = completed.filter(({ type }) => type === "agent_message");
   const answer = messages.at(-1)?.text;
   if (typeof answer !== "string") throw new Error("Model command returned no answer");
-  const toolResults = applicationToolItems.map(({ result }) => ({ content: result.content ?? result, isError: false }));
+  const toolResults = applicationToolItems.map((item) => ({
+    content: item.result.content ?? item.result,
+    isError: item.result.isError === true || item.result.is_error === true
+      || codexApplicationError(item),
+  }));
   return {
     answer,
     calls,

package/semantic/test.d.mts

@@ -52,6 +52,9 @@ export interface ResourceOperation {
 export interface ConversationTurn {
   readonly resourceOperations: readonly (readonly ResourceOperation[])[];
   readonly responses: readonly string[];
+  /** Ordered native Claude visible messages by trial. Other providers currently return empty arrays.
+   * May include the final message; responses remains the separate terminal result. Excludes hidden reasoning. */
+  readonly assistantMessages: readonly (readonly string[])[];
   readonly toolCalls: readonly (readonly ToolCall[])[];
   readonly toolOutcomes: readonly (readonly ToolOutcome[])[];
   readonly elicitations: readonly (readonly ElicitationEvidence[])[];

package/semantic/test.mjs

@@ -148,6 +148,7 @@ export async function createConversation(testContext, options) {
           toolCalls: recordedToolCalls,
           resourceOperations: answer.resourceOperations ?? [],
           assistant: answer.answer,
+          ...(answer.assistantMessages ? { assistantMessages: answer.assistantMessages } : {}),
         });
         const record = {
           turn: trial.record.turns.length + 1,
@@ -155,6 +156,10 @@ export async function createConversation(testContext, options) {
           interactionMode: "native-mcp",
           prompt,
           response: answer.answer,
+          ...(answer.assistantMessages ? {
+            assistantMessages: answer.assistantMessages,
+            assistantMessagesSha256: hash(JSON.stringify(answer.assistantMessages)),
+          } : {}),
           elicitations: answer.elicitations,
           toolCalls: recordedToolCalls,
           resourceOperations: answer.resourceOperations ?? [],
@@ -213,6 +218,7 @@ export async function createConversation(testContext, options) {
       }
       return Object.freeze({
         responses: Object.freeze(trials.map(({ answer }) => answer)),
+        assistantMessages: immutableSnapshot(trials.map(({ record }) => record.assistantMessages ?? [])),
         toolCalls: Object.freeze(trials.map(({ calls }) => Object.freeze(calls))),
         toolOutcomes: Object.freeze(trials.map(({ record }) => immutableSnapshot(record.toolCalls.map(
           ({ name, arguments: args, result, isError }) => ({ name, arguments: args, result, isError }),