@emseepea/testing 0.26.14 → 0.26.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@emseepea/testing",
-  "version": "0.26.14",
+  "version": "0.26.16",
   "description": "MCP integration and semantic testing helpers",
   "license": "MIT",
   "repository": {
@@ -57,7 +57,7 @@
     "test:llm:built": "emseepea-test eval"
   },
   "dependencies": {
-    "@emseepea/server": "0.25.10",
+    "@emseepea/server": "0.26.1",
     "@modelcontextprotocol/client": "2.3.1",
     "@modelcontextprotocol/server": "2.3.1",
     "@modelcontextprotocol/core": "2.3.1"

package/semantic/provider.mjs

@@ -25,15 +25,28 @@ const providerErrorMessages = new Map([
   ["error_max_turns", "Model command exceeded its turn limit"],
 ]);
 const authenticationFailure = /not logged in|failed to authenticate|oauth session expired/i;
-const codexFailureCategories = new Set([
+const providerFailureCategories = new Set([
   "authentication",
+  "billing",
   "configuration",
   "host-storage",
   "permission-or-sandbox",
   "provider",
+  "rate-limit",
   "transport",
   "unknown",
 ]);
+const claudeAssistantFailureCategories = new Map([
+  ["authentication_failed", "authentication"],
+  ["oauth_org_not_allowed", "authentication"],
+  ["account_on_hold", "provider"],
+  ["billing_error", "billing"],
+  ["rate_limit", "rate-limit"],
+  ["overloaded", "provider"],
+  ["invalid_request", "configuration"],
+  ["model_not_found", "provider"],
+  ["server_error", "provider"],
+]);
 const stderrLimit = 32 * 1024;
 const forbiddenItemLimit = 64;
 const hasAuthenticationFailure = (events) => events.some((event) => (
@@ -81,7 +94,7 @@ export function providerSettings(provider) {
     : { effort: "low", maxTurns: null, judgeMaxTurns: 4, permissionMode: "dontAsk", userSettings: false };
 }
 
-export function parseClaudeEvents(stdout, processExitCode = 0) {
+export function parseClaudeEvents(stdout, processExitCode = 0, failureCategory) {
   let events;
   try {
     events = stdout.split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line));
@@ -93,11 +106,15 @@ export function parseClaudeEvents(stdout, processExitCode = 0) {
   const toolUses = events.flatMap(({ message }) => (
     Array.isArray(message?.content) ? message.content.filter(({ type }) => type === "tool_use") : []
   ));
-  if (hasAuthenticationFailure(events)) throw new Error("Model command is not signed in");
-  if (processExitCode !== 0) throw new Error(`Model command exited ${processExitCode}`);
+  if (hasAuthenticationFailure(events)) {
+    throw providerEvidenceError("Model command is not signed in", "authentication");
+  }
+  if (processExitCode !== 0) {
+    throw providerProcessError(processExitCode, claudeFailureCategory(events, result, failureCategory));
+  }
   if (!result) throw new Error("Model command omitted its result event");
   if (result.is_error) {
-    throw new Error(providerErrorMessages.get(result.subtype) ?? "Model command reported an error");
+    throw claudeResultError(events, result);
   }
   if (typeof answer !== "string") throw new Error("Model command returned a non-text answer");
   if (toolUses.length > 0) {
@@ -172,10 +189,12 @@ export function parseNativeClaudeEvents(stdout, advertisedTools, requireInit = f
       responseSha256: hash(response.content),
     };
   });
-  if (hasAuthenticationFailure(events)) throw new Error("Model command is not signed in");
+  if (hasAuthenticationFailure(events)) {
+    throw providerEvidenceError("Model command is not signed in", "authentication");
+  }
   if (!result) throw new Error("Model command omitted its result event");
   if (result.is_error) {
-    throw new Error(providerErrorMessages.get(result.subtype) ?? "Model command reported an error");
+    throw claudeResultError(events, result);
   }
   if (typeof result.result !== "string") throw new Error("Model command returned a non-text answer");
   if ((result.permission_denials?.length ?? 0) > 0) {
@@ -425,12 +444,12 @@ export async function runModel(provider, prompt, directory, signal) {
   if (execution.errorCode === "ABORT_ERR") throw new Error("Model command was cancelled");
   if (execution.errorCode) throw new Error("Model command could not start");
   if (execution.code !== 0 && !execution.stdout) {
-    throw codexProcessError(execution.code, execution.providerFailureCategory);
+    throw providerProcessError(execution.code, execution.providerFailureCategory);
   }
   return provider.startsWith("codex-")
     ? parseCodexEvents(execution.stdout, [], undefined, execution.code,
       providerModel(provider), invocation.providerSecrets, execution.providerFailureCategory)
-    : parseClaudeEvents(execution.stdout, execution.code);
+    : parseClaudeEvents(execution.stdout, execution.code, execution.providerFailureCategory);
 }
 
 export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, processExitCode = 0,
@@ -452,12 +471,9 @@ export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, proc
   const errorText = events.flatMap((event) => [event.message, event.error?.message, event.item?.message])
     .filter((value) => typeof value === "string").join("\n");
   if (authentication.test(errorText)) {
-    throw codexEvidenceError("Model command is not signed in", "authentication");
+    throw providerEvidenceError("Model command is not signed in", "authentication");
   }
-  const failedTool = events.some(({ item }) => item?.type === "mcp_tool_call"
-    && (item.status === "failed" || item.result?.isError === true || item.result?.is_error === true));
-  if (failedTool) throw new Error("Model command reported a failed MCP tool call");
-  if (processExitCode !== 0) throw codexProcessError(processExitCode, providerFailureCategory);
+  if (processExitCode !== 0) throw providerProcessError(processExitCode, providerFailureCategory);
   if (events.some(({ type }) => type === "turn.failed" || type === "error")) {
     throw new Error("Model command reported an error");
   }
@@ -479,6 +495,20 @@ export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, proc
   const forbidden = completed.filter((item) => !["agent_message", "reasoning", "mcp_tool_call"].includes(item?.type));
   if (forbidden.length) throw forbiddenItemError(forbidden);
   const toolItems = completed.filter(({ type }) => type === "mcp_tool_call");
+  const terminalAnswerIndex = completed.findLastIndex(({ type }) => type === "agent_message");
+  const codexApplicationError = (item) => item.status === "failed"
+    && item.server === mcpServerName && advertised.has(item.tool)
+    && item.arguments && typeof item.arguments === "object" && !Array.isArray(item.arguments)
+    && item.result && typeof item.result === "object" && !Array.isArray(item.result)
+    && Object.keys(item.result).sort().join(",") === "content,structured_content"
+    && item.result.structured_content === null
+    && Array.isArray(item.result.content) && item.result.content.length > 0
+    && item.result.content.every((block) => block && block.type === "text"
+      && typeof block.text === "string" && block.text.length > 0)
+    && completed.indexOf(item) < terminalAnswerIndex;
+  if (toolItems.some((item) => item.status === "failed" && !codexApplicationError(item))) {
+    throw new Error("Model command reported a failed MCP tool call");
+  }
   const resourceItems = resourceCatalogue ? toolItems.flatMap((item) => {
     const operation = resourceHelperOperation(item, resourceCatalogue);
     return operation ? [{ item, operation }] : [];
@@ -495,8 +525,10 @@ export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, proc
       || !item.arguments || typeof item.arguments !== "object" || Array.isArray(item.arguments)) {
       throw forbiddenItemError([item]);
     }
-    if (!item.result || typeof item.result !== "object" || item.status === "failed"
-      || item.result.isError === true || item.result.is_error === true) {
+    const applicationError = codexApplicationError(item)
+      || item.result?.isError === true || item.result?.is_error === true;
+    if (!item.result || typeof item.result !== "object"
+      || (item.status !== "completed" && !(item.status === "failed" && applicationError))) {
       throw new Error("Model command reported a failed MCP tool call");
     }
     return { name: item.tool, arguments: item.arguments };
@@ -504,7 +536,11 @@ export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, proc
   const messages = completed.filter(({ type }) => type === "agent_message");
   const answer = messages.at(-1)?.text;
   if (typeof answer !== "string") throw new Error("Model command returned no answer");
-  const toolResults = applicationToolItems.map(({ result }) => ({ content: result.content ?? result, isError: false }));
+  const toolResults = applicationToolItems.map((item) => ({
+    content: item.result.content ?? item.result,
+    isError: item.result.isError === true || item.result.is_error === true
+      || codexApplicationError(item),
+  }));
   return {
     answer,
     calls,
@@ -667,7 +703,7 @@ function runProcess(command, args, options) {
       if (settled) return;
       settled = true;
       clearTimeout(timer);
-      const providerFailureCategory = classifyCodexFailure(stderr.toString("utf8"));
+      const providerFailureCategory = classifyProviderFailure(stderr.toString("utf8"));
       stderr = Buffer.alloc(0);
       resolve({ ...result, providerFailureCategory });
     };
@@ -680,7 +716,7 @@ function runProcess(command, args, options) {
   });
 }
 
-function classifyCodexFailure(stderr) {
+function classifyProviderFailure(stderr) {
   if (/no space left|disk (?:is )?full|storage (?:capacity|quota)|\benospc\b/i.test(stderr)) {
     return "host-storage";
   }
@@ -696,20 +732,45 @@ function classifyCodexFailure(stderr) {
   if (/connection (?:refused|reset)|network|transport|timed? out|\btls\b|certificate|\bdns\b|\beconn/i.test(stderr)) {
     return "transport";
   }
-  if (/provider (?:error|failure|unavailable)|model (?:is )?(?:unavailable|overloaded)|rate limit|service unavailable/i.test(stderr)) {
+  if (/provider (?:error|failure|unavailable)|model (?:is )?(?:unavailable|overloaded)|rate limit|service unavailable|(?:hit|reached) your limit/i.test(stderr)) {
     return "provider";
   }
   return "unknown";
 }
 
-function codexEvidenceError(message, category) {
+function claudeFailureCategory(events, result, fallback) {
+  const end = result ? events.lastIndexOf(result) : events.length;
+  const assistantFailure = events.findLast((event, index) => (
+    index < end
+    && event.type === "assistant"
+    && (event.parent_tool_use_id ?? null) === null
+    && (event.agent_id ?? null) === null
+    && claudeAssistantFailureCategories.has(event.error)
+  ));
+  if (assistantFailure) return claudeAssistantFailureCategories.get(assistantFailure.error);
+  if (!result?.is_error) return fallback;
+  const errors = Array.isArray(result.errors) ? result.errors : [];
+  for (const detail of [result.result, ...errors]) {
+    if (typeof detail !== "string") continue;
+    const category = classifyProviderFailure(detail);
+    if (category !== "unknown") return category;
+  }
+  return fallback;
+}
+
+function claudeResultError(events, result) {
+  const message = providerErrorMessages.get(result.subtype) ?? "Model command reported an error";
+  return providerEvidenceError(message, claudeFailureCategory(events, result));
+}
+
+function providerEvidenceError(message, category) {
   return Object.assign(new Error(message), {
-    providerFailureCategory: codexFailureCategories.has(category) ? category : "unknown",
+    providerFailureCategory: providerFailureCategories.has(category) ? category : "unknown",
   });
 }
 
-function codexProcessError(code, category) {
-  return codexEvidenceError(`Model command exited ${code}`, category);
+function providerProcessError(code, category) {
+  return providerEvidenceError(`Model command exited ${code}`, category);
 }
 
 function forbiddenItemError(items) {

package/semantic/test.mjs

@@ -606,10 +606,12 @@ function recordModelFailure(record, error) {
   const category = error?.providerFailureCategory;
   if ([
     "authentication",
+    "billing",
     "configuration",
     "host-storage",
     "permission-or-sandbox",
     "provider",
+    "rate-limit",
     "transport",
     "unknown",
   ].includes(category)) record.providerFailureCategory = category;