@emseepea/testing 0.20.5 → 0.21.1

package/semantic/cli.mjs

@@ -5,7 +5,7 @@ import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promis
 import { tmpdir } from "node:os";
 import { dirname, join, relative, resolve } from "node:path";
 import { discoverTests } from "./discover.mjs";
-import { modelVersion } from "./provider.mjs";
+import { modelVersion, providerModel, providerSettings } from "./provider.mjs";
 
 const negativeFeedbackObservations = new Set([
   "error",
@@ -32,26 +32,29 @@ for (let i = 2; i < process.argv.length; i += 1) {
   else if (arg.startsWith("--")) throw new Error(`Unknown option: ${arg}`);
   else paths.push(arg);
 }
-if (!["claude-local", "claude-ci"].includes(provider)) throw new Error("Unsupported provider");
+if (!["claude-local", "claude-ci", "codex-local", "codex-ci"].includes(provider)) throw new Error("Unsupported provider");
 if ((smoke && provider === "claude-ci") || (modelCommand && !smoke)) throw new Error("Custom model commands are smoke-only");
 if (!paths.length) paths.push("eval");
 const files = await discoverTests(paths);
 const directory = await mkdtemp(join(tmpdir(), "emseepea-evidence-"));
 const evidence = { authoritative: provider === "claude-ci", provider, smoke,
-  model: "claude-sonnet-4-6", semanticRetries: 0, revision: process.env.GITHUB_SHA,
+  configuredModel: providerModel(provider), modelEvidence: provider.startsWith("codex-") ? "configured" : "observed",
+  settings: providerSettings(provider), semanticRetries: 0, revision: process.env.GITHUB_SHA,
   status: "failed", cases: {}, errors: [], startedAt: new Date().toISOString() };
 try {
   const client = JSON.parse(await readFile(new URL("../package.json", import.meta.resolve("@modelcontextprotocol/client")), "utf8"));
   if (client.name !== "@modelcontextprotocol/client" || client.version !== "2.0.0") throw new Error("Unexpected MCP client version");
-  evidence.dependencies = { mcpClient: client.version, claudeCli: smoke ? "simulated" : await modelVersion() };
+  const cliName = provider.startsWith("codex-") ? "codexCli" : "claudeCli";
+  evidence.dependencies = { mcpClient: client.version, [cliName]: smoke ? "simulated" : await modelVersion(provider) };
   if (provider === "claude-ci" && evidence.dependencies.claudeCli !== "2.1.248") throw new Error("Unexpected Claude CLI version");
   let interrupted = false;
   for (const file of files) {
     const displayFile = relative(process.cwd(), file);
     const code = await new Promise((resolveCode) => {
       const environment = { ...process.env, EMSEEPEA_EVAL_PROVIDER: provider, EMSEEPEA_EVAL_SMOKE: smoke ? "1" : "0",
-        EMSEEPEA_EVIDENCE_DIR: directory, EMSEEPEA_TEST_FILE: displayFile,
-        EMSEEPEA_MODEL_COMMAND: modelCommand ? resolve(modelCommand) : "claude" };
+        EMSEEPEA_EVIDENCE_DIR: directory, EMSEEPEA_TEST_FILE: displayFile };
+      if (modelCommand) environment.EMSEEPEA_MODEL_COMMAND = resolve(modelCommand);
+      else delete environment.EMSEEPEA_MODEL_COMMAND;
       delete environment.NODE_TEST_CONTEXT;
       const child = spawn(process.execPath, ["--test", "--test-concurrency=1", file], {
         stdio: "inherit",
@@ -115,6 +118,9 @@ console.log(`Semantic checks ${evidence.status}; evidence: ${output}`);
 function validRecord(record, authoritative, smoke) {
   const isHash = (value) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
   if (record.status !== "passed" || record.authoritative !== authoritative || record.smoke !== smoke
+    || record.provider !== evidence.provider || record.configuredModel !== evidence.configuredModel
+    || record.modelEvidence !== evidence.modelEvidence
+    || JSON.stringify(record.settings) !== JSON.stringify(evidence.settings)
     || record.mode !== "conversation" || record.answerTrials?.length !== 3
     || !Number.isInteger(record.judgeVerdicts?.length) || record.judgeVerdicts.length < 9
     || record.judgeVerdicts.length % 3 !== 0

package/semantic/provider.mjs

@@ -1,9 +1,10 @@
 import { spawn } from "node:child_process";
 import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { scriptedElicitations } from "./elicitation.mjs";
 
-const model = "claude-sonnet-4-6";
+const claudeModel = "claude-sonnet-4-6";
 const mcpServerName = "emseepea_eval";
 const hash = (value) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
 const providerErrorMessages = new Map([
@@ -19,13 +20,39 @@ const hasAuthenticationFailure = (events) => events.some((event) => (
     && typeof event.result === "string" && authenticationFailure.test(event.result))
 ));
 
-export async function modelVersion() {
-  const result = await runProcess("claude", ["--version"], { env: modelEnvironment({}) });
-  const version = result.stdout.trim().match(/^(\d+\.\d+\.\d+)\b/)?.[1];
-  if (result.code !== 0 || !version) throw new Error("Could not verify Claude CLI version");
+export async function modelVersion(provider = "claude-local") {
+  const codex = provider.startsWith("codex-");
+  const command = codex
+    ? process.env.EMSEEPEA_CODEX_COMMAND ?? process.env.EMSEEPEA_MODEL_COMMAND ?? "codex"
+    : process.env.EMSEEPEA_MODEL_COMMAND ?? "claude";
+  const result = await runProcess(command, ["--version"], { env: modelEnvironment({}) });
+  const version = result.stdout.trim().match(/(?:codex-cli\s+)?(\d+\.\d+\.\d+)\b/)?.[1];
+  if (result.code !== 0 || !version) throw new Error(`Could not verify ${codex ? "Codex" : "Claude"} CLI version`);
   return version;
 }
 
+export function providerModel(provider) {
+  if (!provider.startsWith("codex-")) return claudeModel;
+  const selected = process.env.EMSEEPEA_CODEX_MODEL?.trim();
+  if (!selected) throw new Error("Codex model is not configured");
+  return selected;
+}
+
+export function providerSettings(provider) {
+  return provider.startsWith("codex-")
+    ? {
+        approvalPolicy: "never",
+        loginShell: false,
+        multiAgent: false,
+        sandbox: "read-only",
+        shell: "forbidden",
+        shellEnvironment: "empty",
+        userConfig: "ignored",
+        webSearch: false,
+      }
+    : { effort: "low", maxTurns: 4, permissionMode: "dontAsk", userSettings: false };
+}
+
 export function parseClaudeEvents(stdout, processExitCode = 0) {
   let events;
   try {
@@ -57,8 +84,8 @@ export function parseClaudeEvents(stdout, processExitCode = 0) {
   if (!Number.isInteger(result.num_turns)) throw new Error("Model command returned an invalid turn count");
   if (result.num_turns !== expectedTurns) throw new Error("Model command used an unexpected number of turns");
   if ((result.permission_denials?.length ?? 0) > 0) throw new Error("Model command attempted a forbidden action");
-  const usage = result.modelUsage?.[model];
-  if (usage?.canonicalModel !== model || usage.provider !== "firstParty") {
+  const usage = result.modelUsage?.[claudeModel];
+  if (usage?.canonicalModel !== claudeModel || usage.provider !== "firstParty") {
     throw new Error("Model command did not use the required model");
   }
   return {
@@ -133,8 +160,8 @@ export function parseNativeClaudeEvents(stdout, advertisedTools, requireInit = f
   if (result.num_turns !== toolUses.length + 1) {
     throw new Error("Model command used an unexpected number of turns");
   }
-  const usage = result.modelUsage?.[model];
-  if (usage?.canonicalModel !== model || usage.provider !== "firstParty") {
+  const usage = result.modelUsage?.[claudeModel];
+  if (usage?.canonicalModel !== claudeModel || usage.provider !== "firstParty") {
     throw new Error("Model command did not use the required model");
   }
   return {
@@ -167,6 +194,7 @@ export function parseJudgeVerdict(output) {
 }
 
 export function modelInvocation(provider, prompt, directory) {
+  if (provider.startsWith("codex-")) return codexInvocation(provider, prompt, directory);
   const token = provider === "claude-ci" ? process.env.CLAUDE_CODE_OAUTH_TOKEN?.trim() : undefined;
   if (provider === "claude-ci" && !token) throw new Error("Claude subscription authentication is unavailable");
   const localHome = provider === "claude-local" ? process.env.HOME : undefined;
@@ -177,7 +205,7 @@ export function modelInvocation(provider, prompt, directory) {
     command: process.env.EMSEEPEA_MODEL_COMMAND ?? "claude",
     args: [
       "--print", prompt,
-      "--model", model,
+      "--model", claudeModel,
       "--effort", "low",
       "--max-turns", "4",
       "--safe-mode",
@@ -200,6 +228,9 @@ export function modelInvocation(provider, prompt, directory) {
 }
 
 export function conversationInvocation(provider, directory, url, tools, authToken, context) {
+  if (provider.startsWith("codex-")) {
+    return codexInvocation(provider, undefined, directory, { url, tools, authToken, context });
+  }
   const nativeTools = tools.map(({ name }) => nativeToolName(name));
   const config = nativeTools.length ? {
     mcpServers: {
@@ -218,7 +249,7 @@ export function conversationInvocation(provider, directory, url, tools, authToke
       "--input-format", "stream-json",
       "--output-format", "stream-json",
       "--verbose",
-      "--model", model,
+      "--model", claudeModel,
       "--effort", "low",
       "--max-turns", "4",
       "--strict-mcp-config",
@@ -238,6 +269,9 @@ export function conversationInvocation(provider, directory, url, tools, authToke
 }
 
 export function startModelConversation(provider, directory, url, tools, authToken, context, signal, serverSecrets = []) {
+  if (provider.startsWith("codex-")) {
+    return startCodexConversation(provider, directory, url, tools, authToken, context, signal, serverSecrets);
+  }
   signal?.throwIfAborted();
   const invocation = conversationInvocation(provider, directory, url, tools, authToken, context);
   const child = spawn(invocation.command, invocation.args, {
@@ -362,7 +396,170 @@ export async function runModel(provider, prompt, directory, signal) {
   if (execution.errorCode === "ABORT_ERR") throw new Error("Model command was cancelled");
   if (execution.errorCode) throw new Error("Model command could not start");
   if (execution.code !== 0 && !execution.stdout) throw new Error(`Model command exited ${execution.code}`);
-  return parseClaudeEvents(execution.stdout, execution.code);
+  return provider.startsWith("codex-")
+    ? parseCodexEvents(execution.stdout, [], undefined, execution.code,
+      providerModel(provider), invocation.providerSecrets)
+    : parseClaudeEvents(execution.stdout, execution.code);
+}
+
+export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, processExitCode = 0, selectedModel = "test-model", secrets = []) {
+  let events;
+  try {
+    events = typeof stdout === "string"
+      ? stdout.split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line))
+      : stdout;
+  } catch {
+    throw new Error("Model command returned invalid event data");
+  }
+  if (!Array.isArray(events)) throw new Error("Model command returned invalid event data");
+  const serialized = JSON.stringify(events);
+  if (secrets.some((secret) => typeof secret === "string" && secret.length > 0 && serialized.includes(secret))) {
+    throw new Error("Model command exposed a configured secret");
+  }
+  const authentication = /(?:not logged in|authentication|unauthorized|invalid api key|oauth)/i;
+  const errorText = events.flatMap((event) => [event.message, event.error?.message, event.item?.message])
+    .filter((value) => typeof value === "string").join("\n");
+  if (authentication.test(errorText)) throw new Error("Model command is not signed in");
+  const failedTool = events.some(({ item }) => item?.type === "mcp_tool_call"
+    && (item.status === "failed" || item.result?.isError === true || item.result?.is_error === true));
+  if (failedTool) throw new Error("Model command reported a failed MCP tool call");
+  if (processExitCode !== 0) throw new Error(`Model command exited ${processExitCode}`);
+  if (events.some(({ type }) => type === "turn.failed" || type === "error")) {
+    throw new Error("Model command reported an error");
+  }
+  const started = events.filter(({ type }) => type === "thread.started");
+  const threadId = started[0]?.thread_id;
+  if (started.length !== 1 || typeof threadId !== "string" || !threadId) {
+    throw new Error("Model command omitted its session event");
+  }
+  if (expectedThreadId && threadId !== expectedThreadId) {
+    throw new Error("Model command did not resume the required session");
+  }
+  if (events.filter(({ type }) => type === "turn.started").length !== 1
+    || events.filter(({ type }) => type === "turn.completed").length !== 1) {
+    throw new Error("Model command returned an incomplete turn");
+  }
+  const advertised = new Set(advertisedTools.map(({ name }) => name));
+  const completed = events.filter(({ type }) => type === "item.completed").map(({ item }) => item);
+  const forbidden = completed.filter((item) => !["agent_message", "reasoning", "mcp_tool_call"].includes(item?.type));
+  if (forbidden.length) throw new Error("Model command used a forbidden tool");
+  const toolItems = completed.filter(({ type }) => type === "mcp_tool_call");
+  if (toolItems.length > 3) throw new Error("Model command used more than three tools");
+  const calls = toolItems.map((item) => {
+    if (item.server !== mcpServerName || !advertised.has(item.tool)
+      || !item.arguments || typeof item.arguments !== "object" || Array.isArray(item.arguments)) {
+      throw new Error("Model command used a forbidden tool");
+    }
+    if (!item.result || typeof item.result !== "object" || item.status === "failed"
+      || item.result.isError === true || item.result.is_error === true) {
+      throw new Error("Model command reported a failed MCP tool call");
+    }
+    return { name: item.tool, arguments: item.arguments };
+  });
+  const messages = completed.filter(({ type }) => type === "agent_message");
+  const answer = messages.at(-1)?.text;
+  if (typeof answer !== "string") throw new Error("Model command returned no answer");
+  const toolResults = toolItems.map(({ result }) => ({ content: result.content ?? result, isError: false }));
+  return {
+    answer,
+    calls,
+    toolResults,
+    pathEvidence: calls.map((call, index) => ({
+      method: "tools/call",
+      target: call.name,
+      requestSha256: hash({ method: "tools/call", name: call.name, arguments: call.arguments }),
+      responseSha256: hash(toolResults[index].content),
+    })),
+    models: [selectedModel],
+    modelEvidence: "configured",
+    turnCount: 1,
+    providerTurnCount: calls.length + 1,
+    providerToolCount: calls.length,
+    threadId,
+  };
+}
+
+function startCodexConversation(provider, directory, url, tools, authToken, context, signal, serverSecrets) {
+  let threadId;
+  let closed = false;
+  return Object.freeze({
+    async send(prompt, options) {
+      if (closed) throw new Error("Model conversation is closed");
+      if (options?.elicitations?.length) throw new Error("Codex provider does not support scripted elicitation");
+      const invocation = conversationInvocation(provider, directory, url, tools, authToken, context);
+      const args = threadId
+        ? ["exec", "resume", threadId,
+          ...withoutOption(withoutOption(invocation.args.slice(1), "--cd"), "--sandbox")]
+        : invocation.args;
+      args.push(prompt);
+      const execution = await runProcess(invocation.command, args, {
+        cwd: invocation.cwd, env: invocation.env, signal, killSignal: "SIGKILL",
+      });
+      if (execution.timedOut) throw new Error("Model command timed out");
+      if (execution.outputLimitExceeded) throw new Error("Model command output exceeded its limit");
+      if (execution.errorCode === "ABORT_ERR") throw new Error("Model command was cancelled");
+      if (execution.errorCode) throw new Error("Model conversation could not start");
+      const turn = parseCodexEvents(execution.stdout, tools, threadId, execution.code,
+        providerModel(provider), [...invocation.providerSecrets, authToken, ...serverSecrets]);
+      threadId = turn.threadId;
+      return { ...turn, elicitations: [] };
+    },
+    async close() { closed = true; },
+  });
+}
+
+function codexInvocation(provider, prompt, directory, conversation) {
+  const model = providerModel(provider);
+  const sourceHome = process.env.CODEX_HOME ?? (process.env.HOME ? join(process.env.HOME, ".codex") : undefined);
+  const sourceAuth = sourceHome ? join(sourceHome, "auth.json") : undefined;
+  const apiKey = process.env.OPENAI_API_KEY?.trim();
+  if (provider === "codex-ci" && !apiKey) throw new Error("Codex CI authentication is unavailable");
+  if (provider === "codex-local" && !apiKey && (!sourceAuth || !existsSync(sourceAuth))) {
+    throw new Error("Codex authentication is unavailable");
+  }
+  const codexHome = join(directory, "codex-home");
+  mkdirSync(codexHome, { recursive: true, mode: 0o700 });
+  const rules = join(codexHome, "rules");
+  mkdirSync(rules, { recursive: true, mode: 0o700 });
+  writeFileSync(join(rules, "default.rules"), `prefix_rule(
+    pattern = [["bash", "zsh", "sh", "/bin/bash", "/bin/zsh", "/bin/sh", "/usr/bin/bash", "/usr/bin/zsh", "/usr/bin/sh"], ["-c", "-lc"]],
+    decision = "forbidden",
+    justification = "Only the configured MCP tools are allowed.",
+)\n`, { mode: 0o600 });
+  const isolatedAuth = join(codexHome, "auth.json");
+  if (!apiKey && sourceAuth && !existsSync(isolatedAuth)) symlinkSync(sourceAuth, isolatedAuth);
+  const authSecrets = !apiKey && sourceAuth ? credentialValues(sourceAuth) : [];
+  const args = [
+    "exec", "--json", "--ignore-user-config", "--skip-git-repo-check", "--sandbox", "read-only",
+    "--model", model, "--cd", directory,
+    "-c", 'approval_policy="never"',
+    "-c", 'web_search="disabled"',
+    "-c", 'shell_environment_policy.inherit="none"',
+    "-c", "allow_login_shell=false",
+    "-c", "features.multi_agent=false",
+    "-c", "features.unified_exec=false",
+  ];
+  if (conversation?.context) args.push("-c", `developer_instructions=${JSON.stringify(conversation.context)}`);
+  if (conversation?.tools.length) {
+    args.push("-c", `mcp_servers.${mcpServerName}.url=${JSON.stringify(conversation.url)}`);
+    args.push("-c", `mcp_servers.${mcpServerName}.enabled_tools=${JSON.stringify(conversation.tools.map(({ name }) => name))}`);
+    args.push("-c", `mcp_servers.${mcpServerName}.default_tools_approval_mode="approve"`);
+    for (const { name } of conversation.tools) {
+      args.push("-c", `mcp_servers.${mcpServerName}.tools.${JSON.stringify(name)}.approval_mode="approve"`);
+    }
+    if (conversation.authToken) {
+      args.push("-c", `mcp_servers.${mcpServerName}.bearer_token_env_var="EMSEEPEA_SEMANTIC_MCP_TOKEN"`);
+    }
+  }
+  if (prompt !== undefined) args.push(prompt);
+  return {
+    command: process.env.EMSEEPEA_CODEX_COMMAND ?? process.env.EMSEEPEA_MODEL_COMMAND ?? "codex",
+    args,
+    cwd: directory,
+    providerSecrets: [apiKey, ...authSecrets].filter(Boolean),
+    env: modelEnvironment({ CODEX_HOME: codexHome, ...(apiKey ? { OPENAI_API_KEY: apiKey } : {}),
+      ...(conversation?.authToken ? { EMSEEPEA_SEMANTIC_MCP_TOKEN: conversation.authToken } : {}) }),
+  };
 }
 
 function runProcess(command, args, options) {
@@ -410,3 +607,24 @@ function modelEnvironment(extra) {
 function nativeToolName(name) {
   return `mcp__${mcpServerName}__${name}`;
 }
+
+function withoutOption(args, name) {
+  const index = args.indexOf(name);
+  return index < 0 ? [...args] : [...args.slice(0, index), ...args.slice(index + 2)];
+}
+
+function credentialValues(path) {
+  try {
+    const values = [];
+    const visit = (value, key = "") => {
+      if (typeof value === "string" && /(?:token|secret|key|credential)/i.test(key)) values.push(value);
+      else if (value && typeof value === "object") {
+        for (const [childKey, child] of Object.entries(value)) visit(child, childKey);
+      }
+    };
+    visit(JSON.parse(readFileSync(path, "utf8")));
+    return values;
+  } catch {
+    return [];
+  }
+}

package/semantic/test.d.mts

@@ -38,6 +38,8 @@ export interface ElicitationEvidence {
 export interface SemanticConversation {
   /** Sends this exact user message through the same provider-native MCP conversation. */
   send(prompt: string, options?: { elicitations: readonly ScriptedElicitation[] }): Promise<ConversationTurn>;
+  /** Starts a provider-native conversation with no transcript history against the same running test application. */
+  fresh(): Promise<void>;
 }
 
 export function createConversation(

package/semantic/test.mjs

@@ -10,7 +10,7 @@ import {
   startSemanticServer,
   stopSemanticServer,
 } from "./material.mjs";
-import { parseJudgeVerdict, runModel, startModelConversation } from "./provider.mjs";
+import { parseJudgeVerdict, providerModel, providerSettings, runModel, startModelConversation } from "./provider.mjs";
 
 const hash = (value) => createHash("sha256").update(value).digest("hex");
 const names = new Set();
@@ -28,7 +28,7 @@ export async function createConversation(testContext, options) {
   names.add(key);
 
   const provider = process.env.EMSEEPEA_EVAL_PROVIDER ?? "claude-local";
-  if (!["claude-local", "claude-ci"].includes(provider)) throw new Error("Unsupported model provider");
+  if (!["claude-local", "claude-ci", "codex-local", "codex-ci"].includes(provider)) throw new Error("Unsupported model provider");
   const smoke = process.env.EMSEEPEA_EVAL_SMOKE === "1";
   if (smoke && provider === "claude-ci") throw new Error("Smoke tests cannot qualify a release");
   const file = process.env.EMSEEPEA_TEST_FILE;
@@ -43,7 +43,9 @@ export async function createConversation(testContext, options) {
     authoritative: provider === "claude-ci",
     smoke,
     provider,
-    model: "claude-sonnet-4-6",
+    configuredModel: providerModel(provider),
+    modelEvidence: provider.startsWith("codex-") ? "configured" : "observed",
+    settings: providerSettings(provider),
     semanticRetries: 0,
     status: "failed",
     caseSha256: hash(JSON.stringify({
@@ -72,7 +74,7 @@ export async function createConversation(testContext, options) {
         const serverSecrets = Object.entries(environment ?? {})
           .filter(([key]) => /token|password|secret|credential|api.?key/i.test(key))
           .map(([, value]) => value);
-        state.trials.push({ running, tools, record, directory, history: [], model: undefined, serverSecrets });
+        state.trials.push({ running, tools, record, directory, history: [], model: undefined, serverSecrets, conversation: 1 });
         evidence.answerTrials.push(record);
       } catch (error) {
         await stopSemanticServer(running.child);
@@ -113,6 +115,7 @@ export async function createConversation(testContext, options) {
         });
         const record = {
           turn: trial.record.turns.length + 1,
+          conversation: trial.conversation,
           interactionMode: "native-mcp",
           prompt,
           response: answer.answer,
@@ -125,6 +128,7 @@ export async function createConversation(testContext, options) {
           promptSha256: hash(prompt),
           answerSha256: hash(answer.answer),
           answerModels: answer.models,
+          answerModelEvidence: answer.modelEvidence ?? "observed",
           answerTurnCount: answer.turnCount,
           answerProviderTurnCount: answer.providerTurnCount,
           answerProviderToolCount: answer.providerToolCount,
@@ -176,6 +180,15 @@ export async function createConversation(testContext, options) {
         [privateTurn]: trials,
       });
     },
+    async fresh() {
+      ensureOpen(state);
+      await Promise.all(state.trials.map(async (trial) => {
+        await trial.model?.close();
+        trial.model = undefined;
+        trial.history = [];
+        trial.conversation += 1;
+      }));
+    },
   });
 }
 
@@ -428,6 +441,7 @@ async function judgeTrialMeaning(trial, trialIndex, expected) {
       const response = await isolatedModel(trial.provider, request, "emseepea-judge-", trial.signal);
       Object.assign(record, {
         models: response.models,
+        modelEvidence: response.modelEvidence ?? "observed",
         turnCount: response.turnCount,
         providerTurnCount: response.providerTurnCount,
         providerToolCount: response.providerToolCount,
@@ -453,6 +467,10 @@ function safeModelFailure(error) {
   const message = error instanceof Error ? error.message : "";
   const safeMessages = new Set([
     "Claude subscription authentication is unavailable",
+    "Codex authentication is unavailable",
+    "Codex CI authentication is unavailable",
+    "Codex model is not configured",
+    "Codex provider does not support scripted elicitation",
     "Model command attempted a forbidden action",
     "Model command could not start",
     "Model command exceeded its budget",
@@ -460,6 +478,7 @@ function safeModelFailure(error) {
     "Model command failed during execution",
     "Model command is not signed in",
     "Model command omitted its result event",
+    "Model command omitted its session event",
     "Model command output exceeded its limit",
     "Model command reported an error",
     "Model command returned a non-text answer",
@@ -473,6 +492,10 @@ function safeModelFailure(error) {
     "Model command omitted MCP initialization evidence",
     "Model command omitted an MCP tool result",
     "Model command returned no answer",
+    "Model command returned an incomplete turn",
+    "Model command did not resume the required session",
+    "Model command reported a failed MCP tool call",
+    "Model command exposed a configured secret",
     "Model command used more than three tools",
     "Model conversation already has a pending turn",
     "Model conversation could not start",