@emseepea/testing 0.20.4 → 0.21.0

package/README.md

@@ -57,6 +57,29 @@ advertised-tool text, an answer wrapper, or prepared MCP material. Exact tool
 assertions come from the provider's native MCP events. Follow-up messages use
 the same conversation.
 
+Call `await conversation.fresh()` to discard provider transcript history while
+keeping the same running test application and its saved data. The next `send`
+starts a new provider-native session in each trial.
+
+The runner supports `claude-local`, `claude-ci`, `codex-local`, and `codex-ci`.
+Codex providers require an explicit `EMSEEPEA_CODEX_MODEL`. For example:
+
+```sh
+EMSEEPEA_CODEX_MODEL=gpt-5.5 emseepea-test --provider codex-local eval
+```
+
+`codex-local` uses the existing Codex CLI sign-in without copying it.
+`codex-ci` requires `OPENAI_API_KEY` from the CI secret store. Both providers
+run `codex exec --json`, resume the actual Codex session for follow-ups, ignore
+user configuration, and pre-approve only the discovered tools from the test
+server.
+
+An isolated command policy denies shell launches and spawned shell environments
+inherit no provider credentials.
+
+Codex CLI evidence proves that CLI journey. It does not prove native ChatGPT
+connection, attachment, retrieval, or MCP App rendering behavior.
+
 Optional `context` is application context, not test guidance. Use it only when
 the deployed application supplies the same context. Leaving it out is the best
 default for testing whether tool names, descriptions, and schemas stand on

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@emseepea/testing",
-  "version": "0.20.4",
+  "version": "0.21.0",
   "description": "MCP integration and semantic testing helpers",
   "license": "MIT",
   "repository": {
@@ -47,7 +47,7 @@
     "test:built": "node --test test/*.test.mjs"
   },
   "dependencies": {
-    "@emseepea/server": "0.22.0",
+    "@emseepea/server": "0.22.1",
     "@modelcontextprotocol/client": "2.0.0"
   },
   "devDependencies": {

package/semantic/cli.mjs

@@ -5,7 +5,7 @@ import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promis
 import { tmpdir } from "node:os";
 import { dirname, join, relative, resolve } from "node:path";
 import { discoverTests } from "./discover.mjs";
-import { modelVersion } from "./provider.mjs";
+import { modelVersion, providerModel, providerSettings } from "./provider.mjs";
 
 const negativeFeedbackObservations = new Set([
   "error",
@@ -32,26 +32,29 @@ for (let i = 2; i < process.argv.length; i += 1) {
   else if (arg.startsWith("--")) throw new Error(`Unknown option: ${arg}`);
   else paths.push(arg);
 }
-if (!["claude-local", "claude-ci"].includes(provider)) throw new Error("Unsupported provider");
+if (!["claude-local", "claude-ci", "codex-local", "codex-ci"].includes(provider)) throw new Error("Unsupported provider");
 if ((smoke && provider === "claude-ci") || (modelCommand && !smoke)) throw new Error("Custom model commands are smoke-only");
 if (!paths.length) paths.push("eval");
 const files = await discoverTests(paths);
 const directory = await mkdtemp(join(tmpdir(), "emseepea-evidence-"));
 const evidence = { authoritative: provider === "claude-ci", provider, smoke,
-  model: "claude-sonnet-4-6", semanticRetries: 0, revision: process.env.GITHUB_SHA,
+  configuredModel: providerModel(provider), modelEvidence: provider.startsWith("codex-") ? "configured" : "observed",
+  settings: providerSettings(provider), semanticRetries: 0, revision: process.env.GITHUB_SHA,
   status: "failed", cases: {}, errors: [], startedAt: new Date().toISOString() };
 try {
   const client = JSON.parse(await readFile(new URL("../package.json", import.meta.resolve("@modelcontextprotocol/client")), "utf8"));
   if (client.name !== "@modelcontextprotocol/client" || client.version !== "2.0.0") throw new Error("Unexpected MCP client version");
-  evidence.dependencies = { mcpClient: client.version, claudeCli: smoke ? "simulated" : await modelVersion() };
+  const cliName = provider.startsWith("codex-") ? "codexCli" : "claudeCli";
+  evidence.dependencies = { mcpClient: client.version, [cliName]: smoke ? "simulated" : await modelVersion(provider) };
   if (provider === "claude-ci" && evidence.dependencies.claudeCli !== "2.1.248") throw new Error("Unexpected Claude CLI version");
   let interrupted = false;
   for (const file of files) {
     const displayFile = relative(process.cwd(), file);
     const code = await new Promise((resolveCode) => {
       const environment = { ...process.env, EMSEEPEA_EVAL_PROVIDER: provider, EMSEEPEA_EVAL_SMOKE: smoke ? "1" : "0",
-        EMSEEPEA_EVIDENCE_DIR: directory, EMSEEPEA_TEST_FILE: displayFile,
-        EMSEEPEA_MODEL_COMMAND: modelCommand ? resolve(modelCommand) : "claude" };
+        EMSEEPEA_EVIDENCE_DIR: directory, EMSEEPEA_TEST_FILE: displayFile };
+      if (modelCommand) environment.EMSEEPEA_MODEL_COMMAND = resolve(modelCommand);
+      else delete environment.EMSEEPEA_MODEL_COMMAND;
       delete environment.NODE_TEST_CONTEXT;
       const child = spawn(process.execPath, ["--test", "--test-concurrency=1", file], {
         stdio: "inherit",
@@ -115,6 +118,9 @@ console.log(`Semantic checks ${evidence.status}; evidence: ${output}`);
 function validRecord(record, authoritative, smoke) {
   const isHash = (value) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
   if (record.status !== "passed" || record.authoritative !== authoritative || record.smoke !== smoke
+    || record.provider !== evidence.provider || record.configuredModel !== evidence.configuredModel
+    || record.modelEvidence !== evidence.modelEvidence
+    || JSON.stringify(record.settings) !== JSON.stringify(evidence.settings)
     || record.mode !== "conversation" || record.answerTrials?.length !== 3
     || !Number.isInteger(record.judgeVerdicts?.length) || record.judgeVerdicts.length < 9
     || record.judgeVerdicts.length % 3 !== 0

package/semantic/provider.mjs

@@ -1,9 +1,10 @@
 import { spawn } from "node:child_process";
 import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { scriptedElicitations } from "./elicitation.mjs";
 
-const model = "claude-sonnet-4-6";
+const claudeModel = "claude-sonnet-4-6";
 const mcpServerName = "emseepea_eval";
 const hash = (value) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
 const providerErrorMessages = new Map([
@@ -19,13 +20,39 @@ const hasAuthenticationFailure = (events) => events.some((event) => (
     && typeof event.result === "string" && authenticationFailure.test(event.result))
 ));
 
-export async function modelVersion() {
-  const result = await runProcess("claude", ["--version"], { env: modelEnvironment({}) });
-  const version = result.stdout.trim().match(/^(\d+\.\d+\.\d+)\b/)?.[1];
-  if (result.code !== 0 || !version) throw new Error("Could not verify Claude CLI version");
+export async function modelVersion(provider = "claude-local") {
+  const codex = provider.startsWith("codex-");
+  const command = codex
+    ? process.env.EMSEEPEA_CODEX_COMMAND ?? process.env.EMSEEPEA_MODEL_COMMAND ?? "codex"
+    : process.env.EMSEEPEA_MODEL_COMMAND ?? "claude";
+  const result = await runProcess(command, ["--version"], { env: modelEnvironment({}) });
+  const version = result.stdout.trim().match(/(?:codex-cli\s+)?(\d+\.\d+\.\d+)\b/)?.[1];
+  if (result.code !== 0 || !version) throw new Error(`Could not verify ${codex ? "Codex" : "Claude"} CLI version`);
   return version;
 }
 
+export function providerModel(provider) {
+  if (!provider.startsWith("codex-")) return claudeModel;
+  const selected = process.env.EMSEEPEA_CODEX_MODEL?.trim();
+  if (!selected) throw new Error("Codex model is not configured");
+  return selected;
+}
+
+export function providerSettings(provider) {
+  return provider.startsWith("codex-")
+    ? {
+        approvalPolicy: "never",
+        loginShell: false,
+        multiAgent: false,
+        sandbox: "read-only",
+        shell: "forbidden",
+        shellEnvironment: "empty",
+        userConfig: "ignored",
+        webSearch: false,
+      }
+    : { effort: "low", maxTurns: 4, permissionMode: "dontAsk", userSettings: false };
+}
+
 export function parseClaudeEvents(stdout, processExitCode = 0) {
   let events;
   try {
@@ -57,8 +84,8 @@ export function parseClaudeEvents(stdout, processExitCode = 0) {
   if (!Number.isInteger(result.num_turns)) throw new Error("Model command returned an invalid turn count");
   if (result.num_turns !== expectedTurns) throw new Error("Model command used an unexpected number of turns");
   if ((result.permission_denials?.length ?? 0) > 0) throw new Error("Model command attempted a forbidden action");
-  const usage = result.modelUsage?.[model];
-  if (usage?.canonicalModel !== model || usage.provider !== "firstParty") {
+  const usage = result.modelUsage?.[claudeModel];
+  if (usage?.canonicalModel !== claudeModel || usage.provider !== "firstParty") {
     throw new Error("Model command did not use the required model");
   }
   return {
@@ -133,8 +160,8 @@ export function parseNativeClaudeEvents(stdout, advertisedTools, requireInit = f
   if (result.num_turns !== toolUses.length + 1) {
     throw new Error("Model command used an unexpected number of turns");
   }
-  const usage = result.modelUsage?.[model];
-  if (usage?.canonicalModel !== model || usage.provider !== "firstParty") {
+  const usage = result.modelUsage?.[claudeModel];
+  if (usage?.canonicalModel !== claudeModel || usage.provider !== "firstParty") {
     throw new Error("Model command did not use the required model");
   }
   return {
@@ -167,6 +194,7 @@ export function parseJudgeVerdict(output) {
 }
 
 export function modelInvocation(provider, prompt, directory) {
+  if (provider.startsWith("codex-")) return codexInvocation(provider, prompt, directory);
   const token = provider === "claude-ci" ? process.env.CLAUDE_CODE_OAUTH_TOKEN?.trim() : undefined;
   if (provider === "claude-ci" && !token) throw new Error("Claude subscription authentication is unavailable");
   const localHome = provider === "claude-local" ? process.env.HOME : undefined;
@@ -177,7 +205,7 @@ export function modelInvocation(provider, prompt, directory) {
     command: process.env.EMSEEPEA_MODEL_COMMAND ?? "claude",
     args: [
       "--print", prompt,
-      "--model", model,
+      "--model", claudeModel,
       "--effort", "low",
       "--max-turns", "4",
       "--safe-mode",
@@ -200,6 +228,9 @@ export function modelInvocation(provider, prompt, directory) {
 }
 
 export function conversationInvocation(provider, directory, url, tools, authToken, context) {
+  if (provider.startsWith("codex-")) {
+    return codexInvocation(provider, undefined, directory, { url, tools, authToken, context });
+  }
   const nativeTools = tools.map(({ name }) => nativeToolName(name));
   const config = nativeTools.length ? {
     mcpServers: {
@@ -218,7 +249,7 @@ export function conversationInvocation(provider, directory, url, tools, authToke
       "--input-format", "stream-json",
       "--output-format", "stream-json",
       "--verbose",
-      "--model", model,
+      "--model", claudeModel,
       "--effort", "low",
       "--max-turns", "4",
       "--strict-mcp-config",
@@ -238,6 +269,9 @@ export function conversationInvocation(provider, directory, url, tools, authToke
 }
 
 export function startModelConversation(provider, directory, url, tools, authToken, context, signal, serverSecrets = []) {
+  if (provider.startsWith("codex-")) {
+    return startCodexConversation(provider, directory, url, tools, authToken, context, signal, serverSecrets);
+  }
   signal?.throwIfAborted();
   const invocation = conversationInvocation(provider, directory, url, tools, authToken, context);
   const child = spawn(invocation.command, invocation.args, {
@@ -362,7 +396,170 @@ export async function runModel(provider, prompt, directory, signal) {
   if (execution.errorCode === "ABORT_ERR") throw new Error("Model command was cancelled");
   if (execution.errorCode) throw new Error("Model command could not start");
   if (execution.code !== 0 && !execution.stdout) throw new Error(`Model command exited ${execution.code}`);
-  return parseClaudeEvents(execution.stdout, execution.code);
+  return provider.startsWith("codex-")
+    ? parseCodexEvents(execution.stdout, [], undefined, execution.code,
+      providerModel(provider), invocation.providerSecrets)
+    : parseClaudeEvents(execution.stdout, execution.code);
+}
+
+export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, processExitCode = 0, selectedModel = "test-model", secrets = []) {
+  let events;
+  try {
+    events = typeof stdout === "string"
+      ? stdout.split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line))
+      : stdout;
+  } catch {
+    throw new Error("Model command returned invalid event data");
+  }
+  if (!Array.isArray(events)) throw new Error("Model command returned invalid event data");
+  const serialized = JSON.stringify(events);
+  if (secrets.some((secret) => typeof secret === "string" && secret.length > 0 && serialized.includes(secret))) {
+    throw new Error("Model command exposed a configured secret");
+  }
+  const authentication = /(?:not logged in|authentication|unauthorized|invalid api key|oauth)/i;
+  const errorText = events.flatMap((event) => [event.message, event.error?.message, event.item?.message])
+    .filter((value) => typeof value === "string").join("\n");
+  if (authentication.test(errorText)) throw new Error("Model command is not signed in");
+  const failedTool = events.some(({ item }) => item?.type === "mcp_tool_call"
+    && (item.status === "failed" || item.result?.isError === true || item.result?.is_error === true));
+  if (failedTool) throw new Error("Model command reported a failed MCP tool call");
+  if (processExitCode !== 0) throw new Error(`Model command exited ${processExitCode}`);
+  if (events.some(({ type }) => type === "turn.failed" || type === "error")) {
+    throw new Error("Model command reported an error");
+  }
+  const started = events.filter(({ type }) => type === "thread.started");
+  const threadId = started[0]?.thread_id;
+  if (started.length !== 1 || typeof threadId !== "string" || !threadId) {
+    throw new Error("Model command omitted its session event");
+  }
+  if (expectedThreadId && threadId !== expectedThreadId) {
+    throw new Error("Model command did not resume the required session");
+  }
+  if (events.filter(({ type }) => type === "turn.started").length !== 1
+    || events.filter(({ type }) => type === "turn.completed").length !== 1) {
+    throw new Error("Model command returned an incomplete turn");
+  }
+  const advertised = new Set(advertisedTools.map(({ name }) => name));
+  const completed = events.filter(({ type }) => type === "item.completed").map(({ item }) => item);
+  const forbidden = completed.filter((item) => !["agent_message", "reasoning", "mcp_tool_call"].includes(item?.type));
+  if (forbidden.length) throw new Error("Model command used a forbidden tool");
+  const toolItems = completed.filter(({ type }) => type === "mcp_tool_call");
+  if (toolItems.length > 3) throw new Error("Model command used more than three tools");
+  const calls = toolItems.map((item) => {
+    if (item.server !== mcpServerName || !advertised.has(item.tool)
+      || !item.arguments || typeof item.arguments !== "object" || Array.isArray(item.arguments)) {
+      throw new Error("Model command used a forbidden tool");
+    }
+    if (!item.result || typeof item.result !== "object" || item.status === "failed"
+      || item.result.isError === true || item.result.is_error === true) {
+      throw new Error("Model command reported a failed MCP tool call");
+    }
+    return { name: item.tool, arguments: item.arguments };
+  });
+  const messages = completed.filter(({ type }) => type === "agent_message");
+  const answer = messages.at(-1)?.text;
+  if (typeof answer !== "string") throw new Error("Model command returned no answer");
+  const toolResults = toolItems.map(({ result }) => ({ content: result.content ?? result, isError: false }));
+  return {
+    answer,
+    calls,
+    toolResults,
+    pathEvidence: calls.map((call, index) => ({
+      method: "tools/call",
+      target: call.name,
+      requestSha256: hash({ method: "tools/call", name: call.name, arguments: call.arguments }),
+      responseSha256: hash(toolResults[index].content),
+    })),
+    models: [selectedModel],
+    modelEvidence: "configured",
+    turnCount: 1,
+    providerTurnCount: calls.length + 1,
+    providerToolCount: calls.length,
+    threadId,
+  };
+}
+
+function startCodexConversation(provider, directory, url, tools, authToken, context, signal, serverSecrets) {
+  let threadId;
+  let closed = false;
+  return Object.freeze({
+    async send(prompt, options) {
+      if (closed) throw new Error("Model conversation is closed");
+      if (options?.elicitations?.length) throw new Error("Codex provider does not support scripted elicitation");
+      const invocation = conversationInvocation(provider, directory, url, tools, authToken, context);
+      const args = threadId
+        ? ["exec", "resume", threadId,
+          ...withoutOption(withoutOption(invocation.args.slice(1), "--cd"), "--sandbox")]
+        : invocation.args;
+      args.push(prompt);
+      const execution = await runProcess(invocation.command, args, {
+        cwd: invocation.cwd, env: invocation.env, signal, killSignal: "SIGKILL",
+      });
+      if (execution.timedOut) throw new Error("Model command timed out");
+      if (execution.outputLimitExceeded) throw new Error("Model command output exceeded its limit");
+      if (execution.errorCode === "ABORT_ERR") throw new Error("Model command was cancelled");
+      if (execution.errorCode) throw new Error("Model conversation could not start");
+      const turn = parseCodexEvents(execution.stdout, tools, threadId, execution.code,
+        providerModel(provider), [...invocation.providerSecrets, authToken, ...serverSecrets]);
+      threadId = turn.threadId;
+      return { ...turn, elicitations: [] };
+    },
+    async close() { closed = true; },
+  });
+}
+
+function codexInvocation(provider, prompt, directory, conversation) {
+  const model = providerModel(provider);
+  const sourceHome = process.env.CODEX_HOME ?? (process.env.HOME ? join(process.env.HOME, ".codex") : undefined);
+  const sourceAuth = sourceHome ? join(sourceHome, "auth.json") : undefined;
+  const apiKey = process.env.OPENAI_API_KEY?.trim();
+  if (provider === "codex-ci" && !apiKey) throw new Error("Codex CI authentication is unavailable");
+  if (provider === "codex-local" && !apiKey && (!sourceAuth || !existsSync(sourceAuth))) {
+    throw new Error("Codex authentication is unavailable");
+  }
+  const codexHome = join(directory, "codex-home");
+  mkdirSync(codexHome, { recursive: true, mode: 0o700 });
+  const rules = join(codexHome, "rules");
+  mkdirSync(rules, { recursive: true, mode: 0o700 });
+  writeFileSync(join(rules, "default.rules"), `prefix_rule(
+    pattern = [["bash", "zsh", "sh", "/bin/bash", "/bin/zsh", "/bin/sh", "/usr/bin/bash", "/usr/bin/zsh", "/usr/bin/sh"], ["-c", "-lc"]],
+    decision = "forbidden",
+    justification = "Only the configured MCP tools are allowed.",
+)\n`, { mode: 0o600 });
+  const isolatedAuth = join(codexHome, "auth.json");
+  if (!apiKey && sourceAuth && !existsSync(isolatedAuth)) symlinkSync(sourceAuth, isolatedAuth);
+  const authSecrets = !apiKey && sourceAuth ? credentialValues(sourceAuth) : [];
+  const args = [
+    "exec", "--json", "--ignore-user-config", "--skip-git-repo-check", "--sandbox", "read-only",
+    "--model", model, "--cd", directory,
+    "-c", 'approval_policy="never"',
+    "-c", 'web_search="disabled"',
+    "-c", 'shell_environment_policy.inherit="none"',
+    "-c", "allow_login_shell=false",
+    "-c", "features.multi_agent=false",
+    "-c", "features.unified_exec=false",
+  ];
+  if (conversation?.context) args.push("-c", `developer_instructions=${JSON.stringify(conversation.context)}`);
+  if (conversation?.tools.length) {
+    args.push("-c", `mcp_servers.${mcpServerName}.url=${JSON.stringify(conversation.url)}`);
+    args.push("-c", `mcp_servers.${mcpServerName}.enabled_tools=${JSON.stringify(conversation.tools.map(({ name }) => name))}`);
+    args.push("-c", `mcp_servers.${mcpServerName}.default_tools_approval_mode="approve"`);
+    for (const { name } of conversation.tools) {
+      args.push("-c", `mcp_servers.${mcpServerName}.tools.${JSON.stringify(name)}.approval_mode="approve"`);
+    }
+    if (conversation.authToken) {
+      args.push("-c", `mcp_servers.${mcpServerName}.bearer_token_env_var="EMSEEPEA_SEMANTIC_MCP_TOKEN"`);
+    }
+  }
+  if (prompt !== undefined) args.push(prompt);
+  return {
+    command: process.env.EMSEEPEA_CODEX_COMMAND ?? process.env.EMSEEPEA_MODEL_COMMAND ?? "codex",
+    args,
+    cwd: directory,
+    providerSecrets: [apiKey, ...authSecrets].filter(Boolean),
+    env: modelEnvironment({ CODEX_HOME: codexHome, ...(apiKey ? { OPENAI_API_KEY: apiKey } : {}),
+      ...(conversation?.authToken ? { EMSEEPEA_SEMANTIC_MCP_TOKEN: conversation.authToken } : {}) }),
+  };
 }
 
 function runProcess(command, args, options) {
@@ -410,3 +607,24 @@ function modelEnvironment(extra) {
 function nativeToolName(name) {
   return `mcp__${mcpServerName}__${name}`;
 }
+
+function withoutOption(args, name) {
+  const index = args.indexOf(name);
+  return index < 0 ? [...args] : [...args.slice(0, index), ...args.slice(index + 2)];
+}
+
+function credentialValues(path) {
+  try {
+    const values = [];
+    const visit = (value, key = "") => {
+      if (typeof value === "string" && /(?:token|secret|key|credential)/i.test(key)) values.push(value);
+      else if (value && typeof value === "object") {
+        for (const [childKey, child] of Object.entries(value)) visit(child, childKey);
+      }
+    };
+    visit(JSON.parse(readFileSync(path, "utf8")));
+    return values;
+  } catch {
+    return [];
+  }
+}

package/semantic/test.d.mts

@@ -38,6 +38,8 @@ export interface ElicitationEvidence {
 export interface SemanticConversation {
   /** Sends this exact user message through the same provider-native MCP conversation. */
   send(prompt: string, options?: { elicitations: readonly ScriptedElicitation[] }): Promise<ConversationTurn>;
+  /** Starts a provider-native conversation with no transcript history against the same running test application. */
+  fresh(): Promise<void>;
 }
 
 export function createConversation(

package/semantic/test.mjs

@@ -10,7 +10,7 @@ import {
   startSemanticServer,
   stopSemanticServer,
 } from "./material.mjs";
-import { parseJudgeVerdict, runModel, startModelConversation } from "./provider.mjs";
+import { parseJudgeVerdict, providerModel, providerSettings, runModel, startModelConversation } from "./provider.mjs";
 
 const hash = (value) => createHash("sha256").update(value).digest("hex");
 const names = new Set();
@@ -28,7 +28,7 @@ export async function createConversation(testContext, options) {
   names.add(key);
 
   const provider = process.env.EMSEEPEA_EVAL_PROVIDER ?? "claude-local";
-  if (!["claude-local", "claude-ci"].includes(provider)) throw new Error("Unsupported model provider");
+  if (!["claude-local", "claude-ci", "codex-local", "codex-ci"].includes(provider)) throw new Error("Unsupported model provider");
   const smoke = process.env.EMSEEPEA_EVAL_SMOKE === "1";
   if (smoke && provider === "claude-ci") throw new Error("Smoke tests cannot qualify a release");
   const file = process.env.EMSEEPEA_TEST_FILE;
@@ -43,7 +43,9 @@ export async function createConversation(testContext, options) {
     authoritative: provider === "claude-ci",
     smoke,
     provider,
-    model: "claude-sonnet-4-6",
+    configuredModel: providerModel(provider),
+    modelEvidence: provider.startsWith("codex-") ? "configured" : "observed",
+    settings: providerSettings(provider),
     semanticRetries: 0,
     status: "failed",
     caseSha256: hash(JSON.stringify({
@@ -72,7 +74,7 @@ export async function createConversation(testContext, options) {
         const serverSecrets = Object.entries(environment ?? {})
           .filter(([key]) => /token|password|secret|credential|api.?key/i.test(key))
           .map(([, value]) => value);
-        state.trials.push({ running, tools, record, directory, history: [], model: undefined, serverSecrets });
+        state.trials.push({ running, tools, record, directory, history: [], model: undefined, serverSecrets, conversation: 1 });
         evidence.answerTrials.push(record);
       } catch (error) {
         await stopSemanticServer(running.child);
@@ -113,6 +115,7 @@ export async function createConversation(testContext, options) {
         });
         const record = {
           turn: trial.record.turns.length + 1,
+          conversation: trial.conversation,
           interactionMode: "native-mcp",
           prompt,
           response: answer.answer,
@@ -125,6 +128,7 @@ export async function createConversation(testContext, options) {
           promptSha256: hash(prompt),
           answerSha256: hash(answer.answer),
           answerModels: answer.models,
+          answerModelEvidence: answer.modelEvidence ?? "observed",
           answerTurnCount: answer.turnCount,
           answerProviderTurnCount: answer.providerTurnCount,
           answerProviderToolCount: answer.providerToolCount,
@@ -176,6 +180,15 @@ export async function createConversation(testContext, options) {
         [privateTurn]: trials,
       });
     },
+    async fresh() {
+      ensureOpen(state);
+      await Promise.all(state.trials.map(async (trial) => {
+        await trial.model?.close();
+        trial.model = undefined;
+        trial.history = [];
+        trial.conversation += 1;
+      }));
+    },
   });
 }
 
@@ -428,6 +441,7 @@ async function judgeTrialMeaning(trial, trialIndex, expected) {
       const response = await isolatedModel(trial.provider, request, "emseepea-judge-", trial.signal);
       Object.assign(record, {
         models: response.models,
+        modelEvidence: response.modelEvidence ?? "observed",
         turnCount: response.turnCount,
         providerTurnCount: response.providerTurnCount,
         providerToolCount: response.providerToolCount,
@@ -453,6 +467,10 @@ function safeModelFailure(error) {
   const message = error instanceof Error ? error.message : "";
   const safeMessages = new Set([
     "Claude subscription authentication is unavailable",
+    "Codex authentication is unavailable",
+    "Codex CI authentication is unavailable",
+    "Codex model is not configured",
+    "Codex provider does not support scripted elicitation",
     "Model command attempted a forbidden action",
     "Model command could not start",
     "Model command exceeded its budget",
@@ -460,6 +478,7 @@ function safeModelFailure(error) {
     "Model command failed during execution",
     "Model command is not signed in",
     "Model command omitted its result event",
+    "Model command omitted its session event",
     "Model command output exceeded its limit",
     "Model command reported an error",
     "Model command returned a non-text answer",
@@ -473,6 +492,10 @@ function safeModelFailure(error) {
     "Model command omitted MCP initialization evidence",
     "Model command omitted an MCP tool result",
     "Model command returned no answer",
+    "Model command returned an incomplete turn",
+    "Model command did not resume the required session",
+    "Model command reported a failed MCP tool call",
+    "Model command exposed a configured secret",
     "Model command used more than three tools",
     "Model conversation already has a pending turn",
     "Model conversation could not start",