@emseepea/testing 0.23.1 → 0.23.3

package/README.md

@@ -109,12 +109,39 @@ EMSEEPEA_CODEX_MODEL=gpt-5.5 emseepea-test --provider codex-local eval
 
 `codex-local` uses the existing Codex CLI sign-in without copying it.
 `codex-ci` requires `OPENAI_API_KEY` from the CI secret store. Both providers
-run `codex exec --json`, resume the actual Codex session for follow-ups, ignore
-user configuration, and pre-approve only the discovered tools from the test
-server.
+support Codex CLI 0.162.0 or later, run `codex exec --json`, resume the actual
+Codex session for follow-ups, ignore user configuration, and pre-approve only
+the discovered tools from the test server.
 
 An isolated command policy denies shell launches and spawned shell environments
-inherit no provider credentials.
+inherit no provider credentials. The runner also disables Codex account apps,
+plugins, remote plugins, multi-agent execution, web search, shell tools, and
+unified execution for each one-shot and resumed native invocation.
+
+The invocation configuration sets the default account-app state to disabled,
+omits account-app instructions, and disables Codex app, plugin, and tool
+suggestion features.
+
+Native-client events are authoritative for the tools that were actually used:
+a turn is accepted only when every application MCP call belongs to the
+configured test server and its expected tools.
+
+Codex may make its two built-in resource-discovery calls through the CLI's
+`codex` server. The runner accepts them only when their native results prove the
+resource and template inventories are empty, and excludes them from application
+assertions and raw evidence.
+
+Any non-empty, unrecognised, account-app, plugin, or other foreign call is
+rejected without storing raw account metadata.
+
+Failed Codex runs retain only a fixed failure category: authentication,
+configuration, host storage, permission or sandbox, provider, transport, or
+unknown. The runner inspects at most 32 KiB of standard error to derive that
+category and does not store the standard error.
+
+Rejected native items retain only capped counts from a fixed item-type list.
+Commands, arguments, results, paths, server and tool names, and unknown
+item-type strings are excluded.
 
 Codex CLI evidence proves that CLI journey. It does not prove native ChatGPT
 connection, attachment, retrieval, or MCP App rendering behavior.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@emseepea/testing",
-  "version": "0.23.1",
+  "version": "0.23.3",
   "description": "MCP integration and semantic testing helpers",
   "license": "MIT",
   "repository": {

package/semantic/provider.mjs

@@ -6,6 +6,11 @@ import { scriptedElicitations } from "./elicitation.mjs";
 
 const claudeModel = "claude-sonnet-4-6";
 const mcpServerName = "emseepea_eval";
+const minimumCodexCliVersion = "0.162.0";
+const codexEmptyDiscoveryTools = new Map([
+  ["list_mcp_resources", "resources"],
+  ["list_mcp_resource_templates", "resourceTemplates"],
+]);
 const hash = (value) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
 const providerErrorMessages = new Map([
   ["error_during_execution", "Model command failed during execution"],
@@ -13,6 +18,17 @@ const providerErrorMessages = new Map([
   ["error_max_turns", "Model command exceeded its turn limit"],
 ]);
 const authenticationFailure = /not logged in|failed to authenticate|oauth session expired/i;
+const codexFailureCategories = new Set([
+  "authentication",
+  "configuration",
+  "host-storage",
+  "permission-or-sandbox",
+  "provider",
+  "transport",
+  "unknown",
+]);
+const stderrLimit = 32 * 1024;
+const forbiddenItemLimit = 64;
 const hasAuthenticationFailure = (events) => events.some((event) => (
   (Array.isArray(event.message?.content)
     && event.message.content.some(({ type, text }) => type === "text" && /not logged in/i.test(text ?? "")))
@@ -28,6 +44,9 @@ export async function modelVersion(provider = "claude-local") {
   const result = await runProcess(command, ["--version"], { env: modelEnvironment({}) });
   const version = result.stdout.trim().match(/(?:codex-cli\s+)?(\d+\.\d+\.\d+)\b/)?.[1];
   if (result.code !== 0 || !version) throw new Error(`Could not verify ${codex ? "Codex" : "Claude"} CLI version`);
+  if (codex && compareVersions(version, minimumCodexCliVersion) < 0) {
+    throw new Error(`Codex CLI ${version} is unsupported; use ${minimumCodexCliVersion} or later`);
+  }
   return version;
 }
 
@@ -42,11 +61,16 @@ export function providerSettings(provider) {
   return provider.startsWith("codex-")
     ? {
         approvalPolicy: "never",
+        appInstructions: false,
+        apps: false,
         loginShell: false,
         multiAgent: false,
+        plugins: false,
+        remotePlugin: false,
         sandbox: "read-only",
         shell: "forbidden",
         shellEnvironment: "empty",
+        toolSuggest: false,
         userConfig: "ignored",
         webSearch: false,
       }
@@ -388,14 +412,17 @@ export async function runModel(provider, prompt, directory, signal) {
   if (execution.outputLimitExceeded) throw new Error("Model command output exceeded its limit");
   if (execution.errorCode === "ABORT_ERR") throw new Error("Model command was cancelled");
   if (execution.errorCode) throw new Error("Model command could not start");
-  if (execution.code !== 0 && !execution.stdout) throw new Error(`Model command exited ${execution.code}`);
+  if (execution.code !== 0 && !execution.stdout) {
+    throw codexProcessError(execution.code, execution.providerFailureCategory);
+  }
   return provider.startsWith("codex-")
     ? parseCodexEvents(execution.stdout, [], undefined, execution.code,
-      providerModel(provider), invocation.providerSecrets)
+      providerModel(provider), invocation.providerSecrets, execution.providerFailureCategory)
     : parseClaudeEvents(execution.stdout, execution.code);
 }
 
-export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, processExitCode = 0, selectedModel = "test-model", secrets = []) {
+export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, processExitCode = 0,
+  selectedModel = "test-model", secrets = [], providerFailureCategory) {
   let events;
   try {
     events = typeof stdout === "string"
@@ -412,11 +439,13 @@ export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, proc
   const authentication = /(?:not logged in|authentication|unauthorized|invalid api key|oauth)/i;
   const errorText = events.flatMap((event) => [event.message, event.error?.message, event.item?.message])
     .filter((value) => typeof value === "string").join("\n");
-  if (authentication.test(errorText)) throw new Error("Model command is not signed in");
+  if (authentication.test(errorText)) {
+    throw codexEvidenceError("Model command is not signed in", "authentication");
+  }
   const failedTool = events.some(({ item }) => item?.type === "mcp_tool_call"
     && (item.status === "failed" || item.result?.isError === true || item.result?.is_error === true));
   if (failedTool) throw new Error("Model command reported a failed MCP tool call");
-  if (processExitCode !== 0) throw new Error(`Model command exited ${processExitCode}`);
+  if (processExitCode !== 0) throw codexProcessError(processExitCode, providerFailureCategory);
   if (events.some(({ type }) => type === "turn.failed" || type === "error")) {
     throw new Error("Model command reported an error");
   }
@@ -435,12 +464,18 @@ export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, proc
   const advertised = new Set(advertisedTools.map(({ name }) => name));
   const completed = events.filter(({ type }) => type === "item.completed").map(({ item }) => item);
   const forbidden = completed.filter((item) => !["agent_message", "reasoning", "mcp_tool_call"].includes(item?.type));
-  if (forbidden.length) throw new Error("Model command used a forbidden tool");
+  if (forbidden.length) throw forbiddenItemError(forbidden);
   const toolItems = completed.filter(({ type }) => type === "mcp_tool_call");
-  const calls = toolItems.map((item) => {
+  const discoveryItems = toolItems.filter(({ server, tool }) =>
+    server === "codex" && codexEmptyDiscoveryTools.has(tool));
+  if (discoveryItems.some((item) => !emptyCodexDiscovery(item))) {
+    throw forbiddenItemError(discoveryItems.filter((item) => !emptyCodexDiscovery(item)));
+  }
+  const applicationToolItems = toolItems.filter((item) => !discoveryItems.includes(item));
+  const calls = applicationToolItems.map((item) => {
     if (item.server !== mcpServerName || !advertised.has(item.tool)
       || !item.arguments || typeof item.arguments !== "object" || Array.isArray(item.arguments)) {
-      throw new Error("Model command used a forbidden tool");
+      throw forbiddenItemError([item]);
     }
     if (!item.result || typeof item.result !== "object" || item.status === "failed"
       || item.result.isError === true || item.result.is_error === true) {
@@ -451,7 +486,7 @@ export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, proc
   const messages = completed.filter(({ type }) => type === "agent_message");
   const answer = messages.at(-1)?.text;
   if (typeof answer !== "string") throw new Error("Model command returned no answer");
-  const toolResults = toolItems.map(({ result }) => ({ content: result.content ?? result, isError: false }));
+  const toolResults = applicationToolItems.map(({ result }) => ({ content: result.content ?? result, isError: false }));
   return {
     answer,
     calls,
@@ -471,6 +506,25 @@ export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, proc
   };
 }
 
+function emptyCodexDiscovery(item) {
+  if (item.status !== "completed" || !item.arguments || typeof item.arguments !== "object"
+    || Array.isArray(item.arguments) || Object.keys(item.arguments).length !== 0
+    || !item.result || typeof item.result !== "object" || Array.isArray(item.result)
+    || item.result.structured_content !== null || Object.keys(item.result).length !== 2
+    || !Array.isArray(item.result.content) || item.result.content.length !== 1) return false;
+  const [content] = item.result.content;
+  if (!content || content.type !== "text" || typeof content.text !== "string"
+    || Object.keys(content).length !== 2) return false;
+  try {
+    const parsed = JSON.parse(content.text);
+    const key = codexEmptyDiscoveryTools.get(item.tool);
+    return parsed && typeof parsed === "object" && !Array.isArray(parsed)
+      && Object.keys(parsed).length === 1 && Array.isArray(parsed[key]) && parsed[key].length === 0;
+  } catch {
+    return false;
+  }
+}
+
 function startCodexConversation(provider, directory, url, tools, authToken, context, signal, serverSecrets) {
   let threadId;
   let closed = false;
@@ -492,7 +546,8 @@ function startCodexConversation(provider, directory, url, tools, authToken, cont
       if (execution.errorCode === "ABORT_ERR") throw new Error("Model command was cancelled");
       if (execution.errorCode) throw new Error("Model conversation could not start");
       const turn = parseCodexEvents(execution.stdout, tools, threadId, execution.code,
-        providerModel(provider), [...invocation.providerSecrets, authToken, ...serverSecrets]);
+        providerModel(provider), [...invocation.providerSecrets, authToken, ...serverSecrets],
+        execution.providerFailureCategory);
       threadId = turn.threadId;
       return { ...turn, elicitations: [] };
     },
@@ -528,8 +583,14 @@ function codexInvocation(provider, prompt, directory, conversation) {
     "-c", 'web_search="disabled"',
     "-c", 'shell_environment_policy.inherit="none"',
     "-c", "allow_login_shell=false",
+    "-c", "apps._default.enabled=false",
+    "-c", "include_apps_instructions=false",
+    "-c", "features.apps=false",
     "-c", "features.multi_agent=false",
+    "-c", "features.plugins=false",
+    "-c", "features.remote_plugin=false",
     "-c", "features.shell_tool=false",
+    "-c", "features.tool_suggest=false",
     "-c", "features.unified_exec=false",
   ];
   if (conversation?.context) args.push("-c", `developer_instructions=${JSON.stringify(conversation.context)}`);
@@ -559,8 +620,10 @@ function runProcess(command, args, options) {
   return new Promise((resolve) => {
     const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"], ...options });
     let stdout = "";
+    let stderr = Buffer.alloc(0);
     let timedOut = false;
     let outputLimitExceeded = false;
+    let settled = false;
     const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); }, 180_000);
     timer.unref();
     child.stdout.on("data", (chunk) => {
@@ -570,18 +633,98 @@ function runProcess(command, args, options) {
         child.kill("SIGKILL");
       }
     });
-    child.stderr.resume();
-    child.once("error", (error) => {
+    child.stderr.on("data", (chunk) => {
+      if (stderr.length >= stderrLimit) return;
+      const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+      stderr = Buffer.concat([stderr, bytes.subarray(0, stderrLimit - stderr.length)]);
+    });
+    const finish = (result) => {
+      if (settled) return;
+      settled = true;
       clearTimeout(timer);
-      resolve({ code: 1, errorCode: error.code, outputLimitExceeded, stdout, timedOut });
+      const providerFailureCategory = classifyCodexFailure(stderr.toString("utf8"));
+      stderr = Buffer.alloc(0);
+      resolve({ ...result, providerFailureCategory });
+    };
+    child.once("error", (error) => {
+      finish({ code: 1, errorCode: error.code, outputLimitExceeded, stdout, timedOut });
     });
     child.once("close", (code) => {
-      clearTimeout(timer);
-      resolve({ code: code ?? 1, outputLimitExceeded, stdout, timedOut });
+      finish({ code: code ?? 1, outputLimitExceeded, stdout, timedOut });
     });
   });
 }
 
+function classifyCodexFailure(stderr) {
+  if (/no space left|disk (?:is )?full|storage (?:capacity|quota)|\benospc\b/i.test(stderr)) {
+    return "host-storage";
+  }
+  if (/not logged in|authenticat|unauthorized|invalid api key|oauth/i.test(stderr)) {
+    return "authentication";
+  }
+  if (/permission denied|operation not permitted|sandbox|\beacces\b|\beperm\b/i.test(stderr)) {
+    return "permission-or-sandbox";
+  }
+  if (/configuration|config(?:uration)? error|unknown (?:feature|field|key)|invalid (?:config|value)/i.test(stderr)) {
+    return "configuration";
+  }
+  if (/connection (?:refused|reset)|network|transport|timed? out|\btls\b|certificate|\bdns\b|\beconn/i.test(stderr)) {
+    return "transport";
+  }
+  if (/provider (?:error|failure|unavailable)|model (?:is )?(?:unavailable|overloaded)|rate limit|service unavailable/i.test(stderr)) {
+    return "provider";
+  }
+  return "unknown";
+}
+
+function compareVersions(left, right) {
+  const a = left.split(".").map(Number);
+  const b = right.split(".").map(Number);
+  for (let index = 0; index < Math.max(a.length, b.length); index += 1) {
+    const difference = (a[index] ?? 0) - (b[index] ?? 0);
+    if (difference !== 0) return difference;
+  }
+  return 0;
+}
+
+function codexEvidenceError(message, category) {
+  return Object.assign(new Error(message), {
+    providerFailureCategory: codexFailureCategories.has(category) ? category : "unknown",
+  });
+}
+
+function codexProcessError(code, category) {
+  return codexEvidenceError(`Model command exited ${code}`, category);
+}
+
+function forbiddenItemError(items) {
+  const counts = {
+    commandExecution: 0,
+    computerUse: 0,
+    fileChange: 0,
+    mcpToolCall: 0,
+    other: 0,
+    webSearch: 0,
+  };
+  for (const item of items.slice(0, forbiddenItemLimit)) {
+    const key = new Map([
+      ["command_execution", "commandExecution"],
+      ["computer_tool", "computerUse"],
+      ["file_change", "fileChange"],
+      ["mcp_tool_call", "mcpToolCall"],
+      ["web_search", "webSearch"],
+    ]).get(item?.type) ?? "other";
+    counts[key] += 1;
+  }
+  return Object.assign(new Error("Model command used a forbidden tool"), {
+    forbiddenItemEvidence: Object.freeze({
+      counts: Object.freeze(counts),
+      inspectedCount: Math.min(items.length, forbiddenItemLimit),
+      truncated: items.length > forbiddenItemLimit,
+    }),
+  });
+}
+
 function modelEnvironment(extra) {
   return Object.fromEntries(Object.entries({
     CI: "true",

package/semantic/test.mjs

@@ -178,7 +178,7 @@ export async function createConversation(testContext, options) {
           return;
         }
         const trial = state.trials[index];
-        trial.record.error = safeModelFailure(result.reason);
+        recordModelFailure(trial.record, result.reason);
         if (Array.isArray(result.reason?.elicitations)) trial.record.elicitations = result.reason.elicitations;
         if (Array.isArray(result.reason?.attemptedToolCalls)) {
           trial.record.attemptedToolCalls = result.reason.attemptedToolCalls;
@@ -508,9 +508,11 @@ async function judgeTrialMeaning(trial, trialIndex, expected) {
       record.verdict = verdict;
       outcomes.push({ failed: !verdict.pass, record });
     } catch (error) {
-      record.error = error instanceof SyntaxError || error.message === "Judge returned an invalid verdict"
-        ? "invalid judge verdict"
-        : safeModelFailure(error);
+      if (error instanceof SyntaxError || error.message === "Judge returned an invalid verdict") {
+        record.error = "invalid judge verdict";
+      } else {
+        recordModelFailure(record, error);
+      }
       outcomes.push({ failed: true, record });
     }
   }
@@ -570,6 +572,24 @@ function safeModelFailure(error) {
   return "model invocation failed";
 }
 
+function recordModelFailure(record, error) {
+  record.error = safeModelFailure(error);
+  const category = error?.providerFailureCategory;
+  if ([
+    "authentication",
+    "configuration",
+    "host-storage",
+    "permission-or-sandbox",
+    "provider",
+    "transport",
+    "unknown",
+  ].includes(category)) record.providerFailureCategory = category;
+  const evidence = error?.forbiddenItemEvidence;
+  if (evidence && typeof evidence === "object") {
+    record.forbiddenItemEvidence = structuredClone(evidence);
+  }
+}
+
 async function isolatedModel(provider, prompt, prefix, signal) {
   const directory = await mkdtemp(join(tmpdir(), prefix));
   try {