@emseepea/testing 0.24.0 → 0.25.0

package/README.md

@@ -134,12 +134,35 @@ Native-client events are authoritative for the tools that were actually used:
 a turn is accepted only when every application MCP call belongs to the
 configured test server and its expected tools.
 
-Codex may make its two built-in resource-discovery calls through the CLI's
+With resource capture left off, Codex may make its two built-in resource-discovery calls through the CLI's
 `codex` server. The runner accepts them only when their native results prove the
 resource and template inventories are empty, and excludes them from application
 assertions and raw evidence.
 
-Any non-empty, unrecognised, account-app, plugin, or other foreign call is
+For a Codex conversation that reads resources, set `nativeResources: true` in
+`createConversation` options. This option is off by default; other providers
+reject it.
+
+The runner obtains a bounded resource catalogue using the conversation's
+authentication and records recognized native resource operations separately
+from application-tool calls. `turn.resourceOperations` contains one operation
+array per trial; `assertNoToolCalls` still refers to application tools.
+Unscoped empty Codex discovery remains excluded from resource evidence.
+
+Registered reads retain their actual returned contents and request order in
+the evidence and meaning-judge context. Aggregate contents may have distinct
+URIs, and successful empty reads remain empty. Returned URIs do not authorize
+another read. Failed calls, unknown requested URIs, foreign servers and helper
+names that collide with application tools still fail qualification.
+
+The [retained genuine AI proof](https://github.com/emseepea/emseepea/blob/main/docs/reviews/native-resource-evidence-2026-10-13.json) covers a synthetic mixed tool/resource journey
+across resumed and fresh model sessions. It does not qualify general resource-only
+discovery, authenticated HTTP correlation or a production learner journey.
+
+Fresh model sessions use the same running test application. This option does
+not recover completed turns from a failed model invocation.
+
+Any other non-empty, unrecognised, account-app, plugin, or foreign call is
 rejected without storing raw account metadata.
 
 Failed Codex runs retain only a fixed failure category: authentication,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@emseepea/testing",
-  "version": "0.24.0",
+  "version": "0.25.0",
   "description": "MCP integration and semantic testing helpers",
   "license": "MIT",
   "repository": {
@@ -57,8 +57,10 @@
     "test:llm:built": "emseepea-test eval"
   },
   "dependencies": {
-    "@emseepea/server": "0.23.0",
-    "@modelcontextprotocol/client": "2.2.0"
+    "@emseepea/server": "0.23.1",
+    "@modelcontextprotocol/client": "2.2.0",
+    "@modelcontextprotocol/server": "2.0.0",
+    "@modelcontextprotocol/core": "2.0.0"
   },
   "devDependencies": {
     "playwright": "1.62.1"

package/semantic/case.mjs

@@ -9,6 +9,9 @@ export function validateConversationOptions(value) {
   if (value.context !== undefined && typeof value.context !== "string") {
     throw new Error("context must be text");
   }
+  if (value.nativeResources !== undefined && typeof value.nativeResources !== "boolean") {
+    throw new Error("nativeResources must be boolean");
+  }
   for (const key of ["authToken", "authTokenEnvironment"]) {
     if (value[key] !== undefined && (typeof value[key] !== "string" || !value[key].trim())) {
       throw new Error(`${key} must be non-empty text`);

package/semantic/cli.mjs

@@ -5,7 +5,7 @@ import { tmpdir } from "node:os";
 import { dirname, join, relative, resolve } from "node:path";
 import { validRecord } from "./evidence.mjs";
 import { discoverTests } from "./discover.mjs";
-import { modelVersion, providerModel, providerSettings } from "./provider.mjs";
+import { mcpClientVersion, modelVersion, providerModel, providerSettings } from "./provider.mjs";
 
 const paths = [];
 let provider = process.env.EMSEEPEA_EVAL_PROVIDER ?? "claude-local";
@@ -32,10 +32,9 @@ const evidence = { authoritative: provider === "claude-ci", provider, smoke,
   status: "failed", cases: {}, errors: [], startedAt: new Date().toISOString() };
 try {
   const client = JSON.parse(await readFile(new URL("../package.json", import.meta.resolve("@modelcontextprotocol/client")), "utf8"));
-  if (client.name !== "@modelcontextprotocol/client" || client.version !== "2.2.0") throw new Error("Unexpected MCP client version");
+  const clientVersion = mcpClientVersion(client);
   const cliName = provider.startsWith("codex-") ? "codexCli" : "claudeCli";
-  evidence.dependencies = { mcpClient: client.version, [cliName]: smoke ? "simulated" : await modelVersion(provider) };
-  if (provider === "claude-ci" && evidence.dependencies.claudeCli !== "2.1.248") throw new Error("Unexpected Claude CLI version");
+  evidence.dependencies = { mcpClient: clientVersion, [cliName]: smoke ? "simulated" : await modelVersion(provider) };
   let interrupted = false;
   for (const file of files) {
     const displayFile = relative(process.cwd(), file);

package/semantic/evidence.mjs

@@ -1,4 +1,5 @@
 import { createHash } from "node:crypto";
+import { resourceHelperOperation } from "./resources.mjs";
 import {
   canonicalToolNames,
   normalizeToolSelectionContract,
@@ -21,6 +22,7 @@ const sha256 = (value) => createHash("sha256").update(JSON.stringify(value)).dig
 export function validRecord(record, evidence) {
   const { authoritative, smoke } = evidence;
   const isHash = (value) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value);
+  if (record.nativeResources !== undefined && typeof record.nativeResources !== "boolean") return false;
   if (record.status !== "passed" || record.authoritative !== authoritative || record.smoke !== smoke
     || record.provider !== evidence.provider || record.configuredModel !== evidence.configuredModel
     || record.modelEvidence !== evidence.modelEvidence
@@ -56,13 +58,57 @@ export function validRecord(record, evidence) {
       && validNegativeFeedbackAssertion(turn)
       && validFeedbackDisclosure(turn, record.judgeVerdicts, trial.trial, isHash)
       && turn.toolCalls.every((call) => Object.hasOwn(call, "result") && typeof call.isError === "boolean")
-      && Array.isArray(turn.pathEvidence) && turn.pathEvidence.length === turn.toolCallCount
-      && turn.pathEvidence.every(({ method, target, requestSha256, responseSha256 }, index) => {
-        const call = turn.toolCalls[index];
-        return method === "tools/call" && target === call.name
-          && requestSha256 === sha256({ method, name: call.name, arguments: call.arguments })
-          && responseSha256 === sha256(call.result);
-      })));
+      && validPathEvidence(turn, record, trial, isHash)));
+}
+
+function validPathEvidence(turn, record, trial, isHash) {
+  const operations = turn.resourceOperations ?? [];
+  if (!Array.isArray(operations)) return false;
+  if (record.nativeResources !== true) {
+    if (operations.length) return false;
+  } else {
+    const catalogue = trial.resourceCatalogue;
+    if (!record.provider.startsWith("codex-") || !catalogue
+      || !Array.isArray(catalogue.resources) || !Array.isArray(catalogue.resourceTemplates)
+      || catalogue.resources.length > 4096 || catalogue.resourceTemplates.length > 4096
+      || !catalogue.resources.every(entry => entry && typeof entry.uri === "string" && entry.uri.trim())
+      || !catalogue.resourceTemplates.every(entry => entry && typeof entry.uriTemplate === "string" && entry.uriTemplate.trim())
+      || Buffer.byteLength(JSON.stringify(catalogue)) > 2 * 1_048_576
+      || !isHash(turn.resourceCatalogueSha256) || turn.resourceCatalogueSha256 !== sha256(catalogue)
+      || turn.answerProviderResourceOperationCount !== operations.length
+      || turn.answerProviderTurnCount !== turn.toolCallCount + operations.length + 1
+      || !isHash(turn.materialSha256) || turn.materialSha256 !== sha256(turn.pathEvidence)) return false;
+    const helpers = { "resources/read": "read_mcp_resource", "resources/list": "list_mcp_resources",
+      "resources/templates/list": "list_mcp_resource_templates" };
+    try {
+      for (const operation of operations) {
+        if (!operation || operation.source !== "provider-native-helper"
+          || Object.keys(operation).some(key => !["method", "target", "arguments", "result", "source"].includes(key))) return false;
+        const checked = resourceHelperOperation({ server: "emseepea_eval", status: "completed",
+          tool: helpers[operation.method], arguments: operation.arguments,
+          result: { content: [{ type: "text", text: JSON.stringify(operation.result) }], structured_content: null } }, catalogue);
+        if (!checked || checked.method !== operation.method || checked.target !== operation.target) return false;
+      }
+    } catch { return false; }
+  }
+  if (!Array.isArray(turn.pathEvidence) || turn.pathEvidence.length !== turn.toolCallCount + operations.length) return false;
+  let toolIndex = 0, resourceIndex = 0;
+  for (const path of turn.pathEvidence) {
+    if (!path || typeof path !== "object" || Array.isArray(path)) return false;
+    if (path.method === "tools/call") {
+      const call = turn.toolCalls[toolIndex++];
+      if (!call || path.target !== call.name
+        || path.requestSha256 !== sha256({ method: path.method, name: call.name, arguments: call.arguments })
+        || path.responseSha256 !== sha256(call.result)) return false;
+    } else {
+      const operation = operations[resourceIndex++];
+      if (!operation || path.source !== "provider-native-helper" || path.method !== operation.method
+        || path.target !== operation.target || path.requestSha256 !== sha256(operation.arguments)
+        || path.responseSha256 !== sha256(operation.result)) return false;
+    }
+  }
+  return toolIndex === turn.toolCallCount && resourceIndex === operations.length;
+
 }
 
 function validToolAssertions(turn, isHash) {

package/semantic/material.mjs

@@ -1,6 +1,8 @@
 import { spawn } from "node:child_process";
 import { createHash } from "node:crypto";
 
+import { collectResourceCatalogue } from "./resources.mjs";
+
 import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
 
 const sha256 = (value) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
@@ -115,6 +117,7 @@ export async function listMcpTools(url, testCase, signal) {
     signal?.addEventListener("abort", abort, { once: true });
   });
   try {
+    if (testCase.nativeResources && !client.getServerCapabilities()?.tools) return [];
     const response = await Promise.race([client.listTools(), cancelled]);
     const tools = response.tools.map(({ name, description, inputSchema }) => ({ name, description, inputSchema }));
     tools.sort((left, right) => left.name.localeCompare(right.name));
@@ -128,6 +131,15 @@ export async function listMcpTools(url, testCase, signal) {
   }
 }
 
+export async function listMcpResourceCatalogue(url, testCase, signal) {
+  signal?.throwIfAborted();
+  const client = await openClient(url, testCase);
+  const abort = () => { void client.close().catch(() => {}); };
+  signal?.addEventListener("abort", abort, { once: true });
+  try { return await collectResourceCatalogue(client, signal); }
+  finally { signal?.removeEventListener("abort", abort); await client.close(); }
+}
+
 function requestFor(operation) {
   if (operation.method === "tools/call") {
     return { method: operation.method, name: operation.name, arguments: operation.arguments ?? {} };

package/semantic/provider.mjs

@@ -2,8 +2,16 @@ import { spawn } from "node:child_process";
 import { createHash } from "node:crypto";
 import { existsSync, mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
+import { assertResourceHelperCollisions, resourceHelperOperation } from "./resources.mjs";
 import { scriptedElicitations } from "./elicitation.mjs";
 
+export function mcpClientVersion(client) {
+  if (client?.name !== "@modelcontextprotocol/client" || typeof client.version !== "string" || !client.version.trim()) {
+    throw new Error("Invalid MCP client provenance");
+  }
+  return client.version;
+}
+
 const claudeModel = "claude-sonnet-4-6";
 const mcpServerName = "emseepea_eval";
 const codexEmptyDiscoveryTools = new Map([
@@ -241,9 +249,9 @@ export function modelInvocation(provider, prompt, directory) {
   };
 }
 
-export function conversationInvocation(provider, directory, url, tools, authToken, context) {
+export function conversationInvocation(provider, directory, url, tools, authToken, context, resourceCatalogue) {
   if (provider.startsWith("codex-")) {
-    return codexInvocation(provider, undefined, directory, { url, tools, authToken, context });
+    return codexInvocation(provider, undefined, directory, { url, tools, authToken, context, resourceCatalogue });
   }
   const nativeTools = tools.map(({ name }) => nativeToolName(name));
   const config = nativeTools.length ? {
@@ -281,9 +289,9 @@ export function conversationInvocation(provider, directory, url, tools, authToke
   };
 }
 
-export function startModelConversation(provider, directory, url, tools, authToken, context, signal, serverSecrets = []) {
+export function startModelConversation(provider, directory, url, tools, authToken, context, signal, serverSecrets = [], resourceCatalogue) {
   if (provider.startsWith("codex-")) {
-    return startCodexConversation(provider, directory, url, tools, authToken, context, signal, serverSecrets);
+    return startCodexConversation(provider, directory, url, tools, authToken, context, signal, serverSecrets, resourceCatalogue);
   }
   signal?.throwIfAborted();
   const invocation = conversationInvocation(provider, directory, url, tools, authToken, context);
@@ -418,7 +426,7 @@ export async function runModel(provider, prompt, directory, signal) {
 }
 
 export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, processExitCode = 0,
-  selectedModel = "test-model", secrets = [], providerFailureCategory) {
+  selectedModel = "test-model", secrets = [], providerFailureCategory, resourceCatalogue) {
   let events;
   try {
     events = typeof stdout === "string"
@@ -457,17 +465,23 @@ export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, proc
     || events.filter(({ type }) => type === "turn.completed").length !== 1) {
     throw new Error("Model command returned an incomplete turn");
   }
+  if (resourceCatalogue) assertResourceHelperCollisions(advertisedTools);
   const advertised = new Set(advertisedTools.map(({ name }) => name));
   const completed = events.filter(({ type }) => type === "item.completed").map(({ item }) => item);
   const forbidden = completed.filter((item) => !["agent_message", "reasoning", "mcp_tool_call"].includes(item?.type));
   if (forbidden.length) throw forbiddenItemError(forbidden);
   const toolItems = completed.filter(({ type }) => type === "mcp_tool_call");
-  const discoveryItems = toolItems.filter(({ server, tool }) =>
+  const resourceItems = resourceCatalogue ? toolItems.flatMap((item) => {
+    const operation = resourceHelperOperation(item, resourceCatalogue);
+    return operation ? [{ item, operation }] : [];
+  }) : [];
+  const resourceItemSet = new Set(resourceItems.map(({ item }) => item));
+  const discoveryItems = toolItems.filter((item) => !resourceItemSet.has(item)).filter(({ server, tool }) =>
     server === "codex" && codexEmptyDiscoveryTools.has(tool));
   if (discoveryItems.some((item) => !emptyCodexDiscovery(item))) {
     throw forbiddenItemError(discoveryItems.filter((item) => !emptyCodexDiscovery(item)));
   }
-  const applicationToolItems = toolItems.filter((item) => !discoveryItems.includes(item));
+  const applicationToolItems = toolItems.filter((item) => !discoveryItems.includes(item) && !resourceItemSet.has(item));
   const calls = applicationToolItems.map((item) => {
     if (item.server !== mcpServerName || !advertised.has(item.tool)
       || !item.arguments || typeof item.arguments !== "object" || Array.isArray(item.arguments)) {
@@ -487,17 +501,24 @@ export function parseCodexEvents(stdout, advertisedTools, expectedThreadId, proc
     answer,
     calls,
     toolResults,
-    pathEvidence: calls.map((call, index) => ({
-      method: "tools/call",
-      target: call.name,
-      requestSha256: hash({ method: "tools/call", name: call.name, arguments: call.arguments }),
-      responseSha256: hash(toolResults[index].content),
-    })),
+    resourceOperations: resourceItems.map(({ operation }) => ({ ...operation, source: "provider-native-helper" })),
+    pathEvidence: toolItems.flatMap((item) => {
+      const native = resourceItems.find((entry) => entry.item === item)?.operation;
+      if (native) return [{ method: native.method, target: native.target, source: "provider-native-helper",
+        requestSha256: hash(native.arguments), responseSha256: hash(native.result) }];
+      const index = applicationToolItems.indexOf(item);
+      if (index < 0) return [];
+      const call = calls[index];
+      return [{ method: "tools/call", target: call.name,
+        requestSha256: hash({ method: "tools/call", name: call.name, arguments: call.arguments }),
+        responseSha256: hash(toolResults[index].content) }];
+    }),
     models: [selectedModel],
     modelEvidence: "configured",
     turnCount: 1,
-    providerTurnCount: calls.length + 1,
+    providerTurnCount: calls.length + resourceItems.length + 1,
     providerToolCount: calls.length,
+    providerResourceOperationCount: resourceItems.length,
     threadId,
   };
 }
@@ -521,14 +542,14 @@ function emptyCodexDiscovery(item) {
   }
 }
 
-function startCodexConversation(provider, directory, url, tools, authToken, context, signal, serverSecrets) {
+function startCodexConversation(provider, directory, url, tools, authToken, context, signal, serverSecrets, resourceCatalogue) {
   let threadId;
   let closed = false;
   return Object.freeze({
     async send(prompt, options) {
       if (closed) throw new Error("Model conversation is closed");
       if (options?.elicitations?.length) throw new Error("Codex provider does not support scripted elicitation");
-      const invocation = conversationInvocation(provider, directory, url, tools, authToken, context);
+      const invocation = conversationInvocation(provider, directory, url, tools, authToken, context, resourceCatalogue);
       const args = threadId
         ? ["exec", "resume", threadId,
           ...withoutOption(withoutOption(invocation.args.slice(1), "--cd"), "--sandbox")]
@@ -543,7 +564,7 @@ function startCodexConversation(provider, directory, url, tools, authToken, cont
       if (execution.errorCode) throw new Error("Model conversation could not start");
       const turn = parseCodexEvents(execution.stdout, tools, threadId, execution.code,
         providerModel(provider), [...invocation.providerSecrets, authToken, ...serverSecrets],
-        execution.providerFailureCategory);
+        execution.providerFailureCategory, resourceCatalogue);
       threadId = turn.threadId;
       return { ...turn, elicitations: [] };
     },
@@ -590,9 +611,9 @@ function codexInvocation(provider, prompt, directory, conversation) {
     "-c", "features.unified_exec=false",
   ];
   if (conversation?.context) args.push("-c", `developer_instructions=${JSON.stringify(conversation.context)}`);
-  if (conversation?.tools.length) {
+  if (conversation && (conversation.tools.length || conversation.resourceCatalogue)) {
     args.push("-c", `mcp_servers.${mcpServerName}.url=${JSON.stringify(conversation.url)}`);
-    args.push("-c", `mcp_servers.${mcpServerName}.enabled_tools=${JSON.stringify(conversation.tools.map(({ name }) => name))}`);
+    if (conversation.tools.length) args.push("-c", `mcp_servers.${mcpServerName}.enabled_tools=${JSON.stringify(conversation.tools.map(({ name }) => name))}`);
     args.push("-c", `mcp_servers.${mcpServerName}.default_tools_approval_mode="approve"`);
     for (const { name } of conversation.tools) {
       args.push("-c", `mcp_servers.${mcpServerName}.tools.${JSON.stringify(name)}.approval_mode="approve"`);

package/semantic/resources.mjs

@@ -0,0 +1,98 @@
+import { ReadResourceResultSchema } from "@modelcontextprotocol/core";
+import { ResourceTemplate } from "@modelcontextprotocol/server";
+
+export const nativeResourceHelpers = new Set([
+  "list_mcp_resources", "list_mcp_resource_templates", "read_mcp_resource",
+]);
+const server = "emseepea_eval";
+const limit = 1_048_576;
+
+// The inventory is obtained with the same authentication as the conversation.
+// It bounds provider-reported helpers; it is not independent HTTP-hop evidence.
+export async function collectResourceCatalogue(client, signal) {
+  signal?.throwIfAborted();
+  if (!client.getServerCapabilities()?.resources) return { resources: [], resourceTemplates: [] };
+  const collect = async (method, key) => {
+    const entries = [];
+    const cursors = new Set();
+    let cursor;
+    let bytes = 0;
+    for (let page = 0; page < 32; page += 1) {
+      signal?.throwIfAborted();
+      const response = await client[method](cursor === undefined ? {} : { cursor }, { signal });
+      bytes += Buffer.byteLength(JSON.stringify(response));
+      if (bytes > limit || !Array.isArray(response[key])) throw new Error("Resource catalogue exceeds its bounds");
+      entries.push(...response[key]);
+      if (entries.length > 4096) throw new Error("Resource catalogue exceeds its bounds");
+      if (response.nextCursor === undefined) return entries;
+      if (typeof response.nextCursor !== "string" || !response.nextCursor || cursors.has(response.nextCursor)) {
+        throw new Error("Resource catalogue returned an invalid cursor");
+      }
+      cursors.add(response.nextCursor);
+      cursor = response.nextCursor;
+    }
+    throw new Error("Resource catalogue exceeds its page bound");
+  };
+  const resources = await collect("listResources", "resources");
+  const resourceTemplates = await collect("listResourceTemplates", "resourceTemplates");
+  if (resources.some((entry) => typeof entry.uri !== "string")
+    || resourceTemplates.some((entry) => typeof entry.uriTemplate !== "string")) {
+    throw new Error("Resource catalogue contains invalid entries");
+  }
+  for (const entry of resourceTemplates) new ResourceTemplate(entry.uriTemplate, { list: undefined });
+  return { resources, resourceTemplates };
+}
+
+export function assertResourceHelperCollisions(tools) {
+  if (tools.some(({ name }) => nativeResourceHelpers.has(name))) {
+    throw new Error("Native resource helpers collide with an advertised tool");
+  }
+}
+
+const object = (value) => value && typeof value === "object" && !Array.isArray(value);
+const keys = (value, allowed) => Object.keys(value).every((key) => allowed.includes(key));
+
+export function resourceHelperOperation(item, catalogue) {
+  if (!catalogue || !nativeResourceHelpers.has(item.tool)) return undefined;
+  if (![server, "codex"].includes(item.server) || item.status !== "completed"
+    || !object(item.arguments) || !keys(item.arguments, ["server", "uri"])) return undefined;
+  const reading = item.tool === "read_mcp_resource";
+  if (reading && (item.arguments.server !== server || typeof item.arguments.uri !== "string")) return undefined;
+  if (!reading && (item.arguments.uri !== undefined
+    || (item.arguments.server !== undefined && item.arguments.server !== server))) return undefined;
+  if (!object(item.result) || !keys(item.result, ["content", "structured_content"])
+    || item.result.structured_content !== null || !Array.isArray(item.result.content)
+    || item.result.content.length !== 1) return undefined;
+  const [content] = item.result.content;
+  if (!object(content) || content.type !== "text" || typeof content.text !== "string"
+    || !keys(content, ["type", "text"]) || Buffer.byteLength(content.text) > limit) return undefined;
+  let result;
+  try { result = JSON.parse(content.text); } catch { return undefined; }
+  if (!object(result)) return undefined;
+  if (reading) {
+    const uri = item.arguments.uri;
+    if (!keys(result, ["server", "uri", "contents"]) || result.server !== server || result.uri !== uri
+      || !Array.isArray(result.contents) || !knownUri(uri, catalogue)
+      || !ReadResourceResultSchema.safeParse({ contents: result.contents }).success
+      || result.contents.some((entry) => !object(entry)
+        || (typeof entry.text === "string") === (typeof entry.blob === "string"))) return undefined;
+    // Aggregate reads may return distinct content URIs. They are content from
+    // this registered request, not new catalogue entries or follow-up authority.
+    return { method: "resources/read", target: uri, arguments: item.arguments, result };
+  }
+  const key = item.tool === "list_mcp_resources" ? "resources" : "resourceTemplates";
+  const field = key === "resources" ? "uri" : "uriTemplate";
+  if (!keys(result, ["server", key]) || (result.server !== undefined && result.server !== server)
+    || !Array.isArray(result[key])) return undefined;
+  if (item.server === "codex" && item.arguments.server === undefined
+    && result.server === undefined && result[key].length === 0) return undefined;
+  if (result[key].some((entry) => !object(entry) || entry.server !== server
+    || !catalogue[key].some((known) => known[field] === entry[field]))) return undefined;
+  return { method: key === "resources" ? "resources/list" : "resources/templates/list",
+    target: server, arguments: item.arguments, result };
+}
+
+function knownUri(uri, catalogue) {
+  return catalogue.resources.some((entry) => entry.uri === uri)
+    || catalogue.resourceTemplates.some((entry) => new ResourceTemplate(entry.uriTemplate, { list: undefined }).uriTemplate.match(uri) !== null);
+}

package/semantic/test.d.mts

@@ -35,11 +35,22 @@ export interface ConversationOptions {
   context?: string;
   /** Server environment shared by all trials, or isolated values selected by trial number. */
   environment?: Record<string, string> | ((trial: number) => Record<string, string>);
+  /** Opt in to registered native resource-helper evidence. Currently Codex only. */
+  nativeResources?: boolean;
   authToken?: string;
   authTokenEnvironment?: string;
 }
 
+export interface ResourceOperation {
+  readonly method: "resources/list" | "resources/templates/list" | "resources/read";
+  readonly target: string;
+  readonly source: "provider-native-helper";
+  readonly arguments: Readonly<Record<string, unknown>>;
+  readonly result: unknown;
+}
+
 export interface ConversationTurn {
+  readonly resourceOperations: readonly (readonly ResourceOperation[])[];
   readonly responses: readonly string[];
   readonly toolCalls: readonly (readonly ToolCall[])[];
   readonly toolOutcomes: readonly (readonly ToolOutcome[])[];

package/semantic/test.mjs

@@ -6,11 +6,13 @@ import { dirname, join, resolve } from "node:path";
 import { environmentForTrial, validateConversationOptions } from "./case.mjs";
 import {
   listMcpTools,
+  listMcpResourceCatalogue,
   semanticAuthToken,
   startSemanticServer,
   stopSemanticServer,
 } from "./material.mjs";
 import { parseJudgeVerdict, providerModel, providerSettings, runModel, startModelConversation } from "./provider.mjs";
+import { assertResourceHelperCollisions } from "./resources.mjs";
 import {
   canonicalToolNames,
   normalizeToolSelectionContract,
@@ -46,6 +48,9 @@ export async function createConversation(testContext, options) {
 
   const provider = process.env.EMSEEPEA_EVAL_PROVIDER ?? "claude-local";
   if (!["claude-local", "claude-ci", "codex-local", "codex-ci"].includes(provider)) throw new Error("Unsupported model provider");
+  if (specification.nativeResources && !provider.startsWith("codex-")) {
+    throw new Error("nativeResources requires the Codex provider");
+  }
   const smoke = process.env.EMSEEPEA_EVAL_SMOKE === "1";
   if (smoke && provider === "claude-ci") throw new Error("Smoke tests cannot qualify a release");
   const file = process.env.EMSEEPEA_TEST_FILE;
@@ -57,6 +62,7 @@ export async function createConversation(testContext, options) {
     name,
     file,
     mode: "conversation",
+    ...(specification.nativeResources ? { nativeResources: true } : {}),
     authoritative: provider === "claude-ci",
     smoke,
     provider,
@@ -86,12 +92,15 @@ export async function createConversation(testContext, options) {
       }, testContext.signal);
       try {
         const tools = await listMcpTools(running.url, specification, testContext.signal);
-        const record = { trial, turns: [] };
+        if (specification.nativeResources) assertResourceHelperCollisions(tools);
+        const resourceCatalogue = specification.nativeResources
+          ? await listMcpResourceCatalogue(running.url, specification, testContext.signal) : undefined;
+        const record = { trial, turns: [], ...(resourceCatalogue ? { resourceCatalogue } : {}) };
         const directory = await mkdtemp(join(tmpdir(), "emseepea-conversation-"));
         const serverSecrets = Object.entries(environment ?? {})
           .filter(([key]) => /token|password|secret|credential|api.?key/i.test(key))
           .map(([, value]) => value);
-        state.trials.push({ running, tools, record, directory, history: [], model: undefined, serverSecrets, conversation: 1 });
+        state.trials.push({ running, tools, resourceCatalogue, record, directory, history: [], model: undefined, serverSecrets, conversation: 1 });
         evidence.answerTrials.push(record);
       } catch (error) {
         await stopSemanticServer(running.child);
@@ -118,6 +127,7 @@ export async function createConversation(testContext, options) {
           specification.context,
           testContext.signal,
           trial.serverSecrets,
+          trial.resourceCatalogue,
         );
         const answer = await trial.model.send(prompt, options);
         const calls = answer.calls;
@@ -129,6 +139,7 @@ export async function createConversation(testContext, options) {
         trial.history.push({
           user: prompt,
           toolCalls: recordedToolCalls,
+          resourceOperations: answer.resourceOperations ?? [],
           assistant: answer.answer,
         });
         const record = {
@@ -139,6 +150,9 @@ export async function createConversation(testContext, options) {
           response: answer.answer,
           elicitations: answer.elicitations,
           toolCalls: recordedToolCalls,
+          resourceOperations: answer.resourceOperations ?? [],
+          resourceCatalogueSha256: trial.resourceCatalogue ? hash(JSON.stringify(trial.resourceCatalogue)) : undefined,
+          ...(trial.resourceCatalogue ? { answerProviderResourceOperationCount: answer.providerResourceOperationCount } : {}),
           toolOutcomesSha256: hash(JSON.stringify(recordedToolCalls)),
           promptSha256: hash(prompt),
           answerSha256: hash(answer.answer),
@@ -195,6 +209,7 @@ export async function createConversation(testContext, options) {
         toolOutcomes: Object.freeze(trials.map(({ record }) => immutableSnapshot(record.toolCalls.map(
           ({ name, arguments: args, result, isError }) => ({ name, arguments: args, result, isError }),
         )))),
+        resourceOperations: Object.freeze(trials.map(({ record }) => immutableSnapshot(record.resourceOperations))),
         elicitations: Object.freeze(trials.map(({ record }) => Object.freeze(record.elicitations))),
         [privateTurn]: trials,
       });