@emseepea/testing 0.0.0 → 0.0.1

package/README.md

@@ -1,5 +1,45 @@
-# Em See Pea Testing
+# `@emseepea/testing`
 
-This version only reserves the package name.
+Test what a Model Context Protocol (MCP) server returns and whether a language
+model understands it.
 
-Do not use version `0.0.0`. Install a later release instead.
+The package starts a Streamable HTTP server, connects through the official MCP
+client, and records the MCP operations used by each check. Semantic checks use
+Promptfoo and a model command supplied by the caller.
+
+The package is intended to publish under the `next` tag with the first
+pre-alpha release.
+
+## Add Semantic Checks to an Example
+
+Install the package as a development dependency:
+
+```sh
+npm install --save-dev @emseepea/testing@next
+```
+
+Create `eval.yaml` beside the example. Say which server to start, which MCP
+operation to run, and what a correct answer must mean:
+
+```yaml
+description: Coffee details keep each concept distinct
+server: ./dist/server.js
+operations:
+  - method: tools/call
+    name: get-bean-details
+    arguments:
+      name: Highland Bloom
+question: What are this coffee's origin, variety, process, roast, and tasting notes?
+criticalFacts: [Sample Highlands, Bourbon, natural, medium, berry, cocoa]
+criteria: Keep origin, variety, process, roast, and tasting notes distinct.
+```
+
+Build the example, then run:
+
+```sh
+npx emseepea-test eval.yaml
+```
+
+The check calls the server through the official MCP client. It runs three
+answers and three independent judgments per answer, and saves the evidence
+under `artifacts/llm-eval/`.

package/package.json

@@ -1,11 +1,46 @@
 {
   "name": "@emseepea/testing",
-  "version": "0.0.0",
-  "description": "Reserved package identity for Em See Pea testing helpers",
+  "version": "0.0.1",
+  "description": "MCP integration and semantic testing helpers",
   "license": "MIT",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/windyroad/emseepea.git",
     "directory": "packages/testing"
+  },
+  "homepage": "https://github.com/windyroad/emseepea#readme",
+  "bugs": "https://github.com/windyroad/emseepea/issues",
+  "publishConfig": {
+    "access": "public",
+    "provenance": true,
+    "tag": "next"
+  },
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "bin": {
+    "emseepea-test": "./semantic/cli.mjs"
+  },
+  "files": [
+    "dist",
+    "semantic",
+    "LICENSE",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "test": "npm run build && npm run test:built",
+    "test:built": "node --test test/*.test.mjs"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/client": "2.0.0",
+    "promptfoo": "0.122.1"
+  },
+  "engines": {
+    "node": ">=22"
   }
 }

package/semantic/case.mjs

@@ -0,0 +1,189 @@
+import { readFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const methods = new Set(["tools/call", "resources/read", "prompts/get"]);
+
+export async function loadSemanticCase(path) {
+  const absolutePath = path instanceof URL ? fileURLToPath(path) : resolve(path);
+  const value = parseCase(await readFile(absolutePath, "utf8"));
+  if (!value || typeof value !== "object" || Array.isArray(value)) fail("must be an object");
+  for (const key of ["description", "question", "criteria", "server"]) {
+    if (typeof value[key] !== "string" || value[key].trim() === "") fail(`${key} must be text`);
+  }
+  if (!Array.isArray(value.criticalFacts) || value.criticalFacts.some((fact) => typeof fact !== "string" || !fact)) {
+    fail("criticalFacts must be a non-empty text list");
+  }
+  if (!Array.isArray(value.operations) || value.operations.length === 0) fail("operations must not be empty");
+  for (const operation of value.operations) {
+    if (!operation || typeof operation !== "object" || !methods.has(operation.method)) {
+      fail("each operation needs a supported method");
+    }
+    const target = operation.name ?? operation.uri;
+    if (typeof target !== "string" || target === "") fail("each operation needs a name or uri");
+  }
+  if (value.environment !== undefined && (!value.environment || typeof value.environment !== "object" || Array.isArray(value.environment))) {
+    fail("environment must be an object");
+  }
+  if (value.authTokenEnvironment !== undefined && typeof value.authTokenEnvironment !== "string") {
+    fail("authTokenEnvironment must be text");
+  }
+  if (value.authToken !== undefined && typeof value.authToken !== "string") fail("authToken must be text");
+  return {
+    ...value,
+    path: absolutePath,
+    directory: dirname(absolutePath),
+    server: resolve(dirname(absolutePath), value.server),
+  };
+}
+
+export function requiredPaths(testCase) {
+  return testCase.operations.map((operation) => (
+    `${operation.method}:${operation.name ?? operation.uri}`
+  ));
+}
+
+function fail(message) {
+  throw new Error(`Invalid semantic case: ${message}`);
+}
+
+function parseCase(source) {
+  const lines = source.split(/\r?\n/);
+  const value = {};
+  for (let index = 0; index < lines.length;) {
+    const line = lines[index];
+    if (line.trim() === "" || line.trimStart().startsWith("#")) {
+      index += 1;
+      continue;
+    }
+    const top = /^([A-Za-z][A-Za-z0-9]*):(?:\s*(.*))?$/.exec(line);
+    if (!top) fail(`unsupported line: ${line}`);
+    const [, key, raw = ""] = top;
+    const rest = raw.trimEnd();
+    if (key === "operations") {
+      const parsed = parseOperations(lines, index + 1);
+      value.operations = parsed.value;
+      index = parsed.next;
+    } else if (rest === ">-") {
+      const parsed = parseFoldedText(lines, index + 1);
+      value[key] = parsed.value;
+      index = parsed.next;
+    } else if (key === "environment" && rest === "") {
+      const parsed = parseObject(lines, index + 1, "  ");
+      value.environment = parsed.value;
+      index = parsed.next;
+    } else if (rest === "") {
+      const parsed = parseList(lines, index + 1);
+      value[key] = parsed.value;
+      index = parsed.next;
+    } else if (rest.startsWith("[") && rest.endsWith("]")) {
+      value[key] = rest.slice(1, -1).split(",").map((item) => unquote(item.trim())).filter(Boolean);
+      index += 1;
+    } else {
+      value[key] = unquote(rest);
+      index += 1;
+    }
+  }
+  return value;
+}
+
+function parseOperations(lines, start) {
+  const operations = [];
+  let index = start;
+  while (index < lines.length) {
+    const line = lines[index];
+    if (line.trim() === "") {
+      index += 1;
+      continue;
+    }
+    if (!line.startsWith("  - ")) break;
+    const operation = {};
+    const first = line.slice(4);
+    applyPair(operation, first);
+    index += 1;
+    while (index < lines.length) {
+      const current = lines[index];
+      if (current.trim() === "") {
+        index += 1;
+        continue;
+      }
+      if (current.startsWith("  - ") || !current.startsWith("    ")) break;
+      const pair = /^    ([A-Za-z][A-Za-z0-9]*):(?:\s*(.*))?$/.exec(current);
+      if (!pair) fail(`unsupported operation line: ${current}`);
+      const [, key, raw = ""] = pair;
+      if (key === "arguments" && raw.trimEnd() === "") {
+        const parsed = parseObject(lines, index + 1, "      ", true);
+        operation.arguments = parsed.value;
+        index = parsed.next;
+      } else {
+        operation[key] = unquote(raw.trimEnd());
+        index += 1;
+      }
+    }
+    operations.push(operation);
+  }
+  return { value: operations, next: index };
+}
+
+function parseList(lines, start) {
+  const items = [];
+  let index = start;
+  while (index < lines.length) {
+    const item = /^  -\s+(.+)$/.exec(lines[index]);
+    if (!item) break;
+    items.push(unquote(item[1].trimEnd()));
+    index += 1;
+  }
+  if (items.length === 0) fail("expected a list");
+  return { value: items, next: index };
+}
+
+function parseObject(lines, start, indent, parseScalars = false) {
+  const object = {};
+  let index = start;
+  while (index < lines.length) {
+    if (!lines[index].startsWith(indent)) break;
+    applyPair(object, lines[index].slice(indent.length), parseScalars);
+    index += 1;
+  }
+  return { value: object, next: index };
+}
+
+function parseFoldedText(lines, start) {
+  const parts = [];
+  let index = start;
+  while (index < lines.length) {
+    if (!lines[index].startsWith("  ")) break;
+    parts.push(lines[index].trim());
+    index += 1;
+  }
+  return { value: parts.join(" "), next: index };
+}
+
+function applyPair(target, source, parseScalars = false) {
+  const pair = /^([A-Za-z][A-Za-z0-9_]*):\s*(.*)$/.exec(source);
+  if (!pair) fail(`expected key-value pair: ${source}`);
+  target[pair[1]] = parseScalars ? parseArgumentScalar(pair[2]) : unquote(pair[2].trimEnd());
+}
+
+function parseArgumentScalar(value) {
+  const trimmed = value.trim();
+  if ((trimmed.startsWith('"') && trimmed.endsWith('"')) || (trimmed.startsWith("'") && trimmed.endsWith("'"))) {
+    return unquote(trimmed);
+  }
+  if (trimmed === "true") return true;
+  if (trimmed === "false") return false;
+  if (/^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$/.test(trimmed)) {
+    const number = Number(trimmed);
+    if (Number.isFinite(number)) return number;
+  }
+  return trimmed;
+}
+
+function unquote(value) {
+  const trimmed = value.trim();
+  if ((trimmed.startsWith('"') && trimmed.endsWith('"')) || (trimmed.startsWith("'") && trimmed.endsWith("'"))) {
+    return trimmed.slice(1, -1);
+  }
+  return trimmed;
+}

package/semantic/cli.mjs

@@ -0,0 +1,394 @@
+#!/usr/bin/env node
+
+import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { loadSemanticCase, requiredPaths } from "./case.mjs";
+import { collectMcpMaterial, startSemanticServer, stopSemanticServer } from "./material.mjs";
+import { parseJudgeVerdict, runModel } from "./provider.mjs";
+
+const options = parseArguments(process.argv.slice(2));
+const outputPath = resolve(options.output ?? "artifacts/llm-eval/evidence.json");
+const providerPath = fileURLToPath(new URL("./provider.mjs", import.meta.url));
+const promptfooPath = join(dirname(fileURLToPath(import.meta.resolve("promptfoo"))), "entrypoint.js");
+const startedAt = new Date().toISOString();
+const evidence = {
+  authoritative: options.provider === "claude-ci",
+  cases: {},
+  errors: [],
+  finishedAt: undefined,
+  model: "claude-sonnet-4-6",
+  provider: options.provider,
+  semanticRetries: 0,
+  startedAt,
+  status: "failed",
+};
+
+await mkdir(dirname(outputPath), { recursive: true });
+for (const path of options.casePaths) {
+  try {
+    const result = options.smoke ? await runSmokeCase(path) : await runSemanticCase(path);
+    evidence.cases[result.path] = result.evidence;
+  } catch (error) {
+    evidence.errors.push(error instanceof Error ? error.message : String(error));
+  }
+}
+evidence.finishedAt = new Date().toISOString();
+evidence.status = evidence.errors.length === 0 ? "passed" : "failed";
+await writeFile(outputPath, `${JSON.stringify(evidence, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
+
+if (evidence.errors.length > 0) {
+  process.stderr.write(`Semantic evaluation failed; evidence: ${outputPath}\n`);
+  process.exitCode = 1;
+} else {
+  process.stdout.write(`Semantic evaluation passed for ${options.casePaths.length} case(s); evidence: ${outputPath}\n`);
+}
+
+async function runSemanticCase(path) {
+  const testCase = await loadSemanticCase(path);
+  const directory = await mkdtemp(join(tmpdir(), "emseepea-semantic-"));
+  const configPath = join(directory, "promptfoo.json");
+  const promptfooOutput = join(directory, "promptfoo-output.json");
+  const providerEvidence = join(directory, "provider.jsonl");
+  const config = promptfooConfig(testCase);
+  await writeFile(configPath, JSON.stringify(config), { encoding: "utf8", mode: 0o600 });
+  try {
+    const run = await runProcess(process.execPath, [
+      promptfooPath,
+      "eval",
+      "--config", configPath,
+      "--repeat", "3",
+      "--max-concurrency", "1",
+      "--no-cache",
+      "--no-share",
+      "--no-write",
+      "--no-table",
+      "--no-progress-bar",
+      "--output", promptfooOutput,
+    ], {
+      cwd: directory,
+      env: cleanEnvironment({
+        CLAUDE_CODE_OAUTH_TOKEN: options.provider === "claude-ci"
+          ? process.env.CLAUDE_CODE_OAUTH_TOKEN
+          : undefined,
+        EMSEEPEA_EVAL_PROVIDER: options.provider,
+        EMSEEPEA_EVAL_PROVIDER_EVIDENCE: providerEvidence,
+        EMSEEPEA_MODEL_COMMAND: options.modelCommand,
+        PROMPTFOO_CONFIG_DIR: directory,
+      }),
+      timeoutMs: 38 * 60_000,
+    });
+    if (run.timedOut) throw new Error(`${testCase.description}: Promptfoo timed out`);
+    const records = readJsonLines(await readFile(providerEvidence, "utf8"));
+    const entries = promptfooEntries(JSON.parse(await readFile(promptfooOutput, "utf8")));
+    if (run.code !== 0) {
+      if (process.env.EMSEEPEA_DEBUG === "1") process.stderr.write(run.stderr.slice(-4_000));
+      throw new Error(`${testCase.description}: Promptfoo exited ${run.code}`);
+    }
+    verifyProviderRecords(testCase, records);
+    if (entries.length !== 3 || entries.some(({ success }) => success !== true)) {
+      throw new Error(`${testCase.description}: ${entries.filter(({ success }) => success).length}/${entries.length} trials passed`);
+    }
+    return {
+      path: testCase.path,
+      evidence: {
+        agentTrials: records.filter(({ role }) => role === "agent"),
+        configSha256: createHash("sha256").update(JSON.stringify(config)).digest("hex"),
+        judgeVerdicts: records.filter(({ role }) => role === "judge"),
+        promptfooTrials: entries.map(({ description, score, success }) => ({ description, score, success })),
+      },
+    };
+  } finally {
+    await rm(directory, { force: true, recursive: true });
+  }
+}
+
+function promptfooConfig(testCase) {
+  const judge = {
+    id: `file://${providerPath}`,
+    config: { maxRetries: 0, role: "judge" },
+  };
+  const rubric = [
+    "Judge the meaning of this answer against every criterion below.",
+    "A pass requires every criterion. Ignore harmless wording differences.",
+    "",
+    "Criteria:",
+    "{{criteria}}",
+    "",
+    "Answer:",
+    "<answer>",
+    "{{output}}",
+    "</answer>",
+    "",
+    "Return only JSON with this exact shape:",
+    '{"pass": true or false, "score": 1 or 0, "reason": "one concise sentence"}',
+  ].join("\n");
+  const judgeAssertion = () => ({
+    type: "llm-rubric",
+    value: testCase.criteria,
+    provider: judge,
+    rubricPrompt: rubric,
+  });
+  return {
+    description: testCase.description,
+    providers: [{
+      id: `file://${providerPath}`,
+      config: { maxRetries: 0, role: "agent" },
+    }],
+    prompts: ["{{question}}"],
+    defaultTest: {
+      options: {
+        timeoutMs: 120_000,
+      },
+      assert: [
+        {
+          type: "javascript",
+          value: [
+            "const actual = context.metadata?.pathEvidence ?? [];",
+            "const seen = new Set(actual.map(({ method, target }) => `${method}:${target}`));",
+            "const missing = JSON.parse(context.vars.requiredPaths).filter((path) => !seen.has(path));",
+            "return missing.length === 0",
+            "  ? { pass: true, score: 1, reason: 'required MCP path evidence recorded' }",
+            "  : { pass: false, score: 0, reason: `missing MCP path evidence: ${missing.join(', ')}` };",
+          ].join("\n"),
+        },
+        {
+          type: "javascript",
+          value: [
+            "const answer = String(output).toLowerCase();",
+            "const missing = JSON.parse(context.vars.criticalFacts)",
+            "  .filter((fact) => !answer.includes(fact.toLowerCase()));",
+            "return missing.length === 0",
+            "  ? { pass: true, score: 1, reason: 'critical facts present' }",
+            "  : { pass: false, score: 0, reason: `missing critical facts: ${missing.join(', ')}` };",
+          ].join("\n"),
+        },
+        judgeAssertion(),
+        judgeAssertion(),
+        judgeAssertion(),
+      ],
+    },
+    tests: [{
+      description: testCase.description,
+      vars: {
+        casePath: testCase.path,
+        criteria: testCase.criteria,
+        criticalFacts: JSON.stringify(testCase.criticalFacts),
+        question: testCase.question,
+        requiredPaths: JSON.stringify(requiredPaths(testCase)),
+      },
+    }],
+  };
+}
+
+async function runSmokeCase(path) {
+  const testCase = await loadSemanticCase(path);
+  const records = [];
+  const previousModelCommand = process.env.EMSEEPEA_MODEL_COMMAND;
+  process.env.EMSEEPEA_MODEL_COMMAND = options.modelCommand;
+  try {
+    for (let trial = 1; trial <= 3; trial += 1) {
+      const directory = await mkdtemp(join(tmpdir(), "emseepea-agent-"));
+      let running;
+      try {
+        running = await startSemanticServer(testCase);
+        const material = await collectMcpMaterial(running.url, testCase);
+        const prompt = `${material.text}\n\nAnswer only from that MCP material.\n\nQuestion:\n${testCase.question}`;
+        const result = await runModel(options.provider, prompt, directory);
+        const agentRecord = {
+          casePath: testCase.path,
+          materialSha256: createHash("sha256").update(material.text).digest("hex"),
+          models: result.models,
+          pathEvidence: material.pathEvidence,
+          provider: options.provider,
+          role: "agent",
+          status: "completed",
+          toolCallCount: 0,
+          trial,
+          turnCount: result.turnCount,
+        };
+        records.push(agentRecord);
+        const answer = result.answer;
+        for (let judgment = 1; judgment <= 3; judgment += 1) {
+          const judgeDirectory = await mkdtemp(join(tmpdir(), "emseepea-judge-"));
+          try {
+            const judgeResult = await runModel(options.provider, judgePrompt(testCase, answer), judgeDirectory);
+            records.push({
+              casePath: testCase.path,
+              models: judgeResult.models,
+              provider: options.provider,
+              role: "judge",
+              status: "completed",
+              trial: ((trial - 1) * 3) + judgment,
+              turnCount: judgeResult.turnCount,
+              verdict: parseJudgeVerdict(judgeResult.answer.trim()),
+            });
+          } finally {
+            await rm(judgeDirectory, { force: true, recursive: true });
+          }
+        }
+      } finally {
+        if (running) await stopSemanticServer(running.child);
+        await rm(directory, { force: true, recursive: true });
+      }
+    }
+  } finally {
+    if (previousModelCommand === undefined) delete process.env.EMSEEPEA_MODEL_COMMAND;
+    else process.env.EMSEEPEA_MODEL_COMMAND = previousModelCommand;
+  }
+  verifyProviderRecords(testCase, records);
+  return {
+    path: testCase.path,
+    evidence: {
+      agentTrials: records.filter(({ role }) => role === "agent"),
+      configSha256: createHash("sha256").update(JSON.stringify(promptfooConfig(testCase))).digest("hex"),
+      judgeVerdicts: records.filter(({ role }) => role === "judge"),
+      promptfooTrials: [{ description: testCase.description, score: 1, success: true }],
+      smoke: true,
+    },
+  };
+}
+
+function judgePrompt(testCase, answer) {
+  return [
+    "Judge the meaning of this answer against every criterion below.",
+    "A pass requires every criterion. Ignore harmless wording differences.",
+    "",
+    "Criteria:",
+    testCase.criteria,
+    "",
+    "Answer:",
+    "<answer>",
+    answer,
+    "</answer>",
+    "",
+    "Return only JSON with this exact shape:",
+    '{"pass": true or false, "score": 1 or 0, "reason": "one concise sentence"}',
+  ].join("\n");
+}
+
+function verifyProviderRecords(testCase, records) {
+  const agents = records.filter(({ role }) => role === "agent");
+  const judges = records.filter(({ role }) => role === "judge");
+  if (agents.length !== 3 || judges.length !== 9) {
+    throw new Error(`${testCase.description}: expected 3 answers and 9 judgments; got ${agents.length} and ${judges.length}`);
+  }
+  const required = requiredPaths(testCase);
+  if (agents.some(({ status, toolCallCount, materialSha256, pathEvidence, turnCount }) => (
+    status !== "completed"
+    || toolCallCount !== 0
+    || turnCount !== 1
+    || !/^[a-f0-9]{64}$/.test(materialSha256 ?? "")
+    || !Array.isArray(pathEvidence)
+    || required.some((path) => !pathEvidence.some(({ method, target, requestSha256, responseSha256 }) => (
+      `${method}:${target}` === path
+      && /^[a-f0-9]{64}$/.test(requestSha256 ?? "")
+      && /^[a-f0-9]{64}$/.test(responseSha256 ?? "")
+    )))
+  ))) throw new Error(`${testCase.description}: answer evidence is incomplete`);
+  if (judges.some(({ status, verdict, turnCount }) => (
+    status !== "completed" || turnCount !== 1 || verdict?.pass !== true
+  ))) throw new Error(`${testCase.description}: a judgment failed`);
+}
+
+function parseArguments(args) {
+  const result = { casePaths: [], modelCommand: "claude", output: undefined, provider: "claude-local", smoke: false };
+  for (let index = 0; index < args.length; index += 1) {
+    const value = args[index];
+    if (value === "--provider") result.provider = args[++index];
+    else if (value === "--model-command") result.modelCommand = args[++index];
+    else if (value === "--output") result.output = args[++index];
+    else if (value === "--smoke") result.smoke = true;
+    else if (value?.startsWith("--")) throw new Error(`Unknown option: ${value}`);
+    else result.casePaths.push(value);
+  }
+  if (!new Set(["claude-local", "claude-ci"]).has(result.provider)) {
+    throw new Error(`Unsupported provider: ${result.provider}`);
+  }
+  if (result.casePaths.length === 0) throw new Error("Pass at least one eval.yaml path");
+  if (result.modelCommand.includes("/") || result.modelCommand.includes("\\")) {
+    result.modelCommand = resolve(result.modelCommand);
+  }
+  return result;
+}
+
+function cleanEnvironment(extra) {
+  return Object.fromEntries(Object.entries({
+    CI: "true",
+    HOME: process.env.HOME,
+    LANG: process.env.LANG ?? "C.UTF-8",
+    LOGNAME: process.env.LOGNAME,
+    NO_COLOR: "1",
+    PATH: process.env.PATH,
+    PROMPTFOO_CACHE_ENABLED: "false",
+    PROMPTFOO_DISABLE_REMOTE_GENERATION: "true",
+    PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION: "true",
+    PROMPTFOO_DISABLE_SHARING: "true",
+    PROMPTFOO_DISABLE_TELEMETRY: "true",
+    PROMPTFOO_DISABLE_TEMPLATE_ENV_VARS: "true",
+    PROMPTFOO_DISABLE_UPDATE: "true",
+    PROMPTFOO_SELF_HOSTED: "true",
+    SHELL: process.env.SHELL,
+    TMPDIR: process.env.TMPDIR,
+    USER: process.env.USER,
+    ...extra,
+  }).filter(([, value]) => value !== undefined));
+}
+
+function runProcess(command, args, options = {}) {
+  const { timeoutMs, ...spawnOptions } = options;
+  return new Promise((resolve) => {
+    const grouped = process.platform !== "win32";
+    const child = spawn(command, args, { detached: grouped, stdio: ["ignore", "pipe", "pipe"], ...spawnOptions });
+    let stdout = "";
+    let stderr = "";
+    let timedOut = false;
+    const stop = () => {
+      try {
+        if (grouped) process.kill(-child.pid, "SIGKILL");
+        else child.kill("SIGKILL");
+      } catch (error) {
+        if (error?.code !== "ESRCH") throw error;
+      }
+    };
+    const interrupted = () => stop();
+    process.once("SIGINT", interrupted);
+    process.once("SIGTERM", interrupted);
+    const timer = setTimeout(() => {
+      timedOut = true;
+      stop();
+    }, timeoutMs);
+    child.stdout.on("data", (chunk) => { stdout += chunk; });
+    child.stderr.on("data", (chunk) => { stderr += chunk; });
+    child.once("error", (error) => {
+      clearTimeout(timer);
+      process.off("SIGINT", interrupted);
+      process.off("SIGTERM", interrupted);
+      resolve({ code: 1, error: error.message, stderr, stdout, timedOut });
+    });
+    child.once("close", (code) => {
+      clearTimeout(timer);
+      process.off("SIGINT", interrupted);
+      process.off("SIGTERM", interrupted);
+      resolve({ code: code ?? 1, stderr, stdout, timedOut });
+    });
+  });
+}
+
+function readJsonLines(source) {
+  return source.split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line));
+}
+
+function promptfooEntries(value, found = []) {
+  if (Array.isArray(value)) {
+    if (value.every((entry) => entry && typeof entry === "object" && "success" in entry)) found.push(...value);
+    else for (const entry of value) promptfooEntries(entry, found);
+  } else if (value && typeof value === "object") {
+    for (const entry of Object.values(value)) promptfooEntries(entry, found);
+  }
+  return found;
+}

package/semantic/material.mjs

@@ -0,0 +1,130 @@
+import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+
+import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
+
+const sha256 = (value) => createHash("sha256").update(JSON.stringify(value)).digest("hex");
+
+export async function startSemanticServer(testCase) {
+  const child = spawn(process.execPath, [testCase.server], {
+    cwd: testCase.directory,
+    env: serverEnvironment(testCase.environment),
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+  let output = "";
+  child.stdout.on("data", (chunk) => { output = `${output}${chunk}`.slice(-16_384); });
+  try {
+    const url = await new Promise((resolve, reject) => {
+      const finish = (error, value) => {
+        clearTimeout(timer);
+        child.stdout.off("data", inspect);
+        child.off("error", failed);
+        child.off("exit", exited);
+        if (error) reject(error);
+        else resolve(value);
+      };
+      const inspect = () => {
+        const match = output.match(/http:\/\/127\.0\.0\.1:\d+\/mcp/);
+        if (match?.[0]) finish(undefined, match[0]);
+      };
+      const failed = () => finish(new Error("MCP server did not start"));
+      const exited = () => finish(new Error("MCP server stopped during startup"));
+      const timer = setTimeout(() => finish(new Error("MCP server startup timed out")), 15_000);
+      child.stdout.on("data", inspect);
+      child.once("error", failed);
+      child.once("exit", exited);
+      inspect();
+    });
+    return { child, url };
+  } catch (error) {
+    await stopSemanticServer(child);
+    throw error;
+  }
+}
+
+export async function stopSemanticServer(child) {
+  if (child.exitCode !== null) return;
+  child.kill("SIGTERM");
+  await Promise.race([
+    new Promise((resolve) => child.once("close", resolve)),
+    new Promise((resolve) => setTimeout(resolve, 3_000)),
+  ]);
+  if (child.exitCode === null) child.kill("SIGKILL");
+}
+
+export async function collectMcpMaterial(url, testCase) {
+  const token = testCase.authToken ?? (testCase.authTokenEnvironment
+    ? process.env[testCase.authTokenEnvironment]?.trim()
+    : undefined);
+  if (testCase.authTokenEnvironment && !token) {
+    throw new Error(`Required authentication is unavailable: ${testCase.authTokenEnvironment}`);
+  }
+  const client = new Client(
+    { name: "emseepea-semantic-test", version: "0.0.0" },
+    { versionNegotiation: { mode: { pin: "2026-07-28" } } },
+  );
+  await client.connect(new StreamableHTTPClientTransport(
+    new URL(url),
+    token ? { authProvider: { token: async () => token } } : undefined,
+  ));
+  const evidence = [];
+  const material = [];
+  try {
+    for (const operation of testCase.operations) {
+      const request = requestFor(operation);
+      const response = await perform(client, operation);
+      evidence.push({
+        method: operation.method,
+        target: operation.name ?? operation.uri,
+        requestSha256: sha256(request),
+        responseSha256: sha256(response),
+      });
+      material.push(`Operation: ${JSON.stringify(request)}\nResult: ${JSON.stringify(response)}`);
+    }
+  } finally {
+    await client.close();
+  }
+  return {
+    pathEvidence: evidence,
+    text: ["The following material was retrieved through the official MCP client.", ...material].join("\n\n"),
+  };
+}
+
+function requestFor(operation) {
+  if (operation.method === "tools/call") {
+    return { method: operation.method, name: operation.name, arguments: operation.arguments ?? {} };
+  }
+  if (operation.method === "resources/read") return { method: operation.method, uri: operation.uri };
+  return { method: operation.method, name: operation.name, arguments: operation.arguments ?? {} };
+}
+
+async function perform(client, operation) {
+  if (operation.method === "tools/call") {
+    const progress = [];
+    const result = await client.callTool(
+      { name: operation.name, arguments: operation.arguments ?? {} },
+      { onprogress: (update) => progress.push(update) },
+    );
+    if (result.isError) throw new Error("MCP tool returned an error");
+    return { progress, result };
+  }
+  if (operation.method === "resources/read") return client.readResource({ uri: operation.uri });
+  return client.getPrompt({ name: operation.name, arguments: operation.arguments ?? {} });
+}
+
+function serverEnvironment(extra = {}) {
+  return Object.fromEntries(Object.entries({
+    CI: "true",
+    HOME: process.env.HOME,
+    LANG: process.env.LANG ?? "C.UTF-8",
+    LOGNAME: process.env.LOGNAME,
+    NODE_ENV: "test",
+    NO_COLOR: "1",
+    PATH: process.env.PATH,
+    PORT: "0",
+    SHELL: process.env.SHELL,
+    TMPDIR: process.env.TMPDIR,
+    USER: process.env.USER,
+    ...extra,
+  }).filter(([, value]) => value !== undefined));
+}

package/semantic/provider.mjs

@@ -0,0 +1,190 @@
+import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
+import { appendFile, mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { loadSemanticCase } from "./case.mjs";
+import { collectMcpMaterial, startSemanticServer, stopSemanticServer } from "./material.mjs";
+
+const model = "claude-sonnet-4-6";
+const counters = new Map();
+
+export function parseClaudeEvents(stdout, processExitCode = 0) {
+  const events = stdout.split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line));
+  const result = events.findLast(({ type }) => type === "result");
+  const notLoggedIn = events.some(({ message }) => (
+    Array.isArray(message?.content)
+    && message.content.some(({ type, text }) => type === "text" && /not logged in/i.test(text ?? ""))
+  ));
+  const toolUses = events.flatMap(({ message }) => (
+    Array.isArray(message?.content) ? message.content.filter(({ type }) => type === "tool_use") : []
+  ));
+  if (notLoggedIn) throw new Error("Model command is not signed in");
+  if (processExitCode !== 0) throw new Error(`Model command exited ${processExitCode}`);
+  if (result?.is_error || typeof result?.result !== "string") throw new Error("Model command returned no answer");
+  if (toolUses.length > 0) throw new Error("Model command used a forbidden tool");
+  if (result.num_turns !== 1) throw new Error(`Model command used ${String(result.num_turns)} turns`);
+  if ((result.permission_denials?.length ?? 0) > 0) throw new Error("Model command attempted a forbidden action");
+  const usage = result.modelUsage?.[model];
+  if (usage?.canonicalModel !== model || usage.provider !== "firstParty") {
+    throw new Error("Model command did not use the required model");
+  }
+  return { answer: result.result, models: Object.keys(result.modelUsage), turnCount: result.num_turns };
+}
+
+export function parseJudgeVerdict(output) {
+  const verdict = JSON.parse(output);
+  const keys = verdict && typeof verdict === "object" && !Array.isArray(verdict)
+    ? Object.keys(verdict).sort()
+    : [];
+  if (
+    keys.join(",") !== "pass,reason,score"
+    || !((verdict.pass === true && verdict.score === 1) || (verdict.pass === false && verdict.score === 0))
+    || typeof verdict.reason !== "string"
+    || verdict.reason.trim() === ""
+  ) throw new Error("Judge returned an invalid verdict");
+  return verdict;
+}
+
+export function modelInvocation(provider, prompt, directory) {
+  const token = provider === "claude-ci" ? process.env.CLAUDE_CODE_OAUTH_TOKEN?.trim() : undefined;
+  if (provider === "claude-ci" && !token) throw new Error("Claude subscription authentication is unavailable");
+  const localHome = provider === "claude-local" ? process.env.HOME : undefined;
+  if (provider === "claude-local" && !localHome?.startsWith("/")) {
+    throw new Error("Local model evaluation requires an absolute HOME");
+  }
+  return {
+    command: process.env.EMSEEPEA_MODEL_COMMAND ?? "claude",
+    args: [
+      "--print", prompt,
+      "--model", model,
+      "--effort", "low",
+      "--safe-mode",
+      "--strict-mcp-config",
+      "--disable-slash-commands",
+      "--no-session-persistence",
+      "--permission-mode", "dontAsk",
+      "--setting-sources", "",
+      "--tools", "",
+      "--no-chrome",
+      "--prompt-suggestions", "false",
+      "--output-format", "stream-json",
+      "--verbose",
+    ],
+    cwd: directory,
+    env: modelEnvironment(provider === "claude-ci"
+      ? { CLAUDE_CONFIG_DIR: join(directory, "claude-config"), HOME: directory, CLAUDE_CODE_OAUTH_TOKEN: token }
+      : { HOME: localHome }),
+  };
+}
+
+export default class SemanticProvider {
+  constructor(options = {}) {
+    this.role = options.config?.role ?? "agent";
+    this.provider = process.env.EMSEEPEA_EVAL_PROVIDER ?? "claude-local";
+  }
+
+  id() {
+    return `${this.provider}-${this.role}`;
+  }
+
+  async callApi(prompt, context = {}) {
+    const casePath = String(context.vars?.casePath ?? "");
+    const trial = nextTrial(`${casePath}:${this.role}`);
+    const directory = await mkdtemp(join(tmpdir(), `emseepea-${this.role}-`));
+    let running;
+    try {
+      let effectivePrompt = prompt;
+      let pathEvidence = [];
+      let materialSha256;
+      if (this.role === "agent") {
+        const testCase = await loadSemanticCase(casePath);
+        running = await startSemanticServer(testCase);
+        const material = await collectMcpMaterial(running.url, testCase);
+        effectivePrompt = `${material.text}\n\nAnswer only from that MCP material.\n\nQuestion:\n${prompt}`;
+        pathEvidence = material.pathEvidence;
+        materialSha256 = createHash("sha256").update(material.text).digest("hex");
+      }
+      const result = await runModel(this.provider, effectivePrompt, directory);
+      const output = this.role === "judge" ? result.answer.trim() : result.answer;
+      const verdict = this.role === "judge" ? parseJudgeVerdict(output) : undefined;
+      const metadata = {
+        casePath,
+        materialSha256,
+        models: result.models,
+        pathEvidence,
+        provider: this.provider,
+        role: this.role,
+        toolCallCount: 0,
+        trial,
+        turnCount: result.turnCount,
+      };
+      await recordEvidence({ ...metadata, status: "completed", verdict });
+      return { output, metadata };
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      await recordEvidence({ casePath, error: message, provider: this.provider, role: this.role, status: "error", trial });
+      return { error: message };
+    } finally {
+      if (running) await stopSemanticServer(running.child);
+      await rm(directory, { force: true, recursive: true });
+    }
+  }
+}
+
+export async function runModel(provider, prompt, directory) {
+  const invocation = modelInvocation(provider, prompt, directory);
+  const execution = await runProcess(invocation.command, invocation.args, {
+    cwd: invocation.cwd,
+    env: invocation.env,
+  });
+  if (execution.timedOut) throw new Error("Model command timed out");
+  if (execution.code !== 0 && !execution.stdout) throw new Error(`Model command exited ${execution.code}`);
+  return parseClaudeEvents(execution.stdout, execution.code);
+}
+
+function runProcess(command, args, options) {
+  return new Promise((resolve) => {
+    const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"], ...options });
+    let stdout = "";
+    const timer = setTimeout(() => child.kill("SIGKILL"), 120_000);
+    let timedOut = false;
+    timer.unref();
+    child.stdout.on("data", (chunk) => { stdout += chunk; });
+    child.once("error", (error) => {
+      clearTimeout(timer);
+      resolve({ code: 1, error: error.message, stdout, timedOut });
+    });
+    child.once("close", (code, signal) => {
+      if (signal === "SIGKILL") timedOut = true;
+      clearTimeout(timer);
+      resolve({ code: code ?? 1, stdout, timedOut });
+    });
+  });
+}
+
+function modelEnvironment(extra) {
+  return Object.fromEntries(Object.entries({
+    CI: "true",
+    LANG: process.env.LANG ?? "C.UTF-8",
+    LOGNAME: process.env.LOGNAME,
+    NO_COLOR: "1",
+    PATH: process.env.PATH,
+    SHELL: process.env.SHELL,
+    TMPDIR: process.env.TMPDIR,
+    USER: process.env.USER,
+    ...extra,
+  }).filter(([, value]) => value !== undefined));
+}
+
+function nextTrial(key) {
+  const trial = (counters.get(key) ?? 0) + 1;
+  counters.set(key, trial);
+  return trial;
+}
+
+async function recordEvidence(record) {
+  const path = process.env.EMSEEPEA_EVAL_PROVIDER_EVIDENCE;
+  if (path) await appendFile(path, `${JSON.stringify(record)}\n`, { encoding: "utf8", mode: 0o600 });
+}