@emseepea/testing 0.0.2 → 0.2.0

package/README.md

@@ -1,45 +1,15 @@
 # `@emseepea/testing`
 
-Test what a Model Context Protocol (MCP) server returns and whether a language
-model understands it.
+Test whether an AI selects the right MCP tool and understands its result, not
+just whether the server returns valid data. Write JavaScript tests with ordinary
+imports, setup hooks, loops, and assertions.
 
-The package starts a Streamable HTTP server, connects through the official MCP
-client, and records the MCP operations used by each check. Semantic checks use
-Promptfoo and a model command supplied by the caller.
+Read the [guide to testing AI tool choice and understanding](https://github.com/windyroad/emseepea/blob/main/website/src/content/docs/ai-tests.md)
+for setup, test fields, commands, and what a passing check proves.
 
-The package is intended to publish under the `next` tag with the first
-pre-alpha release.
+Start from the [coffee tool-selection test](https://github.com/windyroad/emseepea/blob/main/examples/basic-no-ui/eval/meaning.test.mjs).
+Keep these tests in `eval/`, separate from ordinary tests in `test/`.
 
-## Add Semantic Checks to an Example
-
-Install the package as a development dependency:
-
-```sh
-npm install --save-dev @emseepea/testing@next
-```
-
-Create `eval.yaml` beside the example. Say which server to start, which MCP
-operation to run, and what a correct answer must mean:
-
-```yaml
-description: Coffee details keep each concept distinct
-server: ./dist/server.js
-operations:
-  - method: tools/call
-    name: get-bean-details
-    arguments:
-      name: Highland Bloom
-question: What are this coffee's origin, variety, process, roast, and tasting notes?
-criticalFacts: [Sample Highlands, Bourbon, natural, medium, berry, cocoa]
-criteria: Keep origin, variety, process, roast, and tasting notes distinct.
-```
-
-Build the example, then run:
-
-```sh
-npx emseepea-test eval.yaml
-```
-
-The check calls the server through the official MCP client. It runs three
-answers and three independent judgments per answer, and saves the evidence
-under `artifacts/llm-eval/`.
+Use `toolSelectionTest` when the AI should choose a tool. Use `semanticTest`
+when test code deliberately prepares MCP resources, prompts, or several results
+for the AI to interpret.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@emseepea/testing",
-  "version": "0.0.2",
+  "version": "0.2.0",
   "description": "MCP integration and semantic testing helpers",
   "license": "MIT",
   "repository": {
@@ -20,6 +20,10 @@
     ".": {
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js"
+    },
+    "./semantic": {
+      "types": "./semantic/test.d.mts",
+      "import": "./semantic/test.mjs"
     }
   },
   "bin": {
@@ -38,8 +42,7 @@
     "test:built": "node --test test/*.test.mjs"
   },
   "dependencies": {
-    "@modelcontextprotocol/client": "2.0.0",
-    "promptfoo": "0.122.1"
+    "@modelcontextprotocol/client": "2.0.0"
   },
   "engines": {
     "node": ">=22"

package/semantic/case.mjs

@@ -1,189 +1,87 @@
-import { readFile } from "node:fs/promises";
-import { dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
+import { dirname } from "node:path";
 
-const methods = new Set(["tools/call", "resources/read", "prompts/get"]);
-
-export async function loadSemanticCase(path) {
-  const absolutePath = path instanceof URL ? fileURLToPath(path) : resolve(path);
-  const value = parseCase(await readFile(absolutePath, "utf8"));
-  if (!value || typeof value !== "object" || Array.isArray(value)) fail("must be an object");
-  for (const key of ["description", "question", "criteria", "server"]) {
-    if (typeof value[key] !== "string" || value[key].trim() === "") fail(`${key} must be text`);
-  }
-  if (!Array.isArray(value.criticalFacts) || value.criticalFacts.some((fact) => typeof fact !== "string" || !fact)) {
-    fail("criticalFacts must be a non-empty text list");
-  }
-  if (!Array.isArray(value.operations) || value.operations.length === 0) fail("operations must not be empty");
-  for (const operation of value.operations) {
-    if (!operation || typeof operation !== "object" || !methods.has(operation.method)) {
-      fail("each operation needs a supported method");
-    }
-    const target = operation.name ?? operation.uri;
-    if (typeof target !== "string" || target === "") fail("each operation needs a name or uri");
+export function validateSemanticCase(value) {
+  const result = validateCommon(value);
+  if (typeof value.exercise !== "function") throw new Error("exercise must be a function");
+  if (!Array.isArray(value.requiredPaths) || !value.requiredPaths.length || value.requiredPaths.some((path) =>
+    typeof path !== "string" || !/^(tools\/call|resources\/read|prompts\/get):.+$/.test(path))) {
+    throw new Error("requiredPaths must name a supported MCP method and target");
   }
-  if (value.environment !== undefined && (!value.environment || typeof value.environment !== "object" || Array.isArray(value.environment))) {
-    fail("environment must be an object");
+  return result;
+}
+
+export function validateToolSelectionCase(value) {
+  const result = validateCommon(value);
+  if (!Array.isArray(value.expectedTools) || value.expectedTools.length < 1 || value.expectedTools.length > 3
+    || value.expectedTools.some((name) => typeof name !== "string" || !name.trim())) {
+    throw new Error("expectedTools must name between one and three tool calls");
   }
-  if (value.authTokenEnvironment !== undefined && typeof value.authTokenEnvironment !== "string") {
-    fail("authTokenEnvironment must be text");
+  if (value.exercise !== undefined || value.requiredPaths !== undefined) {
+    throw new Error("toolSelectionTest chooses calls from expectedTools; do not provide exercise or requiredPaths");
   }
-  if (value.authToken !== undefined && typeof value.authToken !== "string") fail("authToken must be text");
   return {
-    ...value,
-    path: absolutePath,
-    directory: dirname(absolutePath),
-    server: resolve(dirname(absolutePath), value.server),
+    ...result,
+    expectedTools: [...value.expectedTools],
+    requiredPaths: [...new Set(value.expectedTools.map((name) => `tools/call:${name}`))],
   };
 }
 
-export function requiredPaths(testCase) {
-  return testCase.operations.map((operation) => (
-    `${operation.method}:${operation.name ?? operation.uri}`
-  ));
-}
-
-function fail(message) {
-  throw new Error(`Invalid semantic case: ${message}`);
-}
-
-function parseCase(source) {
-  const lines = source.split(/\r?\n/);
-  const value = {};
-  for (let index = 0; index < lines.length;) {
-    const line = lines[index];
-    if (line.trim() === "" || line.trimStart().startsWith("#")) {
-      index += 1;
-      continue;
-    }
-    const top = /^([A-Za-z][A-Za-z0-9]*):(?:\s*(.*))?$/.exec(line);
-    if (!top) fail(`unsupported line: ${line}`);
-    const [, key, raw = ""] = top;
-    const rest = raw.trimEnd();
-    if (key === "operations") {
-      const parsed = parseOperations(lines, index + 1);
-      value.operations = parsed.value;
-      index = parsed.next;
-    } else if (rest === ">-") {
-      const parsed = parseFoldedText(lines, index + 1);
-      value[key] = parsed.value;
-      index = parsed.next;
-    } else if (key === "environment" && rest === "") {
-      const parsed = parseObject(lines, index + 1, "  ");
-      value.environment = parsed.value;
-      index = parsed.next;
-    } else if (rest === "") {
-      const parsed = parseList(lines, index + 1);
-      value[key] = parsed.value;
-      index = parsed.next;
-    } else if (rest.startsWith("[") && rest.endsWith("]")) {
-      value[key] = rest.slice(1, -1).split(",").map((item) => unquote(item.trim())).filter(Boolean);
-      index += 1;
-    } else {
-      value[key] = unquote(rest);
-      index += 1;
-    }
+export function parseToolSelection(output, advertisedTools, expectedTools) {
+  let plan;
+  try { plan = JSON.parse(output); } catch { throw new Error("Tool selection must be valid JSON"); }
+  if (!plan || typeof plan !== "object" || Array.isArray(plan) || Object.keys(plan).join(",") !== "calls"
+    || !Array.isArray(plan.calls) || plan.calls.length < 1 || plan.calls.length > 3) {
+    throw new Error("Tool selection must contain between one and three calls");
   }
-  return value;
-}
-
-function parseOperations(lines, start) {
-  const operations = [];
-  let index = start;
-  while (index < lines.length) {
-    const line = lines[index];
-    if (line.trim() === "") {
-      index += 1;
-      continue;
-    }
-    if (!line.startsWith("  - ")) break;
-    const operation = {};
-    const first = line.slice(4);
-    applyPair(operation, first);
-    index += 1;
-    while (index < lines.length) {
-      const current = lines[index];
-      if (current.trim() === "") {
-        index += 1;
-        continue;
-      }
-      if (current.startsWith("  - ") || !current.startsWith("    ")) break;
-      const pair = /^    ([A-Za-z][A-Za-z0-9]*):(?:\s*(.*))?$/.exec(current);
-      if (!pair) fail(`unsupported operation line: ${current}`);
-      const [, key, raw = ""] = pair;
-      if (key === "arguments" && raw.trimEnd() === "") {
-        const parsed = parseObject(lines, index + 1, "      ", true);
-        operation.arguments = parsed.value;
-        index = parsed.next;
-      } else {
-        operation[key] = unquote(raw.trimEnd());
-        index += 1;
-      }
+  const advertised = new Set(advertisedTools.map(({ name }) => name));
+  const calls = plan.calls.map((call) => {
+    if (!call || typeof call !== "object" || Array.isArray(call)
+      || Object.keys(call).sort().join(",") !== "arguments,name"
+      || typeof call.name !== "string" || !advertised.has(call.name)
+      || !call.arguments || typeof call.arguments !== "object" || Array.isArray(call.arguments)) {
+      throw new Error("Tool selection contains an invalid or unadvertised call");
     }
-    operations.push(operation);
-  }
-  return { value: operations, next: index };
-}
-
-function parseList(lines, start) {
-  const items = [];
-  let index = start;
-  while (index < lines.length) {
-    const item = /^  -\s+(.+)$/.exec(lines[index]);
-    if (!item) break;
-    items.push(unquote(item[1].trimEnd()));
-    index += 1;
+    return { name: call.name, arguments: call.arguments };
+  });
+  if (calls.map(({ name }) => name).join("\n") !== expectedTools.join("\n")) {
+    throw new Error("Model selected the wrong tool sequence");
   }
-  if (items.length === 0) fail("expected a list");
-  return { value: items, next: index };
+  return calls;
 }
 
-function parseObject(lines, start, indent, parseScalars = false) {
-  const object = {};
-  let index = start;
-  while (index < lines.length) {
-    if (!lines[index].startsWith(indent)) break;
-    applyPair(object, lines[index].slice(indent.length), parseScalars);
-    index += 1;
+function validateCommon(value) {
+  if (!value || typeof value !== "object") throw new Error("Semantic test needs options");
+  for (const key of ["question", "criteria"]) {
+    if (typeof value[key] !== "string" || !value[key].trim()) throw new Error(`${key} must be text`);
   }
-  return { value: object, next: index };
-}
-
-function parseFoldedText(lines, start) {
-  const parts = [];
-  let index = start;
-  while (index < lines.length) {
-    if (!lines[index].startsWith("  ")) break;
-    parts.push(lines[index].trim());
-    index += 1;
+  if (!(value.server instanceof URL) || value.server.protocol !== "file:") throw new Error("server must be a file URL");
+  if (!Array.isArray(value.criticalFacts) || !value.criticalFacts.length || value.criticalFacts.some((item) =>
+    !(item instanceof RegExp) && (typeof item !== "string" || !item.trim()))) {
+    throw new Error("criticalFacts must contain text or regular expressions");
   }
-  return { value: parts.join(" "), next: index };
-}
-
-function applyPair(target, source, parseScalars = false) {
-  const pair = /^([A-Za-z][A-Za-z0-9_]*):\s*(.*)$/.exec(source);
-  if (!pair) fail(`expected key-value pair: ${source}`);
-  target[pair[1]] = parseScalars ? parseArgumentScalar(pair[2]) : unquote(pair[2].trimEnd());
-}
-
-function parseArgumentScalar(value) {
-  const trimmed = value.trim();
-  if ((trimmed.startsWith('"') && trimmed.endsWith('"')) || (trimmed.startsWith("'") && trimmed.endsWith("'"))) {
-    return unquote(trimmed);
+  if (value.assertAnswer !== undefined && typeof value.assertAnswer !== "function") throw new Error("assertAnswer must be a function");
+  for (const key of ["authToken", "authTokenEnvironment"]) {
+    if (value[key] !== undefined && (typeof value[key] !== "string" || !value[key].trim())) throw new Error(`${key} must be non-empty text`);
   }
-  if (trimmed === "true") return true;
-  if (trimmed === "false") return false;
-  if (/^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$/.test(trimmed)) {
-    const number = Number(trimmed);
-    if (Number.isFinite(number)) return number;
+  if (value.authToken !== undefined && value.authTokenEnvironment !== undefined) throw new Error("Choose one authentication source");
+  if (value.environment !== undefined && (!value.environment || typeof value.environment !== "object" ||
+    Array.isArray(value.environment) || Object.values(value.environment).some((item) => typeof item !== "string"))) {
+    throw new Error("environment must contain string values");
   }
-  return trimmed;
+  const server = fileURLToPath(value.server);
+  return { ...value, server, directory: dirname(server) };
 }
 
-function unquote(value) {
-  const trimmed = value.trim();
-  if ((trimmed.startsWith('"') && trimmed.endsWith('"')) || (trimmed.startsWith("'") && trimmed.endsWith("'"))) {
-    return trimmed.slice(1, -1);
+export function checkMeaningEvidence(options, answer, paths) {
+  const seen = new Set(paths.map(({ method, target }) => `${method}:${target}`));
+  if (options.requiredPaths.some((path) => !seen.has(path))) throw new Error("Required MCP path evidence is missing");
+  const missingFactIndices = options.criticalFacts.flatMap((fact, index) =>
+    (fact instanceof RegExp ? new RegExp(fact.source, fact.flags).test(answer)
+      : answer.toLowerCase().includes(fact.toLowerCase())) ? [] : [index]);
+  if (missingFactIndices.length) {
+    throw Object.assign(new Error("Answer is missing a required fact"), {
+      code: "missing-critical-facts", missingFactIndices,
+    });
   }
-  return trimmed;
 }