@emseepea/testing 0.17.1 → 0.19.0

package/README.md

@@ -22,6 +22,13 @@ it with `assertToolArguments` for the stable call and `assertFeedback` for the
 feedback observation and important detail. Keep `assertToolCalls` when every
 complete argument should match exactly and no feedback tool is advertised.
 
+For an intentionally optional permission check, use
+`assertToolNames(turn, { oneOf: [["update"], ["permissions", "update"]] })`.
+Supply one to eight explicit sequences. Each trial must match a complete
+sequence in the listed order and count. The evidence retains all alternatives
+and the matched sequence; arguments, meaning, confirmation and feedback still
+need their own assertions.
+
 For a successful turn that advertises feedback, use
 `assertToolCallsWithOptionalFeedback`. It requires the exact ordered primary
 calls and arguments, then allows no feedback call or one trailing
@@ -60,6 +67,33 @@ never production. Resources and prompts need deterministic protocol tests;
 this library does not pretend that manually injecting their content proves a
 native user journey.
 
+## Scripted Confirmation Input
+
+For tools that request form elicitation, supply an explicit simulated human
+response for that turn. The script answers the real native client's request;
+it is not sent to the model and does not replace a tool result.
+
+```js
+const turn = await conversation.send("Create a contact named Synthetic Customer.", {
+  elicitations: [{
+    messageIncludes: ["Create", "Synthetic Customer"],
+    response: { action: "accept", content: { decision: "create" } },
+  }],
+});
+assertNoNegativeFeedback(turn);
+```
+
+Choose fragments that identify the exact proposed effect and recipient where
+applicable. Every fragment must match one actual form request. Unexpected,
+ambiguous, duplicate, and unused scripted responses fail the turn. No request
+is approved by default; URL elicitation is unsupported.
+
+Each of the three trials records the form message, schema, and scripted human
+decision in `turn.elicitations` and saved evidence. This proves the tested
+native client exchange with simulated human input, not compatibility with a
+different client or a person's actual consent. Continue to use isolated,
+synthetic servers; never point these tests at production.
+
 ## Test Extension Composition
 
 Use `startEmseepea` in ordinary tests when you have an app factory and want to
@@ -381,6 +415,7 @@ tool arguments, tool results, assertions, or application context.
 
 Provider events, MCP addresses, configuration, headers, provider and harness
 credentials, environment values, stderr, and home-directory paths are not
-retained. Secrets placed inside test content are not detected or redacted.
+retained. Known configured authentication values are scrubbed from elicitation
+evidence. Arbitrary secrets placed inside test content are not detected.
 Local evidence is written with mode `0600`; repository CI retains it for 14
 days.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@emseepea/testing",
-  "version": "0.17.1",
+  "version": "0.19.0",
   "description": "MCP integration and semantic testing helpers",
   "license": "MIT",
   "repository": {

package/semantic/cli.mjs

@@ -172,6 +172,15 @@ function validToolAssertions(turn, isHash) {
         || (turn.toolCalls.length === 1 && turn.toolCalls[0].name === turn.expectedOptionalTool));
   }
   if (JSON.stringify(turn.selectedTools) !== JSON.stringify(turn.expectedTools)) return false;
+  if (turn.expectedToolAlternatives !== undefined) {
+    const alternatives = turn.expectedToolAlternatives;
+    if (!Array.isArray(alternatives) || alternatives.length < 1 || alternatives.length > 8
+      || !alternatives.every((names) => Array.isArray(names)
+        && names.every((name) => typeof name === "string" && name.trim()))
+      || !alternatives.some((names) => JSON.stringify(names) === JSON.stringify(turn.expectedTools))) {
+      return false;
+    }
+  }
   if (isHash(turn.expectedCallsSha256)) {
     return JSON.stringify(turn.toolCalls.map(({ name, arguments: args }) => ({ name, arguments: args })))
       === JSON.stringify(turn.expectedCalls);
@@ -181,6 +190,8 @@ function validToolAssertions(turn, isHash) {
     tools: turn.expectedTools,
     arguments: turn.expectedArguments,
     feedback: turn.expectedFeedback,
+    ...(turn.expectedToolAlternatives === undefined
+      ? {} : { alternatives: turn.expectedToolAlternatives }),
   })).digest("hex");
   if (expectedHash !== turn.expectedSelectionSha256) return false;
   for (const [name, expected] of Object.entries(turn.expectedArguments ?? {})) {

package/semantic/elicitation.mjs

@@ -0,0 +1,51 @@
+// Scripted human input through the native client's control channel, never model guidance.
+export function redactElicitationEvidence(value, secrets) {
+  let json = JSON.stringify(value);
+  for (const secret of secrets.filter((item) => typeof item === "string" && item)) {
+    // Escape exactly as JSON does, so embedded quotes and backslashes are scrubbed too.
+    const escaped = JSON.stringify(secret).slice(1, -1);
+    json = json.split(escaped).join("[REDACTED]");
+  }
+  return JSON.parse(json);
+}
+
+export function scriptedElicitations(options = {}, secrets = []) {
+  const expected = options.elicitations ?? [];
+  if (!Array.isArray(expected) || expected.length > 8) throw new Error("Invalid scripted elicitations");
+  for (const entry of expected) {
+    if (!entry || !Array.isArray(entry.messageIncludes) || entry.messageIncludes.length === 0
+      || entry.messageIncludes.length > 8
+      || entry.messageIncludes.some((text) => typeof text !== "string" || !text || text.length > 4096)
+      || !entry.response || !["accept", "decline", "cancel"].includes(entry.response.action)
+      || JSON.stringify(entry.response).length > 16_384) throw new Error("Invalid scripted elicitation");
+  }
+  const fixtures = structuredClone(expected);
+  const used = new Set();
+  const evidence = [];
+  return {
+    evidence,
+    respond(request) {
+      if (request?.subtype !== "elicitation" || request.mcp_server_name !== "emseepea_eval"
+        || (request.mode !== undefined && request.mode !== "form")
+        || typeof request.message !== "string" || request.message.length > 16_384
+        || !request.requested_schema || typeof request.requested_schema !== "object"
+        || JSON.stringify(request.requested_schema).length > 16_384) {
+        throw new Error("Unsupported native elicitation request");
+      }
+      const item = redactElicitationEvidence({ mode: "form", message: request.message,
+        requestedSchema: request.requested_schema }, secrets);
+      evidence.push(item);
+      const matches = fixtures.map((entry, index) => ({ entry, index }))
+        .filter(({ entry }) => entry.messageIncludes.every((text) => request.message.includes(text)));
+      if (matches.length !== 1 || used.has(matches[0].index)) throw new Error("Unexpected or duplicate native elicitation");
+      const { entry, index } = matches[0];
+      used.add(index);
+      item.response = redactElicitationEvidence(entry.response, secrets);
+      item.inputSource = "scripted-human";
+      return entry.response;
+    },
+    finish() {
+      if (used.size !== fixtures.length) throw new Error("Unused scripted elicitation response");
+    },
+  };
+}

package/semantic/provider.mjs

@@ -1,6 +1,7 @@
 import { spawn } from "node:child_process";
 import { createHash } from "node:crypto";
 import { join } from "node:path";
+import { scriptedElicitations } from "./elicitation.mjs";
 
 const model = "claude-sonnet-4-6";
 const mcpServerName = "emseepea_eval";
@@ -236,7 +237,7 @@ export function conversationInvocation(provider, directory, url, tools, authToke
   };
 }
 
-export function startModelConversation(provider, directory, url, tools, authToken, context, signal) {
+export function startModelConversation(provider, directory, url, tools, authToken, context, signal, serverSecrets = []) {
   signal?.throwIfAborted();
   const invocation = conversationInvocation(provider, directory, url, tools, authToken, context);
   const child = spawn(invocation.command, invocation.args, {
@@ -250,12 +251,13 @@ export function startModelConversation(provider, directory, url, tools, authToke
   let initialEvents = [];
   let initialized = false;
   let closed = false;
+  const controlIds = new Set();
   const fail = (message) => {
     if (!pending) return;
     clearTimeout(pending.timer);
-    const reject = pending.reject;
+    const { reject, script } = pending;
     pending = undefined;
-    reject(new Error(message));
+    reject(Object.assign(new Error(message), { elicitations: script.evidence }));
   };
   const abort = () => { fail("Model conversation was cancelled"); child.kill("SIGKILL"); };
   signal?.addEventListener("abort", abort, { once: true });
@@ -277,6 +279,21 @@ export function startModelConversation(provider, directory, url, tools, authToke
         child.kill("SIGKILL");
         return;
       }
+      if (event.type === "control_request") {
+        try {
+          if (!pending || typeof event.request_id !== "string" || !event.request_id
+            || controlIds.has(event.request_id)) throw new Error("Unexpected native control request");
+          controlIds.add(event.request_id);
+          const response = pending.script.respond(event.request);
+          child.stdin.write(`${JSON.stringify({ type: "control_response", response: {
+            subtype: "success", request_id: event.request_id, response,
+          } })}\n`);
+        } catch (error) {
+          fail(error.message);
+          child.kill("SIGKILL");
+        }
+        continue;
+      }
       if (!pending) {
         initialEvents.push(event);
         continue;
@@ -284,13 +301,14 @@ export function startModelConversation(provider, directory, url, tools, authToke
       pending.events.push(event);
       if (event.type !== "result") continue;
       clearTimeout(pending.timer);
-      const { events, resolve, reject } = pending;
+      const { events, resolve, reject, script } = pending;
       pending = undefined;
       try {
+        script.finish();
         const turn = parseNativeClaudeEvents(events, tools, !initialized);
         initialized = true;
-        resolve(turn);
-      } catch (error) { reject(error); }
+        resolve({ ...turn, elicitations: script.evidence });
+      } catch (error) { reject(Object.assign(error, { elicitations: script.evidence })); }
     }
   });
   child.once("error", () => fail("Model conversation could not start"));
@@ -299,16 +317,17 @@ export function startModelConversation(provider, directory, url, tools, authToke
     if (pending) fail(`Model conversation exited ${String(code)}`);
   });
   return Object.freeze({
-    send(prompt) {
+    send(prompt, options) {
       if (closed || child.stdin.destroyed) throw new Error("Model conversation is closed");
       if (pending) throw new Error("Model conversation already has a pending turn");
+      const script = scriptedElicitations(options, [authToken, invocation.env.CLAUDE_CODE_OAUTH_TOKEN, ...serverSecrets]);
       return new Promise((resolve, reject) => {
         const timer = setTimeout(() => {
           fail("Model command timed out");
           child.kill("SIGKILL");
         }, 180_000);
         timer.unref();
-        pending = { events: initialEvents, reject, resolve, timer };
+        pending = { events: initialEvents, reject, resolve, timer, script };
         initialEvents = [];
         child.stdin.write(`${JSON.stringify({
           type: "user",

package/semantic/test.d.mts

@@ -18,11 +18,26 @@ export interface ConversationOptions {
 export interface ConversationTurn {
   readonly responses: readonly string[];
   readonly toolCalls: readonly (readonly ToolCall[])[];
+  readonly elicitations: readonly (readonly ElicitationEvidence[])[];
+}
+
+export interface ScriptedElicitation {
+  /** Every fragment must occur in exactly one actual form request. Never sent to the model. */
+  messageIncludes: readonly string[];
+  response: { action: "accept" | "decline" | "cancel"; content?: Record<string, unknown> };
+}
+
+export interface ElicitationEvidence {
+  readonly mode: "form";
+  readonly message: string;
+  readonly requestedSchema: Record<string, unknown>;
+  readonly response?: ScriptedElicitation["response"];
+  readonly inputSource?: "scripted-human";
 }
 
 export interface SemanticConversation {
   /** Sends this exact user message through the same provider-native MCP conversation. */
-  send(prompt: string): Promise<ConversationTurn>;
+  send(prompt: string, options?: { elicitations: readonly ScriptedElicitation[] }): Promise<ConversationTurn>;
 }
 
 export function createConversation(
@@ -39,7 +54,11 @@ export function assertToolCallsWithOptionalFeedback(
 export function assertNoToolCalls(turn: ConversationTurn): void;
 /** Allows no tool call or exactly one call to the named tool in each trial. */
 export function assertOptionalToolCall(turn: ConversationTurn, name: string): void;
-export function assertToolNames(turn: ConversationTurn, expected: readonly string[]): void;
+/** Requires an exact sequence, or one of at most eight explicitly authored sequences. */
+export function assertToolNames(
+  turn: ConversationTurn,
+  expected: readonly string[] | { readonly oneOf: readonly (readonly string[])[] },
+): void;
 export function assertToolArguments(
   turn: ConversationTurn,
   name: string,

package/semantic/test.mjs

@@ -60,15 +60,19 @@ export async function createConversation(testContext, options) {
 
   try {
     for (let trial = 1; trial <= 3; trial += 1) {
+      const environment = environmentForTrial(specification.environment, trial);
       const running = await startSemanticServer({
         ...specification,
-        environment: environmentForTrial(specification.environment, trial),
+        environment,
       }, testContext.signal);
       try {
         const tools = await listMcpTools(running.url, specification, testContext.signal);
         const record = { trial, turns: [] };
         const directory = await mkdtemp(join(tmpdir(), "emseepea-conversation-"));
-        state.trials.push({ running, tools, record, directory, history: [], model: undefined });
+        const serverSecrets = Object.entries(environment ?? {})
+          .filter(([key]) => /token|password|secret|credential|api.?key/i.test(key))
+          .map(([, value]) => value);
+        state.trials.push({ running, tools, record, directory, history: [], model: undefined, serverSecrets });
         evidence.answerTrials.push(record);
       } catch (error) {
         await stopSemanticServer(running.child);
@@ -82,7 +86,7 @@ export async function createConversation(testContext, options) {
   }
 
   return Object.freeze({
-    async send(prompt) {
+    async send(prompt, options) {
       if (typeof prompt !== "string" || !prompt.trim()) throw new Error("send needs a user prompt");
       ensureOpen(state);
       const results = await Promise.allSettled(state.trials.map(async (trial) => {
@@ -94,8 +98,9 @@ export async function createConversation(testContext, options) {
           semanticAuthToken(specification),
           specification.context,
           testContext.signal,
+          trial.serverSecrets,
         );
-        const answer = await trial.model.send(prompt);
+        const answer = await trial.model.send(prompt, options);
         const calls = answer.calls;
         trial.history.push({
           user: prompt,
@@ -111,6 +116,7 @@ export async function createConversation(testContext, options) {
           interactionMode: "native-mcp",
           prompt,
           response: answer.answer,
+          elicitations: answer.elicitations,
           toolCalls: calls.map((call, index) => ({
             ...call,
             result: answer.toolResults[index].content,
@@ -153,6 +159,7 @@ export async function createConversation(testContext, options) {
         }
         const trial = state.trials[index];
         trial.record.error = safeModelFailure(result.reason);
+        if (Array.isArray(result.reason?.elicitations)) trial.record.elicitations = result.reason.elicitations;
         if (Array.isArray(result.reason?.attemptedToolCalls)) {
           trial.record.attemptedToolCalls = result.reason.attemptedToolCalls;
         }
@@ -165,6 +172,7 @@ export async function createConversation(testContext, options) {
       return Object.freeze({
         responses: Object.freeze(trials.map(({ answer }) => answer)),
         toolCalls: Object.freeze(trials.map(({ calls }) => Object.freeze(calls))),
+        elicitations: Object.freeze(trials.map(({ record }) => Object.freeze(record.elicitations))),
         [privateTurn]: trials,
       });
     },
@@ -260,15 +268,28 @@ export function assertOptionalToolCall(turn, name) {
 
 export function assertToolNames(turn, expected) {
   const trials = turnTrials(turn);
-  if (!Array.isArray(expected) || expected.some((name) => typeof name !== "string" || !name.trim())) {
+  const validNames = (names) => Array.isArray(names)
+    && names.every((name) => typeof name === "string" && name.trim());
+  const alternatives = Array.isArray(expected) ? undefined : expected?.oneOf;
+  if (alternatives !== undefined && (!Array.isArray(alternatives)
+    || alternatives.length < 1 || alternatives.length > 8 || !alternatives.every(validNames))) {
+    throw new Error("Expected tool alternatives must contain one to eight string arrays");
+  }
+  if (alternatives === undefined && !validNames(expected)) {
     throw new Error("Expected tool names must be a string array");
   }
+  const sequences = (alternatives ?? [expected]).map((names) => [...names]);
   for (const trial of trials) {
-    trial.record.expectedTools = expected;
+    const actual = trial.calls.map(({ name }) => name);
+    const match = sequences.find((names) => JSON.stringify(names) === JSON.stringify(actual));
+    trial.record.expectedTools = match ?? sequences[0];
+    if (alternatives !== undefined) trial.record.expectedToolAlternatives = sequences;
     recordFlexibleExpectation(trial.record);
   }
   try {
-    for (const trial of trials) assert.deepStrictEqual(trial.calls.map(({ name }) => name), expected);
+    for (const trial of trials) {
+      assert.deepStrictEqual(trial.calls.map(({ name }) => name), trial.record.expectedTools);
+    }
   } catch {
     failAssertion(trials, "tool-name assertion");
     throw new Error("Tool names did not match the expected order and count");
@@ -457,6 +478,12 @@ function safeModelFailure(error) {
     "Model conversation could not start",
     "Model conversation is closed",
     "Model conversation was cancelled",
+    "Unexpected native control request",
+    "Unsupported native elicitation request",
+    "Unexpected or duplicate native elicitation",
+    "Unused scripted elicitation response",
+    "Invalid scripted elicitations",
+    "Invalid scripted elicitation",
   ]);
   if (safeMessages.has(message) || /^Model (?:command|conversation) exited \d{1,3}$/.test(message)) {
     return message.replace(/^./, (character) => character.toLowerCase());
@@ -538,6 +565,8 @@ function recordFlexibleExpectation(record) {
     tools: record.expectedTools,
     arguments: record.expectedArguments,
     feedback: record.expectedFeedback,
+    ...(record.expectedToolAlternatives === undefined
+      ? {} : { alternatives: record.expectedToolAlternatives }),
   }));
 }