@emseepea/testing 0.18.0 → 0.19.0

package/README.md

@@ -22,6 +22,13 @@ it with `assertToolArguments` for the stable call and `assertFeedback` for the
 feedback observation and important detail. Keep `assertToolCalls` when every
 complete argument should match exactly and no feedback tool is advertised.
 
+For an intentionally optional permission check, use
+`assertToolNames(turn, { oneOf: [["update"], ["permissions", "update"]] })`.
+Supply one to eight explicit sequences. Each trial must match a complete
+sequence in the listed order and count. The evidence retains all alternatives
+and the matched sequence; arguments, meaning, confirmation and feedback still
+need their own assertions.
+
 For a successful turn that advertises feedback, use
 `assertToolCallsWithOptionalFeedback`. It requires the exact ordered primary
 calls and arguments, then allows no feedback call or one trailing

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@emseepea/testing",
-  "version": "0.18.0",
+  "version": "0.19.0",
   "description": "MCP integration and semantic testing helpers",
   "license": "MIT",
   "repository": {

package/semantic/cli.mjs

@@ -172,6 +172,15 @@ function validToolAssertions(turn, isHash) {
         || (turn.toolCalls.length === 1 && turn.toolCalls[0].name === turn.expectedOptionalTool));
   }
   if (JSON.stringify(turn.selectedTools) !== JSON.stringify(turn.expectedTools)) return false;
+  if (turn.expectedToolAlternatives !== undefined) {
+    const alternatives = turn.expectedToolAlternatives;
+    if (!Array.isArray(alternatives) || alternatives.length < 1 || alternatives.length > 8
+      || !alternatives.every((names) => Array.isArray(names)
+        && names.every((name) => typeof name === "string" && name.trim()))
+      || !alternatives.some((names) => JSON.stringify(names) === JSON.stringify(turn.expectedTools))) {
+      return false;
+    }
+  }
   if (isHash(turn.expectedCallsSha256)) {
     return JSON.stringify(turn.toolCalls.map(({ name, arguments: args }) => ({ name, arguments: args })))
       === JSON.stringify(turn.expectedCalls);
@@ -181,6 +190,8 @@ function validToolAssertions(turn, isHash) {
     tools: turn.expectedTools,
     arguments: turn.expectedArguments,
     feedback: turn.expectedFeedback,
+    ...(turn.expectedToolAlternatives === undefined
+      ? {} : { alternatives: turn.expectedToolAlternatives }),
   })).digest("hex");
   if (expectedHash !== turn.expectedSelectionSha256) return false;
   for (const [name, expected] of Object.entries(turn.expectedArguments ?? {})) {

package/semantic/test.d.mts

@@ -54,7 +54,11 @@ export function assertToolCallsWithOptionalFeedback(
 export function assertNoToolCalls(turn: ConversationTurn): void;
 /** Allows no tool call or exactly one call to the named tool in each trial. */
 export function assertOptionalToolCall(turn: ConversationTurn, name: string): void;
-export function assertToolNames(turn: ConversationTurn, expected: readonly string[]): void;
+/** Requires an exact sequence, or one of at most eight explicitly authored sequences. */
+export function assertToolNames(
+  turn: ConversationTurn,
+  expected: readonly string[] | { readonly oneOf: readonly (readonly string[])[] },
+): void;
 export function assertToolArguments(
   turn: ConversationTurn,
   name: string,

package/semantic/test.mjs

@@ -268,15 +268,28 @@ export function assertOptionalToolCall(turn, name) {
 
 export function assertToolNames(turn, expected) {
   const trials = turnTrials(turn);
-  if (!Array.isArray(expected) || expected.some((name) => typeof name !== "string" || !name.trim())) {
+  const validNames = (names) => Array.isArray(names)
+    && names.every((name) => typeof name === "string" && name.trim());
+  const alternatives = Array.isArray(expected) ? undefined : expected?.oneOf;
+  if (alternatives !== undefined && (!Array.isArray(alternatives)
+    || alternatives.length < 1 || alternatives.length > 8 || !alternatives.every(validNames))) {
+    throw new Error("Expected tool alternatives must contain one to eight string arrays");
+  }
+  if (alternatives === undefined && !validNames(expected)) {
     throw new Error("Expected tool names must be a string array");
   }
+  const sequences = (alternatives ?? [expected]).map((names) => [...names]);
   for (const trial of trials) {
-    trial.record.expectedTools = expected;
+    const actual = trial.calls.map(({ name }) => name);
+    const match = sequences.find((names) => JSON.stringify(names) === JSON.stringify(actual));
+    trial.record.expectedTools = match ?? sequences[0];
+    if (alternatives !== undefined) trial.record.expectedToolAlternatives = sequences;
     recordFlexibleExpectation(trial.record);
   }
   try {
-    for (const trial of trials) assert.deepStrictEqual(trial.calls.map(({ name }) => name), expected);
+    for (const trial of trials) {
+      assert.deepStrictEqual(trial.calls.map(({ name }) => name), trial.record.expectedTools);
+    }
   } catch {
     failAssertion(trials, "tool-name assertion");
     throw new Error("Tool names did not match the expected order and count");
@@ -552,6 +565,8 @@ function recordFlexibleExpectation(record) {
     tools: record.expectedTools,
     arguments: record.expectedArguments,
     feedback: record.expectedFeedback,
+    ...(record.expectedToolAlternatives === undefined
+      ? {} : { alternatives: record.expectedToolAlternatives }),
   }));
 }