@emseepea/testing 0.6.1 → 0.9.4

package/README.md

@@ -14,6 +14,33 @@ Use `createConversation` inside an ordinary `node:test` test. Send one or more
 user prompts, then assert exact tool calls and response meaning with the exported
 semantic assertions.
 
+Use `assertToolNames` when one tool has intentionally free-form arguments. Pair
+it with `assertToolArguments` for the stable call and `assertFeedback` for the
+feedback observation and important detail. Keep `assertToolCalls` when every
+complete argument should match exactly and no feedback tool is advertised.
+
+For a successful turn that advertises feedback, use
+`assertToolCallsWithOptionalFeedback`. It requires the exact ordered primary
+calls and arguments, then allows no feedback call or one trailing
+`submit-feedback` call. When feedback is present, it also requires a successful
+tool result and semantically checks that the response openly states the
+specific observation. Await this assertion.
+
+Pair it with `assertNoNegativeFeedback` so legitimate positive feedback does
+not make the primary behavior fail. Disclosure costs three judge calls for each
+feedback-bearing trial, with no extra judge calls when feedback is absent and a
+maximum of nine across the three trials.
+
+Use `assertOptionalToolCall(turn, "submit-feedback")` only for a deliberately
+unsuccessful journey where feedback is valid but not required. It accepts no
+call or one feedback call and rejects duplicate feedback or any other tool.
+
+For a successful application journey that advertises `submit-feedback`, call
+`assertNoNegativeFeedback(...turns)` once after its normal assertions. It fails
+if the AI records an error, friction, annoyance, unnecessary difficulty,
+confusion, repetition, an unexpected bad result, or a capability mismatch. The
+evidence records both the expectation and any offending call.
+
 The runner sends each prompt unchanged through one provider-native MCP
 conversation. It does not add tool-selection instructions, a JSON call plan,
 advertised-tool text, an answer wrapper, or prepared MCP material. Exact tool

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@emseepea/testing",
-  "version": "0.6.1",
+  "version": "0.9.4",
   "description": "MCP integration and semantic testing helpers",
   "license": "MIT",
   "repository": {
@@ -41,7 +41,7 @@
     "test:built": "node --test test/*.test.mjs"
   },
   "dependencies": {
-    "@emseepea/server": "0.4.0",
+    "@emseepea/server": "0.7.0",
     "@modelcontextprotocol/client": "2.0.0"
   },
   "engines": {

package/semantic/cli.mjs

@@ -1,11 +1,23 @@
 #!/usr/bin/env node
 import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
 import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { dirname, join, relative, resolve } from "node:path";
 import { discoverTests } from "./discover.mjs";
 import { modelVersion } from "./provider.mjs";
 
+const negativeFeedbackObservations = new Set([
+  "error",
+  "friction",
+  "annoyance",
+  "unnecessary_difficulty",
+  "confusion",
+  "repetition",
+  "unexpected_bad_result",
+  "capability_mismatch",
+]);
+
 const paths = [];
 let provider = process.env.EMSEEPEA_EVAL_PROVIDER ?? "claude-local";
 let output = "artifacts/llm-eval/evidence.json";
@@ -105,7 +117,7 @@ function validRecord(record, authoritative, smoke) {
   if (record.status !== "passed" || record.authoritative !== authoritative || record.smoke !== smoke
     || record.mode !== "conversation" || record.answerTrials?.length !== 3
     || !Number.isInteger(record.judgeVerdicts?.length) || record.judgeVerdicts.length < 9
-    || record.judgeVerdicts.length % 9 !== 0
+    || record.judgeVerdicts.length % 3 !== 0
     || !record.judgeVerdicts.every((judgment) => isHash(judgment.expectationSha256)
       && isHash(judgment.requestSha256) && isHash(judgment.responseSha256)
       && typeof judgment.expectedMeaning === "string" && judgment.expectedMeaning.length > 0
@@ -123,14 +135,100 @@ function validRecord(record, authoritative, smoke) {
       && typeof turn.response === "string"
       && isHash(turn.promptSha256) && isHash(turn.answerSha256)
       && isHash(turn.advertisedToolsSha256) && isHash(turn.selectedCallsSha256)
-      && isHash(turn.expectedCallsSha256)
       && Array.isArray(turn.toolCalls)
-      && JSON.stringify(turn.toolCalls.map(({ name, arguments: args }) => ({ name, arguments: args })))
-        === JSON.stringify(turn.expectedCalls)
-      && turn.toolCalls.every((call) => Object.hasOwn(call, "result"))
-      && JSON.stringify(turn.selectedTools) === JSON.stringify(turn.expectedTools)
+      && JSON.stringify(turn.selectedTools)
+        === JSON.stringify(turn.toolCalls.map(({ name }) => name))
+      && validToolAssertions(turn, isHash)
+      && validNegativeFeedbackAssertion(turn)
+      && validFeedbackDisclosure(turn, record.judgeVerdicts, trial.trial, isHash)
+      && turn.toolCalls.every((call) => Object.hasOwn(call, "result") && typeof call.isError === "boolean")
       && Array.isArray(turn.pathEvidence) && turn.pathEvidence.length === turn.toolCallCount
       && turn.pathEvidence.every(({ method, target, requestSha256, responseSha256 }) =>
         method === "tools/call" && turn.selectedTools.includes(target)
           && isHash(requestSha256) && isHash(responseSha256))));
 }
+
+function validToolAssertions(turn, isHash) {
+  if (turn.expectedOptionalFeedback === true) {
+    if (!Array.isArray(turn.expectedCalls)) return false;
+    const expectedHash = createHash("sha256").update(JSON.stringify({
+      calls: turn.expectedCalls,
+      optionalFeedback: true,
+    })).digest("hex");
+    const calls = turn.toolCalls.map(({ name, arguments: args }) => ({ name, arguments: args }));
+    const primaryCalls = calls.slice(0, turn.expectedCalls?.length);
+    const trailingCalls = calls.slice(turn.expectedCalls?.length);
+    return expectedHash === turn.expectedSelectionSha256
+      && JSON.stringify(primaryCalls) === JSON.stringify(turn.expectedCalls)
+      && (trailingCalls.length === 0
+        || (trailingCalls.length === 1 && trailingCalls[0].name === "submit-feedback"));
+  }
+  if (typeof turn.expectedOptionalTool === "string" && turn.expectedOptionalTool.trim()) {
+    const expectedHash = createHash("sha256").update(JSON.stringify({
+      optionalTool: turn.expectedOptionalTool,
+    })).digest("hex");
+    return expectedHash === turn.expectedSelectionSha256
+      && (turn.toolCalls.length === 0
+        || (turn.toolCalls.length === 1 && turn.toolCalls[0].name === turn.expectedOptionalTool));
+  }
+  if (JSON.stringify(turn.selectedTools) !== JSON.stringify(turn.expectedTools)) return false;
+  if (isHash(turn.expectedCallsSha256)) {
+    return JSON.stringify(turn.toolCalls.map(({ name, arguments: args }) => ({ name, arguments: args })))
+      === JSON.stringify(turn.expectedCalls);
+  }
+  if (!isHash(turn.expectedSelectionSha256)) return false;
+  const expectedHash = createHash("sha256").update(JSON.stringify({
+    tools: turn.expectedTools,
+    arguments: turn.expectedArguments,
+    feedback: turn.expectedFeedback,
+  })).digest("hex");
+  if (expectedHash !== turn.expectedSelectionSha256) return false;
+  for (const [name, expected] of Object.entries(turn.expectedArguments ?? {})) {
+    const matches = turn.toolCalls.filter((call) => call.name === name);
+    if (matches.length !== 1 || JSON.stringify(matches[0].arguments) !== JSON.stringify(expected)) return false;
+  }
+  if (turn.expectedFeedback) {
+    const calls = turn.toolCalls.filter((call) => call.name === "submit-feedback");
+    const detail = calls[0]?.arguments?.detail;
+    if (calls.length !== 1 || !turn.expectedFeedback.observation.includes(calls[0].arguments?.observation)
+      || typeof detail !== "string" || turn.expectedFeedback.detailIncludes.some(
+        (value) => !detail.toLowerCase().includes(value.toLowerCase()),
+      )) return false;
+  }
+  return true;
+}
+
+function validNegativeFeedbackAssertion(turn) {
+  if (turn.expectedNegativeFeedback !== false) return true;
+  const actual = turn.toolCalls.filter((call) => call.name === "submit-feedback"
+    && negativeFeedbackObservations.has(call.arguments?.observation));
+  return actual.length === 0 && JSON.stringify(actual) === JSON.stringify(turn.negativeFeedbackCalls);
+}
+
+function validFeedbackDisclosure(turn, judgments, trial, isHash) {
+  if (turn.expectedOptionalFeedback !== true) return turn.feedbackDisclosure === undefined;
+  const feedback = turn.toolCalls[turn.expectedCalls.length];
+  if (!feedback) return turn.feedbackDisclosure === undefined;
+  if (feedback.isError || typeof feedback.arguments?.observation !== "string"
+    || !feedback.arguments.observation.trim() || typeof feedback.arguments.detail !== "string"
+    || !feedback.arguments.detail.trim()) return false;
+  const expectedMeaning = feedbackDisclosureExpectation(feedback);
+  const expectationSha256 = createHash("sha256").update(expectedMeaning).digest("hex");
+  if (!isHash(turn.feedbackDisclosure?.expectationSha256)
+    || JSON.stringify(turn.feedbackDisclosure) !== JSON.stringify({
+      observation: feedback.arguments.observation,
+      detail: feedback.arguments.detail,
+      expectationSha256,
+    })) return false;
+  const matching = judgments.filter((judgment) => judgment.trial === trial
+    && judgment.turn === turn.turn && judgment.expectationSha256 === expectationSha256);
+  return matching.length === 3 && matching.every((judgment) =>
+    judgment.expectedMeaning === expectedMeaning && judgment.verdict?.pass === true);
+}
+
+function feedbackDisclosureExpectation(call) {
+  return "The final assistant response makes it clear that feedback was submitted, recorded, or noted, "
+    + "and communicates the substance "
+    + `of this specific ${call.arguments.observation} observation: ${call.arguments.detail} `
+    + "A concise, faithful summary is enough. It need not repeat every example, field, or phrase.";
+}

package/semantic/provider.mjs

@@ -84,7 +84,11 @@ export function parseNativeClaudeEvents(stdout, advertisedTools, requireInit = f
     }
     return { name: publicName, arguments: input };
   });
-  if (toolUses.length > 3) throw new Error("Model command used more than three tools");
+  if (toolUses.length > 3) {
+    throw Object.assign(new Error("Model command used more than three tools"), {
+      attemptedToolCalls: calls,
+    });
+  }
   if (init) {
     const available = [...(init.tools ?? [])].sort();
     const expected = [...advertised.keys()].sort();
@@ -134,7 +138,10 @@ export function parseNativeClaudeEvents(stdout, advertisedTools, requireInit = f
   return {
     answer: result.result,
     calls,
-    toolResults: toolUses.map(({ id }) => toolResults.get(id).content),
+    toolResults: toolUses.map(({ id }) => {
+      const toolResult = toolResults.get(id);
+      return { content: toolResult.content, isError: toolResult.is_error === true };
+    }),
     pathEvidence,
     models: Object.keys(result.modelUsage),
     turnCount: 1,

package/semantic/test.d.mts

@@ -31,7 +31,26 @@ export function createConversation(
 ): Promise<SemanticConversation>;
 
 export function assertToolCalls(turn: ConversationTurn, expected: readonly ToolCall[]): void;
+/** Requires the exact primary calls and allows one trailing submit-feedback call. */
+export function assertToolCallsWithOptionalFeedback(
+  turn: ConversationTurn,
+  expected: readonly ToolCall[],
+): Promise<void>;
 export function assertNoToolCalls(turn: ConversationTurn): void;
+/** Allows no tool call or exactly one call to the named tool in each trial. */
+export function assertOptionalToolCall(turn: ConversationTurn, name: string): void;
+export function assertToolNames(turn: ConversationTurn, expected: readonly string[]): void;
+export function assertToolArguments(
+  turn: ConversationTurn,
+  name: string,
+  expected: Record<string, unknown>,
+): void;
+export function assertFeedback(
+  turn: ConversationTurn,
+  expectation: { observation: string | readonly string[]; detailIncludes: readonly string[] },
+): void;
+/** Asserts that successful turns did not submit an error, friction, or other negative observation. */
+export function assertNoNegativeFeedback(...turns: readonly ConversationTurn[]): void;
 export function assertResponseContains(turn: ConversationTurn, expected: string | readonly string[]): void;
 export function assertResponseMeaning(
   turn: ConversationTurn,

package/semantic/test.mjs

@@ -101,7 +101,15 @@ export async function createConversation(testContext, options) {
           );
           const answer = await trial.model.send(prompt);
           const calls = answer.calls;
-          trial.history.push({ user: prompt, assistant: answer.answer });
+          trial.history.push({
+            user: prompt,
+            toolCalls: calls.map((call, index) => ({
+              ...call,
+              result: answer.toolResults[index].content,
+              isError: answer.toolResults[index].isError,
+            })),
+            assistant: answer.answer,
+          });
           const record = {
             turn: trial.record.turns.length + 1,
             interactionMode: "native-mcp",
@@ -109,7 +117,8 @@ export async function createConversation(testContext, options) {
             response: answer.answer,
             toolCalls: calls.map((call, index) => ({
               ...call,
-              result: answer.toolResults[index],
+              result: answer.toolResults[index].content,
+              isError: answer.toolResults[index].isError,
             })),
             promptSha256: hash(prompt),
             answerSha256: hash(answer.answer),
@@ -141,7 +150,12 @@ export async function createConversation(testContext, options) {
           });
         }
       } catch (error) {
-        if (activeTrial) activeTrial.record.error = safeModelFailure(error);
+        if (activeTrial) {
+          activeTrial.record.error = safeModelFailure(error);
+          if (Array.isArray(error?.attemptedToolCalls)) {
+            activeTrial.record.attemptedToolCalls = error.attemptedToolCalls;
+          }
+        }
         state.failed = true;
         evidence.failedPhase = "conversation turn";
         throw new Error(`Semantic test failed during conversation turn: ${name}`);
@@ -157,11 +171,7 @@ export async function createConversation(testContext, options) {
 
 export function assertToolCalls(turn, expected) {
   const trials = turnTrials(turn);
-  if (!Array.isArray(expected) || expected.some((call) => !call || typeof call.name !== "string"
-    || !call.name.trim() || !call.arguments || typeof call.arguments !== "object"
-    || Array.isArray(call.arguments))) {
-    throw new Error("Expected tool calls must have names and object arguments");
-  }
+  validateExpectedCalls(expected);
   for (const trial of trials) {
     trial.record.expectedTools = expected.map(({ name }) => name);
     trial.record.expectedCalls = expected;
@@ -175,10 +185,161 @@ export function assertToolCalls(turn, expected) {
   }
 }
 
+export async function assertToolCallsWithOptionalFeedback(turn, expected) {
+  const trials = turnTrials(turn);
+  validateExpectedCalls(expected);
+  for (const trial of trials) {
+    trial.record.expectedTools = expected.map(({ name }) => name);
+    trial.record.expectedCalls = expected;
+    trial.record.expectedOptionalFeedback = true;
+    trial.record.expectedSelectionSha256 = hash(JSON.stringify({ calls: expected, optionalFeedback: true }));
+    const primaryCalls = trial.calls.slice(0, expected.length);
+    const trailingCalls = trial.calls.slice(expected.length);
+    try {
+      assert.deepStrictEqual(primaryCalls, expected);
+      assert.ok(trailingCalls.length <= 1);
+      if (trailingCalls.length === 1) assert.equal(trailingCalls[0].name, "submit-feedback");
+    } catch {
+      failAssertion(trials, "tool-call assertion");
+      throw new Error("Tool calls did not match the expected calls with at most one trailing feedback call");
+    }
+  }
+  let failed = false;
+  for (let trialIndex = 0; trialIndex < trials.length; trialIndex += 1) {
+    const trial = trials[trialIndex];
+    const feedback = trial.calls[expected.length];
+    if (!feedback) continue;
+    const recordedCall = trial.record.toolCalls[expected.length];
+    if (typeof feedback.arguments.observation !== "string" || !feedback.arguments.observation.trim()
+      || typeof feedback.arguments.detail !== "string" || !feedback.arguments.detail.trim()) {
+      failed = true;
+      continue;
+    }
+    const expectation = feedbackDisclosureExpectation(feedback);
+    trial.record.feedbackDisclosure = {
+      observation: feedback.arguments.observation,
+      detail: feedback.arguments.detail,
+      expectationSha256: hash(expectation),
+    };
+    const disclosureFailed = await judgeTrialMeaning(trial, trialIndex, expectation);
+    if (recordedCall.isError || disclosureFailed) failed = true;
+  }
+  if (failed) {
+    failAssertion(trials, "feedback-disclosure assertion");
+    throw new Error("Optional feedback failed or was not openly described in the response");
+  }
+}
+
 export function assertNoToolCalls(turn) {
   assertToolCalls(turn, []);
 }
 
+export function assertOptionalToolCall(turn, name) {
+  const trials = turnTrials(turn);
+  if (typeof name !== "string" || !name.trim()) {
+    throw new Error("Optional tool-call expectation needs a tool name");
+  }
+  for (const trial of trials) {
+    trial.record.expectedOptionalTool = name;
+    trial.record.expectedSelectionSha256 = hash(JSON.stringify({ optionalTool: name }));
+    if (trial.calls.length > 1 || (trial.calls.length === 1 && trial.calls[0].name !== name)) {
+      failAssertion([trial], "optional tool-call assertion");
+      throw new Error(`Expected no tool call or one ${name} call`);
+    }
+  }
+}
+
+export function assertToolNames(turn, expected) {
+  const trials = turnTrials(turn);
+  if (!Array.isArray(expected) || expected.some((name) => typeof name !== "string" || !name.trim())) {
+    throw new Error("Expected tool names must be a string array");
+  }
+  for (const trial of trials) {
+    trial.record.expectedTools = expected;
+    recordFlexibleExpectation(trial.record);
+  }
+  try {
+    for (const trial of trials) assert.deepStrictEqual(trial.calls.map(({ name }) => name), expected);
+  } catch {
+    failAssertion(trials, "tool-name assertion");
+    throw new Error("Tool names did not match the expected order and count");
+  }
+}
+
+export function assertToolArguments(turn, name, expected) {
+  const trials = turnTrials(turn);
+  if (typeof name !== "string" || !name.trim() || !expected || typeof expected !== "object"
+    || Array.isArray(expected)) {
+    throw new Error("Tool argument expectation needs a name and object arguments");
+  }
+  for (const trial of trials) {
+    const matches = trial.calls.filter((call) => call.name === name);
+    trial.record.expectedArguments ??= {};
+    trial.record.expectedArguments[name] = expected;
+    recordFlexibleExpectation(trial.record);
+    try {
+      assert.equal(matches.length, 1);
+      assert.deepStrictEqual(matches[0].arguments, expected);
+    } catch {
+      failAssertion([trial], "tool-argument assertion");
+      throw new Error(`Arguments for ${name} did not match exactly`);
+    }
+  }
+}
+
+export function assertFeedback(turn, expectation) {
+  const trials = turnTrials(turn);
+  const observations = typeof expectation?.observation === "string"
+    ? [expectation.observation]
+    : expectation?.observation;
+  const detailIncludes = expectation?.detailIncludes;
+  if (!Array.isArray(observations) || observations.length === 0
+    || !Array.isArray(detailIncludes) || detailIncludes.length === 0
+    || [...observations, ...detailIncludes].some((value) => typeof value !== "string" || !value.trim())) {
+    throw new Error("Feedback expectation needs observation and detailIncludes strings");
+  }
+  for (const trial of trials) {
+    trial.record.expectedFeedback = { observation: observations, detailIncludes };
+    recordFlexibleExpectation(trial.record);
+    const calls = trial.calls.filter(({ name }) => name === "submit-feedback");
+    const detail = calls[0]?.arguments?.detail;
+    if (calls.length !== 1
+      || !observations.includes(calls[0].arguments?.observation)
+      || typeof detail !== "string"
+      || detailIncludes.some((value) => !detail.toLowerCase().includes(value.toLowerCase()))) {
+      failAssertion([trial], "feedback assertion");
+      throw new Error("Feedback did not match the expected observation and useful detail");
+    }
+  }
+}
+
+const negativeFeedbackObservations = new Set([
+  "error",
+  "friction",
+  "annoyance",
+  "unnecessary_difficulty",
+  "confusion",
+  "repetition",
+  "unexpected_bad_result",
+  "capability_mismatch",
+]);
+
+export function assertNoNegativeFeedback(...turns) {
+  if (turns.length === 0) throw new Error("No-negative-feedback assertion needs at least one turn");
+  const trials = turns.flatMap(turnTrials);
+  for (const trial of trials) {
+    const offendingCalls = trial.calls.filter((call) =>
+      call.name === "submit-feedback"
+      && negativeFeedbackObservations.has(call.arguments?.observation));
+    trial.record.expectedNegativeFeedback = false;
+    trial.record.negativeFeedbackCalls = offendingCalls;
+    if (offendingCalls.length > 0) {
+      failAssertion([trial], "negative-feedback assertion");
+      throw new Error("A successful example interaction recorded negative feedback");
+    }
+  }
+}
+
 export function assertResponseContains(turn, expected) {
   const trials = turnTrials(turn);
   const values = typeof expected === "string" ? [expected] : expected;
@@ -211,41 +372,7 @@ export async function assertResponseMeaning(turn, expectation) {
   for (let trialIndex = 0; trialIndex < trials.length; trialIndex += 1) {
     const trial = trials[trialIndex];
     trial.record.expectedMeaning = expectation.expected;
-    for (let judgment = 1; judgment <= 3; judgment += 1) {
-      const request = judgePrompt(trial.history, expectation.expected);
-      const record = {
-        trial: trialIndex + 1,
-        turn: trial.record.turn,
-        judgment,
-        expectedMeaning: expectation.expected,
-        expectationSha256: hash(expectation.expected),
-        requestSha256: hash(request),
-      };
-      try {
-        const response = await isolatedModel(
-          trial.provider,
-          request,
-          "emseepea-judge-",
-          trial.signal,
-        );
-        Object.assign(record, {
-          models: response.models,
-          turnCount: response.turnCount,
-          providerTurnCount: response.providerTurnCount,
-          providerToolCount: response.providerToolCount,
-          responseSha256: hash(response.answer),
-        });
-        const verdict = parseJudgeVerdict(response.answer.trim());
-        record.verdict = verdict;
-        if (!verdict.pass) failed = true;
-      } catch (error) {
-        record.error = error instanceof SyntaxError || error.message === "Judge returned an invalid verdict"
-          ? "invalid judge verdict"
-          : safeModelFailure(error);
-        failed = true;
-      }
-      trial.evidence.judgeVerdicts.push(record);
-    }
+    if (await judgeTrialMeaning(trial, trialIndex, expectation.expected)) failed = true;
     trial.record.meaningAssertionCount += 1;
   }
   trials[0].state.meaningAssertions += 1;
@@ -255,6 +382,41 @@ export async function assertResponseMeaning(turn, expectation) {
   }
 }
 
+async function judgeTrialMeaning(trial, trialIndex, expected) {
+  let failed = false;
+  for (let judgment = 1; judgment <= 3; judgment += 1) {
+    const request = judgePrompt(trial.history, expected);
+    const record = {
+      trial: trialIndex + 1,
+      turn: trial.record.turn,
+      judgment,
+      expectedMeaning: expected,
+      expectationSha256: hash(expected),
+      requestSha256: hash(request),
+    };
+    try {
+      const response = await isolatedModel(trial.provider, request, "emseepea-judge-", trial.signal);
+      Object.assign(record, {
+        models: response.models,
+        turnCount: response.turnCount,
+        providerTurnCount: response.providerTurnCount,
+        providerToolCount: response.providerToolCount,
+        responseSha256: hash(response.answer),
+      });
+      const verdict = parseJudgeVerdict(response.answer.trim());
+      record.verdict = verdict;
+      if (!verdict.pass) failed = true;
+    } catch (error) {
+      record.error = error instanceof SyntaxError || error.message === "Judge returned an invalid verdict"
+        ? "invalid judge verdict"
+        : safeModelFailure(error);
+      failed = true;
+    }
+    trial.evidence.judgeVerdicts.push(record);
+  }
+  return failed;
+}
+
 function safeModelFailure(error) {
   const message = error instanceof Error ? error.message : "";
   const safeMessages = new Set([
@@ -313,7 +475,9 @@ async function closeConversation(state, evidence, output) {
   }));
   const complete = !state.failed && state.meaningAssertions > 0 && evidence.answerTrials.length === 3
     && evidence.answerTrials.every(({ turns }) => turns.length > 0
-      && turns.every((turn) => Array.isArray(turn.expectedTools)));
+      && turns.every((turn) => Array.isArray(turn.expectedTools)
+        || turn.expectedOptionalFeedback === true
+        || typeof turn.expectedOptionalTool === "string"));
   if (complete) {
     evidence.status = "passed";
   } else if (!evidence.failedPhase) {
@@ -322,7 +486,7 @@ async function closeConversation(state, evidence, output) {
   await mkdir(dirname(output), { recursive: true });
   await writeFile(output, `${JSON.stringify(evidence, null, 2)}\n`, { mode: 0o600 });
   if (!complete && !state.failed) {
-    throw new Error("Semantic conversation needs exact tool-call assertions for every turn and a meaning assertion");
+    throw new Error("Semantic conversation needs tool-selection assertions for every turn and a meaning assertion");
   }
 }
 
@@ -338,11 +502,34 @@ function turnTrials(turn) {
   return trials;
 }
 
+function validateExpectedCalls(expected) {
+  if (!Array.isArray(expected) || expected.some((call) => !call || typeof call.name !== "string"
+    || !call.name.trim() || !call.arguments || typeof call.arguments !== "object"
+    || Array.isArray(call.arguments))) {
+    throw new Error("Expected tool calls must have names and object arguments");
+  }
+}
+
+function feedbackDisclosureExpectation(call) {
+  return "The final assistant response makes it clear that feedback was submitted, recorded, or noted, "
+    + "and communicates the substance "
+    + `of this specific ${call.arguments.observation} observation: ${call.arguments.detail} `
+    + "A concise, faithful summary is enough. It need not repeat every example, field, or phrase.";
+}
+
 function failAssertion(trials, phase) {
   for (const trial of trials) trial.state.failed = true;
   trials[0].evidence.failedPhase = phase;
 }
 
+function recordFlexibleExpectation(record) {
+  record.expectedSelectionSha256 = hash(JSON.stringify({
+    tools: record.expectedTools,
+    arguments: record.expectedArguments,
+    feedback: record.expectedFeedback,
+  }));
+}
+
 function judgePrompt(history, expected) {
   return [
     "Judge whether the final assistant response communicates the complete expected meaning in this conversation.",