@emseepea/testing 0.5.3 → 0.9.3

package/README.md

@@ -14,6 +14,33 @@ Use `createConversation` inside an ordinary `node:test` test. Send one or more
 user prompts, then assert exact tool calls and response meaning with the exported
 semantic assertions.
 
+Use `assertToolNames` when one tool has intentionally free-form arguments. Pair
+it with `assertToolArguments` for the stable call and `assertFeedback` for the
+feedback observation and important detail. Keep `assertToolCalls` when every
+complete argument should match exactly and no feedback tool is advertised.
+
+For a successful turn that advertises feedback, use
+`assertToolCallsWithOptionalFeedback`. It requires the exact ordered primary
+calls and arguments, then allows no feedback call or one trailing
+`submit-feedback` call. When feedback is present, it also requires a successful
+tool result and semantically checks that the response openly states the
+specific observation. Await this assertion.
+
+Pair it with `assertNoNegativeFeedback` so legitimate positive feedback does
+not make the primary behavior fail. Disclosure costs three judge calls for each
+feedback-bearing trial, with no extra judge calls when feedback is absent and a
+maximum of nine across the three trials.
+
+Use `assertOptionalToolCall(turn, "submit-feedback")` only for a deliberately
+unsuccessful journey where feedback is valid but not required. It accepts no
+call or one feedback call and rejects duplicate feedback or any other tool.
+
+For a successful application journey that advertises `submit-feedback`, call
+`assertNoNegativeFeedback(...turns)` once after its normal assertions. It fails
+if the AI records an error, friction, annoyance, unnecessary difficulty,
+confusion, repetition, an unexpected bad result, or a capability mismatch. The
+evidence records both the expectation and any offending call.
+
 The runner sends each prompt unchanged through one provider-native MCP
 conversation. It does not add tool-selection instructions, a JSON call plan,
 advertised-tool text, an answer wrapper, or prepared MCP material. Exact tool
@@ -30,6 +57,28 @@ never production. Resources and prompts need deterministic protocol tests;
 this library does not pretend that manually injecting their content proves a
 native user journey.
 
+## Test Extension Composition
+
+Use `startEmseepea` in ordinary tests when you have an app factory and want to
+exercise it in the same process. `insecureTestAuthentication` supplies a small
+test-only verifier for protected-access examples:
+
+```js
+import { insecureTestAuthentication, startEmseepea } from "@emseepea/testing";
+
+const running = await startEmseepea(t, await createApp({
+  access: { access: "protected", requiredScopes: ["peas:read"] },
+  authentication: insecureTestAuthentication(["peas:read"]),
+  observability: [{ id: "test", emit: (event) => events.push(event) }],
+}), { token: "test-token" });
+
+const client = await running.connect();
+```
+
+Never deploy `insecureTestAuthentication`. It accepts any supplied bearer token
+and exists only to keep ordinary tests focused on composition and authorization
+flow. Test a real verifier separately against its provider contract.
+
 When a conversation writes to external state, pass an `environment` function
 that returns a separate test database connection for each trial. The trial
 number selects infrastructure only. It is never sent to the model or MCP

package/dist/index.d.ts

@@ -1,4 +1,5 @@
 import { Client } from "@modelcontextprotocol/client";
+import { serveEmseepea, type AuthenticationOptions } from "@emseepea/server";
 export interface TestCleanup {
     after(cleanup: () => Promise<void>): void;
 }
@@ -16,5 +17,8 @@ export interface RunningMcpServer {
     }>;
     url: URL;
 }
+/** Test-only verifier. Never use it in a deployed server. */
+export declare function insecureTestAuthentication(permissions: readonly string[], discovery?: "public" | "protected"): AuthenticationOptions;
+export declare function startEmseepea(test: TestCleanup, app: Parameters<typeof serveEmseepea>[0], options?: Pick<StartMcpServerOptions, "clientName" | "token">): Promise<RunningMcpServer>;
 export declare function startMcpServer(test: TestCleanup, serverUrl: URL, options?: StartMcpServerOptions): Promise<RunningMcpServer>;
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAiC,MAAM,8BAA8B,CAAC;AAErF,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CAC3C;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,IAAI,QAAQ,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvD,GAAG,EAAE,GAAG,CAAC;CACV;AAED,wBAAsB,cAAc,CAClC,IAAI,EAAE,WAAW,EACjB,SAAS,EAAE,GAAG,EACd,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,gBAAgB,CAAC,CAiE3B"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAiC,MAAM,8BAA8B,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,KAAK,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAE7E,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CAC3C;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IAChC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,IAAI,QAAQ,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACvD,GAAG,EAAE,GAAG,CAAC;CACV;AAED,6DAA6D;AAC7D,wBAAgB,0BAA0B,CACxC,WAAW,EAAE,SAAS,MAAM,EAAE,EAC9B,SAAS,GAAE,QAAQ,GAAG,WAAyB,GAC9C,qBAAqB,CAyBvB;AAED,wBAAsB,aAAa,CACjC,IAAI,EAAE,WAAW,EACjB,GAAG,EAAE,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC,EACxC,OAAO,GAAE,IAAI,CAAC,qBAAqB,EAAE,YAAY,GAAG,OAAO,CAAM,GAChE,OAAO,CAAC,gBAAgB,CAAC,CAY3B;AAED,wBAAsB,cAAc,CAClC,IAAI,EAAE,WAAW,EACjB,SAAS,EAAE,GAAG,EACd,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,gBAAgB,CAAC,CAsD3B"}

package/dist/index.js

@@ -1,6 +1,47 @@
 import { spawn } from "node:child_process";
 import { fileURLToPath } from "node:url";
 import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
+import { serveEmseepea } from "@emseepea/server";
+/** Test-only verifier. Never use it in a deployed server. */
+export function insecureTestAuthentication(permissions, discovery = "protected") {
+    const resourceServerUrl = new URL("https://test.example/mcp");
+    return {
+        discovery,
+        verifier: {
+            async verifyAccessToken(token) {
+                return {
+                    token,
+                    clientId: "emseepea-test-client",
+                    scopes: [...permissions],
+                    expiresAt: Math.floor(Date.now() / 1_000) + 60,
+                    resource: resourceServerUrl,
+                };
+            },
+        },
+        metadata: {
+            resourceServerUrl,
+            oauthMetadata: {
+                issuer: "https://auth.test.example",
+                authorization_endpoint: "https://auth.test.example/authorize",
+                token_endpoint: "https://auth.test.example/token",
+                response_types_supported: ["code"],
+            },
+        },
+    };
+}
+export async function startEmseepea(test, app, options = {}) {
+    const running = await serveEmseepea(app, { port: 0 });
+    const clients = [];
+    test.after(async () => {
+        await Promise.allSettled(clients.map((client) => client.close()));
+        await running.close();
+    });
+    return {
+        ...running,
+        output: () => Object.freeze({ stdout: "", stderr: "" }),
+        connect: (token = options.token) => connect(running.url, clients, options.clientName, token),
+    };
+}
 export async function startMcpServer(test, serverUrl, options = {}) {
     const child = spawn(process.execPath, [fileURLToPath(serverUrl)], {
         env: {
@@ -50,14 +91,15 @@ export async function startMcpServer(test, serverUrl, options = {}) {
     return {
         url,
         output: () => Object.freeze({ stdout: output, stderr: errors }),
-        async connect(token = options.token) {
-            const client = new Client({ name: options.clientName ?? "emseepea-test", version: "0.0.0" }, { versionNegotiation: { mode: { pin: "2026-07-28" } } });
-            await client.connect(new StreamableHTTPClientTransport(url, token ? { authProvider: { token: async () => token } } : undefined));
-            clients.push(client);
-            return client;
-        },
+        connect: (token = options.token) => connect(url, clients, options.clientName, token),
     };
 }
+async function connect(url, clients, clientName = "emseepea-test", token) {
+    const client = new Client({ name: clientName, version: "0.0.0" }, { versionNegotiation: { mode: { pin: "2026-07-28" } } });
+    await client.connect(new StreamableHTTPClientTransport(url, token ? { authProvider: { token: async () => token } } : undefined));
+    clients.push(client);
+    return client;
+}
 async function stopProcess(child) {
     if (child.exitCode !== null)
         return;

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAC;AAmBrF,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAAiB,EACjB,SAAc,EACd,UAAiC,EAAE;IAEnC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,EAAE;QAChE,GAAG,EAAE;YACH,GAAG,OAAO,CAAC,GAAG;YACd,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,GAAG;YACT,GAAG,OAAO,CAAC,WAAW;SACvB;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;KAClC,CAAC,CAAC;IACH,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,MAAM,GAAG,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7F,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,MAAM,GAAG,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7F,MAAM,GAAG,GAAG,MAAM,IAAI,OAAO,CAAM,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrD,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,KAAW,EAAE,EAAE;YAC5C,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAClC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAC1B,IAAI,KAAK;gBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;iBACpB,IAAI,KAAK;gBAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC,CAAC;QACF,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YAC7D,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC;gBAAE,MAAM,CAAC,SAAS,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,CAAC,CAAC;QACF,MAAM,MAAM,GAAG,CAAC,KAAY,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,CAAC,IAAmB,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,MAAM,CAAC,IAAI,CAAC,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1G,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,MAAM,EAAE,CAAC,CAAC,EAClE,OAAO,CAAC,gBAAgB,IAAI,MAAM,CACnC,CAAC;QACF,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC3B,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAc,EAAE,EAAE;QAChC,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;QACzB,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE;QACpB,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAClE,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,GAAG;QACH,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QAC/D,KAAK,CAAC,OAAO,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK;YACjC,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,IAAI,eAAe,EAAE,OAAO,EAAE,OAAO,EAAE,EACjE,EAAE,kBAAkB,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,YAAY,EAAE,EAAE,EAAE,CACxD,CAAC;YACF,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,6BAA6B,CACpD,GAAG,EACH,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CACnE,CAAC,CAAC;YACH,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrB,OAAO,MAAM,CAAC;QAChB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,KAA+B;IACxD,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;QAAE,OAAO;IACpC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,MAAM,OAAO,CAAC,IAAI,CAAC;QACjB,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACpE,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;KAC3D,CAAC,CAAC;IACH,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AACrD,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,MAAM,EAAE,6BAA6B,EAAE,MAAM,8BAA8B,CAAC;AACrF,OAAO,EAAE,aAAa,EAA8B,MAAM,kBAAkB,CAAC;AAmB7E,6DAA6D;AAC7D,MAAM,UAAU,0BAA0B,CACxC,WAA8B,EAC9B,YAAoC,WAAW;IAE/C,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,0BAA0B,CAAC,CAAC;IAC9D,OAAO;QACL,SAAS;QACT,QAAQ,EAAE;YACR,KAAK,CAAC,iBAAiB,CAAC,KAAK;gBAC3B,OAAO;oBACL,KAAK;oBACL,QAAQ,EAAE,sBAAsB;oBAChC,MAAM,EAAE,CAAC,GAAG,WAAW,CAAC;oBACxB,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,GAAG,EAAE;oBAC9C,QAAQ,EAAE,iBAAiB;iBAC5B,CAAC;YACJ,CAAC;SACF;QACD,QAAQ,EAAE;YACR,iBAAiB;YACjB,aAAa,EAAE;gBACb,MAAM,EAAE,2BAA2B;gBACnC,sBAAsB,EAAE,qCAAqC;gBAC7D,cAAc,EAAE,iCAAiC;gBACjD,wBAAwB,EAAE,CAAC,MAAM,CAAC;aACnC;SACF;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,IAAiB,EACjB,GAAwC,EACxC,UAA+D,EAAE;IAEjE,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;IACtD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE;QACpB,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAClE,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO;QACL,GAAG,OAAO;QACV,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QACvD,OAAO,EAAE,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC;KAC7F,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAAiB,EACjB,SAAc,EACd,UAAiC,EAAE;IAEnC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,EAAE;QAChE,GAAG,EAAE;YACH,GAAG,OAAO,CAAC,GAAG;YACd,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,GAAG;YACT,GAAG,OAAO,CAAC,WAAW;SACvB;QACD,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;KAClC,CAAC,CAAC;IACH,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,MAAM,GAAG,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7F,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,GAAG,MAAM,GAAG,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7F,MAAM,GAAG,GAAG,MAAM,IAAI,OAAO,CAAM,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrD,MAAM,MAAM,GAAG,CAAC,KAAa,EAAE,KAAW,EAAE,EAAE;YAC5C,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAClC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAC3B,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAC1B,IAAI,KAAK;gBAAE,MAAM,CAAC,KAAK,CAAC,CAAC;iBACpB,IAAI,KAAK;gBAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC,CAAC;QACF,MAAM,OAAO,GAAG,GAAG,EAAE;YACnB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YAC7D,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC;gBAAE,MAAM,CAAC,SAAS,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,CAAC,CAAC;QACF,MAAM,MAAM,GAAG,CAAC,KAAY,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,CAAC,IAAmB,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,MAAM,CAAC,IAAI,CAAC,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1G,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,MAAM,EAAE,CAAC,CAAC,EAClE,OAAO,CAAC,gBAAgB,IAAI,MAAM,CACnC,CAAC;QACF,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC3B,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,KAAc,EAAE,EAAE;QAChC,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;QACzB,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE;QACpB,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAClE,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,GAAG;QACH,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QAC/D,OAAO,EAAE,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC;KACrF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,OAAO,CACpB,GAAQ,EACR,OAAiB,EACjB,UAAU,GAAG,eAAe,EAC5B,KAAc;IAEd,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,EACtC,EAAE,kBAAkB,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,YAAY,EAAE,EAAE,EAAE,CACxD,CAAC;IACF,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,6BAA6B,CACpD,GAAG,EACH,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CACnE,CAAC,CAAC;IACH,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,KAA+B;IACxD,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;QAAE,OAAO;IACpC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,MAAM,OAAO,CAAC,IAAI,CAAC;QACjB,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QACpE,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;KAC3D,CAAC,CAAC;IACH,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AACrD,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@emseepea/testing",
-  "version": "0.5.3",
+  "version": "0.9.3",
   "description": "MCP integration and semantic testing helpers",
   "license": "MIT",
   "repository": {
@@ -41,6 +41,7 @@
     "test:built": "node --test test/*.test.mjs"
   },
   "dependencies": {
+    "@emseepea/server": "0.6.1",
     "@modelcontextprotocol/client": "2.0.0"
   },
   "engines": {

package/semantic/cli.mjs

@@ -1,11 +1,23 @@
 #!/usr/bin/env node
 import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
 import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { dirname, join, relative, resolve } from "node:path";
 import { discoverTests } from "./discover.mjs";
 import { modelVersion } from "./provider.mjs";
 
+const negativeFeedbackObservations = new Set([
+  "error",
+  "friction",
+  "annoyance",
+  "unnecessary_difficulty",
+  "confusion",
+  "repetition",
+  "unexpected_bad_result",
+  "capability_mismatch",
+]);
+
 const paths = [];
 let provider = process.env.EMSEEPEA_EVAL_PROVIDER ?? "claude-local";
 let output = "artifacts/llm-eval/evidence.json";
@@ -105,7 +117,7 @@ function validRecord(record, authoritative, smoke) {
   if (record.status !== "passed" || record.authoritative !== authoritative || record.smoke !== smoke
     || record.mode !== "conversation" || record.answerTrials?.length !== 3
     || !Number.isInteger(record.judgeVerdicts?.length) || record.judgeVerdicts.length < 9
-    || record.judgeVerdicts.length % 9 !== 0
+    || record.judgeVerdicts.length % 3 !== 0
     || !record.judgeVerdicts.every((judgment) => isHash(judgment.expectationSha256)
       && isHash(judgment.requestSha256) && isHash(judgment.responseSha256)
       && typeof judgment.expectedMeaning === "string" && judgment.expectedMeaning.length > 0
@@ -123,14 +135,100 @@ function validRecord(record, authoritative, smoke) {
       && typeof turn.response === "string"
       && isHash(turn.promptSha256) && isHash(turn.answerSha256)
       && isHash(turn.advertisedToolsSha256) && isHash(turn.selectedCallsSha256)
-      && isHash(turn.expectedCallsSha256)
       && Array.isArray(turn.toolCalls)
-      && JSON.stringify(turn.toolCalls.map(({ name, arguments: args }) => ({ name, arguments: args })))
-        === JSON.stringify(turn.expectedCalls)
-      && turn.toolCalls.every((call) => Object.hasOwn(call, "result"))
-      && JSON.stringify(turn.selectedTools) === JSON.stringify(turn.expectedTools)
+      && JSON.stringify(turn.selectedTools)
+        === JSON.stringify(turn.toolCalls.map(({ name }) => name))
+      && validToolAssertions(turn, isHash)
+      && validNegativeFeedbackAssertion(turn)
+      && validFeedbackDisclosure(turn, record.judgeVerdicts, trial.trial, isHash)
+      && turn.toolCalls.every((call) => Object.hasOwn(call, "result") && typeof call.isError === "boolean")
       && Array.isArray(turn.pathEvidence) && turn.pathEvidence.length === turn.toolCallCount
       && turn.pathEvidence.every(({ method, target, requestSha256, responseSha256 }) =>
         method === "tools/call" && turn.selectedTools.includes(target)
           && isHash(requestSha256) && isHash(responseSha256))));
 }
+
+function validToolAssertions(turn, isHash) {
+  if (turn.expectedOptionalFeedback === true) {
+    if (!Array.isArray(turn.expectedCalls)) return false;
+    const expectedHash = createHash("sha256").update(JSON.stringify({
+      calls: turn.expectedCalls,
+      optionalFeedback: true,
+    })).digest("hex");
+    const calls = turn.toolCalls.map(({ name, arguments: args }) => ({ name, arguments: args }));
+    const primaryCalls = calls.slice(0, turn.expectedCalls?.length);
+    const trailingCalls = calls.slice(turn.expectedCalls?.length);
+    return expectedHash === turn.expectedSelectionSha256
+      && JSON.stringify(primaryCalls) === JSON.stringify(turn.expectedCalls)
+      && (trailingCalls.length === 0
+        || (trailingCalls.length === 1 && trailingCalls[0].name === "submit-feedback"));
+  }
+  if (typeof turn.expectedOptionalTool === "string" && turn.expectedOptionalTool.trim()) {
+    const expectedHash = createHash("sha256").update(JSON.stringify({
+      optionalTool: turn.expectedOptionalTool,
+    })).digest("hex");
+    return expectedHash === turn.expectedSelectionSha256
+      && (turn.toolCalls.length === 0
+        || (turn.toolCalls.length === 1 && turn.toolCalls[0].name === turn.expectedOptionalTool));
+  }
+  if (JSON.stringify(turn.selectedTools) !== JSON.stringify(turn.expectedTools)) return false;
+  if (isHash(turn.expectedCallsSha256)) {
+    return JSON.stringify(turn.toolCalls.map(({ name, arguments: args }) => ({ name, arguments: args })))
+      === JSON.stringify(turn.expectedCalls);
+  }
+  if (!isHash(turn.expectedSelectionSha256)) return false;
+  const expectedHash = createHash("sha256").update(JSON.stringify({
+    tools: turn.expectedTools,
+    arguments: turn.expectedArguments,
+    feedback: turn.expectedFeedback,
+  })).digest("hex");
+  if (expectedHash !== turn.expectedSelectionSha256) return false;
+  for (const [name, expected] of Object.entries(turn.expectedArguments ?? {})) {
+    const matches = turn.toolCalls.filter((call) => call.name === name);
+    if (matches.length !== 1 || JSON.stringify(matches[0].arguments) !== JSON.stringify(expected)) return false;
+  }
+  if (turn.expectedFeedback) {
+    const calls = turn.toolCalls.filter((call) => call.name === "submit-feedback");
+    const detail = calls[0]?.arguments?.detail;
+    if (calls.length !== 1 || !turn.expectedFeedback.observation.includes(calls[0].arguments?.observation)
+      || typeof detail !== "string" || turn.expectedFeedback.detailIncludes.some(
+        (value) => !detail.toLowerCase().includes(value.toLowerCase()),
+      )) return false;
+  }
+  return true;
+}
+
+function validNegativeFeedbackAssertion(turn) {
+  if (turn.expectedNegativeFeedback !== false) return true;
+  const actual = turn.toolCalls.filter((call) => call.name === "submit-feedback"
+    && negativeFeedbackObservations.has(call.arguments?.observation));
+  return actual.length === 0 && JSON.stringify(actual) === JSON.stringify(turn.negativeFeedbackCalls);
+}
+
+function validFeedbackDisclosure(turn, judgments, trial, isHash) {
+  if (turn.expectedOptionalFeedback !== true) return turn.feedbackDisclosure === undefined;
+  const feedback = turn.toolCalls[turn.expectedCalls.length];
+  if (!feedback) return turn.feedbackDisclosure === undefined;
+  if (feedback.isError || typeof feedback.arguments?.observation !== "string"
+    || !feedback.arguments.observation.trim() || typeof feedback.arguments.detail !== "string"
+    || !feedback.arguments.detail.trim()) return false;
+  const expectedMeaning = feedbackDisclosureExpectation(feedback);
+  const expectationSha256 = createHash("sha256").update(expectedMeaning).digest("hex");
+  if (!isHash(turn.feedbackDisclosure?.expectationSha256)
+    || JSON.stringify(turn.feedbackDisclosure) !== JSON.stringify({
+      observation: feedback.arguments.observation,
+      detail: feedback.arguments.detail,
+      expectationSha256,
+    })) return false;
+  const matching = judgments.filter((judgment) => judgment.trial === trial
+    && judgment.turn === turn.turn && judgment.expectationSha256 === expectationSha256);
+  return matching.length === 3 && matching.every((judgment) =>
+    judgment.expectedMeaning === expectedMeaning && judgment.verdict?.pass === true);
+}
+
+function feedbackDisclosureExpectation(call) {
+  return "The final assistant response makes it clear that feedback was submitted, recorded, or noted, "
+    + "and communicates the substance "
+    + `of this specific ${call.arguments.observation} observation: ${call.arguments.detail} `
+    + "A concise, faithful summary is enough. It need not repeat every example, field, or phrase.";
+}

package/semantic/provider.mjs

@@ -84,14 +84,21 @@ export function parseNativeClaudeEvents(stdout, advertisedTools, requireInit = f
     }
     return { name: publicName, arguments: input };
   });
-  if (toolUses.length > 3) throw new Error("Model command used more than three tools");
+  if (toolUses.length > 3) {
+    throw Object.assign(new Error("Model command used more than three tools"), {
+      attemptedToolCalls: calls,
+    });
+  }
   if (init) {
     const available = [...(init.tools ?? [])].sort();
     const expected = [...advertised.keys()].sort();
+    const expectedServer = advertised.size > 0
+      ? init.mcp_servers?.length === 1
+        && init.mcp_servers[0]?.name === mcpServerName
+        && init.mcp_servers[0]?.status === "connected"
+      : init.mcp_servers?.length === 0;
     if (JSON.stringify(available) !== JSON.stringify(expected)
-      || init.mcp_servers?.length !== 1
-      || init.mcp_servers[0]?.name !== mcpServerName
-      || init.mcp_servers[0]?.status !== "connected") {
+      || !expectedServer) {
       throw new Error("Model command did not expose exactly the target MCP tools");
     }
   }
@@ -131,7 +138,10 @@ export function parseNativeClaudeEvents(stdout, advertisedTools, requireInit = f
   return {
     answer: result.result,
     calls,
-    toolResults: toolUses.map(({ id }) => toolResults.get(id).content),
+    toolResults: toolUses.map(({ id }) => {
+      const toolResult = toolResults.get(id);
+      return { content: toolResult.content, isError: toolResult.is_error === true };
+    }),
     pathEvidence,
     models: Object.keys(result.modelUsage),
     turnCount: 1,
@@ -189,7 +199,7 @@ export function modelInvocation(provider, prompt, directory) {
 
 export function conversationInvocation(provider, directory, url, tools, authToken, context) {
   const nativeTools = tools.map(({ name }) => nativeToolName(name));
-  const config = {
+  const config = nativeTools.length ? {
     mcpServers: {
       [mcpServerName]: {
         type: "http",
@@ -197,7 +207,7 @@ export function conversationInvocation(provider, directory, url, tools, authToke
         ...(authToken ? { headers: { Authorization: "Bearer ${EMSEEPEA_SEMANTIC_MCP_TOKEN}" } } : {}),
       },
     },
-  };
+  } : undefined;
   const base = modelInvocation(provider, "", directory);
   return {
     ...base,
@@ -210,7 +220,7 @@ export function conversationInvocation(provider, directory, url, tools, authToke
       "--effort", "low",
       "--max-turns", "4",
       "--strict-mcp-config",
-      "--mcp-config", JSON.stringify(config),
+      ...(config ? ["--mcp-config", JSON.stringify(config)] : []),
       "--disable-slash-commands",
       "--no-session-persistence",
       "--permission-mode", "dontAsk",
@@ -221,7 +231,7 @@ export function conversationInvocation(provider, directory, url, tools, authToke
       "--no-chrome",
       "--prompt-suggestions", "false",
     ],
-    env: { ...base.env, ...(authToken ? { EMSEEPEA_SEMANTIC_MCP_TOKEN: authToken } : {}) },
+    env: { ...base.env, ...(config && authToken ? { EMSEEPEA_SEMANTIC_MCP_TOKEN: authToken } : {}) },
   };
 }
 

package/semantic/test.d.mts

@@ -31,7 +31,26 @@ export function createConversation(
 ): Promise<SemanticConversation>;
 
 export function assertToolCalls(turn: ConversationTurn, expected: readonly ToolCall[]): void;
+/** Requires the exact primary calls and allows one trailing submit-feedback call. */
+export function assertToolCallsWithOptionalFeedback(
+  turn: ConversationTurn,
+  expected: readonly ToolCall[],
+): Promise<void>;
 export function assertNoToolCalls(turn: ConversationTurn): void;
+/** Allows no tool call or exactly one call to the named tool in each trial. */
+export function assertOptionalToolCall(turn: ConversationTurn, name: string): void;
+export function assertToolNames(turn: ConversationTurn, expected: readonly string[]): void;
+export function assertToolArguments(
+  turn: ConversationTurn,
+  name: string,
+  expected: Record<string, unknown>,
+): void;
+export function assertFeedback(
+  turn: ConversationTurn,
+  expectation: { observation: string | readonly string[]; detailIncludes: readonly string[] },
+): void;
+/** Asserts that successful turns did not submit an error, friction, or other negative observation. */
+export function assertNoNegativeFeedback(...turns: readonly ConversationTurn[]): void;
 export function assertResponseContains(turn: ConversationTurn, expected: string | readonly string[]): void;
 export function assertResponseMeaning(
   turn: ConversationTurn,

package/semantic/test.mjs

@@ -101,7 +101,15 @@ export async function createConversation(testContext, options) {
           );
           const answer = await trial.model.send(prompt);
           const calls = answer.calls;
-          trial.history.push({ user: prompt, assistant: answer.answer });
+          trial.history.push({
+            user: prompt,
+            toolCalls: calls.map((call, index) => ({
+              ...call,
+              result: answer.toolResults[index].content,
+              isError: answer.toolResults[index].isError,
+            })),
+            assistant: answer.answer,
+          });
           const record = {
             turn: trial.record.turns.length + 1,
             interactionMode: "native-mcp",
@@ -109,7 +117,8 @@ export async function createConversation(testContext, options) {
             response: answer.answer,
             toolCalls: calls.map((call, index) => ({
               ...call,
-              result: answer.toolResults[index],
+              result: answer.toolResults[index].content,
+              isError: answer.toolResults[index].isError,
             })),
             promptSha256: hash(prompt),
             answerSha256: hash(answer.answer),
@@ -141,7 +150,12 @@ export async function createConversation(testContext, options) {
           });
         }
       } catch (error) {
-        if (activeTrial) activeTrial.record.error = safeModelFailure(error);
+        if (activeTrial) {
+          activeTrial.record.error = safeModelFailure(error);
+          if (Array.isArray(error?.attemptedToolCalls)) {
+            activeTrial.record.attemptedToolCalls = error.attemptedToolCalls;
+          }
+        }
         state.failed = true;
         evidence.failedPhase = "conversation turn";
         throw new Error(`Semantic test failed during conversation turn: ${name}`);
@@ -157,11 +171,7 @@ export async function createConversation(testContext, options) {
 
 export function assertToolCalls(turn, expected) {
   const trials = turnTrials(turn);
-  if (!Array.isArray(expected) || expected.some((call) => !call || typeof call.name !== "string"
-    || !call.name.trim() || !call.arguments || typeof call.arguments !== "object"
-    || Array.isArray(call.arguments))) {
-    throw new Error("Expected tool calls must have names and object arguments");
-  }
+  validateExpectedCalls(expected);
   for (const trial of trials) {
     trial.record.expectedTools = expected.map(({ name }) => name);
     trial.record.expectedCalls = expected;
@@ -175,10 +185,161 @@ export function assertToolCalls(turn, expected) {
   }
 }
 
+export async function assertToolCallsWithOptionalFeedback(turn, expected) {
+  const trials = turnTrials(turn);
+  validateExpectedCalls(expected);
+  for (const trial of trials) {
+    trial.record.expectedTools = expected.map(({ name }) => name);
+    trial.record.expectedCalls = expected;
+    trial.record.expectedOptionalFeedback = true;
+    trial.record.expectedSelectionSha256 = hash(JSON.stringify({ calls: expected, optionalFeedback: true }));
+    const primaryCalls = trial.calls.slice(0, expected.length);
+    const trailingCalls = trial.calls.slice(expected.length);
+    try {
+      assert.deepStrictEqual(primaryCalls, expected);
+      assert.ok(trailingCalls.length <= 1);
+      if (trailingCalls.length === 1) assert.equal(trailingCalls[0].name, "submit-feedback");
+    } catch {
+      failAssertion(trials, "tool-call assertion");
+      throw new Error("Tool calls did not match the expected calls with at most one trailing feedback call");
+    }
+  }
+  let failed = false;
+  for (let trialIndex = 0; trialIndex < trials.length; trialIndex += 1) {
+    const trial = trials[trialIndex];
+    const feedback = trial.calls[expected.length];
+    if (!feedback) continue;
+    const recordedCall = trial.record.toolCalls[expected.length];
+    if (typeof feedback.arguments.observation !== "string" || !feedback.arguments.observation.trim()
+      || typeof feedback.arguments.detail !== "string" || !feedback.arguments.detail.trim()) {
+      failed = true;
+      continue;
+    }
+    const expectation = feedbackDisclosureExpectation(feedback);
+    trial.record.feedbackDisclosure = {
+      observation: feedback.arguments.observation,
+      detail: feedback.arguments.detail,
+      expectationSha256: hash(expectation),
+    };
+    const disclosureFailed = await judgeTrialMeaning(trial, trialIndex, expectation);
+    if (recordedCall.isError || disclosureFailed) failed = true;
+  }
+  if (failed) {
+    failAssertion(trials, "feedback-disclosure assertion");
+    throw new Error("Optional feedback failed or was not openly described in the response");
+  }
+}
+
 export function assertNoToolCalls(turn) {
   assertToolCalls(turn, []);
 }
 
+export function assertOptionalToolCall(turn, name) {
+  const trials = turnTrials(turn);
+  if (typeof name !== "string" || !name.trim()) {
+    throw new Error("Optional tool-call expectation needs a tool name");
+  }
+  for (const trial of trials) {
+    trial.record.expectedOptionalTool = name;
+    trial.record.expectedSelectionSha256 = hash(JSON.stringify({ optionalTool: name }));
+    if (trial.calls.length > 1 || (trial.calls.length === 1 && trial.calls[0].name !== name)) {
+      failAssertion([trial], "optional tool-call assertion");
+      throw new Error(`Expected no tool call or one ${name} call`);
+    }
+  }
+}
+
+export function assertToolNames(turn, expected) {
+  const trials = turnTrials(turn);
+  if (!Array.isArray(expected) || expected.some((name) => typeof name !== "string" || !name.trim())) {
+    throw new Error("Expected tool names must be a string array");
+  }
+  for (const trial of trials) {
+    trial.record.expectedTools = expected;
+    recordFlexibleExpectation(trial.record);
+  }
+  try {
+    for (const trial of trials) assert.deepStrictEqual(trial.calls.map(({ name }) => name), expected);
+  } catch {
+    failAssertion(trials, "tool-name assertion");
+    throw new Error("Tool names did not match the expected order and count");
+  }
+}
+
+export function assertToolArguments(turn, name, expected) {
+  const trials = turnTrials(turn);
+  if (typeof name !== "string" || !name.trim() || !expected || typeof expected !== "object"
+    || Array.isArray(expected)) {
+    throw new Error("Tool argument expectation needs a name and object arguments");
+  }
+  for (const trial of trials) {
+    const matches = trial.calls.filter((call) => call.name === name);
+    trial.record.expectedArguments ??= {};
+    trial.record.expectedArguments[name] = expected;
+    recordFlexibleExpectation(trial.record);
+    try {
+      assert.equal(matches.length, 1);
+      assert.deepStrictEqual(matches[0].arguments, expected);
+    } catch {
+      failAssertion([trial], "tool-argument assertion");
+      throw new Error(`Arguments for ${name} did not match exactly`);
+    }
+  }
+}
+
+export function assertFeedback(turn, expectation) {
+  const trials = turnTrials(turn);
+  const observations = typeof expectation?.observation === "string"
+    ? [expectation.observation]
+    : expectation?.observation;
+  const detailIncludes = expectation?.detailIncludes;
+  if (!Array.isArray(observations) || observations.length === 0
+    || !Array.isArray(detailIncludes) || detailIncludes.length === 0
+    || [...observations, ...detailIncludes].some((value) => typeof value !== "string" || !value.trim())) {
+    throw new Error("Feedback expectation needs observation and detailIncludes strings");
+  }
+  for (const trial of trials) {
+    trial.record.expectedFeedback = { observation: observations, detailIncludes };
+    recordFlexibleExpectation(trial.record);
+    const calls = trial.calls.filter(({ name }) => name === "submit-feedback");
+    const detail = calls[0]?.arguments?.detail;
+    if (calls.length !== 1
+      || !observations.includes(calls[0].arguments?.observation)
+      || typeof detail !== "string"
+      || detailIncludes.some((value) => !detail.toLowerCase().includes(value.toLowerCase()))) {
+      failAssertion([trial], "feedback assertion");
+      throw new Error("Feedback did not match the expected observation and useful detail");
+    }
+  }
+}
+
+const negativeFeedbackObservations = new Set([
+  "error",
+  "friction",
+  "annoyance",
+  "unnecessary_difficulty",
+  "confusion",
+  "repetition",
+  "unexpected_bad_result",
+  "capability_mismatch",
+]);
+
+export function assertNoNegativeFeedback(...turns) {
+  if (turns.length === 0) throw new Error("No-negative-feedback assertion needs at least one turn");
+  const trials = turns.flatMap(turnTrials);
+  for (const trial of trials) {
+    const offendingCalls = trial.calls.filter((call) =>
+      call.name === "submit-feedback"
+      && negativeFeedbackObservations.has(call.arguments?.observation));
+    trial.record.expectedNegativeFeedback = false;
+    trial.record.negativeFeedbackCalls = offendingCalls;
+    if (offendingCalls.length > 0) {
+      failAssertion([trial], "negative-feedback assertion");
+      throw new Error("A successful example interaction recorded negative feedback");
+    }
+  }
+}
+
 export function assertResponseContains(turn, expected) {
   const trials = turnTrials(turn);
   const values = typeof expected === "string" ? [expected] : expected;
@@ -211,41 +372,7 @@ export async function assertResponseMeaning(turn, expectation) {
   for (let trialIndex = 0; trialIndex < trials.length; trialIndex += 1) {
     const trial = trials[trialIndex];
     trial.record.expectedMeaning = expectation.expected;
-    for (let judgment = 1; judgment <= 3; judgment += 1) {
-      const request = judgePrompt(trial.history, expectation.expected);
-      const record = {
-        trial: trialIndex + 1,
-        turn: trial.record.turn,
-        judgment,
-        expectedMeaning: expectation.expected,
-        expectationSha256: hash(expectation.expected),
-        requestSha256: hash(request),
-      };
-      try {
-        const response = await isolatedModel(
-          trial.provider,
-          request,
-          "emseepea-judge-",
-          trial.signal,
-        );
-        Object.assign(record, {
-          models: response.models,
-          turnCount: response.turnCount,
-          providerTurnCount: response.providerTurnCount,
-          providerToolCount: response.providerToolCount,
-          responseSha256: hash(response.answer),
-        });
-        const verdict = parseJudgeVerdict(response.answer.trim());
-        record.verdict = verdict;
-        if (!verdict.pass) failed = true;
-      } catch (error) {
-        record.error = error instanceof SyntaxError || error.message === "Judge returned an invalid verdict"
-          ? "invalid judge verdict"
-          : safeModelFailure(error);
-        failed = true;
-      }
-      trial.evidence.judgeVerdicts.push(record);
-    }
+    if (await judgeTrialMeaning(trial, trialIndex, expectation.expected)) failed = true;
     trial.record.meaningAssertionCount += 1;
   }
   trials[0].state.meaningAssertions += 1;
@@ -255,6 +382,41 @@ export async function assertResponseMeaning(turn, expectation) {
   }
 }
 
+async function judgeTrialMeaning(trial, trialIndex, expected) {
+  let failed = false;
+  for (let judgment = 1; judgment <= 3; judgment += 1) {
+    const request = judgePrompt(trial.history, expected);
+    const record = {
+      trial: trialIndex + 1,
+      turn: trial.record.turn,
+      judgment,
+      expectedMeaning: expected,
+      expectationSha256: hash(expected),
+      requestSha256: hash(request),
+    };
+    try {
+      const response = await isolatedModel(trial.provider, request, "emseepea-judge-", trial.signal);
+      Object.assign(record, {
+        models: response.models,
+        turnCount: response.turnCount,
+        providerTurnCount: response.providerTurnCount,
+        providerToolCount: response.providerToolCount,
+        responseSha256: hash(response.answer),
+      });
+      const verdict = parseJudgeVerdict(response.answer.trim());
+      record.verdict = verdict;
+      if (!verdict.pass) failed = true;
+    } catch (error) {
+      record.error = error instanceof SyntaxError || error.message === "Judge returned an invalid verdict"
+        ? "invalid judge verdict"
+        : safeModelFailure(error);
+      failed = true;
+    }
+    trial.evidence.judgeVerdicts.push(record);
+  }
+  return failed;
+}
+
 function safeModelFailure(error) {
   const message = error instanceof Error ? error.message : "";
   const safeMessages = new Set([
@@ -313,7 +475,9 @@ async function closeConversation(state, evidence, output) {
   }));
   const complete = !state.failed && state.meaningAssertions > 0 && evidence.answerTrials.length === 3
     && evidence.answerTrials.every(({ turns }) => turns.length > 0
-      && turns.every((turn) => Array.isArray(turn.expectedTools)));
+      && turns.every((turn) => Array.isArray(turn.expectedTools)
+        || turn.expectedOptionalFeedback === true
+        || typeof turn.expectedOptionalTool === "string"));
   if (complete) {
     evidence.status = "passed";
   } else if (!evidence.failedPhase) {
@@ -322,7 +486,7 @@ async function closeConversation(state, evidence, output) {
   await mkdir(dirname(output), { recursive: true });
   await writeFile(output, `${JSON.stringify(evidence, null, 2)}\n`, { mode: 0o600 });
   if (!complete && !state.failed) {
-    throw new Error("Semantic conversation needs exact tool-call assertions for every turn and a meaning assertion");
+    throw new Error("Semantic conversation needs tool-selection assertions for every turn and a meaning assertion");
   }
 }
 
@@ -338,11 +502,34 @@ function turnTrials(turn) {
   return trials;
 }
 
+function validateExpectedCalls(expected) {
+  if (!Array.isArray(expected) || expected.some((call) => !call || typeof call.name !== "string"
+    || !call.name.trim() || !call.arguments || typeof call.arguments !== "object"
+    || Array.isArray(call.arguments))) {
+    throw new Error("Expected tool calls must have names and object arguments");
+  }
+}
+
+function feedbackDisclosureExpectation(call) {
+  return "The final assistant response makes it clear that feedback was submitted, recorded, or noted, "
+    + "and communicates the substance "
+    + `of this specific ${call.arguments.observation} observation: ${call.arguments.detail} `
+    + "A concise, faithful summary is enough. It need not repeat every example, field, or phrase.";
+}
+
 function failAssertion(trials, phase) {
   for (const trial of trials) trial.state.failed = true;
   trials[0].evidence.failedPhase = phase;
 }
 
+function recordFlexibleExpectation(record) {
+  record.expectedSelectionSha256 = hash(JSON.stringify({
+    tools: record.expectedTools,
+    arguments: record.expectedArguments,
+    feedback: record.expectedFeedback,
+  }));
+}
+
 function judgePrompt(history, expected) {
   return [
     "Judge whether the final assistant response communicates the complete expected meaning in this conversation.",