@b4run/testing 0.12.0 → 0.13.0

package/dist/harness.d.ts

@@ -36,6 +36,15 @@ export interface AgentHarnessOptions {
     readonly record?: boolean;
     /** Upstream base URL for record mode (no /v1 suffix). Default https://api.openai.com. */
     readonly recordUpstream?: string;
+    /**
+     * A JSON Schema the root model's final message must match — the value a
+     * Hashbrown client sends as `hashbrown.responseSchema` on every AG-UI run.
+     * Bound on every turn exactly as the server binds it, so a scripted, live or
+     * recorded run sends the model the same request production does. Only an
+     * `agent` route on a provider that supports it can take one; any other
+     * route fails the run rather than running unconstrained.
+     */
+    readonly responseSchema?: Readonly<Record<string, unknown>>;
 }
 export interface AgentHarness {
     readonly baseUrl: string;

package/dist/harness.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"harness.d.ts","sourceRoot":"","sources":["../src/harness.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,KAAK,aAAa,EASnB,MAAM,oBAAoB,CAAA;AAG3B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAE/C,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAErE,OAAO,EAAE,KAAK,cAAc,EAAoB,MAAM,iBAAiB,CAAA;AA0BvE,qFAAqF;AACrF,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,2DAA2D;IAC3D,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,oEAAoE;IACpE,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,aAAa,EAAE,CAAA;CAC3C;AAED;;;;GAIG;AACH,MAAM,MAAM,6BAA6B,GACrC,WAAW,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,GACxC,CAAC,CACC,GAAG,EAAE,mBAAmB,KAEtB,WAAW,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,GACxC,SAAS,GACT,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAA;AAEtE,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAA;IAC9B;;;;;OAKG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,6BAA6B,CAAA;IAC1D;;;OAGG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAA;IACvB,uFAAuF;IACvF,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAA;IACzB,yFAAyF;IACzF,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAA;CACjC;AAYD,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,GAAG,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,UAAU,GAAG,aAAa,CAAA;KAAE,GAAG,OAAO,CAAC,cAAc,CAAC,CAAA;IAC5F,MAAM,CAAC,IAAI,EAAE;QACX,MAAM,EAAE,SAAS,aAAa,EAAE,CAAA;QAChC,QAAQ,CAAC,EAAE,UAAU,GAAG,aAAa,CAAA;KACtC,GAAG,OAAO,CAAC,cAAc,CAAC,CAAA;IAC3B,KAAK,IAAI,IAAI,CAAA;IACb,KAAK,CAAC,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACxE,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IACtC,4FAA4F;IAC5F,mBAAmB,IAAI,UAAU,CAAA;CAClC;AAED,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,YAAY,CAAC,CA0M5F"}
+{"version":3,"file":"harness.d.ts","sourceRoot":"","sources":["../src/harness.ts"],"names":[],"mappings":"AACA,OAAO,EAGL,KAAK,aAAa,EAUnB,MAAM,oBAAoB,CAAA;AAG3B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAE/C,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAErE,OAAO,EAAE,KAAK,cAAc,EAAoB,MAAM,iBAAiB,CAAA;AA0BvE,qFAAqF;AACrF,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,2DAA2D;IAC3D,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;IACvB,oEAAoE;IACpE,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,aAAa,EAAE,CAAA;CAC3C;AAED;;;;GAIG;AACH,MAAM,MAAM,6BAA6B,GACrC,WAAW,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,GACxC,CAAC,CACC,GAAG,EAAE,mBAAmB,KAEtB,WAAW,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,GACxC,SAAS,GACT,OAAO,CAAC,WAAW,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAA;AAEtE,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,CAAC,EAAE,UAAU,CAAA;IAC9B;;;;;OAKG;IACH,QAAQ,CAAC,iBAAiB,CAAC,EAAE,6BAA6B,CAAA;IAC1D;;;OAGG;IACH,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAA;IACvB,uFAAuF;IACvF,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAA;IACzB,yFAAyF;IACzF,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAA;IAChC;;;;;;;OAOG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;CAC5D;AAYD,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,GAAG,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,UAAU,GAAG,aAAa,CAAA;KAAE,GAAG,OAAO,CAAC,cAAc,CAAC,CAAA;IAC5F,MAAM,CAAC,IAAI,EAAE;QACX,MAAM,EAAE,SAAS,aAAa,EAAE,CAAA;QAChC,QAAQ,CAAC,EAAE,UAAU,GAAG,aAAa,CAAA;KACtC,GAAG,OAAO,CAAC,cAAc,CAAC,CAAA;IAC3B,KAAK,IAAI,IAAI,CAAA;IACb,KAAK,CAAC,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,iBAAiB,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACxE,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IACtC,4FAA4F;IAC5F,mBAAmB,IAAI,UAAU,CAAA;CAClC;AAED,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,YAAY,CAAC,CA2N5F"}

package/dist/harness.js

@@ -1,5 +1,5 @@
 import { randomUUID } from "node:crypto";
-import { __resetMaterializedAgentsForTests, __resetRouteLoadCachesForTests, createRuntimeRegistry, readPendingInterrupts, resolveCheckpointer, resolvePendingResume, resolveSandboxManager, runTypegen, streamResolvedRoute, } from "@b4run/cli/runtime";
+import { __resetMaterializedAgentsForTests, __resetRouteLoadCachesForTests, createRuntimeRegistry, readPendingInterrupts, readResponseFormat, resolveCheckpointer, resolvePendingResume, resolveSandboxManager, runTypegen, streamResolvedRoute, } from "@b4run/cli/runtime";
 import { __clearB4ConfigCacheForTests } from "@b4run/core";
 import { discoverRoutes } from "@b4run/core/node";
 import { createAimock } from "./aimock-runner.js";
@@ -37,6 +37,18 @@ async function resolveMiddlewareContext(option, run) {
 export async function createAgentHarness(options) {
     const live = options.live ?? false;
     const record = options.record ?? false;
+    const responseSchema = options.responseSchema;
+    if (responseSchema !== undefined &&
+        (typeof responseSchema !== "object" || responseSchema === null || Array.isArray(responseSchema))) {
+        throw new Error("createAgentHarness: `responseSchema` must be a JSON Schema object");
+    }
+    // Read through the server's own parser so the bound format — including the
+    // provider-facing schema name — is the one an AG-UI run would carry.
+    const responseFormatResult = readResponseFormat(responseSchema !== undefined ? { hashbrown: { responseSchema } } : undefined);
+    if (!responseFormatResult.ok) {
+        throw new Error(`createAgentHarness: ${responseFormatResult.message}`);
+    }
+    const responseFormat = responseFormatResult.responseFormat;
     // Guard: live mode requires a real API key before doing anything else.
     if (live && !process.env.OPENAI_API_KEY) {
         throw new Error("createAgentHarness({ live: true }) requires OPENAI_API_KEY to be set in the environment");
@@ -144,6 +156,7 @@ export async function createAgentHarness(options) {
             ...(sandboxManager ? { sandboxManager } : {}),
             ...(resolvedResume ? { resume: resolvedResume } : {}),
             ...(middlewareContext !== undefined ? { middlewareContext } : {}),
+            ...(responseFormat !== undefined ? { responseFormat } : {}),
         };
         const stream = streamResolvedRoute(streamArgs);
         const result = await collectRunResult(stream, threadId);

package/dist/record-fixtures.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"record-fixtures.d.ts","sourceRoot":"","sources":["../src/record-fixtures.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAiB,cAAc,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AAErF,2FAA2F;AAC3F,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,OAAO,EAAE;QAChB,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;YAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;SAAE,CAAC,CAAA;KACxF,CAAA;IACD,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAA;CAClC;AAaD;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,SAAS,SAAS,EAAE,GAAG,UAAU,CAYjF"}
+{"version":3,"file":"record-fixtures.d.ts","sourceRoot":"","sources":["../src/record-fixtures.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAiB,cAAc,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AAErF,2FAA2F;AAC3F,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,OAAO,EAAE;QAChB,QAAQ,CAAC,QAAQ,CAAC,EAAE,aAAa,CAAC;YAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;YACrB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;SAC1B,CAAC,CAAA;KACH,CAAA;IACD,QAAQ,CAAC,QAAQ,EAAE,cAAc,CAAA;CAClC;AAaD;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,SAAS,SAAS,EAAE,GAAG,UAAU,CAcjF"}

package/dist/record-fixtures.js

@@ -1,3 +1,4 @@
+import { LLMock } from "@copilotkit/aimock";
 function firstUserMessage(req) {
     for (const m of req.messages ?? []) {
         if (m.role === "user" && typeof m.content === "string")
@@ -17,7 +18,7 @@ function hasToolResult(req) {
  * thread); `hasToolResult` is whether a tool-role message is already present.
  */
 export function recordingsToFixtures(recordings) {
-    return recordings.map((rec, turnIndex) => {
+    const fixtures = recordings.map((rec, turnIndex) => {
         const userMessage = firstUserMessage(rec.request);
         return {
             match: {
@@ -28,4 +29,62 @@ export function recordingsToFixtures(recordings) {
             response: rec.response,
         };
     });
+    assertReplayable(fixtures);
+    return fixtures;
+}
+/** aimock's own load-time error for one fixture, or undefined when it loads. */
+function replayError(fixtures) {
+    try {
+        // The same call replay makes (see createAimock), on a mock that is never
+        // started — so a recording is held to exactly the rules its replay will be.
+        new LLMock({ port: 0 }).addFixturesFromJSON(fixtures);
+        return undefined;
+    }
+    catch (error) {
+        return error instanceof Error ? error.message : String(error);
+    }
+}
+/**
+ * Refuse a recording that replay would refuse (#778). aimock rejects a
+ * fixture whose `content` is an empty string, because its recorder writes one
+ * for several different upstream outcomes — a genuinely empty reply, a
+ * refusal it does not capture, a reply cut off at the token limit — and a
+ * tape cannot say which. Failing here, while the recording is being made,
+ * names the turn instead of leaving a tape that breaks on its next replay.
+ */
+function assertReplayable(fixtures) {
+    const whole = replayError(fixtures);
+    if (whole === undefined)
+        return;
+    const turns = fixtures.flatMap((fixture, turnIndex) => {
+        const error = replayError([fixture]);
+        if (error === undefined)
+            return [];
+        const detail = describeValidationError(error);
+        const where = fixture.match.userMessage !== undefined
+            ? `turn ${turnIndex} of ${JSON.stringify(fixture.match.userMessage)}`
+            : `turn ${turnIndex}`;
+        return [`  - ${where}: ${detail}`];
+    });
+    const lines = turns.length > 0 ? turns.join("\n") : `  - ${describeValidationError(whole)}`;
+    throw new Error(`Recorded fixtures would not replay: aimock rejects them on load.\n${lines}\n` +
+        "An empty assistant message usually means the model refused (a strict response " +
+        "schema reports refusals outside `content`), ran out of output tokens, or was " +
+        "given nothing left to do. If the run's answer is a tool call, export " +
+        "`returnDirect = true` from that tool so the run ends on its result instead of " +
+        "asking the model for a closing message.");
+}
+/** Reduce aimock's `Fixture validation failed: [...]` to its messages. */
+function describeValidationError(error) {
+    const prefix = "Fixture validation failed: ";
+    if (!error.startsWith(prefix))
+        return error;
+    try {
+        const issues = JSON.parse(error.slice(prefix.length));
+        const messages = issues.map((issue) => issue.message).filter((m) => typeof m === "string");
+        return messages.length > 0 ? messages.join("; ") : error;
+    }
+    catch {
+        return error;
+    }
 }

package/dist/thread-access-harness.d.ts

@@ -1,4 +1,4 @@
-import type { ThreadAccessPolicy, ThreadAccessResult, ThreadAction, ThreadOperation, ThreadSubject } from "@b4run/sdk";
+import type { ThreadAccessPolicy, ThreadAccessRequestedWorkspace, ThreadAccessResult, ThreadAction, ThreadOperation, ThreadSubject } from "@b4run/sdk";
 /** A partial `ThreadAccessRequest`: only `action` is required. */
 export interface ThreadAccessCheckSpec {
     readonly action: ThreadAction;
@@ -10,6 +10,13 @@ export interface ThreadAccessCheckSpec {
     readonly method?: string;
     readonly url?: string;
     readonly requestedMetadata?: Readonly<Record<string, unknown>>;
+    /**
+     * The workspace the request stages or chooses — see
+     * `ThreadAccessRequest.requestedWorkspace`: `{ sourceDigest }` for a
+     * `workspace.source.put`, the whole reference for a `thread.create` that
+     * names one.
+     */
+    readonly requestedWorkspace?: ThreadAccessRequestedWorkspace;
     /**
      * Whether this request continues a parked turn — see
      * `ThreadAccessRequest.resuming`. Defaults to `false`, the value every

package/dist/thread-access-harness.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"thread-access-harness.d.ts","sourceRoot":"","sources":["../src/thread-access-harness.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,kBAAkB,EAElB,kBAAkB,EAClB,YAAY,EACZ,eAAe,EACf,aAAa,EACd,MAAM,YAAY,CAAA;AAEnB,kEAAkE;AAClE,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAA;IAC7B,0DAA0D;IAC1D,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,CAAA;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,aAAa,CAAA;IAC/B,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IACnD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,iBAAiB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;IAC9D;;;;;OAKG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;CAC5B;AAED,MAAM,WAAW,mBAAmB;IAClC;;;;OAIG;IACH,KAAK,CAAC,IAAI,EAAE,qBAAqB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAA;CAChE;AAeD;;;;;;;GAOG;AACH,wBAAgB,yBAAyB,CAAC,OAAO,EAAE;IACjD,QAAQ,CAAC,MAAM,EAAE,kBAAkB,CAAA;CACpC,GAAG,mBAAmB,CAmBtB"}
+{"version":3,"file":"thread-access-harness.d.ts","sourceRoot":"","sources":["../src/thread-access-harness.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACV,kBAAkB,EAElB,8BAA8B,EAC9B,kBAAkB,EAClB,YAAY,EACZ,eAAe,EACf,aAAa,EACd,MAAM,YAAY,CAAA;AAEnB,kEAAkE;AAClE,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAA;IAC7B,0DAA0D;IAC1D,QAAQ,CAAC,SAAS,CAAC,EAAE,eAAe,CAAA;IACpC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,aAAa,CAAA;IAC/B,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IACnD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,iBAAiB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAA;IAC9D;;;;;OAKG;IACH,QAAQ,CAAC,kBAAkB,CAAC,EAAE,8BAA8B,CAAA;IAC5D;;;;;OAKG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;CAC5B;AAED,MAAM,WAAW,mBAAmB;IAClC;;;;OAIG;IACH,KAAK,CAAC,IAAI,EAAE,qBAAqB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAA;CAChE;AAeD;;;;;;;GAOG;AACH,wBAAgB,yBAAyB,CAAC,OAAO,EAAE;IACjD,QAAQ,CAAC,MAAM,EAAE,kBAAkB,CAAA;CACpC,GAAG,mBAAmB,CAoBtB"}

package/dist/thread-access-harness.js

@@ -30,6 +30,7 @@ export function createThreadAccessHarness(options) {
                 method: spec.method ?? defaultMethod(spec.action),
                 operation,
                 requestedMetadata: spec.requestedMetadata,
+                requestedWorkspace: spec.requestedWorkspace,
                 resuming: spec.resuming ?? false,
                 thread: spec.thread,
                 threadId: spec.threadId,