@stigmer/runner 0.1.0-dev.20261017020314.g644422724c27

package/src/shared/eval-case/__tests__/case-archive.test.ts

@@ -0,0 +1,159 @@
+/**
+ * Pins how a try reads its case from its source's archive
+ * (`case-archive.ts`): the archive at the eval's digest is fetched, verified
+ * and indexed by cleaned path; a plugin that cannot be read, moved past the
+ * eval's version, records no archive, cannot be fetched, holds no bytes or
+ * serves bytes of another digest is refused with a sentence naming why. An
+ * Eval's archive is read by its reference at the stamped digest, never via
+ * a plugin: over the download URL, inline when the server has no transfer
+ * lane (`FailedPrecondition`) or predates it (`Unimplemented`), refused
+ * naming the Eval when the read is (a caller who may not view it), when it
+ * holds no bytes, or when the bytes are another version's. The index
+ * answers the root (`""`) with every file, an Eval's suite being its root.
+ */
+
+import { createHash } from "node:crypto";
+import { create } from "@bufbuild/protobuf";
+import { Code, ConnectError } from "@connectrpc/connect";
+import { PluginSchema, type Plugin } from "@stigmer/apis/ai/stigmer/agentic/plugin/v1/api_pb";
+import { EvalCaseContextSchema } from "@stigmer/apis/ai/stigmer/agentic/run/v1/eval_case_pb";
+import { buildZip } from "@stigmer/zip-structure/testing";
+import { describe, expect, it } from "vitest";
+import type { StigmerClient } from "../../../client/stigmer-client.js";
+import { ApiResourceKind } from "@stigmer/apis/ai/stigmer/commons/apiresource/apiresourcekind/api_resource_kind_pb";
+import { caseArchiveOf, CaseArchiveError, fetchCaseArchive, joinCasePath } from "../case-archive.js";
+
+const ARCHIVE = buildZip([
+  { name: "evals/fix/case.yaml", content: "name: fix\n" },
+  { name: "evals/fix/resources/b.md", content: "B" },
+  { name: "evals/fix/resources/a.md", content: "A" },
+]);
+const DIGEST = createHash("sha256").update(ARCHIVE).digest("hex");
+
+const context = (digest = DIGEST) => create(EvalCaseContextSchema, { pluginId: "plg_1", pluginDigest: digest });
+
+const pluginRecord = (status: { digest?: string; artifactStorageKey?: string }): Plugin =>
+  create(PluginSchema, { status: { digest: DIGEST, artifactStorageKey: "plugins/abc.zip", ...status } });
+
+/** A client whose download lane is absent, so the unary read answers with `bytes`. */
+function clientServing(fields: {
+  plugin?: () => Promise<Plugin>;
+  bytes?: Uint8Array;
+  fetchError?: Error;
+}): StigmerClient {
+  return {
+    getPlugin: fields.plugin ?? (() => Promise.resolve(pluginRecord({}))),
+    getPluginArtifactDownloadUrl: () => Promise.reject(fields.fetchError ?? new ConnectError("no lane", Code.Unimplemented)),
+    getPluginArtifact: () => Promise.resolve({ artifact: fields.bytes ?? ARCHIVE }),
+  } as unknown as StigmerClient;
+}
+
+describe("fetchCaseArchive", () => {
+  it("indexes the verified archive's files by path", async () => {
+    const archive = await fetchCaseArchive(clientServing({}), context());
+    expect(archive.text("evals/fix/case.yaml")).toBe("name: fix\n");
+    expect(archive.under("evals/fix/resources").map(([path]) => path)).toEqual(["a.md", "b.md"]);
+  });
+
+  it.each([
+    ["the plugin cannot be read", { plugin: () => Promise.reject(new Error("not found")) }, context(), "the plugin could not be read (not found)"],
+    ["the read fails with a non-error", { plugin: () => Promise.reject("gone") }, context(), "the plugin could not be read (gone)"],
+    ["the plugin moved past the eval", {}, context("b".repeat(64)), "the plugin has moved past the eval's version"],
+    ["the plugin records no archive", { plugin: () => Promise.resolve(pluginRecord({ artifactStorageKey: "" })) }, context(), "the plugin records no archive"],
+    ["the fetch fails", { fetchError: new Error("connection reset") }, context(), "the archive could not be fetched (connection reset)"],
+    ["the server holds no bytes", { bytes: new Uint8Array() }, context(), "the server holds no archive for the plugin"],
+    ["the bytes are another version's", { bytes: buildZip([{ name: "x", content: "y" }]) }, context(), "the fetched archive does not match the eval's version"],
+  ] as const)("refuses when %s", async (_why, fields, ctx, detail) => {
+    const refused = fetchCaseArchive(clientServing(fields), ctx);
+    await expect(refused).rejects.toBeInstanceOf(CaseArchiveError);
+    await expect(refused).rejects.toThrow(`the eval run's case could not be read: ${detail}`);
+  });
+
+  it("refuses a plugin whose status carries no digest", async () => {
+    const client = clientServing({ plugin: () => Promise.resolve(create(PluginSchema, {})) });
+    await expect(fetchCaseArchive(client, context())).rejects.toThrow("the plugin has moved past the eval's version");
+  });
+});
+
+const EVAL_ID = "evs_1";
+const evalContext = (digest = DIGEST) => create(EvalCaseContextSchema, { evalId: EVAL_ID, evalDigest: digest, caseDir: "fix" });
+
+/** A client serving an Eval's reads, recording the references they named; `getPlugin` is never asked. */
+function evalClient(fields: {
+  mint?: () => Promise<{ url: string; sizeBytes: bigint }>;
+  inline?: () => Promise<{ artifact: Uint8Array }>;
+  refs?: Array<{ id: string; version: string; kind: number }>;
+}): StigmerClient {
+  const record = (ref: { id: string; version: string; kind: number }) => fields.refs?.push({ id: ref.id, version: ref.version, kind: ref.kind });
+  return {
+    getPlugin: () => Promise.reject(new Error("an Eval's case reads no plugin")),
+    getEvalDownloadUrl: (ref: { id: string; version: string; kind: number }) => {
+      record(ref);
+      return (fields.mint ?? (() => Promise.reject(new ConnectError("no lane", Code.FailedPrecondition))))();
+    },
+    downloadEval: (ref: { id: string; version: string; kind: number }) => {
+      record(ref);
+      return (fields.inline ?? (() => Promise.resolve({ artifact: ARCHIVE })))();
+    },
+  } as unknown as StigmerClient;
+}
+
+const dataUrl = (bytes: Uint8Array): string => `data:application/zip;base64,${Buffer.from(bytes).toString("base64")}`;
+
+describe("fetchCaseArchive for an Eval's case", () => {
+  it("reads the Eval at its stamped digest over the download URL, verified and indexed", async () => {
+    const refs: Array<{ id: string; version: string; kind: number }> = [];
+    const client = evalClient({ mint: () => Promise.resolve({ url: dataUrl(ARCHIVE), sizeBytes: BigInt(ARCHIVE.length) }), refs });
+    const archive = await fetchCaseArchive(client, evalContext());
+    expect(archive.text("evals/fix/case.yaml")).toBe("name: fix\n");
+    expect(refs).toEqual([{ id: EVAL_ID, version: DIGEST, kind: ApiResourceKind.eval }]);
+  });
+
+  it.each([
+    ["has no transfer lane", Code.FailedPrecondition],
+    ["predates the transfer lane", Code.Unimplemented],
+  ])("reads it inline when the server %s", async (_why, code) => {
+    const client = evalClient({ mint: () => Promise.reject(new ConnectError("no lane", code)) });
+    const archive = await fetchCaseArchive(client, evalContext());
+    expect(archive.under("evals/fix/resources").map(([path]) => path)).toEqual(["a.md", "b.md"]);
+  });
+
+  it.each([
+    [
+      "the read is refused",
+      { mint: () => Promise.reject(new ConnectError("unauthorized to get eval", Code.PermissionDenied)) },
+      `eval ${EVAL_ID} could not be read ([permission_denied] unauthorized to get eval)`,
+    ],
+    ["the server holds no bytes", { inline: () => Promise.resolve({ artifact: new Uint8Array() }) }, `the server holds no archive for eval ${EVAL_ID}`],
+    [
+      "the bytes are another version's",
+      { inline: () => Promise.resolve({ artifact: buildZip([{ name: "x", content: "y" }]) }) },
+      "the fetched archive does not match the eval's version",
+    ],
+  ] as const)("refuses when %s, naming the Eval", async (_why, fields, detail) => {
+    const refused = fetchCaseArchive(evalClient(fields), evalContext());
+    await expect(refused).rejects.toBeInstanceOf(CaseArchiveError);
+    await expect(refused).rejects.toThrow(`the eval run's case could not be read: ${detail}`);
+  });
+});
+
+describe("caseArchiveOf", () => {
+  it("answers undefined for a path the archive lacks", () => {
+    const archive = caseArchiveOf([]);
+    expect([archive.read("a"), archive.text("a"), archive.under("a")]).toEqual([undefined, undefined, []]);
+  });
+
+  it("answers the root with every file, an Eval's suite being its archive's root", () => {
+    const archive = caseArchiveOf([
+      { path: "fix/prompt.md", content: new Uint8Array([1]) },
+      { path: "mocks/billing/refund.md", content: new Uint8Array([2]) },
+    ]);
+    expect(archive.under("").map(([path]) => path)).toEqual(["fix/prompt.md", "mocks/billing/refund.md"]);
+  });
+});
+
+describe("joinCasePath", () => {
+  it("joins under a directory, and names a root path as itself", () => {
+    expect([joinCasePath("evals", "mocks/x.md"), joinCasePath("", "mocks/x.md")]).toEqual(["evals/mocks/x.md", "mocks/x.md"]);
+  });
+});

package/src/shared/eval-case/__tests__/mock-endpoint-failure.test.ts

@@ -0,0 +1,65 @@
+/**
+ * Pins the mock endpoint's last resort (`mock-endpoint.ts` `serve`): a
+ * request the MCP transport fails on is logged and answered 500, so a
+ * stand-in's failure is the engine's tool error and never the runner's
+ * crash. The transport is faked here; the endpoint's real paths are pinned
+ * in `mock-endpoint.test.ts`.
+ */
+
+import { create } from "@bufbuild/protobuf";
+import {
+  EvalCaseContextSchema,
+  EvalMockKind,
+  EvalMockServerSchema,
+  EvalMockToolSchema,
+} from "@stigmer/apis/ai/stigmer/agentic/run/v1/eval_case_pb";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { caseArchiveOf } from "../case-archive.js";
+
+vi.mock("@modelcontextprotocol/sdk/server/streamableHttp.js", () => ({
+  StreamableHTTPServerTransport: class {
+    start(): Promise<void> {
+      return Promise.resolve();
+    }
+    close(): Promise<void> {
+      return Promise.resolve();
+    }
+    send(): Promise<void> {
+      return Promise.resolve();
+    }
+    handleRequest(): Promise<void> {
+      return Promise.reject(new Error("transport broke"));
+    }
+  },
+}));
+
+const { startMockEndpoint } = await import("../mock-endpoint.js");
+
+let close: (() => Promise<void>) | undefined;
+afterEach(async () => {
+  await close?.();
+  close = undefined;
+  vi.restoreAllMocks();
+});
+
+describe("the mock endpoint's failed request", () => {
+  it("is logged and answered 500", async () => {
+    const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
+    const context = create(EvalCaseContextSchema, {
+      pluginId: "plg_1",
+      mocks: [
+        create(EvalMockServerSchema, {
+          server: "tracker",
+          segment: "plugin_kit_tracker",
+          tools: [create(EvalMockToolSchema, { tool: "create_issue", file: "evals/mocks/tracker/create_issue.md", kind: EvalMockKind.fixed })],
+        }),
+      ],
+    });
+    const endpoint = await startMockEndpoint({ context, archive: caseArchiveOf([]), agentModel: () => Promise.reject(new Error("unused")) });
+    close = endpoint?.close;
+    const server = endpoint?.servers[0];
+    const response = await fetch(server?.url ?? "", { method: "POST", body: "{}", headers: { ...server?.headers, "content-type": "application/json" } });
+    expect(response.status).toBe(500);
+    expect(warn).toHaveBeenCalledWith("[eval-case mocks] request failed: transport broke");
+  });
+});

package/src/shared/eval-case/__tests__/mock-endpoint.test.ts

@@ -0,0 +1,269 @@
+/**
+ * Pins the eval-case mock endpoint (`mock-endpoint.ts`) through a real
+ * MCP client over loopback: each stand-in lists only its mocked tools, with
+ * the saved schema where there is one; a fixed mock renders its body and
+ * fixtures (the case's before the suite's); an error mock answers as a tool
+ * error; a violated `expect:` stops the try with the guard's sentence; an
+ * agent mock answers through the model, reports its cost, keeps the
+ * server's history, ends the run when the model does, and stops the try
+ * past the shared budget; a call without the token is refused, and so is a
+ * path naming no stand-in; a case with no mocks starts no endpoint; an
+ * unmocked tool, a call before the turn binds the endpoint and a fixture the
+ * archive lacks answer as tool errors. An Eval's stand-ins carry the real
+ * server's home (the agent's own, or a plugin's) and read their fixtures
+ * from a suite at the archive's root.
+ */
+
+import { afterEach, describe, expect, it } from "vitest";
+import { create } from "@bufbuild/protobuf";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
+import {
+  EvalCaseContextSchema,
+  EvalMockAgentHomeSchema,
+  EvalMockKind,
+  EvalMockPluginHomeSchema,
+  EvalMockServerSchema,
+  EvalMockToolSchema,
+} from "@stigmer/apis/ai/stigmer/agentic/run/v1/eval_case_pb";
+
+import type { MockAbort } from "../../../harness/stop-controller.js";
+import type { AgentMockModel, AgentMockRequest } from "../agent-mock.js";
+import { caseArchiveOf } from "../case-archive.js";
+import { type MockEndpoint, startMockEndpoint } from "../mock-endpoint.js";
+
+const encode = (text: string): Uint8Array => new TextEncoder().encode(text);
+
+const ARCHIVE = caseArchiveOf([
+  { path: "evals/mocks/tracker/create_issue.md", content: encode("---\nexpect:\n  title: string\n---\nCreated #4821: {{input.title}}") },
+  { path: "evals/mocks/tracker/close_issue.md", content: encode("---\nerror: true\n---\nNo such issue.") },
+  { path: "evals/mocks/tracker/_tools.json", content: encode(JSON.stringify({ tools: [{ name: "create_issue", description: "Create one", inputSchema: { type: "object", properties: { title: { type: "string" } } } }] })) },
+  { path: "evals/mocks/orders/lookup_order.md", content: encode("{{file:fixtures/{input.id}.json}}") },
+  { path: "evals/mocks/orders/fixtures/7.json", content: encode('{"from":"suite"}') },
+  { path: "evals/fix/mocks/orders/fixtures/7.json", content: encode('{"from":"case"}') },
+  { path: "evals/mocks/orders/_server.md", content: encode("---\ntype: agent\ntools: [refund]\n---\nYou refund orders.") },
+  { path: "evals/mocks/orders/receipt.md", content: encode("{{file:fixtures/missing.json}}") },
+]);
+
+const tool = (fields: { tool: string; file: string; kind: EvalMockKind; error?: boolean; expectJson?: string; abortWhen?: string }) =>
+  create(EvalMockToolSchema, fields);
+
+function contextOf(budget: number) {
+  return create(EvalCaseContextSchema, {
+    pluginId: "plg_1",
+    pluginName: "kit",
+    pluginDigest: "a".repeat(64),
+    suiteDir: "evals",
+    caseDir: "evals/fix",
+    agentMockCallBudget: budget,
+    mocks: [
+      create(EvalMockServerSchema, {
+        server: "tracker",
+        segment: "plugin_kit_tracker",
+        toolsFile: "evals/mocks/tracker/_tools.json",
+        tools: [
+          tool({ tool: "close_issue", file: "evals/mocks/tracker/close_issue.md", kind: EvalMockKind.fixed, error: true }),
+          tool({ tool: "create_issue", file: "evals/mocks/tracker/create_issue.md", kind: EvalMockKind.fixed, expectJson: '{"title":"string"}' }),
+        ],
+      }),
+      create(EvalMockServerSchema, {
+        server: "orders",
+        segment: "plugin_kit_orders",
+        tools: [
+          tool({ tool: "lookup_order", file: "evals/mocks/orders/lookup_order.md", kind: EvalMockKind.fixed }),
+          tool({ tool: "receipt", file: "evals/mocks/orders/receipt.md", kind: EvalMockKind.fixed }),
+          tool({ tool: "refund", file: "evals/mocks/orders/_server.md", kind: EvalMockKind.agent, abortWhen: "the order is unknown" }),
+        ],
+      }),
+    ],
+  });
+}
+
+let endpoint: MockEndpoint | undefined;
+afterEach(async () => {
+  await endpoint?.close();
+  endpoint = undefined;
+});
+
+async function started(budget: number, agentModel: AgentMockModel) {
+  const aborts: MockAbort[] = [];
+  const costs: number[] = [];
+  endpoint = await startMockEndpoint({ context: contextOf(budget), archive: ARCHIVE, agentModel });
+  if (endpoint === undefined) throw new Error("expected an endpoint");
+  endpoint.bind({ onAbort: (abort) => aborts.push(abort), onCost: (cost) => costs.push(cost) });
+  const connect = async (segment: string): Promise<Client> => {
+    const server = endpoint?.servers.find((s) => s.slug === segment);
+    if (server?.url === undefined) throw new Error(`no stand-in ${segment}`);
+    const client = new Client({ name: "test", version: "1.0.0" });
+    await client.connect(new StreamableHTTPClientTransport(new URL(server.url), { requestInit: { headers: server.headers } }));
+    return client;
+  };
+  return { aborts, costs, connect };
+}
+
+const text = (result: Awaited<ReturnType<Client["callTool"]>>): string =>
+  (result.content as { type: string; text: string }[]).map((part) => part.text).join("");
+
+const noModel: AgentMockModel = () => Promise.reject(new Error("no model in this test"));
+
+describe("the mock endpoint", () => {
+  it("registers each stand-in under the plugin server's own name, for its mocked tools only", async () => {
+    const { connect } = await started(4, noModel);
+    expect(endpoint?.servers.map((s) => [s.slug, s.home])).toEqual([
+      ["plugin_kit_tracker", { kind: "plugin", pluginId: "plg_1", plugin: "kit", server: "tracker" }],
+      ["plugin_kit_orders", { kind: "plugin", pluginId: "plg_1", plugin: "kit", server: "orders" }],
+    ]);
+    expect(endpoint?.listed).toEqual([
+      { server: "plugin_kit_tracker", tools: ["close_issue", "create_issue"] },
+      { server: "plugin_kit_orders", tools: ["lookup_order", "receipt", "refund"] },
+    ]);
+    const tracker = await connect("plugin_kit_tracker");
+    const { tools } = await tracker.listTools();
+    expect(tools.map((t) => [t.name, t.description, t.inputSchema])).toEqual([
+      ["close_issue", "The close_issue tool of tracker.", { type: "object", additionalProperties: true }],
+      ["create_issue", "Create one", { type: "object", properties: { title: { type: "string" } } }],
+    ]);
+    await tracker.close();
+  });
+
+  it("answers fixed mocks with their rendered body, fixtures and tool errors", async () => {
+    const { connect, aborts } = await started(4, noModel);
+    const tracker = await connect("plugin_kit_tracker");
+    expect(text(await tracker.callTool({ name: "create_issue", arguments: { title: "Crash" } }))).toBe("Created #4821: Crash");
+    const closed = await tracker.callTool({ name: "close_issue", arguments: {} });
+    expect([closed.isError, text(closed)]).toEqual([true, "No such issue."]);
+    const orders = await connect("plugin_kit_orders");
+    expect(text(await orders.callTool({ name: "lookup_order", arguments: { id: "7" } }))).toBe('{"from":"case"}');
+    expect(aborts).toEqual([]);
+    await tracker.close();
+    await orders.close();
+  });
+
+  it("stops the try when a call breaks the mock's guard", async () => {
+    const { connect, aborts } = await started(4, noModel);
+    const tracker = await connect("plugin_kit_tracker");
+    const result = await tracker.callTool({ name: "create_issue", arguments: { title: 7 } });
+    expect(text(result)).toBe("This call stopped the test: input.title must be a string");
+    expect(aborts).toEqual([{ server: "tracker", tool: "create_issue", reason: "input.title must be a string" }]);
+    await tracker.close();
+  });
+
+  it("answers agent mocks through the model, with the server's history, cost, abort and budget", async () => {
+    const asked: AgentMockRequest[] = [];
+    const model: AgentMockModel = (request) => {
+      asked.push(request);
+      const input = request.input as { id?: string };
+      return Promise.resolve(
+        input.id === "unknown"
+          ? { kind: "abort", reason: "order unknown", costUsd: 0.01 }
+          : { kind: "answer", text: `Refunded ${input.id ?? ""}`, costUsd: 0.02 },
+      );
+    };
+    const { connect, aborts, costs } = await started(2, model);
+    const orders = await connect("plugin_kit_orders");
+    expect(text(await orders.callTool({ name: "refund", arguments: { id: "1" } }))).toBe("Refunded 1");
+    await orders.callTool({ name: "refund", arguments: { id: "unknown" } });
+    await orders.callTool({ name: "refund", arguments: { id: "3" } });
+    expect(asked.map((r) => [r.instructions, r.abortWhen, r.history.length])).toEqual([
+      ["You refund orders.", "the order is unknown", 0],
+      ["You refund orders.", "the order is unknown", 1],
+    ]);
+    expect(costs).toEqual([0.02, 0.01]);
+    expect(aborts).toEqual([
+      { server: "orders", tool: "refund", reason: "order unknown" },
+      { server: "orders", tool: "refund", reason: "the agent mocks' call budget of 2 calls ran out" },
+    ]);
+    await orders.close();
+  });
+
+  it("refuses a request without the turn's token", async () => {
+    await started(4, noModel);
+    const url = endpoint?.servers[0]?.url ?? "";
+    const response = await fetch(url, { method: "POST", body: "{}", headers: { "content-type": "application/json" } });
+    expect(response.status).toBe(401);
+  });
+
+  it("answers 404 for a path that names no stand-in", async () => {
+    await started(4, noModel);
+    const known = endpoint?.servers[0];
+    const url = (known?.url ?? "").replace("plugin_kit_tracker", "plugin_kit_unknown");
+    const response = await fetch(url, { method: "POST", body: "{}", headers: { ...known?.headers, "content-type": "application/json" } });
+    expect(response.status).toBe(404);
+    const root = await fetch(new URL("/", url), { headers: known?.headers });
+    expect(root.status).toBe(404);
+  });
+
+  it("starts nothing for a case that mocks no server", async () => {
+    const context = create(EvalCaseContextSchema, { pluginId: "plg_1" });
+    expect(await startMockEndpoint({ context, archive: ARCHIVE, agentModel: noModel })).toBeUndefined();
+  });
+
+  it("answers an unmocked tool and a missing fixture as tool errors", async () => {
+    const { connect, aborts } = await started(4, noModel);
+    const orders = await connect("plugin_kit_orders");
+    const unmocked = await orders.callTool({ name: "cancel_order", arguments: {} });
+    expect([unmocked.isError, text(unmocked)]).toEqual([true, "the tool 'cancel_order' is not mocked"]);
+    const receipt = await orders.callTool({ name: "receipt", arguments: {} });
+    expect(receipt.isError).toBe(true);
+    expect(text(receipt)).toContain("missing.json");
+    expect(aborts).toEqual([]);
+    await orders.close();
+  });
+
+  it("refuses a call made before the turn binds the endpoint", async () => {
+    endpoint = await startMockEndpoint({ context: contextOf(4), archive: ARCHIVE, agentModel: noModel });
+    const server = endpoint?.servers.find((s) => s.slug === "plugin_kit_tracker");
+    const client = new Client({ name: "test", version: "1.0.0" });
+    await client.connect(new StreamableHTTPClientTransport(new URL(server?.url ?? ""), { requestInit: { headers: server?.headers } }));
+    const early = await client.callTool({ name: "create_issue", arguments: { title: "Crash" } });
+    expect([early.isError, text(early)]).toEqual([true, "the mock is not ready"]);
+    await client.close();
+  });
+});
+
+describe("an Eval's stand-ins", () => {
+  const ROOT_ARCHIVE = caseArchiveOf([
+    { path: "mocks/billing/refund.md", content: encode("{{file:fixtures/ok.json}}") },
+    { path: "mocks/billing/fixtures/ok.json", content: encode('{"refunded":true}') },
+    { path: "mocks/plugin_kit_tracker/create_issue.md", content: encode("Created.") },
+  ]);
+  const evalContext = create(EvalCaseContextSchema, {
+    evalId: "evs_1",
+    evalDigest: "a".repeat(64),
+    suiteDir: "",
+    caseDir: "refund",
+    mocks: [
+      create(EvalMockServerSchema, {
+        server: "billing",
+        segment: "billing",
+        home: { case: "agent", value: create(EvalMockAgentHomeSchema, { agentId: "agt_1", server: "billing" }) },
+        tools: [tool({ tool: "refund", file: "mocks/billing/refund.md", kind: EvalMockKind.fixed })],
+      }),
+      create(EvalMockServerSchema, {
+        server: "plugin_kit_tracker",
+        segment: "plugin_kit_tracker",
+        home: {
+          case: "plugin",
+          value: create(EvalMockPluginHomeSchema, { pluginId: "plg_1", pluginName: "kit", server: "tracker" }),
+        },
+        tools: [tool({ tool: "create_issue", file: "mocks/plugin_kit_tracker/create_issue.md", kind: EvalMockKind.fixed })],
+      }),
+    ],
+  });
+
+  it("carry the real server's home, and read fixtures from a suite at the archive's root", async () => {
+    endpoint = await startMockEndpoint({ context: evalContext, archive: ROOT_ARCHIVE, agentModel: noModel });
+    if (endpoint === undefined) throw new Error("expected an endpoint");
+    endpoint.bind({ onAbort: () => undefined, onCost: () => undefined });
+    expect(endpoint.servers.map((s) => [s.slug, s.home])).toEqual([
+      ["billing", { kind: "agent", agentId: "agt_1", subagent: null, server: "billing" }],
+      ["plugin_kit_tracker", { kind: "plugin", pluginId: "plg_1", plugin: "kit", server: "tracker" }],
+    ]);
+    const billing = endpoint.servers.find((s) => s.slug === "billing");
+    const client = new Client({ name: "test", version: "1.0.0" });
+    await client.connect(new StreamableHTTPClientTransport(new URL(billing?.url ?? ""), { requestInit: { headers: billing?.headers } }));
+    expect(text(await client.callTool({ name: "refund", arguments: {} }))).toBe('{"refunded":true}');
+    await client.close();
+  });
+});
+