@autoview/cli 0.1.0

package/src/agent/toolSurface.ts

@@ -0,0 +1,501 @@
+import { OpenApi } from "@typia/interface";
+
+import {
+  classifyEndpoints,
+  EndpointRole,
+} from "../utils/classifyEndpoints";
+import {
+  COLLECTION_FIELD_NAMES,
+  extractFields,
+  findCollectionField,
+  resolveProperties,
+} from "../utils/extractFields";
+import { IAutoViewEndpoint, toEndpoints } from "../utils/toEndpoints";
+
+/**
+ * Agent tool surface — the deterministic projection of the API's semantic model
+ * (IR) into a set of agent-callable tools (MCP / function-calling shape).
+ *
+ * The same IR that drives the human frontend drives this: resource roles
+ * (read vs write), the resource hierarchy, and the producer→consumer chain (which
+ * tool's output supplies a given tool's id input). A naive OpenAPI→tools dump
+ * has none of that — it emits a flat bag of operations. This emits tools that
+ * carry:
+ *
+ * - `annotations.readOnlyHint` / `destructiveHint` — safety, so an agent (or its
+ *   guardrails) can tell a read from a mutation without parsing prose.
+ * - per-input producer hints — "this `saleId` comes from the `sales.index`
+ *   tool's `id` field" — so the chain is explicit, not inferred.
+ * - only renderable operations (path fragments like `/x#y` that nestia cannot
+ *   express as a typed call are dropped — `toEndpoints` already excludes them).
+ *
+ * Pure and deterministic: same document → same tool surface.
+ */
+export interface IAgentTool {
+  /** Dotted accessor, unique and a valid tool name (`shoppings.sales.index`). */
+  name: string;
+  description: string;
+  /** JSON Schema for the call arguments. */
+  inputSchema: {
+    type: "object";
+    properties: Record<string, unknown>;
+    required: string[];
+  };
+  /** MCP-style behavior hints. */
+  annotations: {
+    readOnlyHint: boolean;
+    destructiveHint: boolean;
+    idempotentHint: boolean;
+  };
+  /** Execution descriptor — how an MCP server / executor invokes the real API. */
+  operation: {
+    method: string;
+    path: string;
+    pathParams: string[];
+    hasQuery: boolean;
+    hasBody: boolean;
+  };
+  /**
+   * Producer links: for each id-like path param, the tool + field that produces
+   * its value. Empty when the surface could not resolve a producer.
+   */
+  producers: Array<{ param: string; tool: string; field: string }>;
+}
+
+const READ_ROLES = new Set<EndpointRole>(["list", "search", "detail"]);
+
+/** Field names that identify a record (the value a path param consumes). */
+const ID_NAMES = ["id", "uuid"];
+
+/** The first identifying field name present (`id` preferred over `uuid`), or null. */
+const firstIdName = (names: Iterable<string>): string | null => {
+  const set = new Set([...names].map((n) => n.toLowerCase()));
+  return ID_NAMES.find((n) => set.has(n)) ?? null;
+};
+
+/**
+ * Whether a read endpoint's response element carries an identifying field
+ * (`id` / `uuid`) — i.e. whether it can PRODUCE an id for a consumer. Handles
+ * both named component responses and INLINE response schemas, and collection
+ * wrappers named generically (`data`/`entries`) OR after the resource
+ * (`{ databases: [...] }`, DigitalOcean), including inline array items.
+ */
+function producedIdField(
+  op: IAutoViewEndpoint,
+  doc: OpenApi.IDocument,
+): string | null {
+  // Named-type path (nestia/AutoBE convention).
+  if (op.responseBody !== null) {
+    const elementFields = op.responseBody.isArray
+      ? extractFields(op.responseBody.typeName, doc)
+      : (() => {
+          const top = extractFields(op.responseBody.typeName, doc);
+          // The collection field is either generically named (`data`/`entries`)
+          // or named after the resource (`{ namespaces: [...] }`, common on
+          // DigitalOcean once nestia promotes the inline response to a named
+          // type). Fall back to the first array field so the latter is found —
+          // producedIdField only runs on list/search reads, so an array IS the
+          // collection.
+          const col =
+            findCollectionField(top) ??
+            top.find((f) => f.kind === "array" && f.ref !== undefined);
+          return col?.ref !== undefined ? extractFields(col.ref, doc) : top;
+        })();
+    const named = firstIdName(elementFields.map((f) => f.name));
+    if (named !== null) return named;
+    // else: the promoted type was a dead/empty ref (nestia emits a `*.GetResponse`
+    // not registered in components, just like `*.GetQuery`) — fall through to the
+    // raw inline response schema, which still carries the real array.
+  }
+  // Inline response schema (DigitalOcean): introspect the raw schema.
+  if (op.responseSchema === null) return null;
+  const { properties } = resolveProperties(op.responseSchema, doc);
+  // The collection array: a known-named one, else the sole/first array property.
+  const arrayEntries = Object.entries(properties).filter(
+    ([, p]) => (p as { type?: unknown }).type === "array",
+  );
+  const collection =
+    arrayEntries.find(([n]) => COLLECTION_FIELD_NAMES.has(n.toLowerCase())) ??
+    arrayEntries[0];
+  if (collection !== undefined) {
+    const items = (collection[1] as { items?: OpenApi.IJsonSchema }).items;
+    if (items !== undefined) {
+      return firstIdName(Object.keys(resolveProperties(items, doc).properties));
+    }
+  }
+  // A single object (detail) returning an inline schema.
+  return firstIdName(Object.keys(properties));
+}
+
+/** Roles whose response is a browsable collection — the only legitimate id
+ *  PRODUCERS. A `detail` read returns one object but needs the id to be called,
+ *  so it cannot produce that id (circular); excluding it keeps navigability honest. */
+const COLLECTION_ROLES = new Set<EndpointRole>(["list", "search"]);
+
+interface IProducer {
+  tool: string;
+  field: string;
+}
+
+/**
+ * The shared producer model used by BOTH the tool surface and the consumability
+ * report, so they never disagree on what is navigable. For each resource, the
+ * collection read (list/search) whose element exposes an id — keyed by the
+ * endpoint's CLASSIFIED resource (so a verb path like `/pet/findByStatus` is
+ * bucketed under `pet`, not `findByStatus`).
+ */
+function producerIndex(document: OpenApi.IDocument): {
+  endpoints: IAutoViewEndpoint[];
+  roleOf: Map<IAutoViewEndpoint, EndpointRole>;
+  producerOf: Map<string, IProducer>;
+  readableResources: Set<string>;
+  nested: Map<string, INestedProducer>;
+} {
+  const endpoints = toEndpoints(document);
+  const roleOf = new Map<IAutoViewEndpoint, EndpointRole>();
+  const resourceOf = new Map<IAutoViewEndpoint, string>();
+  for (const g of classifyEndpoints(endpoints, document)) {
+    for (const c of g.endpoints) {
+      roleOf.set(c.endpoint, c.role);
+      resourceOf.set(c.endpoint, c.chain.at(-1)?.name ?? c.resource);
+    }
+  }
+  const producerOf = new Map<string, IProducer>();
+  const readableResources = new Set<string>();
+  for (const e of endpoints) {
+    const role = roleOf.get(e);
+    const resource = resourceOf.get(e);
+    if (role === undefined || resource === undefined) continue;
+    if (READ_ROLES.has(role)) readableResources.add(resource);
+    if (!COLLECTION_ROLES.has(role)) continue;
+    // A genuine producer ENUMERATES the resource. One that is itself scoped by
+    // that resource's own id (`/files/{file_id}/trash` → resource `files`, but
+    // it requires `file_id`) cannot produce that id — it is circular, so skip it.
+    const selfScoped = e.parameters.some(
+      (p) => resourceOfParam(e.path, p.name) === resource,
+    );
+    if (selfScoped) continue;
+    const field = producedIdField(e, document);
+    if (field !== null && !producerOf.has(resource)) {
+      producerOf.set(resource, { tool: e.accessor.join("."), field });
+    }
+  }
+  const nested = nestedProducers(endpoints, document, roleOf);
+  return { endpoints, roleOf, producerOf, readableResources, nested };
+}
+
+/** The named element type of a read endpoint's response (for recursion). */
+function responseElementTypeName(
+  op: IAutoViewEndpoint,
+  doc: OpenApi.IDocument,
+): string | null {
+  if (op.responseBody === null) return null;
+  if (op.responseBody.isArray) return op.responseBody.typeName;
+  const col = findCollectionField(extractFields(op.responseBody.typeName, doc));
+  return col?.ref ?? op.responseBody.typeName;
+}
+
+/** Where a nested id is obtainable: a read tool, the array path to it, and its id field. */
+interface INestedProducer {
+  tool: string;
+  /** Nested access path to the id-bearing element, e.g. `units[].stocks[]`. */
+  path: string;
+  field: string;
+}
+
+/**
+ * Resources whose ids are obtainable NESTED inside a read response, not from a
+ * dedicated endpoint — `order` detail returns `goods: IShoppingOrderGood[]`,
+ * `sale` returns `units: IShoppingSaleUnit[]` (each unit holds `stocks: […]`).
+ * Walks each read's response type graph (depth-bounded) and, for every
+ * array-field whose element carries an id, records WHICH read tool exposes it
+ * and the nested path — so the tool surface can hand the agent a real producer
+ * hint ("stockId ← sale.get, at units[].stocks[].id"), not just say it exists.
+ */
+function nestedProducers(
+  endpoints: IAutoViewEndpoint[],
+  doc: OpenApi.IDocument,
+  roleOf: Map<IAutoViewEndpoint, EndpointRole>,
+): Map<string, INestedProducer> {
+  const out = new Map<string, INestedProducer>();
+  const visit = (
+    typeName: string,
+    tool: string,
+    parts: string[],
+    depth: number,
+    seen: Set<string>,
+  ): void => {
+    if (depth > 3 || seen.has(typeName)) return;
+    seen.add(typeName);
+    for (const f of extractFields(typeName, doc)) {
+      if (f.kind === "array" && f.ref !== undefined) {
+        const childParts = [...parts, f.name];
+        const idField = firstIdName(
+          extractFields(f.ref, doc).map((x) => x.name),
+        );
+        const key = f.name.toLowerCase();
+        if (idField !== null) {
+          // Prefer the SHALLOWEST path (fewest array hops) — `sale → units[]`
+          // beats `order → goods[].units[]` for the same `units` id, so the
+          // agent is pointed at the most direct producer.
+          const existing = out.get(key);
+          if (existing === undefined || childParts.length < existing.path.split(".").length) {
+            out.set(key, {
+              tool,
+              path: childParts.map((p) => `${p}[]`).join("."),
+              field: idField,
+            });
+          }
+        }
+        visit(f.ref, tool, childParts, depth + 1, seen); // recurse: units → stocks
+      } else if (f.kind === "ref" && f.ref !== undefined) {
+        visit(f.ref, tool, parts, depth + 1, seen);
+      }
+    }
+  };
+  for (const e of endpoints) {
+    const role = roleOf.get(e);
+    if (role === undefined || !READ_ROLES.has(role)) continue;
+    const el = responseElementTypeName(e, doc);
+    if (el !== null) visit(el, e.accessor.join("."), [], 0, new Set());
+  }
+  return out;
+}
+
+/** The static path segment that an `{param}` addresses (`/sales/{saleId}` → `sales`). */
+function resourceOfParam(path: string, param: string): string | null {
+  const segs = path.split("/").filter((s) => s.length > 0);
+  for (let i = 0; i < segs.length; i++) {
+    if (segs[i] === `{${param}}`) return i > 0 ? segs[i - 1]! : null;
+  }
+  return null;
+}
+
+function paramType(schema: OpenApi.IJsonSchema | undefined): "number" | "string" {
+  const type = (schema as { type?: unknown } | undefined)?.type;
+  return type === "integer" || type === "number" ? "number" : "string";
+}
+
+/**
+ * Build the agent tool surface from an OpenAPI document.
+ */
+export function buildToolSurface(document: OpenApi.IDocument): IAgentTool[] {
+  const { endpoints, roleOf, producerOf, nested } = producerIndex(document);
+
+  return endpoints.map((e): IAgentTool => {
+    const role = roleOf.get(e);
+    const readOnly = role !== undefined && READ_ROLES.has(role);
+    const method = e.method.toUpperCase();
+    const name = e.accessor.join(".");
+
+    const properties: Record<string, unknown> = {};
+    const required: string[] = [];
+    const producers: IAgentTool["producers"] = [];
+    for (const p of e.parameters) {
+      const resource = resourceOfParam(e.path, p.name);
+      const direct = resource !== null ? producerOf.get(resource) : undefined;
+      const nestedP =
+        resource !== null ? nested.get(resource.toLowerCase()) : undefined;
+      let description = p.description || `path parameter ${p.name}`;
+      if (direct !== undefined && direct.tool !== name) {
+        // a dedicated list/search produces this id directly
+        producers.push({ param: p.name, tool: direct.tool, field: direct.field });
+        description = `the ${direct.field} of a ${resource}; obtain it from the \`${direct.tool}\` tool — use each item's \`${direct.field}\` field`;
+      } else if (nestedP !== undefined && nestedP.tool !== name) {
+        // the id lives nested inside a parent read's response
+        producers.push({
+          param: p.name,
+          tool: nestedP.tool,
+          field: `${nestedP.path}.${nestedP.field}`,
+        });
+        description = `the ${nestedP.field} of a ${resource}; obtain it from the \`${nestedP.tool}\` tool's response at \`${nestedP.path}.${nestedP.field}\``;
+      }
+      properties[p.name] = { type: paramType(p.schema), description };
+      required.push(p.name);
+    }
+    if (e.query !== null) {
+      properties.query = {
+        type: "object",
+        description: "optional search / filter / pagination",
+      };
+    }
+    if (e.requestBody !== null) {
+      properties.body = {
+        type: "object",
+        description: `request body (${e.requestBody.typeName})`,
+      };
+      required.push("body");
+    }
+
+    const tag = readOnly ? "[READ]" : "[WRITE — mutates server state]";
+    const chainNote =
+      producers.length > 0
+        ? ` Inputs come from: ${producers.map((p) => `${p.param} ← ${p.tool}.${p.field}`).join("; ")}.`
+        : "";
+
+    return {
+      name,
+      description: `${tag} ${e.description || `${method} ${e.path}`}`.trim() + chainNote,
+      inputSchema: { type: "object", properties, required },
+      annotations: {
+        readOnlyHint: readOnly,
+        destructiveHint: method === "DELETE",
+        idempotentHint: method === "GET" || method === "PUT" || method === "DELETE",
+      },
+      operation: {
+        method,
+        path: e.path,
+        pathParams: e.parameters.map((p) => p.name),
+        hasQuery: e.query !== null,
+        hasBody: e.requestBody !== null,
+      },
+      producers,
+    };
+  });
+}
+
+/**
+ * Honest consumability analysis with a 4-way split per path-param input, so the
+ * report never accuses an API of a design gap it cannot actually prove:
+ *
+ * - `resolved`   — a collection read (list/search) produces the id; the tool
+ *   surface declares that producer. Directly navigable, and the agent is told
+ *   exactly where the id comes from.
+ * - `nested`     — the id is obtainable nested inside a parent read's response
+ *   (`order` → `goods[].id`, `sale` → `units[].stocks[].id`), but no dedicated
+ *   list tool produces it. Navigable, just not via a single tool — counted
+ *   honestly, separate from `resolved` so the report matches the tool surface.
+ * - `untraceable`— the resource HAS a read endpoint, but no id could be found in
+ *   its response (e.g. an inline schema this analysis can't fully introspect).
+ *   Undetermined, not blamed.
+ * - `userInput`  — the param is not an entity id but a human-known key
+ *   (`repository_name`, `template_key`, a `scope`/`slug`), supplied by the
+ *   caller — a missing producer is expected, NOT a defect.
+ * - `orphan`     — an entity id (`*_id` / `uuid`) that NO endpoint produces. A
+ *   genuine consumability defect.
+ */
+export interface IConsumabilityAnalysis {
+  tools: number;
+  readTools: number;
+  writeTools: number;
+  referenceInputs: number;
+  resolved: number;
+  nested: number;
+  untraceable: Array<{ tool: string; param: string; resource: string }>;
+  userInputs: Array<{ tool: string; param: string; resource: string }>;
+  orphan: Array<{ tool: string; param: string; resource: string }>;
+}
+
+/**
+ * True when a path param names an entity identifier (`id`, `*_id`, `*Id`,
+ * `uuid`) — the kind that must be produced by another endpoint. A param that is
+ * a human-known key instead (`repository_name`, `template_key`, `scope`, a
+ * `slug`/`code`) is caller-supplied, so its absence of a producer is not a gap.
+ */
+function isIdLikeParam(name: string): boolean {
+  return /(^id$)|(_id$)|([a-z]Id$)|(^uuid$)|(_uuid$)|([a-z]Uuid$)/.test(name);
+}
+
+export function analyzeConsumability(
+  document: OpenApi.IDocument,
+): IConsumabilityAnalysis {
+  // ONE producer model, shared with the tool surface — so the report never
+  // claims an input is navigable that the tools give no producer hint for.
+  const { endpoints, roleOf, producerOf, readableResources, nested } =
+    producerIndex(document);
+
+  const isReadTool = (e: IAutoViewEndpoint): boolean => {
+    const role = roleOf.get(e);
+    return role !== undefined && READ_ROLES.has(role);
+  };
+  // Classify every id input once, then aggregate purely (no mutable counters).
+  const inputs = endpoints.flatMap((e) =>
+    e.parameters.map((p) => {
+      const resource = resourceOfParam(e.path, p.name);
+      const ref = { tool: e.accessor.join("."), param: p.name, resource: resource ?? "?" };
+      const status =
+        resource !== null && producerOf.has(resource)
+          ? "resolved"
+          : resource !== null && nested.has(resource.toLowerCase())
+            ? "nested"
+            : // a non-id param (`username`, `scope`, `*_name`) is a caller-known
+              // key regardless of whether its resource is readable — not a gap.
+              !isIdLikeParam(p.name)
+              ? "userInput"
+              : // an entity id: readable resource but no producer traced =
+                // undetermined; never read at all = a genuine orphan.
+                resource !== null && readableResources.has(resource)
+                ? "untraceable"
+                : "orphan";
+      return { ...ref, status };
+    }),
+  );
+  const readTools = endpoints.filter(isReadTool).length;
+  return {
+    tools: endpoints.length,
+    readTools,
+    writeTools: endpoints.length - readTools,
+    referenceInputs: inputs.length,
+    resolved: inputs.filter((i) => i.status === "resolved").length,
+    nested: inputs.filter((i) => i.status === "nested").length,
+    untraceable: inputs
+      .filter((i) => i.status === "untraceable")
+      .map(({ tool, param, resource }) => ({ tool, param, resource })),
+    userInputs: inputs
+      .filter((i) => i.status === "userInput")
+      .map(({ tool, param, resource }) => ({ tool, param, resource })),
+    orphan: inputs
+      .filter((i) => i.status === "orphan")
+      .map(({ tool, param, resource }) => ({ tool, param, resource })),
+  };
+}
+
+/** Coverage summary of a tool surface — how navigable it is for an agent. */
+export interface IToolSurfaceCoverage {
+  tools: number;
+  /** Tools tagged read vs write. */
+  readTools: number;
+  writeTools: number;
+  /** id-like path-param inputs across the surface. */
+  referenceInputs: number;
+  /** reference inputs with a declared producer (the navigable fraction). */
+  resolvedInputs: number;
+  /** reference inputs with no producer — an agent must guess where they come from. */
+  orphanInputs: number;
+  /**
+   * The orphan inputs in detail: a required id with no tool in the API that
+   * produces it. An agent (or a human) cannot obtain this value from any
+   * endpoint — a consumability defect in the API design.
+   */
+  orphans: Array<{ tool: string; param: string }>;
+}
+
+/**
+ * Deterministic structural coverage of a tool surface — the LLM-independent
+ * verification: is the tool graph navigable (every id input has a declared
+ * producer)? Reported as a report card, distinct from running an agent.
+ */
+export function coverToolSurface(tools: IAgentTool[]): IToolSurfaceCoverage {
+  // Classify every path param once, then aggregate purely (no mutable counters).
+  const inputs = tools.flatMap((t) => {
+    const producerParams = new Set(t.producers.map((p) => p.param));
+    return t.operation.pathParams.map((param) => ({
+      tool: t.name,
+      param,
+      resolved: producerParams.has(param),
+    }));
+  });
+  const readTools = tools.filter((t) => t.annotations.readOnlyHint).length;
+  const resolvedInputs = inputs.filter((i) => i.resolved).length;
+  return {
+    tools: tools.length,
+    readTools,
+    writeTools: tools.length - readTools,
+    referenceInputs: inputs.length,
+    resolvedInputs,
+    orphanInputs: inputs.length - resolvedInputs,
+    orphans: inputs
+      .filter((i) => !i.resolved)
+      .map(({ tool, param }) => ({ tool, param })),
+  };
+}

package/src/agent/verifyAgentTasks.test.ts

@@ -0,0 +1,106 @@
+import http from "http";
+
+import { OpenApiConverter } from "@typia/utils";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+
+import { buildToolSurface } from "./toolSurface";
+import {
+  executeToolCall,
+  functionName,
+  IChatClient,
+  toFunctionTools,
+  verifyAgentTasks,
+} from "./verifyAgentTasks";
+
+function doc(paths: Record<string, unknown>, schemas: Record<string, unknown> = {}) {
+  return OpenApiConverter.upgradeDocument({
+    openapi: "3.0.0", info: { title: "Shop", version: "1.0.0" }, paths, components: { schemas },
+  } as never);
+}
+
+const fixture = doc(
+  {
+    "/sales": { get: { operationId: "index", summary: "List sales.", responses: { 200: { description: "ok", content: { "application/json": { schema: { type: "array", items: { $ref: "#/components/schemas/Sale" } } } } } } } },
+    "/sales/{saleId}": { get: { operationId: "at", summary: "Get a sale.", parameters: [{ name: "saleId", in: "path", required: true, schema: { type: "string" } }], responses: { 200: { description: "ok", content: { "application/json": { schema: { $ref: "#/components/schemas/Sale" } } } } } } },
+  },
+  { Sale: { type: "object", properties: { id: { type: "string" }, title: { type: "string" } }, required: ["id"] } },
+);
+const tools = buildToolSurface(fixture);
+
+const SALES = [{ id: "sale_1", title: "MacBook" }, { id: "sale_2", title: "iPhone" }];
+let server: http.Server;
+let base = "";
+beforeAll(async () => {
+  server = http.createServer((req, res) => {
+    const u = (req.url ?? "").split("?")[0];
+    res.writeHead(200, { "content-type": "application/json" });
+    if (u === "/sales") return res.end(JSON.stringify({ data: SALES }));
+    const m = u!.match(/^\/sales\/([^/]+)$/);
+    return res.end(JSON.stringify(m ? SALES.find((s) => s.id === m[1]) ?? {} : {}));
+  });
+  await new Promise<void>((ok) => server.listen(0, "127.0.0.1", () => ok()));
+  base = `http://127.0.0.1:${(server.address() as { port: number }).port}`;
+});
+afterAll(() => server.close());
+
+describe("toFunctionTools", () => {
+  it("converts the surface to OpenAI function-calling tools", () => {
+    const fns = toFunctionTools(tools);
+    expect(fns).toHaveLength(tools.length);
+    expect(fns[0]!.type).toBe("function");
+    // dotted accessor names are sanitized to OpenAI's `[a-zA-Z0-9_-]` charset
+    expect(fns[0]!.function.name).toBe(functionName(tools[0]!.name));
+    expect(fns[0]!.function.name).not.toContain(".");
+    expect(fns[0]!.function.parameters).toBe(tools[0]!.inputSchema);
+  });
+});
+
+describe("executeToolCall", () => {
+  it("maps a list call to the right HTTP request and returns the body", async () => {
+    const list = tools.find((t) => t.name === "sales.get")!;
+    const res = await executeToolCall(tools, list.name, {}, { baseUrl: base });
+    expect(res.ok).toBe(true);
+    expect(res.text).toContain("MacBook");
+  });
+
+  it("substitutes path params for a detail call", async () => {
+    const detail = tools.find((t) => t.name === "sales.getBySaleid")!;
+    const res = await executeToolCall(tools, detail.name, { saleId: "sale_2" }, { baseUrl: base });
+    expect(res.text).toContain("iPhone");
+  });
+
+  it("returns an error for an unknown tool", async () => {
+    const res = await executeToolCall(tools, "nope", {}, { baseUrl: base });
+    expect(res.ok).toBe(false);
+  });
+});
+
+describe("verifyAgentTasks", () => {
+  it("drives the agent loop (scripted client), executes tools, and grades the task", async () => {
+    // a fake client: turn 1 → call the sales list tool; turn 2 → final answer
+    let turn = 0;
+    const client: IChatClient = {
+      chat: {
+        completions: {
+          create: async () => {
+            turn++;
+            if (turn === 1) {
+              return { choices: [{ message: { content: null, tool_calls: [{ id: "c1", function: { name: "sales.get", arguments: "{}" } }] } }] };
+            }
+            return { choices: [{ message: { content: "There are 2 sales: MacBook and iPhone." } }] };
+          },
+        },
+      },
+    };
+    const results = await verifyAgentTasks(fixture, {
+      client,
+      model: "fake",
+      baseUrl: base,
+      tasks: [{ id: "count", prompt: "How many sales?", expect: (a) => a.includes("2") }],
+    });
+    expect(results).toHaveLength(1);
+    expect(results[0]!.toolCalls).toEqual(["sales.get"]);
+    expect(results[0]!.passed).toBe(true);
+    expect(results[0]!.answer).toContain("MacBook");
+  });
+});