@anteros/core 0.0.44 → 0.0.47

package/lib/agentMemory.ts

@@ -3,6 +3,7 @@ import { parseDuration } from "../utils/func";
 import { syncTtlRetention } from "../database/ttl";
 import { writeDeleteMarkers, replicatesCollection } from "../database/deleteLog";
 import { Redis } from "ioredis";
+import { createHash } from "node:crypto";
 import type { Db } from "mongodb";
 import type {
     AgentMemory,
@@ -11,7 +12,11 @@ import type {
     AgentMemoryProcessor,
     AgentMemoryState,
     AgentMessage,
+    AgentRecallHit,
+    AgentRecallRecord,
+    AgentSemanticRecallOptions,
     AgentThread,
+    CustomAgentMemoryOptions,
     MongoAgentMemoryOptions,
     RedisAgentMemoryOptions,
     RedisClientLike,
@@ -464,8 +469,202 @@ export class MongoAgentMemory implements AgentMemory {
     }
 }
 
+// ─── Custom store ───────────────────────────────────────────────────────
+
+/**
+ * Build a conversation store from your own handlers — the framework applies the
+ * same policy as the built-in stores, so a handler only reads and writes.
+ *
+ * - **scoping** — the key it receives is `resourceId:threadId` when `scoped`
+ *   (default) and a caller identity is present, the bare thread id otherwise;
+ * - **cap** — `save` receives at most `maxMessages` messages (default 100), the
+ *   opening `system`/`developer` message kept first;
+ * - **optional parts** — `list`, `getState`, `setState`, `clearState` and
+ *   `collections` exist on the returned store **only if you provide them**.
+ *
+ * ```ts
+ * import { agents } from "@anteros/core";
+ *
+ * define.Agent({
+ *   // …
+ *   memory: agents.memory.custom({
+ *     name: "sqlite",
+ *     maxMessages: 50,
+ *     get: async (key) => db.thread(key),
+ *     save: async (key, messages) => db.write(key, messages),
+ *     clear: async (key) => db.drop(key),
+ *   }),
+ * });
+ * ```
+ *
+ * A store used through `workingMemory` also needs `getState` / `setState`; without
+ * them the scratchpad is simply inert, as with any other store.
+ */
+function customAgentMemory(options: CustomAgentMemoryOptions): AgentMemory {
+    for (const handler of ["get", "save", "clear"] as const) {
+        if (typeof (options as any)?.[handler] !== "function") {
+            throw new AppError(
+                `A custom memory${options?.name ? ` '${options.name}'` : ""} needs \`${handler}\` — agents.memory.custom({ get, save, clear, … })`,
+                { status: 500, code: "AGENT_MEMORY_HANDLER_REQUIRED" },
+            );
+        }
+    }
+
+    const scoped = options.scoped ?? true;
+    const maxMessages = Math.max(1, options.maxMessages ?? DEFAULT_MAX_MESSAGES);
+    const resolved = (threadId: string, ctx?: AgentMemoryContext) => threadKey(threadId, ctx, scoped);
+
+    const memory: AgentMemory = {
+        get: (threadId, ctx) => options.get(resolved(threadId, ctx), ctx ?? {}),
+        save: (threadId, messages, ctx) =>
+            options.save(resolved(threadId, ctx), trimMessages(messages, maxMessages), ctx ?? {}),
+        clear: (threadId, ctx) => options.clear(resolved(threadId, ctx), ctx ?? {}),
+    };
+
+    if (options.list) memory.list = (ctx) => options.list!(ctx ?? {});
+    if (options.getState) memory.getState = (key, ctx) => options.getState!(key, ctx ?? {});
+    if (options.setState) memory.setState = (key, value, ctx) => options.setState!(key, value, ctx ?? {});
+    if (options.clearState) memory.clearState = (key, ctx) => options.clearState!(key, ctx ?? {});
+    if (options.collections) memory.collections = () => options.collections!();
+
+    return memory;
+}
+
+// ─── Semantic recall ────────────────────────────────────────────────────
+
+/** The text of a message, or `''` when it carries none (tool call, media-only). */
+function recallText(message: AgentMessage): string {
+    const content: any = message.content;
+    if (typeof content === "string") return content.trim();
+    if (Array.isArray(content)) {
+        return content
+            .filter((part: any) => part?.type === "text" && typeof part.text === "string")
+            .map((part: any) => part.text)
+            .join(" ")
+            .trim();
+    }
+    return "";
+}
+
+/** Stable id of an indexed message — two identical turns collapse onto one hit. */
+function recallId(message: AgentMessage, text: string): string {
+    return createHash("sha1").update(`${message.role}\0${text}`).digest("hex");
+}
+
+const RECALL_HEADER = "Relevant past exchanges (semantic recall):";
+
+/**
+ * A `load` / `save` processor that recalls past messages **by meaning** — the
+ * framework's projection of Mastra's `semanticRecall`.
+ *
+ * - `load` embeds the last user message as a **query**, searches your index, and
+ *   prepends the hits as a system message (only when there are hits).
+ * - `save` embeds the new user / assistant / tool messages as **passages** and
+ *   hands them to `upsert`. Without `upsert` it only recalls (index filled
+ *   elsewhere); without `search` it does not run at all.
+ *
+ * Same option names as Mastra: `topK`, `messageRange`, `scope` (`'thread'` or
+ * `'resource'`), `filter`. `messageRange` is forwarded to the store: expanding a
+ * hit into its surrounding messages belongs to the index, not to the prompt.
+ *
+ * ```ts
+ * processors: [
+ *   agents.memory.semanticRecall({
+ *     topK: 3,
+ *     messageRange: 2,
+ *     scope: "resource",
+ *     embed: (texts, kind) => embedder(texts, kind),   // → number[][]
+ *     search: (vector, query) => vectorStore.search(vector, query),
+ *     upsert: (records) => vectorStore.upsert(records),
+ *   }),
+ * ],
+ * ```
+ */
+function semanticRecall(options: AgentSemanticRecallOptions): AgentMemoryProcessor {
+    if (typeof options?.embed !== "function" || typeof options?.search !== "function") {
+        throw new AppError(
+            "A semantic recall processor needs `embed` and `search` — agents.memory.semanticRecall({ embed, search, … })",
+            { status: 500, code: "AGENT_RECALL_HANDLER_REQUIRED" },
+        );
+    }
+
+    const topK = Math.max(1, Math.trunc(options.topK ?? 3));
+    const messageRange = Math.max(0, Math.trunc(options.messageRange ?? 2));
+    const scope = options.scope === "resource" ? "resource" : "thread";
+    const format = options.format
+        ?? ((hits: AgentRecallHit[]) => hits.map((hit) => `- ${hit.text}`).join("\n"));
+
+    // A message is embedded once per process. The id is content-based and stable
+    // across restarts, so the store can deduplicate what it already holds.
+    const indexed = new Set<string>();
+
+    return {
+        id: "semantic-recall",
+
+        async load(messages, ctx) {
+            const question = [...messages].reverse().find((message) => message.role === "user");
+            const text = question ? recallText(question) : "";
+            if (!text) return messages;
+
+            const [vector] = await options.embed([text], "query");
+            if (!vector?.length) return messages;
+
+            const hits = await options.search(vector, {
+                topK,
+                messageRange,
+                scope,
+                ...(options.filter ? { filter: options.filter } : {}),
+                ...(ctx?.thread ? { thread: ctx.thread } : {}),
+                ...(ctx?.resourceId ? { resourceId: ctx.resourceId } : {}),
+            }, ctx);
+            if (!hits?.length) return messages;
+
+            const rendered = format(hits).trim();
+            if (!rendered) return messages;
+
+            return [{ role: "system", content: `${RECALL_HEADER}\n${rendered}` }, ...messages];
+        },
+
+        async save(messages, ctx) {
+            if (typeof options.upsert !== "function") return messages;
+
+            const seen = new Set<string>();
+            const fresh = messages
+                .filter((message) => message.role !== "system" && message.role !== "developer")
+                .map((message) => ({ message, text: recallText(message) }))
+                .filter(({ text }) => !!text)
+                .map((entry) => ({ ...entry, id: recallId(entry.message, entry.text) }))
+                .filter((entry) => {
+                    if (indexed.has(entry.id) || seen.has(entry.id)) return false;
+                    seen.add(entry.id);
+                    return true;
+                });
+            if (!fresh.length) return messages;
+
+            const vectors = await options.embed(fresh.map((entry) => entry.text), "passage");
+            const records: AgentRecallRecord[] = fresh
+                .map((entry, index) => ({
+                    id: entry.id,
+                    text: entry.text,
+                    vector: vectors[index] ?? [],
+                    role: entry.message.role,
+                    ...(ctx?.thread ? { thread: ctx.thread } : {}),
+                    ...(ctx?.resourceId ? { resourceId: ctx.resourceId } : {}),
+                }))
+                .filter((record) => record.vector.length);
+            if (!records.length) return messages;
+
+            await options.upsert(records, ctx);
+            for (const record of records) indexed.add(record.id);
+            return messages;
+        },
+    };
+}
+
 export {
     DEFAULT_MAX_MESSAGES,
+    customAgentMemory,
+    semanticRecall,
     summarize,
     threadKey,
     threadTitle,

package/lib/agents.ts

@@ -3,25 +3,95 @@ import path from "path";
 import fs from "fs/promises";
 import { cfg } from "../server/config";
 import { Agent } from "./agent";
-import { InMemoryAgentMemory, MongoAgentMemory, RedisAgentMemory, summarize } from "./agentMemory";
+import { InMemoryAgentMemory, MongoAgentMemory, RedisAgentMemory, customAgentMemory, semanticRecall, summarize } from "./agentMemory";
 import { importDefinition } from "./load";
 import { resolveSkills } from "./skills";
 import { closeMcpClients } from "./mcpClient";
 import { logger } from "../utils/logger";
 import type { useRest } from "../database/rest";
-import type { AgentDefinition, TenantAgents } from "../types/agent";
+import type { AgentDefinition, AgentTool, TenantAgents } from "../types/agent";
 
 /**
- * Agent loader — `{tenant.dir}/agents/**\/*.agent.ts`.
+ * Agent loader — `{tenant.dir}/ai/agents/**\/*.agent.ts`, the legacy
+ * `{tenant.dir}/agents/**\/*.agent.ts` still read (with a deprecation warning).
  *
- * Each definition is instantiated once per tenant and kept in a process-wide
- * registry keyed by `tenant:agent`. `agents.get('weather')` resolves it at
- * runtime, binding the *calling* `rest` so tools run with the right scope.
+ * Tools live **next to the agent**: `support.agent.ts` + `support.tool.ts` in the
+ * same directory (a sub-directory works the same way). Each definition is
+ * instantiated once per tenant and kept in a process-wide registry keyed by
+ * `tenant:agent`. `agents.get('weather')` resolves it at runtime, binding the
+ * *calling* `rest` so tools run with the right scope.
  */
 
 const registry = new Map<string, Agent>();
 const definitions: AgentDefinition[] = [];
 
+/** Paths scanned, legacy first so the `ai/` definition wins a duplicate id. */
+function agentRoots(tenantDir: string): Array<{ dir: string; legacy: boolean }> {
+    return [
+        { dir: path.join(tenantDir, "agents"), legacy: true },
+        { dir: path.join(tenantDir, "ai", "agents"), legacy: false },
+    ];
+}
+
+/** A deprecation is logged once per tenant and path — `reload()` stays quiet. */
+const warned = new Set<string>();
+function warnLegacy(tenantId: string, from: string, to: string): void {
+    const key = `${tenantId}:${from}`;
+    if (warned.has(key)) return;
+    warned.add(key);
+    logger.warn(`Tenant '${tenantId}': '${from}' is deprecated — move it to '${to}'`);
+}
+
+/**
+ * Tools declared beside an agent — `<basename>.tool.ts` in the same directory.
+ * The file exports one `define.Tool` or an array; `enabled: false` and entries
+ * without an `id` are skipped. Tools declared inline on the agent win on a
+ * name clash.
+ */
+async function loadSiblingTools(agentFile: string): Promise<Record<string, AgentTool> | undefined> {
+    const base = path.basename(agentFile).replace(/\.agent\.ts$/, "");
+    const file = path.join(path.dirname(agentFile), `${base}.tool.ts`);
+    if (!(await fs.exists(file))) return undefined;
+
+    const module = await importDefinition(file, "Agent tool");
+    const exported = Array.isArray(module?.default) ? module.default : [module?.default];
+    const tools: Record<string, AgentTool> = {};
+
+    for (const tool of exported) {
+        if (!tool?._isTool_) {
+            logger.warn(`'${file}' does not export a define.Tool — ignored`);
+            continue;
+        }
+        if (tool.enabled === false) continue;
+        if (!tool.id) {
+            logger.warn(`A tool in '${file}' has no id — ignored (define.Tool requires one)`);
+            continue;
+        }
+        // The description is the whole prompt of the tool: without it the model cannot
+        // choose it, and an unnamed capability in the toolbox is worse than none.
+        if (!String(tool.description ?? "").trim()) {
+            logger.warn(`Tool '${tool.id}' in '${file}' has no description — ignored (the model cannot choose it)`);
+            continue;
+        }
+        tools[tool.id] = tool;
+    }
+
+    return Object.keys(tools).length ? tools : undefined;
+}
+
+/** Merge file tools under the inline ones — an inline tool is an explicit choice. */
+function mergeTools(
+    fromFiles: Record<string, AgentTool> | undefined,
+    inline: AgentDefinition["tools"],
+): AgentDefinition["tools"] {
+    if (!fromFiles) return inline;
+    if (!inline) return fromFiles;
+    const own = Array.isArray(inline)
+        ? Object.fromEntries(inline.map((tool) => [tool.id!, tool]))
+        : inline;
+    return { ...fromFiles, ...own };
+}
+
 async function syncAgents(): Promise<void> {
     registry.clear();
     definitions.length = 0;
@@ -30,58 +100,64 @@ async function syncAgents(): Promise<void> {
     const memories: Array<{ _tenant_: string; collection: string }> = [];
 
     for (const tenant of cfg.tenants ?? []) {
-        const agentsPath = path.join(process.cwd(), tenant.dir, "agents");
-        if (!(await fs.exists(agentsPath))) continue;
-        if (!(await fs.stat(agentsPath)).isDirectory()) continue;
-
-        const glob = new Glob(path.join(agentsPath, "**/*.agent.ts"));
-        for await (const file of glob.scan(".")) {
-            const module = await importDefinition(file, "Agent");
-            const definition: AgentDefinition | undefined = module?.default;
-            if (!definition?._isAgent_) continue;
-            if (definition.enabled === false) continue;
-
-            // `id` and `description` are required — the `Agent` constructor refuses a
-            // definition without them, and a broken file never hides the others
-            const id = definition.id!;
-
-            const key = `${tenant.id}:${id}`;
-            const existing = registry.get(key);
-            if (existing) {
-                logger.warn(`Duplicate agent id '${id}' for tenant '${tenant.id}' — '${file}' replaces the previous definition`);
+        for (const root of agentRoots(tenant.dir)) {
+            const agentsPath = path.join(process.cwd(), root.dir);
+            if (!(await fs.exists(agentsPath))) continue;
+            if (!(await fs.stat(agentsPath)).isDirectory()) continue;
+            if (root.legacy) {
+                warnLegacy(tenant.id, path.join(tenant.dir, "agents"), path.join(tenant.dir, "ai", "agents"));
             }
 
-            try {
-                // `skills` are read **once**, here: a run injects the documents into
-                // its prompt, it never touches the filesystem
-                const baseDir = path.join(process.cwd(), tenant.dir);
-                const skills = await resolveSkills(definition.skills, { baseDir, agentId: id });
-
-                // A referenced sub-agent brings its own declaration: its `skills` are
-                // resolved here too, so delegating is the same as running it directly
-                // (and still never reads a file at run time)
-                for (const referenced of definition.agents
-                    ? (Array.isArray(definition.agents) ? definition.agents : [definition.agents])
-                    : []) {
-                    if (referenced && !referenced._skills_?.length) {
-                        referenced._skills_ = await resolveSkills(referenced.skills, { baseDir, agentId: referenced.id });
-                    }
+            const glob = new Glob(path.join(agentsPath, "**/*.agent.ts"));
+            for await (const file of glob.scan(".")) {
+                const module = await importDefinition(file, "Agent");
+                const definition: AgentDefinition | undefined = module?.default;
+                if (!definition?._isAgent_) continue;
+                if (definition.enabled === false) continue;
+
+                // `id` and `description` are required — the `Agent` constructor refuses a
+                // definition without them, and a broken file never hides the others
+                const id = definition.id!;
+
+                const key = `${tenant.id}:${id}`;
+                const existing = registry.get(key);
+                if (existing) {
+                    logger.warn(`Duplicate agent id '${id}' for tenant '${tenant.id}' — '${file}' replaces the previous definition`);
                 }
 
-                const instance = new Agent({ ...definition, id, _skills_: skills }, { tenant: tenant.id });
-                registry.set(key, instance);
-                definitions.push(instance.getConfig());
+                try {
+                    // `skills` are read **once**, here: a run injects the documents into
+                    // its prompt, it never touches the filesystem
+                    const baseDir = path.join(process.cwd(), tenant.dir);
+                    const skills = await resolveSkills(definition.skills, { baseDir, agentId: id });
+                    const tools = mergeTools(await loadSiblingTools(file), definition.tools);
+
+                    // A referenced sub-agent brings its own declaration: its `skills` are
+                    // resolved here too, so delegating is the same as running it directly
+                    // (and still never reads a file at run time)
+                    for (const referenced of definition.agents
+                        ? (Array.isArray(definition.agents) ? definition.agents : [definition.agents])
+                        : []) {
+                        if (referenced && !referenced._skills_?.length) {
+                            referenced._skills_ = await resolveSkills(referenced.skills, { baseDir, agentId: referenced.id });
+                        }
+                    }
+
+                    const instance = new Agent({ ...definition, id, tools, _skills_: skills }, { tenant: tenant.id });
+                    registry.set(key, instance);
+                    definitions.push(instance.getConfig());
 
-                // A store says which collections it writes to (a Mongo memory does):
-                // published on `cfg` so the replication engine copies them, exactly
-                // like a declared collection — the tenant chooses the name.
-                for (const collection of instance.getMemory()?.collections?.() ?? []) {
-                    if (!memories.some((m) => m._tenant_ === tenant.id && m.collection === collection)) {
-                        memories.push({ _tenant_: tenant.id, collection });
+                    // A store says which collections it writes to (a Mongo memory does):
+                    // published on `cfg` so the replication engine copies them, exactly
+                    // like a declared collection — the tenant chooses the name.
+                    for (const collection of instance.getMemory()?.collections?.() ?? []) {
+                        if (!memories.some((m) => m._tenant_ === tenant.id && m.collection === collection)) {
+                            memories.push({ _tenant_: tenant.id, collection });
+                        }
                     }
+                } catch (err: any) {
+                    console.error(`Failed to load agent '${id}' of tenant '${tenant.id}': ${err?.message}`);
                 }
-            } catch (err: any) {
-                console.error(`Failed to load agent '${id}' of tenant '${tenant.id}': ${err?.message}`);
             }
         }
     }
@@ -156,7 +232,7 @@ const agents = {
     list: (tenantId: string, rest?: InstanceType<typeof useRest>): Agent[] => createAgents(tenantId, rest).list(),
     /** Tenant-scoped registry — the same object a context receives as `agents`. */
     use: createAgents,
-    /** Re-scan `{tenant.dir}/agents` and rebuild the registry. */
+    /** Re-scan `{tenant.dir}/ai/agents` (and the legacy `agents/`) and rebuild the registry. */
     reload: syncAgents,
     /** `{ total, tenants }` — what the boot banner prints. */
     stats: agentsStats,
@@ -164,6 +240,11 @@ const agents = {
         InMemory: InMemoryAgentMemory,
         Mongo: MongoAgentMemory,
         Redis: RedisAgentMemory,
+        // Your own storage — scoping and the message cap are applied for you.
+        custom: customAgentMemory,
+        // A `load`/`save` processor, not a store: recalls past messages by meaning
+        // from **your** vector index (`embed`, `search`, `upsert`).
+        semanticRecall,
         // A `load` processor, not a store: it compacts a thread with **your** model
         // and prompt. Every store above is a backend; this one shapes the prompt.
         summarize,

package/lib/attachment.ts

@@ -1,4 +1,5 @@
 import path from "path";
+import { realpath, stat } from "node:fs/promises";
 import { AppError } from "./error";
 import { openZip, type ZipArchive } from "./zip";
 import type { AgentAttachment, AgentMediaPart, AgentTextPart } from "../types/agent";
@@ -217,10 +218,30 @@ export type NormalizedAttachment = {
  */
 async function materialize(attachment: AgentAttachment, baseDir: string): Promise<NormalizedAttachment> {
     if (typeof attachment === "string") {
-        const file = path.resolve(baseDir, attachment);
-        if (!(await Bun.file(file).exists())) {
-            throw new AppError(`Attachment '${attachment}' not found`, { status: 400, code: "AGENT_FILE_NOT_FOUND" });
+        // A path attachment is a **server-side** feature (scripts, in-process SDK):
+        // it may never escape `filesBaseDir` — not through `..`, not through an
+        // absolute path, and not through a symlink pointing outside (issue #29).
+        const root = path.resolve(baseDir);
+        const candidate = path.resolve(root, attachment);
+        if (candidate !== root && !candidate.startsWith(root + path.sep)) {
+            throw new AppError("Attachment path escapes the base directory", { status: 400, code: "AGENT_FILE_INVALID" });
         }
+
+        let file: string;
+        try {
+            file = await realpath(candidate);
+        } catch {
+            throw new AppError("Attachment not found", { status: 400, code: "AGENT_FILE_NOT_FOUND" });
+        }
+
+        const realRoot = await realpath(root).catch(() => root);
+        if (file !== realRoot && !file.startsWith(realRoot + path.sep)) {
+            throw new AppError("Attachment path escapes the base directory", { status: 400, code: "AGENT_FILE_INVALID" });
+        }
+        if (!(await stat(file)).isFile()) {
+            throw new AppError("Attachment is not a regular file", { status: 400, code: "AGENT_FILE_NOT_FOUND" });
+        }
+
         const bytes = new Uint8Array(await Bun.file(file).arrayBuffer());
         return { name: path.basename(file), mimeType: "", bytes };
     }

package/lib/define.ts

@@ -199,7 +199,11 @@ function Agent(config: AgentConfig & { id: string; description: string }): Agent
  * The `id` is **required** (it is the name the model calls).
  * A `define.McpTool` is accepted by an agent as-is.
  */
-function Tool(tool: AgentToolDefinition): AgentToolDefinition {
+function Tool<S extends import("zod").ZodType>(
+    tool: AgentToolDefinition<import("zod").output<S>> & { inputSchema: S },
+): AgentToolDefinition<import("zod").output<S>>;
+function Tool<TInput = any>(tool: AgentToolDefinition<TInput>): AgentToolDefinition<TInput>;
+function Tool(tool: AgentToolDefinition<any>): AgentToolDefinition<any> {
     return {
         ...tool,
         enabled: tool.enabled ?? true,

package/lib/eval.ts

@@ -0,0 +1,123 @@
+import type { Agent } from "./agent";
+import type {
+    AgentCallOptions,
+    AgentGenerateResult,
+    AgentInput,
+    AgentToolStats,
+    AgentUsage,
+} from "../types/agent";
+
+/** One case of an evaluation — the input, and what this case overrides. */
+export type AgentEvalCase = {
+    /** Named in the report (default: `case-1`, `case-2`…). */
+    name?: string;
+    input: AgentInput;
+    options?: AgentCallOptions;
+};
+
+/** What a grader may return: a bare score, or a score with its reason. */
+export type AgentEvalGrader = (
+    result: AgentGenerateResult,
+    testCase: AgentEvalCase,
+) => number | { score: number; reason?: string } | Promise<number | { score: number; reason?: string }>;
+
+export type AgentEvalCaseReport = {
+    name: string;
+    /** Mean of this case's grader scores, in `0..1`. */
+    score: number;
+    /** The reasons the graders gave, in order (a thrown grader included). */
+    reasons: string[];
+    finishReason: string;
+    usage: AgentUsage;
+    toolStats: AgentToolStats;
+};
+
+export type AgentEvalReport = {
+    /** Mean of the case scores, in `0..1` (`0` for no case). */
+    score: number;
+    /** Cases reaching `passScore`, or more. */
+    passed: number;
+    cases: AgentEvalCaseReport[];
+};
+
+/** A case passes at or above this score. */
+const DEFAULT_PASS_SCORE = 0.5;
+
+/** Keep whatever a grader returned inside `0..1`. */
+function clampScore(score: number): number {
+    if (!Number.isFinite(score)) return 0;
+    return Math.min(1, Math.max(0, score));
+}
+
+function caseName(testCase: AgentEvalCase, index: number): string {
+    return testCase.name?.trim() || `case-${index + 1}`;
+}
+
+/**
+ * Run an agent over a list of cases and score the answers with graders.
+ *
+ * The cases run **sequentially** (a run may write to a thread), each one with
+ * `options` overlaid by its own: `{ ...options, ...testCase.options }`.
+ *
+ * ```ts
+ * const report = await evaluateAgent({
+ *   agent,
+ *   cases: [{ name: 'greeting', input: 'Say hello' }],
+ *   graders: [(result) => (result.text.includes('hello') ? 1 : 0)],
+ * });
+ * ```
+ */
+export async function evaluateAgent(options: {
+    agent: Agent;
+    cases: AgentEvalCase[];
+    graders: AgentEvalGrader[];
+    /** A case passes at or above this score (default 0.5). */
+    passScore?: number;
+    /** Options merged into every case's call. */
+    options?: AgentCallOptions;
+}): Promise<AgentEvalReport> {
+    const { agent, cases, graders } = options;
+    const passScore = options.passScore ?? DEFAULT_PASS_SCORE;
+    const reports: AgentEvalCaseReport[] = [];
+
+    for (const [index, testCase] of cases.entries()) {
+        const result = await agent.generate(testCase.input, { ...options.options, ...testCase.options });
+
+        const scores: number[] = [];
+        const reasons: string[] = [];
+        for (const grader of graders) {
+            try {
+                const outcome = await grader(result, testCase);
+                if (typeof outcome === "number") {
+                    scores.push(clampScore(outcome));
+                } else {
+                    scores.push(clampScore(outcome.score));
+                    if (outcome.reason) reasons.push(outcome.reason);
+                }
+            } catch (err: any) {
+                // One broken grader is a zero, not the end of the evaluation
+                scores.push(0);
+                reasons.push(err?.message ?? String(err));
+            }
+        }
+
+        reports.push({
+            name: caseName(testCase, index),
+            score: scores.length ? scores.reduce((sum, score) => sum + score, 0) / scores.length : 0,
+            reasons,
+            finishReason: result.finishReason,
+            usage: result.usage,
+            toolStats: result.toolStats,
+        });
+    }
+
+    const score = reports.length
+        ? reports.reduce((sum, report) => sum + report.score, 0) / reports.length
+        : 0;
+
+    return {
+        score,
+        passed: reports.filter((report) => report.score >= passScore).length,
+        cases: reports,
+    };
+}