@agent-native/core 0.52.0 → 0.54.0

package/dist/eval/agent-runner.d.ts

@@ -0,0 +1,63 @@
+/**
+ * The headless agent-run seam used by the evals runner.
+ *
+ * This invokes the real `runAgentLoop` as a *caller* — resolving a provider-
+ * agnostic engine + model from the existing registry, converting the app's
+ * actions into engine tools, and collecting the assistant's text + tool calls
+ * off the `send` event stream into a compact `AgentRunOutput`. It deliberately
+ * does NOT modify `production-agent.ts`: everything it needs (`runAgentLoop`,
+ * `actionsToEngineTools`, `ActionEntry`) is already exported from there.
+ *
+ * The factory shape (`createAgentRunner`) keeps the runner unit-testable: tests
+ * inject a fake `runAgentLoop` and a fake engine so no real model is called,
+ * while production wires in the genuine loop. The same factory builds the
+ * `ScorerAnalyzeContext.judge` helper so LLM-judge scorers stream through the
+ * exact same resolved engine.
+ */
+import type { ActionEntry, AgentLoopUsage } from "../agent/production-agent.js";
+import type { AgentEngine, EngineMessage, EngineTool } from "../agent/engine/types.js";
+import type { AgentChatEvent } from "../agent/types.js";
+import type { AgentRunOutput, EvalInput, ScorerAnalyzeContext } from "./types.js";
+/** The slice of `runAgentLoop` the runner depends on — injectable for tests. */
+export type RunAgentLoopFn = (opts: {
+    engine: AgentEngine;
+    model: string;
+    systemPrompt: string;
+    tools: EngineTool[];
+    messages: EngineMessage[];
+    actions: Record<string, ActionEntry>;
+    send: (event: AgentChatEvent) => void;
+    signal: AbortSignal;
+}) => Promise<AgentLoopUsage>;
+export interface AgentRunnerConfig {
+    /** App actions to expose to the agent under test. */
+    actions: Record<string, ActionEntry>;
+    /** System prompt for the run. */
+    systemPrompt?: string;
+    /** Pre-resolved engine; resolved from the registry when omitted. */
+    engine?: AgentEngine;
+    /** Pre-resolved model; resolved from the engine's stored/default when omitted. */
+    model?: string;
+    /** Per-run wall-clock budget in ms (default 120s). */
+    timeoutMs?: number;
+    /**
+     * Seam for tests / custom hosts. Defaults to the real `runAgentLoop`. The
+     * runner never imports `runAgentLoop` directly so this can be swapped.
+     */
+    runLoop?: RunAgentLoopFn;
+}
+export interface AgentRunner {
+    /** Run the agent loop for one eval input and collect a compact output. */
+    runAgent(input: EvalInput): Promise<AgentRunOutput>;
+    /** Analyze context handed to LLM-judge scorers (shares engine/model). */
+    analyzeContext(): ScorerAnalyzeContext;
+    readonly engine: AgentEngine;
+    readonly model: string;
+}
+/**
+ * Build an agent runner, resolving the engine/model once up front so every
+ * eval case (and every LLM-judge scorer) reuses the same provider-agnostic
+ * config. Resolution goes through `resolveEngine` — no model is ever hardcoded.
+ */
+export declare function createAgentRunner(config: AgentRunnerConfig): Promise<AgentRunner>;
+//# sourceMappingURL=agent-runner.d.ts.map

package/dist/eval/agent-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-runner.d.ts","sourceRoot":"","sources":["../../src/eval/agent-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAKhF,OAAO,KAAK,EACV,WAAW,EACX,aAAa,EACb,UAAU,EACX,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAMxD,OAAO,KAAK,EACV,cAAc,EACd,SAAS,EACT,oBAAoB,EACrB,MAAM,YAAY,CAAC;AAKpB,gFAAgF;AAChF,MAAM,MAAM,cAAc,GAAG,CAAC,IAAI,EAAE;IAClC,MAAM,EAAE,WAAW,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACrC,IAAI,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;IACtC,MAAM,EAAE,WAAW,CAAC;CACrB,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;AAE9B,MAAM,WAAW,iBAAiB;IAChC,qDAAqD;IACrD,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACrC,iCAAiC;IACjC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,kFAAkF;IAClF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC;CAC1B;AAED,MAAM,WAAW,WAAW;IAC1B,0EAA0E;IAC1E,QAAQ,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IACpD,yEAAyE;IACzE,cAAc,IAAI,oBAAoB,CAAC;IACvC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAiBD;;;;GAIG;AACH,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,iBAAiB,GACxB,OAAO,CAAC,WAAW,CAAC,CA0GtB"}

package/dist/eval/agent-runner.js

@@ -0,0 +1,142 @@
+/**
+ * The headless agent-run seam used by the evals runner.
+ *
+ * This invokes the real `runAgentLoop` as a *caller* — resolving a provider-
+ * agnostic engine + model from the existing registry, converting the app's
+ * actions into engine tools, and collecting the assistant's text + tool calls
+ * off the `send` event stream into a compact `AgentRunOutput`. It deliberately
+ * does NOT modify `production-agent.ts`: everything it needs (`runAgentLoop`,
+ * `actionsToEngineTools`, `ActionEntry`) is already exported from there.
+ *
+ * The factory shape (`createAgentRunner`) keeps the runner unit-testable: tests
+ * inject a fake `runAgentLoop` and a fake engine so no real model is called,
+ * while production wires in the genuine loop. The same factory builds the
+ * `ScorerAnalyzeContext.judge` helper so LLM-judge scorers stream through the
+ * exact same resolved engine.
+ */
+import { runAgentLoop, actionsToEngineTools, } from "../agent/production-agent.js";
+import { resolveEngine, getStoredModelForEngine, normalizeModelForEngine, } from "../agent/engine/index.js";
+const JUDGE_TIMEOUT_MS = 30_000;
+const DEFAULT_AGENT_TIMEOUT_MS = 120_000;
+function toEngineMessages(input) {
+    const messages = [];
+    for (const turn of input.history ?? []) {
+        messages.push({
+            role: turn.role,
+            content: [{ type: "text", text: turn.text }],
+        });
+    }
+    messages.push({
+        role: "user",
+        content: [{ type: "text", text: input.prompt }],
+    });
+    return messages;
+}
+/**
+ * Build an agent runner, resolving the engine/model once up front so every
+ * eval case (and every LLM-judge scorer) reuses the same provider-agnostic
+ * config. Resolution goes through `resolveEngine` — no model is ever hardcoded.
+ */
+export async function createAgentRunner(config) {
+    const engine = config.engine ?? (await resolveEngine({ engineOption: undefined }));
+    const modelCandidate = config.model ??
+        (await getStoredModelForEngine(engine)) ??
+        engine.defaultModel;
+    const model = normalizeModelForEngine(engine, modelCandidate);
+    const systemPrompt = config.systemPrompt ?? "";
+    const runLoop = config.runLoop ?? runAgentLoop;
+    const timeoutMs = config.timeoutMs ?? DEFAULT_AGENT_TIMEOUT_MS;
+    const tools = actionsToEngineTools(config.actions);
+    async function runAgent(input) {
+        const runId = `eval:${crypto.randomUUID()}`;
+        const messages = toEngineMessages(input);
+        let text = "";
+        const toolCalls = [];
+        let ok = true;
+        let error;
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), timeoutMs);
+        const started = Date.now();
+        const send = (event) => {
+            switch (event.type) {
+                case "text":
+                    text += event.text;
+                    break;
+                case "tool_start":
+                    toolCalls.push(event.tool);
+                    break;
+                case "error":
+                    ok = false;
+                    error = event.error;
+                    break;
+                default:
+                    break;
+            }
+        };
+        try {
+            await runLoop({
+                engine,
+                model,
+                systemPrompt,
+                tools,
+                messages,
+                actions: config.actions,
+                send,
+                signal: controller.signal,
+            });
+        }
+        catch (err) {
+            ok = false;
+            error = err instanceof Error ? err.message : String(err);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+        return {
+            text,
+            toolCalls,
+            ok,
+            error,
+            runId,
+            durationMs: Date.now() - started,
+        };
+    }
+    function analyzeContext() {
+        return {
+            engine,
+            model,
+            async judge(opts) {
+                const controller = new AbortController();
+                const signal = opts.signal ?? controller.signal;
+                const timer = opts.signal
+                    ? undefined
+                    : setTimeout(() => controller.abort(), JUDGE_TIMEOUT_MS);
+                let out = "";
+                try {
+                    const stream = engine.stream({
+                        model,
+                        systemPrompt: opts.systemPrompt ?? "",
+                        messages: [
+                            { role: "user", content: [{ type: "text", text: opts.prompt }] },
+                        ],
+                        tools: [],
+                        abortSignal: signal,
+                        maxOutputTokens: opts.maxOutputTokens ?? 512,
+                        temperature: 0,
+                    });
+                    for await (const event of stream) {
+                        if (event.type === "text-delta")
+                            out += event.text;
+                    }
+                }
+                finally {
+                    if (timer)
+                        clearTimeout(timer);
+                }
+                return out;
+            },
+        };
+    }
+    return { runAgent, analyzeContext, engine, model };
+}
+//# sourceMappingURL=agent-runner.js.map

package/dist/eval/agent-runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-runner.js","sourceRoot":"","sources":["../../src/eval/agent-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAGH,OAAO,EACL,YAAY,EACZ,oBAAoB,GACrB,MAAM,8BAA8B,CAAC;AAOtC,OAAO,EACL,aAAa,EACb,uBAAuB,EACvB,uBAAuB,GACxB,MAAM,0BAA0B,CAAC;AAOlC,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAChC,MAAM,wBAAwB,GAAG,OAAO,CAAC;AAyCzC,SAAS,gBAAgB,CAAC,KAAgB;IACxC,MAAM,QAAQ,GAAoB,EAAE,CAAC;IACrC,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QACvC,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;SAC7C,CAAC,CAAC;IACL,CAAC;IACD,QAAQ,CAAC,IAAI,CAAC;QACZ,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;KAChD,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAyB;IAEzB,MAAM,MAAM,GACV,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,aAAa,CAAC,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;IACtE,MAAM,cAAc,GAClB,MAAM,CAAC,KAAK;QACZ,CAAC,MAAM,uBAAuB,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,CAAC,YAAY,CAAC;IACtB,MAAM,KAAK,GAAG,uBAAuB,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IAC9D,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAK,YAA+B,CAAC;IACnE,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,wBAAwB,CAAC;IAC/D,MAAM,KAAK,GAAG,oBAAoB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnD,KAAK,UAAU,QAAQ,CAAC,KAAgB;QACtC,MAAM,KAAK,GAAG,QAAQ,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;QAEzC,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,IAAI,EAAE,GAAG,IAAI,CAAC;QACd,IAAI,KAAyB,CAAC;QAE9B,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;QAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE3B,MAAM,IAAI,GAAG,CAAC,KAAqB,EAAQ,EAAE;YAC3C,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;gBACnB,KAAK,MAAM;oBACT,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC;oBACnB,MAAM;gBACR,KAAK,YAAY;oBACf,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAC3B,MAAM;gBACR,KAAK,OAAO;oBACV,EAAE,GAAG,KAAK,CAAC;oBACX,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;oBACpB,MAAM;gBACR;oBACE,MAAM;YACV,CAAC;QACH,CAAC,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,OAAO,CAAC;gBACZ,MAAM;gBACN,KAAK;gBACL,YAAY;gBACZ,KAAK;gBACL,QAAQ;gBACR,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,IAAI;gBACJ,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,EAAE,GAAG,KAAK,CAAC;YACX,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3D,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;QAED,OAAO;YACL,IAAI;YACJ,SAAS;YACT,EAAE;YACF,KAAK;YACL,KAAK;YACL,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SACjC,CAAC;IACJ,CAAC;IAED,SAAS,cAAc;QACrB,OAAO;YACL,MAAM;YACN,KAAK;YACL,KAAK,CAAC,KAAK,CAAC,IAAI;gBACd,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;gBACzC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC;gBAChD,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM;oBACvB,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,gBAAgB,CAAC,CAAC;gBAC3D,IAAI,GAAG,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;wBAC3B,KAAK;wBACL,YAAY,EAAE,IAAI,CAAC,YAAY,IAAI,EAAE;wBACrC,QAAQ,EAAE;4BACR,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE;yBACjE;wBACD,KAAK,EAAE,EAAE;wBACT,WAAW,EAAE,MAAM;wBACnB,eAAe,EAAE,IAAI,CAAC,eAAe,IAAI,GAAG;wBAC5C,WAAW,EAAE,CAAC;qBACf,CAAC,CAAC;oBACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;wBACjC,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY;4BAAE,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC;oBACrD,CAAC;gBACH,CAAC;wBAAS,CAAC;oBACT,IAAI,KAAK;wBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;gBACjC,CAAC;gBACD,OAAO,GAAG,CAAC;YACb,CAAC;SACF,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AACrD,CAAC","sourcesContent":["/**\n * The headless agent-run seam used by the evals runner.\n *\n * This invokes the real `runAgentLoop` as a *caller* — resolving a provider-\n * agnostic engine + model from the existing registry, converting the app's\n * actions into engine tools, and collecting the assistant's text + tool calls\n * off the `send` event stream into a compact `AgentRunOutput`. It deliberately\n * does NOT modify `production-agent.ts`: everything it needs (`runAgentLoop`,\n * `actionsToEngineTools`, `ActionEntry`) is already exported from there.\n *\n * The factory shape (`createAgentRunner`) keeps the runner unit-testable: tests\n * inject a fake `runAgentLoop` and a fake engine so no real model is called,\n * while production wires in the genuine loop. The same factory builds the\n * `ScorerAnalyzeContext.judge` helper so LLM-judge scorers stream through the\n * exact same resolved engine.\n */\n\nimport type { ActionEntry, AgentLoopUsage } from \"../agent/production-agent.js\";\nimport {\n  runAgentLoop,\n  actionsToEngineTools,\n} from \"../agent/production-agent.js\";\nimport type {\n  AgentEngine,\n  EngineMessage,\n  EngineTool,\n} from \"../agent/engine/types.js\";\nimport type { AgentChatEvent } from \"../agent/types.js\";\nimport {\n  resolveEngine,\n  getStoredModelForEngine,\n  normalizeModelForEngine,\n} from \"../agent/engine/index.js\";\nimport type {\n  AgentRunOutput,\n  EvalInput,\n  ScorerAnalyzeContext,\n} from \"./types.js\";\n\nconst JUDGE_TIMEOUT_MS = 30_000;\nconst DEFAULT_AGENT_TIMEOUT_MS = 120_000;\n\n/** The slice of `runAgentLoop` the runner depends on — injectable for tests. */\nexport type RunAgentLoopFn = (opts: {\n  engine: AgentEngine;\n  model: string;\n  systemPrompt: string;\n  tools: EngineTool[];\n  messages: EngineMessage[];\n  actions: Record<string, ActionEntry>;\n  send: (event: AgentChatEvent) => void;\n  signal: AbortSignal;\n}) => Promise<AgentLoopUsage>;\n\nexport interface AgentRunnerConfig {\n  /** App actions to expose to the agent under test. */\n  actions: Record<string, ActionEntry>;\n  /** System prompt for the run. */\n  systemPrompt?: string;\n  /** Pre-resolved engine; resolved from the registry when omitted. */\n  engine?: AgentEngine;\n  /** Pre-resolved model; resolved from the engine's stored/default when omitted. */\n  model?: string;\n  /** Per-run wall-clock budget in ms (default 120s). */\n  timeoutMs?: number;\n  /**\n   * Seam for tests / custom hosts. Defaults to the real `runAgentLoop`. The\n   * runner never imports `runAgentLoop` directly so this can be swapped.\n   */\n  runLoop?: RunAgentLoopFn;\n}\n\nexport interface AgentRunner {\n  /** Run the agent loop for one eval input and collect a compact output. */\n  runAgent(input: EvalInput): Promise<AgentRunOutput>;\n  /** Analyze context handed to LLM-judge scorers (shares engine/model). */\n  analyzeContext(): ScorerAnalyzeContext;\n  readonly engine: AgentEngine;\n  readonly model: string;\n}\n\nfunction toEngineMessages(input: EvalInput): EngineMessage[] {\n  const messages: EngineMessage[] = [];\n  for (const turn of input.history ?? []) {\n    messages.push({\n      role: turn.role,\n      content: [{ type: \"text\", text: turn.text }],\n    });\n  }\n  messages.push({\n    role: \"user\",\n    content: [{ type: \"text\", text: input.prompt }],\n  });\n  return messages;\n}\n\n/**\n * Build an agent runner, resolving the engine/model once up front so every\n * eval case (and every LLM-judge scorer) reuses the same provider-agnostic\n * config. Resolution goes through `resolveEngine` — no model is ever hardcoded.\n */\nexport async function createAgentRunner(\n  config: AgentRunnerConfig,\n): Promise<AgentRunner> {\n  const engine =\n    config.engine ?? (await resolveEngine({ engineOption: undefined }));\n  const modelCandidate =\n    config.model ??\n    (await getStoredModelForEngine(engine)) ??\n    engine.defaultModel;\n  const model = normalizeModelForEngine(engine, modelCandidate);\n  const systemPrompt = config.systemPrompt ?? \"\";\n  const runLoop = config.runLoop ?? (runAgentLoop as RunAgentLoopFn);\n  const timeoutMs = config.timeoutMs ?? DEFAULT_AGENT_TIMEOUT_MS;\n  const tools = actionsToEngineTools(config.actions);\n\n  async function runAgent(input: EvalInput): Promise<AgentRunOutput> {\n    const runId = `eval:${crypto.randomUUID()}`;\n    const messages = toEngineMessages(input);\n\n    let text = \"\";\n    const toolCalls: string[] = [];\n    let ok = true;\n    let error: string | undefined;\n\n    const controller = new AbortController();\n    const timer = setTimeout(() => controller.abort(), timeoutMs);\n    const started = Date.now();\n\n    const send = (event: AgentChatEvent): void => {\n      switch (event.type) {\n        case \"text\":\n          text += event.text;\n          break;\n        case \"tool_start\":\n          toolCalls.push(event.tool);\n          break;\n        case \"error\":\n          ok = false;\n          error = event.error;\n          break;\n        default:\n          break;\n      }\n    };\n\n    try {\n      await runLoop({\n        engine,\n        model,\n        systemPrompt,\n        tools,\n        messages,\n        actions: config.actions,\n        send,\n        signal: controller.signal,\n      });\n    } catch (err) {\n      ok = false;\n      error = err instanceof Error ? err.message : String(err);\n    } finally {\n      clearTimeout(timer);\n    }\n\n    return {\n      text,\n      toolCalls,\n      ok,\n      error,\n      runId,\n      durationMs: Date.now() - started,\n    };\n  }\n\n  function analyzeContext(): ScorerAnalyzeContext {\n    return {\n      engine,\n      model,\n      async judge(opts): Promise<string> {\n        const controller = new AbortController();\n        const signal = opts.signal ?? controller.signal;\n        const timer = opts.signal\n          ? undefined\n          : setTimeout(() => controller.abort(), JUDGE_TIMEOUT_MS);\n        let out = \"\";\n        try {\n          const stream = engine.stream({\n            model,\n            systemPrompt: opts.systemPrompt ?? \"\",\n            messages: [\n              { role: \"user\", content: [{ type: \"text\", text: opts.prompt }] },\n            ],\n            tools: [],\n            abortSignal: signal,\n            maxOutputTokens: opts.maxOutputTokens ?? 512,\n            temperature: 0,\n          });\n          for await (const event of stream) {\n            if (event.type === \"text-delta\") out += event.text;\n          }\n        } finally {\n          if (timer) clearTimeout(timer);\n        }\n        return out;\n      },\n    };\n  }\n\n  return { runAgent, analyzeContext, engine, model };\n}\n"]}

package/dist/eval/define-eval.d.ts

@@ -0,0 +1,29 @@
+/**
+ * `defineEval` — declare a named eval test case.
+ *
+ * An eval pairs an input (prompt + optional history/setup) with a list of
+ * scorers and a pass threshold. The runner (see `runner.ts`) actually runs the
+ * agent for the input, scores the output with each scorer, and gates on the
+ * threshold. Authors write `*.eval.ts` files that `export default defineEval(...)`
+ * or export an array of them.
+ *
+ * Example (`evals/greeting.eval.ts`):
+ * ```ts
+ * import { defineEval, contains, llmJudge } from "@agent-native/core/eval";
+ *
+ * export default defineEval({
+ *   name: "greets the user by name",
+ *   input: { prompt: "Say hi to Ada." },
+ *   threshold: 0.7,
+ *   scorers: [
+ *     contains("Ada"),
+ *     llmJudge({ criteria: "friendliness", rubric: "1.0 = warm greeting" }),
+ *   ],
+ * });
+ * ```
+ */
+import type { Eval } from "./types.js";
+/** Default per-scorer pass threshold when an eval doesn't specify one. */
+export declare const DEFAULT_EVAL_THRESHOLD = 0.5;
+export declare function defineEval(spec: Eval): Eval;
+//# sourceMappingURL=define-eval.d.ts.map

package/dist/eval/define-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"define-eval.d.ts","sourceRoot":"","sources":["../../src/eval/define-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,0EAA0E;AAC1E,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C,wBAAgB,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,CAqB3C"}

package/dist/eval/define-eval.js

@@ -0,0 +1,43 @@
+/**
+ * `defineEval` — declare a named eval test case.
+ *
+ * An eval pairs an input (prompt + optional history/setup) with a list of
+ * scorers and a pass threshold. The runner (see `runner.ts`) actually runs the
+ * agent for the input, scores the output with each scorer, and gates on the
+ * threshold. Authors write `*.eval.ts` files that `export default defineEval(...)`
+ * or export an array of them.
+ *
+ * Example (`evals/greeting.eval.ts`):
+ * ```ts
+ * import { defineEval, contains, llmJudge } from "@agent-native/core/eval";
+ *
+ * export default defineEval({
+ *   name: "greets the user by name",
+ *   input: { prompt: "Say hi to Ada." },
+ *   threshold: 0.7,
+ *   scorers: [
+ *     contains("Ada"),
+ *     llmJudge({ criteria: "friendliness", rubric: "1.0 = warm greeting" }),
+ *   ],
+ * });
+ * ```
+ */
+/** Default per-scorer pass threshold when an eval doesn't specify one. */
+export const DEFAULT_EVAL_THRESHOLD = 0.5;
+export function defineEval(spec) {
+    if (!spec.name || typeof spec.name !== "string") {
+        throw new Error("defineEval: `name` is required");
+    }
+    if (!spec.input || typeof spec.input.prompt !== "string") {
+        throw new Error(`defineEval("${spec.name}"): \`input.prompt\` is required`);
+    }
+    if (!Array.isArray(spec.scorers) || spec.scorers.length === 0) {
+        throw new Error(`defineEval("${spec.name}"): at least one scorer is required`);
+    }
+    if (spec.threshold !== undefined &&
+        (spec.threshold < 0 || spec.threshold > 1)) {
+        throw new Error(`defineEval("${spec.name}"): \`threshold\` must be in [0, 1]`);
+    }
+    return spec;
+}
+//# sourceMappingURL=define-eval.js.map

package/dist/eval/define-eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"define-eval.js","sourceRoot":"","sources":["../../src/eval/define-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH,0EAA0E;AAC1E,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C,MAAM,UAAU,UAAU,CAAC,IAAU;IACnC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACzD,MAAM,IAAI,KAAK,CAAC,eAAe,IAAI,CAAC,IAAI,kCAAkC,CAAC,CAAC;IAC9E,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CACb,eAAe,IAAI,CAAC,IAAI,qCAAqC,CAC9D,CAAC;IACJ,CAAC;IACD,IACE,IAAI,CAAC,SAAS,KAAK,SAAS;QAC5B,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,EAC1C,CAAC;QACD,MAAM,IAAI,KAAK,CACb,eAAe,IAAI,CAAC,IAAI,qCAAqC,CAC9D,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC","sourcesContent":["/**\n * `defineEval` — declare a named eval test case.\n *\n * An eval pairs an input (prompt + optional history/setup) with a list of\n * scorers and a pass threshold. The runner (see `runner.ts`) actually runs the\n * agent for the input, scores the output with each scorer, and gates on the\n * threshold. Authors write `*.eval.ts` files that `export default defineEval(...)`\n * or export an array of them.\n *\n * Example (`evals/greeting.eval.ts`):\n * ```ts\n * import { defineEval, contains, llmJudge } from \"@agent-native/core/eval\";\n *\n * export default defineEval({\n *   name: \"greets the user by name\",\n *   input: { prompt: \"Say hi to Ada.\" },\n *   threshold: 0.7,\n *   scorers: [\n *     contains(\"Ada\"),\n *     llmJudge({ criteria: \"friendliness\", rubric: \"1.0 = warm greeting\" }),\n *   ],\n * });\n * ```\n */\n\nimport type { Eval } from \"./types.js\";\n\n/** Default per-scorer pass threshold when an eval doesn't specify one. */\nexport const DEFAULT_EVAL_THRESHOLD = 0.5;\n\nexport function defineEval(spec: Eval): Eval {\n  if (!spec.name || typeof spec.name !== \"string\") {\n    throw new Error(\"defineEval: `name` is required\");\n  }\n  if (!spec.input || typeof spec.input.prompt !== \"string\") {\n    throw new Error(`defineEval(\"${spec.name}\"): \\`input.prompt\\` is required`);\n  }\n  if (!Array.isArray(spec.scorers) || spec.scorers.length === 0) {\n    throw new Error(\n      `defineEval(\"${spec.name}\"): at least one scorer is required`,\n    );\n  }\n  if (\n    spec.threshold !== undefined &&\n    (spec.threshold < 0 || spec.threshold > 1)\n  ) {\n    throw new Error(\n      `defineEval(\"${spec.name}\"): \\`threshold\\` must be in [0, 1]`,\n    );\n  }\n  return spec;\n}\n"]}

package/dist/eval/index.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Public surface for the first-class evals primitive.
+ *
+ * Authors write `*.eval.ts` (or `evals/*.ts`) files that `export default
+ * defineEval(...)`, compose scorers with `createScorer` / the built-ins, and
+ * run them with `agent-native eval` (which gates CI on the thresholds).
+ *
+ * This is complementary to `@agent-native/core`'s observability run-scoring:
+ * that scores real production runs after the fact; this actively runs the
+ * agent against fixed inputs as a deterministic gate. See `types.ts`.
+ */
+export { defineEval, DEFAULT_EVAL_THRESHOLD } from "./define-eval.js";
+export { createScorer, clamp01, exactMatch, contains, usesTool, llmJudge, type LlmJudgeOptions, } from "./scorer.js";
+export { createAgentRunner, type AgentRunner, type AgentRunnerConfig, type RunAgentLoopFn, } from "./agent-runner.js";
+export { runEvalSuite, runEvals, scoreEval, loadEvals, discoverEvalFiles, type RunEvalSuiteOptions, } from "./runner.js";
+export { formatReport } from "./report.js";
+export type { Eval, EvalInput, EvalRunContext, AgentRunOutput, Scorer, ScorerDefinition, ScorerAnalyzeContext, ScorerResult, EvalResultRow, EvalRunReport, } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/eval/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,UAAU,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EACL,YAAY,EACZ,OAAO,EACP,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,EACR,KAAK,eAAe,GACrB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,iBAAiB,EACjB,KAAK,WAAW,EAChB,KAAK,iBAAiB,EACtB,KAAK,cAAc,GACpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,YAAY,EACZ,QAAQ,EACR,SAAS,EACT,SAAS,EACT,iBAAiB,EACjB,KAAK,mBAAmB,GACzB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,YAAY,EACV,IAAI,EACJ,SAAS,EACT,cAAc,EACd,cAAc,EACd,MAAM,EACN,gBAAgB,EAChB,oBAAoB,EACpB,YAAY,EACZ,aAAa,EACb,aAAa,GACd,MAAM,YAAY,CAAC"}

package/dist/eval/index.js

@@ -0,0 +1,17 @@
+/**
+ * Public surface for the first-class evals primitive.
+ *
+ * Authors write `*.eval.ts` (or `evals/*.ts`) files that `export default
+ * defineEval(...)`, compose scorers with `createScorer` / the built-ins, and
+ * run them with `agent-native eval` (which gates CI on the thresholds).
+ *
+ * This is complementary to `@agent-native/core`'s observability run-scoring:
+ * that scores real production runs after the fact; this actively runs the
+ * agent against fixed inputs as a deterministic gate. See `types.ts`.
+ */
+export { defineEval, DEFAULT_EVAL_THRESHOLD } from "./define-eval.js";
+export { createScorer, clamp01, exactMatch, contains, usesTool, llmJudge, } from "./scorer.js";
+export { createAgentRunner, } from "./agent-runner.js";
+export { runEvalSuite, runEvals, scoreEval, loadEvals, discoverEvalFiles, } from "./runner.js";
+export { formatReport } from "./report.js";
+//# sourceMappingURL=index.js.map

package/dist/eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,UAAU,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AACtE,OAAO,EACL,YAAY,EACZ,OAAO,EACP,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,QAAQ,GAET,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,iBAAiB,GAIlB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,YAAY,EACZ,QAAQ,EACR,SAAS,EACT,SAAS,EACT,iBAAiB,GAElB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC","sourcesContent":["/**\n * Public surface for the first-class evals primitive.\n *\n * Authors write `*.eval.ts` (or `evals/*.ts`) files that `export default\n * defineEval(...)`, compose scorers with `createScorer` / the built-ins, and\n * run them with `agent-native eval` (which gates CI on the thresholds).\n *\n * This is complementary to `@agent-native/core`'s observability run-scoring:\n * that scores real production runs after the fact; this actively runs the\n * agent against fixed inputs as a deterministic gate. See `types.ts`.\n */\n\nexport { defineEval, DEFAULT_EVAL_THRESHOLD } from \"./define-eval.js\";\nexport {\n  createScorer,\n  clamp01,\n  exactMatch,\n  contains,\n  usesTool,\n  llmJudge,\n  type LlmJudgeOptions,\n} from \"./scorer.js\";\nexport {\n  createAgentRunner,\n  type AgentRunner,\n  type AgentRunnerConfig,\n  type RunAgentLoopFn,\n} from \"./agent-runner.js\";\nexport {\n  runEvalSuite,\n  runEvals,\n  scoreEval,\n  loadEvals,\n  discoverEvalFiles,\n  type RunEvalSuiteOptions,\n} from \"./runner.js\";\nexport { formatReport } from \"./report.js\";\nexport type {\n  Eval,\n  EvalInput,\n  EvalRunContext,\n  AgentRunOutput,\n  Scorer,\n  ScorerDefinition,\n  ScorerAnalyzeContext,\n  ScorerResult,\n  EvalResultRow,\n  EvalRunReport,\n} from \"./types.js\";\n"]}

package/dist/eval/report.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Human-readable formatting for an eval run report. Kept separate from the
+ * runner so the CLI can print a table while CI consumes the JSON shape.
+ */
+import type { EvalRunReport } from "./types.js";
+/** Render a scored table for the terminal. */
+export declare function formatReport(report: EvalRunReport): string;
+//# sourceMappingURL=report.d.ts.map

package/dist/eval/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/eval/report.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAgBhD,8CAA8C;AAC9C,wBAAgB,YAAY,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,CAqC1D"}

package/dist/eval/report.js

@@ -0,0 +1,44 @@
+/**
+ * Human-readable formatting for an eval run report. Kept separate from the
+ * runner so the CLI can print a table while CI consumes the JSON shape.
+ */
+function bar(score, width = 10) {
+    const filled = Math.round(clamp01(score) * width);
+    return "█".repeat(filled) + "░".repeat(width - filled);
+}
+function clamp01(n) {
+    if (!Number.isFinite(n))
+        return 0;
+    return Math.max(0, Math.min(1, n));
+}
+function pct(score) {
+    return `${Math.round(clamp01(score) * 100)}%`.padStart(4);
+}
+/** Render a scored table for the terminal. */
+export function formatReport(report) {
+    const lines = [];
+    lines.push("");
+    lines.push("  Evals");
+    lines.push("  ─────");
+    for (const row of report.results) {
+        const mark = row.passed ? "✓" : "✗";
+        lines.push("");
+        lines.push(`  ${mark} ${row.eval}  (avg ${pct(row.avgScore)}, threshold ${pct(row.threshold)})`);
+        if (row.error) {
+            lines.push(`      ⚠ run error: ${row.error}`);
+        }
+        for (const s of row.scores) {
+            const smark = s.passed ? "✓" : "✗";
+            const reason = s.reason ? `  — ${s.reason}` : "";
+            lines.push(`      ${smark} ${s.scorer.padEnd(20)} ${bar(s.score)} ${pct(s.score)}${reason}`);
+        }
+    }
+    lines.push("");
+    lines.push("  ─────");
+    const verdict = report.failed === 0 ? "PASS" : "FAIL";
+    lines.push(`  ${verdict}: ${report.passed}/${report.total} evals passed` +
+        (report.failed > 0 ? `, ${report.failed} below threshold` : ""));
+    lines.push("");
+    return lines.join("\n");
+}
+//# sourceMappingURL=report.js.map

package/dist/eval/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/eval/report.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,SAAS,GAAG,CAAC,KAAa,EAAE,KAAK,GAAG,EAAE;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC;IAClD,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,OAAO,CAAC,CAAS;IACxB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,GAAG,CAAC,KAAa;IACxB,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,YAAY,CAAC,MAAqB;IAChD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAEtB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CACR,KAAK,IAAI,IAAI,GAAG,CAAC,IAAI,UAAU,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,GAAG,CAChE,GAAG,CAAC,SAAS,CACd,GAAG,CACL,CAAC;QACF,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,sBAAsB,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAChD,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YACnC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,KAAK,CAAC,IAAI,CACR,SAAS,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAC1D,CAAC,CAAC,KAAK,CACR,GAAG,MAAM,EAAE,CACb,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IACtD,KAAK,CAAC,IAAI,CACR,KAAK,OAAO,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,eAAe;QAC3D,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,MAAM,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAClE,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC","sourcesContent":["/**\n * Human-readable formatting for an eval run report. Kept separate from the\n * runner so the CLI can print a table while CI consumes the JSON shape.\n */\n\nimport type { EvalRunReport } from \"./types.js\";\n\nfunction bar(score: number, width = 10): string {\n  const filled = Math.round(clamp01(score) * width);\n  return \"█\".repeat(filled) + \"░\".repeat(width - filled);\n}\n\nfunction clamp01(n: number): number {\n  if (!Number.isFinite(n)) return 0;\n  return Math.max(0, Math.min(1, n));\n}\n\nfunction pct(score: number): string {\n  return `${Math.round(clamp01(score) * 100)}%`.padStart(4);\n}\n\n/** Render a scored table for the terminal. */\nexport function formatReport(report: EvalRunReport): string {\n  const lines: string[] = [];\n  lines.push(\"\");\n  lines.push(\"  Evals\");\n  lines.push(\"  ─────\");\n\n  for (const row of report.results) {\n    const mark = row.passed ? \"✓\" : \"✗\";\n    lines.push(\"\");\n    lines.push(\n      `  ${mark} ${row.eval}  (avg ${pct(row.avgScore)}, threshold ${pct(\n        row.threshold,\n      )})`,\n    );\n    if (row.error) {\n      lines.push(`      ⚠ run error: ${row.error}`);\n    }\n    for (const s of row.scores) {\n      const smark = s.passed ? \"✓\" : \"✗\";\n      const reason = s.reason ? `  — ${s.reason}` : \"\";\n      lines.push(\n        `      ${smark} ${s.scorer.padEnd(20)} ${bar(s.score)} ${pct(\n          s.score,\n        )}${reason}`,\n      );\n    }\n  }\n\n  lines.push(\"\");\n  lines.push(\"  ─────\");\n  const verdict = report.failed === 0 ? \"PASS\" : \"FAIL\";\n  lines.push(\n    `  ${verdict}: ${report.passed}/${report.total} evals passed` +\n      (report.failed > 0 ? `, ${report.failed} below threshold` : \"\"),\n  );\n  lines.push(\"\");\n  return lines.join(\"\\n\");\n}\n"]}

package/dist/eval/runner.d.ts

@@ -0,0 +1,67 @@
+/**
+ * The evals runner: discover `*.eval.ts` / `evals/*.ts` files, run each eval
+ * through its scorer pipeline against the *real* agent loop, score, and report.
+ *
+ * It is the engine behind `agent-native eval` — when used as a CI deploy gate
+ * the CLI exits non-zero if any eval scores below its threshold.
+ *
+ * Two layers:
+ *   - `scoreEval` / `runEvals` — pure orchestration over an `AgentRunner` and
+ *     a list of evals. Fully unit-testable with an injected runner (no model).
+ *   - `discoverEvalFiles` / `loadEvals` — filesystem discovery + dynamic import
+ *     of author-written eval modules.
+ *
+ * Results are also (best-effort) written to the observability eval store so a
+ * dashboard can surface CI eval history next to production run evals.
+ */
+import type { ActionEntry } from "../agent/production-agent.js";
+import type { AgentRunner } from "./agent-runner.js";
+import type { Eval, EvalResultRow, EvalRunReport } from "./types.js";
+/** Run a single eval: invoke the agent, then score with each scorer. */
+export declare function scoreEval(evalCase: Eval, runner: AgentRunner, opts?: {
+    thresholdOverride?: number;
+}): Promise<EvalResultRow>;
+/** Run a batch of evals against one runner and aggregate a report. */
+export declare function runEvals(evals: Eval[], runner: AgentRunner, opts?: {
+    thresholdOverride?: number;
+    persist?: boolean;
+}): Promise<EvalRunReport>;
+/**
+ * Walk `root` for eval files. Matches two conventions:
+ *   - any `**\/*.eval.ts` (co-located with code), and
+ *   - any `*.ts` directly inside an `evals/` directory.
+ * `pattern` further filters by substring of the relative path.
+ */
+export declare function discoverEvalFiles(root: string, pattern?: string): Promise<string[]>;
+/** Discover and import all eval files under `root`, returning their evals. */
+export declare function loadEvals(root: string, pattern?: string): Promise<{
+    files: string[];
+    evals: Eval[];
+}>;
+export interface RunEvalSuiteOptions {
+    /** App root to discover eval files + actions under. Defaults to cwd. */
+    cwd?: string;
+    /** Substring filter on the eval file path. */
+    pattern?: string;
+    /** Global threshold override (wins over per-eval thresholds). */
+    thresholdOverride?: number;
+    /** App actions to expose to the agent. Auto-discovered when omitted. */
+    actions?: Record<string, ActionEntry>;
+    /** System prompt for runs. */
+    systemPrompt?: string;
+    /** Write results to the observability eval store (default true). */
+    persist?: boolean;
+    /** Pre-built runner (tests inject this to avoid touching engine/loop). */
+    runner?: AgentRunner;
+    /** Pre-loaded evals (tests inject this to skip filesystem discovery). */
+    evals?: Eval[];
+}
+/**
+ * End-to-end: load evals, build a runner, score, report. The CLI wraps this
+ * and maps `report.failed > 0` to a non-zero exit code (the CI gate).
+ */
+export declare function runEvalSuite(opts?: RunEvalSuiteOptions): Promise<{
+    report: EvalRunReport;
+    files: string[];
+}>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/eval/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAKH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAMhE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,KAAK,EAEV,IAAI,EACJ,aAAa,EACb,aAAa,EAEd,MAAM,YAAY,CAAC;AAsCpB,wEAAwE;AACxE,wBAAsB,SAAS,CAC7B,QAAQ,EAAE,IAAI,EACd,MAAM,EAAE,WAAW,EACnB,IAAI,GAAE;IAAE,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAAO,GACxC,OAAO,CAAC,aAAa,CAAC,CAkCxB;AAED,sEAAsE;AACtE,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,IAAI,EAAE,EACb,MAAM,EAAE,WAAW,EACnB,IAAI,GAAE;IAAE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAO,GAC3D,OAAO,CAAC,aAAa,CAAC,CAexB;AA4CD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,EAAE,CAAC,CAqCnB;AA0BD,8EAA8E;AAC9E,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,EAAE,IAAI,EAAE,CAAA;CAAE,CAAC,CAW7C;AAID,MAAM,WAAW,mBAAmB;IAClC,wEAAwE;IACxE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,8CAA8C;IAC9C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iEAAiE;IACjE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACtC,8BAA8B;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0EAA0E;IAC1E,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,yEAAyE;IACzE,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC;CAChB;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAChC,IAAI,GAAE,mBAAwB,GAC7B,OAAO,CAAC;IAAE,MAAM,EAAE,aAAa,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAuBrD"}