@inbrowser/agent 0.0.0-placeholder → 0.2.0

package/dist/eval/run-record.d.ts

@@ -0,0 +1,76 @@
+/**
+ * `RunRecord` — the eval harness's per-trial capture envelope.
+ *
+ * One `RunRecord` is produced for each `runFixture` call. It is the
+ * stable contract every downstream phase-one branch consumes:
+ *
+ *   - `eval/metric-collector` derives the eight metrics from a record.
+ *   - `eval/comparison-report` aggregates records across trials and
+ *     compares two sets.
+ *   - `eval/success-spec-framework` uses `finalWorkspace`, `trace`,
+ *     and `assistantText` as inputs to its spec functions.
+ *
+ * The shape is intentionally narrow and frozen-in-place: everything
+ * the metric collector needs already lives elsewhere (the trace
+ * carries usage, tool calls, and turn boundaries; the final
+ * workspace + runtime describe the agent-visible end state). The
+ * runner does not aggregate, does not derive metrics, and does not
+ * decide pass/fail. Those are downstream concerns by design.
+ *
+ * Browser-safe — no Node imports. The runner that produces these is
+ * also browser-safe in v1; a future on-disk persistence helper would
+ * live behind `@inbrowser/agent/node`.
+ *
+ * Note on naming: `@inbrowser/agent` also exports an unrelated
+ * `RunRecord` from `metrics/runs.ts` (a per-MCP-tool-call NDJSON
+ * record). To avoid breaking that public surface, the eval-side type
+ * is re-exported from the package root as `EvalRunRecord`. Within the
+ * eval barrel and inside this branch, the natural `RunRecord` name is
+ * used.
+ */
+import type { RuntimeState } from '../types/runtime.js';
+import type { TraceEvent } from '../types/trace.js';
+import type { Workspace } from '../types/workspace.js';
+import type { TaskFixture } from './fixture.js';
+export interface RunRecord {
+    /** The input fixture, echoed verbatim so a record is
+     *  self-describing and a downstream reader does not have to
+     *  cross-reference an external fixture id to know what was run. */
+    fixture: TaskFixture;
+    /** Zero-indexed trial number for this fixture. When a batch driver
+     *  runs N trials per fixture, the i-th trial carries `trial: i`. */
+    trial: number;
+    /** Every trace event emitted by the strategy during this run, in
+     *  emission order. Includes `llm_request`, `llm_response`, and
+     *  `turn_dispatch_complete` events. Empty when no events were
+     *  emitted before an early abort. */
+    trace: TraceEvent[];
+    /** The `Workspace` value at the end of the run. Equal to the
+     *  fixture-seeded workspace when no tool produced a workspace
+     *  patch. Frozen by the session, so safe to share by reference. */
+    finalWorkspace: Workspace;
+    /** The `RuntimeState` at the end of the run. Same freezing
+     *  guarantees as `finalWorkspace`. */
+    finalRuntime: RuntimeState;
+    /** Concatenated assistant text across every turn in this run. A
+     *  convenience for spec evaluators that match against assistant
+     *  output — they do not have to re-walk the trace. */
+    assistantText: string;
+    /** Wall-clock ms at run start. Captured by the runner immediately
+     *  before it submits the prompt. */
+    startedAt: number;
+    /** Wall-clock ms at run completion (or termination). Captured by
+     *  the runner immediately after the session event stream drains or
+     *  the run is aborted. */
+    completedAt: number;
+    /** Optional seed echoed from `runFixture`'s input. Threaded for
+     *  traceability only — v1 does not enforce determinism on
+     *  strategies or LLM clients. */
+    seed?: number;
+    /** `null` on a clean finish. A string when the run terminated for
+     *  any non-success reason: external abort, max wall-clock
+     *  exceeded, an unexpected exception thrown by the session, or a
+     *  session-emitted `error` event. */
+    error: string | null;
+}
+//# sourceMappingURL=run-record.d.ts.map

package/dist/eval/run-record.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-record.d.ts","sourceRoot":"","sources":["../../src/eval/run-record.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,MAAM,WAAW,SAAS;IACxB;;uEAEmE;IACnE,OAAO,EAAE,WAAW,CAAC;IACrB;wEACoE;IACpE,KAAK,EAAE,MAAM,CAAC;IACd;;;yCAGqC;IACrC,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB;;uEAEmE;IACnE,cAAc,EAAE,SAAS,CAAC;IAC1B;0CACsC;IACtC,YAAY,EAAE,YAAY,CAAC;IAC3B;;0DAEsD;IACtD,aAAa,EAAE,MAAM,CAAC;IACtB;wCACoC;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB;;8BAE0B;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB;;qCAEiC;IACjC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;yCAGqC;IACrC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB"}

package/dist/eval/run-record.js

@@ -0,0 +1,32 @@
+/**
+ * `RunRecord` — the eval harness's per-trial capture envelope.
+ *
+ * One `RunRecord` is produced for each `runFixture` call. It is the
+ * stable contract every downstream phase-one branch consumes:
+ *
+ *   - `eval/metric-collector` derives the eight metrics from a record.
+ *   - `eval/comparison-report` aggregates records across trials and
+ *     compares two sets.
+ *   - `eval/success-spec-framework` uses `finalWorkspace`, `trace`,
+ *     and `assistantText` as inputs to its spec functions.
+ *
+ * The shape is intentionally narrow and frozen-in-place: everything
+ * the metric collector needs already lives elsewhere (the trace
+ * carries usage, tool calls, and turn boundaries; the final
+ * workspace + runtime describe the agent-visible end state). The
+ * runner does not aggregate, does not derive metrics, and does not
+ * decide pass/fail. Those are downstream concerns by design.
+ *
+ * Browser-safe — no Node imports. The runner that produces these is
+ * also browser-safe in v1; a future on-disk persistence helper would
+ * live behind `@inbrowser/agent/node`.
+ *
+ * Note on naming: `@inbrowser/agent` also exports an unrelated
+ * `RunRecord` from `metrics/runs.ts` (a per-MCP-tool-call NDJSON
+ * record). To avoid breaking that public surface, the eval-side type
+ * is re-exported from the package root as `EvalRunRecord`. Within the
+ * eval barrel and inside this branch, the natural `RunRecord` name is
+ * used.
+ */
+export {};
+//# sourceMappingURL=run-record.js.map

package/dist/eval/run-record.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-record.js","sourceRoot":"","sources":["../../src/eval/run-record.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG"}

package/dist/eval/runner.d.ts

@@ -0,0 +1,140 @@
+/**
+ * `runFixture` + `runFixtures` — the eval harness's session driver.
+ *
+ * Given a `TaskFixture`, an `LlmClient`, and a tool surface, the
+ * runner:
+ *
+ *   1. Builds the seeded `Workspace` from `EMPTY_WORKSPACE +
+ *      fixture.initialState`.
+ *   2. Threads that workspace through the system prompt builder and
+ *      the per-call `ToolContext` so the agent sees the seeded
+ *      state.
+ *   3. Plugs a buffering `Tracer` into the session so every
+ *      `TraceEvent` (request, response, dispatch-complete) lands in
+ *      the captured record.
+ *   4. Submits the fixture's prompt and drains the session event
+ *      stream, shadow-tracking workspace + runtime patches as they
+ *      flow back through `tool_finished` events.
+ *   5. Returns a `RunRecord` with the trace, the final workspace +
+ *      runtime, the concatenated assistant text, timing, and a
+ *      `error` slot that is `null` on a clean finish or a string on
+ *      any non-success outcome.
+ *
+ * Errors never throw out of `runFixture`. An aborted run, a
+ * wall-clock cap exceeded, a session-emitted `error` event, or an
+ * unexpected exception all resolve to a `RunRecord` with `error` set
+ * and a partial trace.
+ *
+ * The runner is browser-safe — no `node:*` imports. Persistence
+ * helpers (if anyone wants them) belong in
+ * `runner-persistence-node.ts` and route through `src/node.ts`. That
+ * file does not exist in v1 by design; downstream consumers can
+ * persist in-memory records however they like.
+ */
+import type { LlmClient } from '../types/llm.js';
+import type { RuntimeState } from '../types/runtime.js';
+import type { AgentStrategy } from '../types/strategy.js';
+import type { ToolDispatch, ToolHandler } from '../types/tools.js';
+import { type Workspace } from '../types/workspace.js';
+import { type TaskFixture } from './fixture.js';
+import type { RunRecord } from './run-record.js';
+/**
+ * Input contract for `runFixture`. The caller supplies the fixture,
+ * the LLM client, and the tool surface; the runner owns everything
+ * else.
+ */
+export interface RunFixtureInput {
+    fixture: TaskFixture;
+    /** The chat client the strategy will drive. Tests pass a canned
+     *  stub; production code wires the real provider adapter. */
+    llm: LlmClient;
+    /** Tool dispatcher the session uses to execute tool calls. Tests
+     *  can pass `createDispatch(createToolRegistry())` for the no-tool
+     *  path. */
+    tools: ToolDispatch;
+    /** Tool handlers the LLM should see this run. Empty when the
+     *  fixture is exercised against a no-tool baseline. */
+    toolList: ToolHandler[];
+    /** Optional strategy override. Defaults to a fresh
+     *  `createReactLoopStrategy()`. */
+    strategy?: AgentStrategy;
+    /** Zero-indexed trial number. Defaults to 0. The batch driver
+     *  passes the trial it is currently on. */
+    trial?: number;
+    /** Optional external abort signal. Wired into the session's
+     *  `submit()` call. */
+    signal?: AbortSignal;
+    /** Optional wall-clock cap in ms. When the run exceeds this, the
+     *  runner aborts the session and resolves with `error` set. Absent
+     *  means no cap. */
+    maxWallClockMs?: number;
+    /** Optional seed echoed back through the `RunRecord` for
+     *  traceability. Not consumed by the runner; strategies + LLM
+     *  clients decide whether to honor it. */
+    seed?: number;
+    /** Optional system-prompt builder. Defaults to the canonical
+     *  fixture-aware prompt — see `defaultSystemPromptBuilder` for the
+     *  shape. */
+    systemPromptBuilder?: (workspace: Workspace, runtime: RuntimeState) => string;
+}
+/**
+ * Drive a single `TaskFixture` end-to-end and resolve with a
+ * `RunRecord` describing the captured run. Never throws.
+ */
+export declare function runFixture(input: RunFixtureInput): Promise<RunRecord>;
+/**
+ * Options for `runFixtures`. Sequential by default; `parallelism` is
+ * accepted for forward-compatibility but v1 honors only `1`. Records
+ * are returned in input order regardless.
+ */
+export interface RunFixturesOptions {
+    /** Number of trials per fixture. Defaults to 1. */
+    trials?: number;
+    /** Forward-compatibility hint. v1 always runs sequentially. A
+     *  future implementation may honor values > 1; downstream branches
+     *  should pass `1` (or omit) until then. */
+    parallelism?: number;
+    /** Optional external abort signal. Forwarded to every `runFixture`
+     *  call; an abort short-circuits the remaining trials. */
+    signal?: AbortSignal;
+    /** Optional wall-clock cap per trial. Forwarded as-is. */
+    maxWallClockMs?: number;
+    /** Optional seed factory. Called once per (fixture, trial). The
+     *  return value is echoed onto the produced `RunRecord.seed`. */
+    seed?: (fixture: TaskFixture, trial: number) => number;
+}
+/**
+ * Static dependencies that every fixture in the batch shares — the
+ * LLM client, the tool dispatch, the tool list, and (optionally) a
+ * strategy factory. Kept separate from `RunFixturesOptions` so the
+ * caller can vary trial-count knobs without re-supplying the heavy
+ * dependencies.
+ */
+export interface RunFixturesDeps {
+    llm: LlmClient;
+    tools: ToolDispatch;
+    toolList: ToolHandler[];
+    /** Optional strategy factory — called once per trial so each trial
+     *  gets a fresh strategy. Defaults to `createReactLoopStrategy()`. */
+    strategy?: () => AgentStrategy;
+    /** Optional system-prompt builder forwarded to every trial. */
+    systemPromptBuilder?: (workspace: Workspace, runtime: RuntimeState) => string;
+}
+/**
+ * Drive every fixture in `fixtures` for `trials` trials each.
+ * Returns records in `(fixture, trial)` order — the i-th fixture's
+ * trials come before the (i+1)-th fixture's trials, and within each
+ * fixture trial 0 comes before trial 1, etc.
+ *
+ * Sequential by design; see `RunFixturesOptions.parallelism`. The
+ * eval harness's job is reproducible measurement, not throughput.
+ */
+export declare function runFixtures(fixtures: readonly TaskFixture[], deps: RunFixturesDeps, options?: RunFixturesOptions): Promise<RunRecord[]>;
+/**
+ * Default system prompt builder used when the caller does not supply
+ * one. Intentionally minimal — production callers should pass their
+ * own. The eval harness uses this only so a test can run end-to-end
+ * without wiring a real prompt.
+ */
+export declare function defaultSystemPromptBuilder(workspace: Workspace, runtime: RuntimeState): string;
+//# sourceMappingURL=runner.d.ts.map

package/dist/eval/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAGxD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,KAAK,EAAe,YAAY,EAAE,WAAW,EAAc,MAAM,mBAAmB,CAAC;AAE5F,OAAO,EAAmB,KAAK,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACxE,OAAO,EAAE,KAAK,WAAW,EAA2B,MAAM,cAAc,CAAC;AACzE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,WAAW,CAAC;IACrB;iEAC6D;IAC7D,GAAG,EAAE,SAAS,CAAC;IACf;;gBAEY;IACZ,KAAK,EAAE,YAAY,CAAC;IACpB;2DACuD;IACvD,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB;uCACmC;IACnC,QAAQ,CAAC,EAAE,aAAa,CAAC;IACzB;+CAC2C;IAC3C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;2BACuB;IACvB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;wBAEoB;IACpB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;8CAE0C;IAC1C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;iBAEa;IACb,mBAAmB,CAAC,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,YAAY,KAAK,MAAM,CAAC;CAC/E;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,SAAS,CAAC,CA0I3E;AAED;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IACjC,mDAAmD;IACnD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;gDAE4C;IAC5C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;8DAC0D;IAC1D,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,0DAA0D;IAC1D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;qEACiE;IACjE,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC;CACxD;AAED;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,SAAS,CAAC;IACf,KAAK,EAAE,YAAY,CAAC;IACpB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB;0EACsE;IACtE,QAAQ,CAAC,EAAE,MAAM,aAAa,CAAC;IAC/B,+DAA+D;IAC/D,mBAAmB,CAAC,EAAE,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,YAAY,KAAK,MAAM,CAAC;CAC/E;AAED;;;;;;;;GAQG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,SAAS,WAAW,EAAE,EAChC,IAAI,EAAE,eAAe,EACrB,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,SAAS,EAAE,CAAC,CAwCtB;AAED;;;;;GAKG;AACH,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,YAAY,GAAG,MAAM,CAO9F"}

package/dist/eval/runner.js

@@ -0,0 +1,310 @@
+/**
+ * `runFixture` + `runFixtures` — the eval harness's session driver.
+ *
+ * Given a `TaskFixture`, an `LlmClient`, and a tool surface, the
+ * runner:
+ *
+ *   1. Builds the seeded `Workspace` from `EMPTY_WORKSPACE +
+ *      fixture.initialState`.
+ *   2. Threads that workspace through the system prompt builder and
+ *      the per-call `ToolContext` so the agent sees the seeded
+ *      state.
+ *   3. Plugs a buffering `Tracer` into the session so every
+ *      `TraceEvent` (request, response, dispatch-complete) lands in
+ *      the captured record.
+ *   4. Submits the fixture's prompt and drains the session event
+ *      stream, shadow-tracking workspace + runtime patches as they
+ *      flow back through `tool_finished` events.
+ *   5. Returns a `RunRecord` with the trace, the final workspace +
+ *      runtime, the concatenated assistant text, timing, and a
+ *      `error` slot that is `null` on a clean finish or a string on
+ *      any non-success outcome.
+ *
+ * Errors never throw out of `runFixture`. An aborted run, a
+ * wall-clock cap exceeded, a session-emitted `error` event, or an
+ * unexpected exception all resolve to a `RunRecord` with `error` set
+ * and a partial trace.
+ *
+ * The runner is browser-safe — no `node:*` imports. Persistence
+ * helpers (if anyone wants them) belong in
+ * `runner-persistence-node.ts` and route through `src/node.ts`. That
+ * file does not exist in v1 by design; downstream consumers can
+ * persist in-memory records however they like.
+ */
+import { createMetricsCollector } from '../metrics.js';
+import { createAgentSession } from '../session.js';
+import { createReactLoopStrategy } from '../strategy.js';
+import { EMPTY_RUNTIME } from '../types/runtime.js';
+import { EMPTY_WORKSPACE } from '../types/workspace.js';
+import { applyWorkspaceOverrides } from './fixture.js';
+/**
+ * Drive a single `TaskFixture` end-to-end and resolve with a
+ * `RunRecord` describing the captured run. Never throws.
+ */
+export async function runFixture(input) {
+    const trial = input.trial ?? 0;
+    const trace = [];
+    const tracer = {
+        emit(event) {
+            trace.push(event);
+        },
+    };
+    // Seed workspace + runtime from the fixture. The session does not
+    // accept a seeded workspace at construction time, so we shadow it:
+    // the seeded value is the one the agent sees (via systemPrompt +
+    // toolContext), and `tool_finished` events carry the patches we
+    // need to overlay back onto the shadow.
+    let shadowWorkspace = applyWorkspaceOverrides(EMPTY_WORKSPACE, input.fixture.initialState);
+    let shadowRuntime = EMPTY_RUNTIME;
+    // Compose the external signal with our own controller so we can
+    // abort cleanly on the wall-clock cap.
+    const internalAbort = new AbortController();
+    const linkedSignal = linkSignals(input.signal, internalAbort.signal);
+    // Wall-clock cap. `unref()` would be nicer but is Node-only; we
+    // just clear the timer on a clean finish.
+    let wallClockTimer;
+    let wallClockExceeded = false;
+    if (typeof input.maxWallClockMs === 'number' && input.maxWallClockMs > 0) {
+        wallClockTimer = setTimeout(() => {
+            wallClockExceeded = true;
+            internalAbort.abort();
+        }, input.maxWallClockMs);
+    }
+    const startedAt = Date.now();
+    let assistantText = '';
+    let error = null;
+    const strategy = input.strategy ?? createReactLoopStrategy();
+    const promptBuilder = input.systemPromptBuilder ?? defaultSystemPromptBuilder;
+    // toolContext factory threads the live shadow workspace + runtime
+    // into every per-call context, so tools that depend on the
+    // workspace see the seeded state rather than the session's empty
+    // baseline.
+    const toolContext = () => ({
+        workspace: shadowWorkspace,
+        runtime: shadowRuntime,
+        signal: linkedSignal,
+    });
+    let session;
+    try {
+        session = createAgentSession({
+            strategy,
+            llm: input.llm,
+            tools: input.tools,
+            toolList: input.toolList,
+            toolContext,
+            // The system prompt is built from the SEEDED workspace, not the
+            // session's empty baseline. The session calls this once per
+            // submit with its own (empty) workspace; we ignore that and
+            // pass our shadow instead.
+            systemPromptBuilder: (_ws, _rt) => promptBuilder(shadowWorkspace, shadowRuntime),
+            metrics: createMetricsCollector(),
+            history: [],
+            tracer,
+        });
+    }
+    catch (e) {
+        if (wallClockTimer)
+            clearTimeout(wallClockTimer);
+        return {
+            fixture: input.fixture,
+            trial,
+            trace,
+            finalWorkspace: shadowWorkspace,
+            finalRuntime: shadowRuntime,
+            assistantText: '',
+            startedAt,
+            completedAt: Date.now(),
+            ...(input.seed !== undefined ? { seed: input.seed } : {}),
+            error: errorMessage(e),
+        };
+    }
+    try {
+        const events = session.submit(input.fixture.prompt, linkedSignal);
+        for await (const ev of events) {
+            if (ev.kind === 'text') {
+                assistantText += ev.chunk;
+            }
+            else if (ev.kind === 'tool_finished') {
+                const result = ev.result;
+                if (result.workspacePatch) {
+                    shadowWorkspace = mergeWorkspace(shadowWorkspace, result.workspacePatch);
+                }
+                if (result.runtimePatch) {
+                    shadowRuntime = mergeRuntime(shadowRuntime, result.runtimePatch);
+                }
+            }
+            else if (ev.kind === 'error') {
+                // The session emitted an error event mid-stream; the stream
+                // closes after this. Record it as the run's error.
+                if (error === null)
+                    error = ev.message;
+            }
+        }
+    }
+    catch (e) {
+        error = errorMessage(e);
+    }
+    finally {
+        if (wallClockTimer)
+            clearTimeout(wallClockTimer);
+    }
+    // Wall-clock cap takes precedence over a generic 'aborted' message
+    // so the caller can tell why the run ended.
+    if (wallClockExceeded && (error === null || /abort/i.test(error))) {
+        error = `runFixture: exceeded maxWallClockMs (${input.maxWallClockMs}ms)`;
+    }
+    else if (error === null &&
+        (linkedSignal.aborted || input.signal?.aborted === true) &&
+        !sawCompletion(trace)) {
+        // External abort with no session-emitted error and no terminal
+        // turn — record an aborted message so consumers do not have to
+        // probe the signal themselves.
+        error = 'runFixture: aborted';
+    }
+    return {
+        fixture: input.fixture,
+        trial,
+        trace,
+        finalWorkspace: shadowWorkspace,
+        finalRuntime: shadowRuntime,
+        assistantText,
+        startedAt,
+        completedAt: Date.now(),
+        ...(input.seed !== undefined ? { seed: input.seed } : {}),
+        error,
+    };
+}
+/**
+ * Drive every fixture in `fixtures` for `trials` trials each.
+ * Returns records in `(fixture, trial)` order — the i-th fixture's
+ * trials come before the (i+1)-th fixture's trials, and within each
+ * fixture trial 0 comes before trial 1, etc.
+ *
+ * Sequential by design; see `RunFixturesOptions.parallelism`. The
+ * eval harness's job is reproducible measurement, not throughput.
+ */
+export async function runFixtures(fixtures, deps, options = {}) {
+    const trials = options.trials ?? 1;
+    const out = [];
+    for (const fixture of fixtures) {
+        for (let trial = 0; trial < trials; trial++) {
+            if (options.signal?.aborted) {
+                // Synthesize a placeholder record so callers can see which
+                // trial was skipped. Trace + state are empty; error
+                // carries the abort.
+                out.push({
+                    fixture,
+                    trial,
+                    trace: [],
+                    finalWorkspace: applyWorkspaceOverrides(EMPTY_WORKSPACE, fixture.initialState),
+                    finalRuntime: EMPTY_RUNTIME,
+                    assistantText: '',
+                    startedAt: Date.now(),
+                    completedAt: Date.now(),
+                    error: 'runFixtures: aborted before trial start',
+                });
+                continue;
+            }
+            const seed = options.seed ? options.seed(fixture, trial) : undefined;
+            const strategy = deps.strategy ? deps.strategy() : undefined;
+            const record = await runFixture({
+                fixture,
+                llm: deps.llm,
+                tools: deps.tools,
+                toolList: deps.toolList,
+                ...(strategy ? { strategy } : {}),
+                trial,
+                ...(options.signal ? { signal: options.signal } : {}),
+                ...(options.maxWallClockMs !== undefined ? { maxWallClockMs: options.maxWallClockMs } : {}),
+                ...(seed !== undefined ? { seed } : {}),
+                ...(deps.systemPromptBuilder ? { systemPromptBuilder: deps.systemPromptBuilder } : {}),
+            });
+            out.push(record);
+        }
+    }
+    return out;
+}
+/**
+ * Default system prompt builder used when the caller does not supply
+ * one. Intentionally minimal — production callers should pass their
+ * own. The eval harness uses this only so a test can run end-to-end
+ * without wiring a real prompt.
+ */
+export function defaultSystemPromptBuilder(workspace, runtime) {
+    const parts = ['You are an evaluation harness agent.'];
+    if (workspace.rules)
+        parts.push(`Rules:\n${workspace.rules}`);
+    if (workspace.code)
+        parts.push(`Code:\n${workspace.code}`);
+    if (workspace.appSource)
+        parts.push(`App:\n${workspace.appSource}`);
+    if (runtime.runSummary)
+        parts.push(`Last run: ${JSON.stringify(runtime.runSummary)}`);
+    return parts.join('\n\n');
+}
+function mergeWorkspace(base, patch) {
+    // Mirror session.ts freezing semantics so consumers see a frozen
+    // shadow workspace too. The cost is one shallow clone per patch,
+    // which is negligible in eval scenarios.
+    return Object.freeze({
+        ...base,
+        ...patch,
+        stitch: Object.freeze({
+            ...base.stitch,
+            ...(patch.stitch ?? {}),
+        }),
+    });
+}
+function mergeRuntime(base, patch) {
+    return Object.freeze({
+        ...base,
+        ...patch,
+        terminal: Object.freeze([
+            ...(patch.terminal ?? base.terminal),
+        ]),
+        uiErrors: Object.freeze([
+            ...(patch.uiErrors ?? base.uiErrors),
+        ]),
+    });
+}
+function errorMessage(e) {
+    if (e instanceof Error)
+        return e.message;
+    if (typeof e === 'string')
+        return e;
+    try {
+        return JSON.stringify(e);
+    }
+    catch {
+        return String(e);
+    }
+}
+/** Combine two abort signals (one optional) into a single signal. */
+function linkSignals(external, internal) {
+    if (!external)
+        return internal;
+    if (external.aborted) {
+        const c = new AbortController();
+        c.abort();
+        return c.signal;
+    }
+    const controller = new AbortController();
+    if (internal.aborted) {
+        controller.abort();
+        return controller.signal;
+    }
+    external.addEventListener('abort', () => controller.abort(), { once: true });
+    internal.addEventListener('abort', () => controller.abort(), { once: true });
+    return controller.signal;
+}
+function sawCompletion(trace) {
+    // A clean finish emits at least one llm_response. If we have any
+    // response trace, treat the run as having reached the strategy
+    // proper rather than being aborted before it started.
+    for (const ev of trace) {
+        if (ev.kind === 'llm_response')
+            return true;
+    }
+    return false;
+}
+//# sourceMappingURL=runner.js.map

package/dist/eval/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AAGzD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAKpD,OAAO,EAAE,eAAe,EAAkB,MAAM,uBAAuB,CAAC;AACxE,OAAO,EAAoB,uBAAuB,EAAE,MAAM,cAAc,CAAC;AA2CzE;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAsB;IACrD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,MAAM,MAAM,GAAW;QACrB,IAAI,CAAC,KAAK;YACR,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,CAAC;KACF,CAAC;IAEF,kEAAkE;IAClE,mEAAmE;IACnE,iEAAiE;IACjE,gEAAgE;IAChE,wCAAwC;IACxC,IAAI,eAAe,GAAc,uBAAuB,CACtD,eAAe,EACf,KAAK,CAAC,OAAO,CAAC,YAAY,CAC3B,CAAC;IACF,IAAI,aAAa,GAAiB,aAAa,CAAC;IAEhD,gEAAgE;IAChE,uCAAuC;IACvC,MAAM,aAAa,GAAG,IAAI,eAAe,EAAE,CAAC;IAC5C,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;IAErE,gEAAgE;IAChE,0CAA0C;IAC1C,IAAI,cAAyD,CAAC;IAC9D,IAAI,iBAAiB,GAAG,KAAK,CAAC;IAC9B,IAAI,OAAO,KAAK,CAAC,cAAc,KAAK,QAAQ,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;QACzE,cAAc,GAAG,UAAU,CAAC,GAAG,EAAE;YAC/B,iBAAiB,GAAG,IAAI,CAAC;YACzB,aAAa,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC;IAC3B,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,KAAK,GAAkB,IAAI,CAAC;IAEhC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,uBAAuB,EAAE,CAAC;IAC7D,MAAM,aAAa,GAAG,KAAK,CAAC,mBAAmB,IAAI,0BAA0B,CAAC;IAE9E,kEAAkE;IAClE,2DAA2D;IAC3D,iEAAiE;IACjE,YAAY;IACZ,MAAM,WAAW,GAAG,GAAgB,EAAE,CAAC,CAAC;QACtC,SAAS,EAAE,eAAe;QAC1B,OAAO,EAAE,aAAa;QACtB,MAAM,EAAE,YAAY;KACrB,CAAC,CAAC;IAEH,IAAI,OAA8C,CAAC;IACnD,IAAI,CAAC;QACH,OAAO,GAAG,kBAAkB,CAAC;YAC3B,QAAQ;YACR,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,WAAW;YACX,gEAAgE;YAChE,4DAA4D;YAC5D,4DAA4D;YAC5D,2BAA2B;YAC3B,mBAAmB,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,aAAa,CAAC,eAAe,EAAE,aAAa,CAAC;YAChF,OAAO,EAAE,sBAAsB,EAAE;YACjC,OAAO,EAAE,EAAE;YACX,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,cAAc;YAAE,YAAY,CAAC,cAAc,CAAC,CAAC;QACjD,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK;YACL,KAAK;YACL,cAAc,EAAE,eAAe;YAC/B,YAAY,EAAE,aAAa;YAC3B,aAAa,EAAE,EAAE;YACjB,SAAS;YACT,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;YACvB,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzD,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC;SACvB,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,MAAM,GAAgC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAC/F,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;YAC9B,IAAI,EAAE,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACvB,aAAa,IAAI,EAAE,CAAC,KAAK,CAAC;YAC5B,CAAC;iBAAM,IAAI,EAAE,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;gBACvC,MAAM,MAAM,GAAG,EAAE,CAAC,MAAoB,CAAC;gBACvC,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;oBAC1B,eAAe,GAAG,cAAc,CAAC,eAAe,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC;gBAC3E,CAAC;gBACD,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;oBACxB,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;gBACnE,CAAC;YACH,CAAC;iBAAM,IAAI,EAAE,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC/B,4DAA4D;gBAC5D,mDAAmD;gBACnD,IAAI,KAAK,KAAK,IAAI;oBAAE,KAAK,GAAG,EAAE,CAAC,OAAO,CAAC;YACzC,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;IAC1B,CAAC;YAAS,CAAC;QACT,IAAI,cAAc;YAAE,YAAY,CAAC,cAAc,CAAC,CAAC;IACnD,CAAC;IAED,mEAAmE;IACnE,4CAA4C;IAC5C,IAAI,iBAAiB,IAAI,CAAC,KAAK,KAAK,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAClE,KAAK,GAAG,wCAAwC,KAAK,CAAC,cAAc,KAAK,CAAC;IAC5E,CAAC;SAAM,IACL,KAAK,KAAK,IAAI;QACd,CAAC,YAAY,CAAC,OAAO,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO,KAAK,IAAI,CAAC;QACxD,CAAC,aAAa,CAAC,KAAK,CAAC,EACrB,CAAC;QACD,+DAA+D;QAC/D,+DAA+D;QAC/D,+BAA+B;QAC/B,KAAK,GAAG,qBAAqB,CAAC;IAChC,CAAC;IAED,OAAO;QACL,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,KAAK;QACL,KAAK;QACL,cAAc,EAAE,eAAe;QAC/B,YAAY,EAAE,aAAa;QAC3B,aAAa;QACb,SAAS;QACT,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;QACvB,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzD,KAAK;KACN,CAAC;AACJ,CAAC;AA0CD;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAgC,EAChC,IAAqB,EACrB,UAA8B,EAAE;IAEhC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;IACnC,MAAM,GAAG,GAAgB,EAAE,CAAC;IAC5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YAC5C,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBAC5B,2DAA2D;gBAC3D,oDAAoD;gBACpD,qBAAqB;gBACrB,GAAG,CAAC,IAAI,CAAC;oBACP,OAAO;oBACP,KAAK;oBACL,KAAK,EAAE,EAAE;oBACT,cAAc,EAAE,uBAAuB,CAAC,eAAe,EAAE,OAAO,CAAC,YAAY,CAAC;oBAC9E,YAAY,EAAE,aAAa;oBAC3B,aAAa,EAAE,EAAE;oBACjB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;oBACrB,WAAW,EAAE,IAAI,CAAC,GAAG,EAAE;oBACvB,KAAK,EAAE,yCAAyC;iBACjD,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YACD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACrE,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7D,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC;gBAC9B,OAAO;gBACP,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjC,KAAK;gBACL,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrD,GAAG,CAAC,OAAO,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3F,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvC,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACvF,CAAC,CAAC;YACH,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,0BAA0B,CAAC,SAAoB,EAAE,OAAqB;IACpF,MAAM,KAAK,GAAa,CAAC,sCAAsC,CAAC,CAAC;IACjE,IAAI,SAAS,CAAC,KAAK;QAAE,KAAK,CAAC,IAAI,CAAC,WAAW,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC;IAC9D,IAAI,SAAS,CAAC,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,UAAU,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3D,IAAI,SAAS,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,SAAS,CAAC,SAAS,EAAE,CAAC,CAAC;IACpE,IAAI,OAAO,CAAC,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACtF,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,cAAc,CAAC,IAAe,EAAE,KAAyB;IAChE,iEAAiE;IACjE,iEAAiE;IACjE,yCAAyC;IACzC,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,GAAG,IAAI;QACP,GAAG,KAAK;QACR,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;YACpB,GAAG,IAAI,CAAC,MAAM;YACd,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC;SACxB,CAAC;KACH,CAAc,CAAC;AAClB,CAAC;AAED,SAAS,YAAY,CAAC,IAAkB,EAAE,KAA4B;IACpE,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,GAAG,IAAI;QACP,GAAG,KAAK;QACR,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC;YACtB,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC;SACrC,CAAwC;QACzC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC;YACtB,GAAG,CAAC,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC;SACrC,CAAwC;KAC1C,CAAiB,CAAC;AACrB,CAAC;AAED,SAAS,YAAY,CAAC,CAAU;IAC9B,IAAI,CAAC,YAAY,KAAK;QAAE,OAAO,CAAC,CAAC,OAAO,CAAC;IACzC,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC;IACpC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC;AACH,CAAC;AAED,qEAAqE;AACrE,SAAS,WAAW,CAAC,QAAiC,EAAE,QAAqB;IAC3E,IAAI,CAAC,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC/B,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,IAAI,eAAe,EAAE,CAAC;QAChC,CAAC,CAAC,KAAK,EAAE,CAAC;QACV,OAAO,CAAC,CAAC,MAAM,CAAC;IAClB,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACrB,UAAU,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B,CAAC;IACD,QAAQ,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7E,QAAQ,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7E,OAAO,UAAU,CAAC,MAAM,CAAC;AAC3B,CAAC;AAED,SAAS,aAAa,CAAC,KAAmB;IACxC,iEAAiE;IACjE,+DAA+D;IAC/D,sDAAsD;IACtD,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,EAAE,CAAC,IAAI,KAAK,cAAc;YAAE,OAAO,IAAI,CAAC;IAC9C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}