@plico-ai/runtime 0.0.0-reserved.0 → 0.1.0

package/dist/engine/memory-journal.js

@@ -0,0 +1,125 @@
+export function createMemoryJournal() {
+    const runs = new Map();
+    const steps = new Map();
+    const waits = new Map();
+    const signals = new Map();
+    return {
+        async begin(runId, inputJson, lease) {
+            if (runs.has(runId))
+                return false;
+            runs.set(runId, { runId, state: "running", inputJson, cancelRequested: false, lease });
+            return true;
+        },
+        async getRun(runId) {
+            const run = runs.get(runId);
+            return (run && {
+                runId,
+                state: run.state,
+                inputJson: run.inputJson,
+                cancelRequested: run.cancelRequested,
+            });
+        },
+        async setState(runId, state) {
+            const run = runs.get(runId);
+            if (run)
+                run.state = state;
+        },
+        async renew(runId, lease) {
+            const run = runs.get(runId);
+            if (run?.lease.workerId === lease.workerId)
+                run.lease = lease;
+        },
+        async claim(runId, lease, expected) {
+            const run = runs.get(runId);
+            if (!run)
+                return false;
+            if (expected &&
+                (run.lease.workerId !== expected.workerId || run.lease.until !== expected.until)) {
+                return false;
+            }
+            run.lease = lease;
+            run.state = "running";
+            return true;
+        },
+        async claimWaiting(runId, lease, expected) {
+            const run = runs.get(runId);
+            if (!run || run.state !== "waiting")
+                return false;
+            if (expected &&
+                (run.lease.workerId !== expected.workerId || run.lease.until !== expected.until))
+                return false;
+            run.lease = lease;
+            run.state = "running";
+            return true;
+        },
+        async listStale(workerId, now) {
+            return [...runs.values()]
+                .filter((run) => run.state === "running" && run.lease.workerId !== workerId && run.lease.until < now)
+                .map((run) => ({ runId: run.runId, held: run.lease }));
+        },
+        async listReady(now) {
+            return [...runs.values()]
+                .filter((run) => {
+                if (run.state !== "waiting")
+                    return false;
+                if (run.cancelRequested)
+                    return true;
+                return [...waits.entries()].some(([key, wait]) => {
+                    if (!key.startsWith(`${run.runId}\u0000`) || wait.outcomeJson !== null)
+                        return false;
+                    return signals.has(key) || (wait.deadline !== null && wait.deadline <= now);
+                });
+            })
+                .map((run) => ({ runId: run.runId, held: run.lease }));
+        },
+        async requestCancel(runId) {
+            const run = runs.get(runId);
+            if (!run || run.cancelRequested || ["completed", "failed", "cancelled"].includes(run.state)) {
+                return false;
+            }
+            run.cancelRequested = true;
+            return true;
+        },
+        async listWaiting() {
+            return [...runs.values()]
+                .filter((run) => run.state === "waiting")
+                .flatMap((run) => [...waits.entries()]
+                .filter(([key, wait]) => key.startsWith(`${run.runId}\0`) && wait.outcomeJson === null)
+                .map(([, wait]) => ({ runId: run.runId, deadline: wait.deadline })));
+        },
+        async getWait(runId, name) {
+            const wait = waits.get(`${runId}\u0000${name}`);
+            return wait && { ...wait };
+        },
+        async openWait(runId, name, deadline) {
+            const key = `${runId}\u0000${name}`;
+            if (!waits.has(key))
+                waits.set(key, { deadline, outcomeJson: null });
+        },
+        async resolveWait(runId, name, outcomeJson) {
+            const wait = waits.get(`${runId}\u0000${name}`);
+            if (!wait || wait.outcomeJson !== null)
+                return false;
+            wait.outcomeJson = outcomeJson;
+            return true;
+        },
+        async putSignal(runId, name, valueJson) {
+            const key = `${runId}\u0000${name}`;
+            if (signals.has(key))
+                return false;
+            signals.set(key, valueJson);
+            return true;
+        },
+        async getSignal(runId, name) {
+            return signals.get(`${runId}\u0000${name}`);
+        },
+        async getStep(runId, index) {
+            return steps.get(`${runId}:${index}`);
+        },
+        async putStep(runId, index, step) {
+            const key = `${runId}:${index}`;
+            if (!steps.has(key))
+                steps.set(key, step);
+        },
+    };
+}

package/dist/engine/sqlite-engine.d.ts

@@ -0,0 +1,7 @@
+import { type EngineOptions } from "./engine.js";
+import type { Engine } from "./types.js";
+export interface SqliteEngineOptions extends Omit<EngineOptions, "journal"> {
+    databasePath: string;
+}
+/** The local engine: runs survive a restart because the journal lives in the project's SQLite file. */
+export declare function createSqliteEngine({ databasePath, ...options }: SqliteEngineOptions): Engine;

package/dist/engine/sqlite-engine.js

@@ -0,0 +1,6 @@
+import { createEngine } from "./engine.js";
+import { createSqliteJournal } from "./sqlite-journal.js";
+/** The local engine: runs survive a restart because the journal lives in the project's SQLite file. */
+export function createSqliteEngine({ databasePath, ...options }) {
+    return createEngine({ ...options, journal: createSqliteJournal({ databasePath }) });
+}

package/dist/engine/sqlite-journal.d.ts

@@ -0,0 +1,5 @@
+import type { Journal } from "./journal.js";
+/** Keeps the journal in the project's SQLite file, in tables private to the engine. */
+export declare function createSqliteJournal({ databasePath }: {
+    databasePath: string;
+}): Journal;

package/dist/engine/sqlite-journal.js

@@ -0,0 +1,136 @@
+import { openDatabase } from "../storage/sqlite/open.js";
+import { acquireOwner, ownerIsAlive } from "../storage/sqlite/owner-lock.js";
+function leaseOf(row) {
+    return { workerId: row.worker_id, until: row.lease_until };
+}
+/** Keeps the journal in the project's SQLite file, in tables private to the engine. */
+export function createSqliteJournal({ databasePath }) {
+    const database = openDatabase(databasePath);
+    function readRun(runId) {
+        return database.prepare("SELECT * FROM engine_runs WHERE run_id = ?").get(runId);
+    }
+    return {
+        async begin(runId, inputJson, lease) {
+            const owner = acquireOwner();
+            const result = database
+                .prepare("INSERT OR IGNORE INTO engine_runs (run_id, state, input_json, worker_id, lease_until, owner_token, owner_path) VALUES (?, 'running', ?, ?, ?, ?, ?)")
+                .run(runId, inputJson, lease.workerId, lease.until, owner.token, owner.path);
+            return result.changes === 1;
+        },
+        async getRun(runId) {
+            const row = readRun(runId);
+            return (row && {
+                runId,
+                state: row.state,
+                inputJson: row.input_json,
+                cancelRequested: row.cancel_requested === 1,
+            });
+        },
+        async setState(runId, state) {
+            database.prepare("UPDATE engine_runs SET state = ? WHERE run_id = ?").run(state, runId);
+        },
+        async renew(runId, lease) {
+            database
+                .prepare("UPDATE engine_runs SET lease_until = ? WHERE run_id = ? AND worker_id = ?")
+                .run(lease.until, runId, lease.workerId);
+        },
+        async claim(runId, lease, expected) {
+            const owner = acquireOwner();
+            const result = expected
+                ? database
+                    .prepare("UPDATE engine_runs SET state = 'running', worker_id = ?, lease_until = ?, owner_token = ?, owner_path = ? WHERE run_id = ? AND worker_id = ? AND lease_until = ?")
+                    .run(lease.workerId, lease.until, owner.token, owner.path, runId, expected.workerId, expected.until)
+                : database
+                    .prepare("UPDATE engine_runs SET state = 'running', worker_id = ?, lease_until = ?, owner_token = ?, owner_path = ? WHERE run_id = ?")
+                    .run(lease.workerId, lease.until, owner.token, owner.path, runId);
+            return result.changes === 1;
+        },
+        async claimWaiting(runId, lease, expected) {
+            const owner = acquireOwner();
+            const sql = `UPDATE engine_runs SET state = 'running', worker_id = ?, lease_until = ?, owner_token = ?, owner_path = ? WHERE run_id = ? AND state = 'waiting'`;
+            const result = expected
+                ? database
+                    .prepare(`${sql} AND worker_id = ? AND lease_until = ?`)
+                    .run(lease.workerId, lease.until, owner.token, owner.path, runId, expected.workerId, expected.until)
+                : database.prepare(sql).run(lease.workerId, lease.until, owner.token, owner.path, runId);
+            return result.changes === 1;
+        },
+        async listStale(workerId, now) {
+            const rows = database
+                .prepare("SELECT * FROM engine_runs WHERE state = 'running' AND worker_id != ?")
+                .all(workerId);
+            // A dead process is recovered at the next start; a live one only after its lease runs out.
+            return rows
+                .filter((row) => row.lease_until < now || !ownerIsAlive(row.owner_token, row.owner_path))
+                .map((row) => ({ runId: row.run_id, held: leaseOf(row) }));
+        },
+        async listReady(now) {
+            const rows = database
+                .prepare(`
+          SELECT DISTINCT r.* FROM engine_runs r
+          JOIN engine_waits w ON w.run_id = r.run_id AND w.outcome_json IS NULL
+          WHERE r.state = 'waiting' AND (
+            r.cancel_requested = 1 OR w.deadline <= ? OR EXISTS (
+              SELECT 1 FROM engine_signals s WHERE s.run_id = w.run_id AND s.name = w.name
+            )
+          )
+        `)
+                .all(now);
+            return rows.map((row) => ({ runId: row.run_id, held: leaseOf(row) }));
+        },
+        async listWaiting() {
+            const rows = database
+                .prepare(`SELECT r.run_id, w.deadline FROM engine_runs r
+        JOIN engine_waits w ON w.run_id = r.run_id AND w.outcome_json IS NULL
+        WHERE r.state = 'waiting'`)
+                .all();
+            return rows.map((row) => ({ runId: row.run_id, deadline: row.deadline }));
+        },
+        async requestCancel(runId) {
+            const result = database
+                .prepare("UPDATE engine_runs SET cancel_requested = 1 WHERE run_id = ? AND cancel_requested = 0 AND state IN ('running', 'waiting')")
+                .run(runId);
+            return result.changes === 1;
+        },
+        async getWait(runId, name) {
+            const row = database
+                .prepare("SELECT deadline, outcome_json FROM engine_waits WHERE run_id = ? AND name = ?")
+                .get(runId, name);
+            return row && { deadline: row.deadline, outcomeJson: row.outcome_json };
+        },
+        async openWait(runId, name, deadline) {
+            database
+                .prepare("INSERT OR IGNORE INTO engine_waits (run_id, name, deadline) VALUES (?, ?, ?)")
+                .run(runId, name, deadline);
+        },
+        async resolveWait(runId, name, outcomeJson) {
+            const result = database
+                .prepare("UPDATE engine_waits SET outcome_json = ? WHERE run_id = ? AND name = ? AND outcome_json IS NULL")
+                .run(outcomeJson, runId, name);
+            return result.changes === 1;
+        },
+        async putSignal(runId, name, valueJson) {
+            const result = database
+                .prepare("INSERT OR IGNORE INTO engine_signals (run_id, name, value_json) VALUES (?, ?, ?)")
+                .run(runId, name, valueJson);
+            return result.changes === 1;
+        },
+        async getSignal(runId, name) {
+            const row = database
+                .prepare("SELECT value_json FROM engine_signals WHERE run_id = ? AND name = ?")
+                .get(runId, name);
+            return row?.value_json;
+        },
+        async getStep(runId, index) {
+            const row = database
+                .prepare("SELECT name, result_json FROM engine_steps WHERE run_id = ? AND step_index = ?")
+                .get(runId, index);
+            return row && { name: row.name, resultJson: row.result_json };
+        },
+        async putStep(runId, index, step) {
+            database
+                .prepare("INSERT OR IGNORE INTO engine_steps (run_id, step_index, name, result_json) VALUES (?, ?, ?, ?)")
+                .run(runId, index, step.name, step.resultJson);
+        },
+    };
+}

package/dist/engine/types.d.ts

@@ -0,0 +1,97 @@
+export interface StepOptions {
+    /** Extra attempts inside one step. Defaults to 0: the step runs once unless the process dies inside it. */
+    retries?: number;
+    /** When supplied, only errors accepted by this predicate are retried. */
+    shouldRetry?: (error: unknown) => boolean;
+}
+export interface StepAttempt {
+    /** Zero-based attempt number within this step. */
+    attempt: number;
+    /** Aborts when the run is cancelled, so a long step can stop early. */
+    signal: AbortSignal;
+}
+export interface RunContext {
+    readonly runId: string;
+    /** Aborts when the run is cancelled. */
+    readonly signal: AbortSignal;
+    /**
+     * Runs `fn` and records its result. When the run is replayed, the recorded result is returned
+     * and `fn` does not run. Results are stored as JSON, so `undefined` properties are dropped.
+     * A step runs at least once: a process that dies inside it runs it again on recovery.
+     */
+    step<T>(name: string, fn: (attempt: StepAttempt) => T | Promise<T>, options?: StepOptions): Promise<T>;
+    /**
+     * Holds the run until a signal named `name` arrives or the timeout passes. A signal that arrived
+     * earlier is used at once. Nothing executes while the run waits: the first time there is nothing
+     * to return, this throws RunSuspended. Let it reach the engine, or catch it and return; the engine
+     * replays the workflow from its recorded steps once the wait can end. Use a unique name per wait.
+     */
+    wait<T = unknown>(name: string, options?: WaitOptions): Promise<WaitOutcome<T>>;
+}
+/** How a run ended for now. A waiting run may carry what the workflow returned before it stopped. */
+export type RunOutcome<O> = {
+    state: "completed";
+    output: O;
+} | {
+    state: "waiting";
+    output?: O;
+} | {
+    state: "cancelled";
+    output?: O;
+};
+/** `ignored`: the run had already finished or been cancelled. */
+export type CancelResult = "accepted" | "ignored" | "unknown";
+/** `ignored`: a decision arrived late, twice, or for a finished run. The first one wins. */
+export type SignalResult = "accepted" | "ignored" | "unknown";
+export interface WaitOptions {
+    /** Ends the wait as a timeout this long after it is first reached. No timeout by default. */
+    timeoutMs?: number;
+}
+/** A timeout is its own outcome, so a caller can tell it from a decision. */
+export type WaitOutcome<T> = {
+    kind: "signal";
+    value: T;
+} | {
+    kind: "timeout";
+};
+export type Workflow<I, O> = (ctx: RunContext, input: I) => Promise<O>;
+export interface RunHandle<O> {
+    readonly runId: string;
+    result(): Promise<RunOutcome<O>>;
+}
+export type ExecutionState = "running" | "waiting" | "completed" | "failed" | "cancelled";
+export interface ExecutionStatus {
+    state: ExecutionState;
+}
+export interface RecoveredRun<O = unknown> {
+    runId: string;
+    state: "completed" | "waiting" | "failed" | "cancelled";
+    error?: string;
+    output?: O;
+}
+export interface Engine {
+    /** Registers the run and begins executing it in the background. */
+    start<I, O>(runId: string, workflow: Workflow<I, O>, input: I): Promise<RunHandle<O>>;
+    /** Starts the run and waits until it completes or stops to wait. */
+    run<I, O>(runId: string, workflow: Workflow<I, O>, input: I): Promise<RunOutcome<O>>;
+    /** Continues an unfinished run from its recorded steps, whoever held it. */
+    resume<I, O>(runId: string, workflow: Workflow<I, O>): Promise<RunOutcome<O>>;
+    /**
+     * Delivers a decision to the wait named `name`. The first signal for a name wins; one that comes
+     * after the wait ended changes nothing. A run waiting in this process continues at once; any other
+     * continues when `recover` next runs.
+     */
+    signal(runId: string, name: string, value: unknown): Promise<SignalResult>;
+    /**
+     * Continues every run whose worker died or let its lease run out, and every waiting run whose
+     * wait can now end. `resolve` picks the workflow for a run; return undefined to leave it alone.
+     */
+    recover<I, O>(resolve: (runId: string, input: I) => Workflow<I, O> | undefined): Promise<RecoveredRun<O>[]>;
+    /**
+     * Cancels a run. A step that is running gets its abort signal fired; a waiting run ends at once
+     * where it holds the run, or when `recover` next runs. The workflow sees RunCancelled at its next
+     * step or wait and returns.
+     */
+    cancel(runId: string): Promise<CancelResult>;
+    status(runId: string): Promise<ExecutionStatus | undefined>;
+}

package/dist/engine/types.js

@@ -0,0 +1 @@
+export {};

package/dist/env.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Loads `<root>/.env` into `process.env` without replacing variables that are already set, and
+ * returns the file path, or `undefined` when the project has none. Values are never logged.
+ */
+export declare function loadProjectEnv(root: string): string | undefined;
+/** The `.env` files this process loaded through {@link loadProjectEnv}. */
+export declare function loadedEnvFiles(): string[];

package/dist/env.js

@@ -0,0 +1,19 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+const loaded = new Set();
+/**
+ * Loads `<root>/.env` into `process.env` without replacing variables that are already set, and
+ * returns the file path, or `undefined` when the project has none. Values are never logged.
+ */
+export function loadProjectEnv(root) {
+    const file = join(root, ".env");
+    if (!existsSync(file))
+        return undefined;
+    process.loadEnvFile(file);
+    loaded.add(file);
+    return file;
+}
+/** The `.env` files this process loaded through {@link loadProjectEnv}. */
+export function loadedEnvFiles() {
+    return [...loaded];
+}

package/dist/errors.d.ts

@@ -0,0 +1,18 @@
+import type { PlicoEventPayloads } from "@plico-ai/contracts";
+export declare class RuntimeDiagnosticError extends Error {
+    readonly payload: PlicoEventPayloads["run.error"];
+    constructor(payload: PlicoEventPayloads["run.error"]);
+}
+export declare class RunWaitingError extends Error {
+    readonly output: string | undefined;
+    constructor(output: string | undefined);
+}
+export declare function errorMessage(error: unknown): string;
+export declare function runtimeErrorPayload(error: unknown): PlicoEventPayloads["run.error"];
+export type ModelErrorKind = "rate_limit" | "overloaded" | "auth" | "bad_request" | "context_length" | "aborted" | "network";
+/** Provider adapters normalize errors to these provider-neutral kinds. */
+export declare class ModelProviderError extends Error {
+    readonly kind: ModelErrorKind;
+    readonly retryable: boolean;
+    constructor(kind: ModelErrorKind, message: string);
+}

package/dist/errors.js

@@ -0,0 +1,40 @@
+export class RuntimeDiagnosticError extends Error {
+    payload;
+    constructor(payload) {
+        super(typeof payload.message === "string" ? payload.message : "Runtime diagnostic error");
+        this.payload = payload;
+    }
+}
+export class RunWaitingError extends Error {
+    output;
+    constructor(output) {
+        super("Run waiting for approval");
+        this.output = output;
+    }
+}
+export function errorMessage(error) {
+    if (error instanceof Error && error.message) {
+        return error.message;
+    }
+    return "Unknown runtime error";
+}
+export function runtimeErrorPayload(error) {
+    if (error instanceof RuntimeDiagnosticError) {
+        return error.payload;
+    }
+    return {
+        phase: "runtime",
+        message: errorMessage(error),
+    };
+}
+/** Provider adapters normalize errors to these provider-neutral kinds. */
+export class ModelProviderError extends Error {
+    kind;
+    retryable;
+    constructor(kind, message) {
+        super(message);
+        this.kind = kind;
+        this.name = "ModelProviderError";
+        this.retryable = kind === "rate_limit" || kind === "overloaded" || kind === "network";
+    }
+}

package/dist/evals/discover.d.ts

@@ -0,0 +1,3 @@
+import type { EvalCase } from "../types/index.js";
+export declare function discoverEvalCases(evalsDir: string): Promise<EvalCase[]>;
+export declare function digestEvalFixtures(evalsDir: string): Promise<string>;

package/dist/evals/discover.js

@@ -0,0 +1,49 @@
+import { createHash } from "node:crypto";
+import { readdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+const fixtureSuffix = ".script.json";
+function toEvalCase(file, parsed) {
+    const fixture = (Array.isArray(parsed) ? { script: parsed } : parsed);
+    if (!Array.isArray(fixture?.script)) {
+        throw new Error(`Invalid eval fixture ${file}: expected a step array or a "script" array.`);
+    }
+    const entry = {
+        name: typeof fixture.name === "string" ? fixture.name : file.slice(0, -fixtureSuffix.length),
+        script: fixture.script,
+    };
+    if (typeof fixture.description === "string")
+        entry.description = fixture.description;
+    if (fixture.expect !== undefined)
+        entry.expect = fixture.expect;
+    return entry;
+}
+async function readFixtures(evalsDir) {
+    let entries;
+    try {
+        entries = await readdir(evalsDir);
+    }
+    catch {
+        return [];
+    }
+    const scriptFiles = entries.filter((e) => e.endsWith(fixtureSuffix)).sort();
+    return Promise.all(scriptFiles.map(async (file) => ({ file, raw: await readFile(join(evalsDir, file), "utf8") })));
+}
+function parseFixture(file, raw) {
+    try {
+        return JSON.parse(raw);
+    }
+    catch (error) {
+        throw new Error(`Invalid eval fixture ${file}: ${error.message}`);
+    }
+}
+export async function discoverEvalCases(evalsDir) {
+    const fixtures = await readFixtures(evalsDir);
+    return fixtures.map(({ file, raw }) => toEvalCase(file, parseFixture(file, raw)));
+}
+export async function digestEvalFixtures(evalsDir) {
+    const hash = createHash("sha256");
+    for (const { file, raw } of await readFixtures(evalsDir)) {
+        hash.update(`${file}\0${raw}\0`);
+    }
+    return hash.digest("hex");
+}

package/dist/evals/index.d.ts

@@ -0,0 +1,3 @@
+export { discoverEvalCases } from "./discover.js";
+export { evalCaseFromRun } from "./promote.js";
+export { runEvals } from "./run-evals.js";

package/dist/evals/index.js

@@ -0,0 +1,3 @@
+export { discoverEvalCases } from "./discover.js";
+export { evalCaseFromRun } from "./promote.js";
+export { runEvals } from "./run-evals.js";

package/dist/evals/promote.d.ts

@@ -0,0 +1,7 @@
+import type { PlicoEvent, RunStatus } from "@plico-ai/contracts";
+import type { EvalCase } from "../types/index.js";
+/**
+ * Turns a stored run into an eval case. Each tool call keeps the output it produced as the mocked
+ * `result`, so the case replays without running any handler. Files a tool wrote are not carried over.
+ */
+export declare function evalCaseFromRun(name: string, status: RunStatus, events: PlicoEvent[]): EvalCase;

package/dist/evals/promote.js

@@ -0,0 +1,32 @@
+/**
+ * Turns a stored run into an eval case. Each tool call keeps the output it produced as the mocked
+ * `result`, so the case replays without running any handler. Files a tool wrote are not carried over.
+ */
+export function evalCaseFromRun(name, status, events) {
+    const script = [];
+    let output;
+    let pending;
+    for (const event of events) {
+        if (event.type === "assistant.output") {
+            script.push({ type: "assistant.output", content: event.payload.content });
+            output = event.payload.content;
+        }
+        else if (event.type === "tool.call") {
+            pending = {
+                type: "tool.call",
+                toolName: event.payload.toolName,
+                arguments: event.payload.arguments,
+            };
+            script.push(pending);
+        }
+        else if (event.type === "tool.result" && pending) {
+            pending.result = event.payload.result;
+            pending = undefined;
+        }
+    }
+    return {
+        name,
+        script,
+        expect: { status, ...(output === undefined ? {} : { outputContains: output }) },
+    };
+}

package/dist/evals/run-evals.d.ts

@@ -0,0 +1,2 @@
+import type { EvalSummary, RunEvalsOptions } from "../types/index.js";
+export declare function runEvals(projectRoot: string, options?: RunEvalsOptions): Promise<EvalSummary>;

package/dist/evals/run-evals.js

@@ -0,0 +1,69 @@
+import { getProjectPaths } from "@plico-ai/core";
+import { runProject } from "../run/index.js";
+import { digestEvalFixtures, discoverEvalCases } from "./discover.js";
+export async function runEvals(projectRoot, options = {}) {
+    const evalsDir = getProjectPaths(projectRoot).directories.evals;
+    const cases = await discoverEvalCases(evalsDir);
+    const fixturesDigest = await digestEvalFixtures(evalsDir);
+    const results = [];
+    for (const evalCase of cases) {
+        const runId = `eval-${evalCase.name.replace(/[^a-z0-9_-]/gi, "_")}-${crypto.randomUUID().slice(0, 8)}`;
+        const runOpts = {
+            script: evalCase.script,
+            mockTools: true,
+            runId,
+        };
+        if (options.eventStore !== undefined)
+            runOpts.eventStore = options.eventStore;
+        const result = await runProject(projectRoot, runOpts);
+        const diagnostics = [];
+        let passed = true;
+        if (evalCase.expect?.status && result.status !== evalCase.expect.status) {
+            passed = false;
+            diagnostics.push(`Expected status "${evalCase.expect.status}" but got "${result.status}"`);
+        }
+        if (!evalCase.expect?.status && result.status !== "completed") {
+            passed = false;
+            diagnostics.push(`Run ended with status "${result.status}"`);
+        }
+        if (evalCase.expect?.outputContains) {
+            const output = result.output ?? "";
+            if (!output.includes(evalCase.expect.outputContains)) {
+                passed = false;
+                diagnostics.push(`Expected output to contain "${evalCase.expect.outputContains}" but got "${output}"`);
+            }
+        }
+        for (const event of result.events) {
+            if (event.type === "run.error" && typeof event.payload.message === "string")
+                diagnostics.push(event.payload.message);
+        }
+        if (result.errors?.length) {
+            for (const err of result.errors) {
+                diagnostics.push(`${err.path}: ${err.message}`);
+            }
+        }
+        const entry = {
+            name: evalCase.name,
+            status: passed ? "pass" : "fail",
+            runId: result.runId,
+            runStatus: result.status,
+        };
+        if (evalCase.description !== undefined)
+            entry.description = evalCase.description;
+        if (result.output !== undefined)
+            entry.output = result.output;
+        if (diagnostics.length > 0)
+            entry.diagnostics = diagnostics;
+        results.push(entry);
+    }
+    const passed = results.filter((r) => r.status === "pass").length;
+    const failed = results.filter((r) => r.status === "fail").length;
+    const summary = { total: results.length, passed, failed, results };
+    await options.eventStore?.persistEvalReport({
+        schemaVersion: 1,
+        ...summary,
+        evaluatedAt: new Date().toISOString(),
+        fixturesDigest,
+    });
+    return summary;
+}