@wix/pathgrade 0.37.0 → 1.0.1

package/dist/runners/vitest-adapter.js

@@ -0,0 +1,108 @@
+import { discoverPathgradeEvalFiles } from '../evals/discovery.js';
+import { buildNormalizedRunSnapshotFromReportGroups } from './model-builders.js';
+import { buildDiagnosticsReport } from '../sdk/diagnostics.js';
+export function createVitestAdapter(options = {}) {
+    return {
+        name: 'vitest',
+        async discover(input) {
+            const include = input.include ?? ['**/*.eval.ts'];
+            const exclude = input.exclude ?? [];
+            const files = discoverPathgradeEvalFiles({
+                cwd: input.cwd,
+                include,
+                exclude,
+            });
+            return {
+                units: files.map(file => ({
+                    id: file,
+                    displayName: file,
+                    sourceRef: file,
+                    native: { file },
+                })),
+            };
+        },
+        async invoke(input) {
+            return {
+                adapterName: this.name,
+                status: input.signal?.aborted ? 'cancelled' : 'completed',
+                exitCode: input.signal?.aborted ? 1 : 0,
+                native: {
+                    testModules: options.testModules ?? [],
+                },
+            };
+        },
+        async collectNormalizedRunSnapshot(run) {
+            return buildNormalizedRunSnapshotFromReportGroups(run, collectVitestReportGroups(readVitestRunNative(run).testModules));
+        },
+    };
+}
+export function collectVitestReportGroups(testModules) {
+    const groupMap = new Map();
+    for (const mod of testModules) {
+        for (const testCase of mod.children.allTests()) {
+            const groupName = getGroupName(testCase);
+            const entry = toReportCaseInput(testCase, groupName);
+            if (!groupMap.has(groupName)) {
+                groupMap.set(groupName, []);
+            }
+            groupMap.get(groupName).push(entry);
+        }
+    }
+    return Array.from(groupMap.entries()).map(([groupName, cases]) => ({
+        groupName,
+        cases,
+    }));
+}
+function readVitestRunNative(run) {
+    if (isVitestRunNative(run.native))
+        return run.native;
+    return { testModules: [] };
+}
+function isVitestRunNative(native) {
+    return typeof native === 'object'
+        && native !== null
+        && Array.isArray(native.testModules);
+}
+function getGroupName(testCase) {
+    const modulePath = testCase.module.relativeModuleId;
+    if (testCase.parent.type === 'suite') {
+        return `${modulePath} > ${testCase.parent.fullName}`;
+    }
+    return modulePath;
+}
+function toReportCaseInput(testCase, groupName) {
+    const pathgradeMeta = testCase.meta().pathgrade;
+    const result = testCase.result();
+    const diagnostics = testCase.diagnostic();
+    const normalized = normalizeState(result.state);
+    const filePath = testCase.module.relativeModuleId;
+    const runnerCaseId = typeof testCase.id === 'string' ? testCase.id : undefined;
+    return {
+        ...(runnerCaseId ? { caseId: runnerCaseId, runnerCaseId } : {}),
+        name: testCase.name,
+        state: normalized.state,
+        runnerDurationMs: diagnostics?.duration ?? 0,
+        sourceRef: filePath,
+        filePath,
+        groupName,
+        evaluations: pathgradeMeta?.map(entry => ({
+            score: entry.score,
+            trial: entry.trial,
+            diagnostics: entry.diagnostics,
+        })),
+        diagnostics: normalized.diagnostics,
+    };
+}
+function normalizeState(state) {
+    if (state === 'passed' || state === 'failed' || state === 'skipped' || state === 'pending') {
+        return { state };
+    }
+    return {
+        state: 'failed',
+        diagnostics: buildDiagnosticsReport({
+            completionReason: state,
+            score: 0,
+            log: [],
+        }),
+    };
+}

package/dist/runners/vitest-invocation.d.ts

@@ -0,0 +1,10 @@
+import type { RunnerInvocationAdapter } from './invocation.js';
+export interface SpawnVitestRequest {
+    argv: string[];
+    env: NodeJS.ProcessEnv;
+    cwd: string;
+}
+export type SpawnVitest = (req: SpawnVitestRequest) => Promise<number> | number;
+export declare function createVitestInvocationAdapter(input?: {
+    spawnVitest?: SpawnVitest;
+}): RunnerInvocationAdapter;

package/dist/runners/vitest-invocation.js

@@ -0,0 +1,44 @@
+export function createVitestInvocationAdapter(input = {}) {
+    const spawnVitest = input.spawnVitest ?? defaultSpawnVitest;
+    return {
+        name: 'vitest',
+        async run(runInput) {
+            if (runInput.selectedFiles && hasPassWithNoTests(runInput.runnerArgs)) {
+                process.stderr.write('pathgrade run: --passWithNoTests cannot be used with pathgrade run --changed. ' +
+                    'The command already exits 0 when no evals are selected; if selected evals resolve to no Vitest files, CI must fail.\n');
+                return 1;
+            }
+            const argv = [
+                'run',
+                ...(runInput.selectedFiles ?? []),
+                ...runInput.runnerArgs,
+            ];
+            return await spawnVitest({
+                argv,
+                env: runInput.env,
+                cwd: runInput.cwd,
+            });
+        },
+    };
+}
+function hasPassWithNoTests(args) {
+    return args.some(arg => {
+        if (arg === '--passWithNoTests')
+            return true;
+        if (!arg.startsWith('--passWithNoTests='))
+            return false;
+        return arg.slice('--passWithNoTests='.length).toLowerCase() !== 'false';
+    });
+}
+async function defaultSpawnVitest(req) {
+    const { spawn } = await import('child_process');
+    return await new Promise(resolve => {
+        const child = spawn('npx', ['vitest', ...req.argv], {
+            stdio: 'inherit',
+            env: req.env,
+            cwd: req.cwd,
+            shell: true,
+        });
+        child.on('close', code => resolve(code ?? 0));
+    });
+}

package/dist/runners/vitest-lifecycle.d.ts

@@ -0,0 +1,43 @@
+import type { Agent, PathgradeTestMeta } from '../sdk/types.js';
+import { type ResultObserverHandle } from '../sdk/result-capture.js';
+import { type CaseContextProviderHandle } from '../sdk/case-context.js';
+import { type LifecycleAgentOwner } from '../sdk/lifecycle.js';
+import type { AdapterLifecycleHooks, EvalResultEvent } from './adapter.js';
+export type VitestAfterEachFn = (fn: (ctx: {
+    task: {
+        id: string;
+        meta: Record<string, unknown>;
+    };
+}) => Promise<void>) => void;
+export type VitestAfterAllFn = (fn: () => Promise<void>) => void;
+export type VitestAroundEachFn = (fn: (runTest: () => Promise<void>, ctx: {
+    task: {
+        id: string;
+        name: string;
+        meta: Record<string, unknown>;
+        suite?: unknown;
+    };
+}) => Promise<void>) => void;
+export interface VitestLifecycleInstallInput {
+    afterEach: VitestAfterEachFn;
+    afterAll?: VitestAfterAllFn;
+    aroundEach?: VitestAroundEachFn;
+    lifecycle?: AdapterLifecycleHooks;
+    subscribeToResults?: (callback: (event: EvalResultEvent) => void) => ResultObserverHandle;
+    installFileContextProvider?: () => CaseContextProviderHandle;
+}
+export interface VitestLifecycleInstallHandle {
+    restore(): void;
+}
+export declare function createVitestLifecycleHooks(): AdapterLifecycleHooks;
+export declare function installVitestLifecycle(input: VitestLifecycleInstallInput): VitestLifecycleInstallHandle;
+export declare function currentVitestAgentOwner(resultCaptureActive: boolean): LifecycleAgentOwner | null;
+export declare function trackVitestAgent(agent: Agent, resultCaptureActive: boolean): void;
+export declare function untrackVitestAgent(agent: Agent): void;
+export declare function releaseVitestAgent(agent: Agent): void;
+export declare function resetVitestLifecycle(handle: VitestLifecycleInstallHandle | null): void;
+declare module 'vitest' {
+    interface TaskMeta {
+        pathgrade?: PathgradeTestMeta[];
+    }
+}

package/dist/runners/vitest-lifecycle.js

@@ -0,0 +1,150 @@
+import path from 'node:path';
+import { subscribeToEvalResults } from '../sdk/result-capture.js';
+import { getCurrentCaseContext, installCaseContextProvider, runWithCaseContext, } from '../sdk/case-context.js';
+import { lifecycleCore } from '../sdk/lifecycle.js';
+export function createVitestLifecycleHooks() {
+    return {
+        onResult: event => recordVitestResult(event.result, event.agent),
+        withCaseContext: (context, run) => runWithCaseContext(toCaseContext(context), run),
+        flushCase: caseId => lifecycleCore.flushCase({ caseId }),
+        cleanupRun: () => lifecycleCore.cleanupAll(),
+    };
+}
+export function installVitestLifecycle(input) {
+    const lifecycle = input.lifecycle ?? createVitestLifecycleHooks();
+    const resultCaptureHandle = (input.subscribeToResults ?? defaultSubscribeToResults)(event => lifecycle.onResult(event));
+    const fileContextHandle = (input.installFileContextProvider ?? defaultInstallFileContextProvider)();
+    let fileContextRestored = false;
+    let resultsUnsubscribed = false;
+    input.aroundEach?.(async (runTest, { task }) => lifecycle.withCaseContext(caseContextForTask(task), runTest));
+    input.afterEach(async ({ task }) => {
+        const results = await lifecycle.flushCase(task.id);
+        if (results.length > 0) {
+            task.meta.pathgrade = results;
+        }
+    });
+    input.afterAll?.(async () => {
+        await lifecycle.cleanupRun();
+        restoreFileContext();
+    });
+    return {
+        restore() {
+            restoreFileContext();
+            if (!resultsUnsubscribed) {
+                resultsUnsubscribed = true;
+                resultCaptureHandle.unsubscribe();
+            }
+        },
+    };
+    function restoreFileContext() {
+        if (fileContextRestored)
+            return;
+        fileContextRestored = true;
+        fileContextHandle.restore();
+    }
+}
+export function currentVitestAgentOwner(resultCaptureActive) {
+    const current = getCurrentCaseContext();
+    if (current.status === 'active') {
+        return current.context.scope === 'runner-case'
+            ? { type: 'runner-case', caseId: current.context.caseId }
+            : { type: 'runner-suite-shared', caseId: current.context.caseId };
+    }
+    const taskId = currentTaskId();
+    if (resultCaptureActive)
+        return null;
+    return taskId ? { type: 'runner-case', caseId: taskId } : { type: 'manual' };
+}
+export function trackVitestAgent(agent, resultCaptureActive) {
+    lifecycleCore.registerAgent(agent, currentVitestAgentOwner(resultCaptureActive));
+}
+export function untrackVitestAgent(agent) {
+    lifecycleCore.untrackAgent(agent);
+}
+export function releaseVitestAgent(agent) {
+    lifecycleCore.releaseAgent(agent);
+}
+export function resetVitestLifecycle(handle) {
+    lifecycleCore.reset();
+    handle?.restore();
+}
+function defaultSubscribeToResults(callback) {
+    return subscribeToEvalResults(({ result, agent }) => callback({ result, agent }), { owner: 'adapter', key: 'vitest-lifecycle' });
+}
+function defaultInstallFileContextProvider() {
+    return installCaseContextProvider(currentFileContext);
+}
+function toCaseContext(context) {
+    return {
+        caseId: context.caseId,
+        caseName: context.caseName,
+        filePath: context.filePath ?? context.sourceRef ?? '',
+        scope: context.scope,
+    };
+}
+function currentTaskId() {
+    try {
+        return globalThis.__vitest_worker__?.current?.id ?? '';
+    }
+    catch {
+        return '';
+    }
+}
+function filePathForTask(task) {
+    let current = task.suite;
+    while (current) {
+        if (typeof current.filepath === 'string')
+            return current.filepath;
+        current = current.suite;
+    }
+    try {
+        return globalThis.__vitest_worker__?.filepath ?? '';
+    }
+    catch {
+        return '';
+    }
+}
+function caseContextForTask(task) {
+    const filePath = filePathForTask(task);
+    return {
+        caseId: task.id,
+        caseName: task.name,
+        filePath,
+        sourceRef: filePath,
+        runnerNativeId: task.id,
+        scope: 'runner-case',
+    };
+}
+function currentFilePath() {
+    try {
+        return globalThis.__vitest_worker__?.filepath ?? '';
+    }
+    catch {
+        return '';
+    }
+}
+function currentFileContext() {
+    const filePath = currentFilePath();
+    if (!filePath)
+        return null;
+    return {
+        caseId: `file:${filePath}`,
+        caseName: path.basename(filePath),
+        filePath,
+        scope: 'runner-suite-shared',
+    };
+}
+function recordVitestResult(result, agent) {
+    const current = getCurrentCaseContext();
+    const taskId = currentTaskId();
+    const owner = lifecycleCore.getAgentOwner(agent);
+    if (owner?.type !== 'runner-case' && (current.status !== 'active' || current.context.scope !== 'runner-case')) {
+        if (!taskId) {
+            lifecycleCore.recordResult(result, agent);
+            return;
+        }
+        lifecycleCore.recordResult(result, agent, { type: 'runner-case', caseId: taskId });
+        return;
+    }
+    lifecycleCore.recordResult(result, agent);
+}

package/dist/sdk/agent.js

@@ -1,6 +1,6 @@
 import { prepareWorkspace } from '../providers/workspace.js';
 import { resolveAgentName, resolveCodexTransport } from './agent-resolution.js';
-import { lifecycle } from '../plugin/lifecycle.js';
+import { lifecycleCore } from './lifecycle.js';
 import { ChatSessionImpl } from './chat.js';
 import { runConversation } from './converse.js';
 import { createPersona } from './persona.js';
@@ -10,6 +10,7 @@ import { createAgentLLM } from '../utils/llm.js';
 import { buildRunSnapshot } from './snapshots.js';
 import { buildModelAgentResultLogEntry } from './agent-result-log.js';
 import { getVisibleAssistantMessage } from './visible-turn.js';
+import { getCurrentCaseContext } from './case-context.js';
 import { createVerboseEmitter } from '../reporters/verbose-emitter.js';
 import fs from 'fs-extra';
 import * as path from 'path';
@@ -293,9 +294,9 @@ class AgentImpl {
         if (this.disposed)
             return;
         this.disposed = true;
-        // Do NOT call lifecycle.untrackAgent here — flush() in afterEach needs
-        // the agent to still be in pendingAgents so it can collect results
-        // (token usage, scorers, diagnostics) into task.meta.pathgrade.
+        // Runner-owned agents stay tracked until flush consumes metadata.
+        // Manual agents have no runner flush, so dispose releases them.
+        lifecycleCore.releaseAgent(this);
         if (this.debugOpt) {
             const dest = typeof this.debugOpt === 'string'
                 ? this.debugOpt
@@ -329,16 +330,14 @@ function slugify(s) {
         .replace(/^-|-$/g, '')
         .toLowerCase();
 }
-function resolveTestContext() {
-    try {
-        const worker = globalThis.__vitest_worker__;
-        const name = worker?.current?.name ? slugify(worker.current.name) : '';
-        const dir = worker?.filepath ? path.dirname(worker.filepath) : '';
-        return { name, dir };
-    }
-    catch {
+function resolveCaseDebugContext() {
+    const current = getCurrentCaseContext();
+    if (current.status !== 'active')
         return { name: '', dir: '' };
-    }
+    return {
+        name: current.context.caseName ? slugify(current.context.caseName) : '',
+        dir: current.context.filePath ? path.dirname(current.context.filePath) : '',
+    };
 }
 export async function createAgent(opts) {
     const agentName = resolveAgentName(opts, process.env);
@@ -346,8 +345,8 @@ export async function createAgent(opts) {
         ? resolveCodexTransport(opts, process.env)
         : undefined;
     const timeoutSetting = opts.timeout ?? 300;
-    // Capture test context now, while vitest state is available
-    const testCtx = opts.debug ? resolveTestContext() : { name: '', dir: '' };
+    // Capture runner context now; adapters own installation and restoration.
+    const testCtx = opts.debug ? resolveCaseDebugContext() : { name: '', dir: '' };
     const { timeout: _, mcpMock, mcpConfigFile, agent: __, debug: ___, model: ____, transport: _____, mcpSafety: ______, ...rest } = opts;
     const workspace = await prepareWorkspace({
         ...rest,
@@ -367,6 +366,6 @@ export async function createAgent(opts) {
         testName: testCtx.name || undefined,
     });
     const agent = new AgentImpl(workspace, agentName, llm, timeoutSetting, opts.conversationWindow, opts.model, opts.debug, debugName, debugBaseDir, verbose, transport, opts.mcpSafety);
-    lifecycle.trackAgent(agent);
+    lifecycleCore.registerAgent(agent);
     return agent;
 }

package/dist/sdk/case-context.d.ts

@@ -0,0 +1,20 @@
+export type CaseContextScope = 'runner-case' | 'runner-suite-shared';
+export interface CaseContext {
+    caseId: string;
+    caseName: string;
+    filePath: string;
+    scope: CaseContextScope;
+}
+export type CurrentCaseContext = {
+    status: 'empty';
+} | {
+    status: 'active';
+    context: CaseContext;
+};
+export interface CaseContextProviderHandle {
+    restore(): void;
+}
+export type CaseContextProvider = () => CaseContext | null | undefined;
+export declare function getCurrentCaseContext(): CurrentCaseContext;
+export declare function runWithCaseContext<T>(context: CaseContext, run: () => T): T;
+export declare function installCaseContextProvider(provider: CaseContextProvider): CaseContextProviderHandle;

package/dist/sdk/case-context.js

@@ -0,0 +1,31 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+const caseContextStorage = new AsyncLocalStorage();
+const providers = [];
+export function getCurrentCaseContext() {
+    const context = caseContextStorage.getStore();
+    if (context)
+        return { status: 'active', context };
+    for (const provider of providers.toReversed()) {
+        const provided = provider();
+        if (provided)
+            return { status: 'active', context: provided };
+    }
+    return { status: 'empty' };
+}
+export function runWithCaseContext(context, run) {
+    return caseContextStorage.run(context, run);
+}
+export function installCaseContextProvider(provider) {
+    providers.push(provider);
+    let restored = false;
+    return {
+        restore() {
+            if (restored)
+                return;
+            restored = true;
+            const index = providers.lastIndexOf(provider);
+            if (index >= 0)
+                providers.splice(index, 1);
+        },
+    };
+}

package/dist/sdk/diagnostics.d.ts

@@ -0,0 +1,40 @@
+import type { LogEntry } from '../types.js';
+import type { ReactionFiredEntry, TurnDetail } from './types.js';
+import { type RuntimePolicyAuditEntry } from './runtime-policy.js';
+export interface DiagnosticsScorer {
+    name: string;
+    type: string;
+    score: number;
+    weight: number;
+    details?: string;
+    status?: 'ok' | 'error' | 'skipped';
+}
+export interface DiagnosticsTurnDetail extends TurnDetail {
+    apiRetries: number;
+}
+export type DiagnosticsRuntimePolicyEntry = RuntimePolicyAuditEntry;
+export interface DiagnosticsReport {
+    turns: number;
+    totalDurationMs: number;
+    completionReason?: string;
+    completionDetail?: string;
+    score?: number;
+    turnDetails: DiagnosticsTurnDetail[];
+    reactionsFired: ReactionFiredEntry[];
+    runtimePoliciesApplied: DiagnosticsRuntimePolicyEntry[];
+    recommendedTimeoutMs: number;
+    warnings: string[];
+    scorers: Array<DiagnosticsScorer & {
+        status: 'ok' | 'error' | 'skipped';
+    }>;
+}
+export interface BuildDiagnosticsReportInput {
+    completionReason?: string;
+    completionDetail?: string;
+    score?: number;
+    turnDetails?: TurnDetail[];
+    reactionsFired?: ReactionFiredEntry[];
+    scorers?: DiagnosticsScorer[];
+    log: LogEntry[];
+}
+export declare function buildDiagnosticsReport(input: BuildDiagnosticsReportInput): DiagnosticsReport;

package/dist/sdk/diagnostics.js

@@ -0,0 +1,80 @@
+import { extractRuntimePolicyAuditEntries } from './runtime-policy.js';
+export function buildDiagnosticsReport(input) {
+    const baseTurnDetails = (input.turnDetails && input.turnDetails.length > 0)
+        ? [...input.turnDetails]
+        : deriveTurnDetailsFromLog(input.log);
+    const retriesByTurn = countRetriesByTurn(input.log);
+    const turnDetails = baseTurnDetails
+        .sort((a, b) => a.turn - b.turn)
+        .map((detail) => ({
+        ...detail,
+        apiRetries: retriesByTurn.get(detail.turn) ?? 0,
+    }));
+    const warnings = turnDetails
+        .filter((detail) => detail.outputLines > 500)
+        .map((detail) => `Turn ${detail.turn}: output exceeded 500 lines (${detail.outputLines} lines)`);
+    return {
+        turns: turnDetails.length,
+        totalDurationMs: turnDetails.reduce((sum, detail) => sum + detail.durationMs, 0),
+        completionReason: input.completionReason,
+        completionDetail: input.completionDetail,
+        score: input.score,
+        turnDetails,
+        reactionsFired: input.reactionsFired ?? [],
+        runtimePoliciesApplied: extractRuntimePolicyAuditEntries(input.log),
+        recommendedTimeoutMs: recommendTimeoutMs(turnDetails),
+        warnings,
+        scorers: (input.scorers ?? []).map((scorer) => ({
+            ...scorer,
+            status: scorer.status ?? 'ok',
+        })),
+    };
+}
+function deriveTurnDetailsFromLog(log) {
+    return log
+        .filter((entry) => entry.type === 'agent_result'
+        && entry.synthetic_blocked_prompt !== true
+        && typeof entry.turn_number === 'number'
+        && typeof entry.duration_ms === 'number'
+        && typeof entry.output_lines === 'number'
+        && typeof entry.output_chars === 'number')
+        .map((entry) => ({
+        turn: entry.turn_number,
+        durationMs: entry.duration_ms,
+        outputLines: entry.output_lines,
+        outputChars: entry.output_chars,
+    }));
+}
+function countRetriesByTurn(log) {
+    const retriesByTurn = new Map();
+    let pendingRetries = 0;
+    for (const entry of log) {
+        if (isRetryEntry(entry)) {
+            pendingRetries++;
+            continue;
+        }
+        if (entry.type === 'agent_result' && typeof entry.turn_number === 'number') {
+            if (pendingRetries > 0) {
+                retriesByTurn.set(entry.turn_number, (retriesByTurn.get(entry.turn_number) ?? 0) + pendingRetries);
+                pendingRetries = 0;
+            }
+            if (!retriesByTurn.has(entry.turn_number)) {
+                retriesByTurn.set(entry.turn_number, 0);
+            }
+        }
+    }
+    return retriesByTurn;
+}
+function isRetryEntry(entry) {
+    if (entry.type !== 'agent_result')
+        return false;
+    const text = `${entry.assistant_message ?? ''} ${entry.output ?? ''}`.toLowerCase();
+    return text.includes('retry') || text.includes('api_retry');
+}
+function recommendTimeoutMs(turnDetails) {
+    if (turnDetails.length <= 1) {
+        return (turnDetails[0]?.durationMs ?? 0) + 30_000;
+    }
+    const totalDurationMs = turnDetails.reduce((sum, detail) => sum + detail.durationMs, 0);
+    return Math.max(totalDurationMs + 200_000, 30_000);
+}

package/dist/sdk/eval-runtime.d.ts

@@ -3,6 +3,10 @@ import type { Agent, RecordedEvalResult } from './types.js';
 export type { LLMPort } from '../utils/llm-types.js';
 export interface EvalRuntime {
     llm: LLMPort;
+    /**
+     * Legacy compatibility seam. `setRuntime({ onResult })` is bridged into
+     * user-owned result observers; live capture is owned by result-capture.
+     */
     onResult: (result: RecordedEvalResult, agent: Agent) => void;
 }
 export declare function getRuntime(): EvalRuntime;

package/dist/sdk/eval-runtime.js

@@ -1,4 +1,5 @@
 import { callLLM } from '../utils/llm.js';
+import { resetUserResultObservers, subscribeToEvalResults, } from './result-capture.js';
 function makeDefaultRuntime() {
     return {
         llm: { call: (prompt, opts) => callLLM(prompt, opts) },
@@ -6,12 +7,21 @@ function makeDefaultRuntime() {
     };
 }
 let current = makeDefaultRuntime();
+let legacyRuntimeResultHandle = null;
 export function getRuntime() {
     return current;
 }
 export function setRuntime(partial) {
-    current = { ...current, ...partial };
+    const { onResult, ...runtimeConfig } = partial;
+    current = { ...current, ...runtimeConfig };
+    if (onResult) {
+        legacyRuntimeResultHandle?.unsubscribe();
+        legacyRuntimeResultHandle = subscribeToEvalResults(({ result, agent }) => onResult(result, agent), { owner: 'user', key: 'legacy-runtime-onResult' });
+    }
 }
 export function resetRuntime() {
     current = makeDefaultRuntime();
+    legacyRuntimeResultHandle?.unsubscribe();
+    legacyRuntimeResultHandle = null;
+    resetUserResultObservers();
 }

package/dist/sdk/evaluate.js

@@ -1,5 +1,6 @@
 import { extractSkillsFromLog } from '../tool-events.js';
 import { getRuntime } from './eval-runtime.js';
+import { emitEvalResult } from './result-capture.js';
 import { runJudgePipeline } from './judge-pipeline.js';
 import { runScorer } from './run-scorer.js';
 import { createLLMClient } from '../utils/llm.js';
@@ -76,7 +77,7 @@ function makeEvaluateAgent() {
             tokenUsage: deltaTokenUsage,
             trial: buildTrialResult(agent.log, { ...evalResult, tokenUsage: deltaTokenUsage }, conversationTokens, conversationCost),
         };
-        getRuntime().onResult(recordedResult, agent);
+        emitEvalResult({ result: recordedResult, agent });
         maybeThrowOnScorerErrors(recordedResult, opts?.onScorerError ?? 'skip');
         return recordedResult;
     };