@wix/pathgrade 1.0.27 → 1.0.29

package/dist/runners/repeated-attempts.js

@@ -0,0 +1,149 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { validateNormalizedRunSnapshot } from './model-validation.js';
+export class RepeatAggregationError extends Error {
+    reason;
+    constructor(reason, message) {
+        super(message);
+        this.reason = reason;
+        this.name = 'RepeatAggregationError';
+    }
+}
+export function mergeRepeatedRunSnapshots(snapshots) {
+    if (snapshots.length === 0) {
+        throw new RepeatAggregationError('incomplete_attempts', 'No repeated-run snapshots were provided.');
+    }
+    const indexed = snapshots.map((snapshot, index) => validateAndIndex(snapshot, index));
+    const expectedKeys = [...indexed[0].keys()];
+    const expectedSet = new Set(expectedKeys);
+    for (let index = 1; index < indexed.length; index++) {
+        const actualKeys = [...indexed[index].keys()];
+        if (actualKeys.length !== expectedKeys.length || actualKeys.some(key => !expectedSet.has(key))) {
+            throw new RepeatAggregationError('case_identity_mismatch', `Attempt ${index + 1} case identities do not exactly match attempt 1.`);
+        }
+    }
+    const runId = `pathgrade:repeated:${randomUUID()}`;
+    const first = snapshots[0];
+    const unitIds = new Map(first.model.units.map((unit, index) => [unit.id, `${runId}:unit-${index + 1}`]));
+    const units = first.model.units.map(unit => ({
+        ...unit,
+        id: unitIds.get(unit.id),
+        runId,
+    }));
+    const cases = expectedKeys.map((repeatKey, caseIndex) => {
+        const sourceCases = indexed.map(casesByKey => casesByKey.get(repeatKey));
+        const firstCase = sourceCases[0];
+        const caseId = `${runId}:case-${caseIndex + 1}-${shortHash(repeatKey)}`;
+        const attempts = sourceCases.map((sourceCase, attemptIndex) => (rewriteAttempt(sourceCase, caseId, attemptIndex + 1)));
+        const scoringPolicy = sourceCases.every(sourceCase => sourceCase.scoringPolicy.kind === 'non-scoring')
+            ? firstCase.scoringPolicy
+            : { kind: 'from-evaluations' };
+        return {
+            ...firstCase,
+            id: caseId,
+            repeatKey,
+            runId,
+            ...(firstCase.unitId ? { unitId: unitIds.get(firstCase.unitId) } : {}),
+            state: aggregateCaseState(sourceCases.map(runCase => runCase.state)),
+            scoringPolicy,
+            attempts,
+        };
+    });
+    return {
+        version: 1,
+        completeness: 'final',
+        model: {
+            run: {
+                ...first.model.run,
+                id: runId,
+                status: aggregateRunStatus(snapshots.map(snapshot => snapshot.model.run.status)),
+                attemptsRequested: snapshots.length,
+                attemptsCompleted: snapshots.length,
+            },
+            units,
+            cases,
+        },
+    };
+}
+function validateAndIndex(snapshot, attemptIndex) {
+    const validation = validateNormalizedRunSnapshot(snapshot, { completeness: 'final' });
+    if (!validation.ok) {
+        throw new RepeatAggregationError('incomplete_attempts', `Attempt ${attemptIndex + 1} snapshot is invalid: ${validation.errors.map(error => `${error.path}: ${error.message}`).join('; ')}`);
+    }
+    const casesByKey = new Map();
+    for (const runCase of snapshot.model.cases) {
+        if (!runCase.repeatKey?.trim()) {
+            throw new RepeatAggregationError('case_identity_mismatch', `Attempt ${attemptIndex + 1} case "${runCase.name}" has no semantic repeat key.`);
+        }
+        if (casesByKey.has(runCase.repeatKey)) {
+            throw new RepeatAggregationError('case_identity_mismatch', `Attempt ${attemptIndex + 1} contains duplicate semantic repeat key "${runCase.repeatKey}".`);
+        }
+        if (runCase.attempts.length !== 1) {
+            throw new RepeatAggregationError('incomplete_attempts', `Attempt ${attemptIndex + 1} case "${runCase.name}" must contain exactly one runner attempt.`);
+        }
+        casesByKey.set(runCase.repeatKey, runCase);
+    }
+    return casesByKey;
+}
+function rewriteAttempt(sourceCase, caseId, attemptIndex) {
+    const source = sourceCase.attempts[0];
+    const attemptId = `${caseId}:attempt-${attemptIndex}`;
+    const evaluations = (source.evaluations ?? []).map((evaluation, index) => ({
+        ...evaluation,
+        id: `${attemptId}:evaluation-${index + 1}`,
+        attemptId,
+    }));
+    if (sourceCase.scoringPolicy.kind === 'score') {
+        if (evaluations.length === 0) {
+            evaluations.push({
+                id: `${attemptId}:evaluation-1`,
+                attemptId,
+                score: sourceCase.scoringPolicy.score,
+            });
+        }
+        else {
+            const terminal = evaluations[evaluations.length - 1];
+            const canonical = {
+                ...terminal,
+                score: sourceCase.scoringPolicy.score,
+                ...(terminal.trial ? {
+                    trial: { ...terminal.trial, reward: sourceCase.scoringPolicy.score },
+                } : {}),
+            };
+            if (canonical.resultKind === 'synthetic_no_evaluation')
+                delete canonical.resultKind;
+            evaluations[evaluations.length - 1] = canonical;
+        }
+    }
+    return {
+        ...source,
+        id: attemptId,
+        caseId,
+        ...(evaluations.length > 0 ? { evaluations } : {}),
+        ...(source.assertions ? {
+            assertions: source.assertions.map((assertion, index) => ({
+                ...assertion,
+                id: `${attemptId}:assertion-${index + 1}`,
+                attemptId,
+            })),
+        } : {}),
+    };
+}
+function aggregateCaseState(states) {
+    if (states.some(state => state === 'failed'))
+        return 'failed';
+    if (states.some(state => state === 'passed'))
+        return 'passed';
+    if (states.some(state => state === 'pending'))
+        return 'pending';
+    return 'skipped';
+}
+function aggregateRunStatus(statuses) {
+    for (const status of ['cancelled', 'timed_out', 'parked', 'failed']) {
+        if (statuses.includes(status))
+            return status;
+    }
+    return 'completed';
+}
+function shortHash(value) {
+    return createHash('sha256').update(value).digest('hex').slice(0, 12);
+}

package/dist/runners/repeated-invocation.d.ts

@@ -0,0 +1,7 @@
+import type { ResolvedPathgradeConfig } from '../config/pathgrade.js';
+import type { RunnerInvocationAdapter } from './invocation.js';
+export declare function withRepeatedAttempts(input: {
+    adapter: RunnerInvocationAdapter;
+    config: ResolvedPathgradeConfig;
+    openBrowser?: () => void | Promise<void>;
+}): RunnerInvocationAdapter;

package/dist/runners/repeated-invocation.js

@@ -0,0 +1,129 @@
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { execFileSync } from 'node:child_process';
+import fs from 'fs-extra';
+import { readSidecarForInvocation } from '../affected/sidecar.js';
+import { getPathgradeDir } from '../reporters/results-path.js';
+import { printReportSummary } from '../reporters/report-summary.js';
+import { buildPathgradeReport } from '../reporting/core.js';
+import { writePathgradeArtifacts } from '../reporting/artifacts.js';
+import { projectNormalizedRunSnapshotToReportInput } from './report-projection.js';
+import { mergeRepeatedRunSnapshots } from './repeated-attempts.js';
+export function withRepeatedAttempts(input) {
+    if (input.config.attempts === 1)
+        return input.adapter;
+    if (!input.adapter.supportsRepeatedAttempts) {
+        throw new Error(`Pathgrade runner adapter "${input.adapter.name}" does not support runner-owned repeated attempts.`);
+    }
+    return {
+        name: input.adapter.name,
+        supportsRepeatedAttempts: true,
+        writeEmptyReport: input.adapter.writeEmptyReport,
+        async run(runInput) {
+            const finalArtifactRoot = getPathgradeDir(runInput.cwd);
+            const attemptRunRoot = path.join(finalArtifactRoot, 'attempts', randomUUID());
+            const snapshots = [];
+            let firstNonzeroExit = 0;
+            let resultCode = 1;
+            try {
+                await Promise.all([
+                    fs.remove(path.join(finalArtifactRoot, 'results.json')),
+                    fs.remove(path.join(finalArtifactRoot, 'traces')),
+                ]);
+                for (let index = 1; index <= input.config.attempts; index++) {
+                    const artifactRoot = path.join(attemptRunRoot, `attempt-${index}`);
+                    const snapshotPath = path.join(artifactRoot, 'normalized-run.json');
+                    const exitCode = await input.adapter.run({
+                        ...runInput,
+                        env: {
+                            ...runInput.env,
+                            PATHGRADE_ATTEMPT_INDEX: String(index),
+                            PATHGRADE_ATTEMPT_COUNT: String(input.config.attempts),
+                            PATHGRADE_ARTIFACT_ROOT: artifactRoot,
+                            PATHGRADE_NORMALIZED_SNAPSHOT_PATH: snapshotPath,
+                        },
+                    });
+                    if (exitCode !== 0 && firstNonzeroExit === 0)
+                        firstNonzeroExit = exitCode;
+                    if (!await fs.pathExists(snapshotPath)) {
+                        throw new Error(`Attempt ${index} did not produce a normalized snapshot at ${snapshotPath}.`);
+                    }
+                    const snapshot = await fs.readJson(snapshotPath);
+                    snapshots.push(snapshot);
+                    if (snapshot.model.run.status === 'cancelled')
+                        break;
+                }
+                if (snapshots.length !== input.config.attempts) {
+                    throw new Error(`Only ${snapshots.length} of ${input.config.attempts} scheduled attempts completed.`);
+                }
+                const merged = mergeRepeatedRunSnapshots(snapshots);
+                const selection = await readSidecarForInvocation(runInput.cwd, runInput.env.PATHGRADE_SELECTION_INVOCATION_ID, message => process.stderr.write(`[pathgrade] ${message}\n`)) ?? undefined;
+                const built = buildPathgradeReport({
+                    threshold: input.config.ci.threshold,
+                    ...projectNormalizedRunSnapshotToReportInput(merged, { selection }),
+                });
+                if (built.report.groups.length === 0 && input.adapter.writeEmptyReport === false) {
+                    resultCode = firstNonzeroExit;
+                }
+                else {
+                    for (const warning of built.warnings) {
+                        process.stderr.write(`  [pathgrade] warning: ${warning}\n`);
+                    }
+                    if ((input.config.reporter ?? 'cli') !== 'json') {
+                        printReportSummary(built.summaries, {
+                            forceVerbose: input.config.diagnostics
+                                || runInput.env.PATHGRADE_DIAGNOSTICS === '1',
+                        });
+                    }
+                    await writePathgradeArtifacts(finalArtifactRoot, built);
+                    process.stdout.write(`\n  Results written to ${finalArtifactRoot}\n\n`);
+                    if (input.config.reporter === 'browser') {
+                        await (input.openBrowser ?? openBrowserViewer)();
+                    }
+                    if (input.config.ci.threshold != null && built.report.status === 'fail') {
+                        process.stdout.write(`\n  CI THRESHOLD FAILED  avg score ${built.report.overall_pass_rate.toFixed(3)} < threshold ${input.config.ci.threshold}\n\n`);
+                        resultCode = firstNonzeroExit || 1;
+                    }
+                    else {
+                        resultCode = firstNonzeroExit;
+                    }
+                }
+            }
+            catch (error) {
+                process.stderr.write(`pathgrade: repeated attempts failed: ${errorMessage(error)}\n`);
+                resultCode = 1;
+            }
+            finally {
+                if (runInput.env.PATHGRADE_KEEP_ATTEMPT_ARTIFACTS !== '1') {
+                    try {
+                        await fs.remove(attemptRunRoot);
+                    }
+                    catch (error) {
+                        process.stderr.write(`pathgrade: could not clean repeated attempt artifacts: ${errorMessage(error)}\n`);
+                        if (resultCode === 0)
+                            resultCode = 1;
+                    }
+                }
+            }
+            return resultCode;
+        },
+    };
+}
+function openBrowserViewer() {
+    const viewerPath = path.resolve(import.meta.dirname, '..', 'viewer.html');
+    try {
+        if (process.platform === 'win32') {
+            execFileSync('cmd', ['/c', 'start', '', viewerPath], { stdio: 'ignore' });
+        }
+        else {
+            execFileSync(process.platform === 'darwin' ? 'open' : 'xdg-open', [viewerPath], { stdio: 'ignore' });
+        }
+        process.stdout.write('  Opened viewer in browser\n\n');
+    }
+    catch {
+        process.stdout.write(`  Open manually: ${viewerPath}\n\n`);
+    }
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}

package/dist/runners/report-projection.js

@@ -6,6 +6,7 @@ export function projectNormalizedRunSnapshotToReportInput(snapshot, options = {}
     }
     const unitsById = new Map(snapshot.model.units.map(unit => [unit.id, unit]));
     const groups = new Map();
+    const repeated = (snapshot.model.run.attemptsRequested ?? 1) > 1;
     for (const runCase of snapshot.model.cases) {
         if (runCase.scoringPolicy.kind === 'non-scoring' && runCase.state !== 'skipped' && runCase.state !== 'pending') {
             continue;
@@ -20,23 +21,31 @@ export function projectNormalizedRunSnapshotToReportInput(snapshot, options = {}
             caseId: runCase.id,
             name: runCase.name,
             state: runCase.state,
-            ...(runCase.state === 'skipped' || runCase.state === 'pending' ? { reportable: runCase.scoringPolicy.kind !== 'non-scoring' } : {}),
+            ...(runCase.state === 'skipped' || runCase.state === 'pending' ? { reportable: repeated || runCase.scoringPolicy.kind !== 'non-scoring' } : {}),
             runnerDurationMs: totalDurationMs(runCase),
-            evaluations: projectedEvaluations(runCase),
+            attempts: runCase.attempts.map((attempt, attemptIndex) => ({
+                attemptId: attempt.id,
+                attemptIndex: attemptIndex + 1,
+                outcome: attempt.outcome,
+                runnerDurationMs: attempt.durationMs ?? 0,
+                evaluations: projectedEvaluations(runCase, attempt.evaluations),
+            })),
         });
         groups.set(key, group);
     }
     return {
         ...(options.selection ? { selection: options.selection } : {}),
+        attemptsRequested: snapshot.model.run.attemptsRequested ?? 1,
+        attemptsCompleted: snapshot.model.run.attemptsCompleted ?? 1,
         groups: [...groups.values()],
     };
 }
 function totalDurationMs(runCase) {
     return runCase.attempts.reduce((sum, attempt) => sum + (attempt.durationMs ?? 0), 0);
 }
-function projectedEvaluations(runCase) {
+function projectedEvaluations(runCase, attemptEvaluations) {
     if (runCase.scoringPolicy.kind === 'non-scoring') {
-        return runCase.attempts.flatMap(attempt => (attempt.evaluations ?? []).map(evaluation => ({
+        return (attemptEvaluations ?? []).map(evaluation => ({
             ...(evaluation.score !== undefined ? { score: evaluation.score } : {}),
             ...(evaluation.trial ? { trial: evaluation.trial } : {}),
             ...(evaluation.diagnostics ? { diagnostics: evaluation.diagnostics } : {}),
@@ -44,12 +53,13 @@ function projectedEvaluations(runCase) {
             ...(evaluation.scoringDurationMs !== undefined ? { scoringDurationMs: evaluation.scoringDurationMs } : {}),
             ...(evaluation.recordedAt ? { recordedAt: evaluation.recordedAt } : {}),
             ...(evaluation.agent ? { agent: evaluation.agent } : {}),
-        })));
+        }));
     }
     if (runCase.scoringPolicy.kind === 'score') {
         return [{ score: runCase.scoringPolicy.score }];
     }
-    const evaluation = runCase.attempts.flatMap(attempt => attempt.evaluations ?? []).at(-1);
+    const evaluation = attemptEvaluations?.findLast(candidate => candidate.resultKind !== 'synthetic_no_evaluation')
+        ?? attemptEvaluations?.at(-1);
     if (!evaluation)
         return [];
     return [{

package/dist/runners/vitest-adapter.js

@@ -79,6 +79,7 @@ function toReportCaseInput(testCase, groupName) {
     const runnerCaseId = typeof testCase.id === 'string' ? testCase.id : undefined;
     return {
         ...(runnerCaseId ? { caseId: runnerCaseId, runnerCaseId } : {}),
+        repeatKey: semanticRepeatKey(filePath, testCase, runnerCaseId),
         name: testCase.name,
         state: normalized.state,
         runnerDurationMs: diagnostics?.duration ?? 0,
@@ -97,6 +98,15 @@ function toReportCaseInput(testCase, groupName) {
         diagnostics: normalized.diagnostics,
     };
 }
+function semanticRepeatKey(filePath, testCase, runnerCaseId) {
+    const parentName = testCase.parent.type === 'suite'
+        ? testCase.parent.fullName
+        : '';
+    return JSON.stringify([
+        'vitest', filePath.replaceAll('\\', '/'), parentName, testCase.name,
+        runnerCaseId ?? null,
+    ]);
+}
 function normalizeState(state) {
     if (state === 'passed' || state === 'failed' || state === 'skipped' || state === 'pending') {
         return { state };

package/dist/runners/vitest-invocation.js

@@ -5,6 +5,8 @@ export function createVitestInvocationAdapter(input = {}) {
     const spawnVitest = input.spawnVitest ?? defaultSpawnVitest;
     return {
         name: 'vitest',
+        supportsRepeatedAttempts: true,
+        writeEmptyReport: false,
         async run(runInput) {
             if (runInput.selectedFiles && hasPassWithNoTests(runInput.runnerArgs)) {
                 process.stderr.write('pathgrade run: --passWithNoTests cannot be used with pathgrade run --changed. ' +

package/dist/sdk/agent-runtime-options.d.ts

@@ -0,0 +1,12 @@
+import { type CompiledMcpMockSession } from './mcp-mock-approvals.js';
+import type { AgentName, AgentOptions, AgentTransport } from './types.js';
+import type { ScenarioArtifact } from '../internal/direct-mcp-v2/types.js';
+import type { PublicScenarioRuntimeFactory } from '../internal/direct-mcp-v2/public-scenario-runtime.js';
+export interface ResolvedAgentRuntimeOptions {
+    agentName: AgentName;
+    transport?: AgentTransport;
+    scenarioArtifact?: ScenarioArtifact;
+    scenarioRuntimeFactory?: PublicScenarioRuntimeFactory;
+    scriptedMcp?: CompiledMcpMockSession;
+}
+export declare function resolveAgentRuntimeOptions(opts: AgentOptions, env: NodeJS.ProcessEnv): ResolvedAgentRuntimeOptions;

package/dist/sdk/agent-runtime-options.js

@@ -0,0 +1,67 @@
+import { resolveAgentName, resolveCodexTransport } from './agent-resolution.js';
+import { compileMcpMockApprovalSession } from './mcp-mock-approvals.js';
+import { compileScenario } from './scenario-machine-v2.js';
+import { startPublicCodexScenarioRuntime } from '../agents/codex-app-server/scenario-mount.js';
+import { validateOpenCodeDeclaration } from '../agents/opencode/contract.js';
+import { createPublicOpenCodeScenarioRuntimeFactory } from '../agents/opencode/scenario.js';
+import { createPublicCursorScenarioRuntimeFactory } from '../agents/cursor-scenario.js';
+export function resolveAgentRuntimeOptions(opts, env) {
+    const agentName = resolveAgentName(opts, env);
+    let scenarioArtifact;
+    if (opts.mcpScenario !== undefined) {
+        if (opts.agent === undefined)
+            throw new Error('mcpScenario requires an explicit agent');
+        if (opts.mcpMock !== undefined || opts.mcpMockApprovalRules !== undefined || opts.mcpConfigFile !== undefined || opts.mcpSafety !== undefined) {
+            throw new Error('mcpScenario cannot be combined with legacy or live MCP options');
+        }
+        const compiled = compileScenario(opts.mcpScenario);
+        if (!compiled.ok)
+            throw new Error(`Scenario compilation failed: ${JSON.stringify(compiled.diagnostics)}`);
+        scenarioArtifact = compiled.artifact;
+    }
+    validateOpenCodeDeclaration(agentName, opts);
+    const transport = agentName === 'codex'
+        ? resolveCodexTransport(opts, env)
+        : undefined;
+    if (scenarioArtifact && agentName === 'codex' && transport !== 'app-server') {
+        throw new Error('mcpScenario requires Codex transport app-server; Codex exec is unsupported');
+    }
+    const scenarioRuntimeFactory = resolveScenarioRuntimeFactory(opts, agentName, transport, scenarioArtifact);
+    const scriptedMcp = resolveScriptedMcp(opts, agentName, transport);
+    return { agentName, transport, scenarioArtifact, scenarioRuntimeFactory, scriptedMcp };
+}
+function resolveScenarioRuntimeFactory(opts, agentName, transport, scenarioArtifact) {
+    if (!scenarioArtifact)
+        return undefined;
+    if (agentName === 'codex')
+        return transport === 'app-server' ? startPublicCodexScenarioRuntime : undefined;
+    if (agentName === 'opencode') {
+        return createPublicOpenCodeScenarioRuntimeFactory({
+            opencodeExecutable: opts.opencodeExecutable,
+            requestedModel: opts.model,
+        });
+    }
+    if (agentName === 'cursor') {
+        return createPublicCursorScenarioRuntimeFactory({ requestedModel: opts.model });
+    }
+    return undefined;
+}
+function resolveScriptedMcp(opts, agentName, transport) {
+    if (opts.mcpMockApprovalRules === undefined)
+        return undefined;
+    if (opts.mcpMock === undefined)
+        throw new Error('mcpMockApprovalRules requires mcpMock');
+    if (opts.mcpConfigFile !== undefined)
+        throw new Error('mcpMockApprovalRules cannot be combined with mcpConfigFile');
+    if (opts.mcpSafety?.runMode !== undefined && opts.mcpSafety.runMode !== 'mock') {
+        throw new Error('mcpMockApprovalRules requires mcpSafety.runMode to be absent or mock');
+    }
+    if (agentName !== 'claude' && !(agentName === 'codex' && transport === 'app-server')) {
+        throw new Error('mcpMockApprovalRules supports Claude or Codex with transport app-server only');
+    }
+    return compileMcpMockApprovalSession({
+        mcpMock: opts.mcpMock,
+        rules: opts.mcpMockApprovalRules,
+        provider: agentName,
+    });
+}

package/dist/sdk/agent.js

@@ -1,5 +1,5 @@
 import { prepareWorkspace } from '../providers/workspace.js';
-import { resolveAgentName, resolveCodexTransport, resolveExecutionTransport, } from './agent-resolution.js';
+import { resolveExecutionTransport, } from './agent-resolution.js';
 import { lifecycleCore } from './lifecycle.js';
 import { ChatSessionImpl } from './chat.js';
 import { runConversation } from './converse.js';
@@ -16,14 +16,10 @@ import { createVerboseEmitter } from '../reporters/verbose-emitter.js';
 import fs from 'fs-extra';
 import * as path from 'path';
 import { cleanDebugRuns, DEFAULT_DEBUG_RETAIN_RUNS, prepareManagedDebugRun, } from '../providers/debug-runs.js';
-import { collectOpenCodeMcpToolNames, validateOpenCodeDeclaration } from '../agents/opencode/contract.js';
+import { collectOpenCodeMcpToolNames } from '../agents/opencode/contract.js';
 import { collectSensitiveEnvValues } from '../tool-event-results.js';
-import { compileMcpMockApprovalSession, } from './mcp-mock-approvals.js';
-import { compileScenario } from './scenario-machine-v2.js';
-import { startPublicCodexScenarioRuntime } from '../agents/codex-app-server/scenario-mount.js';
-import { createPublicOpenCodeScenarioRuntimeFactory } from '../agents/opencode/scenario.js';
-import { createPublicCursorScenarioRuntimeFactory } from '../agents/cursor-scenario.js';
 import { createAskUserHandler } from './ask-bus/handler.js';
+import { resolveAgentRuntimeOptions } from './agent-runtime-options.js';
 /**
  * Test-only injection point: override the sink used by the next emitter
  * built inside `createAgent`. Pass `null` to restore the default (stderr).
@@ -431,56 +427,7 @@ export async function createAgent(opts) {
             throw new Error('Pathgrade debug retainRuns must be a positive integer');
         }
     }
-    const agentName = resolveAgentName(opts, process.env);
-    let scenarioArtifact;
-    if (opts.mcpScenario !== undefined) {
-        if (opts.agent === undefined)
-            throw new Error('mcpScenario requires an explicit agent');
-        if (opts.mcpMock !== undefined || opts.mcpMockApprovalRules !== undefined || opts.mcpConfigFile !== undefined || opts.mcpSafety !== undefined) {
-            throw new Error('mcpScenario cannot be combined with legacy or live MCP options');
-        }
-        const compiled = compileScenario(opts.mcpScenario);
-        if (!compiled.ok)
-            throw new Error(`Scenario compilation failed: ${JSON.stringify(compiled.diagnostics)}`);
-        scenarioArtifact = compiled.artifact;
-    }
-    validateOpenCodeDeclaration(agentName, opts);
-    const transport = agentName === 'codex'
-        ? resolveCodexTransport(opts, process.env)
-        : undefined;
-    if (scenarioArtifact && agentName === 'codex' && transport !== 'app-server') {
-        throw new Error('mcpScenario requires Codex transport app-server; Codex exec is unsupported');
-    }
-    const scenarioRuntimeFactory = !scenarioArtifact
-        ? undefined
-        : agentName === 'codex'
-            ? startPublicCodexScenarioRuntime
-            : agentName === 'opencode'
-                ? createPublicOpenCodeScenarioRuntimeFactory({
-                    opencodeExecutable: opts.opencodeExecutable,
-                    requestedModel: opts.model,
-                })
-                : agentName === 'cursor'
-                    ? createPublicCursorScenarioRuntimeFactory({ requestedModel: opts.model })
-                    : undefined;
-    let scriptedMcp;
-    if (opts.mcpMockApprovalRules !== undefined) {
-        if (opts.mcpMock === undefined)
-            throw new Error('mcpMockApprovalRules requires mcpMock');
-        if (opts.mcpConfigFile !== undefined)
-            throw new Error('mcpMockApprovalRules cannot be combined with mcpConfigFile');
-        if (opts.mcpSafety?.runMode !== undefined && opts.mcpSafety.runMode !== 'mock') {
-            throw new Error('mcpMockApprovalRules requires mcpSafety.runMode to be absent or mock');
-        }
-        if (agentName !== 'claude' && !(agentName === 'codex' && transport === 'app-server')) {
-            throw new Error('mcpMockApprovalRules supports Claude or Codex with transport app-server only');
-        }
-        scriptedMcp = compileMcpMockApprovalSession({
-            mcpMock: opts.mcpMock,
-            rules: opts.mcpMockApprovalRules,
-            provider: agentName,
-        });
-    }
+    const { agentName, transport, scenarioArtifact, scenarioRuntimeFactory, scriptedMcp } = resolveAgentRuntimeOptions(opts, process.env);
     const timeoutSetting = opts.timeout ?? 300;
     // Capture runner context now; adapters own installation and restoration.
     const testCtx = opts.debug ? resolveCaseDebugContext() : { name: '', dir: '' };

package/dist/sdk/case-context.js

@@ -1,6 +1,11 @@
 import { AsyncLocalStorage } from 'node:async_hooks';
-const caseContextStorage = new AsyncLocalStorage();
-const providers = [];
+const storageKey = Symbol.for('@wix/pathgrade/case-context-storage');
+const providersKey = Symbol.for('@wix/pathgrade/case-context-providers');
+const sharedContext = globalThis;
+const caseContextStorage = sharedContext[storageKey]
+    ?? (sharedContext[storageKey] = new AsyncLocalStorage());
+const providers = sharedContext[providersKey]
+    ?? (sharedContext[providersKey] = []);
 export function getCurrentCaseContext() {
     const context = caseContextStorage.getStore();
     if (context)

package/dist/sdk/lifecycle.js

@@ -1,9 +1,14 @@
 import { getCurrentCaseContext } from './case-context.js';
 import { buildDiagnosticsReport } from './diagnostics.js';
 import { countShellCommandsFromLog } from '../tool-events.js';
-const pendingAgents = new Set();
-const agentOwners = new WeakMap();
-const agentResults = new WeakMap();
+const lifecycleStateKey = Symbol.for('@wix/pathgrade/lifecycle-state');
+const sharedLifecycle = globalThis;
+const lifecycleState = sharedLifecycle[lifecycleStateKey] ?? (sharedLifecycle[lifecycleStateKey] = {
+    pendingAgents: new Set(),
+    agentOwners: new WeakMap(),
+    agentResults: new WeakMap(),
+});
+const { pendingAgents, agentOwners, agentResults } = lifecycleState;
 function currentAgentOwner() {
     const current = getCurrentCaseContext();
     if (current.status !== 'active')

package/dist/sdk/result-capture.js

@@ -1,5 +1,8 @@
 import { getCurrentCaseContext } from './case-context.js';
-const observers = new Set();
+const observerRegistryKey = Symbol.for('@wix/pathgrade/eval-result-observers');
+const globalRegistry = globalThis;
+const observers = globalRegistry[observerRegistryKey]
+    ?? (globalRegistry[observerRegistryKey] = new Set());
 export function subscribeToEvalResults(observer, options = {}) {
     if (options.owner === 'adapter' && options.key) {
         removeObserverByOwnerAndKey(options.owner, options.key);