@wix/pathgrade 0.36.0 → 1.0.0

package/dist/providers/workspace.js

@@ -5,6 +5,38 @@ import { createSandbox } from './sandbox.js';
 import { stageMcpConfig } from './mcp-config.js';
 import { sandboxExec } from './sandbox-exec.js';
 import { resolveCredentials } from './credentials.js';
+import { isPortableCopyEntry } from './copy-filter.js';
+const SANDBOX_REMOVE_RETRY_DELAYS_MS = [
+    50,
+    100,
+    250,
+    500,
+    1_000,
+    2_000,
+    3_000,
+    5_000,
+];
+function isRetryableRemoveError(error) {
+    const code = error.code;
+    return code === 'ENOTEMPTY' || code === 'EBUSY' || code === 'EPERM';
+}
+export async function removeSandboxRoot(rootDir, opts = {}) {
+    const remove = opts.remove ?? ((target) => fs.remove(target));
+    const sleep = opts.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    const retryDelaysMs = opts.retryDelaysMs ?? SANDBOX_REMOVE_RETRY_DELAYS_MS;
+    for (let attempt = 0;; attempt++) {
+        try {
+            await remove(rootDir);
+            return;
+        }
+        catch (error) {
+            if (!isRetryableRemoveError(error) || attempt >= retryDelaysMs.length) {
+                throw error;
+            }
+            await sleep(retryDelaysMs[attempt]);
+        }
+    }
+}
 async function copyPathsFromHostHome(pathsToCopy, sandboxHomePath) {
     const realHome = os.homedir();
     for (const relPath of pathsToCopy) {
@@ -13,7 +45,7 @@ async function copyPathsFromHostHome(pathsToCopy, sandboxHomePath) {
             continue;
         const destPath = path.join(sandboxHomePath, relPath);
         await fs.ensureDir(path.dirname(destPath));
-        await fs.copy(srcPath, destPath);
+        await fs.copy(srcPath, destPath, { filter: isPortableCopyEntry });
     }
 }
 export async function linkPathsFromHostHome(pathsToLink, sandboxHomePath) {
@@ -50,19 +82,7 @@ export async function prepareWorkspace(spec) {
                 if (disposed)
                     return;
                 disposed = true;
-                for (let attempt = 0; attempt < 4; attempt++) {
-                    try {
-                        await fs.remove(rootDir);
-                        return;
-                    }
-                    catch (error) {
-                        const code = error.code;
-                        const retryable = code === 'ENOTEMPTY' || code === 'EBUSY' || code === 'EPERM';
-                        if (!retryable || attempt === 3)
-                            throw error;
-                        await new Promise((r) => setTimeout(r, 50 * (attempt + 1)));
-                    }
-                }
+                await removeSandboxRoot(rootDir);
             },
         };
     }

package/dist/reporters/diagnostics.d.ts

@@ -1,46 +1,8 @@
-import type { LogEntry } from '../types.js';
-import type { ReactionFiredEntry, TurnDetail } from '../sdk/types.js';
-import { type RuntimePolicyAuditEntry } from '../sdk/runtime-policy.js';
-export interface DiagnosticsScorer {
-    name: string;
-    type: string;
-    score: number;
-    weight: number;
-    details?: string;
-    status?: 'ok' | 'error' | 'skipped';
-}
-export interface DiagnosticsTurnDetail extends TurnDetail {
-    apiRetries: number;
-}
-export type DiagnosticsRuntimePolicyEntry = RuntimePolicyAuditEntry;
-export interface DiagnosticsReport {
-    turns: number;
-    totalDurationMs: number;
-    completionReason?: string;
-    completionDetail?: string;
-    score?: number;
-    turnDetails: DiagnosticsTurnDetail[];
-    reactionsFired: ReactionFiredEntry[];
-    runtimePoliciesApplied: DiagnosticsRuntimePolicyEntry[];
-    recommendedTimeoutMs: number;
-    warnings: string[];
-    scorers: Array<DiagnosticsScorer & {
-        status: 'ok' | 'error' | 'skipped';
-    }>;
-}
-export interface BuildDiagnosticsReportInput {
-    completionReason?: string;
-    completionDetail?: string;
-    score?: number;
-    turnDetails?: TurnDetail[];
-    reactionsFired?: ReactionFiredEntry[];
-    scorers?: DiagnosticsScorer[];
-    log: LogEntry[];
-}
+import type { DiagnosticsReport } from '../sdk/diagnostics.js';
+export { buildDiagnosticsReport } from '../sdk/diagnostics.js';
 export interface FormatDiagnosticsOptions {
     verbose?: boolean;
     currentTimeoutMs?: number;
 }
-export declare function buildDiagnosticsReport(input: BuildDiagnosticsReportInput): DiagnosticsReport;
 export declare function formatDiagnosticsSummary(report: DiagnosticsReport): string;
 export declare function formatDiagnostics(report: DiagnosticsReport, opts?: FormatDiagnosticsOptions): string;

package/dist/reporters/diagnostics.js

@@ -1,35 +1,4 @@
-import { extractRuntimePolicyAuditEntries } from '../sdk/runtime-policy.js';
-export function buildDiagnosticsReport(input) {
-    const baseTurnDetails = (input.turnDetails && input.turnDetails.length > 0)
-        ? [...input.turnDetails]
-        : deriveTurnDetailsFromLog(input.log);
-    const retriesByTurn = countRetriesByTurn(input.log);
-    const turnDetails = baseTurnDetails
-        .sort((a, b) => a.turn - b.turn)
-        .map((detail) => ({
-        ...detail,
-        apiRetries: retriesByTurn.get(detail.turn) ?? 0,
-    }));
-    const warnings = turnDetails
-        .filter((detail) => detail.outputLines > 500)
-        .map((detail) => `Turn ${detail.turn}: output exceeded 500 lines (${detail.outputLines} lines)`);
-    return {
-        turns: turnDetails.length,
-        totalDurationMs: turnDetails.reduce((sum, detail) => sum + detail.durationMs, 0),
-        completionReason: input.completionReason,
-        completionDetail: input.completionDetail,
-        score: input.score,
-        turnDetails,
-        reactionsFired: input.reactionsFired ?? [],
-        runtimePoliciesApplied: extractRuntimePolicyAuditEntries(input.log),
-        recommendedTimeoutMs: recommendTimeoutMs(turnDetails),
-        warnings,
-        scorers: (input.scorers ?? []).map((scorer) => ({
-            ...scorer,
-            status: scorer.status ?? 'ok',
-        })),
-    };
-}
+export { buildDiagnosticsReport } from '../sdk/diagnostics.js';
 export function formatDiagnosticsSummary(report) {
     const turnsLabel = report.turns === 1 ? '1 turn' : `${report.turns} turns`;
     const durationLabel = `${(report.totalDurationMs / 1000).toFixed(1)}s`;
@@ -84,54 +53,6 @@ export function formatDiagnostics(report, opts = {}) {
     lines.push('', formatTimeoutRecommendation(report.recommendedTimeoutMs, opts.currentTimeoutMs));
     return lines.join('\n');
 }
-function deriveTurnDetailsFromLog(log) {
-    return log
-        .filter((entry) => entry.type === 'agent_result'
-        && entry.synthetic_blocked_prompt !== true
-        && typeof entry.turn_number === 'number'
-        && typeof entry.duration_ms === 'number'
-        && typeof entry.output_lines === 'number'
-        && typeof entry.output_chars === 'number')
-        .map((entry) => ({
-        turn: entry.turn_number,
-        durationMs: entry.duration_ms,
-        outputLines: entry.output_lines,
-        outputChars: entry.output_chars,
-    }));
-}
-function countRetriesByTurn(log) {
-    const retriesByTurn = new Map();
-    let pendingRetries = 0;
-    for (const entry of log) {
-        if (isRetryEntry(entry)) {
-            pendingRetries++;
-            continue;
-        }
-        if (entry.type === 'agent_result' && typeof entry.turn_number === 'number') {
-            if (pendingRetries > 0) {
-                retriesByTurn.set(entry.turn_number, (retriesByTurn.get(entry.turn_number) ?? 0) + pendingRetries);
-                pendingRetries = 0;
-            }
-            if (!retriesByTurn.has(entry.turn_number)) {
-                retriesByTurn.set(entry.turn_number, 0);
-            }
-        }
-    }
-    return retriesByTurn;
-}
-function isRetryEntry(entry) {
-    if (entry.type !== 'agent_result')
-        return false;
-    const text = `${entry.assistant_message ?? ''} ${entry.output ?? ''}`.toLowerCase();
-    return text.includes('retry') || text.includes('api_retry');
-}
-function recommendTimeoutMs(turnDetails) {
-    if (turnDetails.length <= 1) {
-        return (turnDetails[0]?.durationMs ?? 0) + 30_000;
-    }
-    const totalDurationMs = turnDetails.reduce((sum, detail) => sum + detail.durationMs, 0);
-    return Math.max(totalDurationMs + 200_000, 30_000);
-}
 function formatTimeoutRecommendation(recommendedTimeoutMs, currentTimeoutMs) {
     const recommendedSeconds = Math.round(recommendedTimeoutMs / 1000);
     if (currentTimeoutMs == null) {

package/dist/reporters/report-summary.d.ts

@@ -0,0 +1,6 @@
+import type { ReportSummaryGroup } from '../reporting/types.js';
+export interface PrintReportSummaryOptions {
+    forceVerbose?: boolean;
+    currentTimeoutMs?: number;
+}
+export declare function printReportSummary(groups: ReportSummaryGroup[], opts?: PrintReportSummaryOptions): void;

package/dist/reporters/report-summary.js

@@ -0,0 +1,29 @@
+import { fmt } from '../utils/cli.js';
+import { formatDiagnostics, formatDiagnosticsSummary } from './diagnostics.js';
+export function printReportSummary(groups, opts = {}) {
+    console.log(`\n${fmt.bold('── pathgrade summary ')}${fmt.dim('─'.repeat(40))}\n`);
+    for (const group of groups) {
+        const prColor = group.pass_rate >= 0.5 ? fmt.green : fmt.red;
+        console.log(`  ${fmt.bold(group.task)}`);
+        console.log(`    ${fmt.dim('pass rate'.padEnd(12))} ${prColor((group.pass_rate * 100).toFixed(1) + '%')}`);
+        console.log(`    ${fmt.dim(`pass@${group.trial_count}`.padEnd(12))} ${(group.pass_at_k * 100).toFixed(1)}%`);
+        console.log(`    ${fmt.dim(`pass^${group.trial_count}`.padEnd(12))} ${(group.pass_pow_k * 100).toFixed(1)}%`);
+        console.log(`    ${fmt.dim('avg time'.padEnd(12))} ${(group.average_duration_ms / 1000).toFixed(1)}s`);
+        console.log(`    ${fmt.dim('trials'.padEnd(12))} ${group.trial_count}`);
+        console.log();
+        for (const diagnostic of group.diagnostics) {
+            const shouldPrintFull = opts.forceVerbose
+                || diagnostic.state !== 'passed'
+                || diagnostic.report.completionReason === 'timeout'
+                || diagnostic.report.completionReason === 'agent_crashed';
+            console.log(`    ${fmt.bold(diagnostic.caseName)}`);
+            const formatted = shouldPrintFull
+                ? formatDiagnostics(diagnostic.report, { verbose: true, currentTimeoutMs: opts.currentTimeoutMs })
+                : formatDiagnosticsSummary(diagnostic.report);
+            for (const line of formatted.split('\n')) {
+                console.log(`      ${line}`);
+            }
+            console.log();
+        }
+    }
+}

package/dist/reporting/artifacts.d.ts

@@ -0,0 +1,2 @@
+import type { ArtifactWriteResult, PathgradeReportBuildResult } from './types.js';
+export declare function writePathgradeArtifacts(artifactRoot: string, built: PathgradeReportBuildResult): Promise<ArtifactWriteResult>;

package/dist/reporting/artifacts.js

@@ -0,0 +1,20 @@
+import path from 'node:path';
+import fs from 'fs-extra';
+export async function writePathgradeArtifacts(artifactRoot, built) {
+    await fs.ensureDir(path.join(artifactRoot, 'traces'));
+    const gitignorePath = path.join(artifactRoot, '.gitignore');
+    if (!(await fs.pathExists(gitignorePath))) {
+        await fs.writeFile(gitignorePath, '*\n');
+    }
+    const traceFiles = [];
+    for (const trace of built.traces) {
+        await fs.writeJson(path.join(artifactRoot, trace.traceFile), trace.trials, { spaces: 2 });
+        traceFiles.push(trace.traceFile);
+    }
+    const resultsPath = path.join(artifactRoot, 'results.json');
+    await fs.writeJson(resultsPath, built.report, { spaces: 2 });
+    return {
+        resultsPath,
+        traceFiles,
+    };
+}

package/dist/reporting/core.d.ts

@@ -0,0 +1,2 @@
+import type { PathgradeReportBuildResult, ReportRunInput } from './types.js';
+export declare function buildPathgradeReport(input: ReportRunInput): PathgradeReportBuildResult;

package/dist/reporting/core.js

@@ -0,0 +1,173 @@
+import { extractSkillsFromLog } from '../tool-events.js';
+export function buildPathgradeReport(input) {
+    const warnings = [];
+    const consolidatedGroups = [];
+    const traces = [];
+    const summaries = [];
+    const builtGroups = input.groups.map(group => ({
+        groupName: group.groupName,
+        cases: group.cases.map(toBuiltCase),
+    }));
+    for (const group of builtGroups) {
+        for (const testCase of group.cases) {
+            warnings.push(...testCase.warnings);
+        }
+    }
+    const reportableGroups = builtGroups
+        .map(group => ({
+        ...group,
+        cases: group.cases.filter(reportableBuiltCase),
+    }))
+        .filter(group => group.cases.length > 0);
+    for (const group of reportableGroups) {
+        const report = buildEvalReport(group.groupName, group.cases);
+        summaries.push(buildSummary(group.groupName, group.cases, report));
+        const traceFile = `traces/${slug(group.groupName)}.json`;
+        traces.push({ traceFile, trials: report.trials });
+        const strippedTrials = report.trials.map(trial => {
+            const { session_log, conversation, ...rest } = trial;
+            return rest;
+        });
+        const { trials: _trials, ...rest } = report;
+        consolidatedGroups.push({
+            ...rest,
+            trials: strippedTrials,
+            trace_file: traceFile,
+        });
+    }
+    const scores = reportableGroups.flatMap(group => group.cases.map(testCase => testCase.score));
+    const overallPassRate = average(scores);
+    const status = input.threshold != null
+        ? (overallPassRate >= input.threshold ? 'pass' : 'fail')
+        : (reportableGroups.every(group => group.cases.every(testCase => testCase.state === 'passed')) ? 'pass' : 'fail');
+    return {
+        report: {
+            version: 1,
+            timestamp: new Date().toISOString(),
+            ...(input.threshold != null ? { threshold: input.threshold } : {}),
+            overall_pass_rate: overallPassRate,
+            status,
+            groups: consolidatedGroups,
+            ...(input.selection ? { selection: input.selection } : {}),
+        },
+        traces,
+        summaries,
+        warnings,
+    };
+}
+function reportableBuiltCase(testCase) {
+    if (testCase.reportable !== undefined)
+        return testCase.reportable;
+    return testCase.state !== 'skipped' && testCase.state !== 'pending';
+}
+function toBuiltCase(testCase) {
+    if (testCase.evaluations === undefined) {
+        return fallbackCase(testCase, []);
+    }
+    if (testCase.evaluations.length === 0) {
+        return fallbackCase(testCase, [
+            `empty results for "${testCase.name}" — evaluate() may not have been called`,
+        ]);
+    }
+    const evaluation = testCase.evaluations[testCase.evaluations.length - 1];
+    const diagnostics = evaluation.diagnostics ?? testCase.diagnostics;
+    return {
+        name: testCase.name,
+        state: testCase.state,
+        score: evaluation.score,
+        runnerDurationMs: testCase.runnerDurationMs,
+        diagnostics,
+        reportable: testCase.reportable,
+        trial: normalizeTrial({
+            name: testCase.name,
+            score: evaluation.score,
+            runnerDurationMs: testCase.runnerDurationMs,
+            trial: evaluation.trial,
+            diagnostics,
+        }),
+        warnings: [],
+    };
+}
+function fallbackCase(testCase, warnings) {
+    const score = testCase.state === 'passed' ? 1 : 0;
+    return {
+        name: testCase.name,
+        state: testCase.state,
+        score,
+        runnerDurationMs: testCase.runnerDurationMs,
+        diagnostics: testCase.diagnostics,
+        reportable: testCase.reportable,
+        trial: normalizeTrial({
+            name: testCase.name,
+            score,
+            runnerDurationMs: testCase.runnerDurationMs,
+            diagnostics: testCase.diagnostics,
+        }),
+        warnings,
+    };
+}
+function normalizeTrial(input) {
+    const base = input.trial ?? {
+        trial_id: 1,
+        reward: input.score,
+        scorer_results: [],
+        duration_ms: input.runnerDurationMs,
+        n_commands: 0,
+        input_tokens: 0,
+        output_tokens: 0,
+        session_log: [],
+    };
+    const skills = base.skills_used ?? extractSkillsFromLog(base.session_log);
+    return {
+        ...base,
+        name: input.name,
+        duration_ms: base.duration_ms || input.runnerDurationMs,
+        diagnostics: input.diagnostics ?? base.diagnostics,
+        ...(skills.length > 0 ? { skills_used: skills } : {}),
+    };
+}
+function buildEvalReport(groupName, cases) {
+    const trials = cases.map((testCase, index) => ({
+        ...testCase.trial,
+        trial_id: index + 1,
+    }));
+    const passRate = average(cases.map(testCase => testCase.state === 'passed' ? 1 : 0));
+    const skills = new Set();
+    for (const trial of trials) {
+        for (const skill of trial.skills_used ?? []) {
+            skills.add(skill);
+        }
+    }
+    return {
+        task: groupName,
+        pass_rate: passRate,
+        pass_at_k: 1 - Math.pow(1 - passRate, trials.length),
+        pass_pow_k: Math.pow(passRate, trials.length),
+        trials,
+        skills_used: [...skills],
+    };
+}
+function buildSummary(groupName, cases, report) {
+    return {
+        task: groupName,
+        pass_rate: report.pass_rate,
+        pass_at_k: report.pass_at_k,
+        pass_pow_k: report.pass_pow_k,
+        average_duration_ms: average(cases.map(testCase => testCase.runnerDurationMs)),
+        trial_count: cases.length,
+        diagnostics: cases.flatMap(testCase => testCase.diagnostics
+            ? [{ caseName: testCase.name, state: testCase.state, report: testCase.diagnostics }]
+            : []),
+    };
+}
+function average(values) {
+    if (values.length === 0)
+        return 0;
+    return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+function slug(value) {
+    return value
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '-')
+        .replace(/^-+|-+$/g, '') || 'report';
+}

package/dist/reporting/types.d.ts

@@ -0,0 +1,57 @@
+import type { DiagnosticsReport } from '../sdk/diagnostics.js';
+import type { PathgradeReport, PathgradeSelectionReport, TrialResult } from '../types.js';
+export type ReportCaseState = 'passed' | 'failed' | 'skipped' | 'pending';
+export interface ReportRunInput {
+    threshold?: number;
+    selection?: PathgradeSelectionReport;
+    groups: ReportGroupInput[];
+}
+export interface ReportGroupInput {
+    groupName: string;
+    cases: ReportCaseInput[];
+}
+export interface ReportCaseInput {
+    caseId?: string;
+    name: string;
+    state: ReportCaseState;
+    runnerDurationMs: number;
+    sourceRef?: string;
+    filePath?: string;
+    groupName?: string;
+    runnerCaseId?: string;
+    reportable?: boolean;
+    evaluations?: readonly ReportEvaluationInput[];
+    diagnostics?: DiagnosticsReport;
+}
+export interface ReportEvaluationInput {
+    score: number;
+    trial?: TrialResult;
+    diagnostics?: DiagnosticsReport;
+}
+export interface PathgradeReportBuildResult {
+    report: PathgradeReport;
+    traces: Array<{
+        traceFile: string;
+        trials: TrialResult[];
+    }>;
+    summaries: ReportSummaryGroup[];
+    warnings: string[];
+}
+export interface ArtifactWriteResult {
+    resultsPath: string;
+    traceFiles: string[];
+}
+export interface ReportSummaryGroup {
+    task: string;
+    pass_rate: number;
+    pass_at_k: number;
+    pass_pow_k: number;
+    average_duration_ms: number;
+    trial_count: number;
+    diagnostics: ReportSummaryDiagnostics[];
+}
+export interface ReportSummaryDiagnostics {
+    caseName: string;
+    state: ReportCaseState;
+    report: DiagnosticsReport;
+}

package/dist/reporting/types.js

@@ -0,0 +1 @@
+export {};

package/dist/reporting/vitest-edge.d.ts

@@ -0,0 +1 @@
+export { collectVitestReportGroups } from '../runners/vitest-adapter.js';

package/dist/reporting/vitest-edge.js

@@ -0,0 +1 @@
+export { collectVitestReportGroups } from '../runners/vitest-adapter.js';

package/dist/runners/adapter-loader.d.ts

@@ -0,0 +1,17 @@
+import type { ResolvedPathgradeConfig } from '../config/pathgrade.js';
+import { type SpawnVitest } from './vitest-invocation.js';
+import type { RunnerAdapter } from './adapter.js';
+import type { RunnerInvocationAdapter } from './invocation.js';
+export interface ExternalAdapterFactoryInput {
+    config: ResolvedPathgradeConfig;
+}
+export declare function loadRunnerAdapter(input: {
+    adapterName?: string;
+    cwd?: string;
+}): Promise<RunnerAdapter>;
+export declare function loadRunnerInvocationAdapter(input: {
+    adapterName?: string;
+    cwd?: string;
+    config: ResolvedPathgradeConfig;
+    spawnVitest?: SpawnVitest;
+}): Promise<RunnerInvocationAdapter>;

package/dist/runners/adapter-loader.js

@@ -0,0 +1,67 @@
+import { createRequire } from 'node:module';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { createJestInvocationAdapter } from '../adapters/jest/invocation-adapter.js';
+import { createJestAdapter } from '../adapters/jest/runner-adapter.js';
+import { createNodeTestInvocationAdapter } from '../adapters/node-test/invocation-adapter.js';
+import { createVitestInvocationAdapter } from './vitest-invocation.js';
+import { resolveRunnerAdapter } from './selection.js';
+export async function loadRunnerAdapter(input) {
+    const name = input.adapterName ?? 'vitest';
+    if (name === 'vitest' || name === 'node-test') {
+        return resolveRunnerAdapter({ adapterName: name });
+    }
+    if (name === 'jest')
+        return createJestAdapter();
+    const mod = await importExternalAdapter({
+        adapterName: name,
+        cwd: input.cwd ?? process.cwd(),
+    });
+    if (typeof mod.createPathgradeAdapter !== 'function') {
+        throw new Error(`Pathgrade adapter package "${adapterPackageSpecifier(name)}" must export createPathgradeAdapter().`);
+    }
+    return mod.createPathgradeAdapter();
+}
+export async function loadRunnerInvocationAdapter(input) {
+    const name = input.adapterName ?? 'vitest';
+    if (name === 'vitest')
+        return createVitestInvocationAdapter({ spawnVitest: input.spawnVitest });
+    if (name === 'node-test')
+        return createNodeTestInvocationAdapter({ config: input.config });
+    if (name === 'jest')
+        return createJestInvocationAdapter({ config: input.config });
+    const mod = await importExternalAdapter({
+        adapterName: name,
+        cwd: input.cwd ?? process.cwd(),
+    });
+    if (typeof mod.createPathgradeInvocationAdapter !== 'function') {
+        throw new Error(`Pathgrade adapter package "${adapterPackageSpecifier(name)}" must export createPathgradeInvocationAdapter().`);
+    }
+    return mod.createPathgradeInvocationAdapter({ config: input.config });
+}
+function adapterPackageSpecifier(adapterName) {
+    if (adapterName.startsWith('.') || path.isAbsolute(adapterName) || adapterName.includes('/')) {
+        return adapterName;
+    }
+    return `@wix/pathgrade-adapter-${adapterName}`;
+}
+function resolveAdapterSpecifier(input) {
+    if (input.adapterName.startsWith('.') || path.isAbsolute(input.adapterName)) {
+        return path.resolve(input.cwd, input.adapterName);
+    }
+    const specifier = adapterPackageSpecifier(input.adapterName);
+    return createRequire(path.join(input.cwd, 'package.json')).resolve(specifier);
+}
+async function importExternalAdapter(input) {
+    try {
+        const resolved = resolveAdapterSpecifier(input);
+        return await import(pathToFileURL(resolved).href);
+    }
+    catch (err) {
+        throw new Error(`Unsupported Pathgrade runner adapter: ${input.adapterName}. ` +
+            `Tried to load ${adapterPackageSpecifier(input.adapterName)} from ${input.cwd}: ${errMsg(err)}`);
+    }
+}
+function errMsg(err) {
+    return err instanceof Error ? err.message : String(err);
+}

package/dist/runners/adapter.d.ts

@@ -0,0 +1,65 @@
+import type { PathgradeSelectionReport } from '../types.js';
+import type { Agent, PathgradeTestMeta, RecordedEvalResult } from '../sdk/types.js';
+import type { CaseContextScope } from '../sdk/case-context.js';
+import type { NormalizedRunSnapshot } from './model.js';
+export declare const runnerAdapterContractVersion = 1;
+export interface RunnerAdapter {
+    readonly name: string;
+    discover(input: AdapterDiscoveryInput): Promise<AdapterDiscoveryResult>;
+    invoke(input: AdapterInvocationInput): Promise<AdapterRunHandle>;
+    collectNormalizedRunSnapshot(run: AdapterRunHandle): Promise<NormalizedRunSnapshot>;
+}
+export interface AdapterDiscoveryInput {
+    cwd: string;
+    include?: string[];
+    exclude?: string[];
+    selection?: PathgradeSelectionReport;
+}
+export interface AdapterDiscoveryResult {
+    units: AdapterEvalUnit[];
+    diagnostics?: AdapterDiagnostic[];
+}
+export interface AdapterEvalUnit {
+    id: string;
+    displayName: string;
+    sourceRef?: string;
+    native?: unknown;
+}
+export interface AdapterDiagnostic {
+    severity: 'info' | 'warning' | 'error';
+    message: string;
+    sourceRef?: string;
+}
+export interface AdapterInvocationInput {
+    discovered: AdapterDiscoveryResult;
+    argv: string[];
+    env: NodeJS.ProcessEnv;
+    lifecycle: AdapterLifecycleHooks;
+    signal?: AbortSignal;
+}
+export interface AdapterRunHandle {
+    adapterName: string;
+    status: AdapterRunStatus;
+    exitCode: number;
+    native?: unknown;
+    diagnostics?: AdapterDiagnostic[];
+}
+export type AdapterRunStatus = 'completed' | 'failed' | 'cancelled' | 'timed_out' | 'parked';
+export interface AdapterLifecycleHooks {
+    onResult(event: EvalResultEvent): void;
+    withCaseContext<T>(context: AdapterCaseContext, run: () => T): T;
+    flushCase(caseId: string): Promise<PathgradeTestMeta[]>;
+    cleanupRun(): Promise<void>;
+}
+export interface AdapterCaseContext {
+    caseId: string;
+    caseName: string;
+    filePath?: string;
+    sourceRef?: string;
+    runnerNativeId?: string;
+    scope: CaseContextScope;
+}
+export interface EvalResultEvent {
+    result: RecordedEvalResult;
+    agent: Agent;
+}

package/dist/runners/adapter.js

@@ -0,0 +1 @@
+export const runnerAdapterContractVersion = 1;