@wix/pathgrade 0.37.0 → 1.0.0

package/dist/runners/model-validation.js

@@ -0,0 +1,213 @@
+export function validateNormalizedRunSnapshot(snapshot, options) {
+    const errors = [];
+    validateRun(snapshot, options.completeness, errors);
+    const unitIds = validateUnits(snapshot, errors);
+    validateCases(snapshot, options.completeness, unitIds, errors);
+    return { ok: errors.length === 0, errors };
+}
+function validateRun(snapshot, completeness, errors) {
+    if (snapshot.version !== 1) {
+        errors.push({ path: 'version', message: 'Normalized run snapshot version must be 1.' });
+    }
+    if (snapshot.completeness !== completeness) {
+        errors.push({ path: 'completeness', message: `Expected ${completeness} snapshot.` });
+    }
+    if (!snapshot.model.run.id) {
+        errors.push({ path: 'model.run.id', message: 'Run id is required.' });
+    }
+    if (!snapshot.model.run.adapterName) {
+        errors.push({ path: 'model.run.adapterName', message: 'Run adapterName is required.' });
+    }
+    if (completeness === 'final' && snapshot.model.run.status === 'parked' && (snapshot.model.run.diagnostics ?? []).length === 0) {
+        errors.push({ path: 'model.run.diagnostics', message: 'Final parked snapshots require diagnostics explaining why parked is terminal.' });
+    }
+    validateNativeReferences(snapshot.model.run.nativeReferences, 'model.run.nativeReferences', errors);
+    for (const [diagnosticIndex, diagnostic] of (snapshot.model.run.diagnostics ?? []).entries()) {
+        validateNativeReferences(diagnostic.nativeReferences, `model.run.diagnostics.${diagnosticIndex}.nativeReferences`, errors);
+    }
+}
+function validateUnits(snapshot, errors) {
+    const unitIds = new Set();
+    for (const [unitIndex, unit] of snapshot.model.units.entries()) {
+        if (!unit.id) {
+            errors.push({ path: `model.units.${unitIndex}.id`, message: 'Eval unit id is required.' });
+        }
+        else if (unitIds.has(unit.id)) {
+            errors.push({ path: `model.units.${unitIndex}.id`, message: 'Eval unit id must be unique.' });
+        }
+        else {
+            unitIds.add(unit.id);
+        }
+        if (unit.runId !== snapshot.model.run.id) {
+            errors.push({ path: `model.units.${unitIndex}.runId`, message: 'Eval unit runId must reference the run.' });
+        }
+        if (!unit.displayName) {
+            errors.push({ path: `model.units.${unitIndex}.displayName`, message: 'Eval unit displayName is required.' });
+        }
+        validateNativeReferences(unit.nativeReferences, `model.units.${unitIndex}.nativeReferences`, errors);
+        for (const [diagnosticIndex, diagnostic] of (unit.diagnostics ?? []).entries()) {
+            validateNativeReferences(diagnostic.nativeReferences, `model.units.${unitIndex}.diagnostics.${diagnosticIndex}.nativeReferences`, errors);
+        }
+    }
+    return unitIds;
+}
+function validateCases(snapshot, completeness, unitIds, errors) {
+    const caseIds = new Set();
+    const attemptIds = new Set();
+    const evaluationIds = new Set();
+    const assertionIds = new Set();
+    for (const [caseIndex, runCase] of snapshot.model.cases.entries()) {
+        if (!runCase.id) {
+            errors.push({ path: `model.cases.${caseIndex}.id`, message: 'Run case id is required.' });
+        }
+        else if (caseIds.has(runCase.id)) {
+            errors.push({ path: `model.cases.${caseIndex}.id`, message: 'Run case id must be unique.' });
+        }
+        else {
+            caseIds.add(runCase.id);
+        }
+        if (runCase.runId !== snapshot.model.run.id) {
+            errors.push({ path: `model.cases.${caseIndex}.runId`, message: 'Run case runId must reference the run.' });
+        }
+        if (runCase.unitId && !unitIds.has(runCase.unitId)) {
+            errors.push({ path: `model.cases.${caseIndex}.unitId`, message: 'Run case unitId must reference an eval unit.' });
+        }
+        if (!runCase.name) {
+            errors.push({ path: `model.cases.${caseIndex}.name`, message: 'Run case name is required.' });
+        }
+        validateNativeReferences(runCase.nativeReferences, `model.cases.${caseIndex}.nativeReferences`, errors);
+        for (const [diagnosticIndex, diagnostic] of (runCase.diagnostics ?? []).entries()) {
+            validateNativeReferences(diagnostic.nativeReferences, `model.cases.${caseIndex}.diagnostics.${diagnosticIndex}.nativeReferences`, errors);
+        }
+        if (completeness === 'final' && runCase.attempts.length === 0) {
+            errors.push({ path: `model.cases.${caseIndex}.attempts`, message: 'Final run cases require at least one attempt.' });
+        }
+        const evaluations = runCase.attempts.flatMap(attempt => attempt.evaluations ?? []);
+        if (runCase.scoringPolicy.kind === 'from-evaluations' && completeness === 'final' && evaluations.length === 0) {
+            errors.push({ path: `model.cases.${caseIndex}.scoringPolicy`, message: 'from-evaluations scoring requires at least one evaluation.' });
+        }
+        if (runCase.scoringPolicy.kind === 'score' && (!Number.isFinite(runCase.scoringPolicy.score) || runCase.scoringPolicy.score < 0 || runCase.scoringPolicy.score > 1)) {
+            errors.push({ path: `model.cases.${caseIndex}.scoringPolicy.score`, message: 'Scoring policy score must be a finite number in [0, 1].' });
+        }
+        if (runCase.scoringPolicy.kind === 'non-scoring' && runCase.scoringPolicy.reason.trim() === '') {
+            errors.push({ path: `model.cases.${caseIndex}.scoringPolicy.reason`, message: 'Non-scoring cases require a reason.' });
+        }
+        for (const [attemptIndex, attempt] of runCase.attempts.entries()) {
+            validateAttempt({
+                caseIndex,
+                attemptIndex,
+                attempt,
+                runCaseId: runCase.id,
+                runCaseState: runCase.state,
+                attemptIds,
+                evaluationIds,
+                assertionIds,
+                errors,
+            });
+        }
+    }
+}
+function validateAttempt(input) {
+    const attemptPath = `model.cases.${input.caseIndex}.attempts.${input.attemptIndex}`;
+    if (!input.attempt.id) {
+        input.errors.push({ path: `${attemptPath}.id`, message: 'Attempt id is required.' });
+    }
+    else if (input.attemptIds.has(input.attempt.id)) {
+        input.errors.push({ path: `${attemptPath}.id`, message: 'Attempt id must be unique.' });
+    }
+    else {
+        input.attemptIds.add(input.attempt.id);
+    }
+    if (input.attempt.caseId !== input.runCaseId) {
+        input.errors.push({ path: `${attemptPath}.caseId`, message: 'Attempt caseId must reference its run case.' });
+    }
+    if ((input.runCaseState === 'skipped' || input.runCaseState === 'pending') && input.attempt.outcome.kind !== 'not-run') {
+        input.errors.push({ path: `${attemptPath}.outcome`, message: 'Non-executed final cases require a not-run attempt outcome.' });
+    }
+    for (const [diagnosticIndex, diagnostic] of (input.attempt.diagnostics ?? []).entries()) {
+        validateNativeReferences(diagnostic.nativeReferences, `${attemptPath}.diagnostics.${diagnosticIndex}.nativeReferences`, input.errors);
+    }
+    for (const [evaluationIndex, evaluation] of (input.attempt.evaluations ?? []).entries()) {
+        validateEvaluation({
+            attemptPath,
+            evaluationIndex,
+            evaluation,
+            attemptId: input.attempt.id,
+            evaluationIds: input.evaluationIds,
+            errors: input.errors,
+        });
+    }
+    for (const [assertionIndex, assertion] of (input.attempt.assertions ?? []).entries()) {
+        validateAssertion({
+            attemptPath,
+            assertionIndex,
+            assertion,
+            attemptId: input.attempt.id,
+            assertionIds: input.assertionIds,
+            errors: input.errors,
+        });
+    }
+}
+function validateEvaluation(input) {
+    const evaluationPath = `${input.attemptPath}.evaluations.${input.evaluationIndex}`;
+    if (!input.evaluation.id) {
+        input.errors.push({ path: `${evaluationPath}.id`, message: 'Evaluation id is required.' });
+    }
+    else if (input.evaluationIds.has(input.evaluation.id)) {
+        input.errors.push({ path: `${evaluationPath}.id`, message: 'Evaluation id must be unique.' });
+    }
+    else {
+        input.evaluationIds.add(input.evaluation.id);
+    }
+    if (input.evaluation.attemptId !== input.attemptId) {
+        input.errors.push({ path: `${evaluationPath}.attemptId`, message: 'Evaluation attemptId must reference its attempt.' });
+    }
+    if (!Number.isFinite(input.evaluation.score) || input.evaluation.score < 0 || input.evaluation.score > 1) {
+        input.errors.push({ path: `${evaluationPath}.score`, message: 'Evaluation score must be a finite number in [0, 1].' });
+    }
+    validateNativeReferences(input.evaluation.nativeReferences, `${evaluationPath}.nativeReferences`, input.errors);
+}
+function validateAssertion(input) {
+    const assertionPath = `${input.attemptPath}.assertions.${input.assertionIndex}`;
+    if (!input.assertion.id) {
+        input.errors.push({ path: `${assertionPath}.id`, message: 'Assertion id is required.' });
+    }
+    else if (input.assertionIds.has(input.assertion.id)) {
+        input.errors.push({ path: `${assertionPath}.id`, message: 'Assertion id must be unique.' });
+    }
+    else {
+        input.assertionIds.add(input.assertion.id);
+    }
+    if (input.assertion.attemptId !== input.attemptId) {
+        input.errors.push({ path: `${assertionPath}.attemptId`, message: 'Assertion attemptId must reference its attempt.' });
+    }
+    validateNativeReferences(input.assertion.nativeReferences, `${assertionPath}.nativeReferences`, input.errors);
+}
+function validateNativeReferences(references, path, errors) {
+    for (const [index, reference] of (references ?? []).entries()) {
+        const referencePath = `${path}.${index}`;
+        if (!reference.kind) {
+            errors.push({ path: `${referencePath}.kind`, message: 'Native reference kind is required.' });
+        }
+        if (reference.url && !isSafeUrl(reference.url)) {
+            errors.push({ path: `${referencePath}.url`, message: 'Native reference URL must use http: or https:.' });
+        }
+        for (const [key, value] of Object.entries(reference.metadata ?? {})) {
+            if (value !== null
+                && typeof value !== 'string'
+                && typeof value !== 'number'
+                && typeof value !== 'boolean') {
+                errors.push({ path: `${referencePath}.metadata.${key}`, message: 'Native reference metadata values must be flat primitives.' });
+            }
+        }
+    }
+}
+function isSafeUrl(value) {
+    try {
+        const url = new URL(value);
+        return url.protocol === 'http:' || url.protocol === 'https:';
+    }
+    catch {
+        return false;
+    }
+}

package/dist/runners/model.d.ts

@@ -0,0 +1,117 @@
+import type { DiagnosticsReport } from '../sdk/diagnostics.js';
+import type { TrialResult } from '../types.js';
+export interface NormalizedRunSnapshot {
+    version: 1;
+    completeness: SnapshotCompleteness;
+    model: NormalizedRunModel;
+}
+export type SnapshotCompleteness = 'partial' | 'final';
+export interface NormalizedRunModel {
+    run: RunRecord;
+    units: EvalUnitRecord[];
+    cases: RunCaseRecord[];
+}
+export type RunStatus = 'completed' | 'failed' | 'cancelled' | 'timed_out' | 'parked';
+export interface RunRecord {
+    id: string;
+    adapterName: string;
+    status: RunStatus;
+    diagnostics?: Diagnostic[];
+    nativeReferences?: NativeReference[];
+}
+export interface EvalUnitRecord {
+    id: string;
+    runId: string;
+    displayName: string;
+    diagnostics?: Diagnostic[];
+    nativeReferences?: NativeReference[];
+    groupingHints?: GroupingHint[];
+}
+export type RunCaseState = 'passed' | 'failed' | 'skipped' | 'pending';
+export type ScoringPolicy = {
+    kind: 'score';
+    score: number;
+} | {
+    kind: 'non-scoring';
+    reason: string;
+} | {
+    kind: 'from-evaluations';
+};
+export interface RunCaseRecord {
+    id: string;
+    runId: string;
+    unitId?: string;
+    name: string;
+    state: RunCaseState;
+    scoringPolicy: ScoringPolicy;
+    attempts: AttemptRecord[];
+    groupingHints?: GroupingHint[];
+    diagnostics?: Diagnostic[];
+    nativeReferences?: NativeReference[];
+}
+export type AttemptOutcome = {
+    kind: 'passed';
+} | {
+    kind: 'failed';
+    reason?: string;
+} | {
+    kind: 'errored';
+    reason?: string;
+} | {
+    kind: 'timed-out';
+    durationMs?: number;
+} | {
+    kind: 'cancelled';
+    reason?: string;
+} | {
+    kind: 'not-run';
+    reason: 'skipped' | 'pending' | 'filtered';
+};
+export interface AttemptRecord {
+    id: string;
+    caseId: string;
+    outcome: AttemptOutcome;
+    durationMs?: number;
+    evaluations?: EvaluationRecord[];
+    assertions?: AssertionRecord[];
+    diagnostics?: Diagnostic[];
+}
+export interface EvaluationRecord {
+    id: string;
+    attemptId: string;
+    score: number;
+    name?: string;
+    diagnostics?: DiagnosticsReport;
+    trial?: TrialResult;
+    nativeReferences?: NativeReference[];
+}
+export interface AssertionRecord {
+    id: string;
+    attemptId: string;
+    name: string;
+    status: 'passed' | 'failed' | 'skipped' | 'unknown';
+    severity?: 'info' | 'warning' | 'error' | 'fatal';
+    message?: string;
+    evidence?: string;
+    nativeReferences?: NativeReference[];
+}
+export interface Diagnostic {
+    severity: 'info' | 'warning' | 'error';
+    message: string;
+    code?: string;
+    detail?: string;
+    nativeReferences?: NativeReference[];
+}
+export interface NativeReference {
+    kind: string;
+    id?: string;
+    label?: string;
+    url?: string;
+    metadata?: Record<string, string | number | boolean | null>;
+}
+export interface GroupingHint {
+    kind: 'source' | 'suite' | 'dataset' | 'adapter' | 'custom';
+    key: string;
+    label: string;
+    order?: number;
+}

package/dist/runners/model.js

@@ -0,0 +1 @@
+export {};

package/dist/runners/orchestrator.d.ts

@@ -0,0 +1,30 @@
+import type { PathgradeSelectionReport } from '../types.js';
+import type { ReportSummaryGroup } from '../reporting/types.js';
+import type { AdapterDiscoveryInput, AdapterLifecycleHooks, RunnerAdapter } from './adapter.js';
+export type AdapterReporterMode = 'cli' | 'browser' | 'json';
+export interface PathgradeRunOptions {
+    cwd: string;
+    discovery: AdapterDiscoveryInput;
+    runnerArgs: string[];
+    env: NodeJS.ProcessEnv;
+    artifactRoot: string;
+    reporterMode?: AdapterReporterMode;
+    threshold?: number;
+    selection?: PathgradeSelectionReport;
+    lifecycle?: AdapterLifecycleHooks;
+    signal?: AbortSignal;
+    printSummary?: (summaries: ReportSummaryGroup[]) => void;
+    openBrowser?: () => void | Promise<void>;
+    loadSelection?: () => Promise<PathgradeSelectionReport | undefined>;
+    onThresholdFailure?: (input: {
+        overallPassRate: number;
+        threshold: number;
+    }) => void;
+    writeEmptyReport?: boolean;
+    log?: (message: string) => void;
+    warn?: (message: string) => void;
+}
+export declare function runWithAdapter(input: {
+    adapter: RunnerAdapter;
+    options: PathgradeRunOptions;
+}): Promise<number>;

package/dist/runners/orchestrator.js

@@ -0,0 +1,74 @@
+import { buildPathgradeReport } from '../reporting/core.js';
+import { writePathgradeArtifacts } from '../reporting/artifacts.js';
+import { projectNormalizedRunSnapshotToReportInput } from './report-projection.js';
+export async function runWithAdapter(input) {
+    const { adapter, options } = input;
+    const lifecycle = options.lifecycle ?? createNoopLifecycleHooks();
+    try {
+        const selection = options.selection ?? options.discovery.selection;
+        const discovered = await adapter.discover({
+            ...options.discovery,
+            ...(selection ? { selection } : {}),
+        });
+        const run = await adapter.invoke({
+            discovered,
+            argv: options.runnerArgs,
+            env: options.env,
+            lifecycle,
+            signal: options.signal,
+        });
+        const runExitCode = normalizeRunExitCode(run.status, run.exitCode);
+        const reportInput = projectNormalizedRunSnapshotToReportInput(await adapter.collectNormalizedRunSnapshot(run), { selection });
+        let built = buildPathgradeReport({
+            threshold: options.threshold,
+            ...reportInput,
+        });
+        for (const warning of built.warnings) {
+            options.warn?.(`  [pathgrade] warning: ${warning}`);
+        }
+        if (built.report.groups.length === 0 && options.writeEmptyReport === false) {
+            return runExitCode;
+        }
+        const loadedSelection = await options.loadSelection?.();
+        if (loadedSelection) {
+            built = buildPathgradeReport({
+                threshold: options.threshold,
+                selection: loadedSelection,
+                groups: reportInput.groups,
+            });
+        }
+        const mode = options.reporterMode ?? 'cli';
+        if (mode === 'cli' || mode === 'browser') {
+            options.printSummary?.(built.summaries);
+        }
+        await writePathgradeArtifacts(options.artifactRoot, built);
+        options.log?.(`results:${options.artifactRoot}`);
+        if (mode === 'browser') {
+            await options.openBrowser?.();
+        }
+        if (options.threshold != null && built.report.status === 'fail') {
+            options.onThresholdFailure?.({
+                overallPassRate: built.report.overall_pass_rate,
+                threshold: options.threshold,
+            });
+            return runExitCode === 0 ? 1 : runExitCode;
+        }
+        return runExitCode;
+    }
+    finally {
+        await lifecycle.cleanupRun();
+    }
+}
+function normalizeRunExitCode(status, exitCode) {
+    if (status === 'completed')
+        return exitCode;
+    return exitCode === 0 ? 1 : exitCode;
+}
+function createNoopLifecycleHooks() {
+    return {
+        onResult: () => undefined,
+        withCaseContext: (_context, run) => run(),
+        flushCase: async () => [],
+        cleanupRun: async () => undefined,
+    };
+}

package/dist/runners/report-projection.d.ts

@@ -0,0 +1,6 @@
+import type { ReportRunInput } from '../reporting/types.js';
+import type { PathgradeSelectionReport } from '../types.js';
+import type { NormalizedRunSnapshot } from './model.js';
+export declare function projectNormalizedRunSnapshotToReportInput(snapshot: NormalizedRunSnapshot, options?: {
+    selection?: PathgradeSelectionReport;
+}): ReportRunInput;

package/dist/runners/report-projection.js

@@ -0,0 +1,62 @@
+import { validateNormalizedRunSnapshot } from './model-validation.js';
+export function projectNormalizedRunSnapshotToReportInput(snapshot, options = {}) {
+    const validation = validateNormalizedRunSnapshot(snapshot, { completeness: 'final' });
+    if (!validation.ok) {
+        throw new Error(`Invalid normalized run snapshot: ${validation.errors.map(error => `${error.path}: ${error.message}`).join('; ')}`);
+    }
+    const unitsById = new Map(snapshot.model.units.map(unit => [unit.id, unit]));
+    const groups = new Map();
+    for (const runCase of snapshot.model.cases) {
+        if (runCase.scoringPolicy.kind === 'non-scoring' && runCase.state !== 'skipped' && runCase.state !== 'pending') {
+            continue;
+        }
+        const unit = runCase.unitId ? unitsById.get(runCase.unitId) : undefined;
+        const groupName = preferredGroupingLabel(runCase.groupingHints)
+            ?? preferredGroupingLabel(unit?.groupingHints)
+            ?? unit?.displayName;
+        const key = groupName ?? snapshot.model.run.adapterName;
+        const group = groups.get(key) ?? { groupName: key, cases: [] };
+        group.cases.push({
+            caseId: runCase.id,
+            name: runCase.name,
+            state: runCase.state,
+            ...(runCase.state === 'skipped' || runCase.state === 'pending' ? { reportable: runCase.scoringPolicy.kind !== 'non-scoring' } : {}),
+            runnerDurationMs: totalDurationMs(runCase),
+            evaluations: projectedEvaluations(runCase),
+        });
+        groups.set(key, group);
+    }
+    return {
+        ...(options.selection ? { selection: options.selection } : {}),
+        groups: [...groups.values()],
+    };
+}
+function totalDurationMs(runCase) {
+    return runCase.attempts.reduce((sum, attempt) => sum + (attempt.durationMs ?? 0), 0);
+}
+function projectedEvaluations(runCase) {
+    if (runCase.scoringPolicy.kind === 'non-scoring') {
+        return runCase.attempts.flatMap(attempt => (attempt.evaluations ?? []).map(evaluation => ({
+            score: evaluation.score,
+            ...(evaluation.trial ? { trial: evaluation.trial } : {}),
+            ...(evaluation.diagnostics ? { diagnostics: evaluation.diagnostics } : {}),
+        })));
+    }
+    if (runCase.scoringPolicy.kind === 'score') {
+        return [{ score: runCase.scoringPolicy.score }];
+    }
+    const evaluation = runCase.attempts.flatMap(attempt => attempt.evaluations ?? []).at(-1);
+    if (!evaluation)
+        return [];
+    return [{
+            score: evaluation.score,
+            ...(evaluation.trial ? { trial: evaluation.trial } : {}),
+            ...(evaluation.diagnostics ? { diagnostics: evaluation.diagnostics } : {}),
+        }];
+}
+function preferredGroupingLabel(hints) {
+    return hints
+        ?.toSorted((left, right) => (left.order ?? 0) - (right.order ?? 0))
+        .at(0)
+        ?.label;
+}

package/dist/runners/selection.d.ts

@@ -0,0 +1,6 @@
+import type { RunnerAdapter } from './adapter.js';
+import { type VitestAdapterOptions } from './vitest-adapter.js';
+export declare function resolveRunnerAdapter(input: {
+    adapterName?: string;
+    vitest?: VitestAdapterOptions;
+}): RunnerAdapter;

package/dist/runners/selection.js

@@ -0,0 +1,10 @@
+import { createNodeTestAdapter } from '../adapters/node-test/runner-adapter.js';
+import { createVitestAdapter } from './vitest-adapter.js';
+export function resolveRunnerAdapter(input) {
+    const name = input.adapterName ?? 'vitest';
+    if (name === 'vitest')
+        return createVitestAdapter(input.vitest);
+    if (name === 'node-test')
+        return createNodeTestAdapter();
+    throw new Error(`Unsupported Pathgrade runner adapter: ${name}`);
+}

package/dist/runners/vitest-adapter.d.ts

@@ -0,0 +1,8 @@
+import type { TestModule } from 'vitest/node';
+import type { ReportGroupInput } from '../reporting/types.js';
+import type { RunnerAdapter } from './adapter.js';
+export interface VitestAdapterOptions {
+    testModules?: ReadonlyArray<TestModule>;
+}
+export declare function createVitestAdapter(options?: VitestAdapterOptions): RunnerAdapter;
+export declare function collectVitestReportGroups(testModules: ReadonlyArray<TestModule>): ReportGroupInput[];

package/dist/runners/vitest-adapter.js

@@ -0,0 +1,108 @@
+import { discoverPathgradeEvalFiles } from '../evals/discovery.js';
+import { buildNormalizedRunSnapshotFromReportGroups } from './model-builders.js';
+import { buildDiagnosticsReport } from '../sdk/diagnostics.js';
+export function createVitestAdapter(options = {}) {
+    return {
+        name: 'vitest',
+        async discover(input) {
+            const include = input.include ?? ['**/*.eval.ts'];
+            const exclude = input.exclude ?? [];
+            const files = discoverPathgradeEvalFiles({
+                cwd: input.cwd,
+                include,
+                exclude,
+            });
+            return {
+                units: files.map(file => ({
+                    id: file,
+                    displayName: file,
+                    sourceRef: file,
+                    native: { file },
+                })),
+            };
+        },
+        async invoke(input) {
+            return {
+                adapterName: this.name,
+                status: input.signal?.aborted ? 'cancelled' : 'completed',
+                exitCode: input.signal?.aborted ? 1 : 0,
+                native: {
+                    testModules: options.testModules ?? [],
+                },
+            };
+        },
+        async collectNormalizedRunSnapshot(run) {
+            return buildNormalizedRunSnapshotFromReportGroups(run, collectVitestReportGroups(readVitestRunNative(run).testModules));
+        },
+    };
+}
+export function collectVitestReportGroups(testModules) {
+    const groupMap = new Map();
+    for (const mod of testModules) {
+        for (const testCase of mod.children.allTests()) {
+            const groupName = getGroupName(testCase);
+            const entry = toReportCaseInput(testCase, groupName);
+            if (!groupMap.has(groupName)) {
+                groupMap.set(groupName, []);
+            }
+            groupMap.get(groupName).push(entry);
+        }
+    }
+    return Array.from(groupMap.entries()).map(([groupName, cases]) => ({
+        groupName,
+        cases,
+    }));
+}
+function readVitestRunNative(run) {
+    if (isVitestRunNative(run.native))
+        return run.native;
+    return { testModules: [] };
+}
+function isVitestRunNative(native) {
+    return typeof native === 'object'
+        && native !== null
+        && Array.isArray(native.testModules);
+}
+function getGroupName(testCase) {
+    const modulePath = testCase.module.relativeModuleId;
+    if (testCase.parent.type === 'suite') {
+        return `${modulePath} > ${testCase.parent.fullName}`;
+    }
+    return modulePath;
+}
+function toReportCaseInput(testCase, groupName) {
+    const pathgradeMeta = testCase.meta().pathgrade;
+    const result = testCase.result();
+    const diagnostics = testCase.diagnostic();
+    const normalized = normalizeState(result.state);
+    const filePath = testCase.module.relativeModuleId;
+    const runnerCaseId = typeof testCase.id === 'string' ? testCase.id : undefined;
+    return {
+        ...(runnerCaseId ? { caseId: runnerCaseId, runnerCaseId } : {}),
+        name: testCase.name,
+        state: normalized.state,
+        runnerDurationMs: diagnostics?.duration ?? 0,
+        sourceRef: filePath,
+        filePath,
+        groupName,
+        evaluations: pathgradeMeta?.map(entry => ({
+            score: entry.score,
+            trial: entry.trial,
+            diagnostics: entry.diagnostics,
+        })),
+        diagnostics: normalized.diagnostics,
+    };
+}
+function normalizeState(state) {
+    if (state === 'passed' || state === 'failed' || state === 'skipped' || state === 'pending') {
+        return { state };
+    }
+    return {
+        state: 'failed',
+        diagnostics: buildDiagnosticsReport({
+            completionReason: state,
+            score: 0,
+            log: [],
+        }),
+    };
+}

package/dist/runners/vitest-invocation.d.ts

@@ -0,0 +1,10 @@
+import type { RunnerInvocationAdapter } from './invocation.js';
+export interface SpawnVitestRequest {
+    argv: string[];
+    env: NodeJS.ProcessEnv;
+    cwd: string;
+}
+export type SpawnVitest = (req: SpawnVitestRequest) => Promise<number> | number;
+export declare function createVitestInvocationAdapter(input?: {
+    spawnVitest?: SpawnVitest;
+}): RunnerInvocationAdapter;