@wix/pathgrade 0.37.0 → 1.0.0

package/dist/adapters/jest/reporter.js

@@ -0,0 +1,67 @@
+import * as path from 'node:path';
+import { execSync } from 'node:child_process';
+import { fmt, getPathgradeDir, printReportSummary, readSidecar, resolvePathgradeConfig, runWithAdapter, } from '@wix/pathgrade/adapter-kit';
+import { getJestLifecycleMetadata } from './lifecycle.js';
+import { readJestMetadata, removeJestMetadata } from './metadata.js';
+import { createJestAdapter } from './runner-adapter.js';
+export default class PathgradeJestReporter {
+    async onRunComplete(_contexts, results) {
+        const cwd = process.cwd();
+        const config = await resolvePathgradeConfig({ cwd });
+        const mode = config.reporter ?? 'cli';
+        const outputDir = getPathgradeDir(cwd);
+        const metadataByCaseId = new Map([
+            ...readJestMetadata(),
+            ...getJestLifecycleMetadata(),
+        ]);
+        const exitCode = await runWithAdapter({
+            adapter: createJestAdapter({
+                results,
+                metadataByCaseId,
+            }),
+            options: {
+                cwd,
+                discovery: { cwd },
+                runnerArgs: [],
+                env: process.env,
+                artifactRoot: outputDir,
+                reporterMode: mode,
+                threshold: config.ci.threshold,
+                writeEmptyReport: false,
+                warn: warning => console.warn(warning),
+                log: () => console.log(`\n  ${fmt.dim('Results written to')} ${outputDir}\n`),
+                loadSelection: async () => (await readSidecar(cwd, msg => {
+                    console.warn(`[pathgrade] ${msg}`);
+                })) ?? undefined,
+                printSummary: summaries => {
+                    printReportSummary(summaries, {
+                        forceVerbose: config.diagnostics
+                            || process.env.PATHGRADE_DIAGNOSTICS === '1',
+                    });
+                },
+                openBrowser: () => this.openBrowserViewer(),
+                onThresholdFailure: ({ overallPassRate, threshold }) => {
+                    process.exitCode = 1;
+                    console.log(`\n  ${fmt.fail('CI THRESHOLD FAILED')}  avg score ${fmt.bold(overallPassRate.toFixed(3))} < threshold ${fmt.bold(String(threshold))}\n`);
+                },
+            },
+        });
+        if (exitCode !== 0 && (process.exitCode === undefined || process.exitCode === 0)) {
+            process.exitCode = exitCode;
+        }
+        removeJestMetadata();
+    }
+    openBrowserViewer() {
+        const viewerPath = path.resolve(import.meta.dirname, '..', '..', 'viewer.html');
+        try {
+            const openCmd = process.platform === 'darwin' ? 'open'
+                : process.platform === 'win32' ? 'start'
+                    : 'xdg-open';
+            execSync(`${openCmd} "${viewerPath}"`, { stdio: 'ignore' });
+            console.log(`\n  ${fmt.dim('Opened viewer in browser')}\n`);
+        }
+        catch {
+            console.log(`\n  ${fmt.dim('Open manually:')} ${viewerPath}\n`);
+        }
+    }
+}

package/dist/adapters/jest/results.d.ts

@@ -0,0 +1,31 @@
+import type { AdapterRunHandle, NormalizedRunSnapshot, PathgradeTestMeta } from '@wix/pathgrade/adapter-kit';
+export interface JestAggregatedResult {
+    testResults?: JestFileResult[];
+}
+export interface JestFileResult {
+    testFilePath: string;
+    assertionResults?: JestAssertionResult[];
+    testResults?: JestAssertionResult[];
+}
+export interface JestAssertionResult {
+    ancestorTitles?: string[];
+    title: string;
+    fullName?: string;
+    status: string;
+    duration?: number | null;
+    failureMessages?: string[];
+    location?: {
+        line?: number | null;
+        column?: number | null;
+    } | null;
+}
+export declare function normalizeJestRunResults(input: {
+    run: AdapterRunHandle;
+    results: JestAggregatedResult;
+    metadataByCaseId?: Map<string, PathgradeTestMeta[]>;
+}): NormalizedRunSnapshot;
+export declare function jestCaseId(input: {
+    filePath: string;
+    fullName: string;
+    occurrenceCounts?: Map<string, number>;
+}): string;

package/dist/adapters/jest/results.js

@@ -0,0 +1,144 @@
+export function normalizeJestRunResults(input) {
+    const runId = `${input.run.adapterName}:run`;
+    const files = input.results.testResults ?? [];
+    const units = files.map((fileResult, index) => ({
+        id: unitId(index),
+        runId,
+        displayName: fileResult.testFilePath,
+        groupingHints: [{
+                kind: 'source',
+                key: fileResult.testFilePath,
+                label: fileResult.testFilePath,
+                order: index,
+            }],
+        nativeReferences: [{ kind: 'jest-file', id: fileResult.testFilePath }],
+    }));
+    const occurrenceCounts = new Map();
+    return {
+        version: 1,
+        completeness: 'final',
+        model: {
+            run: {
+                id: runId,
+                adapterName: input.run.adapterName,
+                status: input.run.status,
+            },
+            units,
+            cases: files.flatMap((fileResult, fileIndex) => assertionsForFile(fileResult).map(assertion => {
+                const caseId = jestCaseId({
+                    filePath: fileResult.testFilePath,
+                    fullName: assertion.fullName ?? fullCaseName(assertion),
+                    occurrenceCounts,
+                });
+                return runCase({
+                    runId,
+                    unitId: unitId(fileIndex),
+                    caseId,
+                    assertion,
+                    filePath: fileResult.testFilePath,
+                    evaluations: input.metadataByCaseId?.get(caseId) ?? [],
+                });
+            })),
+        },
+    };
+}
+function assertionsForFile(fileResult) {
+    return fileResult.assertionResults ?? fileResult.testResults ?? [];
+}
+export function jestCaseId(input) {
+    const base = `jest:${input.filePath}:${slug(input.fullName)}`;
+    const current = input.occurrenceCounts?.get(base) ?? 0;
+    const occurrence = current + 1;
+    input.occurrenceCounts?.set(base, occurrence);
+    return `${base}:${occurrence}`;
+}
+function runCase(input) {
+    const state = normalizeState(input.assertion.status);
+    const attemptId = `${input.caseId}:attempt-1`;
+    const evaluations = input.evaluations.map((entry, index) => ({
+        id: `${attemptId}:evaluation-${index + 1}`,
+        attemptId,
+        score: entry.score,
+        ...(entry.trial ? { trial: entry.trial } : {}),
+        ...(entry.diagnostics ? { diagnostics: entry.diagnostics } : {}),
+    }));
+    const failureMessage = input.assertion.failureMessages?.find(message => message.trim().length > 0);
+    return {
+        id: input.caseId,
+        runId: input.runId,
+        unitId: input.unitId,
+        name: fullCaseName(input.assertion),
+        state,
+        scoringPolicy: scoringPolicyForCase(state, evaluations),
+        groupingHints: [{
+                kind: 'source',
+                key: input.filePath,
+                label: input.filePath,
+                order: 0,
+            }],
+        nativeReferences: [{
+                kind: 'jest-case',
+                id: input.caseId,
+                metadata: {
+                    status: input.assertion.status,
+                    ...(input.assertion.location?.line ? { line: input.assertion.location.line } : {}),
+                    ...(input.assertion.location?.column ? { column: input.assertion.location.column } : {}),
+                },
+            }],
+        attempts: [{
+                id: attemptId,
+                caseId: input.caseId,
+                outcome: outcomeForCase(state, failureMessage),
+                durationMs: input.assertion.duration ?? 0,
+                ...(evaluations.length > 0 ? { evaluations } : {}),
+                ...(failureMessage ? {
+                    assertions: [{
+                            id: `${attemptId}:assertion-1`,
+                            attemptId,
+                            name: 'Jest assertion failure',
+                            status: 'failed',
+                            severity: 'error',
+                            message: failureMessage,
+                        }],
+                } : {}),
+            }],
+    };
+}
+function fullCaseName(assertion) {
+    const ancestors = assertion.ancestorTitles ?? [];
+    return [...ancestors, assertion.title].filter(Boolean).join(' > ');
+}
+function normalizeState(status) {
+    if (status === 'passed')
+        return 'passed';
+    if (status === 'skipped')
+        return 'skipped';
+    if (status === 'pending' || status === 'todo' || status === 'disabled')
+        return 'pending';
+    return 'failed';
+}
+function scoringPolicyForCase(state, evaluations) {
+    if (state === 'skipped' || state === 'pending')
+        return { kind: 'non-scoring', reason: state };
+    if (evaluations.length > 0)
+        return { kind: 'from-evaluations' };
+    return { kind: 'score', score: state === 'passed' ? 1 : 0 };
+}
+function outcomeForCase(state, failureMessage) {
+    if (state === 'passed')
+        return { kind: 'passed' };
+    if (state === 'failed')
+        return { kind: 'failed', ...(failureMessage ? { reason: failureMessage } : {}) };
+    return { kind: 'not-run', reason: state };
+}
+function unitId(index) {
+    return `jest:unit-${index + 1}`;
+}
+function slug(value) {
+    const clean = value
+        .trim()
+        .toLowerCase()
+        .replaceAll(/[^a-z0-9]+/g, '-')
+        .replaceAll(/^-|-$/g, '');
+    return clean || 'case';
+}

package/dist/adapters/jest/runner-adapter.d.ts

@@ -0,0 +1,9 @@
+import { type JestAggregatedResult } from './results.js';
+import { type PathgradeTestMeta, RunnerAdapter } from '@wix/pathgrade/adapter-kit';
+interface JestRunNative {
+    results?: JestAggregatedResult;
+    metadataByCaseId?: Map<string, PathgradeTestMeta[]>;
+}
+export declare function createJestAdapter(options?: JestRunNative): RunnerAdapter;
+export declare function createPathgradeAdapter(): RunnerAdapter;
+export {};

package/dist/adapters/jest/runner-adapter.js

@@ -0,0 +1,49 @@
+import { normalizeJestRunResults } from './results.js';
+import { discoverPathgradeEvalFiles, } from '@wix/pathgrade/adapter-kit';
+export function createJestAdapter(options = {}) {
+    return {
+        name: 'jest',
+        async discover(input) {
+            const include = input.include ?? ['**/*.eval.ts'];
+            const exclude = input.exclude ?? [];
+            const files = discoverPathgradeEvalFiles({
+                cwd: input.cwd,
+                include,
+                exclude,
+            });
+            return {
+                units: files.map(file => ({
+                    id: file,
+                    displayName: file,
+                    sourceRef: file,
+                    native: { file, cwd: input.cwd },
+                })),
+            };
+        },
+        async invoke(input) {
+            return {
+                adapterName: this.name,
+                status: input.signal?.aborted ? 'cancelled' : 'completed',
+                exitCode: input.signal?.aborted ? 1 : 0,
+                native: options,
+            };
+        },
+        async collectNormalizedRunSnapshot(run) {
+            const native = readJestRunNative(run);
+            return normalizeJestRunResults({
+                run,
+                results: native.results ?? { testResults: [] },
+                metadataByCaseId: native.metadataByCaseId,
+            });
+        },
+    };
+}
+export function createPathgradeAdapter() {
+    return createJestAdapter();
+}
+function readJestRunNative(run) {
+    if (typeof run.native === 'object' && run.native !== null) {
+        return run.native;
+    }
+    return {};
+}

package/dist/adapters/jest/setup.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/adapters/jest/setup.js

@@ -0,0 +1,10 @@
+import { installJestLifecycle } from './lifecycle.js';
+const jestGlobals = globalThis;
+if (jestGlobals.beforeEach && jestGlobals.afterEach && jestGlobals.expect?.getState) {
+    installJestLifecycle({
+        beforeEach: jestGlobals.beforeEach,
+        afterEach: jestGlobals.afterEach,
+        afterAll: jestGlobals.afterAll,
+        getState: () => jestGlobals.expect?.getState?.() ?? {},
+    });
+}

package/dist/adapters/node-test/index.d.ts

@@ -0,0 +1,8 @@
+import type { TestContext } from 'node:test';
+type NodeTestFn = (t: TestContext) => unknown | Promise<unknown>;
+type NodeTestOptions = Record<string, unknown>;
+export declare function test(name: string, fn: NodeTestFn): void;
+export declare function test(name: string, options: NodeTestOptions, fn: NodeTestFn): void;
+export default test;
+export { createNodeTestAdapter } from './runner-adapter.js';
+export { createNodeTestInvocationAdapter } from './invocation-adapter.js';

package/dist/adapters/node-test/index.js

@@ -0,0 +1,106 @@
+import nodeTest from 'node:test';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { subscribeToEvalResults } from '../../sdk/result-capture.js';
+import { createRunnerLifecycleHooks } from '../../runners/lifecycle-hooks.js';
+const lifecycle = createRunnerLifecycleHooks();
+let subscribed = false;
+let caseCounter = 0;
+export function test(name, optionsOrFn, maybeFn) {
+    installResultCapture();
+    const options = typeof optionsOrFn === 'function' ? undefined : optionsOrFn;
+    const fn = typeof optionsOrFn === 'function' ? optionsOrFn : maybeFn;
+    const sourceRef = callerSourceRef();
+    const caseId = `${sourceRef}:${name}:${++caseCounter}`;
+    nodeTest(name, options, async (ctx) => {
+        const startedAt = performance.now();
+        let state = 'passed';
+        let thrown;
+        try {
+            await lifecycle.withCaseContext({
+                caseId,
+                caseName: name,
+                filePath: sourceRef,
+                sourceRef,
+                runnerNativeId: caseId,
+                scope: 'runner-case',
+            }, async () => {
+                await fn?.(ctx);
+            });
+        }
+        catch (err) {
+            state = 'failed';
+            thrown = err;
+        }
+        finally {
+            const evaluations = await lifecycle.flushCase(caseId);
+            writeCase({
+                caseId,
+                name,
+                state,
+                runnerDurationMs: performance.now() - startedAt,
+                sourceRef,
+                filePath: sourceRef,
+                groupName: sourceRef,
+                runnerCaseId: caseId,
+                evaluations: toReportEvaluations(evaluations),
+            });
+        }
+        if (thrown)
+            throw thrown;
+    });
+}
+export default test;
+export { createNodeTestAdapter } from './runner-adapter.js';
+export { createNodeTestInvocationAdapter } from './invocation-adapter.js';
+function installResultCapture() {
+    if (subscribed)
+        return;
+    subscribed = true;
+    subscribeToEvalResults(event => lifecycle.onResult(event), {
+        owner: 'adapter',
+        key: 'node-test-lifecycle',
+    });
+}
+function toReportEvaluations(evaluations) {
+    if (evaluations.length === 0)
+        return undefined;
+    return evaluations.map(entry => ({
+        score: entry.score,
+        trial: entry.trial,
+        diagnostics: entry.diagnostics,
+    }));
+}
+function writeCase(testCase) {
+    const resultsPath = process.env.PATHGRADE_NODE_TEST_RESULTS;
+    if (!resultsPath)
+        return;
+    fs.mkdirSync(path.dirname(resultsPath), { recursive: true });
+    const existing = fs.existsSync(resultsPath)
+        ? JSON.parse(fs.readFileSync(resultsPath, 'utf8'))
+        : [];
+    existing.push(testCase);
+    fs.writeFileSync(resultsPath, `${JSON.stringify(existing, null, 2)}\n`);
+}
+function callerSourceRef() {
+    const stack = new Error().stack ?? '';
+    const cwd = realpath(process.env.PATHGRADE_NODE_TEST_CWD ?? process.cwd());
+    for (const line of stack.split('\n')) {
+        const match = line.match(/\(?((?:file:\/\/)?[^() ]+\.eval\.[cm]?[jt]s)(?::\d+)?(?::\d+)?\)?/);
+        if (!match)
+            continue;
+        const raw = match[1];
+        const abs = realpath(raw.startsWith('file://') ? fileURLToPath(raw) : raw);
+        return path.relative(cwd, abs).replaceAll(path.sep, '/');
+    }
+    return 'node-test';
+}
+function realpath(value) {
+    try {
+        return fs.realpathSync(value);
+    }
+    catch {
+        return value;
+    }
+}

package/dist/adapters/node-test/invocation-adapter.d.ts

@@ -0,0 +1,5 @@
+import type { ResolvedPathgradeConfig } from '../../config/pathgrade.js';
+import type { RunnerInvocationAdapter } from '../../runners/invocation.js';
+export declare function createNodeTestInvocationAdapter(input: {
+    config: ResolvedPathgradeConfig;
+}): RunnerInvocationAdapter;

package/dist/adapters/node-test/invocation-adapter.js

@@ -0,0 +1,47 @@
+import { readSidecar } from '../../affected/sidecar.js';
+import { getPathgradeDir } from '../../reporters/results-path.js';
+import { printReportSummary } from '../../reporters/report-summary.js';
+import { runWithAdapter } from '../../runners/orchestrator.js';
+import { createNodeTestAdapter } from './runner-adapter.js';
+export function createNodeTestInvocationAdapter(input) {
+    return {
+        name: 'node-test',
+        async run(runInput) {
+            const selection = await readSidecar(runInput.cwd, msg => {
+                process.stderr.write(`[pathgrade] ${msg}\n`);
+            }) ?? undefined;
+            const include = runInput.selectedFiles && runInput.selectedFiles.length > 0
+                ? runInput.selectedFiles
+                : input.config.evals.include;
+            return await runWithAdapter({
+                adapter: createNodeTestAdapter(),
+                options: {
+                    cwd: runInput.cwd,
+                    discovery: {
+                        cwd: runInput.cwd,
+                        include,
+                        exclude: runInput.selectedFiles ? [] : input.config.evals.exclude,
+                    },
+                    runnerArgs: runInput.runnerArgs,
+                    env: runInput.env,
+                    artifactRoot: getPathgradeDir(runInput.cwd),
+                    reporterMode: input.config.reporter ?? 'cli',
+                    threshold: input.config.ci.threshold,
+                    selection,
+                    printSummary: summaries => {
+                        printReportSummary(summaries, {
+                            forceVerbose: input.config.diagnostics
+                                || runInput.env.PATHGRADE_DIAGNOSTICS === '1',
+                        });
+                    },
+                    log: () => {
+                        process.stdout.write(`\n  Results written to ${getPathgradeDir(runInput.cwd)}\n\n`);
+                    },
+                    onThresholdFailure: ({ overallPassRate, threshold }) => {
+                        process.stdout.write(`\n  CI THRESHOLD FAILED  avg score ${overallPassRate.toFixed(3)} < threshold ${threshold}\n\n`);
+                    },
+                },
+            });
+        },
+    };
+}

package/dist/adapters/node-test/runner-adapter.d.ts

@@ -0,0 +1,2 @@
+import type { RunnerAdapter } from '../../runners/adapter.js';
+export declare function createNodeTestAdapter(): RunnerAdapter;

package/dist/adapters/node-test/runner-adapter.js

@@ -0,0 +1,116 @@
+import * as path from 'node:path';
+import * as os from 'node:os';
+import { spawn } from 'node:child_process';
+import fs from 'fs-extra';
+import { discoverPathgradeEvalFiles } from '../../evals/discovery.js';
+import { buildNormalizedRunSnapshotFromReportGroups } from '../../runners/model-builders.js';
+export function createNodeTestAdapter() {
+    return {
+        name: 'node-test',
+        async discover(input) {
+            const include = input.include ?? ['**/*.eval.ts'];
+            const exclude = input.exclude ?? [];
+            const files = discoverPathgradeEvalFiles({
+                cwd: input.cwd,
+                include,
+                exclude,
+            });
+            return {
+                units: files.map(file => ({
+                    id: file,
+                    displayName: file,
+                    sourceRef: file,
+                    native: { file, cwd: input.cwd },
+                })),
+            };
+        },
+        async invoke(input) {
+            const resultsDir = await fs.mkdtemp(path.join(os.tmpdir(), 'pathgrade-node-test-'));
+            const resultsPath = path.join(resultsDir, 'cases.json');
+            const cwd = readDiscoveredCwd(input.discovered.units[0]?.native) ?? process.cwd();
+            const files = input.discovered.units
+                .map(unit => unit.sourceRef)
+                .filter((sourceRef) => typeof sourceRef === 'string');
+            if (files.length === 0) {
+                await fs.writeJson(resultsPath, []);
+                return {
+                    adapterName: this.name,
+                    status: 'completed',
+                    exitCode: 0,
+                    native: { resultsPath },
+                };
+            }
+            const exitCode = await spawnNodeTest({
+                testCwd: cwd,
+                files,
+                argv: input.argv,
+                env: {
+                    ...input.env,
+                    PATHGRADE_NODE_TEST_RESULTS: resultsPath,
+                    PATHGRADE_NODE_TEST_CWD: cwd,
+                },
+            });
+            return {
+                adapterName: this.name,
+                status: input.signal?.aborted ? 'cancelled' : (exitCode === 0 ? 'completed' : 'failed'),
+                exitCode,
+                native: { resultsPath },
+            };
+        },
+        async collectNormalizedRunSnapshot(run) {
+            const native = readNodeTestRunNative(run);
+            const cases = await readCases(native.resultsPath);
+            const groupMap = new Map();
+            for (const testCase of cases) {
+                const groupName = testCase.groupName ?? testCase.filePath ?? testCase.sourceRef ?? 'node-test';
+                if (!groupMap.has(groupName))
+                    groupMap.set(groupName, []);
+                groupMap.get(groupName).push(testCase);
+            }
+            return buildNormalizedRunSnapshotFromReportGroups(run, Array.from(groupMap.entries()).map(([groupName, groupedCases]) => ({
+                groupName,
+                cases: groupedCases,
+            })));
+        },
+    };
+}
+async function spawnNodeTest(input) {
+    const tsxLoader = import.meta.resolve('tsx');
+    const args = [
+        '--import',
+        tsxLoader,
+        '--test',
+        ...input.files,
+        ...input.argv,
+    ];
+    return await new Promise(resolve => {
+        const child = spawn(process.execPath, args, {
+            cwd: input.testCwd,
+            env: input.env,
+            stdio: 'inherit',
+        });
+        child.on('close', code => resolve(code ?? 0));
+        child.on('error', () => resolve(1));
+    });
+}
+function readNodeTestRunNative(run) {
+    if (typeof run.native === 'object'
+        && run.native !== null
+        && typeof run.native.resultsPath === 'string') {
+        return run.native;
+    }
+    return { resultsPath: '' };
+}
+async function readCases(resultsPath) {
+    if (!resultsPath || !(await fs.pathExists(resultsPath)))
+        return [];
+    return await fs.readJson(resultsPath);
+}
+function readDiscoveredCwd(native) {
+    if (typeof native === 'object'
+        && native !== null
+        && typeof native.cwd === 'string') {
+        return native.cwd;
+    }
+    return undefined;
+}

package/dist/adapters/vitest/index.d.ts

@@ -0,0 +1,9 @@
+import type { PathgradePluginOptions } from '../../sdk/types.js';
+export type { PathgradePluginOptions };
+/**
+ * Vitest plugin for pathgrade evaluations.
+ *
+ * Configures test file patterns, timeout, auto-dispose lifecycle hooks,
+ * and the pathgrade reporter for aggregate statistics.
+ */
+export declare function pathgrade(opts?: PathgradePluginOptions): any;