@wix/pathgrade 0.29.0

package/dist/pathgrade.js

@@ -0,0 +1,243 @@
+#!/usr/bin/env node
+/**
+ * pathgrade CLI — thin wrapper around vitest
+ *
+ * Usage:
+ *   pathgrade run [-- vitest-args]   Run evals via vitest (loads .env, validates API keys)
+ *   pathgrade init [--force]         Generate eval scaffolding
+ *   pathgrade preview [browser]      View results (CLI default, or browser)
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { spawn } from 'child_process';
+import { runInit } from './commands/init.js';
+import { runAnalyze } from './commands/analyze.js';
+import { runValidate, runValidateAffected } from './commands/validate.js';
+import { parsePathgradeRunArgs } from './commands/run-args.js';
+import { runPreview } from './commands/preview.js';
+import { runPreviewReactions } from './commands/preview-reactions.js';
+import { runReport } from './commands/report.js';
+import { runAffected } from './commands/affected.js';
+import { runChanged } from './commands/run-changed.js';
+import { clearSidecar } from './affected/sidecar.js';
+import { fmt } from './utils/cli.js';
+import { shutdown } from './utils/shutdown.js';
+function loadDotenv() {
+    const envPath = path.resolve(process.cwd(), '.env');
+    if (!fs.existsSync(envPath))
+        return;
+    const content = fs.readFileSync(envPath, 'utf-8');
+    for (const line of content.split('\n')) {
+        const trimmed = line.trim();
+        if (!trimmed || trimmed.startsWith('#'))
+            continue;
+        const eqIdx = trimmed.indexOf('=');
+        if (eqIdx === -1)
+            continue;
+        const key = trimmed.slice(0, eqIdx).trim();
+        let value = trimmed.slice(eqIdx + 1).trim();
+        // Strip surrounding quotes
+        if ((value.startsWith('"') && value.endsWith('"')) ||
+            (value.startsWith("'") && value.endsWith("'"))) {
+            value = value.slice(1, -1);
+        }
+        if (!(key in process.env)) {
+            process.env[key] = value;
+        }
+    }
+}
+function validateApiKeys() {
+    const hasAnthropic = !!process.env.ANTHROPIC_API_KEY;
+    const hasOpenAI = !!process.env.OPENAI_API_KEY;
+    const hasClaude = !!process.env.HOME; // Claude CLI uses OS keychain, just check it exists
+    if (!hasAnthropic && !hasOpenAI) {
+        console.log(`\n  ${fmt.dim('warning:')} No API keys found. Set ANTHROPIC_API_KEY or OPENAI_API_KEY in .env or environment.\n` +
+            `  ${fmt.dim('         Claude CLI auth (keychain) and Codex cached login (~/.codex/auth.json) may still work if installed.')}\n`);
+    }
+}
+async function main() {
+    shutdown.install();
+    const args = process.argv.slice(2);
+    const command = args[0];
+    if (command === '--help' || command === '-h') {
+        printHelp();
+        return;
+    }
+    if (command === '--version' || command === '-v') {
+        const pkg = JSON.parse(await import('fs').then(fs => fs.promises.readFile(new URL('../package.json', import.meta.url), 'utf-8')));
+        console.log(pkg.version);
+        return;
+    }
+    if (command === 'analyze') {
+        const skillFlag = args.find(a => a.startsWith('--skill='));
+        const dirFlag = args.find(a => a.startsWith('--dir='));
+        const skill = skillFlag ? skillFlag.split('=')[1] : undefined;
+        const dir = dirFlag ? dirFlag.split('=')[1] : undefined;
+        const exitCode = await runAnalyze(process.cwd(), { skill, dir });
+        process.exitCode = exitCode;
+        return;
+    }
+    if (command === 'validate') {
+        const validateArgs = args.slice(1);
+        if (validateArgs.includes('--affected')) {
+            const exitCode = await runValidateAffected(process.cwd());
+            process.exitCode = exitCode;
+            return;
+        }
+        const filePath = validateArgs[0];
+        if (!filePath) {
+            console.error('Usage: pathgrade validate <file.eval.ts>  |  pathgrade validate --affected');
+            process.exitCode = 1;
+            return;
+        }
+        const exitCode = await runValidate(path.resolve(filePath));
+        process.exitCode = exitCode;
+        return;
+    }
+    if (command === 'init') {
+        const hasForce = args.includes('--force');
+        await runInit(process.cwd(), { force: hasForce });
+        return;
+    }
+    if (command === 'preview') {
+        const previewArgs = args.slice(1);
+        const mode = previewArgs.includes('browser') ? 'browser' : 'cli';
+        const lastFlag = previewArgs.find(a => a.startsWith('--last='));
+        const filterFlag = previewArgs.find(a => a.startsWith('--filter='));
+        const last = lastFlag ? parseInt(lastFlag.split('=')[1], 10) : undefined;
+        const filter = filterFlag ? filterFlag.split('=')[1] : undefined;
+        await runPreview(process.cwd(), mode, undefined, { last, filter });
+        return;
+    }
+    if (command === 'preview-reactions') {
+        await runPreviewReactions(args.slice(1));
+        return;
+    }
+    if (command === 'affected') {
+        const affectedArgs = args.slice(1);
+        const changedFilesFlag = affectedArgs.find(a => a.startsWith('--changed-files='));
+        const sinceFlag = affectedArgs.find(a => a.startsWith('--since='));
+        const changedFilesPath = changedFilesFlag
+            ? changedFilesFlag.split('=').slice(1).join('=')
+            : undefined;
+        const since = sinceFlag ? sinceFlag.split('=').slice(1).join('=') : undefined;
+        const explain = affectedArgs.includes('--explain');
+        const json = affectedArgs.includes('--json');
+        const exitCode = await runAffected({
+            cwd: process.cwd(),
+            changedFilesPath,
+            since,
+            explain,
+            json,
+        });
+        process.exitCode = exitCode;
+        return;
+    }
+    if (command === 'report') {
+        const reportArgs = args.slice(1);
+        const resultsPathFlag = reportArgs.find(a => a.startsWith('--results-path='));
+        const commentIdFlag = reportArgs.find(a => a.startsWith('--comment-id='));
+        const noComment = reportArgs.includes('--no-comment');
+        const resultsPath = resultsPathFlag ? resultsPathFlag.split('=').slice(1).join('=') : undefined;
+        const commentId = commentIdFlag ? commentIdFlag.split('=').slice(1).join('=') : undefined;
+        await runReport(process.cwd(), { resultsPath, commentId, noComment });
+        return;
+    }
+    if (command === 'run' || !command || command.startsWith('-')) {
+        // pathgrade run [--changed [--since=…|--changed-files=…]] [--] [vitest-args]
+        loadDotenv();
+        validateApiKeys();
+        const parsed = parsePathgradeRunArgs(command === 'run' ? args.slice(1) : args);
+        for (const warning of parsed.warnings ?? []) {
+            console.error(`pathgrade: ${warning}`);
+        }
+        if (parsed.changed) {
+            const exitCode = await runChanged({
+                cwd: process.cwd(),
+                parsed,
+                spawnVitest: ({ argv }) => new Promise(resolve => {
+                    const child = spawn('npx', ['vitest', ...argv], {
+                        stdio: 'inherit',
+                        env: {
+                            ...process.env,
+                            ...(parsed.forceDiagnostics ? { PATHGRADE_DIAGNOSTICS: '1' } : {}),
+                            ...(parsed.forceVerbose ? { PATHGRADE_VERBOSE: '1' } : {}),
+                        },
+                        shell: true,
+                    });
+                    child.on('close', code => resolve(code ?? 0));
+                }),
+            });
+            process.exitCode = exitCode;
+            return;
+        }
+        // Plain `pathgrade run` — clear any stale selection sidecar from a
+        // previous `--changed` run so it doesn't leak into this full-suite
+        // run (the reporter would otherwise merge old metadata).
+        await clearSidecar(process.cwd());
+        const child = spawn('npx', ['vitest', 'run', ...parsed.vitestArgs], {
+            stdio: 'inherit',
+            env: {
+                ...process.env,
+                ...(parsed.forceDiagnostics ? { PATHGRADE_DIAGNOSTICS: '1' } : {}),
+                ...(parsed.forceVerbose ? { PATHGRADE_VERBOSE: '1' } : {}),
+            },
+            shell: true,
+        });
+        child.on('close', (code) => {
+            process.exitCode = code ?? 0;
+        });
+        return;
+    }
+    console.error(`Unknown command: ${command}`);
+    console.error('Run "pathgrade --help" for usage.');
+    process.exitCode = 1;
+}
+function printHelp() {
+    console.log(`
+  pathgrade - Evaluate AI agent skills with vitest
+
+  Usage:
+    pathgrade run [-- vitest-args]   Run evals (loads .env, delegates to vitest)
+                     [--changed]               Run only evals affected by the current PR/change-set
+                     [--since=<ref>]           Override base ref (implies git mode)
+                     [--changed-files=<path>]  Use an explicit newline-delimited file list
+                     [--quiet]                 Suppress the run-start summary
+                     [--verbose|-v]            Stream live per-turn events to stderr during the run
+    pathgrade init [--force]         Generate eval scaffolding
+    pathgrade analyze [--skill=X]    Analyze skills and output JSON
+    pathgrade validate <file>        Validate an .eval.ts file
+    pathgrade validate --affected    Strict: every eval must be anchored or have valid __pathgradeMeta
+    pathgrade preview [browser]      View results (CLI default, or browser)
+                     [--last=N]     Show only the N most recent reports
+                     [--filter=X]   Filter reports by test name (substring)
+    pathgrade preview-reactions      Preview reactions against a snapshot
+    pathgrade report                 Format .pathgrade/results.json as a markdown PR comment
+                     [--results-path=<path>]   Override results.json location
+                     [--no-comment]            Print markdown to stdout; do not post
+                     [--comment-id=<id>]       Override comment marker (default: $GITHUB_WORKFLOW:$GITHUB_JOB)
+    pathgrade affected               Print eval files affected by a change-set (one per line)
+                     [--since=<ref>]           Diff <ref>...HEAD (overrides git auto-detection)
+                     [--changed-files=<path>]  Newline-delimited repo-relative file list
+                     [--explain]               Print human-readable per-eval decision to stderr
+                     [--json]                  Emit structured JSON (snake_case) to stdout
+
+  Environment:
+    PATHGRADE_AGENT=codex            Override agent for all trials
+    ANTHROPIC_API_KEY                API key for Claude
+    OPENAI_API_KEY                   API key for Codex (optional if Codex CLI is already logged in)
+
+  Examples:
+    pathgrade run                    # run all *.eval.ts files
+    pathgrade run --diagnostics      # print full diagnostics for passing evals too
+    pathgrade run --verbose          # stream per-turn events live to stderr while evals run
+    pathgrade run -- --grep superlint   # filter by test name
+    pathgrade init                   # scaffold eval files
+    pathgrade preview browser        # open web UI
+    pathgrade preview-reactions --snapshot ./pathgrade-debug/run-snapshot.json --reactions ./reactions.ts
+`);
+}
+main().catch(err => {
+    console.error(err);
+    process.exit(1);
+});

package/dist/plugin/index.d.ts

@@ -0,0 +1,9 @@
+import type { PathgradePluginOptions } from '../sdk/types.js';
+export type { PathgradePluginOptions };
+/**
+ * Vitest plugin for pathgrade evaluations.
+ *
+ * Configures test file patterns, timeout, auto-dispose lifecycle hooks,
+ * and the pathgrade reporter for aggregate statistics.
+ */
+export declare function pathgrade(opts?: PathgradePluginOptions): any;

package/dist/plugin/index.js

@@ -0,0 +1,81 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { configDefaults } from 'vitest/config';
+import { PathgradeReporter } from './reporter.js';
+import { discoverPathgradeEvalFiles } from '../evals/discovery.js';
+const DEFAULT_EXCLUDE = [...configDefaults.exclude, '.worktrees/**', 'worktrees/**'];
+function resolveSetupFile() {
+    // Prefer .ts (running from source via vitest) over .js (built dist)
+    const tsPath = path.resolve(import.meta.dirname, 'setup.ts');
+    if (fs.existsSync(tsPath))
+        return tsPath;
+    return path.resolve(import.meta.dirname, 'setup.js');
+}
+/**
+ * Vitest plugin for pathgrade evaluations.
+ *
+ * Configures test file patterns, timeout, auto-dispose lifecycle hooks,
+ * and the pathgrade reporter for aggregate statistics.
+ */
+export function pathgrade(opts) {
+    const timeoutSec = opts?.timeout ?? 300;
+    // Verbose live-streaming seam: the SDK (createAgent) reads
+    // `process.env.PATHGRADE_VERBOSE` to decide whether to emit. Wiring the
+    // plugin option through the env var keeps a single source of truth and
+    // preserves CLI/env precedence — we only set the env if the user hasn't
+    // already set it via their shell or the `pathgrade run --verbose` CLI.
+    if (opts?.verbose === true && process.env.PATHGRADE_VERBOSE === undefined) {
+        process.env.PATHGRADE_VERBOSE = '1';
+    }
+    return {
+        name: 'pathgrade',
+        // Surface the user's options to pre-vitest CLI commands
+        // (`pathgrade affected`, `pathgrade run --changed`) so they can
+        // read `affected.global` after loading vitest.config.ts.
+        // See `src/affected/config.ts`.
+        __pathgradeOptions: opts ?? {},
+        config() {
+            return {
+                resolve: {
+                    alias: resolveLocalAliases(),
+                },
+                test: {
+                    include: opts?.include ?? ['**/*.eval.ts'],
+                    exclude: opts?.exclude ?? DEFAULT_EXCLUDE,
+                    testTimeout: (timeoutSec + 30) * 1000,
+                    setupFiles: [resolveSetupFile()],
+                    reporters: [
+                        'default',
+                        new PathgradeReporter(opts),
+                    ],
+                },
+            };
+        },
+        configResolved(resolved) {
+            const root = resolved.root ?? process.cwd();
+            const include = resolved.test?.include ?? opts?.include ?? ['**/*.eval.ts'];
+            const exclude = resolved.test?.exclude ?? opts?.exclude ?? DEFAULT_EXCLUDE;
+            const discovered = discoverPathgradeEvalFiles({ cwd: root, include, exclude });
+            if (!resolved.test) {
+                resolved.test = {};
+            }
+            resolved.test.include = discovered;
+        },
+    };
+}
+/**
+ * When running from source (not installed via npm), resolve
+ * `@wix/pathgrade` imports to the local src/ directory so examples
+ * can use canonical package imports while developing locally.
+ */
+function resolveLocalAliases() {
+    const sdkSource = path.resolve(import.meta.dirname, '..', 'sdk', 'index.ts');
+    if (!fs.existsSync(sdkSource))
+        return undefined;
+    const src = path.resolve(import.meta.dirname, '..');
+    return {
+        '@wix/pathgrade/mcp-mock': path.join(src, 'core', 'mcp-mock.ts'),
+        '@wix/pathgrade/plugin': path.join(src, 'plugin', 'index.ts'),
+        '@wix/pathgrade': path.join(src, 'sdk', 'index.ts'),
+    };
+}

package/dist/plugin/lifecycle.d.ts

@@ -0,0 +1,45 @@
+import type { Agent, PathgradeTestMeta, RecordedEvalResult } from '../sdk/types.js';
+declare module 'vitest' {
+    interface TaskMeta {
+        pathgrade?: PathgradeTestMeta[];
+    }
+}
+type AfterEachFn = (fn: (ctx: {
+    task: {
+        id: string;
+        meta: Record<string, unknown>;
+    };
+}) => Promise<void>) => void;
+type AfterAllFn = (fn: () => Promise<void>) => void;
+declare function trackAgent(agent: Agent): void;
+declare function untrackAgent(agent: Agent): void;
+/**
+ * Flush results and dispose agents for the current test. Agents created in
+ * this test's scope are disposed after flushing. Shared agents (created in a
+ * different scope, e.g. module-level or beforeAll) have their pending results
+ * flushed but are kept alive for subsequent tests.
+ */
+declare function flush(task: {
+    id: string;
+    meta: Pick<import('vitest').TaskMeta, 'pathgrade'>;
+}): Promise<void>;
+declare function onResult(result: RecordedEvalResult, agent: Agent): void;
+/**
+ * Dispose any agents still pending at end-of-file. Shared agents (created at
+ * module scope or in beforeAll) have a stored task id that won't match any
+ * per-test id in afterEach, so afterAll is their only disposal opportunity.
+ * Without this, sandboxes leak and `debug: true` folders never get written.
+ */
+declare function flushAll(): Promise<void>;
+declare function reset(): void;
+declare function install(afterEach: AfterEachFn, afterAll?: AfterAllFn): void;
+export declare const lifecycle: {
+    trackAgent: typeof trackAgent;
+    untrackAgent: typeof untrackAgent;
+    flush: typeof flush;
+    flushAll: typeof flushAll;
+    onResult: typeof onResult;
+    reset: typeof reset;
+    install: typeof install;
+};
+export {};

package/dist/plugin/lifecycle.js

@@ -0,0 +1,146 @@
+import { setRuntime } from '../sdk/eval-runtime.js';
+import { buildDiagnosticsReport } from '../reporters/diagnostics.js';
+// Agent tracking: each test creates its own agent(s) and calls evaluate() on them.
+// Results are keyed by agent reference, which is unique per test.
+const pendingAgents = new Set();
+const agentTaskIds = new WeakMap();
+const agentResults = new WeakMap();
+function currentTaskId() {
+    try {
+        return globalThis.__vitest_worker__?.current?.id ?? '';
+    }
+    catch {
+        return '';
+    }
+}
+function trackAgent(agent) {
+    pendingAgents.add(agent);
+    const taskId = currentTaskId();
+    if (taskId)
+        agentTaskIds.set(agent, taskId);
+}
+function untrackAgent(agent) {
+    pendingAgents.delete(agent);
+}
+/**
+ * Flush results and dispose agents for the current test. Agents created in
+ * this test's scope are disposed after flushing. Shared agents (created in a
+ * different scope, e.g. module-level or beforeAll) have their pending results
+ * flushed but are kept alive for subsequent tests.
+ */
+async function flush(task) {
+    const results = [];
+    const toDispose = [];
+    for (const agent of pendingAgents) {
+        const storedId = agentTaskIds.get(agent);
+        const belongsToThisTest = !storedId || storedId === task.id;
+        const meta = agentResults.get(agent);
+        if (meta && meta.length > 0) {
+            // Always flush pending results, even for shared agents
+            results.push(...meta);
+            agentResults.delete(agent);
+        }
+        if (belongsToThisTest) {
+            if (!meta || meta.length === 0) {
+                // evaluate() was never called — synthesize a trial from agent data
+                // so the reporter still surfaces token usage and command counts.
+                const synthTrial = synthesizeTrialFromAgent(agent);
+                if (synthTrial) {
+                    results.push(synthTrial);
+                }
+            }
+            pendingAgents.delete(agent);
+            toDispose.push(agent);
+        }
+    }
+    if (results.length > 0) {
+        task.meta.pathgrade = results;
+    }
+    await Promise.all(toDispose.map((a) => a.dispose().catch(() => { })));
+}
+function onResult(result, agent) {
+    if (!agentResults.has(agent)) {
+        agentResults.set(agent, []);
+    }
+    const conversationEnd = [...agent.log].reverse().find((entry) => entry.type === 'conversation_end');
+    const completionReason = conversationEnd?.completion_reason ?? (agent.log.some((entry) => entry.type === 'agent_result') ? 'completed' : undefined);
+    agentResults.get(agent).push({
+        score: result.score,
+        scorers: result.scorers,
+        trial: result.trial,
+        diagnostics: buildDiagnosticsReport({
+            completionReason,
+            completionDetail: conversationEnd?.completion_detail,
+            turnDetails: conversationEnd?.turn_details,
+            reactionsFired: conversationEnd?.reactions_fired,
+            score: result.score,
+            scorers: result.scorers,
+            log: agent.log,
+        }),
+    });
+}
+/**
+ * When evaluate() is never called, synthesize a minimal trial from the agent's
+ * log so the reporter still surfaces command count and token usage.
+ */
+function synthesizeTrialFromAgent(agent) {
+    // Only synthesize if the agent actually ran (has log entries)
+    if (agent.log.length === 0)
+        return null;
+    const nCommands = agent.log.filter((e) => e.type === 'command').length;
+    const tokenUsage = agent.llm.tokenUsage;
+    const conversationEnd = [...agent.log].reverse().find((entry) => entry.type === 'conversation_end');
+    const completionReason = conversationEnd?.completion_reason ?? (agent.log.some((entry) => entry.type === 'agent_result') ? 'completed' : undefined);
+    return {
+        score: 1, // Will be overridden by vitest pass/fail in the reporter
+        scorers: [],
+        trial: {
+            trial_id: 0,
+            reward: 1,
+            scorer_results: [],
+            duration_ms: 0,
+            n_commands: nCommands,
+            input_tokens: tokenUsage?.inputTokens ?? 0,
+            output_tokens: tokenUsage?.outputTokens ?? 0,
+            session_log: [...agent.log],
+        },
+        diagnostics: buildDiagnosticsReport({
+            completionReason,
+            completionDetail: conversationEnd?.completion_detail,
+            turnDetails: conversationEnd?.turn_details,
+            reactionsFired: conversationEnd?.reactions_fired,
+            score: 1,
+            scorers: [],
+            log: agent.log,
+        }),
+    };
+}
+/**
+ * Dispose any agents still pending at end-of-file. Shared agents (created at
+ * module scope or in beforeAll) have a stored task id that won't match any
+ * per-test id in afterEach, so afterAll is their only disposal opportunity.
+ * Without this, sandboxes leak and `debug: true` folders never get written.
+ */
+async function flushAll() {
+    const toDispose = [...pendingAgents];
+    pendingAgents.clear();
+    await Promise.all(toDispose.map((a) => a.dispose().catch(() => { })));
+}
+function reset() {
+    pendingAgents.clear();
+}
+function install(afterEach, afterAll) {
+    setRuntime({ onResult });
+    afterEach(async ({ task }) => flush(task));
+    if (afterAll)
+        afterAll(async () => flushAll());
+}
+export const lifecycle = {
+    trackAgent,
+    untrackAgent,
+    flush,
+    flushAll,
+    onResult,
+    reset,
+    install,
+};

package/dist/plugin/reporter.d.ts

@@ -0,0 +1,36 @@
+import type { Reporter, TestModule } from 'vitest/node';
+import type { PathgradePluginOptions } from '../sdk/types.js';
+/**
+ * Custom vitest reporter that layers pathgrade aggregate statistics
+ * on top of vitest's default output.
+ */
+export declare class PathgradeReporter implements Reporter {
+    private opts;
+    constructor(opts?: PathgradePluginOptions);
+    onTestRunEnd(testModules: ReadonlyArray<TestModule>): Promise<void>;
+    private collectGroups;
+    /**
+     * Group key is the parent suite's full name (file + describe hierarchy).
+     * Tests without a describe wrapper are grouped by module path.
+     */
+    private getGroupKey;
+    private toTestEntry;
+    private printCliSummary;
+    private writeJsonResults;
+    private buildEvalReport;
+    private toTrialResult;
+    private slug;
+    private openBrowserViewer;
+    private computePassRate;
+    /**
+     * pass@k = 1 - (1 - p)^k
+     * Probability of at least 1 success in k trials.
+     */
+    private computePassAtK;
+    /**
+     * pass^k = p^k
+     * Probability of all k trials succeeding.
+     */
+    private computePassPowK;
+    private computeAvgDuration;
+}