@wix/pathgrade 0.37.0 → 1.0.1

package/dist/config/pathgrade.d.ts

@@ -0,0 +1,48 @@
+export interface PathgradeConfig {
+    runner?: {
+        adapter?: string;
+        args?: string[];
+    };
+    evals?: {
+        include?: string[];
+        exclude?: string[];
+    };
+    affected?: {
+        global?: string[];
+    };
+    reporter?: 'cli' | 'browser' | 'json';
+    diagnostics?: boolean;
+    verbose?: boolean;
+    ci?: {
+        threshold?: number;
+    };
+}
+export interface ResolvedPathgradeConfig {
+    runner: {
+        adapter: string;
+        args: string[];
+    };
+    evals: {
+        include: string[];
+        exclude: string[];
+    };
+    affected: {
+        global: string[];
+    };
+    reporter?: 'cli' | 'browser' | 'json';
+    diagnostics: boolean;
+    verbose: boolean;
+    ci: {
+        threshold?: number;
+    };
+}
+export declare const DEFAULT_EVAL_INCLUDE: string[];
+export declare const DEFAULT_EVAL_EXCLUDE: string[];
+export declare function defaultPathgradeConfig(): ResolvedPathgradeConfig;
+export declare function resolvePathgradeConfig(input: {
+    cwd: string;
+    cli?: PathgradeConfig;
+    configPath?: string;
+    legacyVitestConfigPath?: string;
+    warn?: (message: string) => void;
+}): Promise<ResolvedPathgradeConfig>;

package/dist/config/pathgrade.js

@@ -0,0 +1,204 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import { createJiti } from 'jiti';
+export const DEFAULT_EVAL_INCLUDE = ['**/*.eval.ts'];
+export const DEFAULT_EVAL_EXCLUDE = [
+    '**/node_modules/**',
+    '**/.git/**',
+    '.worktrees/**',
+    'worktrees/**',
+    '**/fixtures/**',
+];
+const PATHGRADE_CONFIG_CANDIDATES = [
+    'pathgrade.config.ts',
+    'pathgrade.config.mts',
+    'pathgrade.config.js',
+    'pathgrade.config.mjs',
+];
+export function defaultPathgradeConfig() {
+    return {
+        runner: {
+            adapter: 'vitest',
+            args: [],
+        },
+        evals: {
+            include: [...DEFAULT_EVAL_INCLUDE],
+            exclude: [...DEFAULT_EVAL_EXCLUDE],
+        },
+        affected: {
+            global: [],
+        },
+        diagnostics: false,
+        verbose: false,
+        ci: {},
+    };
+}
+export async function resolvePathgradeConfig(input) {
+    const fileConfig = await loadPathgradeConfigFile(input.cwd, input.configPath);
+    const legacyConfig = await loadLegacyVitestConfig(input.cwd, input.legacyVitestConfigPath, input.warn);
+    return mergePathgradeConfig(mergePathgradeConfig(mergePathgradeConfig(defaultPathgradeConfig(), legacyConfig), fileConfig), input.cli);
+}
+function mergePathgradeConfig(base, override) {
+    if (!override)
+        return base;
+    return {
+        runner: {
+            adapter: override.runner?.adapter ?? base.runner.adapter,
+            args: override.runner?.args ?? base.runner.args,
+        },
+        evals: {
+            include: override.evals?.include ?? base.evals.include,
+            exclude: override.evals?.exclude ?? base.evals.exclude,
+        },
+        affected: {
+            global: override.affected?.global ?? base.affected.global,
+        },
+        reporter: override.reporter ?? base.reporter,
+        diagnostics: override.diagnostics ?? base.diagnostics,
+        verbose: override.verbose ?? base.verbose,
+        ci: {
+            threshold: override.ci?.threshold ?? base.ci.threshold,
+        },
+    };
+}
+async function loadPathgradeConfigFile(cwd, configPath) {
+    const resolved = configPath
+        ? path.resolve(cwd, configPath)
+        : PATHGRADE_CONFIG_CANDIDATES
+            .map(candidate => path.join(cwd, candidate))
+            .find(candidate => fs.existsSync(candidate));
+    if (!resolved)
+        return undefined;
+    try {
+        const jiti = createJiti(cwd, { interopDefault: true });
+        const loaded = await jiti.import(resolved, { default: true });
+        return validatePathgradeConfig(loaded, path.relative(cwd, resolved));
+    }
+    catch (err) {
+        if (err instanceof InvalidPathgradeConfigError) {
+            throw err;
+        }
+        throw new Error(`pathgrade: failed to load ${path.relative(cwd, resolved)}: ${errMsg(err)}`);
+    }
+}
+function isObject(value) {
+    return value !== null && typeof value === 'object';
+}
+class InvalidPathgradeConfigError extends Error {
+}
+function validatePathgradeConfig(value, label) {
+    if (!isObject(value)) {
+        throw invalidConfig(label, 'default export must be an object');
+    }
+    validateOptionalObject(value.runner, label, 'runner');
+    const runner = asOptionalObject(value.runner);
+    validateOptionalString(runner?.adapter, label, 'runner.adapter');
+    validateOptionalStringArray(runner?.args, label, 'runner.args');
+    validateOptionalObject(value.evals, label, 'evals');
+    const evals = asOptionalObject(value.evals);
+    validateOptionalStringArray(evals?.include, label, 'evals.include');
+    validateOptionalStringArray(evals?.exclude, label, 'evals.exclude');
+    validateOptionalObject(value.affected, label, 'affected');
+    const affected = asOptionalObject(value.affected);
+    validateOptionalStringArray(affected?.global, label, 'affected.global');
+    if (value.reporter !== undefined && !['cli', 'browser', 'json'].includes(String(value.reporter))) {
+        throw invalidConfig(label, 'reporter must be one of cli, browser, or json');
+    }
+    validateOptionalBoolean(value.diagnostics, label, 'diagnostics');
+    validateOptionalBoolean(value.verbose, label, 'verbose');
+    validateOptionalObject(value.ci, label, 'ci');
+    const ci = asOptionalObject(value.ci);
+    validateOptionalNumber(ci?.threshold, label, 'ci.threshold');
+    return value;
+}
+function asOptionalObject(value) {
+    return isObject(value) ? value : undefined;
+}
+function validateOptionalObject(value, label, field) {
+    if (value !== undefined && !isObject(value)) {
+        throw invalidConfig(label, `${field} must be an object`);
+    }
+}
+function validateOptionalString(value, label, field) {
+    if (value !== undefined && typeof value !== 'string') {
+        throw invalidConfig(label, `${field} must be a string`);
+    }
+}
+function validateOptionalStringArray(value, label, field) {
+    if (value !== undefined && (!Array.isArray(value) || value.some(item => typeof item !== 'string'))) {
+        throw invalidConfig(label, `${field} must be an array of strings`);
+    }
+}
+function validateOptionalBoolean(value, label, field) {
+    if (value !== undefined && typeof value !== 'boolean') {
+        throw invalidConfig(label, `${field} must be a boolean`);
+    }
+}
+function validateOptionalNumber(value, label, field) {
+    if (value !== undefined && typeof value !== 'number') {
+        throw invalidConfig(label, `${field} must be a number`);
+    }
+}
+function invalidConfig(label, reason) {
+    return new InvalidPathgradeConfigError(`pathgrade: invalid ${label}: ${reason}`);
+}
+function errMsg(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+const VITEST_CONFIG_CANDIDATES = [
+    'vitest.config.ts',
+    'vitest.config.mts',
+    'vitest.config.js',
+    'vitest.config.mjs',
+];
+async function loadLegacyVitestConfig(cwd, configPath, warn = () => { }) {
+    const resolved = configPath
+        ? path.resolve(cwd, configPath)
+        : VITEST_CONFIG_CANDIDATES
+            .map(candidate => path.join(cwd, candidate))
+            .find(candidate => fs.existsSync(candidate));
+    if (!resolved)
+        return undefined;
+    let loaded;
+    try {
+        const jiti = createJiti(cwd, { interopDefault: true });
+        loaded = await jiti.import(resolved, { default: true });
+    }
+    catch (err) {
+        const message = `pathgrade: failed to load ${path.relative(cwd, resolved)}: ${errMsg(err)}`;
+        if (configPath)
+            throw new Error(message);
+        warn(message);
+        return undefined;
+    }
+    const plugins = findPluginsList(loaded);
+    const pathgradePlugin = plugins?.find(plugin => isPathgradePlugin(plugin));
+    const opts = isObject(pathgradePlugin)
+        ? pathgradePlugin.__pathgradeOptions
+        : undefined;
+    if (!isObject(opts))
+        return undefined;
+    return {
+        evals: {
+            ...(Array.isArray(opts.include) ? { include: opts.include } : {}),
+            ...(Array.isArray(opts.exclude) ? { exclude: opts.exclude } : {}),
+        },
+        affected: {
+            global: isObject(opts.affected) && Array.isArray(opts.affected.global)
+                ? opts.affected.global
+                : [],
+        },
+    };
+}
+function findPluginsList(config) {
+    if (!isObject(config))
+        return null;
+    if (Array.isArray(config.plugins))
+        return config.plugins;
+    if (isObject(config.test) && Array.isArray(config.test.plugins))
+        return config.test.plugins;
+    return null;
+}
+function isPathgradePlugin(plugin) {
+    return isObject(plugin) && (plugin.name === 'pathgrade' || '__pathgradeOptions' in plugin);
+}

package/dist/pathgrade.d.ts

@@ -1,9 +1,9 @@
 #!/usr/bin/env node
 /**
- * pathgrade CLI — thin wrapper around vitest
+ * pathgrade CLI — runner-neutral wrapper around Pathgrade adapters
  *
  * Usage:
- *   pathgrade run [-- vitest-args]   Run evals via vitest (loads .env, validates API keys)
+ *   pathgrade run [-- runner-args]   Run evals via the selected adapter
  *   pathgrade init [--force]         Generate eval scaffolding
  *   pathgrade preview [browser]      View results (CLI default, or browser)
  */

package/dist/pathgrade.js

@@ -1,15 +1,14 @@
 #!/usr/bin/env node
 /**
- * pathgrade CLI — thin wrapper around vitest
+ * pathgrade CLI — runner-neutral wrapper around Pathgrade adapters
  *
  * Usage:
- *   pathgrade run [-- vitest-args]   Run evals via vitest (loads .env, validates API keys)
+ *   pathgrade run [-- runner-args]   Run evals via the selected adapter
  *   pathgrade init [--force]         Generate eval scaffolding
  *   pathgrade preview [browser]      View results (CLI default, or browser)
  */
 import * as fs from 'fs';
 import * as path from 'path';
-import { spawn } from 'child_process';
 import { runInit } from './commands/init.js';
 import { runAnalyze } from './commands/analyze.js';
 import { runValidate, runValidateAffected } from './commands/validate.js';
@@ -20,6 +19,8 @@ import { runReport } from './commands/report.js';
 import { runAffected } from './commands/affected.js';
 import { runChanged } from './commands/run-changed.js';
 import { clearSidecar } from './affected/sidecar.js';
+import { resolvePathgradeConfig } from './config/pathgrade.js';
+import { loadRunnerInvocationAdapter } from './runners/adapter-loader.js';
 import { fmt } from './utils/cli.js';
 import { shutdown } from './utils/shutdown.js';
 function loadDotenv() {
@@ -52,7 +53,7 @@ function validateApiKeys() {
     const hasClaude = !!process.env.HOME; // Claude CLI uses OS keychain, just check it exists
     if (!hasAnthropic && !hasOpenAI) {
         console.log(`\n  ${fmt.dim('warning:')} No API keys found. Set ANTHROPIC_API_KEY or OPENAI_API_KEY in .env or environment.\n` +
-            `  ${fmt.dim('         Claude CLI auth (keychain) and Codex cached login (~/.codex/auth.json) may still work if installed.')}\n`);
+            `  ${fmt.dim('         Claude CLI auth (keychain) and Codex exec cached login (~/.codex/auth.json) may still work if installed.')}\n`);
     }
 }
 async function main() {
@@ -144,7 +145,7 @@ async function main() {
         return;
     }
     if (command === 'run' || !command || command.startsWith('-')) {
-        // pathgrade run [--changed [--since=…|--changed-files=…]] [--] [vitest-args]
+        // pathgrade run [--changed [--since=…|--changed-files=…]] [--] [runner-args]
         loadDotenv();
         validateApiKeys();
         const parsed = parsePathgradeRunArgs(command === 'run' ? args.slice(1) : args);
@@ -155,18 +156,6 @@ async function main() {
             const exitCode = await runChanged({
                 cwd: process.cwd(),
                 parsed,
-                spawnVitest: ({ argv }) => new Promise(resolve => {
-                    const child = spawn('npx', ['vitest', ...argv], {
-                        stdio: 'inherit',
-                        env: {
-                            ...process.env,
-                            ...(parsed.forceDiagnostics ? { PATHGRADE_DIAGNOSTICS: '1' } : {}),
-                            ...(parsed.forceVerbose ? { PATHGRADE_VERBOSE: '1' } : {}),
-                        },
-                        shell: true,
-                    });
-                    child.on('close', code => resolve(code ?? 0));
-                }),
             });
             process.exitCode = exitCode;
             return;
@@ -175,18 +164,28 @@ async function main() {
         // previous `--changed` run so it doesn't leak into this full-suite
         // run (the reporter would otherwise merge old metadata).
         await clearSidecar(process.cwd());
-        const child = spawn('npx', ['vitest', 'run', ...parsed.vitestArgs], {
-            stdio: 'inherit',
-            env: {
-                ...process.env,
-                ...(parsed.forceDiagnostics ? { PATHGRADE_DIAGNOSTICS: '1' } : {}),
-                ...(parsed.forceVerbose ? { PATHGRADE_VERBOSE: '1' } : {}),
-            },
-            shell: true,
-        });
-        child.on('close', (code) => {
-            process.exitCode = code ?? 0;
-        });
+        const env = {
+            ...process.env,
+            ...(parsed.forceDiagnostics ? { PATHGRADE_DIAGNOSTICS: '1' } : {}),
+            ...(parsed.forceVerbose ? { PATHGRADE_VERBOSE: '1' } : {}),
+        };
+        try {
+            const config = await resolvePathgradeConfig({ cwd: process.cwd() });
+            const runner = await loadRunnerInvocationAdapter({
+                adapterName: parsed.adapterName ?? config.runner.adapter,
+                cwd: process.cwd(),
+                config,
+            });
+            process.exitCode = await runner.run({
+                cwd: process.cwd(),
+                runnerArgs: [...config.runner.args, ...parsed.runnerArgs],
+                env,
+            });
+        }
+        catch (err) {
+            console.error(err instanceof Error ? err.message : String(err));
+            process.exitCode = 1;
+        }
         return;
     }
     console.error(`Unknown command: ${command}`);
@@ -195,13 +194,15 @@ async function main() {
 }
 function printHelp() {
     console.log(`
-  pathgrade - Evaluate AI agent skills with vitest
+  pathgrade - Evaluate AI agent skills with a runner adapter
 
   Usage:
-    pathgrade run [-- vitest-args]   Run evals (loads .env, delegates to vitest)
+    pathgrade run [-- runner-args]   Run evals (loads .env, delegates to the selected adapter)
                      [--changed]               Run only evals affected by the current PR/change-set
                      [--since=<ref>]           Override base ref (implies git mode)
                      [--changed-files=<path>]  Use an explicit newline-delimited file list
+                     [--adapter=<name|path>]   Select built-in or third-party runner adapter
+                     [--diagnostics]           Print full diagnostics for passing evals too
                      [--quiet]                 Suppress the run-start summary
                      [--verbose|-v]            Stream live per-turn events to stderr during the run
     pathgrade init [--force]         Generate eval scaffolding
@@ -223,9 +224,9 @@ function printHelp() {
                      [--json]                  Emit structured JSON (snake_case) to stdout
 
   Environment:
-    PATHGRADE_AGENT=codex            Override agent for all trials
-    ANTHROPIC_API_KEY                API key for Claude
-    OPENAI_API_KEY                   API key for Codex (optional if Codex CLI is already logged in)
+    PATHGRADE_AGENT=codex            Fallback agent when createAgent({ agent }) is omitted
+    ANTHROPIC_API_KEY                API key for Claude and Anthropic-backed judges
+    OPENAI_API_KEY                   API key for Codex and OpenAI-backed judges
 
   Examples:
     pathgrade run                    # run all *.eval.ts files

package/dist/plugin/index.d.ts

@@ -1,9 +1,2 @@
-import type { PathgradePluginOptions } from '../sdk/types.js';
-export type { PathgradePluginOptions };
-/**
- * Vitest plugin for pathgrade evaluations.
- *
- * Configures test file patterns, timeout, auto-dispose lifecycle hooks,
- * and the pathgrade reporter for aggregate statistics.
- */
-export declare function pathgrade(opts?: PathgradePluginOptions): any;
+export { pathgrade } from '../adapters/vitest/index.js';
+export type { PathgradePluginOptions } from '../adapters/vitest/index.js';

package/dist/plugin/index.js

@@ -1,81 +1 @@
-import * as fs from 'fs';
-import * as path from 'path';
-import { configDefaults } from 'vitest/config';
-import { PathgradeReporter } from './reporter.js';
-import { discoverPathgradeEvalFiles } from '../evals/discovery.js';
-const DEFAULT_EXCLUDE = [...configDefaults.exclude, '.worktrees/**', 'worktrees/**'];
-function resolveSetupFile() {
-    // Prefer .ts (running from source via vitest) over .js (built dist)
-    const tsPath = path.resolve(import.meta.dirname, 'setup.ts');
-    if (fs.existsSync(tsPath))
-        return tsPath;
-    return path.resolve(import.meta.dirname, 'setup.js');
-}
-/**
- * Vitest plugin for pathgrade evaluations.
- *
- * Configures test file patterns, timeout, auto-dispose lifecycle hooks,
- * and the pathgrade reporter for aggregate statistics.
- */
-export function pathgrade(opts) {
-    const timeoutSec = opts?.timeout ?? 300;
-    // Verbose live-streaming seam: the SDK (createAgent) reads
-    // `process.env.PATHGRADE_VERBOSE` to decide whether to emit. Wiring the
-    // plugin option through the env var keeps a single source of truth and
-    // preserves CLI/env precedence — we only set the env if the user hasn't
-    // already set it via their shell or the `pathgrade run --verbose` CLI.
-    if (opts?.verbose === true && process.env.PATHGRADE_VERBOSE === undefined) {
-        process.env.PATHGRADE_VERBOSE = '1';
-    }
-    return {
-        name: 'pathgrade',
-        // Surface the user's options to pre-vitest CLI commands
-        // (`pathgrade affected`, `pathgrade run --changed`) so they can
-        // read `affected.global` after loading vitest.config.ts.
-        // See `src/affected/config.ts`.
-        __pathgradeOptions: opts ?? {},
-        config() {
-            return {
-                resolve: {
-                    alias: resolveLocalAliases(),
-                },
-                test: {
-                    include: opts?.include ?? ['**/*.eval.ts'],
-                    exclude: opts?.exclude ?? DEFAULT_EXCLUDE,
-                    testTimeout: (timeoutSec + 30) * 1000,
-                    setupFiles: [resolveSetupFile()],
-                    reporters: [
-                        'default',
-                        new PathgradeReporter(opts),
-                    ],
-                },
-            };
-        },
-        configResolved(resolved) {
-            const root = resolved.root ?? process.cwd();
-            const include = resolved.test?.include ?? opts?.include ?? ['**/*.eval.ts'];
-            const exclude = resolved.test?.exclude ?? opts?.exclude ?? DEFAULT_EXCLUDE;
-            const discovered = discoverPathgradeEvalFiles({ cwd: root, include, exclude });
-            if (!resolved.test) {
-                resolved.test = {};
-            }
-            resolved.test.include = discovered;
-        },
-    };
-}
-/**
- * When running from source (not installed via npm), resolve
- * `@wix/pathgrade` imports to the local src/ directory so examples
- * can use canonical package imports while developing locally.
- */
-function resolveLocalAliases() {
-    const sdkSource = path.resolve(import.meta.dirname, '..', 'sdk', 'index.ts');
-    if (!fs.existsSync(sdkSource))
-        return undefined;
-    const src = path.resolve(import.meta.dirname, '..');
-    return {
-        '@wix/pathgrade/mcp-mock': path.join(src, 'core', 'mcp-mock.ts'),
-        '@wix/pathgrade/plugin': path.join(src, 'plugin', 'index.ts'),
-        '@wix/pathgrade': path.join(src, 'sdk', 'index.ts'),
-    };
-}
+export { pathgrade } from '../adapters/vitest/index.js';

package/dist/plugin/lifecycle.d.ts

@@ -1,45 +1 @@
-import type { Agent, PathgradeTestMeta, RecordedEvalResult } from '../sdk/types.js';
-declare module 'vitest' {
-    interface TaskMeta {
-        pathgrade?: PathgradeTestMeta[];
-    }
-}
-type AfterEachFn = (fn: (ctx: {
-    task: {
-        id: string;
-        meta: Record<string, unknown>;
-    };
-}) => Promise<void>) => void;
-type AfterAllFn = (fn: () => Promise<void>) => void;
-declare function trackAgent(agent: Agent): void;
-declare function untrackAgent(agent: Agent): void;
-/**
- * Flush results and dispose agents for the current test. Agents created in
- * this test's scope are disposed after flushing. Shared agents (created in a
- * different scope, e.g. module-level or beforeAll) have their pending results
- * flushed but are kept alive for subsequent tests.
- */
-declare function flush(task: {
-    id: string;
-    meta: Pick<import('vitest').TaskMeta, 'pathgrade'>;
-}): Promise<void>;
-declare function onResult(result: RecordedEvalResult, agent: Agent): void;
-/**
- * Dispose any agents still pending at end-of-file. Shared agents (created at
- * module scope or in beforeAll) have a stored task id that won't match any
- * per-test id in afterEach, so afterAll is their only disposal opportunity.
- * Without this, sandboxes leak and `debug: true` folders never get written.
- */
-declare function flushAll(): Promise<void>;
-declare function reset(): void;
-declare function install(afterEach: AfterEachFn, afterAll?: AfterAllFn): void;
-export declare const lifecycle: {
-    trackAgent: typeof trackAgent;
-    untrackAgent: typeof untrackAgent;
-    flush: typeof flush;
-    flushAll: typeof flushAll;
-    onResult: typeof onResult;
-    reset: typeof reset;
-    install: typeof install;
-};
-export {};
+export { lifecycle } from '../adapters/vitest/lifecycle.js';