@wix/pathgrade 1.0.7 → 1.0.9

package/README.md

@@ -312,7 +312,8 @@ const result = await agent.runConversation({
 Pathgrade exposes a few useful features that are easy to miss from the basic examples:
 
 - `createAgent({ skillDir, workspace })` stages a real skill and a fixture workspace into the sandbox, which is how Pathgrade's skill examples are evaluated.
-- `createAgent({ debug: true })` preserves the final workspace under `pathgrade-debug/<test-name>/`; when you use `runConversation()`, it also writes `run-snapshot.json`.
+- `createAgent({ debug: true })` preserves the final workspace under the backward-compatible `pathgrade-debug/<test-name>/` path; when you use `runConversation()`, it also writes `run-snapshot.json`.
+- `createAgent({ debug: { retainRuns: 5 } })` opts into managed run retention under `pathgrade-debug/runs/<run-id>/<test-name>/`. Use `pathgrade clean --debug`, `--keep=N`, and `--dry-run` to clean marked, inactive debug runs safely.
 - `evaluate.fromSnapshot(snapshotPath, scorers)` re-runs grading against a saved snapshot without re-running the agent.
 - `previewReactions(messages, reactions)` lets you inspect which scripted reactions would fire offline.
 - `conversationWindow` on agents and personas keeps long transcripts bounded with summarization instead of sending the full conversation every turn.

package/bin/pathgrade.js

@@ -1,2 +1,7 @@
 #!/usr/bin/env node
-import '../dist/pathgrade.js';
+import { runPathgradeCli } from '../dist/pathgrade.js';
+
+runPathgradeCli().catch(error => {
+  console.error(error);
+  process.exit(1);
+});

package/dist/commands/clean.d.ts

@@ -0,0 +1,8 @@
+import { type CleanDebugRunsResult } from '../providers/debug-runs.js';
+export interface CleanCommandOptions {
+    debug: boolean;
+    keep?: number;
+    dryRun?: boolean;
+}
+export declare function parseCleanArgs(args: string[]): CleanCommandOptions;
+export declare function runClean(cwd: string, options: CleanCommandOptions): Promise<CleanDebugRunsResult>;

package/dist/commands/clean.js

@@ -0,0 +1,94 @@
+import fs from 'fs-extra';
+import path from 'node:path';
+import { cleanDebugRuns, DEBUG_ROOT_MARKER, } from '../providers/debug-runs.js';
+function parseKeep(value) {
+    if (value === undefined || !/^\d+$/.test(value)) {
+        throw new Error('pathgrade clean: --keep must be a non-negative integer');
+    }
+    const parsed = Number(value);
+    if (!Number.isSafeInteger(parsed)) {
+        throw new Error('pathgrade clean: --keep must be a non-negative integer');
+    }
+    return parsed;
+}
+export function parseCleanArgs(args) {
+    let debug = false;
+    let dryRun = false;
+    let keep;
+    for (let index = 0; index < args.length; index++) {
+        const arg = args[index];
+        if (arg === '--debug') {
+            debug = true;
+            continue;
+        }
+        if (arg === '--dry-run') {
+            dryRun = true;
+            continue;
+        }
+        if (arg === '--keep') {
+            keep = parseKeep(args[++index]);
+            continue;
+        }
+        if (arg.startsWith('--keep=')) {
+            keep = parseKeep(arg.slice('--keep='.length));
+            continue;
+        }
+        throw new Error(`pathgrade clean: unknown option ${arg}`);
+    }
+    return { debug, ...(keep === undefined ? {} : { keep }), dryRun };
+}
+const SKIPPED_DIRECTORY_NAMES = new Set(['.git', '.worktrees', 'node_modules']);
+async function findDebugRoots(cwd) {
+    const roots = [];
+    async function visit(dir) {
+        let stat;
+        try {
+            stat = await fs.lstat(dir);
+        }
+        catch {
+            return;
+        }
+        if (!stat.isDirectory() || stat.isSymbolicLink())
+            return;
+        let entries;
+        try {
+            entries = await fs.readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        if (entries.some(entry => entry.isFile() && !entry.isSymbolicLink() && entry.name === DEBUG_ROOT_MARKER)) {
+            roots.push(dir);
+            return;
+        }
+        await Promise.all(entries.map(async (entry) => {
+            if (!entry.isDirectory() || entry.isSymbolicLink() || SKIPPED_DIRECTORY_NAMES.has(entry.name))
+                return;
+            await visit(path.join(dir, entry.name));
+        }));
+    }
+    await visit(path.resolve(cwd));
+    return roots;
+}
+export async function runClean(cwd, options) {
+    if (!options.debug) {
+        throw new Error('pathgrade clean requires --debug');
+    }
+    const roots = await findDebugRoots(cwd);
+    const results = await Promise.all(roots.map(rootDir => cleanDebugRuns({
+        rootDir,
+        keep: options.keep,
+        dryRun: options.dryRun,
+    })));
+    return results.reduce((total, result) => ({
+        removed: total.removed + result.removed,
+        retained: total.retained + result.retained,
+        active: total.active + result.active,
+        dryRun: total.dryRun,
+    }), {
+        removed: 0,
+        retained: 0,
+        active: 0,
+        dryRun: options.dryRun ?? false,
+    });
+}

package/dist/commands/init.d.ts

@@ -1,3 +1,5 @@
-export declare function runInit(dir: string, opts?: {
+export interface InitOptions {
     force?: boolean;
-}): Promise<void>;
+    commandName?: string;
+}
+export declare function runInit(dir: string, opts?: InitOptions): Promise<void>;

package/dist/commands/init.js

@@ -22,6 +22,11 @@ async function findExistingEvalFile(dir) {
     return null;
 }
 export async function runInit(dir, opts = {}) {
+    const commandName = opts.commandName?.trim() || 'pathgrade';
+    const skills = await detectSkills(dir);
+    const dirName = path.basename(dir);
+    const evalName = skills.length === 1 ? skills[0].name : dirName;
+    const evalPath = resolveEvalPath(dir, evalName);
     const existing = await findExistingEvalFile(dir);
     if (existing) {
         if (opts.force) {
@@ -33,16 +38,11 @@ export async function runInit(dir, opts = {}) {
             throw new Error(`${name} already exists`);
         }
     }
-    console.log('\npathgrade init\n');
-    // Detect skills
-    const skills = await detectSkills(dir);
-    // Derive eval filename: <skill-name>.eval.ts or <dirname>.eval.ts
-    const dirName = path.basename(dir);
+    console.log(`\n${commandName} init\n`);
     if (skills.length === 0) {
         console.log('  No SKILL.md found. Creating a generic template.');
         console.log('     Place a SKILL.md in this directory for better scaffolding.\n');
-        const evalPath = path.join(dir, `${dirName}.eval.ts`);
-        await writeTemplate(evalPath, 'my-skill', 'Describe what the agent should do with this skill.');
+        await writeTemplate(evalPath, 'my-skill', 'Describe what the agent should do with this skill.', commandName);
         return;
     }
     console.log(`  Found ${skills.length} skill(s): ${skills.map(s => s.name).join(', ')}\n`);
@@ -61,17 +61,15 @@ export async function runInit(dir, opts = {}) {
     const openaiKey = process.env.OPENAI_API_KEY;
     const hasApiKey = !!(anthropicKey || openaiKey);
     const cliAvailable = await isClaudeCliAvailable();
-    const evalName = skills.length === 1 ? skills[0].name : dirName;
-    const evalPath = path.join(dir, `${evalName}.eval.ts`);
     if (hasApiKey || cliAvailable) {
         const { Spinner, fmt } = await import('../utils/cli.js');
         const label = 'generating eval with available LLM backend';
         const spinner = new Spinner('init', label);
         try {
             const config = await generateWithLLM(skills);
-            await fs.writeFile(evalPath, config, 'utf-8');
+            await writeEvalFile(evalPath, config);
             spinner.stop(fmt.green(`created ${path.basename(evalPath)}`));
-            console.log(`     Review and edit the file, then run: pathgrade\n`);
+            console.log(`     Review and edit the file, then run: ${commandName}\n`);
             return;
         }
         catch (err) {
@@ -86,9 +84,9 @@ export async function runInit(dir, opts = {}) {
     const skill = skills[0];
     const taskName = `test-${skill.name}`;
     const instruction = extractInstructionHint(skill.skillMd);
-    await writeTemplate(evalPath, taskName, instruction);
+    await writeTemplate(evalPath, taskName, instruction, commandName);
 }
-async function writeTemplate(evalPath, taskName, instruction) {
+async function writeTemplate(evalPath, taskName, instruction, commandName) {
     const templatePath = path.join(import.meta.dirname, '..', '..', 'templates', 'eval.ts.template');
     let template;
     if (await fs.pathExists(templatePath)) {
@@ -101,9 +99,38 @@ async function writeTemplate(evalPath, taskName, instruction) {
     const result = template
         .replace(/\{\{TASK_NAME\}\}/g, taskName)
         .replace(/\{\{INSTRUCTION\}\}/g, instruction);
-    await fs.writeFile(evalPath, result, 'utf-8');
+    await writeEvalFile(evalPath, result);
     console.log(`  Created ${path.basename(evalPath)}.`);
-    console.log(`     Edit the file to define your eval tasks, then run: pathgrade\n`);
+    console.log(`     Edit the file to define your eval tasks, then run: ${commandName}\n`);
+}
+function resolveEvalPath(dir, evalName) {
+    const baseDir = path.resolve(dir);
+    if (!evalName
+        || evalName === '.'
+        || evalName === '..'
+        || path.basename(evalName) !== evalName
+        || evalName.includes('/')
+        || evalName.includes('\\')) {
+        throw new Error(`Invalid skill name for eval filename: ${JSON.stringify(evalName)}`);
+    }
+    const evalPath = path.resolve(baseDir, `${evalName}.eval.ts`);
+    if (path.dirname(evalPath) !== baseDir) {
+        throw new Error(`Eval filename escapes the target directory: ${JSON.stringify(evalName)}`);
+    }
+    return evalPath;
+}
+async function writeEvalFile(evalPath, content) {
+    try {
+        const stat = await fs.lstat(evalPath);
+        if (stat.isSymbolicLink()) {
+            throw new Error(`Refusing to write eval through symbolic link: ${path.basename(evalPath)}`);
+        }
+    }
+    catch (error) {
+        if (error.code !== 'ENOENT')
+            throw error;
+    }
+    await fs.writeFile(evalPath, content, 'utf-8');
 }
 /**
  * Extract a reasonable instruction hint from SKILL.md content.

package/dist/commands/run-changed.js

@@ -20,15 +20,13 @@ import { writeSidecar } from '../affected/sidecar.js';
 import { discoverPathgradeEvalFiles } from '../evals/discovery.js';
 import { resolvePathgradeConfig } from '../config/pathgrade.js';
 import { loadRunnerInvocationAdapter } from '../runners/adapter-loader.js';
+import { buildRunnerEnv } from './runner-env.js';
 export async function runChanged(opts) {
     const { cwd, parsed } = opts;
     const selectionInvocationId = randomUUID();
-    const runnerEnv = {
-        ...process.env,
-        ...(parsed.forceDiagnostics ? { PATHGRADE_DIAGNOSTICS: '1' } : {}),
-        ...(parsed.forceVerbose ? { PATHGRADE_VERBOSE: '1' } : {}),
+    const runnerEnv = buildRunnerEnv(parsed, {
         PATHGRADE_SELECTION_INVOCATION_ID: selectionInvocationId,
-    };
+    });
     const configPath = findVitestConfigArg(parsed.runnerArgs);
     let config;
     let runnerInvocation;

package/dist/commands/runner-env.d.ts

@@ -0,0 +1,2 @@
+import type { PathgradeRunArgs } from './run-args.js';
+export declare function buildRunnerEnv(parsed: Pick<PathgradeRunArgs, 'forceDiagnostics' | 'forceVerbose'>, additions?: Record<string, string>): NodeJS.ProcessEnv;

package/dist/commands/runner-env.js

@@ -0,0 +1,10 @@
+import { resolveDebugRunId } from '../providers/debug-runs.js';
+export function buildRunnerEnv(parsed, additions = {}) {
+    return {
+        ...process.env,
+        PATHGRADE_DEBUG_RUN_ID: resolveDebugRunId(),
+        ...(parsed.forceDiagnostics ? { PATHGRADE_DIAGNOSTICS: '1' } : {}),
+        ...(parsed.forceVerbose ? { PATHGRADE_VERBOSE: '1' } : {}),
+        ...additions,
+    };
+}

package/dist/commands/validate.js

@@ -15,6 +15,8 @@ import fs from 'fs-extra';
 import { findSkillRoot } from '../affected/anchor.js';
 import { parsePathgradeMeta } from '../affected/meta.js';
 import { discoverEvalFiles } from './affected.js';
+import * as ts from 'typescript';
+import { hasEvalSdkImport } from '../evals/sdk-import.js';
 /**
  * Run the validate command. Returns exit code (0 = valid, 1 = errors).
  */
@@ -40,8 +42,12 @@ export async function runValidate(filePath, opts = {}) {
         }
     }
     // Check 3: Imports
-    if (!content.includes("from '@wix/pathgrade'") && !content.includes('from "@wix/pathgrade"')) {
-        errors.push({ check: 'imports-pathgrade', message: "No import from '@wix/pathgrade' found." });
+    const sourceFile = ts.createSourceFile(filePath, content, ts.ScriptTarget.Latest, false, ts.ScriptKind.TS);
+    if (!hasEvalSdkImport(sourceFile)) {
+        errors.push({
+            check: 'imports-pathgrade',
+            message: 'No recognizable eval SDK import found.',
+        });
     }
     // Check 4: describe/it blocks
     if (!/describe\s*\(/.test(content) || !/it\s*\(/.test(content)) {

package/dist/evals/discovery.js

@@ -2,8 +2,8 @@ import * as fs from 'fs';
 import * as path from 'path';
 import picomatch from 'picomatch';
 import * as ts from 'typescript';
+import { hasEvalSdkImport } from './sdk-import.js';
 const EVAL_SUFFIX = '.eval.ts';
-const PATHGRADE_PACKAGE = '@wix/pathgrade';
 export function discoverPathgradeEvalFiles(opts) {
     const { cwd, include, exclude } = opts;
     const root = path.resolve(cwd);
@@ -48,16 +48,8 @@ export function isPathgradeEval(absPath) {
     const source = fs.readFileSync(absPath, 'utf8');
     const sourceFile = ts.createSourceFile(absPath, source, ts.ScriptTarget.Latest, 
     /* setParentNodes */ false, ts.ScriptKind.TS);
-    for (const stmt of sourceFile.statements) {
-        if (isPathgradeImport(stmt) || isPathgradeMetaExport(stmt))
-            return true;
-    }
-    return false;
-}
-function isPathgradeImport(stmt) {
-    if (!ts.isImportDeclaration(stmt))
-        return false;
-    return ts.isStringLiteral(stmt.moduleSpecifier) && stmt.moduleSpecifier.text === PATHGRADE_PACKAGE;
+    return hasEvalSdkImport(sourceFile)
+        || sourceFile.statements.some(isPathgradeMetaExport);
 }
 function isPathgradeMetaExport(stmt) {
     if (!ts.isVariableStatement(stmt))

package/dist/evals/sdk-import.d.ts

@@ -0,0 +1,3 @@
+import * as ts from 'typescript';
+/** Detects the Pathgrade SDK shape without depending on a module specifier. */
+export declare function hasEvalSdkImport(sourceFile: ts.SourceFile): boolean;

package/dist/evals/sdk-import.js

@@ -0,0 +1,76 @@
+import * as ts from 'typescript';
+const PATHGRADE_PACKAGE = '@wix/pathgrade';
+const SUPPORTING_SDK_IMPORTS = new Set([
+    'Agent',
+    'check',
+    'judge',
+    'score',
+    'toolUsage',
+]);
+/** Detects the Pathgrade SDK shape without depending on a module specifier. */
+export function hasEvalSdkImport(sourceFile) {
+    const sdkSignalsBySource = new Map();
+    const namespaceSources = new Map();
+    for (const stmt of sourceFile.statements) {
+        if (!ts.isImportDeclaration(stmt))
+            continue;
+        if (ts.isStringLiteral(stmt.moduleSpecifier)
+            && stmt.moduleSpecifier.text === PATHGRADE_PACKAGE)
+            return true;
+        const bindings = stmt.importClause?.namedBindings;
+        const importSource = ts.isStringLiteral(stmt.moduleSpecifier)
+            ? stmt.moduleSpecifier.text
+            : stmt.moduleSpecifier.getText(sourceFile);
+        if (bindings && ts.isNamespaceImport(bindings)) {
+            namespaceSources.set(bindings.name.text, importSource);
+            continue;
+        }
+        if (bindings && ts.isNamedImports(bindings)) {
+            const importedNames = sdkSignalsBySource.get(importSource) ?? new Set();
+            for (const element of bindings.elements) {
+                importedNames.add(element.propertyName?.text ?? element.name.text);
+            }
+            sdkSignalsBySource.set(importSource, importedNames);
+        }
+    }
+    function addNamespaceMember(namespace, member) {
+        const importSource = namespaceSources.get(namespace);
+        if (!importSource)
+            return;
+        const members = sdkSignalsBySource.get(importSource) ?? new Set();
+        members.add(member);
+        sdkSignalsBySource.set(importSource, members);
+    }
+    function visit(node) {
+        if (ts.isPropertyAccessExpression(node)
+            && ts.isIdentifier(node.expression)
+            && namespaceSources.has(node.expression.text)) {
+            addNamespaceMember(node.expression.text, node.name.text);
+        }
+        if (ts.isQualifiedName(node)
+            && ts.isIdentifier(node.left)
+            && namespaceSources.has(node.left.text)) {
+            addNamespaceMember(node.left.text, node.right.text);
+        }
+        if (ts.isVariableDeclaration(node)
+            && ts.isObjectBindingPattern(node.name)
+            && node.initializer
+            && ts.isIdentifier(node.initializer)
+            && namespaceSources.has(node.initializer.text)) {
+            for (const element of node.name.elements) {
+                const importedName = element.propertyName ?? element.name;
+                if (ts.isIdentifier(importedName)) {
+                    addNamespaceMember(node.initializer.text, importedName.text);
+                }
+            }
+        }
+        ts.forEachChild(node, visit);
+    }
+    visit(sourceFile);
+    return [...sdkSignalsBySource.values()].some(matchesSdkShape);
+}
+function matchesSdkShape(names) {
+    return names.has('evaluate')
+        && (names.has('createAgent')
+            || [...SUPPORTING_SDK_IMPORTS].some(name => names.has(name)));
+}

package/dist/pathgrade.d.ts

@@ -7,4 +7,8 @@
  *   pathgrade init [--force]         Generate eval scaffolding
  *   pathgrade preview [browser]      View results (CLI default, or browser)
  */
-export {};
+export interface PathgradeCliOptions {
+    name?: string;
+    version?: string;
+}
+export declare function runPathgradeCli(options?: PathgradeCliOptions): Promise<void>;

package/dist/pathgrade.js

@@ -9,6 +9,7 @@
  */
 import * as fs from 'fs';
 import * as path from 'path';
+import { fileURLToPath } from 'node:url';
 import { runInit } from './commands/init.js';
 import { runAnalyze } from './commands/analyze.js';
 import { runValidate, runValidateAffected } from './commands/validate.js';
@@ -18,11 +19,13 @@ import { runPreviewReactions } from './commands/preview-reactions.js';
 import { runReport } from './commands/report.js';
 import { runAffected } from './commands/affected.js';
 import { runChanged } from './commands/run-changed.js';
+import { parseCleanArgs, runClean } from './commands/clean.js';
 import { clearSidecar } from './affected/sidecar.js';
 import { resolvePathgradeConfig } from './config/pathgrade.js';
 import { loadRunnerInvocationAdapter } from './runners/adapter-loader.js';
 import { fmt } from './utils/cli.js';
 import { shutdown } from './utils/shutdown.js';
+import { buildRunnerEnv } from './commands/runner-env.js';
 function loadDotenv() {
     const envPath = path.resolve(process.cwd(), '.env');
     if (!fs.existsSync(envPath))
@@ -56,15 +59,20 @@ function validateApiKeys() {
             `  ${fmt.dim('         Claude CLI auth (keychain) and Codex exec cached login (~/.codex/auth.json) may still work if installed.')}\n`);
     }
 }
-async function main() {
+export async function runPathgradeCli(options = {}) {
+    const cliName = options.name?.trim() || 'pathgrade';
     shutdown.install();
     const args = process.argv.slice(2);
     const command = args[0];
     if (command === '--help' || command === '-h') {
-        printHelp();
+        printHelp(cliName);
         return;
     }
     if (command === '--version' || command === '-v') {
+        if (options.version) {
+            console.log(options.version);
+            return;
+        }
         const pkg = JSON.parse(await import('fs').then(fs => fs.promises.readFile(new URL('../package.json', import.meta.url), 'utf-8')));
         console.log(pkg.version);
         return;
@@ -87,7 +95,7 @@ async function main() {
         }
         const filePath = validateArgs[0];
         if (!filePath) {
-            console.error('Usage: pathgrade validate <file.eval.ts>  |  pathgrade validate --affected');
+            console.error(`Usage: ${cliName} validate <file.eval.ts>  |  ${cliName} validate --affected`);
             process.exitCode = 1;
             return;
         }
@@ -97,7 +105,17 @@ async function main() {
     }
     if (command === 'init') {
         const hasForce = args.includes('--force');
-        await runInit(process.cwd(), { force: hasForce });
+        await runInit(process.cwd(), {
+            force: hasForce,
+            commandName: cliName,
+        });
+        return;
+    }
+    if (command === 'clean') {
+        const result = await runClean(process.cwd(), parseCleanArgs(args.slice(1)));
+        const action = result.dryRun ? 'would remove' : 'removed';
+        console.log(`${cliName}: ${action} ${result.removed} debug run(s); ` +
+            `retained ${result.retained}; active ${result.active}`);
         return;
     }
     if (command === 'preview') {
@@ -150,7 +168,7 @@ async function main() {
         validateApiKeys();
         const parsed = parsePathgradeRunArgs(command === 'run' ? args.slice(1) : args);
         for (const warning of parsed.warnings ?? []) {
-            console.error(`pathgrade: ${warning}`);
+            console.error(`${cliName}: ${warning}`);
         }
         if (parsed.changed) {
             const exitCode = await runChanged({
@@ -164,11 +182,7 @@ async function main() {
         // previous `--changed` run so it doesn't leak into this full-suite
         // run (the reporter would otherwise merge old metadata).
         await clearSidecar(process.cwd());
-        const env = {
-            ...process.env,
-            ...(parsed.forceDiagnostics ? { PATHGRADE_DIAGNOSTICS: '1' } : {}),
-            ...(parsed.forceVerbose ? { PATHGRADE_VERBOSE: '1' } : {}),
-        };
+        const env = buildRunnerEnv(parsed);
         try {
             const config = await resolvePathgradeConfig({ cwd: process.cwd() });
             const runner = await loadRunnerInvocationAdapter({
@@ -189,15 +203,15 @@ async function main() {
         return;
     }
     console.error(`Unknown command: ${command}`);
-    console.error('Run "pathgrade --help" for usage.');
+    console.error(`Run "${cliName} --help" for usage.`);
     process.exitCode = 1;
 }
-function printHelp() {
+function printHelp(cliName) {
     console.log(`
-  pathgrade - Evaluate AI agent skills with a runner adapter
+  ${cliName} - Evaluate AI agent skills with a runner adapter
 
   Usage:
-    pathgrade run [-- runner-args]   Run evals (loads .env, delegates to the selected adapter)
+    ${cliName} run [-- runner-args]   Run evals (loads .env, delegates to the selected adapter)
                      [--changed]               Run only evals affected by the current PR/change-set
                      [--since=<ref>]           Override base ref (implies git mode)
                      [--changed-files=<path>]  Use an explicit newline-delimited file list
@@ -205,19 +219,22 @@ function printHelp() {
                      [--diagnostics]           Print full diagnostics for passing evals too
                      [--quiet]                 Suppress the run-start summary
                      [--verbose|-v]            Stream live per-turn events to stderr during the run
-    pathgrade init [--force]         Generate eval scaffolding
-    pathgrade analyze [--skill=X]    Analyze skills and output JSON
-    pathgrade validate <file>        Validate an .eval.ts file
-    pathgrade validate --affected    Strict: every eval must be anchored or have valid __pathgradeMeta
-    pathgrade preview [browser]      View results (CLI default, or browser)
+    ${cliName} init [--force]         Generate eval scaffolding
+    ${cliName} analyze [--skill=X]    Analyze skills and output JSON
+    ${cliName} validate <file>        Validate an .eval.ts file
+    ${cliName} validate --affected    Strict: every eval must be anchored or have valid __pathgradeMeta
+    ${cliName} clean --debug          Remove completed debug runs
+                     [--keep=N]      Keep the N newest completed debug runs
+                     [--dry-run]     Report removals without changing files
+    ${cliName} preview [browser]      View results (CLI default, or browser)
                      [--last=N]     Show only the N most recent reports
                      [--filter=X]   Filter reports by test name (substring)
-    pathgrade preview-reactions      Preview reactions against a snapshot
-    pathgrade report                 Format .pathgrade/results.json as a markdown PR comment
+    ${cliName} preview-reactions      Preview reactions against a snapshot
+    ${cliName} report                 Format .pathgrade/results.json as a markdown PR comment
                      [--results-path=<path>]   Override results.json location
                      [--no-comment]            Print markdown to stdout; do not post
                      [--comment-id=<id>]       Override comment marker (default: $GITHUB_WORKFLOW:$GITHUB_JOB)
-    pathgrade affected               Print eval files affected by a change-set (one per line)
+    ${cliName} affected               Print eval files affected by a change-set (one per line)
                      [--since=<ref>]           Diff <ref>...HEAD (overrides git auto-detection)
                      [--changed-files=<path>]  Newline-delimited repo-relative file list
                      [--explain]               Print human-readable per-eval decision to stderr
@@ -229,16 +246,19 @@ function printHelp() {
     OPENAI_API_KEY                   API key for Codex and OpenAI-backed judges
 
   Examples:
-    pathgrade run                    # run all *.eval.ts files
-    pathgrade run --diagnostics      # print full diagnostics for passing evals too
-    pathgrade run --verbose          # stream per-turn events live to stderr while evals run
-    pathgrade run -- --grep superlint   # filter by test name
-    pathgrade init                   # scaffold eval files
-    pathgrade preview browser        # open web UI
-    pathgrade preview-reactions --snapshot ./pathgrade-debug/run-snapshot.json --reactions ./reactions.ts
+    ${cliName} run                    # run all *.eval.ts files
+    ${cliName} run --diagnostics      # print full diagnostics for passing evals too
+    ${cliName} run --verbose          # stream per-turn events live to stderr while evals run
+    ${cliName} run -- --grep superlint   # filter by test name
+    ${cliName} init                   # scaffold eval files
+    ${cliName} preview browser        # open web UI
+    ${cliName} preview-reactions --snapshot ./pathgrade-debug/run-snapshot.json --reactions ./reactions.ts
 `);
 }
-main().catch(err => {
-    console.error(err);
-    process.exit(1);
-});
+if (process.argv[1]
+    && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+    runPathgradeCli().catch(err => {
+        console.error(err);
+        process.exit(1);
+    });
+}

package/dist/providers/debug-runs.d.ts

@@ -0,0 +1,23 @@
+export declare const DEBUG_ROOT_MARKER = ".pathgrade-debug-root.json";
+export declare const DEBUG_RUN_MARKER = ".pathgrade-debug-run.json";
+export declare const DEFAULT_DEBUG_RETAIN_RUNS = 3;
+export declare function resolveDebugRunId(): string;
+export interface CleanDebugRunsResult {
+    removed: number;
+    retained: number;
+    active: number;
+    dryRun: boolean;
+}
+export declare function prepareManagedDebugRun(input: {
+    rootDir: string;
+    debugName: string;
+}): Promise<{
+    destination: string;
+    rootDir: string;
+    runId: string;
+}>;
+export declare function cleanDebugRuns(input: {
+    rootDir: string;
+    keep?: number;
+    dryRun?: boolean;
+}): Promise<CleanDebugRunsResult>;

package/dist/providers/debug-runs.js

@@ -0,0 +1,208 @@
+import fs from 'fs-extra';
+import { randomUUID } from 'node:crypto';
+import path from 'node:path';
+export const DEBUG_ROOT_MARKER = '.pathgrade-debug-root.json';
+export const DEBUG_RUN_MARKER = '.pathgrade-debug-run.json';
+export const DEFAULT_DEBUG_RETAIN_RUNS = 3;
+let generatedRunId;
+function createRunId() {
+    generatedRunId ??= `${new Date().toISOString().replace(/[:.]/g, '-')}-${Math.random().toString(36).slice(2, 8)}`;
+    return generatedRunId;
+}
+export function resolveDebugRunId() {
+    const configured = process.env.PATHGRADE_DEBUG_RUN_ID;
+    if (configured && /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(configured))
+        return configured;
+    return createRunId();
+}
+async function readJsonFile(filePath) {
+    try {
+        const stat = await fs.lstat(filePath);
+        if (!stat.isFile() || stat.isSymbolicLink())
+            return undefined;
+        return await fs.readJson(filePath);
+    }
+    catch {
+        return undefined;
+    }
+}
+async function isOwnedDebugRoot(rootDir) {
+    try {
+        const stat = await fs.lstat(rootDir);
+        if (!stat.isDirectory() || stat.isSymbolicLink())
+            return false;
+    }
+    catch {
+        return false;
+    }
+    const marker = await readJsonFile(path.join(rootDir, DEBUG_ROOT_MARKER));
+    return marker?.version === 1;
+}
+async function ensureOwnedDebugRoot(rootDir) {
+    await fs.ensureDir(rootDir);
+    const rootStat = await fs.lstat(rootDir);
+    if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) {
+        throw new Error(`Pathgrade debug root must be a real directory: ${rootDir}`);
+    }
+    const markerPath = path.join(rootDir, DEBUG_ROOT_MARKER);
+    let markerExists = false;
+    try {
+        const markerStat = await fs.lstat(markerPath);
+        markerExists = true;
+        if (!markerStat.isFile() || markerStat.isSymbolicLink()) {
+            throw new Error(`Pathgrade debug root has an unsafe ownership marker: ${rootDir}`);
+        }
+    }
+    catch (error) {
+        if (error.code !== 'ENOENT')
+            throw error;
+    }
+    if (!markerExists) {
+        await fs.writeJson(markerPath, { version: 1 }, { spaces: 2 });
+        return;
+    }
+    const existing = await readJsonFile(markerPath);
+    if (existing?.version !== 1) {
+        throw new Error(`Pathgrade debug root has an unsupported ownership marker: ${rootDir}`);
+    }
+}
+async function isRealDirectory(directory) {
+    try {
+        const stat = await fs.lstat(directory);
+        return stat.isDirectory() && !stat.isSymbolicLink();
+    }
+    catch {
+        return false;
+    }
+}
+async function ensureRealChildDirectory(directory) {
+    try {
+        await fs.mkdir(directory);
+    }
+    catch (error) {
+        if (error.code !== 'EEXIST')
+            throw error;
+    }
+    if (!await isRealDirectory(directory)) {
+        throw new Error(`Pathgrade managed debug path must be a real directory: ${directory}`);
+    }
+}
+async function ensureRunMarker(runDir, runId) {
+    const markerPath = path.join(runDir, DEBUG_RUN_MARKER);
+    const temporaryMarker = `${markerPath}.${process.pid}.${randomUUID()}.tmp`;
+    await fs.writeJson(temporaryMarker, {
+        version: 1,
+        runId,
+        createdAt: new Date().toISOString(),
+    }, { spaces: 2 });
+    try {
+        await fs.link(temporaryMarker, markerPath);
+    }
+    catch (error) {
+        if (error.code !== 'EEXIST')
+            throw error;
+    }
+    finally {
+        await fs.remove(temporaryMarker);
+    }
+    const marker = await readJsonFile(markerPath);
+    if (marker?.version !== 1 ||
+        marker.runId !== runId ||
+        typeof marker.createdAt !== 'string' ||
+        !Number.isFinite(Date.parse(marker.createdAt))) {
+        throw new Error(`Pathgrade debug run has an unsafe ownership marker: ${runDir}`);
+    }
+}
+export async function prepareManagedDebugRun(input) {
+    const rootDir = path.resolve(input.rootDir);
+    await ensureOwnedDebugRoot(rootDir);
+    const runId = resolveDebugRunId();
+    const runsDir = path.join(rootDir, 'runs');
+    const runDir = path.join(runsDir, runId);
+    const activeDir = path.join(runDir, '.pathgrade-active');
+    await ensureRealChildDirectory(runsDir);
+    await ensureRealChildDirectory(runDir);
+    await ensureRealChildDirectory(activeDir);
+    await fs.writeFile(path.join(activeDir, String(process.pid)), '');
+    await ensureRunMarker(runDir, runId);
+    return {
+        destination: input.debugName ? path.join(runDir, input.debugName) : runDir,
+        rootDir,
+        runId,
+    };
+}
+async function readOwnedRun(runDir) {
+    const marker = await readJsonFile(path.join(runDir, DEBUG_RUN_MARKER));
+    if (marker?.version !== 1 ||
+        typeof marker.runId !== 'string' ||
+        typeof marker.createdAt !== 'string' ||
+        !Number.isFinite(Date.parse(marker.createdAt))) {
+        return undefined;
+    }
+    if (path.basename(runDir) !== marker.runId)
+        return undefined;
+    return marker;
+}
+function isProcessAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code !== 'ESRCH';
+    }
+}
+async function isRunActive(runDir) {
+    const activeDir = path.join(runDir, '.pathgrade-active');
+    const entries = await fs.readdir(activeDir, { withFileTypes: true }).catch(() => []);
+    return entries.some(entry => {
+        if (!entry.isFile() || entry.isSymbolicLink() || !/^\d+$/.test(entry.name))
+            return false;
+        const pid = Number(entry.name);
+        return Number.isSafeInteger(pid) && pid > 0 && isProcessAlive(pid);
+    });
+}
+export async function cleanDebugRuns(input) {
+    const dryRun = input.dryRun ?? false;
+    if (!await isOwnedDebugRoot(input.rootDir)) {
+        return { removed: 0, retained: 0, active: 0, dryRun };
+    }
+    const runsDir = path.join(input.rootDir, 'runs');
+    if (!await isRealDirectory(runsDir)) {
+        return { removed: 0, retained: 0, active: 0, dryRun };
+    }
+    const entries = await fs.readdir(runsDir, { withFileTypes: true }).catch(() => []);
+    const ownedRuns = [];
+    for (const entry of entries) {
+        if (!entry.isDirectory() || entry.isSymbolicLink())
+            continue;
+        const dir = path.join(runsDir, entry.name);
+        const marker = await readOwnedRun(dir);
+        if (marker)
+            ownedRuns.push({ dir, marker, active: await isRunActive(dir) });
+    }
+    const activeRuns = ownedRuns.filter(run => run.active);
+    const completedRuns = ownedRuns.filter(run => !run.active);
+    completedRuns.sort((a, b) => b.marker.createdAt.localeCompare(a.marker.createdAt));
+    const keep = input.keep ?? 0;
+    const retained = completedRuns.slice(0, keep);
+    const removable = completedRuns.slice(keep);
+    let removed = removable.length;
+    let newlyActive = 0;
+    if (!dryRun) {
+        const removalResults = await Promise.all(removable.map(async (run) => {
+            if (await isRunActive(run.dir))
+                return false;
+            await fs.remove(run.dir);
+            return true;
+        }));
+        removed = removalResults.filter(Boolean).length;
+        newlyActive = removalResults.length - removed;
+    }
+    return {
+        removed,
+        retained: retained.length,
+        active: activeRuns.length + newlyActive,
+        dryRun,
+    };
+}

package/dist/sdk/agent.js

@@ -14,6 +14,7 @@ import { getCurrentCaseContext } from './case-context.js';
 import { createVerboseEmitter } from '../reporters/verbose-emitter.js';
 import fs from 'fs-extra';
 import * as path from 'path';
+import { cleanDebugRuns, DEFAULT_DEBUG_RETAIN_RUNS, prepareManagedDebugRun, } from '../providers/debug-runs.js';
 import { collectOpenCodeMcpToolNames, validateOpenCodeDeclaration } from '../agents/opencode-contract.js';
 /**
  * Test-only injection point: override the sink used by the next emitter
@@ -319,26 +320,46 @@ class AgentImpl {
         // Runner-owned agents stay tracked until flush consumes metadata.
         // Manual agents have no runner flush, so dispose releases them.
         lifecycleCore.releaseAgent(this);
-        await this.activeChatSession?.dispose?.();
-        this.activeChatSession = undefined;
-        if (this.debugOpt) {
-            const dest = typeof this.debugOpt === 'string'
-                ? this.debugOpt
-                : path.join(this.debugBaseDir, 'pathgrade-debug', this.debugName);
-            await fs.remove(dest);
-            await fs.copy(this.ws.path, dest);
-            if (this.interactionMode === 'runConversation' && this.lastConversationResult) {
-                const snapshot = buildRunSnapshot({
-                    agent: this.agentName,
-                    messages: this._messages,
-                    log: this._log,
-                    conversationResult: this.lastConversationResult,
-                    workspace: dest,
-                });
-                await fs.writeJSON(path.join(dest, 'run-snapshot.json'), snapshot, { spaces: 2 });
+        try {
+            await this.activeChatSession?.dispose?.();
+            this.activeChatSession = undefined;
+            if (this.debugOpt) {
+                const managedOptions = typeof this.debugOpt === 'object' ? this.debugOpt : undefined;
+                const managed = managedOptions
+                    ? await prepareManagedDebugRun({
+                        rootDir: managedOptions.directory
+                            ? path.resolve(this.debugBaseDir, managedOptions.directory)
+                            : path.join(this.debugBaseDir, 'pathgrade-debug'),
+                        debugName: this.debugName,
+                    })
+                    : undefined;
+                const dest = typeof this.debugOpt === 'string'
+                    ? this.debugOpt
+                    : managed?.destination ?? path.join(this.debugBaseDir, 'pathgrade-debug', this.debugName);
+                await fs.remove(dest);
+                await fs.copy(this.ws.path, dest);
+                if (this.interactionMode === 'runConversation' && this.lastConversationResult) {
+                    const snapshot = buildRunSnapshot({
+                        agent: this.agentName,
+                        messages: this._messages,
+                        log: this._log,
+                        conversationResult: this.lastConversationResult,
+                        workspace: dest,
+                    });
+                    await fs.writeJSON(path.join(dest, 'run-snapshot.json'), snapshot, { spaces: 2 });
+                }
+                if (managed) {
+                    const retainRuns = managedOptions.retainRuns ?? DEFAULT_DEBUG_RETAIN_RUNS;
+                    await cleanDebugRuns({
+                        rootDir: managed.rootDir,
+                        keep: Math.max(0, retainRuns - 1),
+                    });
+                }
             }
         }
-        await this.ws.dispose();
+        finally {
+            await this.ws.dispose();
+        }
     }
 }
 /**
@@ -364,6 +385,12 @@ function resolveCaseDebugContext() {
     };
 }
 export async function createAgent(opts) {
+    if (typeof opts.debug === 'object') {
+        const retainRuns = opts.debug.retainRuns ?? DEFAULT_DEBUG_RETAIN_RUNS;
+        if (!Number.isSafeInteger(retainRuns) || retainRuns < 1) {
+            throw new Error('Pathgrade debug retainRuns must be a positive integer');
+        }
+    }
     const agentName = resolveAgentName(opts, process.env);
     validateOpenCodeDeclaration(agentName, opts);
     const transport = agentName === 'codex'

package/dist/sdk/index.d.ts

@@ -21,7 +21,7 @@ export { emitEvalResult, resetAllResultObserversForTests, resetUserResultObserve
 export { getAgentCapabilities } from './types.js';
 export type { AgentTransport, AgentCapabilities, AgentName, McpRunMode, McpSafetyOptions, McpToolPolicy, McpToolPolicyRule, } from './types.js';
 export type { AskBus, AskBatch, AskQuestion, AskOption, AskAnswer, AskResolution, AskBatchSnapshot, AskAnswerSnapshot, AskResolutionSnapshot, AskHandle, AskHandler, AskSource, AskLifecycle, AskAnswerSource, Unsubscribe as AskBusUnsubscribe, } from './ask-bus/types.js';
-export type { Agent, AgentOptions, Message, Scorer, CheckScorer, ScoreScorer, JudgeScorer, ToolUsageScorer, ScorerContext, EvalResult, ScorerResultEntry, ScorerStatus, ChatSession, ConversationResult, ConverseOptions, UntilPredicate, UntilContext, Reaction, TextReaction, AskUserReaction, AskUserQuestion, AskUserOption, ReactionPreviewEntry, TextReactionPreviewEntry, AskUserReactionPreviewEntry, ReactionPreviewResult, ReactionPreviewTurn, StepScorer, Persona, PersonaConfig, ConversationWindowConfig, TurnDetail, ReactionFiredEntry, PathgradePluginOptions, PathgradeMeta, TurnTiming, TokenUsage, EvaluateOptions, ReactionPreviewStatus, ScoreResult, JudgeInput, CodeJudgeToolName, ToolExpectation, SessionArtifactMatchOptions, SessionArtifactContent, SessionArtifacts, RecordedEvalResult, PathgradeTestMeta, EvaluationResultKind, AgentExecutionMetadata, AgentExecutionTransport, AgentInteractionMode, } from './types.js';
+export type { Agent, AgentOptions, DebugOptions, Message, Scorer, CheckScorer, ScoreScorer, JudgeScorer, ToolUsageScorer, ScorerContext, EvalResult, ScorerResultEntry, ScorerStatus, ChatSession, ConversationResult, ConverseOptions, UntilPredicate, UntilContext, Reaction, TextReaction, AskUserReaction, AskUserQuestion, AskUserOption, ReactionPreviewEntry, TextReactionPreviewEntry, AskUserReactionPreviewEntry, ReactionPreviewResult, ReactionPreviewTurn, StepScorer, Persona, PersonaConfig, ConversationWindowConfig, TurnDetail, ReactionFiredEntry, PathgradePluginOptions, PathgradeMeta, TurnTiming, TokenUsage, EvaluateOptions, ReactionPreviewStatus, ScoreResult, JudgeInput, CodeJudgeToolName, ToolExpectation, SessionArtifactMatchOptions, SessionArtifactContent, SessionArtifacts, RecordedEvalResult, PathgradeTestMeta, EvaluationResultKind, AgentExecutionMetadata, AgentExecutionTransport, AgentInteractionMode, } from './types.js';
 export type { ConversationWindow, ConversationWindowOptions } from './conversation-window.js';
 export type { JudgePipelineOptions } from './judge-pipeline.js';
 export type { RunScorerOptions } from './run-scorer.js';

package/dist/sdk/types.d.ts

@@ -31,8 +31,8 @@ export interface AgentOptions {
     mcpConfigFile?: string;
     /** Configure the conversation window for transcript-based agents. Set false to disable. */
     conversationWindow?: ConversationWindowConfig | false;
-    /** Copy workspace to a persistent location before cleanup. true = ./pathgrade-debug/{test-name}/, string = custom path. */
-    debug?: boolean | string;
+    /** Preserve the workspace. true uses the legacy path; strings are exact paths; objects enable managed retention. */
+    debug?: boolean | string | DebugOptions;
     /**
      * Glob patterns to ignore when copying workspace and skill directories.
      * Replaces the default ignore list entirely. Pass `[]` to disable filtering.
@@ -53,6 +53,12 @@ export interface AgentOptions {
      */
     mcpSafety?: McpSafetyOptions;
 }
+export interface DebugOptions {
+    /** Managed debug root. Relative paths resolve next to the eval file. */
+    directory?: string;
+    /** Maximum managed runs to retain, including the current run. Default: 3. */
+    retainRuns?: number;
+}
 export type { McpRunMode, McpSafetyOptions, McpToolPolicy, McpToolPolicyRule, } from './mcp-safety.js';
 export interface ConversationWindowConfig {
     /** Number of recent messages to keep verbatim. Default: 4 */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/pathgrade",
-  "version": "1.0.7",
+  "version": "1.0.9",
   "packageManager": "yarn@4.12.0",
   "description": "Evaluate whether AI agents discover and use your skills correctly",
   "exports": {
@@ -44,6 +44,10 @@
       "types": "./dist/core/mcp-mock.d.ts",
       "default": "./dist/core/mcp-mock.js"
     },
+    "./cli": {
+      "types": "./dist/pathgrade.d.ts",
+      "default": "./dist/pathgrade.js"
+    },
     "./package.json": "./package.json"
   },
   "bin": "bin/pathgrade.js",
@@ -128,5 +132,5 @@
     "typescript": "^5.9.3",
     "zod": "4.3.6"
   },
-  "falconPackageHash": "6dec3f88035544895dd8e99741e3c9a69e73651415b4999e32ddefd6"
+  "falconPackageHash": "130e5449f0131fe2bba40bf0237a7620315714dfe492f7e5e9d7cc14"
 }