@wix/pathgrade 0.29.0

package/dist/commands/validate.js

@@ -0,0 +1,233 @@
+/**
+ * `pathgrade validate` command.
+ *
+ * Validates a .eval.ts file for structural correctness without running it.
+ * Uses regex/string pattern matching — no AST parsing, no LLM calls.
+ *
+ * `runValidateAffected` (Issue 13) is a strict mode: instead of validating
+ * one positional file, it iterates every discovered eval and reports
+ * onMissing (no SKILL.md + no meta) and malformed-meta as hard errors,
+ * for teams that want CI-level declaration enforcement.
+ */
+import { createRequire } from 'node:module';
+import * as path from 'path';
+import fs from 'fs-extra';
+import { findSkillRoot } from '../affected/anchor.js';
+import { parsePathgradeMeta } from '../affected/meta.js';
+import { discoverEvalFiles } from './affected.js';
+/**
+ * Run the validate command. Returns exit code (0 = valid, 1 = errors).
+ */
+export async function runValidate(filePath, opts = {}) {
+    const errors = [];
+    const warnings = [];
+    // Check 1: File exists
+    if (!await fs.pathExists(filePath)) {
+        errors.push({ check: 'file-exists', message: `File not found: ${filePath}` });
+        return writeResult({ valid: false, errors, warnings });
+    }
+    // Check 1b: File extension
+    if (!filePath.endsWith('.eval.ts')) {
+        errors.push({ check: 'file-extension', message: `File must have .eval.ts extension, got: ${filePath}` });
+        return writeResult({ valid: false, errors, warnings });
+    }
+    const content = await fs.readFile(filePath, 'utf-8');
+    // Check 2: TypeScript syntax (optional, skip in tests)
+    if (!opts.skipTsc) {
+        const tscResult = await runTsc(filePath);
+        if (!tscResult.ok) {
+            errors.push({ check: 'typescript-compiles', message: tscResult.message });
+        }
+    }
+    // Check 3: Imports
+    if (!content.includes("from '@wix/pathgrade'") && !content.includes('from "@wix/pathgrade"')) {
+        errors.push({ check: 'imports-pathgrade', message: "No import from '@wix/pathgrade' found." });
+    }
+    // Check 4: describe/it blocks
+    if (!/describe\s*\(/.test(content) || !/it\s*\(/.test(content)) {
+        errors.push({ check: 'has-describe-it', message: 'No describe() or it() blocks found. Eval must use Vitest structure.' });
+    }
+    // Check 5: createAgent
+    if (!/createAgent\s*\(/.test(content)) {
+        errors.push({ check: 'has-create-agent', message: 'No createAgent() call found.' });
+    }
+    // Check 6: evaluate
+    if (!/evaluate\s*\(/.test(content)) {
+        errors.push({ check: 'has-evaluate', message: 'No evaluate() call found.' });
+    }
+    // Check 7: Deterministic scorer
+    if (!/\bcheck\s*\(/.test(content) && !/\bscore\s*\(/.test(content)) {
+        errors.push({ check: 'has-deterministic-scorer', message: 'No check() or score() scorer found. Add at least one deterministic scorer.' });
+    }
+    // Check 8: Scorer names non-empty
+    const scorerCallPattern = /\b(?:check|score|judge|toolUsage)\s*\(\s*(['"])(.*?)\1/g;
+    let scorerMatch;
+    while ((scorerMatch = scorerCallPattern.exec(content)) !== null) {
+        if (scorerMatch[2].trim() === '') {
+            errors.push({ check: 'scorer-names-non-empty', message: `Scorer has empty name at position ${scorerMatch.index}.` });
+            break; // One error is enough
+        }
+    }
+    // Check 9: Instruction non-empty (>10 chars)
+    const instructionPattern = /(?:prompt|startChat)\s*\(\s*(['"`])([\s\S]*?)\1/g;
+    let instrMatch;
+    let foundInstruction = false;
+    while ((instrMatch = instructionPattern.exec(content)) !== null) {
+        foundInstruction = true;
+        if (instrMatch[2].trim().length <= 10) {
+            errors.push({ check: 'instruction-non-empty', message: `Instruction is too short (${instrMatch[2].trim().length} chars). Write a meaningful instruction (>10 chars).` });
+            break;
+        }
+    }
+    // Also check template literal instructions
+    if (!foundInstruction) {
+        const templateInstrPattern = /(?:prompt|startChat)\s*\(\s*`([\s\S]*?)`/g;
+        let tmplMatch;
+        while ((tmplMatch = templateInstrPattern.exec(content)) !== null) {
+            foundInstruction = true;
+            if (tmplMatch[1].trim().length <= 10) {
+                errors.push({ check: 'instruction-non-empty', message: `Instruction is too short (${tmplMatch[1].trim().length} chars). Write a meaningful instruction (>10 chars).` });
+                break;
+            }
+        }
+    }
+    // Check 10: Filename consistency
+    const filenameWarnings = checkFilenameConsistency(content);
+    warnings.push(...filenameWarnings);
+    const valid = errors.length === 0;
+    return writeResult({ valid, errors, warnings });
+}
+function checkFilenameConsistency(content) {
+    const warnings = [];
+    // Extract all instruction text from prompt()/startChat() calls
+    const instrTexts = [];
+    // Match single/double quoted strings
+    const instrQuoted = /(?:prompt|startChat)\s*\(\s*(['"])([\s\S]*?)\1/g;
+    let m;
+    while ((m = instrQuoted.exec(content)) !== null) {
+        instrTexts.push(m[2]);
+    }
+    // Match template literals
+    const instrTemplate = /(?:prompt|startChat)\s*\(\s*`([\s\S]*?)`/g;
+    while ((m = instrTemplate.exec(content)) !== null) {
+        instrTexts.push(m[1]);
+    }
+    const instructionText = instrTexts.join(' ');
+    if (!instructionText)
+        return warnings;
+    // Find all check() call regions by matching check( ... the callback body
+    // Strategy: find each check( occurrence, then scan forward for filenames
+    const checkCallPattern = /\bcheck\s*\(/g;
+    while ((m = checkCallPattern.exec(content)) !== null) {
+        // Extract a reasonable window after the check( — up to 500 chars covers most callbacks
+        const window = content.substring(m.index, m.index + 500);
+        const filenamePattern = /['"`](?:.*\/)?([a-zA-Z0-9_.-]+\.[a-zA-Z]{1,5})['"`]/g;
+        let fm;
+        while ((fm = filenamePattern.exec(window)) !== null) {
+            const filename = fm[1];
+            // Skip the scorer name (first string after check(), and common non-file patterns
+            if (fm.index < 10)
+                continue; // likely the scorer name
+            if (filename.startsWith('0.'))
+                continue;
+            if (!instructionText.includes(filename)) {
+                warnings.push({
+                    check: 'filename-consistency',
+                    message: `Scorer checks for '${filename}' but the instruction does not mention this filename.`,
+                });
+            }
+        }
+    }
+    return warnings;
+}
+function writeResult(result) {
+    process.stdout.write(JSON.stringify(result));
+    return result.valid ? 0 : 1;
+}
+/**
+ * Strict affected-selection validator. Exits 1 if any eval is missing
+ * both a SKILL.md ancestor and `__pathgradeMeta`, or has malformed meta.
+ */
+export async function runValidateAffected(cwd) {
+    const evalFiles = discoverEvalFiles(cwd);
+    let errors = 0;
+    for (const evalFile of evalFiles) {
+        const abs = path.resolve(cwd, evalFile);
+        const skillRoot = findSkillRoot(abs, cwd);
+        let parseError = null;
+        let hasMeta = false;
+        try {
+            const meta = parsePathgradeMeta(abs);
+            hasMeta = meta !== null;
+        }
+        catch (err) {
+            parseError = err instanceof Error ? err.message : String(err);
+        }
+        if (parseError) {
+            errors++;
+            process.stdout.write(`${evalFile} — ❌ malformed __pathgradeMeta: ${parseError}\n`);
+            continue;
+        }
+        if (skillRoot) {
+            process.stdout.write(`${evalFile} — ✅ anchored at ${skillRoot}\n`);
+            continue;
+        }
+        if (hasMeta) {
+            process.stdout.write(`${evalFile} — ✅ __pathgradeMeta present (no SKILL.md anchor)\n`);
+            continue;
+        }
+        errors++;
+        process.stdout.write(`${evalFile} — ❌ no SKILL.md ancestor and no __pathgradeMeta\n`);
+    }
+    process.stdout.write(`\n${evalFiles.length} evals, ${errors} errors\n`);
+    return errors === 0 ? 0 : 1;
+}
+async function runTsc(filePath) {
+    const { execSync } = await import('child_process');
+    const path = await import('path');
+    const { writeFileSync, unlinkSync } = await import('fs');
+    const os = await import('os');
+    // Resolve pathgrade's own tsc binary — avoids npx indirection and the
+    // "This is not the tsc command you are looking for" error when the
+    // consumer hasn't installed typescript themselves. `createRequire` is
+    // needed because this module is emitted as ESM ("type": "module") and
+    // `require` is not a free binding there — issue #39.
+    const localRequire = createRequire(import.meta.url);
+    const tscPath = localRequire.resolve('typescript/bin/tsc');
+    // Resolve pathgrade's own @types so Node builtins (fs, path, __dirname)
+    // compile without the consumer needing to install @types/node.
+    const pathgradeTypesDir = path.dirname(localRequire.resolve('@types/node/package.json'));
+    const pathgradeTypeRoots = path.dirname(pathgradeTypesDir);
+    const consumerTypeRoots = path.join(process.cwd(), 'node_modules', '@types');
+    // Generate a temp tsconfig instead of passing files on the CLI.
+    // This avoids TS5112 ("tsconfig.json is present but will not be loaded
+    // if files are specified on commandline") in TypeScript 5.6+.
+    const tmpConfig = path.join(os.tmpdir(), `pathgrade-tsc-${Date.now()}-${Math.random().toString(36).slice(2)}.json`);
+    const tsconfig = {
+        compilerOptions: {
+            noEmit: true,
+            esModuleInterop: true,
+            module: 'nodenext',
+            moduleResolution: 'nodenext',
+            typeRoots: [pathgradeTypeRoots, consumerTypeRoots],
+            skipLibCheck: true,
+        },
+        files: [path.resolve(filePath)],
+    };
+    writeFileSync(tmpConfig, JSON.stringify(tsconfig));
+    try {
+        execSync(`node "${tscPath}" --project "${tmpConfig}"`, { stdio: 'pipe', timeout: 30000 });
+        return { ok: true, message: '' };
+    }
+    catch (err) {
+        const output = (err.stdout?.toString() || '') + (err.stderr?.toString() || '');
+        const lines = output.split('\n').filter((l) => l.trim()).slice(0, 5);
+        return { ok: false, message: lines.join('\n') || 'TypeScript compilation failed.' };
+    }
+    finally {
+        try {
+            unlinkSync(tmpConfig);
+        }
+        catch { }
+    }
+}

package/dist/core/mcp-mock.d.ts

@@ -0,0 +1,3 @@
+import { MockMcpServerConfig, MockMcpServerDescriptor } from './mcp-mock.types.js';
+export type { MockMcpTool, MockMcpServerConfig, MockMcpServerDescriptor } from './mcp-mock.types.js';
+export declare function mockMcpServer(config: MockMcpServerConfig): MockMcpServerDescriptor;

package/dist/core/mcp-mock.js

@@ -0,0 +1,23 @@
+export function mockMcpServer(config) {
+    if (!config.name || typeof config.name !== 'string') {
+        throw new Error('mockMcpServer: name must be a non-empty string');
+    }
+    if (!Array.isArray(config.tools) || config.tools.length === 0) {
+        throw new Error('mockMcpServer: must have at least one tool');
+    }
+    for (let i = 0; i < config.tools.length; i++) {
+        const tool = config.tools[i];
+        if (!tool.name || typeof tool.name !== 'string') {
+            throw new Error(`mockMcpServer: tools[${i}].name must be a non-empty string`);
+        }
+        if (tool.when !== undefined) {
+            try {
+                new RegExp(tool.when, 'i');
+            }
+            catch (e) {
+                throw new Error(`mockMcpServer: tools[${i}].when is not a valid regex: ${e.message}`);
+            }
+        }
+    }
+    return { __type: 'mock_mcp_server', config };
+}

package/dist/core/mcp-mock.types.d.ts

@@ -0,0 +1,15 @@
+export interface MockMcpTool {
+    name: string;
+    description?: string;
+    inputSchema?: Record<string, unknown>;
+    when?: string;
+    response: unknown;
+}
+export interface MockMcpServerConfig {
+    name: string;
+    tools: MockMcpTool[];
+}
+export interface MockMcpServerDescriptor {
+    __type: 'mock_mcp_server';
+    config: MockMcpServerConfig;
+}

package/dist/core/mcp-mock.types.js

@@ -0,0 +1 @@
+export {};

package/dist/core/skills.d.ts

@@ -0,0 +1,15 @@
+export interface DetectedSkill {
+    name: string;
+    path: string;
+    skillMd: string;
+}
+/**
+ * Detect skills in the given directory.
+ *
+ * Looks in these locations (in order):
+ * 1. SKILL.md in the root directory (the dir itself IS the skill)
+ * 2. skill/SKILL.md (example layout with eval.ts at the root)
+ * 3. skills/{name}/SKILL.md (sub-skills)
+ * 4. .agents/skills/{name}/SKILL.md
+ */
+export declare function detectSkills(dir: string): Promise<DetectedSkill[]>;

package/dist/core/skills.js

@@ -0,0 +1,76 @@
+/**
+ * Skill detection — finds skills in the current directory or standard locations.
+ */
+import fs from 'fs-extra';
+import * as path from 'path';
+/**
+ * Detect skills in the given directory.
+ *
+ * Looks in these locations (in order):
+ * 1. SKILL.md in the root directory (the dir itself IS the skill)
+ * 2. skill/SKILL.md (example layout with eval.ts at the root)
+ * 3. skills/{name}/SKILL.md (sub-skills)
+ * 4. .agents/skills/{name}/SKILL.md
+ */
+export async function detectSkills(dir) {
+    const skills = [];
+    // 1. Check if the directory itself is a skill
+    const rootSkillMd = path.join(dir, 'SKILL.md');
+    if (await fs.pathExists(rootSkillMd)) {
+        const content = await fs.readFile(rootSkillMd, 'utf-8');
+        const name = parseSkillName(content) || path.basename(dir);
+        skills.push({ name, path: dir, skillMd: content });
+        return skills; // If root is a skill, don't look for sub-skills
+    }
+    // 2. Check singular skill/ directory used by example layouts
+    const nestedSkillDir = path.join(dir, 'skill');
+    const nestedSkillMd = path.join(nestedSkillDir, 'SKILL.md');
+    if (await fs.pathExists(nestedSkillMd)) {
+        const content = await fs.readFile(nestedSkillMd, 'utf-8');
+        const name = parseSkillName(content) || 'skill';
+        skills.push({ name, path: nestedSkillDir, skillMd: content });
+        return skills;
+    }
+    // 3. Check skills/ and agent skill subdirectories
+    const searchDirs = ['skills', '.agents/skills', '.claude/skills'];
+    for (const searchDir of searchDirs) {
+        const fullSearchDir = path.join(dir, searchDir);
+        if (!await fs.pathExists(fullSearchDir))
+            continue;
+        const entries = await fs.readdir(fullSearchDir, { withFileTypes: true });
+        for (const entry of entries) {
+            if (!entry.isDirectory())
+                continue;
+            const skillMdPath = path.join(fullSearchDir, entry.name, 'SKILL.md');
+            if (await fs.pathExists(skillMdPath)) {
+                const content = await fs.readFile(skillMdPath, 'utf-8');
+                const name = parseSkillName(content) || entry.name;
+                skills.push({
+                    name,
+                    path: path.join(fullSearchDir, entry.name),
+                    skillMd: content,
+                });
+            }
+        }
+    }
+    return skills;
+}
+/**
+ * Parse the skill name from SKILL.md content.
+ * Looks for YAML frontmatter `name:` or first `# Heading`.
+ */
+function parseSkillName(content) {
+    // Try YAML frontmatter
+    const fmMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
+    if (fmMatch) {
+        const nameMatch = fmMatch[1].match(/^name:\s*(.+)$/m);
+        if (nameMatch)
+            return nameMatch[1].trim().replace(/^['"]|['"]$/g, '');
+    }
+    // Try first heading
+    const headingMatch = content.match(/^#\s+(.+)$/m);
+    if (headingMatch) {
+        return headingMatch[1].trim().toLowerCase().replace(/\s+/g, '-');
+    }
+    return undefined;
+}

package/dist/evals/discovery.d.ts

@@ -0,0 +1,7 @@
+export interface EvalDiscoveryOptions {
+    cwd: string;
+    include: string[];
+    exclude: string[];
+}
+export declare function discoverPathgradeEvalFiles(opts: EvalDiscoveryOptions): string[];
+export declare function isPathgradeEval(absPath: string): boolean;

package/dist/evals/discovery.js

@@ -0,0 +1,69 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import picomatch from 'picomatch';
+import * as ts from 'typescript';
+const EVAL_SUFFIX = '.eval.ts';
+const PATHGRADE_PACKAGE = '@wix/pathgrade';
+export function discoverPathgradeEvalFiles(opts) {
+    const { cwd, include, exclude } = opts;
+    const root = path.resolve(cwd);
+    const includeMatchers = include.map(g => picomatch(g, { dot: true }));
+    const excludeMatchers = exclude.map(g => picomatch(g, { dot: true }));
+    const results = [];
+    function isIncluded(relPath) {
+        return includeMatchers.some(m => m(relPath));
+    }
+    function isExcluded(relPath) {
+        return excludeMatchers.some(m => m(relPath));
+    }
+    function walk(absDir, relDir) {
+        let entries;
+        try {
+            entries = fs.readdirSync(absDir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const ent of entries) {
+            const rel = relDir ? `${relDir}/${ent.name}` : ent.name;
+            if (isExcluded(rel))
+                continue;
+            const abs = path.join(absDir, ent.name);
+            if (ent.isDirectory()) {
+                walk(abs, rel);
+                continue;
+            }
+            if (!ent.isFile() || !ent.name.endsWith(EVAL_SUFFIX) || !isIncluded(rel))
+                continue;
+            if (!isPathgradeEval(abs))
+                continue;
+            results.push(rel);
+        }
+    }
+    walk(root, '');
+    results.sort();
+    return results;
+}
+export function isPathgradeEval(absPath) {
+    const source = fs.readFileSync(absPath, 'utf8');
+    const sourceFile = ts.createSourceFile(absPath, source, ts.ScriptTarget.Latest, 
+    /* setParentNodes */ false, ts.ScriptKind.TS);
+    for (const stmt of sourceFile.statements) {
+        if (isPathgradeImport(stmt) || isPathgradeMetaExport(stmt))
+            return true;
+    }
+    return false;
+}
+function isPathgradeImport(stmt) {
+    if (!ts.isImportDeclaration(stmt))
+        return false;
+    return ts.isStringLiteral(stmt.moduleSpecifier) && stmt.moduleSpecifier.text === PATHGRADE_PACKAGE;
+}
+function isPathgradeMetaExport(stmt) {
+    if (!ts.isVariableStatement(stmt))
+        return false;
+    const hasExport = stmt.modifiers?.some(m => m.kind === ts.SyntaxKind.ExportKeyword);
+    if (!hasExport)
+        return false;
+    return stmt.declarationList.declarations.some(decl => ts.isIdentifier(decl.name) && decl.name.text === '__pathgradeMeta');
+}

package/dist/mcp-mock-server.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/mcp-mock-server.js

@@ -0,0 +1,116 @@
+import * as fs from 'fs';
+import * as readline from 'readline';
+function loadFixture(fixturePath) {
+    const content = fs.readFileSync(fixturePath, 'utf-8');
+    return JSON.parse(content);
+}
+function buildToolSchemas(fixture) {
+    const seen = new Set();
+    const schemas = [];
+    for (const tool of fixture.tools) {
+        if (seen.has(tool.name))
+            continue;
+        seen.add(tool.name);
+        schemas.push({
+            name: tool.name,
+            description: tool.description || `Mock tool: ${tool.name}`,
+            inputSchema: tool.inputSchema || { type: 'object' },
+        });
+    }
+    return schemas;
+}
+function matchTool(fixture, toolName, args) {
+    const inputStr = JSON.stringify(args);
+    let fallback;
+    for (const entry of fixture.tools) {
+        if (entry.name !== toolName)
+            continue;
+        if (entry.when) {
+            if (new RegExp(entry.when, 'i').test(inputStr)) {
+                return { response: entry.response };
+            }
+        }
+        else if (!fallback) {
+            fallback = entry;
+        }
+    }
+    if (fallback)
+        return { response: fallback.response };
+    return null;
+}
+function formatResponse(response) {
+    return typeof response === 'string' ? response : JSON.stringify(response);
+}
+function sendResponse(id, result) {
+    const msg = JSON.stringify({ jsonrpc: '2.0', id, result });
+    process.stdout.write(msg + '\n');
+}
+function sendError(id, code, message) {
+    const msg = JSON.stringify({ jsonrpc: '2.0', id, error: { code, message } });
+    process.stdout.write(msg + '\n');
+}
+function main() {
+    const fixturePath = process.argv[2];
+    if (!fixturePath) {
+        process.stderr.write('Usage: mcp-mock-server <fixture.json>\n');
+        process.exit(1);
+    }
+    let fixture;
+    try {
+        fixture = loadFixture(fixturePath);
+    }
+    catch (e) {
+        process.stderr.write(`Failed to load fixture: ${e.message}\n`);
+        process.exit(1);
+    }
+    const toolSchemas = buildToolSchemas(fixture);
+    const rl = readline.createInterface({ input: process.stdin });
+    rl.on('line', (line) => {
+        if (!line.trim())
+            return;
+        let msg;
+        try {
+            msg = JSON.parse(line);
+        }
+        catch {
+            return;
+        }
+        // Notifications (no id) — acknowledge silently
+        if (msg.id === undefined)
+            return;
+        switch (msg.method) {
+            case 'initialize':
+                sendResponse(msg.id, {
+                    protocolVersion: '2024-11-05',
+                    capabilities: { tools: {} },
+                    serverInfo: { name: fixture.name, version: '1.0.0' },
+                });
+                break;
+            case 'tools/list':
+                sendResponse(msg.id, { tools: toolSchemas });
+                break;
+            case 'tools/call': {
+                const toolName = msg.params?.name;
+                const toolArgs = msg.params?.arguments ?? {};
+                const match = matchTool(fixture, toolName, toolArgs);
+                if (match) {
+                    sendResponse(msg.id, {
+                        content: [{ type: 'text', text: formatResponse(match.response) }],
+                    });
+                }
+                else {
+                    sendError(msg.id, -32602, `Unknown tool: ${toolName}`);
+                }
+                break;
+            }
+            case 'ping':
+                sendResponse(msg.id, {});
+                break;
+            default:
+                sendError(msg.id, -32601, `Method not found: ${msg.method}`);
+                break;
+        }
+    });
+    rl.on('close', () => process.exit(0));
+}
+main();

package/dist/pathgrade.d.ts

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+/**
+ * pathgrade CLI — thin wrapper around vitest
+ *
+ * Usage:
+ *   pathgrade run [-- vitest-args]   Run evals via vitest (loads .env, validates API keys)
+ *   pathgrade init [--force]         Generate eval scaffolding
+ *   pathgrade preview [browser]      View results (CLI default, or browser)
+ */
+export {};