@wix/pathgrade 0.29.0

package/dist/analytics/engine.js

@@ -0,0 +1,66 @@
+import fs from 'fs-extra';
+import * as path from 'path';
+/**
+ * Calculates Normalized Gain (NG) as defined in the Skill Eval paper.
+ * NG = (p_with - p_without) / (1 - p_without)
+ */
+export function calculateNormalizedGain(pWith, pWithout) {
+    if (pWithout === 1) {
+        return pWith === 1 ? 0 : -1;
+    }
+    return (pWith - pWithout) / (1 - pWithout);
+}
+export class AnalyticsEngine {
+    async loadReports(logDir) {
+        const files = await fs.readdir(logDir);
+        const reports = [];
+        for (const file of files) {
+            if (file.endsWith('.json')) {
+                const report = await fs.readJSON(path.join(logDir, file));
+                reports.push(report);
+            }
+        }
+        return reports;
+    }
+    aggregate(reports) {
+        const taskGroups = {};
+        for (const report of reports) {
+            if (!taskGroups[report.task]) {
+                taskGroups[report.task] = { withSkill: [], withoutSkill: [] };
+            }
+            const hasSkills = report.skills_used && report.skills_used.length > 0;
+            if (hasSkills) {
+                taskGroups[report.task].withSkill.push(report);
+            }
+            else {
+                taskGroups[report.task].withoutSkill.push(report);
+            }
+        }
+        const stats = [];
+        for (const [task, data] of Object.entries(taskGroups)) {
+            const allReports = [...data.withSkill, ...data.withoutSkill];
+            const avgWith = data.withSkill.length > 0
+                ? data.withSkill.reduce((a, b) => a + b.pass_rate, 0) / data.withSkill.length
+                : 0;
+            const avgWithout = data.withoutSkill.length > 0
+                ? data.withoutSkill.reduce((a, b) => a + b.pass_rate, 0) / data.withoutSkill.length
+                : 0;
+            const allTrials = allReports.flatMap(r => r.trials);
+            const avgDurationMs = allTrials.length > 0
+                ? allTrials.reduce((s, t) => s + (t.duration_ms || 0), 0) / allTrials.length
+                : 0;
+            const avgCommands = allTrials.length > 0
+                ? allTrials.reduce((s, t) => s + (t.n_commands || 0), 0) / allTrials.length
+                : 0;
+            stats.push({
+                task,
+                passRateWithSkill: avgWith,
+                passRateNoSkill: avgWithout,
+                normalizedGain: calculateNormalizedGain(avgWith, avgWithout),
+                avgDurationMs,
+                avgCommands
+            });
+        }
+        return stats;
+    }
+}

package/dist/commands/affected.d.ts

@@ -0,0 +1,31 @@
+/**
+ * `pathgrade affected` — the selection primitive.
+ *
+ * Input precedence for the change-set (Issue 6):
+ *   1. `--changed-files=<path>` wins unconditionally (non-git contexts).
+ *   2. `--since=<ref>` diffs `<ref>...HEAD`.
+ *   3. No flags → auto-derive base ref from GitHub Actions env + git merge-base.
+ *
+ * Output contract: one selected path per stdout line, sorted lexicographically.
+ * The resolved base-ref one-liner goes to stderr so it doesn't pollute the
+ * composable stdout list. Issues 7/8/10 wire warnings, global short-circuit,
+ * and `--explain` / `--json` presentation.
+ */
+export interface RunAffectedOptions {
+    /** Absolute path to the repo root (CLI passes `process.cwd()`). */
+    cwd: string;
+    /** Newline-delimited list of repo-relative changed files (overrides git). */
+    changedFilesPath?: string;
+    /** Git ref to diff against (`<ref>...HEAD`). Overrides auto-detection. */
+    since?: string;
+    /** Emit a human-readable summary to stderr. Stdout contract unchanged. */
+    explain?: boolean;
+    /** Emit structured JSON to stdout instead of the plain list. */
+    json?: boolean;
+}
+export declare function runAffected(opts: RunAffectedOptions): Promise<number>;
+/**
+ * Discover repo-relative `*.eval.ts` files under `cwd`, honoring the same
+ * exclude defaults the vitest plugin uses.
+ */
+export declare function discoverEvalFiles(cwd: string): string[];

package/dist/commands/affected.js

@@ -0,0 +1,150 @@
+/**
+ * `pathgrade affected` — the selection primitive.
+ *
+ * Input precedence for the change-set (Issue 6):
+ *   1. `--changed-files=<path>` wins unconditionally (non-git contexts).
+ *   2. `--since=<ref>` diffs `<ref>...HEAD`.
+ *   3. No flags → auto-derive base ref from GitHub Actions env + git merge-base.
+ *
+ * Output contract: one selected path per stdout line, sorted lexicographically.
+ * The resolved base-ref one-liner goes to stderr so it doesn't pollute the
+ * composable stdout list. Issues 7/8/10 wire warnings, global short-circuit,
+ * and `--explain` / `--json` presentation.
+ */
+import * as fs from 'fs';
+import { configDefaults } from 'vitest/config';
+import { selectAffected } from '../affected/select.js';
+import { resolveBaseRef, computeChangedFiles } from '../affected/git.js';
+import { loadAffectedConfig } from '../affected/config.js';
+import { formatExplain, formatJson } from '../affected/format.js';
+import { discoverPathgradeEvalFiles } from '../evals/discovery.js';
+const DEFAULT_EXCLUDE = [
+    ...configDefaults.exclude,
+    '.worktrees/**',
+    'worktrees/**',
+    '**/node_modules/**',
+    '**/fixtures/**',
+];
+export async function runAffected(opts) {
+    const { cwd } = opts;
+    let changes;
+    try {
+        const resolved = resolveChangeSet(opts);
+        if ('error' in resolved) {
+            process.stderr.write(`${resolved.error}\n`);
+            return 1;
+        }
+        changes = resolved;
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        process.stderr.write(`pathgrade affected: ${msg}\n`);
+        return 1;
+    }
+    if (changes.baseRefLine) {
+        process.stderr.write(`${changes.baseRefLine}\n`);
+    }
+    const evalFiles = discoverEvalFiles(cwd);
+    const config = await loadAffectedConfig(cwd, {
+        onWarning: w => process.stderr.write(`${w}\n`),
+    });
+    let result;
+    try {
+        result = selectAffected({
+            evalFiles,
+            changedFiles: changes.changedFiles,
+            repoRoot: cwd,
+            baseRef: changes.baseRef,
+            global: config.global,
+        });
+    }
+    catch (err) {
+        // Malformed `__pathgradeMeta` — the PRD requires a hard error.
+        const msg = err instanceof Error ? err.message : String(err);
+        process.stderr.write(`pathgrade affected: ${msg}\n`);
+        return 1;
+    }
+    // Default mode still prints warnings + global summary to stderr so the
+    // Issue 5/6 behavior is preserved when neither flag is set.
+    if (!opts.explain) {
+        for (const w of result.warnings) {
+            process.stderr.write(`warning: ${w}\n`);
+        }
+        if (result.globalMatch) {
+            process.stderr.write(`pathgrade: global match: \`${result.globalMatch}\` — selecting all ${result.selected.length} evals\n`);
+        }
+    }
+    if (opts.explain) {
+        process.stderr.write(formatExplain(result));
+    }
+    if (opts.json) {
+        process.stdout.write(formatJson(result));
+    }
+    else {
+        printSelectedToStdout(result);
+    }
+    return 0;
+}
+/**
+ * Decide where the change-set comes from and produce it.
+ *
+ * Returns `{ error }` when git auto-detection fails (so the caller can
+ * surface a clean stderr message); throws only on actual I/O / programmer
+ * errors (e.g., unreadable `--changed-files` path).
+ */
+function resolveChangeSet(opts) {
+    const { changedFilesPath, since } = opts;
+    if (changedFilesPath) {
+        let changedFiles;
+        try {
+            changedFiles = readChangedFilesList(changedFilesPath);
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            throw new Error(`could not read --changed-files: ${msg}`);
+        }
+        return { baseRef: 'explicit', changedFiles };
+    }
+    if (since) {
+        const changedFiles = computeChangedFiles(since);
+        return {
+            baseRef: `${since}`,
+            changedFiles,
+            baseRefLine: `pathgrade: base = ${since} (--since)`,
+        };
+    }
+    // Auto-derive from git.
+    const resolution = resolveBaseRef();
+    if ('error' in resolution)
+        return { error: resolution.error };
+    const shortSha = resolution.sha.slice(0, 7);
+    const changedFiles = computeChangedFiles(resolution.sha);
+    return {
+        baseRef: `${resolution.base}@${shortSha}`,
+        changedFiles,
+        baseRefLine: `pathgrade: base = ${resolution.base}@${shortSha} (merge-base with HEAD)`,
+    };
+}
+function readChangedFilesList(filePath) {
+    const content = fs.readFileSync(filePath, 'utf-8');
+    return content
+        .split('\n')
+        .map(l => l.trim())
+        .filter(l => l.length > 0);
+}
+/**
+ * Discover repo-relative `*.eval.ts` files under `cwd`, honoring the same
+ * exclude defaults the vitest plugin uses.
+ */
+export function discoverEvalFiles(cwd) {
+    return discoverPathgradeEvalFiles({
+        cwd,
+        include: ['**/*.eval.ts'],
+        exclude: DEFAULT_EXCLUDE,
+    });
+}
+function printSelectedToStdout(result) {
+    for (const entry of result.selected) {
+        process.stdout.write(`${entry.file}\n`);
+    }
+}

package/dist/commands/analyze.d.ts

@@ -0,0 +1,21 @@
+export interface AnalyzeOptions {
+    skill?: string;
+    dir?: string;
+}
+export interface SkillAnalysis {
+    skillName: string;
+    description: string;
+    procedures: string[];
+    expectedOutputs: string[];
+    suggestedScorers: Array<{
+        type: 'check' | 'judge';
+        name: string;
+        hint: string;
+    }>;
+    workspaceHint: string;
+    hasFixtures: boolean;
+}
+/**
+ * Run the analyze command. Returns exit code (0 = success, 1 = error).
+ */
+export declare function runAnalyze(dir: string, opts?: AnalyzeOptions): Promise<number>;

package/dist/commands/analyze.js

@@ -0,0 +1,167 @@
+/**
+ * `pathgrade analyze` command.
+ *
+ * Reads SKILL.md files in a directory and outputs structured JSON analysis to stdout.
+ * No LLM calls — purely deterministic parsing.
+ */
+import fs from 'fs-extra';
+import * as path from 'path';
+import { detectSkills } from '../core/skills.js';
+/**
+ * Run the analyze command. Returns exit code (0 = success, 1 = error).
+ */
+export async function runAnalyze(dir, opts = {}) {
+    const targetDir = opts.dir || dir;
+    const skills = await detectSkills(targetDir);
+    if (skills.length === 0) {
+        process.stdout.write(JSON.stringify({
+            error: 'no-skill-found',
+            message: 'No SKILL.md found. Create one or use --dir to point to a skill directory.',
+        }));
+        return 1;
+    }
+    if (opts.skill) {
+        const match = skills.find(s => s.name === opts.skill);
+        if (!match) {
+            process.stdout.write(JSON.stringify({
+                error: 'skill-not-found',
+                message: `Skill '${opts.skill}' not found. Available: ${skills.map(s => s.name).join(', ')}`,
+            }));
+            return 1;
+        }
+        const analysis = analyzeSkill(match, targetDir);
+        process.stdout.write(JSON.stringify(analysis));
+        return 0;
+    }
+    if (skills.length === 1) {
+        const analysis = analyzeSkill(skills[0], targetDir);
+        process.stdout.write(JSON.stringify(analysis));
+    }
+    else {
+        const analyses = skills.map(s => analyzeSkill(s, targetDir));
+        process.stdout.write(JSON.stringify(analyses));
+    }
+    return 0;
+}
+function analyzeSkill(skill, dir) {
+    const { name, description } = parseFrontmatter(skill.skillMd);
+    const procedures = extractProcedures(skill.skillMd);
+    const expectedOutputs = extractExpectedOutputs(procedures);
+    const suggestedScorers = buildSuggestedScorers(expectedOutputs);
+    const workspaceHint = extractWorkspaceHint(procedures);
+    const hasFixtures = fs.pathExistsSync(path.join(skill.path, 'fixtures')) ||
+        fs.pathExistsSync(path.join(skill.path, 'test', 'fixtures'));
+    return {
+        skillName: name || skill.name,
+        description: description || extractFirstParagraph(skill.skillMd),
+        procedures,
+        expectedOutputs,
+        suggestedScorers,
+        workspaceHint,
+        hasFixtures,
+    };
+}
+function parseFrontmatter(content) {
+    const fmMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
+    if (!fmMatch)
+        return {};
+    const fm = fmMatch[1];
+    const nameMatch = fm.match(/^name:\s*(.+)$/m);
+    const descMatch = fm.match(/^description:\s*(.+)$/m);
+    return {
+        name: nameMatch ? nameMatch[1].trim().replace(/^['"]|['"]$/g, '') : undefined,
+        description: descMatch ? descMatch[1].trim().replace(/^['"]|['"]$/g, '') : undefined,
+    };
+}
+function extractFirstParagraph(content) {
+    // Strip frontmatter
+    const stripped = content.replace(/^---\s*\n[\s\S]*?\n---\s*\n?/, '');
+    const lines = stripped.split('\n');
+    let foundHeading = false;
+    const paragraphLines = [];
+    for (const line of lines) {
+        if (line.startsWith('# ') && !foundHeading) {
+            foundHeading = true;
+            continue;
+        }
+        if (foundHeading) {
+            if (line.trim() === '' && paragraphLines.length > 0)
+                break;
+            if (line.startsWith('#'))
+                break;
+            if (line.trim())
+                paragraphLines.push(line.trim());
+        }
+    }
+    return paragraphLines.join(' ');
+}
+function extractProcedures(content) {
+    const lines = content.split('\n');
+    const procedures = [];
+    let inProcedureSection = false;
+    for (const line of lines) {
+        // Check for headings containing procedure/step/how to
+        if (/^#{1,3}\s+/i.test(line)) {
+            const headingText = line.replace(/^#{1,3}\s+/, '').toLowerCase();
+            inProcedureSection = /procedure|step|how to/i.test(headingText);
+            continue;
+        }
+        if (inProcedureSection) {
+            // Numbered list: "1. Do something" or "- Do something"
+            const listMatch = line.match(/^\s*(?:\d+\.|[-*])\s+(.+)/);
+            if (listMatch) {
+                procedures.push(listMatch[1].trim());
+            }
+            // Empty line after list items ends the section only if we have procedures
+            if (line.trim() === '' && procedures.length > 0) {
+                // Continue — might be spacing between items
+            }
+        }
+    }
+    return procedures;
+}
+const CREATION_VERBS = /\b(?:create|write|generate|save|produce|output)\b/i;
+const FILENAME_PATTERN = /`([^`]+\.[a-zA-Z]{1,5})`|"([^"]+\.[a-zA-Z]{1,5})"|'([^']+\.[a-zA-Z]{1,5})'/g;
+function extractExpectedOutputs(procedures) {
+    const outputs = [];
+    for (const proc of procedures) {
+        if (!CREATION_VERBS.test(proc))
+            continue;
+        let match;
+        FILENAME_PATTERN.lastIndex = 0;
+        while ((match = FILENAME_PATTERN.exec(proc)) !== null) {
+            const filename = match[1] || match[2] || match[3];
+            if (filename && !outputs.includes(filename)) {
+                outputs.push(filename);
+            }
+        }
+    }
+    return outputs;
+}
+function buildSuggestedScorers(expectedOutputs) {
+    const scorers = [];
+    for (const output of expectedOutputs) {
+        const baseName = output.replace(/\.[^.]+$/, '').replace(/[^a-zA-Z0-9]+/g, '-');
+        scorers.push({
+            type: 'check',
+            name: `${baseName}-exists`,
+            hint: `Check ${output} exists in workspace`,
+        });
+    }
+    // Always add a judge scorer
+    scorers.push({
+        type: 'judge',
+        name: 'quality',
+        hint: 'Evaluate overall quality of the agent\'s work',
+    });
+    return scorers;
+}
+function extractWorkspaceHint(procedures) {
+    const text = procedures.join(' ').toLowerCase();
+    const keywords = ['workspace', 'project', 'directory', 'fixture', 'file'];
+    const found = keywords.filter(k => text.includes(k));
+    if (found.length > 0) {
+        return `Skill operates on ${found.join(', ')} resources in the workspace`;
+    }
+    return '';
+}

package/dist/commands/init.d.ts

@@ -0,0 +1,3 @@
+export declare function runInit(dir: string, opts?: {
+    force?: boolean;
+}): Promise<void>;

package/dist/commands/init.js

@@ -0,0 +1,242 @@
+/**
+ * `pathgrade init` command.
+ *
+ * Detects skills in the current directory and generates a *.eval.ts config.
+ * With an API key, uses an LLM to generate intelligent eval tasks.
+ * Without an API key, generates a well-commented template.
+ */
+import fs from 'fs-extra';
+import * as path from 'path';
+import { detectSkills } from '../core/skills.js';
+import { parseEnvFile } from '../utils/env.js';
+import { callLLM, isClaudeCliAvailable } from '../utils/llm.js';
+/** Find any existing eval config file in a directory. */
+async function findExistingEvalFile(dir) {
+    const files = await fs.readdir(dir);
+    const evalFile = files.find(f => f.endsWith('.eval.ts'));
+    if (evalFile)
+        return path.join(dir, evalFile);
+    const legacy = path.join(dir, 'eval.ts');
+    if (await fs.pathExists(legacy))
+        return legacy;
+    return null;
+}
+export async function runInit(dir, opts = {}) {
+    const existing = await findExistingEvalFile(dir);
+    if (existing) {
+        if (opts.force) {
+            await fs.remove(existing);
+        }
+        else {
+            const name = path.basename(existing);
+            console.error(`  ${name} already exists. Use --force to overwrite.`);
+            throw new Error(`${name} already exists`);
+        }
+    }
+    console.log('\npathgrade init\n');
+    // Detect skills
+    const skills = await detectSkills(dir);
+    // Derive eval filename: <skill-name>.eval.ts or <dirname>.eval.ts
+    const dirName = path.basename(dir);
+    if (skills.length === 0) {
+        console.log('  No SKILL.md found. Creating a generic template.');
+        console.log('     Place a SKILL.md in this directory for better scaffolding.\n');
+        const evalPath = path.join(dir, `${dirName}.eval.ts`);
+        await writeTemplate(evalPath, 'my-skill', 'Describe what the agent should do with this skill.');
+        return;
+    }
+    console.log(`  Found ${skills.length} skill(s): ${skills.map(s => s.name).join(', ')}\n`);
+    // Load .env file if present
+    const envPath = path.join(dir, '.env');
+    if (await fs.pathExists(envPath)) {
+        const envVars = parseEnvFile(await fs.readFile(envPath, 'utf-8'));
+        for (const [key, value] of Object.entries(envVars)) {
+            if (!process.env[key]) {
+                process.env[key] = value;
+            }
+        }
+    }
+    // Try LLM-powered scaffold — auto-detect from available API key or CLI
+    const anthropicKey = process.env.ANTHROPIC_API_KEY;
+    const openaiKey = process.env.OPENAI_API_KEY;
+    const hasApiKey = !!(anthropicKey || openaiKey);
+    const cliAvailable = await isClaudeCliAvailable();
+    const evalName = skills.length === 1 ? skills[0].name : dirName;
+    const evalPath = path.join(dir, `${evalName}.eval.ts`);
+    if (hasApiKey || cliAvailable) {
+        const { Spinner, fmt } = await import('../utils/cli.js');
+        const label = 'generating eval with available LLM backend';
+        const spinner = new Spinner('init', label);
+        try {
+            const config = await generateWithLLM(skills);
+            await fs.writeFile(evalPath, config, 'utf-8');
+            spinner.stop(fmt.green(`created ${path.basename(evalPath)}`));
+            console.log(`     Review and edit the file, then run: pathgrade\n`);
+            return;
+        }
+        catch (err) {
+            spinner.stop(fmt.red(`AI generation failed: ${err.message}`));
+            console.log('     Falling back to template.\n');
+        }
+    }
+    else {
+        console.log('  Install Claude CLI or set an API key for AI-powered eval generation.\n');
+    }
+    // Fallback: template-based scaffold
+    const skill = skills[0];
+    const taskName = `test-${skill.name}`;
+    const instruction = extractInstructionHint(skill.skillMd);
+    await writeTemplate(evalPath, taskName, instruction);
+}
+async function writeTemplate(evalPath, taskName, instruction) {
+    const templatePath = path.join(import.meta.dirname, '..', '..', 'templates', 'eval.ts.template');
+    let template;
+    if (await fs.pathExists(templatePath)) {
+        template = await fs.readFile(templatePath, 'utf-8');
+    }
+    else {
+        // Inline fallback if template file not found (e.g., if installed globally)
+        template = getInlineTemplate();
+    }
+    const result = template
+        .replace(/\{\{TASK_NAME\}\}/g, taskName)
+        .replace(/\{\{INSTRUCTION\}\}/g, instruction);
+    await fs.writeFile(evalPath, result, 'utf-8');
+    console.log(`  Created ${path.basename(evalPath)}.`);
+    console.log(`     Edit the file to define your eval tasks, then run: pathgrade\n`);
+}
+/**
+ * Extract a reasonable instruction hint from SKILL.md content.
+ */
+function extractInstructionHint(skillMd) {
+    // Try to get the first paragraph after the main heading
+    const lines = skillMd.split('\n');
+    let foundHeading = false;
+    const paragraphLines = [];
+    for (const line of lines) {
+        if (line.startsWith('# ') && !foundHeading) {
+            foundHeading = true;
+            continue;
+        }
+        if (foundHeading) {
+            if (line.trim() === '' && paragraphLines.length > 0)
+                break;
+            if (line.startsWith('#'))
+                break;
+            if (line.trim())
+                paragraphLines.push(line.trim());
+        }
+    }
+    if (paragraphLines.length > 0) {
+        return `TODO: Write an instruction based on this skill.\n      Skill description: ${paragraphLines.join(' ')}`;
+    }
+    return 'TODO: Write an instruction for the agent.';
+}
+/**
+ * Build the init prompt for eval.ts generation.
+ */
+function buildInitPrompt(skills) {
+    const skillSummaries = skills.map(s => `## Skill: ${s.name}\n\n${s.skillMd}`).join('\n\n---\n\n');
+    return `You are an expert at creating evaluation tasks for AI agent skills.
+
+Given the following skill definition(s), generate a single \`.eval.ts\` file
+using the current PathGrade runtime API from \`@wix/pathgrade\`.
+
+For each task:
+- Write a realistic user instruction
+- Define workspace files if needed
+- Create an agent with \`createAgent(...)\`
+- Run the agent with \`agent.prompt(...)\`
+- Score the result with \`evaluate(agent, [...])\`
+- Use deterministic scorers via \`check()\` or \`score()\`
+- Use qualitative scoring via \`judge()\`
+
+IMPORTANT SCORING RULES:
+- Deterministic scorers must only check observable outcomes
+- Use \`check()\` for pass/fail assertions and \`score()\` for partial credit
+- Judge scorers use \`judge(name, { rubric, weight })\`
+- For workspace files, only reference files that exist in the skill directory or that the agent will create
+
+CRITICAL — FILENAME CONSISTENCY:
+- The instruction MUST tell the agent exactly what filenames to create
+- Deterministic scorers MUST only check filenames explicitly mentioned in the instruction
+- NEVER check for a hardcoded filename that the instruction does not mention
+
+${skillSummaries}
+
+Respond with ONLY the \`.eval.ts\` file content. Start with the import
+statement. Use this format:
+
+import { describe, it, expect } from 'vitest';
+import { createAgent, check, judge, evaluate } from '@wix/pathgrade';
+
+describe('<task-name>', () => {
+  it('scores the agent result', async () => {
+    const agent = await createAgent({
+      agent: 'claude',
+      timeout: 300,
+      skillDir: __dirname,
+    });
+
+    try {
+      await agent.prompt(\`<realistic user instruction>\`);
+
+      const result = await evaluate(agent, [
+        check('required artifact exists', async ({ workspace }) => true),
+        judge('workflow quality', {
+          rubric: \`<evaluation criteria>\`,
+          weight: 0.3,
+        }),
+      ]);
+
+      expect(result.score).toBeGreaterThanOrEqual(0);
+    } finally {
+      await agent.dispose();
+    }
+  });
+});`;
+}
+/**
+ * Generate eval.ts content using the shared LLM boundary.
+ * This keeps init aligned with the CLI-first provider selection used
+ * elsewhere in the runtime.
+ */
+async function generateWithLLM(skills) {
+    const prompt = buildInitPrompt(skills);
+    const result = await callLLM(prompt);
+    // Extract TypeScript from response (strip markdown code fences if present)
+    const tsContent = result.text.replace(/```(?:typescript|ts)?\n?/g, '').replace(/```\n?/g, '').trim();
+    return tsContent + '\n';
+}
+function getInlineTemplate() {
+    return `import { describe, it, expect } from 'vitest';
+import { createAgent, check, judge, evaluate } from '@wix/pathgrade';
+
+describe('{{TASK_NAME}}', () => {
+  it('scores the agent result', async () => {
+    const agent = await createAgent({
+      agent: 'claude',
+      timeout: 300,
+      skillDir: __dirname,
+      // workspace: path.join(__dirname, 'test', 'fixtures', 'my-fixture'),
+    });
+
+    try {
+      await agent.prompt(\`{{INSTRUCTION}}\`);
+
+      const result = await evaluate(agent, [
+        check('TODO: add deterministic scorer', async () => false),
+        judge('TODO: add rubric scorer', {
+          rubric: \`TODO: Write evaluation criteria.\`,
+          weight: 0.3,
+        }),
+      ]);
+
+      expect(result.score).toBeGreaterThanOrEqual(0);
+    } finally {
+      await agent.dispose();
+    }
+  });
+});
+`;
+}

package/dist/commands/preview-reactions.d.ts

@@ -0,0 +1,10 @@
+type PreviewFormat = 'cli' | 'json';
+export interface PreviewReactionsCommandOptions {
+    snapshot: string;
+    reactions: string;
+    format?: PreviewFormat;
+    cwd?: string;
+    write?: (chunk: string) => void;
+}
+export declare function runPreviewReactions(input: string[] | PreviewReactionsCommandOptions): Promise<number>;
+export {};