@sx4im/skillcheck 0.1.0

package/dist/src/cli.js

@@ -0,0 +1,146 @@
+import { NvidiaNimClient } from './adapters/nvidia-nim.js';
+import { runCorpus } from './corpus.js';
+import { evalSkill } from './eval.js';
+import { runM0Gate } from './m0/run.js';
+import { runRot } from './rot.js';
+import { verifyResult } from './verify.js';
+function printHelp() {
+    console.log(`skillcheck
+
+Usage:
+  skillcheck m0
+  skillcheck eval <path> [--tasks N] [--trials K] [--output file.json] [--task-suite file.json]
+    [--runner model] [--grader model] [--generator model]
+  skillcheck verify <result.json> [--sample n]
+  skillcheck corpus run --corpus corpus.json [--results dir] [--tasks N] [--trials K]
+    [--concurrency N] [--runner model] [--limit N]
+  skillcheck rot [--results dir] [--output file.json] [--model model] [--corpus corpus.yaml]
+    [--tasks N] [--trials K]
+
+M0 is the hardcoded spike. eval is the M1 forced-injection evaluator.`);
+}
+function readOption(argv, name) {
+    const index = argv.indexOf(name);
+    if (index === -1) {
+        return undefined;
+    }
+    const value = argv[index + 1];
+    if (!value || value.startsWith('--')) {
+        throw new Error(`Missing value for ${name}`);
+    }
+    return value;
+}
+function readNumberOption(argv, name, fallback) {
+    const value = readOption(argv, name);
+    if (value === undefined) {
+        return fallback;
+    }
+    const parsed = Number(value);
+    if (!Number.isInteger(parsed) || parsed <= 0) {
+        throw new Error(`${name} must be a positive integer`);
+    }
+    return parsed;
+}
+function readOptionalNumberOption(argv, name) {
+    const value = readOption(argv, name);
+    if (value === undefined) {
+        return undefined;
+    }
+    const parsed = Number(value);
+    if (!Number.isInteger(parsed) || parsed <= 0) {
+        throw new Error(`${name} must be a positive integer`);
+    }
+    return parsed;
+}
+function parseEvalOptions(argv) {
+    const inputPath = argv[3];
+    if (!inputPath || inputPath.startsWith('--')) {
+        throw new Error('Usage: skillcheck eval <path> [--tasks N] [--trials K] [--output file.json]');
+    }
+    const mode = readOption(argv, '--mode') ?? 'forced';
+    if (mode !== 'forced') {
+        throw new Error('Only --mode forced is supported in v1');
+    }
+    return {
+        inputPath,
+        output: readOption(argv, '--output'),
+        tasks: readNumberOption(argv, '--tasks', 10),
+        trials: readNumberOption(argv, '--trials', 3),
+        mode,
+        runner: readOption(argv, '--runner'),
+        grader: readOption(argv, '--grader'),
+        generator: readOption(argv, '--generator'),
+        taskSuite: readOption(argv, '--task-suite')
+    };
+}
+function parseRotOptions(argv) {
+    return {
+        resultsDir: readOption(argv, '--results') ?? 'results',
+        output: readOption(argv, '--output') ?? 'results/rot/report.json',
+        model: readOption(argv, '--model'),
+        corpus: readOption(argv, '--corpus'),
+        tasks: readNumberOption(argv, '--tasks', 10),
+        trials: readNumberOption(argv, '--trials', 3)
+    };
+}
+function parseCorpusRunOptions(argv) {
+    const corpus = readOption(argv, '--corpus');
+    if (!corpus) {
+        throw new Error('Usage: skillcheck corpus run --corpus corpus.json [--results dir]');
+    }
+    return {
+        corpus,
+        outputDir: readOption(argv, '--results') ?? 'results/corpus',
+        tasks: readNumberOption(argv, '--tasks', 10),
+        trials: readNumberOption(argv, '--trials', 3),
+        concurrency: readNumberOption(argv, '--concurrency', 2),
+        runner: readOption(argv, '--runner'),
+        limit: readOptionalNumberOption(argv, '--limit')
+    };
+}
+export async function main(argv) {
+    const command = argv[2];
+    if (!command || command === '--help' || command === '-h') {
+        printHelp();
+        return;
+    }
+    if (command === 'm0') {
+        const report = await runM0Gate((config) => new NvidiaNimClient(config));
+        console.log(JSON.stringify(report, null, 2));
+        process.exitCode = report.passed ? 0 : 1;
+        return;
+    }
+    if (command === 'eval') {
+        const result = await evalSkill(parseEvalOptions(argv));
+        console.log(JSON.stringify(result, null, 2));
+        return;
+    }
+    if (command === 'verify') {
+        const resultPath = argv[3];
+        if (!resultPath || resultPath.startsWith('--')) {
+            throw new Error('Usage: skillcheck verify <result.json> [--sample n]');
+        }
+        const result = await verifyResult({
+            resultPath,
+            sample: readNumberOption(argv, '--sample', 3)
+        });
+        console.log(JSON.stringify(result, null, 2));
+        return;
+    }
+    if (command === 'corpus') {
+        const subcommand = argv[3];
+        if (subcommand !== 'run') {
+            throw new Error('Usage: skillcheck corpus run --corpus corpus.json [--results dir]');
+        }
+        const result = await runCorpus(parseCorpusRunOptions(argv));
+        console.log(JSON.stringify(result, null, 2));
+        return;
+    }
+    if (command === 'rot') {
+        const result = await runRot(parseRotOptions(argv));
+        console.log(JSON.stringify(result, null, 2));
+        return;
+    }
+    throw new Error(`Unknown command: ${command}`);
+}
+//# sourceMappingURL=cli.js.map

package/dist/src/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../packages/cli/src/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAyB,MAAM,aAAa,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAoB,MAAM,WAAW,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,MAAM,EAAmB,MAAM,UAAU,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,SAAS,SAAS;IAChB,OAAO,CAAC,GAAG,CAAC;;;;;;;;;;;;sEAYwD,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,UAAU,CAAC,IAAc,EAAE,IAAY;IAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAC9B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAc,EAAE,IAAY,EAAE,QAAgB;IACtE,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACrC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAc,EAAE,IAAY;IAC5D,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACrC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,6BAA6B,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAc;IACtC,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAC1B,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,6EAA6E,CAAC,CAAC;IACjG,CAAC;IAED,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,QAAQ,CAAC;IACpD,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO;QACL,SAAS;QACT,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC;QAC5C,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QAC7C,IAAI;QACJ,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,aAAa,CAAC;QAC1C,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,cAAc,CAAC;KAC5C,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,IAAc;IACrC,OAAO;QACL,UAAU,EAAE,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,IAAI,SAAS;QACtD,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,yBAAyB;QACjE,KAAK,EAAE,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC;QAClC,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC;QAC5C,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;KAC9C,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAc;IAC3C,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAC5C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;IACvF,CAAC;IAED,OAAO;QACL,MAAM;QACN,SAAS,EAAE,UAAU,CAAC,IAAI,EAAE,WAAW,CAAC,IAAI,gBAAgB;QAC5D,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC;QAC5C,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;QAC7C,WAAW,EAAE,gBAAgB,CAAC,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;QACvD,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;QACpC,KAAK,EAAE,wBAAwB,CAAC,IAAI,EAAE,SAAS,CAAC;KACjD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,IAAc;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAExB,IAAI,CAAC,OAAO,IAAI,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACzD,SAAS,EAAE,CAAC;QACZ,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC;QACxE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;QACvD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;YAChC,UAAU;YACV,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;SAC9C,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,UAAU,KAAK,KAAK,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;QACvF,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5D,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;AACjD,CAAC"}

package/dist/src/corpus.d.ts

@@ -0,0 +1,43 @@
+export interface CorpusSkill {
+    id: string;
+    path: string;
+    source?: string;
+    repo?: string;
+    commit?: string;
+}
+export interface CorpusManifest {
+    name: string;
+    repo?: string;
+    commit?: string;
+    skills: CorpusSkill[];
+}
+export interface CorpusRunOptions {
+    corpus: string;
+    outputDir: string;
+    tasks: number;
+    trials: number;
+    concurrency: number;
+    runner?: string;
+    limit?: number;
+}
+export interface CorpusRunEntry {
+    id: string;
+    source: string;
+    repo?: string;
+    commit?: string;
+    path: string;
+    input_path: string;
+    output_path: string;
+}
+export interface CorpusRunReport {
+    corpus: string;
+    output_dir: string;
+    tasks: number;
+    trials: number;
+    concurrency: number;
+    runner?: string;
+    skills: CorpusRunEntry[];
+}
+export declare function slugify(value: string): string;
+export declare function parseCorpusManifest(text: string): CorpusManifest;
+export declare function runCorpus(options: CorpusRunOptions): Promise<CorpusRunReport>;

package/dist/src/corpus.js

@@ -0,0 +1,233 @@
+import { execFile as execFileCallback } from 'node:child_process';
+import { mkdir, readFile, stat, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { promisify } from 'node:util';
+import { evalSkill } from './eval.js';
+const execFile = promisify(execFileCallback);
+export function slugify(value) {
+    return value
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '-')
+        .replace(/^-|-$/g, '');
+}
+function unquote(value) {
+    return value.trim().replace(/^["']|["']$/g, '');
+}
+function assertCorpusManifest(value) {
+    const manifest = value;
+    if (!manifest.name || !Array.isArray(manifest.skills)) {
+        throw new Error('Corpus manifest JSON must include name and skills');
+    }
+    for (const [index, skill] of manifest.skills.entries()) {
+        if (!skill.id || !skill.path) {
+            throw new Error(`Corpus skill ${index + 1} must include id and path`);
+        }
+    }
+    return manifest;
+}
+export function parseCorpusManifest(text) {
+    const trimmed = text.trim();
+    if (trimmed.startsWith('{')) {
+        return assertCorpusManifest(JSON.parse(trimmed));
+    }
+    const manifest = { name: '', skills: [] };
+    let currentSkill;
+    for (const rawLine of text.split('\n')) {
+        const withoutComment = rawLine.replace(/\s+#.*$/, '');
+        if (!withoutComment.trim()) {
+            continue;
+        }
+        const topLevel = /^([A-Za-z_]+):\s*(.*)$/.exec(withoutComment);
+        if (topLevel && !rawLine.startsWith(' ')) {
+            const [, key, value = ''] = topLevel;
+            if (key === 'name')
+                manifest.name = unquote(value);
+            if (key === 'repo')
+                manifest.repo = unquote(value);
+            if (key === 'commit')
+                manifest.commit = unquote(value);
+            continue;
+        }
+        const skillStart = /^\s*-\s*id:\s*(.+)$/.exec(withoutComment);
+        if (skillStart) {
+            currentSkill = { id: unquote(skillStart[1]) };
+            manifest.skills.push(currentSkill);
+            continue;
+        }
+        const skillField = /^\s+([A-Za-z_]+):\s*(.+)$/.exec(withoutComment);
+        if (skillField && currentSkill) {
+            const [, key, value] = skillField;
+            if (key === 'path')
+                currentSkill.path = unquote(value);
+            if (key === 'id')
+                currentSkill.id = unquote(value);
+            if (key === 'source')
+                currentSkill.source = unquote(value);
+            if (key === 'repo')
+                currentSkill.repo = unquote(value);
+            if (key === 'commit')
+                currentSkill.commit = unquote(value);
+        }
+    }
+    if (!manifest.name || manifest.skills.some((skill) => !skill.id || !skill.path)) {
+        throw new Error('Corpus manifest YAML must include name and skills with id/path');
+    }
+    return manifest;
+}
+async function pathExists(filePath) {
+    try {
+        await stat(filePath);
+        return true;
+    }
+    catch (error) {
+        if (error.code === 'ENOENT') {
+            return false;
+        }
+        throw error;
+    }
+}
+async function hasValidJsonFile(filePath) {
+    try {
+        JSON.parse(await readFile(filePath, 'utf8'));
+        return true;
+    }
+    catch (error) {
+        if (error.code === 'ENOENT') {
+            return false;
+        }
+        if (error instanceof SyntaxError) {
+            return false;
+        }
+        throw error;
+    }
+}
+function repoSlug(repo) {
+    return slugify(repo.replace(/^https:\/\/github\.com\//, '').replace(/\.git$/, ''));
+}
+function sparseDir(skillPath) {
+    const dir = path.posix.dirname(skillPath);
+    return dir === '.' ? skillPath : dir;
+}
+function sourceName(manifest, skill) {
+    if (skill.source) {
+        return skill.source;
+    }
+    const repo = skill.repo ?? manifest.repo;
+    return repo ? repo.replace(/^https:\/\/github\.com\//, '').replace(/\.git$/, '') : manifest.name;
+}
+function sourceLabel(manifest, skill) {
+    const repo = skill.repo ?? manifest.repo;
+    const commit = skill.commit ?? manifest.commit ?? 'HEAD';
+    if (!repo) {
+        return `${sourceName(manifest, skill)}:${skill.path}`;
+    }
+    return `${repo.replace(/\.git$/, '')}/blob/${commit}/${skill.path}`;
+}
+async function prepareGitSource(source, repo, commit, paths) {
+    const checkoutDir = path.join('.cache', 'sources', `${slugify(source)}-${repoSlug(repo)}-${(commit ?? 'head').slice(0, 12)}`);
+    const sparseDirs = [...new Set(paths.map(sparseDir))].sort();
+    if (!(await pathExists(path.join(checkoutDir, '.git')))) {
+        await mkdir(path.dirname(checkoutDir), { recursive: true });
+        await execFile('git', ['clone', '--filter=blob:none', '--no-checkout', repo, checkoutDir]);
+    }
+    await execFile('git', ['-C', checkoutDir, 'sparse-checkout', 'init', '--cone']);
+    await execFile('git', ['-C', checkoutDir, 'sparse-checkout', 'set', ...sparseDirs]);
+    if (commit) {
+        await execFile('git', ['-C', checkoutDir, 'fetch', '--depth', '1', 'origin', commit]);
+        await execFile('git', ['-C', checkoutDir, 'checkout', '--quiet', commit]);
+    }
+    else {
+        await execFile('git', ['-C', checkoutDir, 'checkout', '--quiet']);
+    }
+    return checkoutDir;
+}
+async function prepareSources(corpusPath, manifest, skills) {
+    const roots = new Map();
+    const grouped = new Map();
+    for (const skill of skills) {
+        const repo = skill.repo ?? manifest.repo;
+        const commit = skill.commit ?? manifest.commit;
+        const source = sourceName(manifest, skill);
+        const key = repo ? `${repo}#${commit ?? 'HEAD'}` : `local:${path.dirname(path.resolve(corpusPath))}`;
+        const current = grouped.get(key) ?? { source, repo, commit, paths: [] };
+        current.paths.push(skill.path);
+        grouped.set(key, current);
+    }
+    for (const [key, group] of grouped) {
+        roots.set(key, group.repo
+            ? await prepareGitSource(group.source, group.repo, group.commit, group.paths)
+            : path.dirname(path.resolve(corpusPath)));
+    }
+    return roots;
+}
+async function writeJson(filePath, value) {
+    await mkdir(path.dirname(filePath), { recursive: true });
+    await writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`);
+}
+export async function runCorpus(options) {
+    const manifest = parseCorpusManifest(await readFile(options.corpus, 'utf8'));
+    const skills = manifest.skills.slice(0, options.limit ?? manifest.skills.length);
+    const roots = await prepareSources(options.corpus, manifest, skills);
+    const entries = new Array(skills.length);
+    async function runOne(skill, index) {
+        const repo = skill.repo ?? manifest.repo;
+        const commit = skill.commit ?? manifest.commit;
+        const source = sourceName(manifest, skill);
+        const key = repo ? `${repo}#${commit ?? 'HEAD'}` : `local:${path.dirname(path.resolve(options.corpus))}`;
+        const root = roots.get(key);
+        if (!root) {
+            throw new Error(`No prepared source root for ${source}`);
+        }
+        const outputPath = path.join(options.outputDir, `${slugify(source)}-${slugify(skill.id)}.json`);
+        const inputPath = path.join(root, skill.path);
+        const entry = {
+            id: skill.id,
+            source,
+            ...(repo ? { repo } : {}),
+            ...(commit ? { commit } : {}),
+            path: skill.path,
+            input_path: inputPath,
+            output_path: outputPath
+        };
+        if (await hasValidJsonFile(outputPath)) {
+            console.error(`[corpus] skip ${source}/${skill.id}; existing result`);
+            entries[index] = entry;
+            return;
+        }
+        console.error(`[corpus] eval ${source}/${skill.id}`);
+        await evalSkill({
+            inputPath,
+            output: outputPath,
+            tasks: options.tasks,
+            trials: options.trials,
+            mode: 'forced',
+            runner: options.runner,
+            sourceLabel: sourceLabel(manifest, skill)
+        });
+        entries[index] = entry;
+    }
+    let nextIndex = 0;
+    const workerCount = Math.min(options.concurrency, skills.length);
+    await Promise.all(Array.from({ length: workerCount }, async () => {
+        while (nextIndex < skills.length) {
+            const index = nextIndex;
+            nextIndex += 1;
+            await runOne(skills[index], index);
+        }
+    }));
+    if (entries.some((entry) => !entry)) {
+        throw new Error('Corpus run completed without producing all result entries');
+    }
+    const report = {
+        corpus: options.corpus,
+        output_dir: options.outputDir,
+        tasks: options.tasks,
+        trials: options.trials,
+        concurrency: options.concurrency,
+        ...(options.runner ? { runner: options.runner } : {}),
+        skills: entries
+    };
+    await writeJson(path.join(options.outputDir, 'summary.json'), report);
+    return report;
+}
+//# sourceMappingURL=corpus.js.map

package/dist/src/corpus.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpus.js","sourceRoot":"","sources":["../../packages/cli/src/corpus.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpE,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,MAAM,QAAQ,GAAG,SAAS,CAAC,gBAAgB,CAAC,CAAC;AA+C7C,MAAM,UAAU,OAAO,CAAC,KAAa;IACnC,OAAO,KAAK;SACT,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAC3B,CAAC;AAED,SAAS,OAAO,CAAC,KAAa;IAC5B,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc;IAC1C,MAAM,QAAQ,GAAG,KAAgC,CAAC;IAClD,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACvE,CAAC;IACD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,gBAAgB,KAAK,GAAG,CAAC,2BAA2B,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IACD,OAAO,QAA0B,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,OAAO,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAY,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,QAAQ,GAAmB,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IAC1D,IAAI,YAA8C,CAAC;IAEnD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACvC,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3B,SAAS;QACX,CAAC;QAED,MAAM,QAAQ,GAAG,wBAAwB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC/D,IAAI,QAAQ,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACzC,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,GAAG,EAAE,CAAC,GAAG,QAAQ,CAAC;YACrC,IAAI,GAAG,KAAK,MAAM;gBAAE,QAAQ,CAAC,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;YACnD,IAAI,GAAG,KAAK,MAAM;gBAAE,QAAQ,CAAC,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;YACnD,IAAI,GAAG,KAAK,QAAQ;gBAAE,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;YACvD,SAAS;QACX,CAAC;QAED,MAAM,UAAU,GAAG,qBAAqB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC9D,IAAI,UAAU,EAAE,CAAC;YACf,YAAY,GAAG,EAAE,EAAE,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;YAC/C,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,YAA2B,CAAC,CAAC;YAClD,SAAS;QACX,CAAC;QAED,MAAM,UAAU,GAAG,2BAA2B,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACpE,IAAI,UAAU,IAAI,YAAY,EAAE,CAAC;YAC/B,MAAM,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,UAAU,CAAC;YAClC,IAAI,GAAG,KAAK,MAAM;gBAAE,YAAY,CAAC,IAAI,GAAG,OAAO,CAAC,KAAM,CAAC,CAAC;YACxD,IAAI,GAAG,KAAK,IAAI;gBAAE,YAAY,CAAC,EAAE,GAAG,OAAO,CAAC,KAAM,CAAC,CAAC;YACpD,IAAI,GAAG,KAAK,QAAQ;gBAAE,YAAY,CAAC,MAAM,GAAG,OAAO,CAAC,KAAM,CAAC,CAAC;YAC5D,IAAI,GAAG,KAAK,MAAM;gBAAE,YAAY,CAAC,IAAI,GAAG,OAAO,CAAC,KAAM,CAAC,CAAC;YACxD,IAAI,GAAG,KAAK,QAAQ;gBAAE,YAAY,CAAC,MAAM,GAAG,OAAO,CAAC,KAAM,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAChF,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;IACpF,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,QAAgB;IACxC,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,QAAgB;IAC9C,IAAI,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,KAAK,YAAY,WAAW,EAAE,CAAC;YACjC,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,0BAA0B,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;AACrF,CAAC;AAED,SAAS,SAAS,CAAC,SAAiB;IAClC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC1C,OAAO,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC;AACvC,CAAC;AAED,SAAS,UAAU,CAAC,QAAwB,EAAE,KAAkB;IAC9D,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,OAAO,KAAK,CAAC,MAAM,CAAC;IACtB,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC;IACzC,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,0BAA0B,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC;AACnG,CAAC;AAED,SAAS,WAAW,CAAC,QAAwB,EAAE,KAAkB;IAC/D,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC;IACzC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,IAAI,MAAM,CAAC;IACzD,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,GAAG,UAAU,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;IACxD,CAAC;IACD,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,SAAS,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;AACtE,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,MAAc,EAAE,IAAY,EAAE,MAA0B,EAAE,KAAe;IACvG,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9H,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAE7D,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;QACxD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,oBAAoB,EAAE,eAAe,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;IAC7F,CAAC;IAED,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;IAChF,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE,KAAK,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC;IACpF,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;QACtF,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5E,CAAC;SAAM,CAAC;QACN,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,UAAkB,EAClB,QAAwB,EACxB,MAAqB;IAErB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,MAAM,OAAO,GAAG,IAAI,GAAG,EAA+E,CAAC;IAEvG,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC;QACzC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC;QAC/C,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,MAAM,IAAI,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QACrG,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACxE,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/B,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC5B,CAAC;IAED,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACnC,KAAK,CAAC,GAAG,CACP,GAAG,EACH,KAAK,CAAC,IAAI;YACR,CAAC,CAAC,MAAM,gBAAgB,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC;YAC7E,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAC3C,CAAC;IACJ,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,KAAc;IACvD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,MAAM,SAAS,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB;IACvD,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACjF,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IACrE,MAAM,OAAO,GAAqB,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE3D,KAAK,UAAU,MAAM,CAAC,KAAkB,EAAE,KAAa;QACrD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC;QACzC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC;QAC/C,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,MAAM,IAAI,MAAM,EAAE,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;QACzG,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;QAChG,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAmB;YAC5B,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,MAAM;YACN,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7B,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,UAAU,EAAE,SAAS;YACrB,WAAW,EAAE,UAAU;SACxB,CAAC;QAEF,IAAI,MAAM,gBAAgB,CAAC,UAAU,CAAC,EAAE,CAAC;YACvC,OAAO,CAAC,KAAK,CAAC,iBAAiB,MAAM,IAAI,KAAK,CAAC,EAAE,mBAAmB,CAAC,CAAC;YACtE,OAAO,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;YACvB,OAAO;QACT,CAAC;QAED,OAAO,CAAC,KAAK,CAAC,iBAAiB,MAAM,IAAI,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;QACrD,MAAM,SAAS,CAAC;YACd,SAAS;YACT,MAAM,EAAE,UAAU;YAClB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,WAAW,EAAE,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC;SAC1C,CAAC,CAAC;QAEH,OAAO,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;IACzB,CAAC;IAED,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACjE,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,KAAK,IAAI,EAAE;QAC7C,OAAO,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,SAAS,CAAC;YACxB,SAAS,IAAI,CAAC,CAAC;YACf,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,CAAE,EAAE,KAAK,CAAC,CAAC;QACtC,CAAC;IACH,CAAC,CAAC,CACH,CAAC;IAEF,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC/E,CAAC;IAED,MAAM,MAAM,GAAoB;QAC9B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,UAAU,EAAE,OAAO,CAAC,SAAS;QAC7B,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,MAAM,EAAE,OAA2B;KACpC,CAAC;IACF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAAC;IACtE,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/src/deterministic.d.ts

@@ -0,0 +1,7 @@
+import type { GeneratedTask } from './types.js';
+export interface DeterministicGrade {
+    score: number;
+    reason: string;
+    pass: boolean;
+}
+export declare function gradeDeterministically(task: GeneratedTask, output: string): DeterministicGrade;

package/dist/src/deterministic.js

@@ -0,0 +1,25 @@
+export function gradeDeterministically(task, output) {
+    if (task.criterionType !== 'deterministic') {
+        throw new Error(`Task ${task.id} is not deterministic`);
+    }
+    if (task.criterion.startsWith('regex:')) {
+        const pattern = task.criterion.slice('regex:'.length);
+        const pass = new RegExp(pattern, 's').test(output);
+        return {
+            score: pass ? 1 : 0,
+            reason: pass ? `matched regex ${pattern}` : `did not match regex ${pattern}`,
+            pass
+        };
+    }
+    if (task.criterion.startsWith('includes:')) {
+        const expected = task.criterion.slice('includes:'.length);
+        const pass = output.includes(expected);
+        return {
+            score: pass ? 1 : 0,
+            reason: pass ? `included ${expected}` : `did not include ${expected}`,
+            pass
+        };
+    }
+    throw new Error(`Unsupported deterministic criterion for ${task.id}: ${task.criterion}`);
+}
+//# sourceMappingURL=deterministic.js.map

package/dist/src/deterministic.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"deterministic.js","sourceRoot":"","sources":["../../packages/cli/src/deterministic.ts"],"names":[],"mappings":"AAQA,MAAM,UAAU,sBAAsB,CAAC,IAAmB,EAAE,MAAc;IACxE,IAAI,IAAI,CAAC,aAAa,KAAK,eAAe,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,QAAQ,IAAI,CAAC,EAAE,uBAAuB,CAAC,CAAC;IAC1D,CAAC;IAED,IAAI,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtD,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnD,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACnB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,iBAAiB,OAAO,EAAE,CAAC,CAAC,CAAC,uBAAuB,OAAO,EAAE;YAC5E,IAAI;SACL,CAAC;IACJ,CAAC;IAED,IAAI,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAC1D,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACvC,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACnB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,YAAY,QAAQ,EAAE,CAAC,CAAC,CAAC,mBAAmB,QAAQ,EAAE;YACrE,IAAI;SACL,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,2CAA2C,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;AAC3F,CAAC"}

package/dist/src/env.d.ts

@@ -0,0 +1,12 @@
+export interface NvidiaConfig {
+    apiKey: string;
+    baseUrl: string;
+    timeoutMs: number;
+    requestDelayMs: number;
+    maxAttempts: number;
+    maxRetryDelayMs: number;
+    generatorModel: string;
+    graderModel: string;
+    runnerModel: string;
+}
+export declare function loadNvidiaConfig(): NvidiaConfig;

package/dist/src/env.js

@@ -0,0 +1,39 @@
+import dotenv from 'dotenv';
+dotenv.config();
+function requireEnv(name) {
+    const value = process.env[name]?.trim();
+    if (!value) {
+        throw new Error(`Missing required environment variable: ${name}`);
+    }
+    return value;
+}
+export function loadNvidiaConfig() {
+    const timeoutMs = Number(process.env.NVIDIA_TIMEOUT_MS?.trim() || 120000);
+    if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
+        throw new Error('NVIDIA_TIMEOUT_MS must be a positive number when set');
+    }
+    const requestDelayMs = Number(process.env.NVIDIA_REQUEST_DELAY_MS?.trim() || 5000);
+    if (!Number.isFinite(requestDelayMs) || requestDelayMs < 0) {
+        throw new Error('NVIDIA_REQUEST_DELAY_MS must be a non-negative number when set');
+    }
+    const maxAttempts = Number(process.env.NVIDIA_MAX_ATTEMPTS?.trim() || 8);
+    if (!Number.isInteger(maxAttempts) || maxAttempts <= 0) {
+        throw new Error('NVIDIA_MAX_ATTEMPTS must be a positive integer when set');
+    }
+    const maxRetryDelayMs = Number(process.env.NVIDIA_MAX_RETRY_DELAY_MS?.trim() || 60000);
+    if (!Number.isFinite(maxRetryDelayMs) || maxRetryDelayMs <= 0) {
+        throw new Error('NVIDIA_MAX_RETRY_DELAY_MS must be a positive number when set');
+    }
+    return {
+        apiKey: requireEnv('NVIDIA_API_KEY'),
+        baseUrl: process.env.NVIDIA_BASE_URL?.trim() || 'https://integrate.api.nvidia.com/v1',
+        timeoutMs,
+        requestDelayMs,
+        maxAttempts,
+        maxRetryDelayMs,
+        generatorModel: requireEnv('NVIDIA_GENERATOR_MODEL'),
+        graderModel: requireEnv('NVIDIA_GRADER_MODEL'),
+        runnerModel: requireEnv('NVIDIA_RUNNER_MODEL')
+    };
+}
+//# sourceMappingURL=env.js.map

package/dist/src/env.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"env.js","sourceRoot":"","sources":["../../packages/cli/src/env.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,MAAM,CAAC,MAAM,EAAE,CAAC;AAchB,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC;IACxC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,0CAA0C,IAAI,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,CAAC;IAC1E,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC;IACnF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;IACpF,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IACzE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,eAAe,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,CAAC;IACvF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,CAAC;IAED,OAAO;QACL,MAAM,EAAE,UAAU,CAAC,gBAAgB,CAAC;QACpC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,EAAE,IAAI,qCAAqC;QACrF,SAAS;QACT,cAAc;QACd,WAAW;QACX,eAAe;QACf,cAAc,EAAE,UAAU,CAAC,wBAAwB,CAAC;QACpD,WAAW,EAAE,UAAU,CAAC,qBAAqB,CAAC;QAC9C,WAAW,EAAE,UAAU,CAAC,qBAAqB,CAAC;KAC/C,CAAC;AACJ,CAAC"}

package/dist/src/eval.d.ts

@@ -0,0 +1,13 @@
+export interface EvalOptions {
+    inputPath: string;
+    output?: string;
+    tasks: number;
+    trials: number;
+    mode: 'forced';
+    runner?: string;
+    grader?: string;
+    generator?: string;
+    taskSuite?: string;
+    sourceLabel?: string;
+}
+export declare function evalSkill(options: EvalOptions): Promise<unknown>;