@sx4im/skillcheck 0.1.0

package/dist/src/normalize.js

@@ -0,0 +1,109 @@
+import { readdir, readFile, stat } from 'node:fs/promises';
+import path from 'node:path';
+import { sha256 } from './hash.js';
+const SUPPORTED_FILES = [
+    { file: 'SKILL.md', format: 'SKILL.md' },
+    { file: 'AGENTS.md', format: 'AGENTS.md' },
+    { file: 'CLAUDE.md', format: 'CLAUDE.md' },
+    { file: '.cursorrules', format: '.cursorrules' }
+];
+async function resolveSkillFile(inputPath) {
+    const stats = await stat(inputPath);
+    if (stats.isDirectory()) {
+        for (const candidate of SUPPORTED_FILES) {
+            const filePath = path.join(inputPath, candidate.file);
+            try {
+                const candidateStats = await stat(filePath);
+                if (candidateStats.isFile()) {
+                    return { filePath, format: candidate.format };
+                }
+            }
+            catch (error) {
+                if (error.code !== 'ENOENT') {
+                    throw error;
+                }
+            }
+        }
+        throw new Error(`No supported skill file found in ${inputPath}`);
+    }
+    const basename = path.basename(inputPath);
+    const supported = SUPPORTED_FILES.find((candidate) => candidate.file === basename);
+    if (!supported) {
+        throw new Error(`Unsupported skill file: ${inputPath}`);
+    }
+    return { filePath: inputPath, format: supported.format };
+}
+function extractFrontMatter(text) {
+    if (!text.startsWith('---\n')) {
+        return {};
+    }
+    const end = text.indexOf('\n---', 4);
+    if (end === -1) {
+        return {};
+    }
+    const fields = {};
+    for (const line of text.slice(4, end).split('\n')) {
+        const match = /^([A-Za-z0-9_-]+):\s*(.+)$/.exec(line.trim());
+        if (match) {
+            fields[match[1].toLowerCase()] = match[2].trim().replace(/^["']|["']$/g, '');
+        }
+    }
+    return fields;
+}
+function firstHeading(text) {
+    return text
+        .split('\n')
+        .map((line) => /^#\s+(.+)$/.exec(line.trim())?.[1]?.trim())
+        .find(Boolean);
+}
+function inferDomain(text) {
+    return firstHeading(text) || 'general agent skill';
+}
+function extractDomain(text) {
+    const frontMatter = extractFrontMatter(text);
+    const declared = frontMatter.domain ||
+        frontMatter.description ||
+        frontMatter.when_to_use ||
+        frontMatter['when-to-use'];
+    if (declared) {
+        return declared;
+    }
+    const descriptionLine = /^description:\s*(.+)$/im.exec(text)?.[1]?.trim();
+    if (descriptionLine) {
+        return descriptionLine;
+    }
+    const whenLine = /^(when_to_use|when to use|when-to-use):\s*(.+)$/im.exec(text)?.[2]?.trim();
+    if (whenLine) {
+        return whenLine;
+    }
+    return inferDomain(text);
+}
+async function listAssets(skillFilePath) {
+    const dir = path.dirname(skillFilePath);
+    const skillName = path.basename(skillFilePath);
+    try {
+        const entries = await readdir(dir, { withFileTypes: true });
+        return entries
+            .filter((entry) => entry.name !== skillName && !entry.name.startsWith('.git'))
+            .map((entry) => entry.name)
+            .sort();
+    }
+    catch {
+        return [];
+    }
+}
+export async function normalizeSkill(inputPath) {
+    const { filePath, format } = await resolveSkillFile(inputPath);
+    const instructions = await readFile(filePath, 'utf8');
+    const name = firstHeading(instructions) || path.basename(path.dirname(filePath)) || path.basename(filePath);
+    return {
+        name,
+        sourcePath: filePath,
+        format,
+        instructions,
+        domain: extractDomain(instructions),
+        assets: await listAssets(filePath),
+        versionHash: sha256(instructions)
+    };
+}
+//# sourceMappingURL=normalize.js.map

package/dist/src/normalize.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalize.js","sourceRoot":"","sources":["../../packages/cli/src/normalize.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AAGnC,MAAM,eAAe,GAAiD;IACpE,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE;IACxC,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE;IAC1C,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE;IAC1C,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE;CACjD,CAAC;AAEF,KAAK,UAAU,gBAAgB,CAAC,SAAiB;IAC/C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC;IACpC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;QACxB,KAAK,MAAM,SAAS,IAAI,eAAe,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;YACtD,IAAI,CAAC;gBACH,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC5C,IAAI,cAAc,CAAC,MAAM,EAAE,EAAE,CAAC;oBAC5B,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC;gBAChD,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBACvD,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,oCAAoC,SAAS,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;IACnF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,2BAA2B,SAAS,EAAE,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC;AAC3D,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9B,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IACrC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;QACf,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,4BAA4B,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,WAAW,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,IAAY;IAChC,OAAO,IAAI;SACR,KAAK,CAAC,IAAI,CAAC;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;SAC1D,IAAI,CAAC,OAAO,CAAC,CAAC;AACnB,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,OAAO,YAAY,CAAC,IAAI,CAAC,IAAI,qBAAqB,CAAC;AACrD,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,QAAQ,GACZ,WAAW,CAAC,MAAM;QAClB,WAAW,CAAC,WAAW;QACvB,WAAW,CAAC,WAAW;QACvB,WAAW,CAAC,aAAa,CAAC,CAAC;IAC7B,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,eAAe,GAAG,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;IAC1E,IAAI,eAAe,EAAE,CAAC;QACpB,OAAO,eAAe,CAAC;IACzB,CAAC;IAED,MAAM,QAAQ,GAAG,mDAAmD,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;IAC7F,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC;AAC3B,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,aAAqB;IAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;IAC/C,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,OAAO,OAAO;aACX,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC7E,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;aAC1B,IAAI,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,SAAiB;IACpD,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,IAAI,GAAG,YAAY,CAAC,YAAY,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAE5G,OAAO;QACL,IAAI;QACJ,UAAU,EAAE,QAAQ;QACpB,MAAM;QACN,YAAY;QACZ,MAAM,EAAE,aAAa,CAAC,YAAY,CAAC;QACnC,MAAM,EAAE,MAAM,UAAU,CAAC,QAAQ,CAAC;QAClC,WAAW,EAAE,MAAM,CAAC,YAAY,CAAC;KAClC,CAAC;AACJ,CAAC"}

package/dist/src/rot.d.ts

@@ -0,0 +1,62 @@
+type Verdict = 'helps' | 'placebo' | 'harms';
+type RotStatus = 'new' | 'stable' | 'rot';
+interface StoredResult {
+    skill: {
+        name: string;
+        source: string;
+        format: string;
+        commit_hash: string;
+        domain: string;
+    };
+    config: {
+        runner_model: string;
+        runner_version?: string;
+    };
+    result: {
+        effect_pp: number;
+        ci_pp: [number, number];
+        verdict: Verdict;
+    };
+    run_date: string;
+}
+export interface RotHistoryEntry {
+    result_id: string;
+    file_path: string;
+    runner_model: string;
+    runner_version: string;
+    run_date: string;
+    effect_pp: number;
+    ci_pp: [number, number];
+    verdict: Verdict;
+}
+export interface RotSkillReport {
+    key: string;
+    skill: StoredResult['skill'];
+    status: RotStatus;
+    latest: RotHistoryEntry;
+    changed_from?: RotHistoryEntry;
+    history: RotHistoryEntry[];
+}
+export interface RotReport {
+    generated_at: string;
+    results_dir: string;
+    model?: string;
+    summary: {
+        skills: number;
+        new: number;
+        stable: number;
+        rot: number;
+    };
+    skills: RotSkillReport[];
+}
+export interface RotOptions {
+    resultsDir: string;
+    output?: string;
+    model?: string;
+    corpus?: string;
+    tasks: number;
+    trials: number;
+}
+export declare function buildRotReport(resultsDir: string, model?: string): Promise<RotReport>;
+export declare function runRot(options: RotOptions): Promise<RotReport>;
+export {};

package/dist/src/rot.js

@@ -0,0 +1,156 @@
+import { mkdir, readdir, readFile, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { runCorpus } from './corpus.js';
+function slugify(value) {
+    return value
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '-')
+        .replace(/^-|-$/g, '');
+}
+function resultKey(result) {
+    return `${slugify(result.skill.name)}:${result.skill.commit_hash}`;
+}
+function isVerdict(value) {
+    return value === 'helps' || value === 'placebo' || value === 'harms';
+}
+function asStoredResult(value) {
+    const candidate = value;
+    if (!candidate.skill?.name ||
+        !candidate.skill.source ||
+        !candidate.skill.format ||
+        !candidate.skill.commit_hash ||
+        !candidate.skill.domain ||
+        !candidate.config?.runner_model ||
+        !candidate.result?.ci_pp ||
+        !isVerdict(candidate.result.verdict) ||
+        typeof candidate.result.effect_pp !== 'number' ||
+        !candidate.run_date) {
+        return undefined;
+    }
+    return candidate;
+}
+async function listJsonFiles(dir) {
+    let entries;
+    try {
+        entries = await readdir(dir, { withFileTypes: true });
+    }
+    catch (error) {
+        if (error.code === 'ENOENT') {
+            return [];
+        }
+        throw error;
+    }
+    const files = await Promise.all(entries.map(async (entry) => {
+        const fullPath = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            return listJsonFiles(fullPath);
+        }
+        return entry.isFile() && entry.name.endsWith('.json') ? [fullPath] : [];
+    }));
+    return files.flat();
+}
+async function readStoredResults(resultsDir) {
+    const files = await listJsonFiles(resultsDir);
+    const results = [];
+    for (const file of files) {
+        let parsed;
+        try {
+            parsed = JSON.parse(await readFile(file, 'utf8'));
+        }
+        catch {
+            continue;
+        }
+        const result = asStoredResult(parsed);
+        if (result) {
+            results.push({ file, result });
+        }
+    }
+    return results;
+}
+function compareHistory(a, b) {
+    const dateDelta = Date.parse(a.run_date) - Date.parse(b.run_date);
+    return dateDelta === 0 ? a.file_path.localeCompare(b.file_path) : dateDelta;
+}
+function historyEntry(resultsDir, file, result) {
+    const relative = path.relative(resultsDir, file);
+    return {
+        result_id: `${slugify(result.skill.name)}-${slugify(path.dirname(relative))}-${slugify(path.basename(relative, '.json'))}`,
+        file_path: relative,
+        runner_model: result.config.runner_model,
+        runner_version: result.config.runner_version ?? result.config.runner_model,
+        run_date: result.run_date,
+        effect_pp: result.result.effect_pp,
+        ci_pp: result.result.ci_pp,
+        verdict: result.result.verdict
+    };
+}
+function summarizeSkill(key, skill, history) {
+    const sorted = [...history].sort(compareHistory);
+    const latest = sorted.at(-1);
+    if (!latest) {
+        throw new Error(`No history for ${key}`);
+    }
+    const changedFrom = [...sorted.slice(0, -1)].reverse().find((entry) => entry.verdict === 'helps');
+    const status = sorted.length === 1 ? 'new' : changedFrom && latest.verdict !== 'helps' ? 'rot' : 'stable';
+    return {
+        key,
+        skill,
+        status,
+        latest,
+        ...(status === 'rot' ? { changed_from: changedFrom } : {}),
+        history: sorted
+    };
+}
+export async function buildRotReport(resultsDir, model) {
+    const absoluteResultsDir = path.resolve(resultsDir);
+    const storedResults = await readStoredResults(absoluteResultsDir);
+    const grouped = new Map();
+    for (const { file, result } of storedResults) {
+        const key = resultKey(result);
+        const current = grouped.get(key) ?? { skill: result.skill, history: [] };
+        current.history.push(historyEntry(absoluteResultsDir, file, result));
+        grouped.set(key, current);
+    }
+    const skills = [...grouped.entries()]
+        .map(([key, group]) => summarizeSkill(key, group.skill, group.history))
+        .sort((a, b) => a.skill.name.localeCompare(b.skill.name));
+    return {
+        generated_at: new Date().toISOString(),
+        results_dir: resultsDir,
+        ...(model ? { model } : {}),
+        summary: {
+            skills: skills.length,
+            new: skills.filter((skill) => skill.status === 'new').length,
+            stable: skills.filter((skill) => skill.status === 'stable').length,
+            rot: skills.filter((skill) => skill.status === 'rot').length
+        },
+        skills
+    };
+}
+async function rerunCorpus(options) {
+    if (!options.corpus) {
+        return;
+    }
+    const runId = `${new Date().toISOString().replace(/[:.]/g, '-')}-${slugify(options.model ?? 'env-runner')}`;
+    await runCorpus({
+        corpus: options.corpus,
+        outputDir: path.join(options.resultsDir, 'rot-runs', runId),
+        tasks: options.tasks,
+        trials: options.trials,
+        concurrency: 2,
+        runner: options.model
+    });
+}
+async function writeJson(filePath, value) {
+    await mkdir(path.dirname(filePath), { recursive: true });
+    await writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`);
+}
+export async function runRot(options) {
+    await rerunCorpus(options);
+    const report = await buildRotReport(options.resultsDir, options.model);
+    if (options.output) {
+        await writeJson(options.output, report);
+    }
+    return report;
+}
+//# sourceMappingURL=rot.js.map

package/dist/src/rot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rot.js","sourceRoot":"","sources":["../../packages/cli/src/rot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvE,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAmExC,SAAS,OAAO,CAAC,KAAa;IAC5B,OAAO,KAAK;SACT,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAC3B,CAAC;AAED,SAAS,SAAS,CAAC,MAAoB;IACrC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;AACrE,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,OAAO,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,OAAO,CAAC;AACvE,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,MAAM,SAAS,GAAG,KAA8B,CAAC;IACjD,IACE,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI;QACtB,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM;QACvB,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM;QACvB,CAAC,SAAS,CAAC,KAAK,CAAC,WAAW;QAC5B,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM;QACvB,CAAC,SAAS,CAAC,MAAM,EAAE,YAAY;QAC/B,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK;QACxB,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;QACpC,OAAO,SAAS,CAAC,MAAM,CAAC,SAAS,KAAK,QAAQ;QAC9C,CAAC,SAAS,CAAC,QAAQ,EACnB,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,SAAyB,CAAC;AACnC,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,GAAW;IACtC,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5C,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1E,CAAC,CAAC,CACH,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;AACtB,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,UAAkB;IACjD,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,UAAU,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAkD,EAAE,CAAC;IAElE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAY,CAAC;QAC/D,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QACtC,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,cAAc,CAAC,CAAkB,EAAE,CAAkB;IAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAClE,OAAO,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC9E,CAAC;AAED,SAAS,YAAY,CAAC,UAAkB,EAAE,IAAY,EAAE,MAAoB;IAC1E,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACjD,OAAO;QACL,SAAS,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,EAAE;QAC1H,SAAS,EAAE,QAAQ;QACnB,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,YAAY;QACxC,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,MAAM,CAAC,YAAY;QAC1E,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS;QAClC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK;QAC1B,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO;KAC/B,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,GAAW,EAAE,KAA4B,EAAE,OAA0B;IAC3F,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAC;IAC3C,CAAC;IACD,MAAM,WAAW,GAAG,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC;IAClG,MAAM,MAAM,GAAc,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,IAAI,MAAM,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC;IAErH,OAAO;QACL,GAAG;QACH,KAAK;QACL,MAAM;QACN,MAAM;QACN,GAAG,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1D,OAAO,EAAE,MAAM;KAChB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,UAAkB,EAAE,KAAc;IACrE,MAAM,kBAAkB,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,aAAa,GAAG,MAAM,iBAAiB,CAAC,kBAAkB,CAAC,CAAC;IAClE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAwE,CAAC;IAEhG,KAAK,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,aAAa,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;QAC9B,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QACzE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,kBAAkB,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;QACrE,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC5B,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;SAClC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;SACtE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAE5D,OAAO;QACL,YAAY,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACtC,WAAW,EAAE,UAAU;QACvB,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE;YACP,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,MAAM;YAC5D,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;YAClE,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,MAAM;SAC7D;QACD,MAAM;KACP,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,OAAmB;IAC5C,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,OAAO;IACT,CAAC;IAED,MAAM,KAAK,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,IAAI,YAAY,CAAC,EAAE,CAAC;IAC5G,MAAM,SAAS,CAAC;QACd,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,UAAU,EAAE,KAAK,CAAC;QAC3D,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,WAAW,EAAE,CAAC;QACd,MAAM,EAAE,OAAO,CAAC,KAAK;KACtB,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,KAAc;IACvD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,MAAM,SAAS,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,OAAmB;IAC9C,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;IAC3B,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACvE,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/src/run.d.ts

@@ -0,0 +1,5 @@
+import type { NvidiaNimClient } from './adapters/nvidia-nim.js';
+import type { JsonCache } from './cache.js';
+import type { NvidiaConfig } from './env.js';
+import type { GeneratedTask, NormalizedSkill, TrialOutput } from './types.js';
+export declare function runTrials(skill: NormalizedSkill, tasks: GeneratedTask[], trials: number, config: NvidiaConfig, client: NvidiaNimClient, cache: JsonCache): Promise<TrialOutput[]>;

package/dist/src/run.js

@@ -0,0 +1,47 @@
+import { hashJson } from './hash.js';
+function messagesForArm(skill, task, withSkill) {
+    if (!withSkill) {
+        return [{ role: 'user', content: task.prompt }];
+    }
+    return [
+        {
+            role: 'system',
+            content: `You are completing an evaluation task. Apply the following skill instructions when relevant.\n\n${skill.instructions}`
+        },
+        { role: 'user', content: task.prompt }
+    ];
+}
+async function runOne(skill, task, trial, arm, config, client, cache) {
+    const messages = messagesForArm(skill, task, arm === 'with_skill');
+    const response = await cache.getOrSet('runner', { model: config.runnerModel, temperature: 0.7, maxTokens: 1200, messages }, () => client.complete({
+        model: config.runnerModel,
+        messages,
+        temperature: 0.7,
+        maxTokens: 1200
+    }));
+    const transcriptHash = `sha256:${hashJson({ taskId: task.id, trial, arm, messages, output: response.content })}`;
+    return {
+        taskId: task.id,
+        trial,
+        arm,
+        output: response.content,
+        model: response.model,
+        promptTokens: response.usage?.promptTokens ?? 0,
+        completionTokens: response.usage?.completionTokens ?? 0,
+        totalTokens: response.usage?.totalTokens ?? 0,
+        transcriptHash
+    };
+}
+export async function runTrials(skill, tasks, trials, config, client, cache) {
+    const outputs = [];
+    for (const task of tasks) {
+        for (let trial = 1; trial <= trials; trial += 1) {
+            console.error(`[eval] run ${task.id} trial ${trial}/${trials} with_skill`);
+            outputs.push(await runOne(skill, task, trial, 'with_skill', config, client, cache));
+            console.error(`[eval] run ${task.id} trial ${trial}/${trials} no_skill`);
+            outputs.push(await runOne(skill, task, trial, 'no_skill', config, client, cache));
+        }
+    }
+    return outputs;
+}
+//# sourceMappingURL=run.js.map

package/dist/src/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../../packages/cli/src/run.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAGrC,SAAS,cAAc,CAAC,KAAsB,EAAE,IAAmB,EAAE,SAAkB;IACrF,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,OAAO;QACL;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,mGAAmG,KAAK,CAAC,YAAY,EAAE;SACjI;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE;KACvC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,MAAM,CACnB,KAAsB,EACtB,IAAmB,EACnB,KAAa,EACb,GAAuB,EACvB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,KAAK,YAAY,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,WAAW,EAAE,WAAW,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CAC/H,MAAM,CAAC,QAAQ,CAAC;QACd,KAAK,EAAE,MAAM,CAAC,WAAW;QACzB,QAAQ;QACR,WAAW,EAAE,GAAG;QAChB,SAAS,EAAE,IAAI;KAChB,CAAC,CACH,CAAC;IACF,MAAM,cAAc,GAAG,UAAU,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;IAEjH,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,KAAK;QACL,GAAG;QACH,MAAM,EAAE,QAAQ,CAAC,OAAO;QACxB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,YAAY,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC;QAC/C,gBAAgB,EAAE,QAAQ,CAAC,KAAK,EAAE,gBAAgB,IAAI,CAAC;QACvD,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC;QAC7C,cAAc;KACf,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,KAAsB,EACtB,KAAsB,EACtB,MAAc,EACd,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,OAAO,GAAkB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAChD,OAAO,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,EAAE,UAAU,KAAK,IAAI,MAAM,aAAa,CAAC,CAAC;YAC3E,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;YACpF,OAAO,CAAC,KAAK,CAAC,cAAc,IAAI,CAAC,EAAE,UAAU,KAAK,IAAI,MAAM,WAAW,CAAC,CAAC;YACzE,OAAO,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;QACpF,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/src/score.d.ts

@@ -0,0 +1,14 @@
+export interface PairedObservation {
+    withSkillPass: boolean;
+    noSkillPass: boolean;
+}
+export interface ScoreSummary {
+    effectPp: number;
+    ciPp: [number, number];
+    verdict: 'helps' | 'placebo' | 'harms';
+    withSkillPass: number;
+    noSkillPass: number;
+}
+export declare function scorePairedObservations(observations: PairedObservation[], iterations?: number, seed?: number): ScoreSummary;
+export declare function effectInsideCi(summary: ScoreSummary): boolean;
+export declare function ciOverlapsZero(summary: ScoreSummary): boolean;

package/dist/src/score.js

@@ -0,0 +1,59 @@
+function createSeededRandom(seed) {
+    let state = seed >>> 0;
+    return () => {
+        state = (state * 1664525 + 1013904223) >>> 0;
+        return state / 0x100000000;
+    };
+}
+function quantile(sortedValues, q) {
+    if (sortedValues.length === 0) {
+        throw new Error('Cannot compute quantile of an empty sample');
+    }
+    const index = (sortedValues.length - 1) * q;
+    const lower = Math.floor(index);
+    const upper = Math.ceil(index);
+    if (lower === upper) {
+        return sortedValues[lower];
+    }
+    const weight = index - lower;
+    return sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight;
+}
+function toPp(value) {
+    return Number((value * 100).toFixed(2));
+}
+export function scorePairedObservations(observations, iterations = 1000, seed = 1337) {
+    if (observations.length === 0) {
+        throw new Error('Cannot score zero observations');
+    }
+    const withSkillPass = observations.filter((observation) => observation.withSkillPass).length / observations.length;
+    const noSkillPass = observations.filter((observation) => observation.noSkillPass).length / observations.length;
+    const effect = withSkillPass - noSkillPass;
+    const random = createSeededRandom(seed);
+    const bootstrapEffects = [];
+    for (let iteration = 0; iteration < iterations; iteration += 1) {
+        let total = 0;
+        for (let index = 0; index < observations.length; index += 1) {
+            const sample = observations[Math.floor(random() * observations.length)];
+            total += Number(sample.withSkillPass) - Number(sample.noSkillPass);
+        }
+        bootstrapEffects.push(total / observations.length);
+    }
+    bootstrapEffects.sort((a, b) => a - b);
+    const lower = toPp(quantile(bootstrapEffects, 0.025));
+    const upper = toPp(quantile(bootstrapEffects, 0.975));
+    const verdict = lower > 0 ? 'helps' : upper < 0 ? 'harms' : 'placebo';
+    return {
+        effectPp: toPp(effect),
+        ciPp: [lower, upper],
+        verdict,
+        withSkillPass: Number(withSkillPass.toFixed(4)),
+        noSkillPass: Number(noSkillPass.toFixed(4))
+    };
+}
+export function effectInsideCi(summary) {
+    return summary.effectPp >= summary.ciPp[0] && summary.effectPp <= summary.ciPp[1];
+}
+export function ciOverlapsZero(summary) {
+    return summary.ciPp[0] <= 0 && summary.ciPp[1] >= 0;
+}
+//# sourceMappingURL=score.js.map

package/dist/src/score.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.js","sourceRoot":"","sources":["../../packages/cli/src/score.ts"],"names":[],"mappings":"AAaA,SAAS,kBAAkB,CAAC,IAAY;IACtC,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC,CAAC;IACvB,OAAO,GAAG,EAAE;QACV,KAAK,GAAG,CAAC,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,KAAK,GAAG,WAAW,CAAC;IAC7B,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,YAAsB,EAAE,CAAS;IACjD,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,KAAK,GAAG,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC;QACpB,OAAO,YAAY,CAAC,KAAK,CAAE,CAAC;IAC9B,CAAC;IAED,MAAM,MAAM,GAAG,KAAK,GAAG,KAAK,CAAC;IAC7B,OAAO,YAAY,CAAC,KAAK,CAAE,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,YAAY,CAAC,KAAK,CAAE,GAAG,MAAM,CAAC;AAC7E,CAAC;AAED,SAAS,IAAI,CAAC,KAAa;IACzB,OAAO,MAAM,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;AAC1C,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,YAAiC,EACjC,UAAU,GAAG,IAAI,EACjB,IAAI,GAAG,IAAI;IAEX,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IAED,MAAM,aAAa,GACjB,YAAY,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;IAC/F,MAAM,WAAW,GACf,YAAY,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC;IAC7F,MAAM,MAAM,GAAG,aAAa,GAAG,WAAW,CAAC;IAE3C,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,gBAAgB,GAAa,EAAE,CAAC;IAEtC,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,UAAU,EAAE,SAAS,IAAI,CAAC,EAAE,CAAC;QAC/D,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,YAAY,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAC5D,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,CAAE,CAAC;YACzE,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACrE,CAAC;QACD,gBAAgB,CAAC,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACrD,CAAC;IAED,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC,CAAC;IACtD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC,CAAC;IACtD,MAAM,OAAO,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;IAEtE,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;QACtB,IAAI,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC;QACpB,OAAO;QACP,aAAa,EAAE,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/C,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;KAC5C,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,OAAqB;IAClD,OAAO,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACpF,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,OAAqB;IAClD,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACtD,CAAC"}

package/dist/src/types.d.ts

@@ -0,0 +1,41 @@
+export type SkillFormat = 'SKILL.md' | 'AGENTS.md' | '.cursorrules' | 'CLAUDE.md';
+export type CriterionType = 'rubric' | 'deterministic';
+export interface NormalizedSkill {
+    name: string;
+    sourcePath: string;
+    format: SkillFormat;
+    instructions: string;
+    domain: string;
+    assets: string[];
+    versionHash: string;
+}
+export interface GeneratedTask {
+    id: string;
+    prompt: string;
+    criterionType: CriterionType;
+    criterion: string;
+}
+export interface TrialOutput {
+    taskId: string;
+    trial: number;
+    arm: 'with_skill' | 'no_skill';
+    output: string;
+    model: string;
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    transcriptHash: string;
+}
+export interface GradedOutput extends TrialOutput {
+    score: number;
+    reason: string;
+    pass: boolean;
+}
+export interface TaskBreakdown {
+    id: string;
+    prompt: string;
+    criterion_type: CriterionType;
+    criterion: string;
+    arm_a_pass_rate: number;
+    arm_b_pass_rate: number;
+}

package/dist/src/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/src/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../packages/cli/src/types.ts"],"names":[],"mappings":""}

package/dist/src/verify.d.ts

@@ -0,0 +1,5 @@
+export interface VerifyOptions {
+    resultPath: string;
+    sample: number;
+}
+export declare function verifyResult(options: VerifyOptions): Promise<unknown>;

package/dist/src/verify.js

@@ -0,0 +1,71 @@
+import { readFile } from 'node:fs/promises';
+import { NvidiaNimClient } from './adapters/nvidia-nim.js';
+import { JsonCache } from './cache.js';
+import { loadNvidiaConfig } from './env.js';
+import { gradeOutputs } from './grade.js';
+import { normalizeSkill } from './normalize.js';
+import { runTrials } from './run.js';
+import { scorePairedObservations } from './score.js';
+function parseTasks(text) {
+    const value = JSON.parse(text);
+    const tasks = Array.isArray(value) ? value : value.tasks;
+    if (!Array.isArray(tasks)) {
+        throw new Error('Task suite must be an array or object with tasks');
+    }
+    return tasks.map((task, index) => {
+        const item = task;
+        const criterionType = item.criterionType ?? item.criterion_type ?? 'rubric';
+        if (criterionType !== 'rubric' && criterionType !== 'deterministic') {
+            throw new Error(`Unsupported criterion type in task ${index + 1}`);
+        }
+        return {
+            id: String(item.id ?? `t${String(index + 1).padStart(3, '0')}`),
+            prompt: String(item.prompt ?? ''),
+            criterionType,
+            criterion: String(item.criterion ?? '')
+        };
+    });
+}
+function pairedObservations(graded) {
+    const byPair = new Map();
+    for (const item of graded) {
+        const key = `${item.taskId}:${item.trial}`;
+        const current = byPair.get(key) ?? {};
+        if (item.arm === 'with_skill') {
+            current.withSkillPass = item.pass;
+        }
+        else {
+            current.noSkillPass = item.pass;
+        }
+        byPair.set(key, current);
+    }
+    return [...byPair.values()].map((item) => {
+        if (typeof item.withSkillPass !== 'boolean' || typeof item.noSkillPass !== 'boolean') {
+            throw new Error('Incomplete A/B pair while verifying');
+        }
+        return item;
+    });
+}
+export async function verifyResult(options) {
+    const published = JSON.parse(await readFile(options.resultPath, 'utf8'));
+    const config = loadNvidiaConfig();
+    const client = new NvidiaNimClient(config);
+    const cache = new JsonCache();
+    const skill = await normalizeSkill(published.skill.source);
+    const tasks = parseTasks(await readFile(published.reproducibility.task_suite_path, 'utf8')).slice(0, options.sample);
+    const outputs = await runTrials(skill, tasks, published.config.trials, config, client, cache);
+    const graded = await gradeOutputs(tasks, outputs, config, client, cache);
+    const score = scorePairedObservations(pairedObservations(graded));
+    const [lower, upper] = published.result.ci_pp;
+    const passed = score.effectPp >= lower && score.effectPp <= upper;
+    return {
+        passed,
+        sample: tasks.length,
+        published_effect_pp: published.result.effect_pp,
+        published_ci_pp: published.result.ci_pp,
+        verify_effect_pp: score.effectPp,
+        verify_ci_pp: score.ciPp,
+        verify_verdict: score.verdict
+    };
+}
+//# sourceMappingURL=verify.js.map

package/dist/src/verify.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verify.js","sourceRoot":"","sources":["../../packages/cli/src/verify.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,uBAAuB,EAA0B,MAAM,YAAY,CAAC;AAQ7E,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAE,KAA6B,CAAC,KAAK,CAAC;IAClF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG,IAA+B,CAAC;QAC7C,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,cAAc,IAAI,QAAQ,CAAC;QAC5E,IAAI,aAAa,KAAK,QAAQ,IAAI,aAAa,KAAK,eAAe,EAAE,CAAC;YACpE,MAAM,IAAI,KAAK,CAAC,sCAAsC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;YACjC,aAAa;YACb,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;SACxC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAsB;IAChD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsC,CAAC;IAC7D,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,GAAG,KAAK,YAAY,EAAE,CAAC;YAC9B,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC;QAClC,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACvC,IAAI,OAAO,IAAI,CAAC,aAAa,KAAK,SAAS,IAAI,OAAO,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACrF,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,OAAO,IAAyB,CAAC;IACnC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAsB;IACvD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,CAKtE,CAAC;IACF,MAAM,MAAM,GAAG,gBAAgB,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,IAAI,SAAS,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,QAAQ,CAAC,SAAS,CAAC,eAAe,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IACrH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAC9F,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IACzE,MAAM,KAAK,GAAG,uBAAuB,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC;IAClE,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC;IAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,IAAI,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC;IAElE,OAAO;QACL,MAAM;QACN,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,mBAAmB,EAAE,SAAS,CAAC,MAAM,CAAC,SAAS;QAC/C,eAAe,EAAE,SAAS,CAAC,MAAM,CAAC,KAAK;QACvC,gBAAgB,EAAE,KAAK,CAAC,QAAQ;QAChC,YAAY,EAAE,KAAK,CAAC,IAAI;QACxB,cAAc,EAAE,KAAK,CAAC,OAAO;KAC9B,CAAC;AACJ,CAAC"}