@sx4im/skillcheck 0.1.0

package/dist/src/eval.js

@@ -0,0 +1,155 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { NvidiaNimClient } from './adapters/nvidia-nim.js';
+import { JsonCache } from './cache.js';
+import { loadNvidiaConfig } from './env.js';
+import { generateTasks } from './generate.js';
+import { hashJson } from './hash.js';
+import { gradeOutputs } from './grade.js';
+import { normalizeSkill } from './normalize.js';
+import { runTrials } from './run.js';
+import { scorePairedObservations } from './score.js';
+function applyModelOverrides(config, options) {
+    return {
+        ...config,
+        runnerModel: options.runner ?? config.runnerModel,
+        graderModel: options.grader ?? config.graderModel,
+        generatorModel: options.generator ?? config.generatorModel
+    };
+}
+function pairedObservations(graded) {
+    const byPair = new Map();
+    for (const item of graded) {
+        const key = `${item.taskId}:${item.trial}`;
+        const current = byPair.get(key) ?? {};
+        if (item.arm === 'with_skill') {
+            current.withSkillPass = item.pass;
+        }
+        else {
+            current.noSkillPass = item.pass;
+        }
+        byPair.set(key, current);
+    }
+    return [...byPair.values()].map((item) => {
+        if (typeof item.withSkillPass !== 'boolean' || typeof item.noSkillPass !== 'boolean') {
+            throw new Error('Incomplete A/B pair while scoring');
+        }
+        return item;
+    });
+}
+function taskBreakdowns(tasks, graded) {
+    return tasks.map((task) => {
+        const taskGrades = graded.filter((item) => item.taskId === task.id);
+        const withSkill = taskGrades.filter((item) => item.arm === 'with_skill');
+        const noSkill = taskGrades.filter((item) => item.arm === 'no_skill');
+        return {
+            id: task.id,
+            prompt: task.prompt,
+            criterion_type: task.criterionType,
+            criterion: task.criterion,
+            arm_a_pass_rate: withSkill.filter((item) => item.pass).length / withSkill.length,
+            arm_b_pass_rate: noSkill.filter((item) => item.pass).length / noSkill.length
+        };
+    });
+}
+function mean(values) {
+    return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+async function writeJson(filePath, value) {
+    await mkdir(path.dirname(filePath), { recursive: true });
+    await writeFile(filePath, `${JSON.stringify(value, null, 2)}\n`);
+}
+function parseTaskSuite(text) {
+    const value = JSON.parse(text);
+    const tasks = Array.isArray(value)
+        ? value
+        : typeof value === 'object' && value !== null && Array.isArray(value.tasks)
+            ? value.tasks
+            : undefined;
+    if (!tasks) {
+        throw new Error('Task suite must be an array or an object with a tasks array');
+    }
+    return tasks.map((task, index) => {
+        const item = task;
+        const criterionType = item.criterionType ?? item.criterion_type ?? 'rubric';
+        if (criterionType !== 'rubric' && criterionType !== 'deterministic') {
+            throw new Error(`Unsupported criterion type in task ${index + 1}`);
+        }
+        return {
+            id: String(item.id ?? `t${String(index + 1).padStart(3, '0')}`),
+            prompt: String(item.prompt ?? ''),
+            criterionType,
+            criterion: String(item.criterion ?? '')
+        };
+    });
+}
+export async function evalSkill(options) {
+    const baseConfig = loadNvidiaConfig();
+    const config = applyModelOverrides(baseConfig, options);
+    const client = new NvidiaNimClient(config);
+    const cache = new JsonCache();
+    const skill = await normalizeSkill(options.inputPath);
+    const tasks = options.taskSuite
+        ? parseTaskSuite(await readFile(options.taskSuite, 'utf8')).slice(0, options.tasks)
+        : await generateTasks({ domain: skill.domain, count: options.tasks }, config, client, cache);
+    const taskSuiteHash = hashJson({ skill: skill.versionHash, tasks });
+    const taskSuitePath = options.taskSuite ?? `results/tasks/${taskSuiteHash}.json`;
+    await writeJson(taskSuitePath, tasks);
+    const outputs = await runTrials(skill, tasks, options.trials, config, client, cache);
+    const graded = await gradeOutputs(tasks, outputs, config, client, cache);
+    const score = scorePairedObservations(pairedObservations(graded));
+    const breakdowns = taskBreakdowns(tasks, graded);
+    const withSkillTokens = graded.filter((item) => item.arm === 'with_skill').map((item) => item.promptTokens);
+    const noSkillTokens = graded.filter((item) => item.arm === 'no_skill').map((item) => item.promptTokens);
+    const tokenOverhead = Math.max(0, Math.round(mean(withSkillTokens) - mean(noSkillTokens)));
+    const valuePer1kTokens = tokenOverhead === 0 ? 0 : Number((score.effectPp / (tokenOverhead / 1000)).toFixed(2));
+    const runDate = new Date().toISOString().slice(0, 10);
+    const result = {
+        skill: {
+            name: skill.name,
+            source: options.sourceLabel ?? skill.sourcePath,
+            format: skill.format,
+            commit_hash: skill.versionHash,
+            domain: skill.domain
+        },
+        config: {
+            runner_model: config.runnerModel,
+            runner_version: config.runnerModel,
+            grader_model: config.graderModel,
+            grader_version: config.graderModel,
+            generator_model: config.generatorModel,
+            trials: options.trials,
+            tasks: options.tasks,
+            temperature: 0.7,
+            mode: 'forced-injection'
+        },
+        result: {
+            effect_pp: score.effectPp,
+            ci_pp: score.ciPp,
+            verdict: score.verdict,
+            with_skill_pass: score.withSkillPass,
+            no_skill_pass: score.noSkillPass,
+            token_overhead: tokenOverhead,
+            value_per_1k_tokens: valuePer1kTokens
+        },
+        tasks: breakdowns,
+        reproducibility: {
+            task_suite_path: taskSuitePath,
+            transcript_hashes: graded.map((item) => item.transcriptHash)
+        },
+        history: [
+            {
+                runner_version: config.runnerModel,
+                run_date: runDate,
+                effect_pp: score.effectPp,
+                verdict: score.verdict
+            }
+        ],
+        run_date: runDate
+    };
+    if (options.output) {
+        await writeJson(options.output, result);
+    }
+    return result;
+}
+//# sourceMappingURL=eval.js.map

package/dist/src/eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../packages/cli/src/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAqB,MAAM,UAAU,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,uBAAuB,EAA0B,MAAM,YAAY,CAAC;AAgB7E,SAAS,mBAAmB,CAAC,MAAoB,EAAE,OAAoB;IACrE,OAAO;QACL,GAAG,MAAM;QACT,WAAW,EAAE,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,WAAW;QACjD,WAAW,EAAE,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,WAAW;QACjD,cAAc,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,cAAc;KAC3D,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAsB;IAChD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsC,CAAC;IAC7D,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,GAAG,KAAK,YAAY,EAAE,CAAC;YAC9B,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC;QACpC,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC;QAClC,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACvC,IAAI,OAAO,IAAI,CAAC,aAAa,KAAK,SAAS,IAAI,OAAO,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACrF,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,IAAyB,CAAC;IACnC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,KAAsB,EAAE,MAAsB;IACpE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,YAAY,CAAC,CAAC;QACzE,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC;QACrE,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,cAAc,EAAE,IAAI,CAAC,aAAa;YAClC,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,eAAe,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM;YAChF,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;SAC7E,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,IAAI,CAAC,MAAgB;IAC5B,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACjG,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,KAAc;IACvD,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,MAAM,SAAS,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAChC,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAE,KAA6B,CAAC,KAAK,CAAC;YAClG,CAAC,CAAE,KAA8B,CAAC,KAAK;YACvC,CAAC,CAAC,SAAS,CAAC;IAChB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACjF,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC/B,MAAM,IAAI,GAAG,IAA+B,CAAC;QAC7C,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,cAAc,IAAI,QAAQ,CAAC;QAC5E,IAAI,aAAa,KAAK,QAAQ,IAAI,aAAa,KAAK,eAAe,EAAE,CAAC;YACpE,MAAM,IAAI,KAAK,CAAC,sCAAsC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;YACjC,aAAa;YACb,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;SACxC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAoB;IAClD,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAC;IACtC,MAAM,MAAM,GAAG,mBAAmB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,IAAI,SAAS,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAEtD,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS;QAC7B,CAAC,CAAC,cAAc,CAAC,MAAM,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC;QACnF,CAAC,CAAC,MAAM,aAAa,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IAC/F,MAAM,aAAa,GAAG,QAAQ,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;IACpE,MAAM,aAAa,GAAG,OAAO,CAAC,SAAS,IAAI,iBAAiB,aAAa,OAAO,CAAC;IACjF,MAAM,SAAS,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;IAEtC,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IACrF,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;IACzE,MAAM,KAAK,GAAG,uBAAuB,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC;IAClE,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACjD,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5G,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACxG,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAC3F,MAAM,gBAAgB,GAAG,aAAa,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IAChH,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEtD,MAAM,MAAM,GAAG;QACb,KAAK,EAAE;YACL,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,MAAM,EAAE,OAAO,CAAC,WAAW,IAAI,KAAK,CAAC,UAAU;YAC/C,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;SACrB;QACD,MAAM,EAAE;YACN,YAAY,EAAE,MAAM,CAAC,WAAW;YAChC,cAAc,EAAE,MAAM,CAAC,WAAW;YAClC,YAAY,EAAE,MAAM,CAAC,WAAW;YAChC,cAAc,EAAE,MAAM,CAAC,WAAW;YAClC,eAAe,EAAE,MAAM,CAAC,cAAc;YACtC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,WAAW,EAAE,GAAG;YAChB,IAAI,EAAE,kBAAkB;SACzB;QACD,MAAM,EAAE;YACN,SAAS,EAAE,KAAK,CAAC,QAAQ;YACzB,KAAK,EAAE,KAAK,CAAC,IAAI;YACjB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,eAAe,EAAE,KAAK,CAAC,aAAa;YACpC,aAAa,EAAE,KAAK,CAAC,WAAW;YAChC,cAAc,EAAE,aAAa;YAC7B,mBAAmB,EAAE,gBAAgB;SACtC;QACD,KAAK,EAAE,UAAU;QACjB,eAAe,EAAE;YACf,eAAe,EAAE,aAAa;YAC9B,iBAAiB,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC;SAC7D;QACD,OAAO,EAAE;YACP;gBACE,cAAc,EAAE,MAAM,CAAC,WAAW;gBAClC,QAAQ,EAAE,OAAO;gBACjB,SAAS,EAAE,KAAK,CAAC,QAAQ;gBACzB,OAAO,EAAE,KAAK,CAAC,OAAO;aACvB;SACF;QACD,QAAQ,EAAE,OAAO;KAClB,CAAC;IAEF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/src/generate.d.ts

@@ -0,0 +1,9 @@
+import type { NvidiaNimClient } from './adapters/nvidia-nim.js';
+import type { JsonCache } from './cache.js';
+import type { NvidiaConfig } from './env.js';
+import type { GeneratedTask } from './types.js';
+export interface TaskGenerationInput {
+    domain: string;
+    count: number;
+}
+export declare function generateTasks(input: TaskGenerationInput, config: NvidiaConfig, client: NvidiaNimClient, cache: JsonCache): Promise<GeneratedTask[]>;

package/dist/src/generate.js

@@ -0,0 +1,94 @@
+import { hashJson } from './hash.js';
+function extractJsonPayload(text) {
+    const trimmed = text.trim();
+    if (trimmed.startsWith('[') || trimmed.startsWith('{')) {
+        return JSON.parse(trimmed);
+    }
+    const arrayStart = trimmed.indexOf('[');
+    const arrayEnd = trimmed.lastIndexOf(']');
+    const objectStart = trimmed.indexOf('{');
+    const objectEnd = trimmed.lastIndexOf('}');
+    const useObject = objectStart !== -1 && (arrayStart === -1 || objectStart < arrayStart);
+    const start = useObject ? objectStart : arrayStart;
+    const end = useObject ? objectEnd : arrayEnd;
+    if (start === -1 || end === -1 || end <= start) {
+        throw new Error(`Generator did not return JSON: ${trimmed.slice(0, 200)}`);
+    }
+    return JSON.parse(trimmed.slice(start, end + 1));
+}
+function validateTasks(value, count) {
+    const array = Array.isArray(value)
+        ? value
+        : typeof value === 'object' && value !== null && Array.isArray(value.tasks)
+            ? value.tasks
+            : undefined;
+    if (!array) {
+        throw new Error('Generated task payload must be an array');
+    }
+    return array.slice(0, count).map((item, index) => {
+        if (typeof item !== 'object' || item === null) {
+            throw new Error(`Generated task ${index} is not an object`);
+        }
+        const task = item;
+        const prompt = String(task.prompt ?? '').trim();
+        const criterion = String(task.criterion ?? task.rubric ?? '').trim();
+        if (!prompt || !criterion) {
+            throw new Error(`Generated task ${index} is missing prompt or criterion`);
+        }
+        return {
+            id: String(task.id ?? `t${String(index + 1).padStart(3, '0')}`),
+            prompt,
+            criterionType: 'rubric',
+            criterion
+        };
+    });
+}
+function seededShuffle(items, seedText) {
+    let state = parseInt(hashJson(seedText).slice(0, 8), 16) >>> 0;
+    const random = () => {
+        state = (state * 1664525 + 1013904223) >>> 0;
+        return state / 0x100000000;
+    };
+    const copy = [...items];
+    for (let index = copy.length - 1; index > 0; index -= 1) {
+        const swapIndex = Math.floor(random() * (index + 1));
+        [copy[index], copy[swapIndex]] = [copy[swapIndex], copy[index]];
+    }
+    return copy;
+}
+export async function generateTasks(input, config, client, cache) {
+    const generatedCount = input.count * 2;
+    let lastError;
+    for (let attempt = 1; attempt <= 3; attempt += 1) {
+        const response = await cache.getOrSet('generator', { model: config.generatorModel, input, generatedCount, promptVersion: 7, attempt }, () => client.complete({
+            model: config.generatorModel,
+            temperature: 0.4,
+            maxTokens: 8000,
+            responseFormat: 'json_object',
+            chatTemplateKwargs: { thinking: false },
+            messages: [
+                {
+                    role: 'system',
+                    content: 'Generate independent evaluation tasks. You only know the declared domain, never skill instructions. Return only valid compact JSON.'
+                },
+                {
+                    role: 'user',
+                    content: `Declared domain:\n${input.domain}\n\nGenerate ${generatedCount} concise tasks. Return exactly {"tasks":[{"id":"t1","prompt":"one concrete task under 80 words","criterion":"one pass/fail rubric under 60 words"}]}. Keep every criterion a single string, not an array. Do not include markdown or commentary.`
+                }
+            ]
+        }));
+        try {
+            const generated = validateTasks(extractJsonPayload(response.content), generatedCount);
+            return seededShuffle(generated, `${input.domain}:${config.generatorModel}`).slice(0, input.count).map((task, index) => ({
+                ...task,
+                id: `t${String(index + 1).padStart(3, '0')}`
+            }));
+        }
+        catch (error) {
+            lastError = error;
+            console.error(`[eval] generator returned invalid JSON on attempt ${attempt}/3`);
+        }
+    }
+    throw lastError instanceof Error ? lastError : new Error('Generator did not return valid tasks after retries');
+}
+//# sourceMappingURL=generate.js.map

package/dist/src/generate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generate.js","sourceRoot":"","sources":["../../packages/cli/src/generate.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAQrC,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACvD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,WAAW,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,IAAI,WAAW,GAAG,UAAU,CAAC,CAAC;IACxF,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;IACnD,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC7C,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,IAAI,GAAG,IAAI,KAAK,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,kCAAkC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,aAAa,CAAC,KAAc,EAAE,KAAa;IAClD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAChC,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAE,KAA6B,CAAC,KAAK,CAAC;YAClG,CAAC,CAAE,KAA8B,CAAC,KAAK;YACvC,CAAC,CAAC,SAAS,CAAC;IAChB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IAED,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC/C,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,mBAAmB,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,IAAI,GAAG,IAA+B,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACrE,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,iCAAiC,CAAC,CAAC;QAC5E,CAAC;QACD,OAAO;YACL,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM;YACN,aAAa,EAAE,QAAQ;YACvB,SAAS;SACV,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,aAAa,CAAI,KAAU,EAAE,QAAgB;IACpD,IAAI,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,KAAK,GAAG,CAAC,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,KAAK,GAAG,WAAW,CAAC;IAC7B,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACxB,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAE,EAAE,IAAI,CAAC,KAAK,CAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAA0B,EAC1B,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,cAAc,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;IACvC,IAAI,SAAkB,CAAC;IACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CACnC,WAAW,EACX,EAAE,KAAK,EAAE,MAAM,CAAC,cAAc,EAAE,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,EAAE,OAAO,EAAE,EAClF,GAAG,EAAE,CACH,MAAM,CAAC,QAAQ,CAAC;YACd,KAAK,EAAE,MAAM,CAAC,cAAc;YAC5B,WAAW,EAAE,GAAG;YAChB,SAAS,EAAE,IAAI;YACf,cAAc,EAAE,aAAa;YAC7B,kBAAkB,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;YACvC,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,qIAAqI;iBAC/I;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,qBAAqB,KAAK,CAAC,MAAM,gBAAgB,cAAc,kPAAkP;iBAC3T;aACF;SACF,CAAC,CACL,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,aAAa,CAAC,kBAAkB,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC,CAAC;YACtF,OAAO,aAAa,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;gBACtH,GAAG,IAAI;gBACP,EAAE,EAAE,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;aAC7C,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,CAAC;YAClB,OAAO,CAAC,KAAK,CAAC,qDAAqD,OAAO,IAAI,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IAED,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;AACjH,CAAC"}

package/dist/src/grade.d.ts

@@ -0,0 +1,5 @@
+import type { NvidiaNimClient } from './adapters/nvidia-nim.js';
+import type { JsonCache } from './cache.js';
+import type { NvidiaConfig } from './env.js';
+import type { GeneratedTask, GradedOutput, TrialOutput } from './types.js';
+export declare function gradeOutputs(tasks: GeneratedTask[], outputs: TrialOutput[], config: NvidiaConfig, client: NvidiaNimClient, cache: JsonCache): Promise<GradedOutput[]>;

package/dist/src/grade.js

@@ -0,0 +1,112 @@
+import { gradeDeterministically } from './deterministic.js';
+import { hashJson } from './hash.js';
+function parseGrade(text) {
+    const trimmed = text.trim();
+    if (trimmed.startsWith('{')) {
+        const end = trimmed.lastIndexOf('}');
+        if (end === -1) {
+            throw new Error('Grader JSON object was not closed');
+        }
+        const parsed = JSON.parse(trimmed.slice(0, end + 1));
+        const score = Number(parsed.score);
+        if (!Number.isFinite(score)) {
+            throw new Error('Grader JSON missing numeric score');
+        }
+        return {
+            score: Math.max(0, Math.min(1, score)),
+            reason: String(parsed.reason ?? '')
+        };
+    }
+    const lower = trimmed.toLowerCase();
+    const score = /\b(score|grade)\s*[:=]\s*1\b/.test(lower) || /\b(pass|passes|meets)\b/.test(lower) ? 1 : 0;
+    return {
+        score: Math.max(0, Math.min(1, score)),
+        reason: `non-json grader response: ${trimmed.slice(0, 160)}`
+    };
+}
+function seededShuffle(items, seedText) {
+    let state = parseInt(hashJson(seedText).slice(0, 8), 16) >>> 0;
+    const random = () => {
+        state = (state * 1664525 + 1013904223) >>> 0;
+        return state / 0x100000000;
+    };
+    const copy = [...items];
+    for (let index = copy.length - 1; index > 0; index -= 1) {
+        const swapIndex = Math.floor(random() * (index + 1));
+        [copy[index], copy[swapIndex]] = [copy[swapIndex], copy[index]];
+    }
+    return copy;
+}
+export async function gradeOutputs(tasks, outputs, config, client, cache) {
+    const taskById = new Map(tasks.map((task) => [task.id, task]));
+    const shuffled = seededShuffle(outputs, hashJson(outputs.map((output) => output.transcriptHash)));
+    const graded = new Map();
+    for (const output of shuffled) {
+        const task = taskById.get(output.taskId);
+        if (!task) {
+            throw new Error(`Missing task for output ${output.taskId}`);
+        }
+        if (task.criterionType === 'deterministic') {
+            const grade = gradeDeterministically(task, output.output);
+            graded.set(output.transcriptHash, {
+                ...output,
+                ...grade
+            });
+            continue;
+        }
+        console.error(`[eval] grade ${output.taskId} trial ${output.trial}`);
+        let grade;
+        let lastError;
+        for (let attempt = 1; attempt <= 3; attempt += 1) {
+            const response = await cache.getOrSet('grader', {
+                model: config.graderModel,
+                criterion: task.criterion,
+                output: output.output,
+                promptVersion: 6,
+                responseFormat: 'json_object',
+                attempt
+            }, () => client.complete({
+                model: config.graderModel,
+                temperature: 0,
+                maxTokens: 240,
+                responseFormat: 'json_object',
+                chatTemplateKwargs: { thinking: false },
+                messages: [
+                    {
+                        role: 'system',
+                        content: 'You are a blind evaluator. Grade only the provided output against the success criterion. You do not know which experimental arm produced it. Do not explain your reasoning. Return only valid JSON.'
+                    },
+                    {
+                        role: 'user',
+                        content: `Success criterion:\n${task.criterion}\n\nOutput to grade:\n${output.output}\n\nReturn exactly one JSON object with this shape: {"score":0,"reason":"brief reason"}. Use score 1 only if the output satisfies the criterion; otherwise use 0. Do not include markdown or commentary.`
+                    }
+                ]
+            }));
+            try {
+                grade = parseGrade(response.content);
+                break;
+            }
+            catch (error) {
+                lastError = error;
+                console.error(`[eval] grader returned invalid JSON on attempt ${attempt}/3`);
+            }
+        }
+        if (!grade) {
+            throw lastError instanceof Error ? lastError : new Error('Grader did not return a valid grade after retries');
+        }
+        graded.set(output.transcriptHash, {
+            ...output,
+            score: grade.score,
+            reason: grade.reason,
+            pass: grade.score >= 0.5
+        });
+    }
+    return outputs.map((output) => {
+        const item = graded.get(output.transcriptHash);
+        if (!item) {
+            throw new Error(`Missing grade for ${output.transcriptHash}`);
+        }
+        return item;
+    });
+}
+//# sourceMappingURL=grade.js.map

package/dist/src/grade.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grade.js","sourceRoot":"","sources":["../../packages/cli/src/grade.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAQrC,SAAS,UAAU,CAAC,IAAY;IAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAA4B,CAAC;QAChF,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;SACpC,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1G,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACtC,MAAM,EAAE,6BAA6B,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;KAC7D,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAI,KAAU,EAAE,QAAgB;IACpD,IAAI,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,KAAK,GAAG,CAAC,KAAK,GAAG,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC7C,OAAO,KAAK,GAAG,WAAW,CAAC;IAC7B,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACxB,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAE,EAAE,IAAI,CAAC,KAAK,CAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAsB,EACtB,OAAsB,EACtB,MAAoB,EACpB,MAAuB,EACvB,KAAgB;IAEhB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAClG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAE/C,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,2BAA2B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,KAAK,eAAe,EAAE,CAAC;YAC3C,MAAM,KAAK,GAAG,sBAAsB,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YAC1D,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE;gBAChC,GAAG,MAAM;gBACT,GAAG,KAAK;aACT,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QAED,OAAO,CAAC,KAAK,CAAC,gBAAgB,MAAM,CAAC,MAAM,UAAU,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QACrE,IAAI,KAA+B,CAAC;QACpC,IAAI,SAAkB,CAAC;QACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YACjD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,CACnC,QAAQ,EACR;gBACE,KAAK,EAAE,MAAM,CAAC,WAAW;gBACzB,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,aAAa,EAAE,CAAC;gBAChB,cAAc,EAAE,aAAa;gBAC7B,OAAO;aACR,EACD,GAAG,EAAE,CACH,MAAM,CAAC,QAAQ,CAAC;gBACd,KAAK,EAAE,MAAM,CAAC,WAAW;gBACzB,WAAW,EAAE,CAAC;gBACd,SAAS,EAAE,GAAG;gBACd,cAAc,EAAE,aAAa;gBAC7B,kBAAkB,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE;gBACvC,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,QAAQ;wBACd,OAAO,EACL,qMAAqM;qBACxM;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,uBAAuB,IAAI,CAAC,SAAS,yBAAyB,MAAM,CAAC,MAAM,0MAA0M;qBAC/R;iBACF;aACF,CAAC,CACL,CAAC;YAEF,IAAI,CAAC;gBACH,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBACrC,MAAM;YACR,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,CAAC;gBAClB,OAAO,CAAC,KAAK,CAAC,kDAAkD,OAAO,IAAI,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC;QACD,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;QAChH,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE;YAChC,GAAG,MAAM;YACT,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,IAAI,EAAE,KAAK,CAAC,KAAK,IAAI,GAAG;SACzB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,qBAAqB,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/src/hash.d.ts

@@ -0,0 +1,2 @@
+export declare function sha256(value: string): string;
+export declare function hashJson(value: unknown): string;

package/dist/src/hash.js

@@ -0,0 +1,8 @@
+import { createHash } from 'node:crypto';
+export function sha256(value) {
+    return createHash('sha256').update(value).digest('hex');
+}
+export function hashJson(value) {
+    return sha256(JSON.stringify(value));
+}
+//# sourceMappingURL=hash.js.map

package/dist/src/hash.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hash.js","sourceRoot":"","sources":["../../packages/cli/src/hash.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,UAAU,MAAM,CAAC,KAAa;IAClC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,KAAc;IACrC,OAAO,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;AACvC,CAAC"}

package/dist/src/m0/hardcoded.d.ts

@@ -0,0 +1,7 @@
+export interface M0Task {
+    id: string;
+    prompt: string;
+    expected: 'VALID' | 'INVALID';
+}
+export declare const M0_SKILL = "For SkillCheck Canary SKU classification tasks, apply this private rule exactly.\n\nA candidate is VALID only when all conditions hold:\n1. It has the exact form SC- followed by four digits, followed by a hyphen, followed by one lowercase checksum letter.\n2. Sum the four digits.\n3. The checksum letter must be q when the digit sum modulo 3 is 0, r when it is 1, and s when it is 2.\n\nFor every task, answer exactly VALID or INVALID. Do not explain.";
+export declare const M0_TASKS: M0Task[];

package/dist/src/m0/hardcoded.js

@@ -0,0 +1,51 @@
+export const M0_SKILL = `For SkillCheck Canary SKU classification tasks, apply this private rule exactly.
+
+A candidate is VALID only when all conditions hold:
+1. It has the exact form SC- followed by four digits, followed by a hyphen, followed by one lowercase checksum letter.
+2. Sum the four digits.
+3. The checksum letter must be q when the digit sum modulo 3 is 0, r when it is 1, and s when it is 2.
+
+For every task, answer exactly VALID or INVALID. Do not explain.`;
+export const M0_TASKS = [
+    {
+        id: 'm0-001',
+        prompt: 'Under the private SkillCheck Canary SKU rule, classify this candidate. Candidate: SC-1200-q. Answer exactly VALID or INVALID.',
+        expected: 'VALID'
+    },
+    {
+        id: 'm0-002',
+        prompt: 'Under the private SkillCheck Canary SKU rule, classify this candidate. Candidate: SC-1201-r. Answer exactly VALID or INVALID.',
+        expected: 'VALID'
+    },
+    {
+        id: 'm0-003',
+        prompt: 'Under the private SkillCheck Canary SKU rule, classify this candidate. Candidate: SC-1202-s. Answer exactly VALID or INVALID.',
+        expected: 'VALID'
+    },
+    {
+        id: 'm0-004',
+        prompt: 'Under the private SkillCheck Canary SKU rule, classify this candidate. Candidate: SC-9999-q. Answer exactly VALID or INVALID.',
+        expected: 'VALID'
+    },
+    {
+        id: 'm0-005',
+        prompt: 'Under the private SkillCheck Canary SKU rule, classify this candidate. Candidate: SC-1200-r. Answer exactly VALID or INVALID.',
+        expected: 'INVALID'
+    },
+    {
+        id: 'm0-006',
+        prompt: 'Under the private SkillCheck Canary SKU rule, classify this candidate. Candidate: SC-1202-q. Answer exactly VALID or INVALID.',
+        expected: 'INVALID'
+    },
+    {
+        id: 'm0-007',
+        prompt: 'Under the private SkillCheck Canary SKU rule, classify this candidate. Candidate: SC-12A2-s. Answer exactly VALID or INVALID.',
+        expected: 'INVALID'
+    },
+    {
+        id: 'm0-008',
+        prompt: 'Under the private SkillCheck Canary SKU rule, classify this candidate. Candidate: SC-5555-s. Answer exactly VALID or INVALID.',
+        expected: 'VALID'
+    }
+];
+//# sourceMappingURL=hardcoded.js.map

package/dist/src/m0/hardcoded.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hardcoded.js","sourceRoot":"","sources":["../../../packages/cli/src/m0/hardcoded.ts"],"names":[],"mappings":"AAMA,MAAM,CAAC,MAAM,QAAQ,GAAG;;;;;;;iEAOyC,CAAC;AAElE,MAAM,CAAC,MAAM,QAAQ,GAAa;IAChC;QACE,EAAE,EAAE,QAAQ;QACZ,MAAM,EAAE,+HAA+H;QACvI,QAAQ,EAAE,OAAO;KAClB;IACD;QACE,EAAE,EAAE,QAAQ;QACZ,MAAM,EAAE,+HAA+H;QACvI,QAAQ,EAAE,OAAO;KAClB;IACD;QACE,EAAE,EAAE,QAAQ;QACZ,MAAM,EAAE,+HAA+H;QACvI,QAAQ,EAAE,OAAO;KAClB;IACD;QACE,EAAE,EAAE,QAAQ;QACZ,MAAM,EAAE,+HAA+H;QACvI,QAAQ,EAAE,OAAO;KAClB;IACD;QACE,EAAE,EAAE,QAAQ;QACZ,MAAM,EAAE,+HAA+H;QACvI,QAAQ,EAAE,SAAS;KACpB;IACD;QACE,EAAE,EAAE,QAAQ;QACZ,MAAM,EAAE,+HAA+H;QACvI,QAAQ,EAAE,SAAS;KACpB;IACD;QACE,EAAE,EAAE,QAAQ;QACZ,MAAM,EAAE,+HAA+H;QACvI,QAAQ,EAAE,SAAS;KACpB;IACD;QACE,EAAE,EAAE,QAAQ;QACZ,MAAM,EAAE,+HAA+H;QACvI,QAAQ,EAAE,OAAO;KAClB;CACF,CAAC"}

package/dist/src/m0/run.d.ts

@@ -0,0 +1,38 @@
+import type { NvidiaNimClient } from '../adapters/nvidia-nim.js';
+import { loadNvidiaConfig } from '../env.js';
+import { type PairedObservation, type ScoreSummary } from '../score.js';
+import { type M0Task } from './hardcoded.js';
+interface ArmResult {
+    taskId: string;
+    trial: number;
+    arm: 'with_skill' | 'no_skill';
+    expected: M0Task['expected'];
+    output: string;
+    pass: boolean;
+}
+interface M0RunResult {
+    runId: string;
+    skillLabel: string;
+    score: ScoreSummary;
+    observations: PairedObservation[];
+    armResults: ArmResult[];
+}
+export interface M0GateReport {
+    config: {
+        runnerModel: string;
+        trials: number;
+        tasks: number;
+        temperature: number;
+    };
+    repeatability: {
+        passed: boolean;
+        runs: M0RunResult[];
+    };
+    emptyControl: {
+        passed: boolean;
+        run: M0RunResult;
+    };
+    passed: boolean;
+}
+export declare function runM0Gate(clientFactory: (config: ReturnType<typeof loadNvidiaConfig>) => NvidiaNimClient): Promise<M0GateReport>;
+export {};

package/dist/src/m0/run.js

@@ -0,0 +1,102 @@
+import { loadNvidiaConfig } from '../env.js';
+import { ciOverlapsZero, effectInsideCi, scorePairedObservations } from '../score.js';
+import { M0_SKILL, M0_TASKS } from './hardcoded.js';
+const K_TRIALS = 3;
+const TEMPERATURE = 0.7;
+const MAX_TOKENS = 16;
+function buildMessages(skill, prompt) {
+    if (!skill.trim()) {
+        return [{ role: 'user', content: prompt }];
+    }
+    return [
+        {
+            role: 'system',
+            content: `You are the runner in an A/B skill evaluation. Follow any skill instructions below when they apply.\n\n${skill}`
+        },
+        { role: 'user', content: prompt }
+    ];
+}
+function outputPasses(output, expected) {
+    const escaped = expected.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    return new RegExp(`^\\s*${escaped}\\s*\\.?\\s*$`, 'i').test(output);
+}
+async function runArm(client, model, skill, task, trial, arm) {
+    const response = await client.complete({
+        model,
+        messages: buildMessages(skill, task.prompt),
+        temperature: TEMPERATURE,
+        maxTokens: MAX_TOKENS
+    });
+    return {
+        taskId: task.id,
+        trial,
+        arm,
+        expected: task.expected,
+        output: response.content.trim(),
+        pass: outputPasses(response.content, task.expected)
+    };
+}
+async function runSkillOnce(client, model, skill, skillLabel, runId) {
+    const armResults = [];
+    const observations = [];
+    for (const task of M0_TASKS) {
+        for (let trial = 1; trial <= K_TRIALS; trial += 1) {
+            console.error(`[m0] ${runId} ${task.id} trial ${trial}/${K_TRIALS} with_skill`);
+            const withSkill = await runArm(client, model, skill, task, trial, 'with_skill');
+            console.error(`[m0] ${runId} ${task.id} trial ${trial}/${K_TRIALS} no_skill`);
+            const noSkill = await runArm(client, model, '', task, trial, 'no_skill');
+            armResults.push(withSkill, noSkill);
+            observations.push({
+                withSkillPass: withSkill.pass,
+                noSkillPass: noSkill.pass
+            });
+        }
+    }
+    return {
+        runId,
+        skillLabel,
+        score: scorePairedObservations(observations, 1000, hashSeed(runId)),
+        observations,
+        armResults
+    };
+}
+function hashSeed(value) {
+    let hash = 2166136261;
+    for (const char of value) {
+        hash ^= char.charCodeAt(0);
+        hash = Math.imul(hash, 16777619);
+    }
+    return hash >>> 0;
+}
+export async function runM0Gate(clientFactory) {
+    const config = loadNvidiaConfig();
+    const client = clientFactory(config);
+    const startedAt = new Date().toISOString();
+    const repeatRuns = [];
+    for (let index = 1; index <= 3; index += 1) {
+        console.error(`[m0] starting repeatability run ${index}/3`);
+        repeatRuns.push(await runSkillOnce(client, config.runnerModel, M0_SKILL, 'canary-sku-skill', `${startedAt}-repeat-${index}`));
+    }
+    console.error('[m0] starting empty-control run');
+    const emptyRun = await runSkillOnce(client, config.runnerModel, '', 'empty-skill', `${startedAt}-empty-control`);
+    const repeatabilityPassed = repeatRuns.every((run) => effectInsideCi(run.score));
+    const emptyControlPassed = ciOverlapsZero(emptyRun.score);
+    return {
+        config: {
+            runnerModel: config.runnerModel,
+            trials: K_TRIALS,
+            tasks: M0_TASKS.length,
+            temperature: TEMPERATURE
+        },
+        repeatability: {
+            passed: repeatabilityPassed,
+            runs: repeatRuns
+        },
+        emptyControl: {
+            passed: emptyControlPassed,
+            run: emptyRun
+        },
+        passed: repeatabilityPassed && emptyControlPassed
+    };
+}
+//# sourceMappingURL=run.js.map

package/dist/src/m0/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../../../packages/cli/src/m0/run.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EACL,cAAc,EACd,cAAc,EACd,uBAAuB,EAGxB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAe,MAAM,gBAAgB,CAAC;AAEjE,MAAM,QAAQ,GAAG,CAAC,CAAC;AACnB,MAAM,WAAW,GAAG,GAAG,CAAC;AACxB,MAAM,UAAU,GAAG,EAAE,CAAC;AAqCtB,SAAS,aAAa,CAAC,KAAa,EAAE,MAAc;IAClD,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAClB,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,OAAO;QACL;YACE,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,0GAA0G,KAAK,EAAE;SAC3H;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE;KAClC,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,MAAc,EAAE,QAA4B;IAChE,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;IAChE,OAAO,IAAI,MAAM,CAAC,QAAQ,OAAO,eAAe,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACtE,CAAC;AAED,KAAK,UAAU,MAAM,CACnB,MAAuB,EACvB,KAAa,EACb,KAAa,EACb,IAAY,EACZ,KAAa,EACb,GAAqB;IAErB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC;QACrC,KAAK;QACL,QAAQ,EAAE,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC;QAC3C,WAAW,EAAE,WAAW;QACxB,SAAS,EAAE,UAAU;KACtB,CAAC,CAAC;IAEH,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,KAAK;QACL,GAAG;QACH,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE;QAC/B,IAAI,EAAE,YAAY,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC;KACpD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,MAAuB,EACvB,KAAa,EACb,KAAa,EACb,UAAkB,EAClB,KAAa;IAEb,MAAM,UAAU,GAAgB,EAAE,CAAC;IACnC,MAAM,YAAY,GAAwB,EAAE,CAAC;IAE7C,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,QAAQ,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAClD,OAAO,CAAC,KAAK,CAAC,QAAQ,KAAK,IAAI,IAAI,CAAC,EAAE,UAAU,KAAK,IAAI,QAAQ,aAAa,CAAC,CAAC;YAChF,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;YAChF,OAAO,CAAC,KAAK,CAAC,QAAQ,KAAK,IAAI,IAAI,CAAC,EAAE,UAAU,KAAK,IAAI,QAAQ,WAAW,CAAC,CAAC;YAC9E,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;YAEzE,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACpC,YAAY,CAAC,IAAI,CAAC;gBAChB,aAAa,EAAE,SAAS,CAAC,IAAI;gBAC7B,WAAW,EAAE,OAAO,CAAC,IAAI;aAC1B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK;QACL,UAAU;QACV,KAAK,EAAE,uBAAuB,CAAC,YAAY,EAAE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;QACnE,YAAY;QACZ,UAAU;KACX,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,IAAI,IAAI,GAAG,UAAU,CAAC;IACtB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,IAAI,KAAK,CAAC,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,aAA+E;IAC7G,MAAM,MAAM,GAAG,gBAAgB,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,UAAU,GAAkB,EAAE,CAAC;IAErC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC3C,OAAO,CAAC,KAAK,CAAC,mCAAmC,KAAK,IAAI,CAAC,CAAC;QAC5D,UAAU,CAAC,IAAI,CACb,MAAM,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE,QAAQ,EAAE,kBAAkB,EAAE,GAAG,SAAS,WAAW,KAAK,EAAE,CAAC,CAC7G,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE,EAAE,EAAE,aAAa,EAAE,GAAG,SAAS,gBAAgB,CAAC,CAAC;IACjH,MAAM,mBAAmB,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IACjF,MAAM,kBAAkB,GAAG,cAAc,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAE1D,OAAO;QACL,MAAM,EAAE;YACN,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,MAAM,EAAE,QAAQ;YAChB,KAAK,EAAE,QAAQ,CAAC,MAAM;YACtB,WAAW,EAAE,WAAW;SACzB;QACD,aAAa,EAAE;YACb,MAAM,EAAE,mBAAmB;YAC3B,IAAI,EAAE,UAAU;SACjB;QACD,YAAY,EAAE;YACZ,MAAM,EAAE,kBAAkB;YAC1B,GAAG,EAAE,QAAQ;SACd;QACD,MAAM,EAAE,mBAAmB,IAAI,kBAAkB;KAClD,CAAC;AACJ,CAAC"}

package/dist/src/normalize.d.ts

@@ -0,0 +1,2 @@
+import type { NormalizedSkill } from './types.js';
+export declare function normalizeSkill(inputPath: string): Promise<NormalizedSkill>;