@skill-harness/cli 0.32.4 → 0.33.0

package/assets/report.template.html

@@ -177,6 +177,7 @@ function render() {
       const errorNote = g.errors > 0 ? ` — ${g.errors} infrastructure error${g.errors === 1 ? "" : "s"}` : "";
       gradeHtml = `${g.letter} (${g.pct}%) · ${g.passed}/${g.total}${suspectNote}${errorNote} ${badge}`;
     }
+    const summaryHtml = col.summaryLines ? `<details><summary>Evidence, profiles and usage</summary><div class="meta">${col.summaryLines.map(line => escapeHtml(line)).join("<br>")}</div></details>` : "";
     const m = col.metrics;
     const metricHtml = m && m.total_reps > 0
       ? `<div class='grade' title='subject metrics reported by ${m.subject_metrics_reps}/${m.total_reps} reps'>` +
@@ -200,7 +201,7 @@ function render() {
           : `lift 0 · no effect`;
       const inc = ls.inconclusive > 0 ? ` · ${ls.inconclusive} inconclusive` : "";
       const part = col.lift.partial ? " · partial" : "";
-      html += `<th><div>${escapeHtml(col.label)}</div><div class='grade'>${gradeHtml}</div>${metricHtml}` +
+      html += `<th><div>${escapeHtml(col.label)}</div><div class='grade'>${gradeHtml}</div>${metricHtml}${summaryHtml}` +
         `<div class='lift ${cls}' title='vs red baseline ${escapeHtml(col.lift.redTimestamp)} — ${ls.kept} passed without the skill too'>${body}${inc}${part}</div></th>`;
       return;
     }
@@ -210,7 +211,7 @@ function render() {
     if (none) {
       liftHtml = `<div class='lift none' title='${escapeHtml(none.title)}'>${escapeHtml(none.text)}</div>`;
     }
-    html += `<th><div>${escapeHtml(col.label)}</div><div class='grade'>${gradeHtml}</div>${metricHtml}${liftHtml}</th>`;
+    html += `<th><div>${escapeHtml(col.label)}</div><div class='grade'>${gradeHtml}</div>${metricHtml}${summaryHtml}${liftHtml}</th>`;
   });
   html += "</tr></thead><tbody>";
   for (const scn of DATA.scenarios) {

package/dist/cli.js

@@ -1,8 +1,9 @@
 #!/usr/bin/env node
+import { withThinking, modelProfileLabel } from "@skill-harness/core";
 import { readFileSync, existsSync, mkdirSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
 import { basename, dirname, join, resolve } from "node:path";
 import { tmpdir } from "node:os";
-import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectReport, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, computeCoverage, formatCoverage, HARNESS_VERSION, defaultJudge, recordedJudgeOrDefault, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, isScoredMode, resolveArm, isFreeOfflineCommand, diagnoseRun, loadCalibration, runCalibration, planCalibration, forkSavedRun, withProcessScope, } from "@skill-harness/core";
+import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectReport, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, computeCoverage, formatCoverage, HARNESS_VERSION, defaultJudge, recordedJudgeOrDefault, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, isScoredMode, resolveArm, isFreeOfflineCommand, diagnoseRun, requestedSubject, loadCalibration, runCalibration, planCalibration, forkSavedRun, withProcessScope, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 import { serveReview } from "./serve.js";
 import { cmdDecision } from "./decision.js";
@@ -173,7 +174,7 @@ export async function cmdRun(args, adapterOverride) {
     // any subject tokens are spent: a refusal that arrives after the model has been
     // paid for is a worse version of the problem it exists to prevent.
     const judgeFlagRun = flagStr(args, "judge");
-    const judge = parseModelRef(judgeFlagRun ?? defaultJudge());
+    const judge = withThinking(parseModelRef(judgeFlagRun ?? defaultJudge()), flagStr(args, "judge-thinking"));
     assertJudgeAllowed(judge, {
         source: judgeFlagRun ? "--judge" : "the default judge (SKILL_HARNESS_JUDGE or the baked value)",
         allowMetered: flagBool(args, "allow-metered-judge"),
@@ -185,6 +186,7 @@ export async function cmdRun(args, adapterOverride) {
     // that was never the problem. Free, offline validation first — it is also why
     // CI, which has no `pi`, could not test this refusal at all.
     const only = selectedIds(args);
+    const models = resolveModels(args).map(token => ({ token, model: withThinking(parseModelRef(token), flagStr(args, "thinking")) }));
     const harnessName = flagStr(args, "harness", "pi");
     const adapter = adapterOverride ?? getAdapter(harnessName);
     if (!(await adapter.available()))
@@ -195,7 +197,6 @@ export async function cmdRun(args, adapterOverride) {
     const label = flagStr(args, "label") || null;
     const parallel = Math.max(1, Number(flagStr(args, "parallel", "1")) || 1);
     const { reps, passThreshold } = parseRunTuning(args);
-    const modelTokens = resolveModels(args);
     // Resolved ONCE, before the skill loop: a typo'd arm name fails before any
     // token is spent on any skill.
     const armName = flagStr(args, "arm") || null;
@@ -213,12 +214,11 @@ export async function cmdRun(args, adapterOverride) {
         // that look comparable and are not. Checked per skill, before its first token.
         assertNotDowngraded(dirname(skill.testsDir), "run");
         const spec = loadSpec(skill.specPath);
-        for (const token of modelTokens) {
-            const model = parseModelRef(token);
+        for (const { token, model } of models) {
             // The version is on the banner because a stale global install is otherwise
             // invisible: a 0.1.0 binary grades a 0.3.x corpus, produces plausible
             // numbers, and nothing on screen says which tool made them.
-            console.log(`\n▶ ${spec.skill} · ${harnessName}:${token} · mode=${mode} · judge=${judge.provider}:${judge.model} · skill-harness ${HARNESS_VERSION}`);
+            console.log(`\n▶ ${spec.skill} · ${harnessName}:${token} · mode=${mode} · subject=${modelProfileLabel(model)} · judge=${modelProfileLabel(judge)} · skill-harness ${HARNESS_VERSION}`);
             const summary = await runSkillModel({
                 spec,
                 skillDir: skill.dir,
@@ -280,10 +280,10 @@ export async function cmdGrade(args, adapterOverride) {
     // the current Pi default.
     const judgeFlag = flagStr(args, "judge");
     const onlyUnparsed = flagBool(args, "unparsed-only");
-    if (onlyUnparsed && judgeFlag !== undefined)
+    if (onlyUnparsed && (judgeFlag !== undefined || flagStr(args, "judge-thinking") !== undefined))
         throw new Error("--unparsed-only uses the recorded judge; --judge cannot be combined with it");
     const resolvedRecordedJudge = judgeFlag ? undefined : recordedJudgeOrDefault(prev?.judge);
-    const judge = judgeFlag ? parseModelRef(judgeFlag) : resolvedRecordedJudge.judge;
+    const judge = withThinking(judgeFlag ? parseModelRef(judgeFlag) : resolvedRecordedJudge.judge, flagStr(args, "judge-thinking"));
     if (onlyUnparsed && resolvedRecordedJudge?.migratedFrom)
         throw new Error("--unparsed-only cannot use this retired recorded judge; use grade to migrate it explicitly");
     if (resolvedRecordedJudge?.migratedFrom) {
@@ -369,7 +369,7 @@ export async function cmdRegate(args, adapterOverride) {
         const spec = loadSpec(specPath);
         const prev = readResults(runDir);
         const resolvedRecordedJudge = judgeFlag ? undefined : recordedJudgeOrDefault(prev.judge);
-        const judge = judgeFlag ? parseModelRef(judgeFlag) : resolvedRecordedJudge.judge;
+        const judge = withThinking(judgeFlag ? parseModelRef(judgeFlag) : resolvedRecordedJudge.judge, flagStr(args, "judge-thinking"));
         if (resolvedRecordedJudge?.migratedFrom) {
             console.error(`skill-harness: recorded judge ${resolvedRecordedJudge.migratedFrom.provider}:${resolvedRecordedJudge.migratedFrom.model} was removed; re-judging through Pi with ${judge.provider}:${judge.model}`);
         }
@@ -751,6 +751,7 @@ export function help() {
     return `skill-harness ${HARNESS_VERSION} — test/optimize loop for agent skills (pi harness)
 
   run    <skill|all> --skills <root> [--model prov:model ...] [--models file] [--only A1,D2]
+                     [--thinking LEVEL] [--judge-thinking LEVEL]
                      [--mode red|green|force] [--judge prov:model] [--harness pi] [--label name] [--parallel N] [--reps N] [--pass-threshold T]
                      [--canary]  green only: one instructed-loading probe; autonomous selection stays untested
                      [--structured]  request structured capture (Pi uses it by default for subject usage)
@@ -816,7 +817,16 @@ export async function cmdDiagnose(args) {
     const recorded = readResults(runDir);
     const { skills, specs } = roots(args);
     const selected = resolveSkill(skills, recorded.skill, specs);
-    const report = diagnoseRun(runDir, selected.dir, selected.specPath);
+    if ((args.multi.model?.length ?? 0) > 1)
+        throw new Error("diagnose previews one subject profile at a time");
+    const modelToken = args.multi.model?.[0] ?? flagStr(args, "model"), judgeToken = flagStr(args, "judge");
+    if (modelToken?.includes(","))
+        throw new Error("diagnose previews one subject profile at a time");
+    const profiles = {
+        subject: modelToken !== undefined || Object.hasOwn(args.flags, "thinking") ? withThinking(modelToken !== undefined ? parseModelRef(modelToken) : requestedSubject(recorded), flagStr(args, "thinking")) : undefined,
+        judge: judgeToken !== undefined || Object.hasOwn(args.flags, "judge-thinking") ? withThinking(judgeToken !== undefined ? parseModelRef(judgeToken) : recorded.judge, flagStr(args, "judge-thinking")) : undefined,
+    };
+    const report = diagnoseRun(runDir, selected.dir, selected.specPath, profiles);
     console.log(format === "text" ? formatDiagnosis(report) : JSON.stringify(report, null, 2));
 }
 export async function cmdCalibrate(args, adapterOverride) {
@@ -830,7 +840,7 @@ export async function cmdCalibrate(args, adapterOverride) {
     if (flagBool(args, "plan")) {
         if (!judgeToken)
             throw new Error("Calibration --plan needs an explicit --judge; it makes zero provider calls.");
-        const plan = planCalibration(calibration, { judge: parseModelRef(judgeToken), only, reuse });
+        const plan = planCalibration(calibration, { judge: withThinking(parseModelRef(judgeToken), flagStr(args, "judge-thinking")), only, reuse });
         console.log(JSON.stringify({ ...plan, providerCalls: 0 }, null, 2));
         if (!plan.ready)
             process.exitCode = 1;
@@ -848,7 +858,7 @@ export async function cmdCalibrate(args, adapterOverride) {
     const adapter = adapterOverride ?? getAdapter("pi");
     if (!await adapter.available())
         throw new Error("Pi is required for judge calibration. Configure the selected model in Pi first.");
-    const result = await runCalibration(calibration, { adapter, judge: parseModelRef(judgeToken), only, reuse, output: resolve(output), allowMeteredJudge: flagBool(args, "allow-metered-judge"), onProgress: p => console.log(`Judging control ${p.index + 1}/${p.total}: ${p.id}`) });
+    const result = await runCalibration(calibration, { adapter, judge: withThinking(parseModelRef(judgeToken), flagStr(args, "judge-thinking")), only, reuse, output: resolve(output), allowMeteredJudge: flagBool(args, "allow-metered-judge"), onProgress: p => console.log(`Judging control ${p.index + 1}/${p.total}: ${p.id}`) });
     console.log(JSON.stringify(result, null, 2));
     if (result.status !== "matched")
         process.exitCode = 1;

package/dist/diagnose-format.js

@@ -1,3 +1,4 @@
+import { modelProfileLabel } from "@skill-harness/core";
 /** Human view of the same offline report; never a second scoring policy. */
 export function formatDiagnosis(report) {
     const grouped = report.criterionDimensions;
@@ -9,6 +10,10 @@ export function formatDiagnosis(report) {
         `Coverage: ${grouped.coverage.availableVotes}/${grouped.coverage.expectedVotes} known-size votes available; ${grouped.coverage.unknownScenarioSizes} scenario sizes unknown.`,
         `Mapping: ${grouped.metadataSha256}`,
     ];
+    if (report.summary) {
+        const summary = report.summary;
+        lines.push(`Subject requested: ${modelProfileLabel(summary.subject.requested)}; effective thinking: unknown.`, `Judge requested: ${modelProfileLabel(summary.judge.requested)}; effective thinking: unknown.`, `Retained token observations: ${summary.usage.subject.recordedInvocations} subject invocations (total unknown); ${summary.usage.judge.recordedInvocations}/${summary.usage.judge.expectedInvocations ?? "unknown"} judge attempts. Full per-counter coverage: --format json. Costs are separate.`, summary.limits);
+    }
     for (const scenario of grouped.scenarios) {
         lines.push("", `${scenario.id}: recorded ${scenario.original.judge_verdict}; override ${scenario.original.override ?? "none"}; objective ${scenario.original.objective?.status ?? "not recorded"}`);
         for (const [dimension, counts] of Object.entries(scenario.counts)) {

package/package.json

@@ -1,50 +1,50 @@
 {
-    "name": "@skill-harness/cli",
-    "version": "0.32.4",
-    "description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
-    "type": "module",
-    "license": "MIT",
-    "main": "./dist/cli.js",
-    "bin": {
-        "skill-harness": "dist/cli.js"
-    },
-    "exports": {
-        ".": "./dist/cli.js",
-        "./serve": "./dist/serve.js"
-    },
-    "keywords": [
-        "agent-skills",
-        "skill",
-        "llm",
-        "testing",
-        "eval",
-        "grading",
-        "pi",
-        "ci",
-        "harness"
-    ],
-    "files": [
-        "dist/**/*.js",
-        "dist/**/*.d.ts",
-        "assets",
-        "LICENSE",
-        "README.md"
-    ],
-    "repository": {
-        "type": "git",
-        "url": "git+https://github.com/mojomanyana/skill-harness.git"
-    },
-    "publishConfig": {
-        "access": "public"
-    },
-    "engines": {
-        "node": ">=20"
-    },
-    "scripts": {
-        "prepack": "node ../../scripts/release-pack.mjs --guard"
-    },
-    "dependencies": {
-        "@skill-harness/core": "0.32.4",
-        "@skill-harness/adapters": "0.32.4"
-    }
+  "name": "@skill-harness/cli",
+  "version": "0.33.0",
+  "description": "skill-harness CLI \u2014 run, grade, review, and lint agent-skill scenarios on the pi harness",
+  "type": "module",
+  "license": "MIT",
+  "main": "./dist/cli.js",
+  "bin": {
+    "skill-harness": "dist/cli.js"
+  },
+  "exports": {
+    ".": "./dist/cli.js",
+    "./serve": "./dist/serve.js"
+  },
+  "keywords": [
+    "agent-skills",
+    "skill",
+    "llm",
+    "testing",
+    "eval",
+    "grading",
+    "pi",
+    "ci",
+    "harness"
+  ],
+  "files": [
+    "dist/**/*.js",
+    "dist/**/*.d.ts",
+    "assets",
+    "LICENSE",
+    "README.md"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/mojomanyana/skill-harness.git"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "prepack": "node ../../scripts/release-pack.mjs --guard"
+  },
+  "dependencies": {
+    "@skill-harness/core": "0.33.0",
+    "@skill-harness/adapters": "0.33.0"
+  }
 }