@skill-harness/cli 0.1.2 → 0.3.0

package/assets/report.grade.js

@@ -75,3 +75,55 @@ export function gradeColumn(col, shipBar, critical) {
 
   return { passed, total, pct, letter, ship, criticalFails, bFails, suspect };
 }
+
+/**
+ * Red-vs-green class for ONE scenario — mirrors `classify` in
+ * packages/core/src/lift.ts exactly. Kept in the client because the author
+ * flips green verdicts live in this UI: a server-computed class would freeze at
+ * page load and could show "gained" beside a cell the author just marked FAIL.
+ *
+ * `liftCell` supplies the red side (verdict + redSuspect, from the baseline run,
+ * which this view never edits); `greenCell` is the live report cell, so its
+ * override and suspect state are read through the same effective()/suspect rule
+ * the grader uses.
+ */
+export function liftClass(liftCell, greenCell) {
+  const conclusive = (verdict, suspect) =>
+    !suspect && verdict !== "ERROR" && verdict !== "JUDGE-AMBIGUOUS";
+
+  const redOk = conclusive(liftCell.red, liftCell.redSuspect);
+  const greenOk = conclusive(effective(greenCell), !!greenCell.suspect && !greenCell.override);
+  if (!redOk || !greenOk) return "inconclusive";
+
+  const redPass = liftCell.red === "PASS";
+  const greenPass = effective(greenCell) === "PASS";
+  if (redPass && greenPass) return "kept";
+  if (!redPass && greenPass) return "gained";
+  if (redPass && !greenPass) return "regressed";
+  return "both-fail";
+}
+
+/**
+ * Aggregate a column's lift over the live cells — mirrors computeLift's counters
+ * in packages/core/src/lift.ts. Only scenarios the baseline also covered are
+ * counted, matching the server's intersection rule.
+ */
+export function liftSummary(col) {
+  const out = { gained: 0, regressed: 0, kept: 0, bothFail: 0, inconclusive: 0, compared: 0, redPassed: 0, greenPassed: 0, delta: 0 };
+  if (!col.lift) return null;
+  for (const id of Object.keys(col.lift.cells)) {
+    const greenCell = col.cells[id];
+    if (!greenCell) continue;
+    const liftCell = col.lift.cells[id];
+    const cls = liftClass(liftCell, greenCell);
+    out.compared++;
+    if (cls === "both-fail") out.bothFail++;
+    else out[cls]++;
+    if (cls !== "inconclusive") {
+      if (liftCell.red === "PASS") out.redPassed++;
+      if (effective(greenCell) === "PASS") out.greenPassed++;
+    }
+  }
+  out.delta = out.greenPassed - out.redPassed;
+  return out;
+}

package/assets/report.template.html

@@ -35,6 +35,14 @@
   .badge { display: inline-block; padding: 1px 7px; border-radius: 999px; font-size: 11px; font-weight: 700; }
   .badge.ship { background: var(--pass); color: #fff; }
   .badge.no { background: var(--fail); color: #fff; }
+  /* red-vs-green lift: what the skill itself changed */
+  .lift { font-size: 11px; margin-top: 3px; font-weight: 600; }
+  .lift.up { color: var(--pass); }
+  .lift.down { color: var(--fail); }
+  .lift.flat, .lift.none { color: var(--dim); font-weight: 500; }
+  .cell .lm { display: block; font-size: 10px; font-weight: 700; }
+  .cell .lm.up { color: var(--pass); }
+  .cell .lm.down { color: var(--fail); }
   aside { width: 0; transition: width .15s ease; overflow: hidden; border-left: 1px solid var(--line); background: var(--panel); }
   aside.open { width: 460px; }
   .panel { width: 460px; padding: 16px 18px; }
@@ -153,14 +161,43 @@ function render() {
       const suspectNote = g.suspect > 0 ? ` — ${g.suspect} suspect` : "";
       gradeHtml = `${g.letter} (${g.pct}%) · ${g.passed}/${g.total}${suspectNote} ${badge}`;
     }
-    html += `<th><div>${escapeHtml(col.label)}</div><div class='grade'>${gradeHtml}</div></th>`;
+    // Lift answers "does this skill do anything?", which the grade alone cannot:
+    // a column can score A because the model never needed the skill. Recomputed
+    // from the live cells (see liftSummary) so overrides move it immediately.
+    const ls = liftSummary(col);
+    let liftHtml = "";
+    if (ls && ls.compared > 0) {
+      const moved = ls.gained > 0 || ls.regressed > 0;
+      const cls = !moved ? "flat" : ls.delta > 0 ? "up" : ls.delta < 0 ? "down" : "flat";
+      // All-inconclusive is no measurement, not a zero effect — don't claim "no
+      // effect" when nothing was measurable (mirrors liftHeadline in lift.ts).
+      const body = ls.compared === ls.inconclusive
+        ? `lift — nothing conclusive`
+        : moved
+          ? `lift ${ls.delta > 0 ? "+" : ""}${ls.delta} · ${ls.gained}↑ ${ls.regressed}↓`
+          : `lift 0 · no effect`;
+      const inc = ls.inconclusive > 0 ? ` · ${ls.inconclusive} inconclusive` : "";
+      const part = col.lift.partial ? " · partial" : "";
+      html += `<th><div>${escapeHtml(col.label)}</div><div class='grade'>${gradeHtml}</div>` +
+        `<div class='lift ${cls}' title='vs red baseline ${escapeHtml(col.lift.redTimestamp)} — ${ls.kept} passed without the skill too'>${body}${inc}${part}</div></th>`;
+      return;
+    }
+    if (col.mode === "green") {
+      // "not measured" is a different claim from "measured no effect" — say which.
+      liftHtml = `<div class='lift none' title='run the same scenarios with --mode red to get a baseline'>no red baseline</div>`;
+    }
+    html += `<th><div>${escapeHtml(col.label)}</div><div class='grade'>${gradeHtml}</div>${liftHtml}</th>`;
   });
   html += "</tr></thead><tbody>";
   for (const scn of DATA.scenarios) {
     html += `<tr><td class='scn'>${scn.id}${scn.critical ? "<span class='crit' title='critical'>⚠</span>" : ""}<div class='grade'>${escapeHtml(scn.title)}</div></td>`;
     DATA.columns.forEach((col) => {
       const cell = col.cells[scn.id];
-      if (!cell) { html += `<td class='cell empty'>–</td>`; continue; }
+      // `return`, not `continue`: this is a forEach callback, so `continue` is a
+      // SyntaxError that takes the entire inline script — the whole review UI —
+      // down with it. Guarded by report.test.ts's "rendered report is valid
+      // JavaScript".
+      if (!cell) { html += `<td class='cell empty'>–</td>`; return; }
       const v = effective(cell);
       const sel = selected && selected.colIndex === col.index && selected.scenarioId === scn.id ? " sel" : "";
       // _unsaved wins regardless of override value, so a failed *clear*
@@ -171,7 +208,17 @@ function render() {
       const misfired = cell.clean != null && cell.clean < cell.reps ? ` · ${cell.reps - cell.clean} misfired` : "";
       const reps = cell.reps ? `<span class='reps'>${cell.passes}/${cell.clean}${misfired}${cell.flakiness ? ` · flaky ${cell.flakiness.toFixed(2)}` : ""}</span>` : "";
       const suspectBadge = cell.suspect && !cell.override ? `<span class='ov suspect'>suspect</span>` : "";
-      html += `<td class='cell ${v}${sel}' data-col='${col.index}' data-id='${scn.id}'>${v}${ov}${suspectBadge}${reps}</td>`;
+      // Per-cell lift marker: only the two classes that mean the skill changed
+      // the outcome. kept/both-fail/inconclusive are left unmarked — marking
+      // every cell would bury the signal this column exists to show.
+      const lc = col.lift && col.lift.cells[scn.id] ? liftClass(col.lift.cells[scn.id], cell) : null;
+      const liftMark =
+        lc === "gained"
+          ? `<span class='lm up' title='red baseline ${col.lift.cells[scn.id].red} → PASS with the skill'>↑ skill</span>`
+          : lc === "regressed"
+            ? `<span class='lm down' title='red baseline PASS → ${effective(cell)} with the skill'>↓ skill</span>`
+            : "";
+      html += `<td class='cell ${v}${sel}' data-col='${col.index}' data-id='${scn.id}'>${v}${ov}${suspectBadge}${liftMark}${reps}</td>`;
     });
     html += "</tr>";
   }

package/dist/cli.d.ts

@@ -12,6 +12,8 @@ export declare function parseRunTuning(args: Args): {
     passThreshold: number;
 };
 export declare function cmdGrade(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
+export declare function cmdInit(args: Args): Promise<void>;
+export declare function cmdSuggest(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
 /** Exit-code contract: 0 = clean (no findings), 1 = >=1 finding, or a resolution error (unknown skill/root, no skills with a spec). */
 export declare function cmdLint(args: Args): Promise<void>;
 export declare function main(argv: string[]): Promise<void>;

package/dist/cli.js

@@ -1,12 +1,14 @@
 #!/usr/bin/env node
-import { readFileSync, existsSync, appendFileSync } from "node:fs";
-import { dirname, join } from "node:path";
+import { readFileSync, existsSync, appendFileSync, mkdirSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
+import { basename, dirname, join, resolve } from "node:path";
+import { tmpdir } from "node:os";
 import yaml from "js-yaml";
-import { discover, resolveSkill, loadSpec, parseSpec, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, } from "@skill-harness/core";
+import { discover, resolveSkill, loadSpec, parseSpec, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, specPathForRunDir, collectLift, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 import { serveReview } from "./serve.js";
 const DEFAULT_MODEL = "fireworks:accounts/fireworks/models/deepseek-v4-pro";
 const DEFAULT_JUDGE = "anthropic:claude-opus-4-8";
+const DEFAULT_SUGGEST_MODEL = "claude-code:claude-opus-4-8";
 const REPEATABLE = new Set(["model", "turn", "check"]);
 function parseArgs(argv) {
     const _ = [];
@@ -120,6 +122,8 @@ async function cmdRun(args) {
     const label = flagStr(args, "label") || null;
     const parallel = Math.max(1, Number(flagStr(args, "parallel", "1")) || 1);
     const { reps, passThreshold } = parseRunTuning(args);
+    const onlyRaw = flagStr(args, "only");
+    const only = onlyRaw ? onlyRaw.split(",").map((x) => x.trim()).filter(Boolean) : undefined;
     const modelTokens = resolveModels(args);
     const skills = target === "all"
         ? discover(root).filter((s) => s.hasSpec)
@@ -148,18 +152,26 @@ async function cmdRun(args) {
                 concurrency: parallel,
                 reps,
                 passThreshold,
+                only,
                 onProgress: (m) => console.log(m),
             });
             summaries.push(summary);
-            console.log("\n" + formatScorecard(summary) + "\n");
+            // Lift is derived from what's on disk, so it picks up a red baseline from
+            // any earlier run — the tag dir (<harness>-<modelslug>) is the join key.
+            const tag = basename(dirname(summary.runDir));
+            const lift = collectLift(skill.dir).find((l) => l.tag === tag);
+            console.log("\n" + formatScorecard(summary, lift) + "\n");
         }
     }
     console.log(`\nReview interactively:  skill-harness review ${skills[0]?.name ?? "<skill>"} --skills ${root}`);
 }
 export async function cmdGrade(args, adapterOverride) {
     const runDir = args._[0];
-    if (!runDir || !existsSync(runDir))
-        throw new Error("usage: skill-harness grade <run-dir> [--judge prov:model]");
+    if (!runDir)
+        throw new Error("usage: skill-harness grade <run-dir> [--judge prov:model] [--suspect-only]");
+    if (!existsSync(runDir)) {
+        throw new Error(`run dir not found: ${resolve(runDir)} (relative paths resolve against the cwd)`);
+    }
     // spec lives at <runDir>/../../../specification.yaml  (results/<tag>/<ts> -> tests/)
     const testsDir = dirname(dirname(dirname(runDir)));
     const specPath = join(testsDir, "specification.yaml");
@@ -171,13 +183,44 @@ export async function cmdGrade(args, adapterOverride) {
     const judgeFlag = flagStr(args, "judge");
     const judge = judgeFlag ? parseModelRef(judgeFlag) : (prev?.judge ?? parseModelRef(DEFAULT_JUDGE));
     const adapter = adapterOverride ?? getAdapter(prev?.harness ?? "pi");
-    const results = await regradeRun({ runDir, spec, adapter, judge, specDir: testsDir, now: nowIso });
+    const results = await regradeRun({
+        runDir, spec, adapter, judge, specDir: testsDir, now: nowIso,
+        onlySuspect: args.flags["suspect-only"] === true || args.flags["suspect-only"] === "true",
+    });
     for (const s of results.scenarios) {
         console.log(`  ${s.id} → ${s.judge_verdict}: ${s.judge_reason}`);
     }
     const g = results.effective_grade;
     console.log(`\n  re-graded with ${judge.provider}:${judge.model} → ${g.letter} (${g.pct}%) ${g.ship ? "SHIP" : "NOT READY"}`);
 }
+/**
+ * Re-score saved runs against the current spec's thresholds — no model or judge calls.
+ * Reps are the measurement; thresholds are policy. When policy changes, recompute rather
+ * than reconcile two numbers in prose.
+ */
+async function cmdRescore(args) {
+    const runDirs = args._;
+    if (runDirs.length === 0)
+        throw new Error("usage: skill-harness rescore <run-dir> [<run-dir> ...]");
+    let moved = 0;
+    for (const raw of runDirs) {
+        const runDir = resolve(raw);
+        if (!existsSync(runDir))
+            throw new Error(`run dir not found: ${runDir} (relative paths resolve against the cwd)`);
+        const spec = loadSpec(specPathForRunDir(runDir));
+        const { results, changes } = rescoreRun({ runDir, spec, now: nowIso });
+        const g = results.effective_grade;
+        console.log(`\n${results.skill} · ${results.model}`);
+        for (const c of changes) {
+            console.log(`  ${c.id}: ${c.from} → ${c.to}  (${c.passes}/${c.clean} @ threshold ${c.toThreshold})`);
+        }
+        if (changes.length === 0)
+            console.log("  (no verdict changed)");
+        moved += changes.length;
+        console.log(`  → ${g.letter} (${g.pct}%) ${g.passed}/${g.total} ${g.ship ? "SHIP" : "NOT READY"}`);
+    }
+    console.log(`\n${runDirs.length} run(s) re-scored, ${moved} verdict(s) moved.`);
+}
 async function cmdReview(args) {
     const root = flagStr(args, "skills", process.cwd());
     const target = args._[0];
@@ -222,6 +265,95 @@ async function cmdAddTest(args) {
     appendFileSync(skill.specPath, block, "utf8");
     console.log(`added scenario ${id} to ${skill.specPath}`);
 }
+/** Write a spec to disk, creating its tests/ dir. The single choke point for spec
+ *  writes (init/suggest) so a future atomic-write/backup/audit change lands in one place. */
+function writeSpecFile(specPath, text) {
+    mkdirSync(dirname(specPath), { recursive: true });
+    writeFileSync(specPath, text, "utf8");
+}
+export async function cmdInit(args) {
+    const root = flagStr(args, "skills", process.cwd());
+    const target = args._[0];
+    if (!target)
+        throw new Error("usage: skill-harness init <skill> --skills <root> [--force]");
+    const skill = resolveSkill(root, target);
+    const force = flagStr(args, "force") !== undefined;
+    if (skill.hasSpec && !force) {
+        throw new Error(`${skill.specPath} exists — edit it, or pass --force to overwrite`);
+    }
+    const text = renderTemplateSpec(skill.name);
+    parseSpec(text, skill.specPath); // guard: the template must always be valid
+    writeSpecFile(skill.specPath, text);
+    console.log(`wrote template ${skill.specPath} — fill it in, or run \`skill-harness suggest ${skill.name}\` to LLM-draft it.`);
+}
+export async function cmdSuggest(args, adapterOverride) {
+    const root = flagStr(args, "skills", process.cwd());
+    const target = args._[0];
+    if (!target)
+        throw new Error("usage: skill-harness suggest <skill> --skills <root> [--model prov:model] [--force]");
+    // resolveSkill throws a SKILL.md-specific error when the directory exists but
+    // lacks one, so we don't reimplement that check here; a resolved skill always
+    // has a SKILL.md at skill.dir.
+    const skill = resolveSkill(root, target);
+    const skillMd = readFileSync(join(skill.dir, "SKILL.md"), "utf8");
+    // Overwrite without --force only when the target is absent or an *unedited*
+    // template. A file that still carries the sentinel but no longer matches the
+    // pristine template has been hand-edited — refuse it so we never clobber work.
+    const force = flagStr(args, "force") !== undefined;
+    if (skill.hasSpec && !force) {
+        const existing = readFileSync(skill.specPath, "utf8");
+        if (existing !== renderTemplateSpec(skill.name)) {
+            const hint = isTemplateSpec(existing)
+                ? "looks like an edited template — pass --force to overwrite (or delete your edits)"
+                : "already has real content — pass --force to overwrite";
+            throw new Error(`${skill.specPath} ${hint}`);
+        }
+    }
+    const model = parseModelRef(flagStr(args, "model", DEFAULT_SUGGEST_MODEL));
+    const adapter = adapterOverride ?? getAdapter("pi");
+    const cwd = mkdtempSync(join(tmpdir(), "sh-suggest-cwd-"));
+    try {
+        const basePrompt = buildSuggestPrompt(skill.name, skillMd);
+        let text = null;
+        let count = 0;
+        let lastErr = "";
+        for (let attempt = 0; attempt < 2 && text === null; attempt++) {
+            const prompt = attempt === 0
+                ? basePrompt
+                : `${basePrompt}\n\nYour previous reply was rejected: ${lastErr}. Return corrected JSON only.`;
+            const raw = await adapter.judge({ model, prompt, cwd });
+            // A `[judge error` prefix is a hard adapter failure (auth, exec, credits) —
+            // retrying won't help, so fail fast with a model hint. An empty reply is
+            // treated as a transient miss and gets the same retry as a bad-JSON reply.
+            if (raw.startsWith("[judge error")) {
+                throw new Error(`model ${model.provider}:${model.model} failed — ${raw.trim()} (try --model fireworks:...)`);
+            }
+            if (!raw.trim()) {
+                lastErr = "model produced no output";
+                continue;
+            }
+            try {
+                const draft = parseSuggestDraft(raw);
+                const candidate = renderDraftSpec(skill.name, draft);
+                parseSpec(candidate, skill.specPath); // validate before writing
+                text = candidate;
+                count = draft.scenarios.length;
+            }
+            catch (e) {
+                lastErr = e instanceof Error ? e.message : String(e);
+            }
+        }
+        if (text === null) {
+            throw new Error(`could not get a valid spec from ${model.provider}:${model.model} after 2 attempts (${lastErr}) — try \`skill-harness init ${skill.name}\` for a manual template`);
+        }
+        writeSpecFile(skill.specPath, text);
+        console.log(`drafted ${count} scenario(s) → ${skill.specPath}`);
+        console.log(`review it (especially the proposed critical set), then \`skill-harness run ${skill.name} --skills ${root}\``);
+    }
+    finally {
+        rmSync(cwd, { recursive: true, force: true });
+    }
+}
 /** Exit-code contract: 0 = clean (no findings), 1 = >=1 finding, or a resolution error (unknown skill/root, no skills with a spec). */
 export async function cmdLint(args) {
     const root = flagStr(args, "skills", process.cwd());
@@ -269,11 +401,14 @@ export async function cmdLint(args) {
 // ---------------------------------------------------------------- dispatch
 const HELP = `skill-harness — test/optimize loop for agent skills (pi harness)
 
-  run    <skill|all> --skills <root> [--model prov:model ...] [--models file]
+  run    <skill|all> --skills <root> [--model prov:model ...] [--models file] [--only A1,D2]
                      [--mode red|green|force] [--judge prov:model] [--harness pi] [--label name] [--parallel N] [--reps N] [--pass-threshold T]
-  grade  <run-dir>   [--judge prov:model]      re-grade saved transcripts (neutral judge)
+  grade  <run-dir>   [--judge prov:model] [--suspect-only]   re-grade saved transcripts (neutral judge)
+  rescore <run-dir>...                          re-score saved reps vs current spec thresholds (free)
   review <skill>     --skills <root> [--port N] serve the interactive review UI
   add-test <skill>   --skills <root> --id ID --title T --turn ... --check ... [--critical] [--mode seeded --fixture path]
+  init   <skill>     --skills <root> [--force]     scaffold a commented template spec (free, offline)
+  suggest <skill>    --skills <root> [--model prov:model] [--force]  LLM-draft a spec from SKILL.md (spends tokens)
   list   --skills <root>                        discovered skills + spec status
   lint   <skill|all> --skills <root>           validate specs/fixtures + results-consistency (CI gate; exits non-zero on findings)
 
@@ -284,8 +419,11 @@ export async function main(argv) {
     switch (cmd) {
         case "run": return cmdRun(args);
         case "grade": return cmdGrade(args);
+        case "rescore": return cmdRescore(args);
         case "review": return cmdReview(args);
         case "add-test": return cmdAddTest(args);
+        case "init": return cmdInit(args);
+        case "suggest": return cmdSuggest(args);
         case "list": return cmdList(args);
         case "lint": return cmdLint(args);
         case undefined:

package/dist/serve.js

@@ -3,7 +3,7 @@ import { readFileSync, existsSync } from "node:fs";
 import { join, dirname } from "node:path";
 import { fileURLToPath } from "node:url";
 import { spawn } from "node:child_process";
-import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, findJudgeRawFiles, effectiveThreshold, } from "@skill-harness/core";
+import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, findJudgeRawFiles, effectiveThreshold, envFlag, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 /** Locate assets/report.template.html relative to dist/ or src/. */
@@ -202,7 +202,7 @@ export async function serveReview(opts) {
     console.log(`  → ${link}`);
     console.log(`  flip verdicts + add notes in the browser; saves persist to results.yaml.`);
     console.log(`  Ctrl-C to stop.\n`);
-    if (opts.open !== false && !process.env.SKILL_CHECK_NO_OPEN)
+    if (opts.open !== false && !envFlag("NO_OPEN"))
         tryOpen(link);
     return { port: port, close: () => server.close() };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/cli",
-    "version": "0.1.2",
+    "version": "0.3.0",
     "description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
     "type": "module",
     "license": "MIT",
@@ -34,7 +34,7 @@
         "prepack": "rm -rf ./assets && cp -r ../../assets ./assets && cp ../../LICENSE ./LICENSE"
     },
     "dependencies": {
-        "@skill-harness/core": "0.1.2",
-        "@skill-harness/adapters": "0.1.2"
+        "@skill-harness/core": "0.3.0",
+        "@skill-harness/adapters": "0.3.0"
     }
 }