@skill-harness/cli 0.3.2 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -143,7 +143,9 @@ export function liftNoneBadge(col) {
143
143
  if (col.lift) {
144
144
  return { text: "lift not comparable", title: col.liftHeadline || "nothing in the red baseline could be compared" };
145
145
  }
146
- if (col.mode === "green") {
146
+ // Green and force are both skill-delivered, so both can be missing a baseline.
147
+ // A red column is the baseline and gets no badge at all.
148
+ if (col.mode === "green" || col.mode === "force") {
147
149
  return { text: "no red baseline", title: "run the same scenarios with --mode red to get a baseline" };
148
150
  }
149
151
  return null;
@@ -126,10 +126,10 @@ function render() {
126
126
  const grades = DATA.columns.map((col) => gradeColumn(col, DATA.shipBar, DATA.critical));
127
127
  const queue = [];
128
128
  DATA.columns.forEach((col) => {
129
- // A suspect cell in a red/force column can't be re-judged (/rejudge 400s
130
- // "only green runs can be re-judged") — don't queue it. The matrix below
131
- // still shows suspect badges for every column regardless of mode.
132
- if (col.mode && col.mode !== "green") return;
129
+ // A suspect cell in a red column can't be re-judged from here (/rejudge 400s
130
+ // for an unscored baseline) — don't queue it. The matrix below still shows
131
+ // suspect badges for every column regardless of mode.
132
+ if (col.mode && col.mode !== "green" && col.mode !== "force") return;
133
133
  for (const scn of DATA.scenarios) {
134
134
  const cell = col.cells[scn.id];
135
135
  if (cell && cell.suspect && !cell.override) queue.push({ col: col.index, id: scn.id, label: col.label, reason: cell.judge_reason });
@@ -150,8 +150,8 @@ function render() {
150
150
  DATA.columns.forEach((col, i) => {
151
151
  const g = grades[i];
152
152
  let gradeHtml;
153
- if (col.mode && col.mode !== "green") {
154
- // Only green runs are scored; a red/force column has no ship grade.
153
+ if (col.mode && col.mode !== "green" && col.mode !== "force") {
154
+ // Green and force are scored; a red baseline column has no ship grade.
155
155
  gradeHtml = `<span class='badge no'>not scored (${escapeHtml(col.mode)})</span>`;
156
156
  if (g.suspect > 0) gradeHtml += ` — ${g.suspect} suspect`;
157
157
  } else {
@@ -244,7 +244,7 @@ async function openPanel(colIndex, scenarioId) {
244
244
  panel.innerHTML = `
245
245
  <button class="close" id="closeBtn">×</button>
246
246
  <h2>${scn.id} · ${escapeHtml(scn.title)}</h2>
247
- <div class="meta">${escapeHtml(col.label)} · judge ${escapeHtml(col.judge.provider + ":" + col.judge.model)}</div>
247
+ <div class="meta">${escapeHtml(col.label)} · mode ${escapeHtml(col.mode || "green")} · judge ${escapeHtml(col.judge.provider + ":" + col.judge.model)}</div>
248
248
  <div class="reason"><b>judge:</b> ${cell.judge_verdict} — ${escapeHtml(cell.judge_reason || "(no reason)")}</div>
249
249
  ${cell.suspect ? `<div class="reason" style="color:#b45309"><b>⚠ suspect:</b> judge listed no failed item — re-judge before trusting this FAIL</div>` : ""}
250
250
  <div class="toggle">
@@ -358,7 +358,12 @@ function renderTrends(data) {
358
358
  const badge = last.ship ? "<span class='badge ship'>SHIP</span>" : "<span class='badge no'>NOT READY</span>";
359
359
  const trunc = m.truncated ? ` <span class='dim'>(last ${m.runs.length})</span>` : "";
360
360
  const skippedNote = m.skipped > 0 ? ` <span class='dim'>(${m.skipped} unreadable)</span>` : "";
361
- html += `<div class="tmodel"><div class="tmodel-h">${escapeHtml(m.model)} ${sparkline(m.runs)} ${last.letter} (${last.pct}%) ${badge}${trunc}${skippedNote}</div>`;
361
+ // The mode is part of the series identity, not decoration: a tag that moved
362
+ // from green to force delivery has TWO series here, and the same skill text
363
+ // scores differently under each — so an unlabelled pair of sparklines would
364
+ // read as one history that jumped.
365
+ const modeNote = m.mode ? ` <span class='dim'>${escapeHtml(m.mode)}</span>` : "";
366
+ html += `<div class="tmodel"><div class="tmodel-h">${escapeHtml(m.model)}${modeNote} — ${sparkline(m.runs)} ${last.letter} (${last.pct}%) ${badge}${trunc}${skippedNote}</div>`;
362
367
  html += "<table class='tgrid'><thead><tr><th></th>";
363
368
  for (const run of m.runs) html += `<th title="${escapeHtml(run.label || run.timestamp)}">${escapeHtml((run.label || run.timestamp).slice(0, 8))}</th>`;
364
369
  html += "</tr></thead><tbody>";
package/dist/cli.d.ts CHANGED
@@ -6,14 +6,32 @@ export interface Args {
6
6
  multi: Record<string, string[]>;
7
7
  }
8
8
  export declare function flagStr(args: Args, key: string, fallback?: string): string | undefined;
9
+ /** A boolean flag: bare `--flag`, or an explicit `--flag=true` / `--flag=1`. */
10
+ export declare function flagBool(args: Args, key: string): boolean;
9
11
  /** Parse the run's reps + pass-threshold flags. Throws on an invalid provided value. */
10
12
  export declare function parseRunTuning(args: Args): {
11
13
  reps: number;
12
14
  passThreshold: number;
13
15
  };
16
+ export declare function cmdRun(args: Args): Promise<void>;
14
17
  export declare function cmdGrade(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
18
+ /**
19
+ * Re-evaluate needle gates against the saved staged diffs — free, except for the reps
20
+ * whose gate verdict flips from fail to pass, which the judge never saw and must now
21
+ * be shown. Prints the cost before making those calls.
22
+ */
23
+ export declare function cmdRegate(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
15
24
  export declare function cmdInit(args: Args): Promise<void>;
16
25
  export declare function cmdSuggest(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
17
26
  /** Exit-code contract: 0 = clean (no findings), 1 = >=1 finding, or a resolution error (unknown skill/root, no skills with a spec). */
18
27
  export declare function cmdLint(args: Args): Promise<void>;
28
+ /**
29
+ * The help text, rendered per call rather than frozen at module load.
30
+ *
31
+ * The `defaults:` line reports the judge that the *next* command will actually
32
+ * use, which `SKILL_HARNESS_JUDGE` can change after this module was imported. A
33
+ * help screen that prints a default the tool won't use is worse than one that
34
+ * prints none.
35
+ */
36
+ export declare function help(): string;
19
37
  export declare function main(argv: string[]): Promise<void>;
package/dist/cli.js CHANGED
@@ -3,11 +3,13 @@ import { readFileSync, existsSync, appendFileSync, mkdirSync, writeFileSync, mkd
3
3
  import { basename, dirname, join, resolve } from "node:path";
4
4
  import { tmpdir } from "node:os";
5
5
  import yaml from "js-yaml";
6
- import { discover, resolveSkill, loadSpec, parseSpec, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, specPathForRunDir, collectLift, } from "@skill-harness/core";
6
+ import { discover, resolveSkill, loadSpec, parseSpec, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, HARNESS_VERSION, defaultJudge, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, } from "@skill-harness/core";
7
7
  import { getAdapter } from "@skill-harness/adapters";
8
8
  import { serveReview } from "./serve.js";
9
9
  const DEFAULT_MODEL = "fireworks:accounts/fireworks/models/deepseek-v4-pro";
10
- const DEFAULT_JUDGE = "anthropic:claude-opus-4-8";
10
+ // The judge default lives in core (`defaultJudge()`), which resolves
11
+ // SKILL_HARNESS_JUDGE over a baked value — it was duplicated in three places
12
+ // before, and the pi extension's copy could disagree with this one.
11
13
  const DEFAULT_SUGGEST_MODEL = "claude-code:claude-opus-4-8";
12
14
  const REPEATABLE = new Set(["model", "turn", "check"]);
13
15
  function parseArgs(argv) {
@@ -48,6 +50,11 @@ export function flagStr(args, key, fallback) {
48
50
  return "";
49
51
  return fallback;
50
52
  }
53
+ /** A boolean flag: bare `--flag`, or an explicit `--flag=true` / `--flag=1`. */
54
+ export function flagBool(args, key) {
55
+ const v = args.flags[key];
56
+ return v === true || v === "true" || v === "1";
57
+ }
51
58
  function resolveModels(args) {
52
59
  const models = [...(args.multi.model ?? [])];
53
60
  const file = flagStr(args, "models");
@@ -108,17 +115,26 @@ async function cmdList(args) {
108
115
  }
109
116
  console.log(`\n● = testable · ○ = no spec yet · ✗ = spec present but invalid`);
110
117
  }
111
- async function cmdRun(args) {
118
+ export async function cmdRun(args) {
112
119
  const root = flagStr(args, "skills", process.cwd());
113
120
  const target = args._[0];
114
121
  if (!target)
115
122
  throw new Error("usage: skill-harness run <skill|all> --skills <root>");
123
+ // Judge policy is checked first, ahead of the harness/PATH check and long before
124
+ // any subject tokens are spent: a refusal that arrives after the model has been
125
+ // paid for is a worse version of the problem it exists to prevent.
126
+ const judgeFlagRun = flagStr(args, "judge");
127
+ const judge = parseModelRef(judgeFlagRun ?? defaultJudge());
128
+ assertJudgeAllowed(judge, {
129
+ source: judgeFlagRun ? "--judge" : "the default judge (SKILL_HARNESS_JUDGE or the baked value)",
130
+ allowMetered: flagBool(args, "allow-metered-judge"),
131
+ });
116
132
  const harnessName = flagStr(args, "harness", "pi");
117
133
  const adapter = getAdapter(harnessName);
118
134
  if (!(await adapter.available()))
119
135
  throw new Error(`harness \`${harnessName}\` is not on PATH`);
120
136
  const mode = flagStr(args, "mode", "green") || "green";
121
- const judge = parseModelRef(flagStr(args, "judge", DEFAULT_JUDGE));
137
+ const canary = flagBool(args, "canary");
122
138
  const label = flagStr(args, "label") || null;
123
139
  const parallel = Math.max(1, Number(flagStr(args, "parallel", "1")) || 1);
124
140
  const { reps, passThreshold } = parseRunTuning(args);
@@ -134,10 +150,16 @@ async function cmdRun(args) {
134
150
  console.log(`skip ${skill.name}: no spec`);
135
151
  continue;
136
152
  }
153
+ // A run from an older tool than the records already here would produce numbers
154
+ // that look comparable and are not. Checked per skill, before its first token.
155
+ assertNotDowngraded(skill.dir, "run");
137
156
  const spec = loadSpec(skill.specPath);
138
157
  for (const token of modelTokens) {
139
158
  const model = parseModelRef(token);
140
- console.log(`\n▶ ${spec.skill} · ${harnessName}:${token} · mode=${mode} · judge=${judge.provider}:${judge.model}`);
159
+ // The version is on the banner because a stale global install is otherwise
160
+ // invisible: a 0.1.0 binary grades a 0.3.x corpus, produces plausible
161
+ // numbers, and nothing on screen says which tool made them.
162
+ console.log(`\n▶ ${spec.skill} · ${harnessName}:${token} · mode=${mode} · judge=${judge.provider}:${judge.model} · skill-harness ${HARNESS_VERSION}`);
141
163
  const summary = await runSkillModel({
142
164
  spec,
143
165
  skillDir: skill.dir,
@@ -153,6 +175,7 @@ async function cmdRun(args) {
153
175
  reps,
154
176
  passThreshold,
155
177
  only,
178
+ canary,
156
179
  onProgress: (m) => console.log(m),
157
180
  });
158
181
  summaries.push(summary);
@@ -181,11 +204,25 @@ export async function cmdGrade(args, adapterOverride) {
181
204
  // an explicit --judge flag still wins; with no prior results, fall back to
182
205
  // the CLI default.
183
206
  const judgeFlag = flagStr(args, "judge");
184
- const judge = judgeFlag ? parseModelRef(judgeFlag) : (prev?.judge ?? parseModelRef(DEFAULT_JUDGE));
207
+ const judge = judgeFlag ? parseModelRef(judgeFlag) : (prev?.judge ?? parseModelRef(defaultJudge()));
208
+ // A regrade reuses the judge the run RECORDED, so a run that names a metered judge
209
+ // bills on every later regrade with no flag typed anywhere. Latent rather than live
210
+ // in the reference corpus (all ~140 committed runs there record `claude-code`), but
211
+ // it is the one path where the cost decision was made by a file, not a person.
212
+ assertJudgeAllowed(judge, {
213
+ source: judgeFlag ? "--judge" : prev?.judge ? "the run's recorded judge" : "the default judge",
214
+ allowMetered: flagBool(args, "allow-metered-judge"),
215
+ });
185
216
  const adapter = adapterOverride ?? getAdapter(prev?.harness ?? "pi");
217
+ // Warn rather than refuse: re-grading is cheap, it writes no new measurement of the
218
+ // model, and it is one of the ways someone diagnoses a stale install in the first
219
+ // place. Blocking the diagnosis would be the wrong trade.
220
+ const stale = downgradeWarning(dirname(testsDir));
221
+ if (stale)
222
+ console.error(stale);
186
223
  const results = await regradeRun({
187
224
  runDir, spec, adapter, judge, specDir: testsDir, now: nowIso,
188
- onlySuspect: args.flags["suspect-only"] === true || args.flags["suspect-only"] === "true",
225
+ onlySuspect: flagBool(args, "suspect-only"),
189
226
  });
190
227
  for (const s of results.scenarios) {
191
228
  console.log(` ${s.id} → ${s.judge_verdict}: ${s.judge_reason}`);
@@ -221,6 +258,50 @@ async function cmdRescore(args) {
221
258
  }
222
259
  console.log(`\n${runDirs.length} run(s) re-scored, ${moved} verdict(s) moved.`);
223
260
  }
261
+ /**
262
+ * Re-evaluate needle gates against the saved staged diffs — free, except for the reps
263
+ * whose gate verdict flips from fail to pass, which the judge never saw and must now
264
+ * be shown. Prints the cost before making those calls.
265
+ */
266
+ export async function cmdRegate(args, adapterOverride) {
267
+ const runDirs = args._;
268
+ if (runDirs.length === 0)
269
+ throw new Error("usage: skill-harness regate <run-dir> [<run-dir> ...] [--judge prov:model]");
270
+ const judgeFlag = flagStr(args, "judge");
271
+ let moved = 0;
272
+ let calls = 0;
273
+ for (const raw of runDirs) {
274
+ const runDir = resolve(raw);
275
+ if (!existsSync(runDir))
276
+ throw new Error(`run dir not found: ${runDir} (relative paths resolve against the cwd)`);
277
+ const specPath = specPathForRunDir(runDir);
278
+ const spec = loadSpec(specPath);
279
+ const prev = readResults(runDir);
280
+ const judge = judgeFlag ? parseModelRef(judgeFlag) : (prev.judge ?? parseModelRef(defaultJudge()));
281
+ assertJudgeAllowed(judge, {
282
+ source: judgeFlag ? "--judge" : "the run's recorded judge",
283
+ allowMetered: flagBool(args, "allow-metered-judge"),
284
+ });
285
+ const { results, changes, judgeCalls } = await regateRun({
286
+ runDir, spec, specDir: dirname(specPath),
287
+ adapter: adapterOverride ?? getAdapter(prev.harness ?? "pi"),
288
+ judge, now: nowIso,
289
+ });
290
+ const g = results.effective_grade;
291
+ console.log(`\n${results.skill} · ${results.model}`);
292
+ for (const c of changes) {
293
+ console.log(` ${c.id}: ${c.from} → ${c.to} (gate ${c.gate}${c.judged ? ", re-judged from the saved transcript" : ", no judge call"})`);
294
+ }
295
+ if (changes.length === 0)
296
+ console.log(" (no verdict changed)");
297
+ console.log(` → ${g.letter} (${g.pct}%) ${g.passed}/${g.total} ${g.ship ? "SHIP" : "NOT READY"}`);
298
+ moved += changes.length;
299
+ calls += judgeCalls;
300
+ }
301
+ // The cost line matters: regate is advertised as free, and it is — except for the
302
+ // flipped reps, which it must not spend silently.
303
+ console.log(`\n${runDirs.length} run(s) re-gated, ${moved} verdict(s) moved, ${calls} judge call(s) (no model re-runs).`);
304
+ }
224
305
  async function cmdReview(args) {
225
306
  const root = flagStr(args, "skills", process.cwd());
226
307
  const target = args._[0];
@@ -399,12 +480,23 @@ export async function cmdLint(args) {
399
480
  process.exitCode = findings.length > 0 ? 1 : 0;
400
481
  }
401
482
  // ---------------------------------------------------------------- dispatch
402
- const HELP = `skill-harness — test/optimize loop for agent skills (pi harness)
483
+ /**
484
+ * The help text, rendered per call rather than frozen at module load.
485
+ *
486
+ * The `defaults:` line reports the judge that the *next* command will actually
487
+ * use, which `SKILL_HARNESS_JUDGE` can change after this module was imported. A
488
+ * help screen that prints a default the tool won't use is worse than one that
489
+ * prints none.
490
+ */
491
+ export function help() {
492
+ return `skill-harness ${HARNESS_VERSION} — test/optimize loop for agent skills (pi harness)
403
493
 
404
494
  run <skill|all> --skills <root> [--model prov:model ...] [--models file] [--only A1,D2]
405
495
  [--mode red|green|force] [--judge prov:model] [--harness pi] [--label name] [--parallel N] [--reps N] [--pass-threshold T]
496
+ [--canary] green only: spend ONE probe proving the skill reached the model, and abort the run if it did not
406
497
  grade <run-dir> [--judge prov:model] [--suspect-only] re-grade saved transcripts (neutral judge)
407
498
  rescore <run-dir>... re-score saved reps vs current spec thresholds (free)
499
+ regate <run-dir>... [--judge prov:model] re-evaluate diff needles against the saved diffs (free; judges only reps whose gate flipped)
408
500
  review <skill> --skills <root> [--port N] serve the interactive review UI
409
501
  add-test <skill> --skills <root> --id ID --title T --turn ... --check ... [--critical] [--mode seeded --fixture path]
410
502
  init <skill> --skills <root> [--force] scaffold a commented template spec (free, offline)
@@ -412,7 +504,15 @@ const HELP = `skill-harness — test/optimize loop for agent skills (pi harness)
412
504
  list --skills <root> discovered skills + spec status
413
505
  lint <skill|all> --skills <root> validate specs/fixtures + results-consistency (CI gate; exits non-zero on findings)
414
506
 
415
- defaults: model=${DEFAULT_MODEL} judge=${DEFAULT_JUDGE} mode=green harness=pi`;
507
+ version print ${HARNESS_VERSION} and exit (also --version / -v)
508
+
509
+ defaults: model=${DEFAULT_MODEL} judge=${defaultJudge()} mode=green harness=pi
510
+ green and force are both scored; red is the unscored baseline. green delivery depends on the
511
+ harness version (pi >= 0.83.0 discloses only the skill's description and loads the body on demand),
512
+ so --mode force is the delivery that cannot silently degrade — and --canary proves green per run.
513
+ the judge default is Opus on your Claude subscription (\`claude-code\` → \`claude -p\`), not a
514
+ metered API key. Set SKILL_HARNESS_JUDGE to change it for a repo or a shell; --judge wins over both.`;
515
+ }
416
516
  export async function main(argv) {
417
517
  const cmd = argv[0];
418
518
  const args = parseArgs(argv.slice(1));
@@ -420,21 +520,29 @@ export async function main(argv) {
420
520
  case "run": return cmdRun(args);
421
521
  case "grade": return cmdGrade(args);
422
522
  case "rescore": return cmdRescore(args);
523
+ case "regate": return cmdRegate(args);
423
524
  case "review": return cmdReview(args);
424
525
  case "add-test": return cmdAddTest(args);
425
526
  case "init": return cmdInit(args);
426
527
  case "suggest": return cmdSuggest(args);
427
528
  case "list": return cmdList(args);
428
529
  case "lint": return cmdLint(args);
530
+ case "version":
531
+ case "--version":
532
+ case "-v":
533
+ // Bare version, one line, nothing else: this is what a script or a confused
534
+ // user greps to find out whether the binary on PATH is the one they think.
535
+ console.log(HARNESS_VERSION);
536
+ return;
429
537
  case undefined:
430
538
  case "help":
431
539
  case "--help":
432
540
  case "-h":
433
- console.log(HELP);
541
+ console.log(help());
434
542
  return;
435
543
  default:
436
544
  console.error(`unknown command: ${cmd}\n`);
437
- console.log(HELP);
545
+ console.log(help());
438
546
  process.exitCode = 1;
439
547
  }
440
548
  }
package/dist/serve.js CHANGED
@@ -3,7 +3,7 @@ import { readFileSync, existsSync } from "node:fs";
3
3
  import { join, dirname } from "node:path";
4
4
  import { fileURLToPath } from "node:url";
5
5
  import { spawn } from "node:child_process";
6
- import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, findJudgeRawFiles, effectiveThreshold, envFlag, } from "@skill-harness/core";
6
+ import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, envFlag, } from "@skill-harness/core";
7
7
  import { getAdapter } from "@skill-harness/adapters";
8
8
  const __dirname = dirname(fileURLToPath(import.meta.url));
9
9
  /** Locate assets/report.template.html relative to dist/ or src/. */
@@ -99,9 +99,13 @@ export async function serveReview(opts) {
99
99
  return;
100
100
  }
101
101
  const results = readResults(column.runDir);
102
- if (results.mode !== "green") {
102
+ // Skill-delivered runs only (green or force). A red baseline's transcripts
103
+ // can be re-judged too — `skill-harness grade <run-dir>` does it — but a
104
+ // baseline has no grade for this endpoint to report back, and the button
105
+ // sits under a scorecard.
106
+ if (!isScoredMode(results.mode)) {
103
107
  res.writeHead(400, { "content-type": "application/json" });
104
- res.end(JSON.stringify({ ok: false, error: "only green runs can be re-judged" }));
108
+ res.end(JSON.stringify({ ok: false, error: `only scored runs (green/force) can be re-judged here — for a ${results.mode} run use \`skill-harness grade\`` }));
105
109
  return;
106
110
  }
107
111
  const specPath = join(opts.skillDir, "tests", "specification.yaml");
@@ -126,13 +130,21 @@ export async function serveReview(opts) {
126
130
  try {
127
131
  const rr = await regradeScenario({
128
132
  runDir: column.runDir, spec, scenario, adapter, judge: results.judge,
129
- specDir: dirname(specPath), threshold,
133
+ specDir: dirname(specPath), threshold, mode: results.mode,
130
134
  });
131
135
  const merged = results.scenarios.map((s) => s.id === body.scenarioId ? { ...rr, override: s.override, note: s.note } : s);
132
136
  const written = writeResults(column.runDir, {
133
137
  skill: results.skill, harness: results.harness, model: results.model, judge: results.judge,
134
138
  timestamp: results.timestamp, label: results.label, mode: results.mode, scenarios: merged,
135
- }, { shipBar: spec.ship_bar, critical: spec.critical });
139
+ partial: results.partial,
140
+ // Provenance survives a UI re-judge, same as it does through `grade`.
141
+ harness_cli_version: results.harness_cli_version, delivery_canary: results.delivery_canary,
142
+ // Recorded hashes were being dropped here entirely, which silently
143
+ // retired the staleness gate for any run re-judged from the UI. Carried,
144
+ // with the one `rubric:` key this re-judge actually applied refreshed —
145
+ // the same doctrine `grade` follows (see refreshRubricHashes).
146
+ source_hashes: refreshRubricHashes(results.source_hashes, spec, [body.scenarioId]),
147
+ }, scoreContextFor(results, spec));
136
148
  ensureResultsGitignore(join(opts.skillDir, "tests", "results"));
137
149
  const g = written.effective_grade;
138
150
  appendJournal(column.runDir, { event: "score", ts: new Date().toISOString(), passed: g.passed, total: g.total, pct: g.pct, letter: g.letter, ship: g.ship, note: g.note });
@@ -167,11 +179,11 @@ export async function serveReview(opts) {
167
179
  return;
168
180
  }
169
181
  // writeResults recomputes effective_grade override-aware against the CURRENT
170
- // spec's ship bar — a saved override can never leave a stale grade. Only
171
- // green runs are scored (PR #1 finding: /save must not grade red/force runs).
182
+ // spec's ship bar — a saved override can never leave a stale grade. Scored
183
+ // modes only: /save must not put a grade on a red baseline (PR #1 finding),
184
+ // and since 0.5.0 "scored" includes force.
172
185
  const spec = loadSpec(join(opts.skillDir, "tests", "specification.yaml"));
173
- const ctx = patched.mode === "green" ? { shipBar: spec.ship_bar, critical: spec.critical } : null;
174
- writeResults(column.runDir, patched, ctx);
186
+ writeResults(column.runDir, patched, scoreContextFor(patched, spec));
175
187
  // Unconditional: a results root created before schema-2/journal.jsonl existed
176
188
  // may still have a stale .gitignore body — every save (not just overrides)
177
189
  // must roll it forward so journal.jsonl doesn't end up tracked.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@skill-harness/cli",
3
- "version": "0.3.2",
3
+ "version": "0.5.0",
4
4
  "description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -44,7 +44,7 @@
44
44
  "prepack": "rm -rf ./assets && mkdir -p ./assets && cp ../../assets/report.* ./assets/ && cp ../../LICENSE ./LICENSE"
45
45
  },
46
46
  "dependencies": {
47
- "@skill-harness/core": "0.3.2",
48
- "@skill-harness/adapters": "0.3.2"
47
+ "@skill-harness/core": "0.5.0",
48
+ "@skill-harness/adapters": "0.5.0"
49
49
  }
50
50
  }