@skill-harness/cli 0.3.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/report.grade.js +3 -1
- package/assets/report.template.html +13 -8
- package/dist/cli.d.ts +18 -0
- package/dist/cli.js +119 -11
- package/dist/serve.js +21 -9
- package/package.json +3 -3
package/assets/report.grade.js
CHANGED
|
@@ -143,7 +143,9 @@ export function liftNoneBadge(col) {
|
|
|
143
143
|
if (col.lift) {
|
|
144
144
|
return { text: "lift not comparable", title: col.liftHeadline || "nothing in the red baseline could be compared" };
|
|
145
145
|
}
|
|
146
|
-
|
|
146
|
+
// Green and force are both skill-delivered, so both can be missing a baseline.
|
|
147
|
+
// A red column is the baseline and gets no badge at all.
|
|
148
|
+
if (col.mode === "green" || col.mode === "force") {
|
|
147
149
|
return { text: "no red baseline", title: "run the same scenarios with --mode red to get a baseline" };
|
|
148
150
|
}
|
|
149
151
|
return null;
|
|
@@ -126,10 +126,10 @@ function render() {
|
|
|
126
126
|
const grades = DATA.columns.map((col) => gradeColumn(col, DATA.shipBar, DATA.critical));
|
|
127
127
|
const queue = [];
|
|
128
128
|
DATA.columns.forEach((col) => {
|
|
129
|
-
// A suspect cell in a red
|
|
130
|
-
//
|
|
131
|
-
//
|
|
132
|
-
if (col.mode && col.mode !== "green") return;
|
|
129
|
+
// A suspect cell in a red column can't be re-judged from here (/rejudge 400s
|
|
130
|
+
// for an unscored baseline) — don't queue it. The matrix below still shows
|
|
131
|
+
// suspect badges for every column regardless of mode.
|
|
132
|
+
if (col.mode && col.mode !== "green" && col.mode !== "force") return;
|
|
133
133
|
for (const scn of DATA.scenarios) {
|
|
134
134
|
const cell = col.cells[scn.id];
|
|
135
135
|
if (cell && cell.suspect && !cell.override) queue.push({ col: col.index, id: scn.id, label: col.label, reason: cell.judge_reason });
|
|
@@ -150,8 +150,8 @@ function render() {
|
|
|
150
150
|
DATA.columns.forEach((col, i) => {
|
|
151
151
|
const g = grades[i];
|
|
152
152
|
let gradeHtml;
|
|
153
|
-
if (col.mode && col.mode !== "green") {
|
|
154
|
-
//
|
|
153
|
+
if (col.mode && col.mode !== "green" && col.mode !== "force") {
|
|
154
|
+
// Green and force are scored; a red baseline column has no ship grade.
|
|
155
155
|
gradeHtml = `<span class='badge no'>not scored (${escapeHtml(col.mode)})</span>`;
|
|
156
156
|
if (g.suspect > 0) gradeHtml += ` — ${g.suspect} suspect`;
|
|
157
157
|
} else {
|
|
@@ -244,7 +244,7 @@ async function openPanel(colIndex, scenarioId) {
|
|
|
244
244
|
panel.innerHTML = `
|
|
245
245
|
<button class="close" id="closeBtn">×</button>
|
|
246
246
|
<h2>${scn.id} · ${escapeHtml(scn.title)}</h2>
|
|
247
|
-
<div class="meta">${escapeHtml(col.label)} · judge ${escapeHtml(col.judge.provider + ":" + col.judge.model)}</div>
|
|
247
|
+
<div class="meta">${escapeHtml(col.label)} · mode ${escapeHtml(col.mode || "green")} · judge ${escapeHtml(col.judge.provider + ":" + col.judge.model)}</div>
|
|
248
248
|
<div class="reason"><b>judge:</b> ${cell.judge_verdict} — ${escapeHtml(cell.judge_reason || "(no reason)")}</div>
|
|
249
249
|
${cell.suspect ? `<div class="reason" style="color:#b45309"><b>⚠ suspect:</b> judge listed no failed item — re-judge before trusting this FAIL</div>` : ""}
|
|
250
250
|
<div class="toggle">
|
|
@@ -358,7 +358,12 @@ function renderTrends(data) {
|
|
|
358
358
|
const badge = last.ship ? "<span class='badge ship'>SHIP</span>" : "<span class='badge no'>NOT READY</span>";
|
|
359
359
|
const trunc = m.truncated ? ` <span class='dim'>(last ${m.runs.length})</span>` : "";
|
|
360
360
|
const skippedNote = m.skipped > 0 ? ` <span class='dim'>(${m.skipped} unreadable)</span>` : "";
|
|
361
|
-
|
|
361
|
+
// The mode is part of the series identity, not decoration: a tag that moved
|
|
362
|
+
// from green to force delivery has TWO series here, and the same skill text
|
|
363
|
+
// scores differently under each — so an unlabelled pair of sparklines would
|
|
364
|
+
// read as one history that jumped.
|
|
365
|
+
const modeNote = m.mode ? ` <span class='dim'>${escapeHtml(m.mode)}</span>` : "";
|
|
366
|
+
html += `<div class="tmodel"><div class="tmodel-h">${escapeHtml(m.model)}${modeNote} — ${sparkline(m.runs)} ${last.letter} (${last.pct}%) ${badge}${trunc}${skippedNote}</div>`;
|
|
362
367
|
html += "<table class='tgrid'><thead><tr><th></th>";
|
|
363
368
|
for (const run of m.runs) html += `<th title="${escapeHtml(run.label || run.timestamp)}">${escapeHtml((run.label || run.timestamp).slice(0, 8))}</th>`;
|
|
364
369
|
html += "</tr></thead><tbody>";
|
package/dist/cli.d.ts
CHANGED
|
@@ -6,14 +6,32 @@ export interface Args {
|
|
|
6
6
|
multi: Record<string, string[]>;
|
|
7
7
|
}
|
|
8
8
|
export declare function flagStr(args: Args, key: string, fallback?: string): string | undefined;
|
|
9
|
+
/** A boolean flag: bare `--flag`, or an explicit `--flag=true` / `--flag=1`. */
|
|
10
|
+
export declare function flagBool(args: Args, key: string): boolean;
|
|
9
11
|
/** Parse the run's reps + pass-threshold flags. Throws on an invalid provided value. */
|
|
10
12
|
export declare function parseRunTuning(args: Args): {
|
|
11
13
|
reps: number;
|
|
12
14
|
passThreshold: number;
|
|
13
15
|
};
|
|
16
|
+
export declare function cmdRun(args: Args): Promise<void>;
|
|
14
17
|
export declare function cmdGrade(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
|
|
18
|
+
/**
|
|
19
|
+
* Re-evaluate needle gates against the saved staged diffs — free, except for the reps
|
|
20
|
+
* whose gate verdict flips from fail to pass, which the judge never saw and must now
|
|
21
|
+
* be shown. Prints the cost before making those calls.
|
|
22
|
+
*/
|
|
23
|
+
export declare function cmdRegate(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
|
|
15
24
|
export declare function cmdInit(args: Args): Promise<void>;
|
|
16
25
|
export declare function cmdSuggest(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
|
|
17
26
|
/** Exit-code contract: 0 = clean (no findings), 1 = >=1 finding, or a resolution error (unknown skill/root, no skills with a spec). */
|
|
18
27
|
export declare function cmdLint(args: Args): Promise<void>;
|
|
28
|
+
/**
|
|
29
|
+
* The help text, rendered per call rather than frozen at module load.
|
|
30
|
+
*
|
|
31
|
+
* The `defaults:` line reports the judge that the *next* command will actually
|
|
32
|
+
* use, which `SKILL_HARNESS_JUDGE` can change after this module was imported. A
|
|
33
|
+
* help screen that prints a default the tool won't use is worse than one that
|
|
34
|
+
* prints none.
|
|
35
|
+
*/
|
|
36
|
+
export declare function help(): string;
|
|
19
37
|
export declare function main(argv: string[]): Promise<void>;
|
package/dist/cli.js
CHANGED
|
@@ -3,11 +3,13 @@ import { readFileSync, existsSync, appendFileSync, mkdirSync, writeFileSync, mkd
|
|
|
3
3
|
import { basename, dirname, join, resolve } from "node:path";
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
5
|
import yaml from "js-yaml";
|
|
6
|
-
import { discover, resolveSkill, loadSpec, parseSpec, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, specPathForRunDir, collectLift, } from "@skill-harness/core";
|
|
6
|
+
import { discover, resolveSkill, loadSpec, parseSpec, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, HARNESS_VERSION, defaultJudge, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, } from "@skill-harness/core";
|
|
7
7
|
import { getAdapter } from "@skill-harness/adapters";
|
|
8
8
|
import { serveReview } from "./serve.js";
|
|
9
9
|
const DEFAULT_MODEL = "fireworks:accounts/fireworks/models/deepseek-v4-pro";
|
|
10
|
-
|
|
10
|
+
// The judge default lives in core (`defaultJudge()`), which resolves
|
|
11
|
+
// SKILL_HARNESS_JUDGE over a baked value — it was duplicated in three places
|
|
12
|
+
// before, and the pi extension's copy could disagree with this one.
|
|
11
13
|
const DEFAULT_SUGGEST_MODEL = "claude-code:claude-opus-4-8";
|
|
12
14
|
const REPEATABLE = new Set(["model", "turn", "check"]);
|
|
13
15
|
function parseArgs(argv) {
|
|
@@ -48,6 +50,11 @@ export function flagStr(args, key, fallback) {
|
|
|
48
50
|
return "";
|
|
49
51
|
return fallback;
|
|
50
52
|
}
|
|
53
|
+
/** A boolean flag: bare `--flag`, or an explicit `--flag=true` / `--flag=1`. */
|
|
54
|
+
export function flagBool(args, key) {
|
|
55
|
+
const v = args.flags[key];
|
|
56
|
+
return v === true || v === "true" || v === "1";
|
|
57
|
+
}
|
|
51
58
|
function resolveModels(args) {
|
|
52
59
|
const models = [...(args.multi.model ?? [])];
|
|
53
60
|
const file = flagStr(args, "models");
|
|
@@ -108,17 +115,26 @@ async function cmdList(args) {
|
|
|
108
115
|
}
|
|
109
116
|
console.log(`\n● = testable · ○ = no spec yet · ✗ = spec present but invalid`);
|
|
110
117
|
}
|
|
111
|
-
async function cmdRun(args) {
|
|
118
|
+
export async function cmdRun(args) {
|
|
112
119
|
const root = flagStr(args, "skills", process.cwd());
|
|
113
120
|
const target = args._[0];
|
|
114
121
|
if (!target)
|
|
115
122
|
throw new Error("usage: skill-harness run <skill|all> --skills <root>");
|
|
123
|
+
// Judge policy is checked first, ahead of the harness/PATH check and long before
|
|
124
|
+
// any subject tokens are spent: a refusal that arrives after the model has been
|
|
125
|
+
// paid for is a worse version of the problem it exists to prevent.
|
|
126
|
+
const judgeFlagRun = flagStr(args, "judge");
|
|
127
|
+
const judge = parseModelRef(judgeFlagRun ?? defaultJudge());
|
|
128
|
+
assertJudgeAllowed(judge, {
|
|
129
|
+
source: judgeFlagRun ? "--judge" : "the default judge (SKILL_HARNESS_JUDGE or the baked value)",
|
|
130
|
+
allowMetered: flagBool(args, "allow-metered-judge"),
|
|
131
|
+
});
|
|
116
132
|
const harnessName = flagStr(args, "harness", "pi");
|
|
117
133
|
const adapter = getAdapter(harnessName);
|
|
118
134
|
if (!(await adapter.available()))
|
|
119
135
|
throw new Error(`harness \`${harnessName}\` is not on PATH`);
|
|
120
136
|
const mode = flagStr(args, "mode", "green") || "green";
|
|
121
|
-
const
|
|
137
|
+
const canary = flagBool(args, "canary");
|
|
122
138
|
const label = flagStr(args, "label") || null;
|
|
123
139
|
const parallel = Math.max(1, Number(flagStr(args, "parallel", "1")) || 1);
|
|
124
140
|
const { reps, passThreshold } = parseRunTuning(args);
|
|
@@ -134,10 +150,16 @@ async function cmdRun(args) {
|
|
|
134
150
|
console.log(`skip ${skill.name}: no spec`);
|
|
135
151
|
continue;
|
|
136
152
|
}
|
|
153
|
+
// A run from an older tool than the records already here would produce numbers
|
|
154
|
+
// that look comparable and are not. Checked per skill, before its first token.
|
|
155
|
+
assertNotDowngraded(skill.dir, "run");
|
|
137
156
|
const spec = loadSpec(skill.specPath);
|
|
138
157
|
for (const token of modelTokens) {
|
|
139
158
|
const model = parseModelRef(token);
|
|
140
|
-
|
|
159
|
+
// The version is on the banner because a stale global install is otherwise
|
|
160
|
+
// invisible: a 0.1.0 binary grades a 0.3.x corpus, produces plausible
|
|
161
|
+
// numbers, and nothing on screen says which tool made them.
|
|
162
|
+
console.log(`\n▶ ${spec.skill} · ${harnessName}:${token} · mode=${mode} · judge=${judge.provider}:${judge.model} · skill-harness ${HARNESS_VERSION}`);
|
|
141
163
|
const summary = await runSkillModel({
|
|
142
164
|
spec,
|
|
143
165
|
skillDir: skill.dir,
|
|
@@ -153,6 +175,7 @@ async function cmdRun(args) {
|
|
|
153
175
|
reps,
|
|
154
176
|
passThreshold,
|
|
155
177
|
only,
|
|
178
|
+
canary,
|
|
156
179
|
onProgress: (m) => console.log(m),
|
|
157
180
|
});
|
|
158
181
|
summaries.push(summary);
|
|
@@ -181,11 +204,25 @@ export async function cmdGrade(args, adapterOverride) {
|
|
|
181
204
|
// an explicit --judge flag still wins; with no prior results, fall back to
|
|
182
205
|
// the CLI default.
|
|
183
206
|
const judgeFlag = flagStr(args, "judge");
|
|
184
|
-
const judge = judgeFlag ? parseModelRef(judgeFlag) : (prev?.judge ?? parseModelRef(
|
|
207
|
+
const judge = judgeFlag ? parseModelRef(judgeFlag) : (prev?.judge ?? parseModelRef(defaultJudge()));
|
|
208
|
+
// A regrade reuses the judge the run RECORDED, so a run that names a metered judge
|
|
209
|
+
// bills on every later regrade with no flag typed anywhere. Latent rather than live
|
|
210
|
+
// in the reference corpus (all ~140 committed runs there record `claude-code`), but
|
|
211
|
+
// it is the one path where the cost decision was made by a file, not a person.
|
|
212
|
+
assertJudgeAllowed(judge, {
|
|
213
|
+
source: judgeFlag ? "--judge" : prev?.judge ? "the run's recorded judge" : "the default judge",
|
|
214
|
+
allowMetered: flagBool(args, "allow-metered-judge"),
|
|
215
|
+
});
|
|
185
216
|
const adapter = adapterOverride ?? getAdapter(prev?.harness ?? "pi");
|
|
217
|
+
// Warn rather than refuse: re-grading is cheap, it writes no new measurement of the
|
|
218
|
+
// model, and it is one of the ways someone diagnoses a stale install in the first
|
|
219
|
+
// place. Blocking the diagnosis would be the wrong trade.
|
|
220
|
+
const stale = downgradeWarning(dirname(testsDir));
|
|
221
|
+
if (stale)
|
|
222
|
+
console.error(stale);
|
|
186
223
|
const results = await regradeRun({
|
|
187
224
|
runDir, spec, adapter, judge, specDir: testsDir, now: nowIso,
|
|
188
|
-
onlySuspect: args
|
|
225
|
+
onlySuspect: flagBool(args, "suspect-only"),
|
|
189
226
|
});
|
|
190
227
|
for (const s of results.scenarios) {
|
|
191
228
|
console.log(` ${s.id} → ${s.judge_verdict}: ${s.judge_reason}`);
|
|
@@ -221,6 +258,50 @@ async function cmdRescore(args) {
|
|
|
221
258
|
}
|
|
222
259
|
console.log(`\n${runDirs.length} run(s) re-scored, ${moved} verdict(s) moved.`);
|
|
223
260
|
}
|
|
261
|
+
/**
|
|
262
|
+
* Re-evaluate needle gates against the saved staged diffs — free, except for the reps
|
|
263
|
+
* whose gate verdict flips from fail to pass, which the judge never saw and must now
|
|
264
|
+
* be shown. Prints the cost before making those calls.
|
|
265
|
+
*/
|
|
266
|
+
export async function cmdRegate(args, adapterOverride) {
|
|
267
|
+
const runDirs = args._;
|
|
268
|
+
if (runDirs.length === 0)
|
|
269
|
+
throw new Error("usage: skill-harness regate <run-dir> [<run-dir> ...] [--judge prov:model]");
|
|
270
|
+
const judgeFlag = flagStr(args, "judge");
|
|
271
|
+
let moved = 0;
|
|
272
|
+
let calls = 0;
|
|
273
|
+
for (const raw of runDirs) {
|
|
274
|
+
const runDir = resolve(raw);
|
|
275
|
+
if (!existsSync(runDir))
|
|
276
|
+
throw new Error(`run dir not found: ${runDir} (relative paths resolve against the cwd)`);
|
|
277
|
+
const specPath = specPathForRunDir(runDir);
|
|
278
|
+
const spec = loadSpec(specPath);
|
|
279
|
+
const prev = readResults(runDir);
|
|
280
|
+
const judge = judgeFlag ? parseModelRef(judgeFlag) : (prev.judge ?? parseModelRef(defaultJudge()));
|
|
281
|
+
assertJudgeAllowed(judge, {
|
|
282
|
+
source: judgeFlag ? "--judge" : "the run's recorded judge",
|
|
283
|
+
allowMetered: flagBool(args, "allow-metered-judge"),
|
|
284
|
+
});
|
|
285
|
+
const { results, changes, judgeCalls } = await regateRun({
|
|
286
|
+
runDir, spec, specDir: dirname(specPath),
|
|
287
|
+
adapter: adapterOverride ?? getAdapter(prev.harness ?? "pi"),
|
|
288
|
+
judge, now: nowIso,
|
|
289
|
+
});
|
|
290
|
+
const g = results.effective_grade;
|
|
291
|
+
console.log(`\n${results.skill} · ${results.model}`);
|
|
292
|
+
for (const c of changes) {
|
|
293
|
+
console.log(` ${c.id}: ${c.from} → ${c.to} (gate ${c.gate}${c.judged ? ", re-judged from the saved transcript" : ", no judge call"})`);
|
|
294
|
+
}
|
|
295
|
+
if (changes.length === 0)
|
|
296
|
+
console.log(" (no verdict changed)");
|
|
297
|
+
console.log(` → ${g.letter} (${g.pct}%) ${g.passed}/${g.total} ${g.ship ? "SHIP" : "NOT READY"}`);
|
|
298
|
+
moved += changes.length;
|
|
299
|
+
calls += judgeCalls;
|
|
300
|
+
}
|
|
301
|
+
// The cost line matters: regate is advertised as free, and it is — except for the
|
|
302
|
+
// flipped reps, which it must not spend silently.
|
|
303
|
+
console.log(`\n${runDirs.length} run(s) re-gated, ${moved} verdict(s) moved, ${calls} judge call(s) (no model re-runs).`);
|
|
304
|
+
}
|
|
224
305
|
async function cmdReview(args) {
|
|
225
306
|
const root = flagStr(args, "skills", process.cwd());
|
|
226
307
|
const target = args._[0];
|
|
@@ -399,12 +480,23 @@ export async function cmdLint(args) {
|
|
|
399
480
|
process.exitCode = findings.length > 0 ? 1 : 0;
|
|
400
481
|
}
|
|
401
482
|
// ---------------------------------------------------------------- dispatch
|
|
402
|
-
|
|
483
|
+
/**
|
|
484
|
+
* The help text, rendered per call rather than frozen at module load.
|
|
485
|
+
*
|
|
486
|
+
* The `defaults:` line reports the judge that the *next* command will actually
|
|
487
|
+
* use, which `SKILL_HARNESS_JUDGE` can change after this module was imported. A
|
|
488
|
+
* help screen that prints a default the tool won't use is worse than one that
|
|
489
|
+
* prints none.
|
|
490
|
+
*/
|
|
491
|
+
export function help() {
|
|
492
|
+
return `skill-harness ${HARNESS_VERSION} — test/optimize loop for agent skills (pi harness)
|
|
403
493
|
|
|
404
494
|
run <skill|all> --skills <root> [--model prov:model ...] [--models file] [--only A1,D2]
|
|
405
495
|
[--mode red|green|force] [--judge prov:model] [--harness pi] [--label name] [--parallel N] [--reps N] [--pass-threshold T]
|
|
496
|
+
[--canary] green only: spend ONE probe proving the skill reached the model, and abort the run if it did not
|
|
406
497
|
grade <run-dir> [--judge prov:model] [--suspect-only] re-grade saved transcripts (neutral judge)
|
|
407
498
|
rescore <run-dir>... re-score saved reps vs current spec thresholds (free)
|
|
499
|
+
regate <run-dir>... [--judge prov:model] re-evaluate diff needles against the saved diffs (free; judges only reps whose gate flipped)
|
|
408
500
|
review <skill> --skills <root> [--port N] serve the interactive review UI
|
|
409
501
|
add-test <skill> --skills <root> --id ID --title T --turn ... --check ... [--critical] [--mode seeded --fixture path]
|
|
410
502
|
init <skill> --skills <root> [--force] scaffold a commented template spec (free, offline)
|
|
@@ -412,7 +504,15 @@ const HELP = `skill-harness — test/optimize loop for agent skills (pi harness)
|
|
|
412
504
|
list --skills <root> discovered skills + spec status
|
|
413
505
|
lint <skill|all> --skills <root> validate specs/fixtures + results-consistency (CI gate; exits non-zero on findings)
|
|
414
506
|
|
|
415
|
-
|
|
507
|
+
version print ${HARNESS_VERSION} and exit (also --version / -v)
|
|
508
|
+
|
|
509
|
+
defaults: model=${DEFAULT_MODEL} judge=${defaultJudge()} mode=green harness=pi
|
|
510
|
+
green and force are both scored; red is the unscored baseline. green delivery depends on the
|
|
511
|
+
harness version (pi >= 0.83.0 discloses only the skill's description and loads the body on demand),
|
|
512
|
+
so --mode force is the delivery that cannot silently degrade — and --canary proves green per run.
|
|
513
|
+
the judge default is Opus on your Claude subscription (\`claude-code\` → \`claude -p\`), not a
|
|
514
|
+
metered API key. Set SKILL_HARNESS_JUDGE to change it for a repo or a shell; --judge wins over both.`;
|
|
515
|
+
}
|
|
416
516
|
export async function main(argv) {
|
|
417
517
|
const cmd = argv[0];
|
|
418
518
|
const args = parseArgs(argv.slice(1));
|
|
@@ -420,21 +520,29 @@ export async function main(argv) {
|
|
|
420
520
|
case "run": return cmdRun(args);
|
|
421
521
|
case "grade": return cmdGrade(args);
|
|
422
522
|
case "rescore": return cmdRescore(args);
|
|
523
|
+
case "regate": return cmdRegate(args);
|
|
423
524
|
case "review": return cmdReview(args);
|
|
424
525
|
case "add-test": return cmdAddTest(args);
|
|
425
526
|
case "init": return cmdInit(args);
|
|
426
527
|
case "suggest": return cmdSuggest(args);
|
|
427
528
|
case "list": return cmdList(args);
|
|
428
529
|
case "lint": return cmdLint(args);
|
|
530
|
+
case "version":
|
|
531
|
+
case "--version":
|
|
532
|
+
case "-v":
|
|
533
|
+
// Bare version, one line, nothing else: this is what a script or a confused
|
|
534
|
+
// user greps to find out whether the binary on PATH is the one they think.
|
|
535
|
+
console.log(HARNESS_VERSION);
|
|
536
|
+
return;
|
|
429
537
|
case undefined:
|
|
430
538
|
case "help":
|
|
431
539
|
case "--help":
|
|
432
540
|
case "-h":
|
|
433
|
-
console.log(
|
|
541
|
+
console.log(help());
|
|
434
542
|
return;
|
|
435
543
|
default:
|
|
436
544
|
console.error(`unknown command: ${cmd}\n`);
|
|
437
|
-
console.log(
|
|
545
|
+
console.log(help());
|
|
438
546
|
process.exitCode = 1;
|
|
439
547
|
}
|
|
440
548
|
}
|
package/dist/serve.js
CHANGED
|
@@ -3,7 +3,7 @@ import { readFileSync, existsSync } from "node:fs";
|
|
|
3
3
|
import { join, dirname } from "node:path";
|
|
4
4
|
import { fileURLToPath } from "node:url";
|
|
5
5
|
import { spawn } from "node:child_process";
|
|
6
|
-
import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, findJudgeRawFiles, effectiveThreshold, envFlag, } from "@skill-harness/core";
|
|
6
|
+
import { collectReport, renderReport, collectTrends, readResults, writeResults, applyOverride, preserveTranscript, findTranscriptFiles, ensureResultsGitignore, appendJournal, loadSpec, regradeScenario, refreshRubricHashes, findJudgeRawFiles, effectiveThreshold, scoreContextFor, isScoredMode, envFlag, } from "@skill-harness/core";
|
|
7
7
|
import { getAdapter } from "@skill-harness/adapters";
|
|
8
8
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
9
9
|
/** Locate assets/report.template.html relative to dist/ or src/. */
|
|
@@ -99,9 +99,13 @@ export async function serveReview(opts) {
|
|
|
99
99
|
return;
|
|
100
100
|
}
|
|
101
101
|
const results = readResults(column.runDir);
|
|
102
|
-
|
|
102
|
+
// Skill-delivered runs only (green or force). A red baseline's transcripts
|
|
103
|
+
// can be re-judged too — `skill-harness grade <run-dir>` does it — but a
|
|
104
|
+
// baseline has no grade for this endpoint to report back, and the button
|
|
105
|
+
// sits under a scorecard.
|
|
106
|
+
if (!isScoredMode(results.mode)) {
|
|
103
107
|
res.writeHead(400, { "content-type": "application/json" });
|
|
104
|
-
res.end(JSON.stringify({ ok: false, error:
|
|
108
|
+
res.end(JSON.stringify({ ok: false, error: `only scored runs (green/force) can be re-judged here — for a ${results.mode} run use \`skill-harness grade\`` }));
|
|
105
109
|
return;
|
|
106
110
|
}
|
|
107
111
|
const specPath = join(opts.skillDir, "tests", "specification.yaml");
|
|
@@ -126,13 +130,21 @@ export async function serveReview(opts) {
|
|
|
126
130
|
try {
|
|
127
131
|
const rr = await regradeScenario({
|
|
128
132
|
runDir: column.runDir, spec, scenario, adapter, judge: results.judge,
|
|
129
|
-
specDir: dirname(specPath), threshold,
|
|
133
|
+
specDir: dirname(specPath), threshold, mode: results.mode,
|
|
130
134
|
});
|
|
131
135
|
const merged = results.scenarios.map((s) => s.id === body.scenarioId ? { ...rr, override: s.override, note: s.note } : s);
|
|
132
136
|
const written = writeResults(column.runDir, {
|
|
133
137
|
skill: results.skill, harness: results.harness, model: results.model, judge: results.judge,
|
|
134
138
|
timestamp: results.timestamp, label: results.label, mode: results.mode, scenarios: merged,
|
|
135
|
-
|
|
139
|
+
partial: results.partial,
|
|
140
|
+
// Provenance survives a UI re-judge, same as it does through `grade`.
|
|
141
|
+
harness_cli_version: results.harness_cli_version, delivery_canary: results.delivery_canary,
|
|
142
|
+
// Recorded hashes were being dropped here entirely, which silently
|
|
143
|
+
// retired the staleness gate for any run re-judged from the UI. Carried,
|
|
144
|
+
// with the one `rubric:` key this re-judge actually applied refreshed —
|
|
145
|
+
// the same doctrine `grade` follows (see refreshRubricHashes).
|
|
146
|
+
source_hashes: refreshRubricHashes(results.source_hashes, spec, [body.scenarioId]),
|
|
147
|
+
}, scoreContextFor(results, spec));
|
|
136
148
|
ensureResultsGitignore(join(opts.skillDir, "tests", "results"));
|
|
137
149
|
const g = written.effective_grade;
|
|
138
150
|
appendJournal(column.runDir, { event: "score", ts: new Date().toISOString(), passed: g.passed, total: g.total, pct: g.pct, letter: g.letter, ship: g.ship, note: g.note });
|
|
@@ -167,11 +179,11 @@ export async function serveReview(opts) {
|
|
|
167
179
|
return;
|
|
168
180
|
}
|
|
169
181
|
// writeResults recomputes effective_grade override-aware against the CURRENT
|
|
170
|
-
// spec's ship bar — a saved override can never leave a stale grade.
|
|
171
|
-
//
|
|
182
|
+
// spec's ship bar — a saved override can never leave a stale grade. Scored
|
|
183
|
+
// modes only: /save must not put a grade on a red baseline (PR #1 finding),
|
|
184
|
+
// and since 0.5.0 "scored" includes force.
|
|
172
185
|
const spec = loadSpec(join(opts.skillDir, "tests", "specification.yaml"));
|
|
173
|
-
|
|
174
|
-
writeResults(column.runDir, patched, ctx);
|
|
186
|
+
writeResults(column.runDir, patched, scoreContextFor(patched, spec));
|
|
175
187
|
// Unconditional: a results root created before schema-2/journal.jsonl existed
|
|
176
188
|
// may still have a stale .gitignore body — every save (not just overrides)
|
|
177
189
|
// must roll it forward so journal.jsonl doesn't end up tracked.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skill-harness/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
"prepack": "rm -rf ./assets && mkdir -p ./assets && cp ../../assets/report.* ./assets/ && cp ../../LICENSE ./LICENSE"
|
|
45
45
|
},
|
|
46
46
|
"dependencies": {
|
|
47
|
-
"@skill-harness/core": "0.
|
|
48
|
-
"@skill-harness/adapters": "0.
|
|
47
|
+
"@skill-harness/core": "0.5.0",
|
|
48
|
+
"@skill-harness/adapters": "0.5.0"
|
|
49
49
|
}
|
|
50
50
|
}
|