@skill-harness/cli 0.32.2 → 0.32.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +8 -3
- package/dist/diagnose-format.d.ts +3 -0
- package/dist/diagnose-format.js +32 -0
- package/package.json +3 -3
package/dist/cli.js
CHANGED
|
@@ -6,6 +6,7 @@ import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModel
|
|
|
6
6
|
import { getAdapter } from "@skill-harness/adapters";
|
|
7
7
|
import { serveReview } from "./serve.js";
|
|
8
8
|
import { cmdDecision } from "./decision.js";
|
|
9
|
+
import { formatDiagnosis } from "./diagnose-format.js";
|
|
9
10
|
const DEFAULT_MODEL = "fireworks:accounts/fireworks/models/deepseek-v4-pro";
|
|
10
11
|
// The judge default lives in core (`defaultJudge()`), which resolves
|
|
11
12
|
// SKILL_HARNESS_JUDGE over a baked value — it was duplicated in three places
|
|
@@ -768,7 +769,7 @@ export function help() {
|
|
|
768
769
|
preflight <skill|all> --skills <root> [--specs <root>] eval checks (${free("preflight")})
|
|
769
770
|
calibrate <manifest.json> [--run --judge p:m --out NEW_DIR] validate or judge reviewed controls
|
|
770
771
|
[--plan --judge p:m] [--only c1,c2] [--reuse PRIOR_DIR ...] exact historical reuse, no new subject calls
|
|
771
|
-
diagnose <run-dir> --skills <root> [--specs <root>] inspect saved evidence (${free("diagnose")})
|
|
772
|
+
diagnose <run-dir> --skills <root> [--specs <root>] [--format json|text] inspect saved evidence (${free("diagnose")})
|
|
772
773
|
Paid commands accept --max-memory-mb N / --max-write-mb N for sampled Linux process limits.
|
|
773
774
|
run/grade/calibrate save bounded private resources-*.json; unsupported host metrics stay unknown.
|
|
774
775
|
In Pi: /skill-harness create <idea> guided skill + optional agent + eval creation
|
|
@@ -807,12 +808,16 @@ export async function cmdPreflight(args) {
|
|
|
807
808
|
}
|
|
808
809
|
export async function cmdDiagnose(args) {
|
|
809
810
|
if (!args._[0])
|
|
810
|
-
throw new Error("usage: skill-harness diagnose <run-dir> --skills <root> [--specs <root>]");
|
|
811
|
+
throw new Error("usage: skill-harness diagnose <run-dir> --skills <root> [--specs <root>] [--format json|text]");
|
|
812
|
+
const format = flagStr(args, "format") ?? "json";
|
|
813
|
+
if (format !== "json" && format !== "text")
|
|
814
|
+
throw new Error("diagnose --format must be json or text");
|
|
811
815
|
const runDir = resolve(args._[0]);
|
|
812
816
|
const recorded = readResults(runDir);
|
|
813
817
|
const { skills, specs } = roots(args);
|
|
814
818
|
const selected = resolveSkill(skills, recorded.skill, specs);
|
|
815
|
-
|
|
819
|
+
const report = diagnoseRun(runDir, selected.dir, selected.specPath);
|
|
820
|
+
console.log(format === "text" ? formatDiagnosis(report) : JSON.stringify(report, null, 2));
|
|
816
821
|
}
|
|
817
822
|
export async function cmdCalibrate(args, adapterOverride) {
|
|
818
823
|
if (!args._[0])
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/** Human view of the same offline report; never a second scoring policy. */
|
|
2
|
+
export function formatDiagnosis(report) {
|
|
3
|
+
const grouped = report.criterionDimensions;
|
|
4
|
+
const lines = [
|
|
5
|
+
`${grouped.skill} · ${grouped.model} · ${grouped.mode}`,
|
|
6
|
+
"Saved criterion votes — current author-declared grouping, not a new assessment.",
|
|
7
|
+
"Counts are criterion observations, not passing answers. Mixed criteria remain indivisible.",
|
|
8
|
+
"No model calls; saved results and acceptance rules unchanged.",
|
|
9
|
+
`Coverage: ${grouped.coverage.availableVotes}/${grouped.coverage.expectedVotes} known-size votes available; ${grouped.coverage.unknownScenarioSizes} scenario sizes unknown.`,
|
|
10
|
+
`Mapping: ${grouped.metadataSha256}`,
|
|
11
|
+
];
|
|
12
|
+
for (const scenario of grouped.scenarios) {
|
|
13
|
+
lines.push("", `${scenario.id}: recorded ${scenario.original.judge_verdict}; override ${scenario.original.override ?? "none"}; objective ${scenario.original.objective?.status ?? "not recorded"}`);
|
|
14
|
+
for (const [dimension, counts] of Object.entries(scenario.counts)) {
|
|
15
|
+
if (counts.pass + counts.fail + counts.unavailable > 0)
|
|
16
|
+
lines.push(` ${dimension}: ${counts.pass} PASS, ${counts.fail} FAIL, ${counts.unavailable} unavailable`);
|
|
17
|
+
}
|
|
18
|
+
const reasons = new Set([scenario.unavailableReason, ...scenario.repetitions.map(rep => rep.unavailableReason)].filter(Boolean));
|
|
19
|
+
for (const reason of reasons)
|
|
20
|
+
lines.push(` Unavailable: ${reason}`);
|
|
21
|
+
}
|
|
22
|
+
if (report.issues.length) {
|
|
23
|
+
lines.push("", "Evidence and next checks:");
|
|
24
|
+
for (const issue of report.issues)
|
|
25
|
+
lines.push(`- ${issue.scenario ? issue.scenario + ": " : ""}${issue.category}: ${issue.evidence}\n Next: ${issue.next}`);
|
|
26
|
+
}
|
|
27
|
+
lines.push("", "Reuse plan:");
|
|
28
|
+
for (const item of report.reusePlan.scenarios)
|
|
29
|
+
lines.push(`- ${item.id}: ${item.action} — ${item.note}`);
|
|
30
|
+
return lines.join("\n");
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=diagnose-format.js.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skill-harness/cli",
|
|
3
|
-
"version": "0.32.
|
|
3
|
+
"version": "0.32.4",
|
|
4
4
|
"description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
"prepack": "node ../../scripts/release-pack.mjs --guard"
|
|
45
45
|
},
|
|
46
46
|
"dependencies": {
|
|
47
|
-
"@skill-harness/core": "0.32.
|
|
48
|
-
"@skill-harness/adapters": "0.32.
|
|
47
|
+
"@skill-harness/core": "0.32.4",
|
|
48
|
+
"@skill-harness/adapters": "0.32.4"
|
|
49
49
|
}
|
|
50
50
|
}
|