@skill-harness/cli 0.20.0 → 0.21.0

package/dist/cli.d.ts

@@ -22,7 +22,7 @@ export declare function releaseExitCode(summaries: Array<{
         };
     };
 }>): 0 | 1;
-export declare function cmdRun(args: Args): Promise<void>;
+export declare function cmdRun(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
 export declare function cmdGrade(args: Args, adapterOverride?: HarnessAdapter): Promise<void>;
 /**
  * Re-evaluate needle gates against the saved staged diffs — no subject call. A rep

package/dist/cli.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import { readFileSync, existsSync, mkdirSync, writeFileSync, mkdtempSync, rmSync } from "node:fs";
-import { basename, dirname, join, resolve, relative } from "node:path";
+import { basename, dirname, join, resolve } from "node:path";
 import { tmpdir } from "node:os";
 import { discover, resolveSkill, loadSpec, parseSpec, appendScenario, parseModelRef, runSkillModel, formatScorecard, readResults, regradeRun, lintSkill, failsGate, renderTemplateSpec, isTemplateSpec, renderDraftSpec, buildSuggestPrompt, parseSuggestDraft, rescoreRun, regateRun, specPathForRunDir, collectLift, collectStability, boundaryCells, stabilityNote, PATH_LEGEND, restampSkill, computeCoverage, formatCoverage, HARNESS_VERSION, defaultJudge, recordedJudgeOrDefault, assertJudgeAllowed, assertNotDowngraded, downgradeWarning, isScoredMode, resolveArm, isFreeOfflineCommand, } from "@skill-harness/core";
 import { getAdapter } from "@skill-harness/adapters";
@@ -82,6 +82,20 @@ function resolveModels(args) {
 function nowIso() {
     return new Date().toISOString();
 }
+function roots(args) {
+    return {
+        skills: flagStr(args, "skills", process.cwd()),
+        specs: flagStr(args, "specs", process.env.SKILL_HARNESS_SPECS) || undefined,
+    };
+}
+function discoverUsable(skillsRoot, specsRoot) {
+    const skills = discover(skillsRoot, specsRoot);
+    const duplicate = skills.find((skill) => skill.specSource === "both");
+    if (duplicate) {
+        throw new Error(`skill \`${duplicate.name}\` has a specification.yaml in both --skills and --specs; remove one (specs are not merged)`);
+    }
+    return skills;
+}
 /** Parse the run's reps + pass-threshold flags. Throws on an invalid provided value. */
 export function parseRunTuning(args) {
     let reps = 1;
@@ -104,31 +118,41 @@ export function parseRunTuning(args) {
 }
 // ---------------------------------------------------------------- commands
 async function cmdList(args) {
-    const root = flagStr(args, "skills", process.cwd());
-    const skills = discover(root);
+    const { skills: root, specs } = roots(args);
+    const skills = discover(root, specs);
+    let duplicate = false;
     console.log(`skills under ${root}:`);
     for (const s of skills) {
+        if (s.specSource === "both") {
+            duplicate = true;
+            console.log(`  ✗ ${s.name}  INVALID: specification.yaml exists in both --skills and --specs (specs are not merged)`);
+            continue;
+        }
         if (!s.hasSpec) {
-            console.log(`  ○ ${s.name}  (no spec)`);
+            const localOnly = specs && s.specSource === "skills" ? "; spec found under --skills" : "";
+            console.log(`  ○ ${s.name}  (no spec${localOnly})`);
             continue;
         }
         try {
             const spec = loadSpec(s.specPath);
             const seeded = spec.scenarios.filter((x) => x.mode === "seeded").length;
             const seededNote = seeded ? `, ${seeded} seeded` : "";
-            console.log(`  ● ${s.name}  (${spec.scenarios.length} scenarios${seededNote})`);
+            const sourceNote = specs ? `; spec: --${s.specSource}` : "";
+            console.log(`  ● ${s.name}  (${spec.scenarios.length} scenarios${seededNote}${sourceNote})`);
         }
         catch (e) {
             console.log(`  ✗ ${s.name}  INVALID: ${e instanceof Error ? e.message : e}`);
         }
     }
     console.log(`\n● = testable · ○ = no spec yet · ✗ = spec present but invalid`);
+    if (duplicate)
+        process.exitCode = 1;
 }
 export function releaseExitCode(summaries) {
     return summaries.some(({ results }) => isScoredMode(results.mode) && !results.partial && !results.effective_grade.ship) ? 1 : 0;
 }
-export async function cmdRun(args) {
-    const root = flagStr(args, "skills", process.cwd());
+export async function cmdRun(args, adapterOverride) {
+    const { skills: root, specs } = roots(args);
     const target = args._[0];
     if (!target)
         throw new Error("usage: skill-harness run <skill|all> --skills <root>");
@@ -150,7 +174,7 @@ export async function cmdRun(args) {
     const onlyRaw = flagStr(args, "only");
     const only = onlyRaw ? onlyRaw.split(",").map((x) => x.trim()).filter(Boolean) : undefined;
     const harnessName = flagStr(args, "harness", "pi");
-    const adapter = getAdapter(harnessName);
+    const adapter = adapterOverride ?? getAdapter(harnessName);
     if (!(await adapter.available()))
         throw new Error(`harness \`${harnessName}\` is not on PATH`);
     const mode = flagStr(args, "mode", "green") || "green";
@@ -165,8 +189,8 @@ export async function cmdRun(args) {
     const armName = flagStr(args, "arm") || null;
     const arm = resolveArm(root, armName);
     const skills = target === "all"
-        ? discover(root).filter((s) => s.hasSpec)
-        : [resolveSkill(root, target)];
+        ? discoverUsable(root, specs).filter((s) => s.hasSpec)
+        : [resolveSkill(root, target, specs)];
     const summaries = [];
     for (const skill of skills) {
         if (!skill.hasSpec) {
@@ -175,7 +199,7 @@ export async function cmdRun(args) {
         }
         // A run from an older tool than the records already here would produce numbers
         // that look comparable and are not. Checked per skill, before its first token.
-        assertNotDowngraded(skill.dir, "run");
+        assertNotDowngraded(dirname(skill.testsDir), "run");
         const spec = loadSpec(skill.specPath);
         for (const token of modelTokens) {
             const model = parseModelRef(token);
@@ -187,6 +211,8 @@ export async function cmdRun(args) {
                 spec,
                 skillDir: skill.dir,
                 specPath: skill.specPath,
+                testsDir: skill.testsDir,
+                recordSourceRoots: skill.specSource === "specs",
                 adapter,
                 model,
                 modelToken: token,
@@ -208,15 +234,16 @@ export async function cmdRun(args) {
             // Lift is derived from what's on disk, so it picks up a red baseline from
             // any earlier run — the tag dir (<harness>-<modelslug>) is the join key.
             const tag = basename(dirname(summary.runDir));
-            const lift = collectLift(skill.dir).find((l) => l.tag === tag);
+            const resultsSkillDir = dirname(skill.testsDir);
+            const lift = collectLift(resultsSkillDir).find((l) => l.tag === tag);
             // Stability is derived from history INCLUDING the run just written, and scoped to
             // this tag + mode: another model's flips under this model's scorecard would be a
             // worse error than not reporting them at all.
-            const stability = collectStability(skill.dir).filter((c) => c.tag === tag && c.mode === summary.results.mode);
+            const stability = collectStability(resultsSkillDir).filter((c) => c.tag === tag && c.mode === summary.results.mode);
             console.log("\n" + formatScorecard(summary, lift, stability) + "\n");
         }
     }
-    console.log(`\nReview interactively:  skill-harness review ${skills[0]?.name ?? "<skill>"} --skills ${root}`);
+    console.log(`\nReview interactively:  skill-harness review ${skills[0]?.name ?? "<skill>"} --skills ${root}${specs ? ` --specs ${specs}` : ""}`);
     // A full delivered run is a release gate. NOT READY — including one critical
     // failure hidden by a high aggregate — must be machine-visible to CI. Red
     // baselines and partial branch feedback are deliberately excluded.
@@ -363,9 +390,9 @@ export async function cmdRegate(args, adapterOverride) {
  * inventing freshness is the one thing this gate must never do.
  */
 async function cmdRestamp(args) {
-    const root = flagStr(args, "skills", process.cwd());
+    const { skills: root, specs } = roots(args);
     const target = args._[0] ?? "all";
-    const skills = target === "all" ? discover(root).filter((s) => s.hasSpec) : [resolveSkill(root, target)];
+    const skills = target === "all" ? discoverUsable(root, specs).filter((s) => s.hasSpec) : [resolveSkill(root, target, specs)];
     if (skills.length === 0)
         throw new Error(`no skills with a spec under ${root}`);
     let runs = 0;
@@ -374,7 +401,7 @@ async function cmdRestamp(args) {
     let unchanged = 0;
     let partial = 0;
     for (const skill of skills) {
-        const r = restampSkill(skill.dir, { from: flagStr(args, "from") });
+        const r = restampSkill(skill.dir, { from: flagStr(args, "from"), testsDir: skill.testsDir });
         runs += r.runs;
         upgraded += r.upgraded;
         unprovable += r.unprovable;
@@ -396,7 +423,7 @@ async function cmdRestamp(args) {
         `${partial > 0 ? ` (${partial} upgraded only in part.)` : ""} No models were called.`);
 }
 async function cmdStability(args) {
-    const root = flagStr(args, "skills", process.cwd());
+    const { skills: root, specs } = roots(args);
     const target = args._[0] ?? "all";
     const windowRaw = flagStr(args, "window");
     const window = windowRaw ? Number(windowRaw) : undefined;
@@ -404,12 +431,12 @@ async function cmdStability(args) {
         throw new Error(`--window must be an integer >= 2 (got \`${windowRaw}\`) — one run has no run-over-run step`);
     }
     const showAll = flagBool(args, "all");
-    const skills = target === "all" ? discover(root).filter((s) => s.hasSpec) : [resolveSkill(root, target)];
+    const skills = target === "all" ? discoverUsable(root, specs).filter((s) => s.hasSpec) : [resolveSkill(root, target, specs)];
     if (skills.length === 0)
         throw new Error(`no skills with a spec under ${root}`);
     let boundaries = 0;
     for (const skill of skills) {
-        const all = collectStability(skill.dir, { window });
+        const all = collectStability(dirname(skill.testsDir), { window });
         if (all.length === 0) {
             console.log(`\n${skill.name}: no scored runs yet — stability needs at least two runs of the same skill × model × mode`);
             continue;
@@ -450,20 +477,20 @@ async function cmdStability(args) {
     }
 }
 async function cmdReview(args) {
-    const root = flagStr(args, "skills", process.cwd());
+    const { skills: root, specs } = roots(args);
     const target = args._[0];
     if (!target)
         throw new Error("usage: skill-harness review <skill> --skills <root>");
-    const skill = resolveSkill(root, target);
+    const skill = resolveSkill(root, target, specs);
     const port = Number(flagStr(args, "port", "0")) || 0;
-    await serveReview({ skillDir: skill.dir, skillName: skill.name, port });
+    await serveReview({ skillDir: dirname(skill.testsDir), skillName: skill.name, port });
 }
 async function cmdAddTest(args) {
-    const root = flagStr(args, "skills", process.cwd());
+    const { skills: root, specs } = roots(args);
     const target = args._[0];
     if (!target)
         throw new Error("usage: skill-harness add-test <skill> --skills <root> --id ... --title ... --turn ... --check ...");
-    const skill = resolveSkill(root, target);
+    const skill = resolveSkill(root, target, specs);
     if (!skill.hasSpec)
         throw new Error(`${target} has no spec yet — create tests/specification.yaml first`);
     const id = flagStr(args, "id");
@@ -497,12 +524,12 @@ async function cmdAddTest(args) {
  * reddens CI for it teaches people to add a token `covers:` to silence it.
  */
 async function cmdCoverage(args) {
-    const root = flagStr(args, "skills", process.cwd());
+    const { skills: root, specs } = roots(args);
     const target = args._[0];
     if (!target)
         throw new Error("usage: skill-harness coverage <skill|all> --skills <root> [--strict]");
     const strict = flagBool(args, "strict");
-    const skills = target === "all" ? discover(root).filter((s) => s.hasSpec) : [resolveSkill(root, target)];
+    const skills = target === "all" ? discoverUsable(root, specs).filter((s) => s.hasSpec) : [resolveSkill(root, target, specs)];
     let anyUncovered = false;
     let anyBroken = false;
     for (const skill of skills) {
@@ -510,13 +537,13 @@ async function cmdCoverage(args) {
             continue;
         const spec = loadSpec(skill.specPath);
         const specDir = dirname(skill.specPath);
+        const skillRef = "../SKILL.md";
         const report = computeCoverage({
             specDir,
             scenarios: spec.scenarios,
-            // SKILL.md lives one level above tests/, and is the file `covers` almost
-            // always points at, so report on it even when nothing references it —
-            // otherwise a skill with zero `covers` reports 0 sections and looks fine.
-            baseFiles: [relative(specDir, join(skill.dir, "SKILL.md")).split("\\").join("/")],
+            // Keep the spec-facing path stable even when SKILL.md lives in another repo.
+            baseFiles: [skillRef],
+            fileOverrides: { [skillRef]: join(skill.dir, "SKILL.md") },
         });
         console.log(formatCoverage(report, spec.skill));
         if (report.uncovered.length)
@@ -543,11 +570,11 @@ function writeSpecFile(specPath, text) {
     writeFileSync(specPath, text, "utf8");
 }
 export async function cmdInit(args) {
-    const root = flagStr(args, "skills", process.cwd());
+    const { skills: root, specs } = roots(args);
     const target = args._[0];
     if (!target)
         throw new Error("usage: skill-harness init <skill> --skills <root> [--force]");
-    const skill = resolveSkill(root, target);
+    const skill = resolveSkill(root, target, specs);
     const force = flagStr(args, "force") !== undefined;
     if (skill.hasSpec && !force) {
         throw new Error(`${skill.specPath} exists — edit it, or pass --force to overwrite`);
@@ -558,14 +585,14 @@ export async function cmdInit(args) {
     console.log(`wrote template ${skill.specPath} — fill it in, or run \`skill-harness suggest ${skill.name}\` to LLM-draft it.`);
 }
 export async function cmdSuggest(args, adapterOverride) {
-    const root = flagStr(args, "skills", process.cwd());
+    const { skills: root, specs } = roots(args);
     const target = args._[0];
     if (!target)
         throw new Error("usage: skill-harness suggest <skill> --skills <root> [--model prov:model] [--force]");
     // resolveSkill throws a SKILL.md-specific error when the directory exists but
     // lacks one, so we don't reimplement that check here; a resolved skill always
     // has a SKILL.md at skill.dir.
-    const skill = resolveSkill(root, target);
+    const skill = resolveSkill(root, target, specs);
     const skillMd = readFileSync(join(skill.dir, "SKILL.md"), "utf8");
     // Overwrite without --force only when the target is absent or an *unedited*
     // template. A file that still carries the sentinel but no longer matches the
@@ -619,7 +646,7 @@ export async function cmdSuggest(args, adapterOverride) {
         }
         writeSpecFile(skill.specPath, text);
         console.log(`drafted ${count} scenario(s) → ${skill.specPath}`);
-        console.log(`review it (especially the proposed critical set), then \`skill-harness run ${skill.name} --skills ${root}\``);
+        console.log(`review it (especially the proposed critical set), then \`skill-harness run ${skill.name} --skills ${root}${specs ? ` --specs ${specs}` : ""}\``);
     }
     finally {
         rmSync(cwd, { recursive: true, force: true });
@@ -635,30 +662,31 @@ export async function cmdSuggest(args, adapterOverride) {
  * teach everyone to stop reading it.
  */
 export async function cmdLint(args) {
-    const root = flagStr(args, "skills", process.cwd());
+    const { skills: root, specs } = roots(args);
     const target = args._[0] ?? "all";
-    let skillDirs;
+    let skills;
     try {
-        skillDirs = target === "all"
-            ? discover(root).filter((s) => s.hasSpec).map((s) => s.dir)
-            : [resolveSkill(root, target).dir];
+        skills = target === "all"
+            ? discoverUsable(root, specs).filter((s) => s.hasSpec)
+            : [resolveSkill(root, target, specs)];
     }
     catch (e) {
         console.error(`error: ${e instanceof Error ? e.message : e}`);
         process.exitCode = 1;
         return;
     }
-    if (skillDirs.length === 0) {
+    if (skills.length === 0) {
         console.error(`no skills with a spec under ${root}`);
         process.exitCode = 1;
         return;
     }
     const gha = process.env.GITHUB_ACTIONS === "true";
     const findings = [];
-    for (const dir of skillDirs) {
+    for (const skill of skills) {
+        const dir = skill.dir;
         let f;
         try {
-            f = lintSkill(dir);
+            f = lintSkill(dir, skill.testsDir);
         }
         catch (e) {
             f = [{ skill: dir, code: "lint-error", message: e instanceof Error ? e.message : String(e) }];
@@ -676,7 +704,7 @@ export async function cmdLint(args) {
     }
     const gating = findings.filter(failsGate).length;
     const notes = findings.length - gating;
-    console.log(`\n${skillDirs.length} skill(s), ${gating} finding(s)${notes > 0 ? `, ${notes} note(s) (do not fail the gate)` : ""}`);
+    console.log(`\n${skills.length} skill(s), ${gating} finding(s)${notes > 0 ? `, ${notes} note(s) (do not fail the gate)` : ""}`);
     process.exitCode = gating > 0 ? 1 : 0;
 }
 // ---------------------------------------------------------------- dispatch
@@ -715,6 +743,9 @@ export function help() {
   lint   <skill|all> --skills <root>           validate specs/fixtures + results-consistency (${free("lint")}; CI gate; exits non-zero on findings)
   coverage <skill|all> --skills <root> [--strict]   which instruction sections have a declared test (${free("coverage")})
 
+  Every command with --skills also accepts --specs <root> (default: SKILL_HARNESS_SPECS).
+  Skill text comes from --skills; test assets and results come from --specs.
+
   version  print ${HARNESS_VERSION} and exit (also --version / -v)
 
 defaults: model=${DEFAULT_MODEL}  judge=${defaultJudge()}  mode=green  harness=pi

package/dist/serve.js

@@ -175,6 +175,7 @@ export async function serveReview(opts) {
                         // with the one `rubric:` key this re-judge actually applied refreshed —
                         // the same doctrine `grade` follows (see refreshRubricHashes).
                         source_hashes: refreshRubricHashes(results.source_hashes, spec, [body.scenarioId]),
+                        source_hash_roots: results.source_hash_roots,
                     }, scoreContextFor(results, spec));
                     ensureResultsGitignore(join(opts.skillDir, "tests", "results"));
                     const g = written.effective_grade;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/cli",
-    "version": "0.20.0",
+    "version": "0.21.0",
     "description": "skill-harness CLI — run, grade, review, and lint agent-skill scenarios on the pi harness",
     "type": "module",
     "license": "MIT",
@@ -44,7 +44,7 @@
         "prepack": "node ../../scripts/release-pack.mjs --guard"
     },
     "dependencies": {
-        "@skill-harness/core": "0.20.0",
-        "@skill-harness/adapters": "0.20.0"
+        "@skill-harness/core": "0.21.0",
+        "@skill-harness/adapters": "0.21.0"
     }
 }