@skill-harness/core 0.20.0 → 0.21.0

package/dist/discover.d.ts

@@ -1,8 +1,10 @@
 export interface DiscoveredSkill {
     name: string;
     dir: string;
+    testsDir: string;
     hasSpec: boolean;
     specPath: string;
+    specSource: "skills" | "specs" | "both" | null;
 }
 /**
  * Scan a skills root. A "skill" is any immediate subdirectory containing a
@@ -16,10 +18,10 @@ export interface DiscoveredSkill {
  * and a normal answer. The adapter refuses such a path too (see requireSkillDir),
  * but the honest fix is here, where the path is built.
  */
-export declare function discover(root: string): DiscoveredSkill[];
+export declare function discover(root: string, specsRoot?: string): DiscoveredSkill[];
 /**
  * Resolve a single skill by name; throws a helpful error if absent or specless.
  * A directory that exists but lacks a SKILL.md gets a specific error (rather than
  * the generic "no skill") so callers don't reimplement the SKILL.md existence check.
  */
-export declare function resolveSkill(root: string, name: string): DiscoveredSkill;
+export declare function resolveSkill(root: string, name: string, specsRoot?: string): DiscoveredSkill;

package/dist/discover.js

@@ -12,11 +12,15 @@ import { join, resolve } from "node:path";
  * and a normal answer. The adapter refuses such a path too (see requireSkillDir),
  * but the honest fix is here, where the path is built.
  */
-export function discover(root) {
+export function discover(root, specsRoot) {
     const absRoot = resolve(root);
     if (!existsSync(absRoot) || !statSync(absRoot).isDirectory()) {
         throw new Error(`skills root is not a directory: ${root}`);
     }
+    const absSpecsRoot = specsRoot ? resolve(specsRoot) : undefined;
+    if (absSpecsRoot && (!existsSync(absSpecsRoot) || !statSync(absSpecsRoot).isDirectory())) {
+        throw new Error(`specs root is not a directory: ${specsRoot}`);
+    }
     const skills = [];
     for (const name of readdirSync(absRoot)) {
         if (name.startsWith("."))
@@ -26,8 +30,22 @@ export function discover(root) {
             continue;
         if (!existsSync(join(dir, "SKILL.md")))
             continue;
-        const specPath = join(dir, "tests", "specification.yaml");
-        skills.push({ name, dir, hasSpec: existsSync(specPath), specPath });
+        const localSpecPath = join(dir, "tests", "specification.yaml");
+        const overlayTestsDir = absSpecsRoot ? join(absSpecsRoot, name, "tests") : join(dir, "tests");
+        const overlaySpecPath = join(overlayTestsDir, "specification.yaml");
+        const local = existsSync(localSpecPath);
+        const overlay = Boolean(absSpecsRoot && existsSync(overlaySpecPath));
+        const specSource = local && overlay ? "both" : overlay ? "specs" : local ? "skills" : null;
+        skills.push({
+            name,
+            dir,
+            // An explicit --specs is authoritative. The local path is inspected only so
+            // list can report it and duplicate specs can be refused rather than merged.
+            testsDir: overlayTestsDir,
+            hasSpec: absSpecsRoot ? overlay && !local : local,
+            specPath: overlaySpecPath,
+            specSource,
+        });
     }
     skills.sort((a, b) => a.name.localeCompare(b.name));
     return skills;
@@ -37,8 +55,8 @@ export function discover(root) {
  * A directory that exists but lacks a SKILL.md gets a specific error (rather than
  * the generic "no skill") so callers don't reimplement the SKILL.md existence check.
  */
-export function resolveSkill(root, name) {
-    const skill = discover(root).find((s) => s.name === name);
+export function resolveSkill(root, name, specsRoot) {
+    const skill = discover(root, specsRoot).find((s) => s.name === name);
     if (!skill) {
         const dir = join(resolve(root), name);
         if (existsSync(dir) && statSync(dir).isDirectory() && !existsSync(join(dir, "SKILL.md"))) {
@@ -46,6 +64,9 @@ export function resolveSkill(root, name) {
         }
         throw new Error(`no skill \`${name}\` under ${root}`);
     }
+    if (skill.specSource === "both") {
+        throw new Error(`skill \`${name}\` has a specification.yaml in both --skills and --specs; remove one (specs are not merged)`);
+    }
     return skill;
 }
 //# sourceMappingURL=discover.js.map

package/dist/instruction-coverage.d.ts

@@ -85,6 +85,8 @@ export interface CoverageOptions {
     scenarios: Scenario[];
     /** Instruction files to report on even if nothing references them. */
     baseFiles?: string[];
+    /** Resolve selected spec-relative references from another root (for split skill/spec repos). */
+    fileOverrides?: Record<string, string>;
 }
 /**
  * Build the coverage report.

package/dist/instruction-coverage.js

@@ -115,7 +115,7 @@ export function computeCoverage(opts) {
     const readSections = (file) => {
         if (fileSections.has(file))
             return fileSections.get(file);
-        const abs = isAbsolute(file) ? file : resolve(opts.specDir, file);
+        const abs = opts.fileOverrides?.[file] ?? (isAbsolute(file) ? file : resolve(opts.specDir, file));
         if (!existsSync(abs))
             return null;
         const sections = parseSections(readFileSync(abs, "utf8"));

package/dist/lint.d.ts

@@ -26,7 +26,7 @@ export declare function failsGate(f: LintFinding): boolean;
  * becomes a single `code:"spec"` finding. Returns ALL findings so the CLI can
  * report every problem across every skill.
  */
-export declare function lintSkill(skillDir: string): LintFinding[];
+export declare function lintSkill(skillDir: string, testsDir?: string): LintFinding[];
 /**
  * All committed run dirs under a skill's tests/results (<tag>/<timestamp>/results.yaml).
  * Empty if none. Never throws: unreadable/dangling entries (e.g. a broken symlink, or a

package/dist/lint.js

@@ -37,8 +37,8 @@ function isFile(p) {
  * becomes a single `code:"spec"` finding. Returns ALL findings so the CLI can
  * report every problem across every skill.
  */
-export function lintSkill(skillDir) {
-    const specPath = join(skillDir, "tests", "specification.yaml");
+export function lintSkill(skillDir, testsDir = join(skillDir, "tests")) {
+    const specPath = join(testsDir, "specification.yaml");
     const findings = [];
     let spec;
     try {
@@ -55,7 +55,8 @@ export function lintSkill(skillDir) {
     // kinds added after it shipped. Reported as a finding (so CI sees it) rather than
     // thrown, because refusing to lint is worse than linting with a caveat, and it is
     // one of the few ways this situation announces itself at all.
-    const stale = downgradeWarning(skillDir);
+    const resultsSkillDir = dirname(testsDir);
+    const stale = downgradeWarning(resultsSkillDir);
     if (stale) {
         findings.push({ skill, code: "consistency", message: stale.replace(/^warning: /, "") });
     }
@@ -152,7 +153,11 @@ export function lintSkill(skillDir) {
         // uncovered section is only reported by `coverage --strict`. Renaming a heading
         // is the usual cause, so the finding names the near-misses.
         if (s.covers?.length) {
-            const report = computeCoverage({ specDir, scenarios: [s] });
+            const report = computeCoverage({
+                specDir,
+                scenarios: [s],
+                fileOverrides: { "../SKILL.md": join(skillDir, "SKILL.md") },
+            });
             for (const b of report.broken) {
                 const hint = b.didYouMean.length ? ` — did you mean ${b.didYouMean.map((x) => `#${x}`).join(", ")}?` : "";
                 findings.push({
@@ -183,7 +188,7 @@ export function lintSkill(skillDir) {
     // malformed fields — e.g. `scenarios: null`) is caught and surfaces as a `consistency`
     // finding instead of throwing (lintSkill never throws) or being silently dropped (a
     // broken committed artifact must fail the gate, not pass it).
-    const resultsRoot = join(skillDir, "tests", "results");
+    const resultsRoot = join(testsDir, "results");
     for (const runDir of enumerateRunDirs(resultsRoot)) {
         try {
             const raw = yaml.load(readFileSync(resultsPath(runDir), "utf8"));
@@ -316,7 +321,7 @@ export function lintSkill(skillDir) {
     // is on disk. Wrapped: lintSkill must never throw, and a stability read touches every
     // run file in the tree.
     try {
-        for (const cell of boundaryCells(stabilityFrom(collectScoredRuns(skillDir), spec))) {
+        for (const cell of boundaryCells(stabilityFrom(collectScoredRuns(resultsSkillDir), spec))) {
             findings.push({
                 skill, scenario: cell.id, code: "stability", severity: "info",
                 message: `${cell.tag} mode=${cell.mode}: ${stabilityNote(cell)}`,

package/dist/regate.js

@@ -284,6 +284,7 @@ export async function regateRun(opts) {
         // Only the `gates:` keys of the scenarios actually re-evaluated. Stimulus, rubric
         // and policy were not re-decided here, so their hashes stay exactly as recorded.
         source_hashes: refreshGateHashes(prev.source_hashes, targets),
+        source_hash_roots: prev.source_hash_roots,
         scenarios,
     }, ctx);
     appendJournal(opts.runDir, {

package/dist/regrade.js

@@ -230,6 +230,7 @@ export async function regradeRun(opts) {
         // *stimulus* hashes stay" — see refreshRubricHashes.
         partial: prev?.partial,
         source_hashes: refreshRubricHashes(prev?.source_hashes, spec, targets),
+        source_hash_roots: prev?.source_hash_roots,
         scenarios: scenarioResults,
     }, ctx);
     const g = results.effective_grade;

package/dist/rescore.js

@@ -98,6 +98,7 @@ export function rescoreRun(opts) {
         // that is what makes `rescore` the honest remedy lint names for it. Stimulus,
         // rubric and gate hashes are untouched: none of them was re-evaluated here.
         source_hashes: refreshPolicyHashes(prev.source_hashes, opts.spec),
+        source_hash_roots: prev.source_hash_roots,
         scenarios,
     }, ctx);
     appendJournal(opts.runDir, {

package/dist/restamp.d.ts

@@ -63,6 +63,8 @@ export interface RestampReport {
 export interface RestampOptions {
     /** Git ref holding the bytes the runs measured, for a tree that has already been edited. */
     from?: string;
+    /** Tests/results location; defaults to <skillDir>/tests. */
+    testsDir?: string;
 }
 /** Upgrade every provable record under one skill's tests/results. */
 export declare function restampSkill(skillDir: string, opts?: RestampOptions): RestampReport;

package/dist/restamp.js

@@ -101,7 +101,8 @@ function writeAtomically(path, text) {
 /** Upgrade every provable record under one skill's tests/results. */
 export function restampSkill(skillDir, opts = {}) {
     const report = { runs: 0, upgraded: 0, unprovable: 0, unchanged: 0, partial: 0, added: [] };
-    const specPath = join(skillDir, "tests", "specification.yaml");
+    const testsDir = opts.testsDir ?? join(skillDir, "tests");
+    const specPath = join(testsDir, "specification.yaml");
     const spec = loadSpec(specPath);
     const specDir = dirname(specPath);
     // Only `system_prompt_file` paths, resolved from the CURRENT spec. The other
@@ -132,7 +133,7 @@ export function restampSkill(skillDir, opts = {}) {
             currentPrompt,
         });
     }
-    for (const runDir of enumerateRunDirs(join(skillDir, "tests", "results"))) {
+    for (const runDir of enumerateRunDirs(join(testsDir, "results"))) {
         report.runs++;
         let doc;
         try {

package/dist/results.d.ts

@@ -238,6 +238,8 @@ export interface ResultsFile {
      * simply don't carry it, and are never retroactively flagged.
      */
     source_hashes?: Record<string, string>;
+    /** For split-root runs, identifies whether each hash came from --skills or --specs. */
+    source_hash_roots?: Record<string, "skills" | "specs">;
     effective_grade: GradeSummary;
     scenarios: ScenarioResult[];
     /** Adapter-computed model-visible prompt observations. Required for schema v3. */

package/dist/results.js

@@ -212,6 +212,7 @@ export function finalizeResults(draft, ctx) {
         ...(draft.arm ? { arm: draft.arm } : {}),
         ...(draft.partial ? { partial: true } : {}),
         ...(draft.source_hashes ? { source_hashes: draft.source_hashes } : {}),
+        ...(draft.source_hash_roots ? { source_hash_roots: draft.source_hash_roots } : {}),
         effective_grade,
         scenarios: draft.scenarios,
         ...(draft.subject_invocations ? { subject_invocations: draft.subject_invocations } : {}),

package/dist/run.d.ts

@@ -8,6 +8,10 @@ export interface RunOptions {
     spec: Spec;
     skillDir: string;
     specPath: string;
+    /** Tests/results location; defaults to <skillDir>/tests for the single-root layout. */
+    testsDir?: string;
+    /** Record per-hash skills/specs provenance for a split-root run. */
+    recordSourceRoots?: boolean;
     adapter: HarnessAdapter;
     model: ModelRef;
     modelToken: string;

package/dist/run.js

@@ -1,6 +1,6 @@
 import { mkdirSync, writeFileSync, readFileSync } from "node:fs";
 import { dirname, join, resolve } from "node:path";
-import { sourceHashes } from "./sources.js";
+import { sourceHashes, sourceHashRoots } from "./sources.js";
 import { judgeResemblesSubject } from "./grade.js";
 import { NONE_ARM, seedArmDefinitions } from "./arms.js";
 import { runDirFor, transcriptPath, diffPath, tracePath, writeResults, ensureResultsGitignore, scoreContextFor, effectiveThreshold, isScoredMode, } from "./results.js";
@@ -68,7 +68,7 @@ export async function runSkillModel(opts) {
             `(${model.provider}:${model.model}) — verdicts may be inflated. Use a distinct judge.`);
     }
     const arm = opts.arm ?? NONE_ARM;
-    const runDir = runDirFor(skillDir, adapter.name, model, timestamp, arm.name);
+    const runDir = runDirFor(dirname(opts.testsDir ?? join(skillDir, "tests")), adapter.name, model, timestamp, arm.name);
     mkdirSync(runDir, { recursive: true });
     ensureResultsGitignore(dirname(dirname(runDir))); // .../tests/results/.gitignore
     // Which harness CLI delivered the skill, asked once per run and recorded with the
@@ -145,6 +145,7 @@ export async function runSkillModel(opts) {
         return outcomesToResult(scenario.id, grouped[si], repCounts[si], threshold);
     });
     const ctx = scoreContextFor({ mode, partial }, spec);
+    const hashes = sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios, judgePersona: spec.judge_persona });
     const results = writeResults(runDir, {
         skill: spec.skill,
         harness: adapter.name,
@@ -158,7 +159,8 @@ export async function runSkillModel(opts) {
         ...(partial ? { partial: true } : {}),
         // Only the scenarios this run actually measured: a --only run must not claim
         // coverage of scenarios it skipped.
-        source_hashes: sourceHashes({ skillDir, specDir: dirname(opts.specPath), scenarios, judgePersona: spec.judge_persona }),
+        source_hashes: hashes,
+        ...(opts.recordSourceRoots ? { source_hash_roots: sourceHashRoots(hashes) } : {}),
         scenarios: scenarioResults,
         ...(arm.name === NONE_ARM.name ? {} : {
             arm: {

package/dist/sources.d.ts

@@ -151,6 +151,8 @@ export interface SourceContext {
  * problem to report, not run's to crash on.
  */
 export declare function sourceHashes(ctx: SourceContext): Record<string, string>;
+/** Per-hash root provenance for split --skills/--specs runs. */
+export declare function sourceHashRoots(hashes: Record<string, string>): Record<string, "skills" | "specs">;
 /**
  * The current hash for a recorded key, or null when the source is gone.
  *

package/dist/sources.js

@@ -348,6 +348,10 @@ export function sourceHashes(ctx) {
     }
     return hashes;
 }
+/** Per-hash root provenance for split --skills/--specs runs. */
+export function sourceHashRoots(hashes) {
+    return Object.fromEntries(Object.keys(hashes).map((key) => [key, key === SKILL_KEY || key === SKILL_PROMPT_KEY ? "skills" : "specs"]));
+}
 /**
  * The current hash for a recorded key, or null when the source is gone.
  *

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.20.0",
+    "version": "0.21.0",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",

package/schemas/results-v2.schema.json

@@ -80,6 +80,12 @@
         "type": "string"
       }
     },
+    "source_hash_roots": {
+      "type": "object",
+      "additionalProperties": {
+        "enum": ["skills", "specs"]
+      }
+    },
     "effective_grade": {
       "$ref": "#/$defs/grade"
     },

package/schemas/results-v3.schema.json

@@ -81,6 +81,12 @@
         "type": "string"
       }
     },
+    "source_hash_roots": {
+      "type": "object",
+      "additionalProperties": {
+        "enum": ["skills", "specs"]
+      }
+    },
     "effective_grade": {
       "$ref": "#/$defs/grade"
     },