@skill-harness/core 0.1.0

package/dist/spec.js

@@ -0,0 +1,175 @@
+import { readFileSync } from "node:fs";
+import yaml from "js-yaml";
+/** Thrown on any validation failure. Message always carries the spec file path. */
+export class SpecError extends Error {
+    constructor(message, file) {
+        super(`${file}: ${message}`);
+        this.name = "SpecError";
+    }
+}
+function isStringArray(v) {
+    return Array.isArray(v) && v.every((x) => typeof x === "string");
+}
+/**
+ * Require a non-empty list of strings, with a targeted error. A common authoring
+ * trap: an unquoted "key: value" list item parses as a YAML mapping, not a string
+ * — call that out explicitly so the fix (quote the item) is obvious.
+ */
+function assertStringList(v, id, field, file) {
+    if (!Array.isArray(v) || v.length === 0) {
+        throw new SpecError(`scenario \`${id}\` needs at least one \`${field}\` entry`, file);
+    }
+    const i = v.findIndex((x) => typeof x !== "string");
+    if (i >= 0) {
+        const bad = v[i];
+        const hint = bad !== null && typeof bad === "object"
+            ? ` — item #${i + 1} parsed as a YAML mapping; an unquoted ": " does that, so quote the item`
+            : ` — item #${i + 1} is not a string`;
+        throw new SpecError(`scenario \`${id}\` \`${field}\` items must all be strings${hint}`, file);
+    }
+}
+/** Resolve a scenario's `env.workspace` into a WorkspaceKind, applying defaults. */
+function resolveWorkspace(env, mode, fixture, id, file) {
+    const raw = env && typeof env === "object" ? env.workspace : undefined;
+    if (raw === undefined) {
+        // Default: a seeded scenario runs in its fixture repo; everything else is bare.
+        if (mode === "seeded" && fixture)
+            return { fixture };
+        return "none";
+    }
+    if (raw === "none") {
+        if (mode === "seeded") {
+            throw new SpecError(`seeded scenario \`${id}\` cannot use env.workspace: none — seeded gates need a git repo ` +
+                `(omit env to use its fixture, or use empty-git/fixture:<path>)`, file);
+        }
+        return raw;
+    }
+    if (raw === "empty-git")
+        return raw;
+    if (typeof raw === "string" && raw.startsWith("fixture:")) {
+        const p = raw.slice("fixture:".length).trim();
+        if (!p)
+            throw new SpecError(`scenario \`${id}\` env.workspace fixture path is empty`, file);
+        return { fixture: p };
+    }
+    throw new SpecError(`scenario \`${id}\` env.workspace must be none | empty-git | fixture:<path>`, file);
+}
+/** Parse + validate a specification.yaml from its raw text. `file` is used in error messages. */
+export function parseSpec(text, file) {
+    let doc;
+    try {
+        doc = yaml.load(text);
+    }
+    catch (e) {
+        throw new SpecError(`not valid YAML — ${e.message}`, file);
+    }
+    if (doc === null || typeof doc !== "object") {
+        throw new SpecError("spec must be a YAML mapping", file);
+    }
+    const o = doc;
+    if (typeof o.skill !== "string" || o.skill.length === 0) {
+        throw new SpecError("missing or invalid `skill` (string)", file);
+    }
+    if (typeof o.judge_persona !== "string" || o.judge_persona.length === 0) {
+        throw new SpecError("missing or invalid `judge_persona` (string)", file);
+    }
+    const sb = o.ship_bar;
+    if (!sb || typeof sb !== "object") {
+        throw new SpecError("missing `ship_bar` mapping", file);
+    }
+    if (typeof sb.total !== "number" || typeof sb.min_pass !== "number") {
+        throw new SpecError("`ship_bar` requires numeric `total` and `min_pass`", file);
+    }
+    const ship_bar = {
+        total: sb.total,
+        min_pass: sb.min_pass,
+        no_critical_fail: sb.no_critical_fail !== false, // default true
+    };
+    const critical = o.critical === undefined ? [] : o.critical;
+    if (!isStringArray(critical)) {
+        throw new SpecError("`critical` must be a list of scenario ids (strings)", file);
+    }
+    if (!Array.isArray(o.scenarios)) {
+        throw new SpecError("missing `scenarios` (list)", file);
+    }
+    const seen = new Set();
+    const scenarios = o.scenarios.map((raw, i) => {
+        if (raw === null || typeof raw !== "object") {
+            throw new SpecError(`scenario #${i + 1} is not a mapping`, file);
+        }
+        const s = raw;
+        const id = s.id;
+        if (typeof id !== "string" || id.length === 0) {
+            throw new SpecError(`scenario #${i + 1} missing \`id\` (string)`, file);
+        }
+        if (seen.has(id)) {
+            throw new SpecError(`duplicate scenario id \`${id}\``, file);
+        }
+        seen.add(id);
+        if (typeof s.title !== "string" || s.title.length === 0) {
+            throw new SpecError(`scenario \`${id}\` missing \`title\``, file);
+        }
+        const mode = s.mode === undefined ? "inline" : s.mode;
+        if (mode !== "inline" && mode !== "seeded") {
+            throw new SpecError(`scenario \`${id}\` has invalid \`mode\` (inline|seeded)`, file);
+        }
+        assertStringList(s.turns, id, "turns", file);
+        assertStringList(s.checklist, id, "checklist", file);
+        const critFlag = s.critical === true || critical.includes(id);
+        const scenario = {
+            id,
+            title: s.title,
+            critical: critFlag,
+            mode,
+            turns: s.turns,
+            checklist: s.checklist,
+            workspace: "none",
+        };
+        if (mode === "seeded") {
+            if (typeof s.fixture !== "string" || s.fixture.length === 0) {
+                throw new SpecError(`seeded scenario \`${id}\` requires a \`fixture\` path`, file);
+            }
+            scenario.fixture = s.fixture;
+            const a = s.assert;
+            if (a) {
+                const assertObj = {};
+                if (a.vitest !== undefined)
+                    assertObj.vitest = a.vitest === true;
+                if (a.diff_contains !== undefined) {
+                    if (!isStringArray(a.diff_contains)) {
+                        throw new SpecError(`seeded scenario \`${id}\` \`assert.diff_contains\` must be strings`, file);
+                    }
+                    assertObj.diff_contains = a.diff_contains;
+                }
+                scenario.assert = assertObj;
+            }
+        }
+        scenario.workspace = resolveWorkspace(s.env, mode, scenario.fixture, id, file);
+        if (s.reps !== undefined) {
+            if (typeof s.reps !== "number" || !Number.isInteger(s.reps) || s.reps < 1) {
+                throw new SpecError(`scenario \`${id}\` \`reps\` must be a positive integer`, file);
+            }
+            scenario.reps = s.reps;
+        }
+        if (s.pass_threshold !== undefined) {
+            if (typeof s.pass_threshold !== "number" || s.pass_threshold < 0 || s.pass_threshold > 1) {
+                throw new SpecError(`scenario \`${id}\` \`pass_threshold\` must be a number in [0, 1]`, file);
+            }
+            scenario.passThreshold = s.pass_threshold;
+        }
+        return scenario;
+    });
+    return { skill: o.skill, judge_persona: o.judge_persona, ship_bar, critical, scenarios };
+}
+/** Load + validate a specification.yaml from disk. */
+export function loadSpec(file) {
+    let text;
+    try {
+        text = readFileSync(file, "utf8");
+    }
+    catch (e) {
+        throw new SpecError(`cannot read spec file — ${e.message}`, file);
+    }
+    return parseSpec(text, file);
+}
+//# sourceMappingURL=spec.js.map

package/dist/trends.d.ts

@@ -0,0 +1,54 @@
+import { type ResultsFile } from "./results.js";
+import type { Verdict } from "./score.js";
+export interface TrendCell {
+    verdict: Verdict;
+    suspect: boolean;
+    flakiness?: number;
+}
+export interface TrendRun {
+    timestamp: string;
+    label: string | null;
+    grade: ResultsFile["effective_grade"];
+    cells: Record<string, TrendCell>;
+}
+export interface TrendModel {
+    model: string;
+    tag: string;
+    runs: TrendRun[];
+    truncated: boolean;
+    skipped: number;
+}
+export interface TrendData {
+    skill: string;
+    scenarios: {
+        id: string;
+        title: string;
+        critical: boolean;
+    }[];
+    models: TrendModel[];
+}
+/**
+ * Per model-tag, read the full run history (not just the latest) from
+ * <skillDir>/tests/results/, chronologically (timestamp-slug dir names sort
+ * correctly), keeping the most recent `limit` runs. Each run's cell carries the
+ * override-aware verdict + suspect (matching `effectiveVerdicts`'s canonical
+ * rule: an override resolves a misfire) + reps flakiness. Read-only; no
+ * absolute paths in the result.
+ *
+ * Only scored (mode === "green") runs are included in the history — a
+ * red/force run has no real grade (`effective_grade` is a "not scored"
+ * placeholder; see run.ts) and would otherwise plot as a misleading 0% dip in
+ * the sparkline/grid. Non-green runs are deliberately excluded, which is
+ * distinct from `skipped`: a run's mode can only be known after reading its
+ * results.yaml, so every candidate run-dir in the tag is read (not just the
+ * most recent `limit`) before filtering to green and applying the `limit`
+ * window — trends is a bounded, on-demand, local view, so this extra read
+ * cost is acceptable. If a tag has zero green runs, it's omitted entirely.
+ *
+ * A run whose `results.yaml` fails to parse (e.g. an interrupted non-atomic
+ * write) is logged via `console.warn` and skipped — never surfaced or thrown —
+ * and counted in that model's `skipped`. `truncated` reflects only the
+ * run-count cap on the green-run history (more green runs existed than
+ * `limit`), not parse-skips or mode-excluded runs.
+ */
+export declare function collectTrends(skillDir: string, limit?: number): TrendData;

package/dist/trends.js

@@ -0,0 +1,102 @@
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { loadSpec } from "./spec.js";
+import { readResults, effectiveVerdicts } from "./results.js";
+/** A directory that exists right now; false (never throws) if it vanished concurrently (e.g. ENOENT). */
+function isDir(p) {
+    try {
+        return statSync(p).isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Per model-tag, read the full run history (not just the latest) from
+ * <skillDir>/tests/results/, chronologically (timestamp-slug dir names sort
+ * correctly), keeping the most recent `limit` runs. Each run's cell carries the
+ * override-aware verdict + suspect (matching `effectiveVerdicts`'s canonical
+ * rule: an override resolves a misfire) + reps flakiness. Read-only; no
+ * absolute paths in the result.
+ *
+ * Only scored (mode === "green") runs are included in the history — a
+ * red/force run has no real grade (`effective_grade` is a "not scored"
+ * placeholder; see run.ts) and would otherwise plot as a misleading 0% dip in
+ * the sparkline/grid. Non-green runs are deliberately excluded, which is
+ * distinct from `skipped`: a run's mode can only be known after reading its
+ * results.yaml, so every candidate run-dir in the tag is read (not just the
+ * most recent `limit`) before filtering to green and applying the `limit`
+ * window — trends is a bounded, on-demand, local view, so this extra read
+ * cost is acceptable. If a tag has zero green runs, it's omitted entirely.
+ *
+ * A run whose `results.yaml` fails to parse (e.g. an interrupted non-atomic
+ * write) is logged via `console.warn` and skipped — never surfaced or thrown —
+ * and counted in that model's `skipped`. `truncated` reflects only the
+ * run-count cap on the green-run history (more green runs existed than
+ * `limit`), not parse-skips or mode-excluded runs.
+ */
+export function collectTrends(skillDir, limit = 20) {
+    const specPath = join(skillDir, "tests", "specification.yaml");
+    const spec = loadSpec(specPath);
+    const scenarios = spec.scenarios.map((s) => ({ id: s.id, title: s.title, critical: s.critical }));
+    const resultsRoot = join(skillDir, "tests", "results");
+    const models = [];
+    if (existsSync(resultsRoot)) {
+        const tags = readdirSync(resultsRoot)
+            .filter((n) => isDir(join(resultsRoot, n)))
+            .sort();
+        for (const tag of tags) {
+            const tagDir = join(resultsRoot, tag);
+            const runDirs = readdirSync(tagDir)
+                .map((n) => join(tagDir, n))
+                .filter((p) => isDir(p) && existsSync(join(p, "results.yaml")))
+                .sort(); // timestamp-slug dir names ⇒ chronological ascending
+            if (runDirs.length === 0)
+                continue;
+            // Read every candidate run (mode isn't knowable from the dir name) and
+            // filter to green (scored) runs before applying the `limit` window —
+            // filtering after the slice would let red/force runs consume window
+            // slots, undercounting the green history even when more exists.
+            const greenRuns = [];
+            let skipped = 0;
+            for (const rd of runDirs) {
+                let r;
+                try {
+                    r = readResults(rd);
+                }
+                catch (e) {
+                    // A corrupt/truncated results.yaml must not take down the whole
+                    // trends view — skip that run, but surface the failure.
+                    console.warn(`skill-harness trends: skipping unreadable run ${rd}: ${e instanceof Error ? e.message : e}`);
+                    skipped++;
+                    continue;
+                }
+                if (r.mode !== "green")
+                    continue; // not scored — deliberate exclusion, not a skip
+                greenRuns.push(r);
+            }
+            if (greenRuns.length === 0)
+                continue;
+            const truncated = greenRuns.length > limit;
+            const kept = greenRuns.slice(-limit); // most recent `limit`, newest last
+            const runs = [];
+            let model = "";
+            for (const r of kept) {
+                // effectiveVerdicts is the single source of truth for the
+                // override-aware verdict/suspect rule (suspect = s.suspect &&
+                // s.override == null — an override resolves the misfire); zip in
+                // flakiness from the matching ScenarioResult.
+                const verdicts = effectiveVerdicts(r.scenarios);
+                const cells = {};
+                r.scenarios.forEach((s, i) => {
+                    cells[s.id] = { verdict: verdicts[i].verdict, suspect: verdicts[i].suspect ?? false, flakiness: s.flakiness };
+                });
+                runs.push({ timestamp: r.timestamp, label: r.label, grade: r.effective_grade, cells });
+                model = r.model; // last successfully-read run (kept is ascending) wins
+            }
+            models.push({ model, tag, runs, truncated, skipped });
+        }
+    }
+    return { skill: spec.skill, scenarios, models };
+}
+//# sourceMappingURL=trends.js.map

package/dist/util/exec.d.ts

@@ -0,0 +1,14 @@
+export interface ExecResult {
+    stdout: string;
+    stderr: string;
+    code: number | null;
+}
+export interface ExecOpts {
+    cwd?: string;
+    timeoutMs?: number;
+    env?: NodeJS.ProcessEnv;
+}
+/** Spawn a command, capture stdout/stderr. Never throws on non-zero exit; returns the code. */
+export declare function exec(cmd: string, args: string[], opts?: ExecOpts): Promise<ExecResult>;
+/** True if a binary is resolvable on PATH. */
+export declare function onPath(bin: string): boolean;

package/dist/util/exec.js

@@ -0,0 +1,43 @@
+import { spawn } from "node:child_process";
+import { existsSync } from "node:fs";
+import { join, delimiter } from "node:path";
+/** Spawn a command, capture stdout/stderr. Never throws on non-zero exit; returns the code. */
+export function exec(cmd, args, opts = {}) {
+    return new Promise((resolve, reject) => {
+        // stdin must be ignored (not inherited): pi blocks reading stdin even in
+        // --print mode if its stdin is an open pipe/tty.
+        const child = spawn(cmd, args, {
+            cwd: opts.cwd,
+            env: opts.env ?? process.env,
+            stdio: ["ignore", "pipe", "pipe"],
+        });
+        let stdout = "";
+        let stderr = "";
+        let timer;
+        if (opts.timeoutMs) {
+            timer = setTimeout(() => {
+                child.kill("SIGKILL");
+                stderr += `\n[skill-harness] killed after ${opts.timeoutMs}ms timeout`;
+            }, opts.timeoutMs);
+        }
+        child.stdout.on("data", (d) => (stdout += d.toString()));
+        child.stderr.on("data", (d) => (stderr += d.toString()));
+        child.on("error", (e) => {
+            if (timer)
+                clearTimeout(timer);
+            reject(e);
+        });
+        child.on("close", (code) => {
+            if (timer)
+                clearTimeout(timer);
+            resolve({ stdout, stderr, code });
+        });
+    });
+}
+/** True if a binary is resolvable on PATH. */
+export function onPath(bin) {
+    const dirs = (process.env.PATH ?? "").split(delimiter);
+    const exts = process.platform === "win32" ? ["", ".exe", ".cmd", ".bat"] : [""];
+    return dirs.some((d) => d && exts.some((ext) => existsSync(join(d, bin + ext))));
+}
+//# sourceMappingURL=exec.js.map

package/dist/workspace.d.ts

@@ -0,0 +1,17 @@
+/** How a scenario's working directory is prepared. */
+export type WorkspaceKind = "none" | "empty-git" | {
+    fixture: string;
+};
+export interface Workspace {
+    cwd: string;
+    cleanup(): void;
+}
+/**
+ * Create an isolated temp-dir working directory for one scenario. `none` is an
+ * empty dir (no git); `empty-git` initialises a clean repo; `{ fixture }` copies
+ * the fixture (relative paths resolve against `specDir`) then initialises a repo
+ * with a baseline commit. Child processes run here, never in the user's home.
+ */
+export declare function createWorkspace(kind: WorkspaceKind, opts: {
+    specDir: string;
+}): Workspace;

package/dist/workspace.js

@@ -0,0 +1,42 @@
+import { cpSync, existsSync, mkdtempSync, rmSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { tmpdir } from "node:os";
+import { isAbsolute, join, resolve } from "node:path";
+const GIT_TIMEOUT_MS = 30_000;
+/** git init + a baseline commit, so a later `git diff --cached` shows only edits. */
+function gitBaseline(cwd) {
+    execFileSync("git", ["init", "-q"], { cwd, timeout: GIT_TIMEOUT_MS });
+    execFileSync("git", ["add", "-A"], { cwd, timeout: GIT_TIMEOUT_MS });
+    execFileSync("git", ["-c", "user.email=sc@local", "-c", "user.name=skill-check", "commit", "-q", "--allow-empty", "-m", "baseline"], { cwd, timeout: GIT_TIMEOUT_MS });
+}
+/**
+ * Create an isolated temp-dir working directory for one scenario. `none` is an
+ * empty dir (no git); `empty-git` initialises a clean repo; `{ fixture }` copies
+ * the fixture (relative paths resolve against `specDir`) then initialises a repo
+ * with a baseline commit. Child processes run here, never in the user's home.
+ */
+export function createWorkspace(kind, opts) {
+    const cwd = mkdtempSync(join(tmpdir(), "sc-ws-"));
+    const cleanup = () => rmSync(cwd, { recursive: true, force: true });
+    try {
+        if (kind === "none") {
+            // empty isolated dir; nothing to set up
+        }
+        else if (kind === "empty-git") {
+            gitBaseline(cwd);
+        }
+        else {
+            const src = isAbsolute(kind.fixture) ? kind.fixture : resolve(opts.specDir, kind.fixture);
+            if (!existsSync(src))
+                throw new Error(`fixture not found: ${src}`);
+            cpSync(src, cwd, { recursive: true });
+            gitBaseline(cwd);
+        }
+    }
+    catch (e) {
+        cleanup(); // never leak a temp dir on a setup failure
+        throw e;
+    }
+    return { cwd, cleanup };
+}
+//# sourceMappingURL=workspace.js.map

package/package.json

@@ -0,0 +1,17 @@
+{
+    "name": "@skill-harness/core",
+    "version": "0.1.0",
+    "type": "module",
+    "license": "MIT",
+    "main": "./dist/index.js",
+    "types": "./dist/index.d.ts",
+    "exports": { ".": "./dist/index.js" },
+    "files": ["dist/**/*.js", "dist/**/*.d.ts", "LICENSE"],
+    "repository": { "type": "git", "url": "git+https://github.com/mojomanyana/skill-harness.git" },
+    "publishConfig": { "access": "public" },
+    "engines": { "node": ">=20" },
+    "scripts": {
+        "prepack": "cp ../../LICENSE ./LICENSE"
+    },
+    "dependencies": { "js-yaml": "^4.1.0" }
+}