@skill-harness/core 0.1.2 → 0.2.1

package/dist/scaffold.js

@@ -0,0 +1,188 @@
+/** Marker written into an `init` template's first comment. Its presence tells
+ *  `suggest` the file is an unadopted template it may overwrite without --force. */
+export const TEMPLATE_SENTINEL = "skill-harness: generated template";
+/** Render a commented, empty-but-valid specification.yaml for a skill. */
+export function renderTemplateSpec(skillName) {
+    return `# ${TEMPLATE_SENTINEL} — \`suggest\` will overwrite this file while
+# this line is present; delete it once you start editing by hand.
+skill: ${skillName}
+
+# How the LLM judge should role-play when grading transcripts.
+judge_persona: >
+  a careful, fair reviewer.
+
+# The ship bar: what it takes to SHIP.
+#   total    = scenarios counted toward the bar
+#   min_pass = minimum passes required
+#   no_critical_fail = a critical-id fail blocks SHIP even if min_pass is met
+ship_bar:
+  total: 1
+  min_pass: 1
+  no_critical_fail: true
+
+# Scenario ids that block the ship if they fail (or set \`critical: true\` per scenario).
+critical: []
+
+scenarios:
+  # A* = baseline capability · B* = under-pressure / adversarial
+  - id: A1
+    title: describe what this scenario checks
+    # critical: true            # uncomment to gate the ship on this scenario
+    turns:
+      - "the user's first message"
+      # - "a follow-up message for a multi-turn scenario"
+    checklist:
+      - "an observable thing the response must do"
+`;
+}
+/** True if the text still carries the template sentinel (i.e. an unadopted template). */
+export function isTemplateSpec(text) {
+    return text.includes(TEMPLATE_SENTINEL);
+}
+/** Render a populated spec from an LLM draft. Strings are JSON-encoded (valid YAML
+ *  flow scalars) so colons/quotes never break the file. Carries no sentinel. */
+export function renderDraftSpec(skillName, draft) {
+    const scenarioBlocks = draft.scenarios
+        .map((s) => {
+        const turns = s.turns.map((t) => `      - ${JSON.stringify(t)}`).join("\n");
+        const checks = s.checklist.map((c) => `      - ${JSON.stringify(c)}`).join("\n");
+        return `  - id: ${s.id}\n    title: ${JSON.stringify(s.title)}\n    turns:\n${turns}\n    checklist:\n${checks}`;
+    })
+        .join("\n");
+    const proposed = draft.proposed_critical.length
+        ? `# proposed critical: [${draft.proposed_critical.join(", ")}] — move ids into \`critical: []\` below after review.`
+        : `# proposed critical: (none) — mark any ship-gating scenarios in \`critical: []\` below.`;
+    return `skill: ${skillName}
+
+# REVIEW: does this judge persona fit the skill? Edit freely.
+judge_persona: ${JSON.stringify(draft.judge_persona)}
+
+# REVIEW: tune the ship bar before your first run.
+ship_bar:
+  total: ${draft.ship_bar.total}
+  min_pass: ${draft.ship_bar.min_pass}
+  no_critical_fail: ${draft.ship_bar.no_critical_fail}
+
+${proposed}
+critical: []
+
+scenarios:
+${scenarioBlocks}
+`;
+}
+export function buildSuggestPrompt(skillName, skillMd) {
+    return `You are drafting a test specification for an agent skill named "${skillName}".
+Below is its SKILL.md. Propose scenarios that check whether an agent following this
+skill behaves correctly, including at least one adversarial / under-pressure case.
+
+Return ONLY a JSON object (no prose, no markdown fences) with exactly this shape:
+{
+  "judge_persona": "<how a judge should role-play when grading transcripts>",
+  "ship_bar": { "total": <int>, "min_pass": <int>, "no_critical_fail": true },
+  "proposed_critical": ["<scenario id you think should gate the ship>", ...],
+  "scenarios": [
+    { "id": "A1", "title": "<short title>",
+      "turns": ["<the user's message>", "<optional follow-up turns>"],
+      "checklist": ["<an observable thing the response must do>", ...] }
+  ]
+}
+Use ids A1, A2, ... for baseline scenarios and B1, B2, ... for adversarial ones.
+Every scenario needs at least one turn and one checklist item.
+
+--- SKILL.md ---
+${skillMd}`;
+}
+/** Ids are interpolated raw into YAML (see renderDraftSpec); restrict the character
+ *  set so a crafted id can never inject extra YAML keys (e.g. `critical: true`). */
+const SAFE_ID = /^[A-Za-z0-9_-]+$/;
+function asStringArray(v, ctx) {
+    if (!Array.isArray(v) || v.length === 0 || v.some((x) => typeof x !== "string")) {
+        throw new Error(`${ctx} must be a non-empty array of strings`);
+    }
+    return v;
+}
+/** Extract the first complete top-level JSON object from a model reply, tolerating
+ *  surrounding prose or ```json fences — including trailing text that itself
+ *  contains braces (e.g. "…} Does {this} work?"). Scans brace depth while skipping
+ *  string contents, so the object ends at its own matching `}`, not the last `}`
+ *  anywhere in the reply. */
+function extractJsonObject(raw) {
+    const start = raw.indexOf("{");
+    if (start < 0)
+        throw new Error("no JSON object in model output");
+    let depth = 0;
+    let inStr = false;
+    let escaped = false;
+    for (let i = start; i < raw.length; i++) {
+        const ch = raw[i];
+        if (inStr) {
+            if (escaped)
+                escaped = false;
+            else if (ch === "\\")
+                escaped = true;
+            else if (ch === '"')
+                inStr = false;
+            continue;
+        }
+        if (ch === '"')
+            inStr = true;
+        else if (ch === "{")
+            depth++;
+        else if (ch === "}" && --depth === 0)
+            return raw.slice(start, i + 1);
+    }
+    throw new Error("no complete JSON object in model output");
+}
+export function parseSuggestDraft(raw) {
+    let obj;
+    try {
+        obj = JSON.parse(extractJsonObject(raw));
+    }
+    catch (e) {
+        if (e instanceof Error && e.message.includes("JSON object in model output"))
+            throw e;
+        throw new Error(`model output is not valid JSON — ${e.message}`);
+    }
+    if (typeof obj.judge_persona !== "string" || !obj.judge_persona.trim()) {
+        throw new Error("judge_persona must be a non-empty string");
+    }
+    const sb = obj.ship_bar;
+    if (!sb || typeof sb.total !== "number" || typeof sb.min_pass !== "number") {
+        throw new Error("ship_bar must have numeric total and min_pass");
+    }
+    if (sb.min_pass > sb.total) {
+        throw new Error(`ship_bar.min_pass (${sb.min_pass}) cannot exceed total (${sb.total})`);
+    }
+    const proposed = Array.isArray(obj.proposed_critical)
+        ? obj.proposed_critical.filter((x) => typeof x === "string" && SAFE_ID.test(x))
+        : [];
+    if (!Array.isArray(obj.scenarios) || obj.scenarios.length === 0) {
+        throw new Error("scenarios must be a non-empty array");
+    }
+    const seen = new Set();
+    const scenarios = obj.scenarios.map((raw2, i) => {
+        const s = raw2;
+        if (typeof s.id !== "string" || !s.id.trim())
+            throw new Error(`scenario #${i + 1} needs a string id`);
+        if (!SAFE_ID.test(s.id))
+            throw new Error(`scenario id \`${s.id}\` must be alphanumeric (A-Z a-z 0-9 _ -)`);
+        if (seen.has(s.id))
+            throw new Error(`duplicate scenario id \`${s.id}\``);
+        seen.add(s.id);
+        if (typeof s.title !== "string" || !s.title.trim())
+            throw new Error(`scenario ${s.id} needs a title`);
+        return {
+            id: s.id,
+            title: s.title,
+            turns: asStringArray(s.turns, `scenario ${s.id} turns`),
+            checklist: asStringArray(s.checklist, `scenario ${s.id} checklist`),
+        };
+    });
+    return {
+        judge_persona: obj.judge_persona,
+        ship_bar: { total: sb.total, min_pass: sb.min_pass, no_critical_fail: sb.no_critical_fail !== false },
+        proposed_critical: proposed,
+        scenarios,
+    };
+}
+//# sourceMappingURL=scaffold.js.map

package/dist/score.d.ts

@@ -1,5 +1,9 @@
 import type { ShipBar } from "./spec.js";
-export type Verdict = "PASS" | "FAIL" | "ERROR";
+/**
+ * "JUDGE-AMBIGUOUS": the judge emitted conflicting verdicts for one transcript. It is
+ * never a pass and never silently resolved — it marks the run for a rejudge.
+ */
+export type Verdict = "PASS" | "FAIL" | "ERROR" | "JUDGE-AMBIGUOUS";
 export interface ScenarioVerdict {
     id: string;
     verdict: Verdict;

package/dist/seeded.js

@@ -1,5 +1,6 @@
 import { exec } from "./util/exec.js";
-const VITEST_TIMEOUT_MS = Number(process.env.SKILL_CHECK_VITEST_TIMEOUT_MS ?? 120_000);
+import { envNum } from "./util/env.js";
+const VITEST_TIMEOUT_MS = envNum("VITEST_TIMEOUT_MS", 120_000);
 /**
  * Run a seeded scenario inside a caller-prepared workspace: let the harness edit
  * the repo, then evaluate objective gates (staged-diff contains + optional vitest

package/dist/spec.d.ts

@@ -14,6 +14,8 @@ export interface Scenario {
     fixture?: string;
     assert?: SeededAssert;
     workspace: WorkspaceKind;
+    remote: boolean;
+    systemPromptFile?: string;
     reps?: number;
     passThreshold?: number;
 }

package/dist/spec.js

@@ -54,6 +54,24 @@ function resolveWorkspace(env, mode, fixture, id, file) {
     }
     throw new SpecError(`scenario \`${id}\` env.workspace must be none | empty-git | fixture:<path>`, file);
 }
+/**
+ * Resolve `env.remote`. A remote needs a repo to attach to, so it is only meaningful
+ * with empty-git or a fixture — asking for one on a bare cwd is an authoring mistake,
+ * not something to silently ignore.
+ */
+function resolveRemote(env, workspace, id, file) {
+    const raw = env && typeof env === "object" ? env.remote : undefined;
+    if (raw === undefined)
+        return false;
+    if (typeof raw !== "boolean") {
+        throw new SpecError(`scenario \`${id}\` env.remote must be true or false`, file);
+    }
+    if (raw && workspace === "none") {
+        throw new SpecError(`scenario \`${id}\` sets env.remote but has no repo to attach it to — ` +
+            `use env.workspace: empty-git or fixture:<path>`, file);
+    }
+    return raw;
+}
 /** Parse + validate a specification.yaml from its raw text. `file` is used in error messages. */
 export function parseSpec(text, file) {
     let doc;
@@ -124,6 +142,7 @@ export function parseSpec(text, file) {
             turns: s.turns,
             checklist: s.checklist,
             workspace: "none",
+            remote: false,
         };
         if (mode === "seeded") {
             if (typeof s.fixture !== "string" || s.fixture.length === 0) {
@@ -145,6 +164,19 @@ export function parseSpec(text, file) {
             }
         }
         scenario.workspace = resolveWorkspace(s.env, mode, scenario.fixture, id, file);
+        scenario.remote = resolveRemote(s.env, scenario.workspace, id, file);
+        if (s.system_prompt_file !== undefined) {
+            if (typeof s.system_prompt_file !== "string" || !s.system_prompt_file.trim()) {
+                throw new SpecError(`scenario \`${id}\` \`system_prompt_file\` must be a non-empty string`, file);
+            }
+            // A subagent has no turn two. Testing an agent file across multiple turns would
+            // measure conversation armor the single-shot contract deliberately drops.
+            if (scenario.turns.length !== 1) {
+                throw new SpecError(`scenario \`${id}\` uses system_prompt_file, so it must have exactly one turn ` +
+                    `(got ${scenario.turns.length}) — an agent definition is single-shot by contract`, file);
+            }
+            scenario.systemPromptFile = s.system_prompt_file.trim();
+        }
         if (s.reps !== undefined) {
             if (typeof s.reps !== "number" || !Number.isInteger(s.reps) || s.reps < 1) {
                 throw new SpecError(`scenario \`${id}\` \`reps\` must be a positive integer`, file);

package/dist/util/env.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Environment-variable resolution for the `SKILL_HARNESS_*` namespace.
+ *
+ * The tuning vars predate the skill-check → skill-harness rename and shipped in
+ * 0.1.x under a `SKILL_CHECK_` prefix. Anyone who set one of those in a shell
+ * profile or CI config must keep working, so every name is resolved new-first
+ * with a legacy fallback and a one-time notice pointing at the new spelling.
+ *
+ * Call these with the *suffix* only (`"PI_TIMEOUT_MS"`), never a full name — the
+ * prefixes are this module's business, which is what keeps ROADMAP rule 5 (no
+ * new `SKILL_CHECK_*` names) enforceable by grepping for the prefix.
+ */
+/** Test seam: clears the once-per-suffix warning memo. */
+export declare function __resetEnvWarnings(): void;
+/**
+ * Resolve `SKILL_HARNESS_<suffix>`, falling back to `SKILL_CHECK_<suffix>`.
+ *
+ * An empty value counts as unset: exporting a var as `""` is a common way to
+ * neutralize it in CI, and honoring it as "set" would make the legacy fallback
+ * unreachable for exactly those users.
+ */
+export declare function readEnv(suffix: string): string | undefined;
+/**
+ * Resolve a positive-integer var, e.g. a timeout in ms.
+ *
+ * A set-but-unparseable value warns and yields `fallback` rather than passing
+ * `NaN` down: the pre-rename `Number(process.env.X ?? default)` turned a typo
+ * into a NaN timeout, which silently disabled the timeout it was feeding.
+ */
+export declare function envNum(suffix: string, fallback: number): number;
+/** Resolve a boolean var: any non-empty value is on. */
+export declare function envFlag(suffix: string): boolean;

package/dist/util/env.js

@@ -0,0 +1,67 @@
+/**
+ * Environment-variable resolution for the `SKILL_HARNESS_*` namespace.
+ *
+ * The tuning vars predate the skill-check → skill-harness rename and shipped in
+ * 0.1.x under a `SKILL_CHECK_` prefix. Anyone who set one of those in a shell
+ * profile or CI config must keep working, so every name is resolved new-first
+ * with a legacy fallback and a one-time notice pointing at the new spelling.
+ *
+ * Call these with the *suffix* only (`"PI_TIMEOUT_MS"`), never a full name — the
+ * prefixes are this module's business, which is what keeps ROADMAP rule 5 (no
+ * new `SKILL_CHECK_*` names) enforceable by grepping for the prefix.
+ */
+const NEW_PREFIX = "SKILL_HARNESS_";
+const LEGACY_PREFIX = "SKILL_CHECK_";
+/** Suffixes already warned about, so a module-scope read doesn't nag per call. */
+const warned = new Set();
+/** Test seam: clears the once-per-suffix warning memo. */
+export function __resetEnvWarnings() {
+    warned.clear();
+}
+function warnOnce(key, message) {
+    if (warned.has(key))
+        return;
+    warned.add(key);
+    process.stderr.write(`skill-harness: ${message}\n`);
+}
+/**
+ * Resolve `SKILL_HARNESS_<suffix>`, falling back to `SKILL_CHECK_<suffix>`.
+ *
+ * An empty value counts as unset: exporting a var as `""` is a common way to
+ * neutralize it in CI, and honoring it as "set" would make the legacy fallback
+ * unreachable for exactly those users.
+ */
+export function readEnv(suffix) {
+    const fresh = process.env[NEW_PREFIX + suffix];
+    if (fresh)
+        return fresh;
+    const legacy = process.env[LEGACY_PREFIX + suffix];
+    if (legacy) {
+        warnOnce(`legacy:${suffix}`, `${LEGACY_PREFIX}${suffix} is the pre-rename name and still honored; rename it to ${NEW_PREFIX}${suffix}.`);
+        return legacy;
+    }
+    return undefined;
+}
+/**
+ * Resolve a positive-integer var, e.g. a timeout in ms.
+ *
+ * A set-but-unparseable value warns and yields `fallback` rather than passing
+ * `NaN` down: the pre-rename `Number(process.env.X ?? default)` turned a typo
+ * into a NaN timeout, which silently disabled the timeout it was feeding.
+ */
+export function envNum(suffix, fallback) {
+    const raw = readEnv(suffix);
+    if (raw === undefined)
+        return fallback;
+    const n = Number(raw);
+    if (!Number.isFinite(n) || n <= 0) {
+        warnOnce(`malformed:${suffix}`, `${NEW_PREFIX}${suffix}=${JSON.stringify(raw)} is not a positive number; using ${fallback}.`);
+        return fallback;
+    }
+    return n;
+}
+/** Resolve a boolean var: any non-empty value is on. */
+export function envFlag(suffix) {
+    return readEnv(suffix) !== undefined;
+}
+//# sourceMappingURL=env.js.map

package/dist/workspace.d.ts

@@ -10,8 +10,12 @@ export interface Workspace {
  * Create an isolated temp-dir working directory for one scenario. `none` is an
  * empty dir (no git); `empty-git` initialises a clean repo; `{ fixture }` copies
  * the fixture (relative paths resolve against `specDir`) then initialises a repo
- * with a baseline commit. Child processes run here, never in the user's home.
+ * with a baseline commit. A fixture may carry top-level `_staged/` and
+ * `_uncommitted/` subdirectories, applied after that commit to start the scenario
+ * from a dirty tree. `opts.remote` additionally wires a local bare `origin`.
+ * Child processes run here, never in the user's home.
  */
 export declare function createWorkspace(kind: WorkspaceKind, opts: {
     specDir: string;
+    remote?: boolean;
 }): Workspace;

package/dist/workspace.js

@@ -1,36 +1,112 @@
-import { cpSync, existsSync, mkdtempSync, rmSync } from "node:fs";
+import { cpSync, existsSync, mkdtempSync, readdirSync, rmSync } from "node:fs";
 import { execFileSync } from "node:child_process";
 import { tmpdir } from "node:os";
 import { isAbsolute, join, resolve } from "node:path";
 const GIT_TIMEOUT_MS = 30_000;
-/** git init + a baseline commit, so a later `git diff --cached` shows only edits. */
+/**
+ * Fixture subdirectories copied over the workspace AFTER the baseline commit, so their
+ * files land as pending changes rather than history. Lets a scenario start from a dirty
+ * tree — "I fixed the typo, commit it" needs the fix present but not yet committed, and
+ * "commit my staged changes" needs a populated index. `_staged/` contents are added to
+ * the index, `_uncommitted/` contents are left unstaged; a fixture may use either or
+ * both. Neither marker directory ever appears in the workspace.
+ */
+const UNCOMMITTED_DIR = "_uncommitted";
+const STAGED_DIR = "_staged";
+const MARKERS = [STAGED_DIR, UNCOMMITTED_DIR];
+/**
+ * A misspelled marker (`_uncommited/`) used to be copied into the baseline commit as a
+ * literal directory: the tree came up clean, the scenario silently measured the opposite
+ * of its intent, and nothing reported it. Any top-level `_name/` is therefore treated as
+ * a marker claim and must be a real one.
+ *
+ * Only a SINGLE leading underscore counts, so `__tests__` and `__pycache__` — ordinary
+ * directories a fixture may legitimately contain — are not marker claims. Nested
+ * `pkg/_staged/` is likewise ordinary content: markers are top-level only.
+ */
+function assertKnownMarkers(src) {
+    const suspects = readdirSync(src, { withFileTypes: true })
+        .filter((e) => e.isDirectory() && /^_[A-Za-z]/.test(e.name) && !MARKERS.includes(e.name))
+        .map((e) => e.name);
+    if (suspects.length > 0) {
+        throw new Error(`fixture ${src}: unknown marker director${suspects.length > 1 ? "ies" : "y"} ` +
+            `${suspects.map((s) => `\`${s}/\``).join(", ")} — known markers are ` +
+            `${MARKERS.map((m) => `\`${m}/\``).join(" and ")}. Rename it, or move it deeper if it is ordinary content.`);
+    }
+}
+/**
+ * git init + a baseline commit, so a later `git diff --cached` shows only edits.
+ * Pinned to `main`: the host's init.defaultBranch is not ours to depend on, and
+ * scenarios say things like "I'm on the main branch".
+ */
 function gitBaseline(cwd) {
-    execFileSync("git", ["init", "-q"], { cwd, timeout: GIT_TIMEOUT_MS });
+    execFileSync("git", ["init", "-q", "-b", "main"], { cwd, timeout: GIT_TIMEOUT_MS });
     execFileSync("git", ["add", "-A"], { cwd, timeout: GIT_TIMEOUT_MS });
     execFileSync("git", ["-c", "user.email=sh@local", "-c", "user.name=skill-harness", "commit", "-q", "--allow-empty", "-m", "baseline"], { cwd, timeout: GIT_TIMEOUT_MS });
 }
+/**
+ * Wire a bare repo in its own temp dir as `origin` and push the baseline, so `main`
+ * tracks `origin/main` and the baseline commit genuinely exists upstream.
+ *
+ * Without this, a git fixture has no remote — and a model reasonably reads a missing
+ * upstream as "solo throwaway repo", which is how git-ops A4 passes or fails depending
+ * on the run. A real remote makes "shared work" a fact of the fixture rather than a
+ * claim in the prompt, and needs no network.
+ */
+function addLocalRemote(cwd) {
+    const bare = mkdtempSync(join(tmpdir(), "sc-remote-")) + ".git";
+    execFileSync("git", ["init", "-q", "--bare", "-b", "main", bare], { timeout: GIT_TIMEOUT_MS });
+    execFileSync("git", ["remote", "add", "origin", bare], { cwd, timeout: GIT_TIMEOUT_MS });
+    execFileSync("git", ["push", "-q", "-u", "origin", "main"], { cwd, timeout: GIT_TIMEOUT_MS });
+    return bare;
+}
 /**
  * Create an isolated temp-dir working directory for one scenario. `none` is an
  * empty dir (no git); `empty-git` initialises a clean repo; `{ fixture }` copies
  * the fixture (relative paths resolve against `specDir`) then initialises a repo
- * with a baseline commit. Child processes run here, never in the user's home.
+ * with a baseline commit. A fixture may carry top-level `_staged/` and
+ * `_uncommitted/` subdirectories, applied after that commit to start the scenario
+ * from a dirty tree. `opts.remote` additionally wires a local bare `origin`.
+ * Child processes run here, never in the user's home.
  */
 export function createWorkspace(kind, opts) {
     const cwd = mkdtempSync(join(tmpdir(), "sc-ws-"));
-    const cleanup = () => rmSync(cwd, { recursive: true, force: true });
+    let bare = null;
+    const cleanup = () => {
+        rmSync(cwd, { recursive: true, force: true });
+        if (bare)
+            rmSync(bare, { recursive: true, force: true }); // the remote is ours too
+    };
     try {
         if (kind === "none") {
             // empty isolated dir; nothing to set up
         }
         else if (kind === "empty-git") {
             gitBaseline(cwd);
+            if (opts.remote)
+                bare = addLocalRemote(cwd);
         }
         else {
             const src = isAbsolute(kind.fixture) ? kind.fixture : resolve(opts.specDir, kind.fixture);
             if (!existsSync(src))
                 throw new Error(`fixture not found: ${src}`);
-            cpSync(src, cwd, { recursive: true });
+            assertKnownMarkers(src);
+            const pending = [STAGED_DIR, UNCOMMITTED_DIR].map((d) => join(src, d));
+            cpSync(src, cwd, {
+                recursive: true,
+                filter: (from) => !pending.includes(from), // committed baseline only
+            });
             gitBaseline(cwd);
+            // Before the pending changes land, so the remote holds the baseline only.
+            if (opts.remote)
+                bare = addLocalRemote(cwd);
+            const [staged, uncommitted] = pending;
+            if (existsSync(staged)) {
+                cpSync(staged, cwd, { recursive: true });
+                execFileSync("git", ["add", "-A"], { cwd, timeout: GIT_TIMEOUT_MS });
+            }
+            if (existsSync(uncommitted))
+                cpSync(uncommitted, cwd, { recursive: true });
         }
     }
     catch (e) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.1.2",
+    "version": "0.2.1",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",