@skill-harness/core 0.6.0 → 0.7.0

package/dist/spec-write.js

@@ -0,0 +1,106 @@
+import { createHash } from "node:crypto";
+import { readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import yaml from "js-yaml";
+import { parseSpec } from "./spec.js";
+/**
+ * The single choke point for appending a scenario to an existing
+ * `specification.yaml`.
+ *
+ * Two callers need this — `add-test` and capture promotion — and a second
+ * implementation is how they would drift into disagreeing about what a valid
+ * write is. Everything here is deliberately append-shaped: a spec is
+ * hand-authored and full of comments, and a round trip through
+ * `yaml.load`/`yaml.dump` would silently reformat it and drop every comment the
+ * author wrote. So the existing bytes are never re-serialized — the new block is
+ * concatenated onto them and the *result* is validated before anything is
+ * written.
+ */
+/** Thrown when the spec on disk moved between the caller reading it and writing. */
+export class ConcurrentSpecModification extends Error {
+    constructor(specPath) {
+        super(`${specPath} changed on disk since it was read — refusing to append. ` +
+            `Re-read the spec and retry; appending now would validate against a file that no longer exists.`);
+        this.name = "ConcurrentSpecModification";
+    }
+}
+/** Thrown when the scenario being appended collides with one already in the spec. */
+export class DuplicateScenarioId extends Error {
+    constructor(id, specPath) {
+        super(`scenario id \`${id}\` already exists in ${specPath}`);
+        this.name = "DuplicateScenarioId";
+    }
+}
+/** SHA-256 of spec text. Callers hold one across a read→confirm→write cycle. */
+export function specSha256(text) {
+    return createHash("sha256").update(text, "utf8").digest("hex");
+}
+/**
+ * Render one scenario as a YAML block that can be concatenated onto a spec.
+ *
+ * Dumps `{ scenarios: [scenario] }` and strips the top-level key, leaving the
+ * correctly-indented list item. Going through `yaml.dump` rather than string
+ * templating is what makes arbitrary user text — quotes, colons, newlines,
+ * leading dashes — safe to embed.
+ */
+export function renderScenarioBlock(scenario) {
+    const dumped = yaml.dump({ scenarios: [scenario] }, { lineWidth: -1, noRefs: true });
+    return "\n" + dumped.replace(/^scenarios:\n/, "");
+}
+/**
+ * Validate and atomically append a scenario.
+ *
+ * Order matters and is load-bearing: read → detect concurrent modification →
+ * reject duplicate id → build → **validate the merged text** → write. The
+ * validation is on the merged result, not the block alone, because a block that
+ * parses in isolation can still break the file it lands in.
+ *
+ * The write is temp-file-plus-rename rather than `appendFileSync`. An append
+ * interrupted partway through leaves a syntactically broken spec on disk; a
+ * rename either happened or did not.
+ */
+export function appendScenario(opts) {
+    const { specPath, scenario, baseSha256 } = opts;
+    const current = readFileSync(specPath, "utf8");
+    if (baseSha256 !== undefined && specSha256(current) !== baseSha256) {
+        throw new ConcurrentSpecModification(specPath);
+    }
+    const id = scenario.id;
+    if (typeof id !== "string" || id.trim() === "") {
+        throw new Error("scenario needs a non-empty string `id`");
+    }
+    const existing = parseSpec(current, specPath);
+    if (existing.scenarios.some((s) => s.id === id)) {
+        throw new DuplicateScenarioId(id, specPath);
+    }
+    const block = renderScenarioBlock(scenario);
+    const merged = current + block;
+    parseSpec(merged, specPath); // throws if the append broke the spec
+    atomicWrite(specPath, merged);
+    return { id, sha256: specSha256(merged), block };
+}
+/**
+ * Write via a sibling temp file and rename.
+ *
+ * Sibling, not `/tmp`: `rename(2)` is only atomic within a filesystem, and a
+ * cross-device rename would silently degrade to copy-then-delete — exactly the
+ * torn write this exists to prevent.
+ */
+function atomicWrite(path, text) {
+    const tmp = join(dirname(path), `.${Date.now()}-${process.pid}.specwrite.tmp`);
+    try {
+        writeFileSync(tmp, text, "utf8");
+        renameSync(tmp, path);
+    }
+    catch (err) {
+        try {
+            unlinkSync(tmp);
+        }
+        catch {
+            // Best effort: the original file is untouched either way, and masking the
+            // real failure with a cleanup error would hide why the write failed.
+        }
+        throw err;
+    }
+}
+//# sourceMappingURL=spec-write.js.map

package/dist/spec.d.ts

@@ -1,4 +1,5 @@
 import type { WorkspaceKind } from "./workspace.js";
+import { type TraceAssert } from "./trace-gates.js";
 export type ScenarioMode = "inline" | "seeded";
 export interface SeededAssert {
     vitest?: boolean;
@@ -28,9 +29,37 @@ export interface Scenario {
     checklist: string[];
     fixture?: string;
     assert?: SeededAssert;
+    /**
+     * Objective assertions over the execution trace.
+     *
+     * Deliberately NOT part of `SeededAssert`: the other gates read a staged git
+     * diff and are meaningless without a fixture, while a trace exists for any run.
+     * Declaring it opts the scenario into structured (`--mode json`) execution.
+     */
+    traceAssert?: TraceAssert;
     workspace: WorkspaceKind;
     remote: boolean;
     systemPromptFile?: string;
+    /**
+     * `env.extensions`: pi extension files to load, resolved relative to the spec dir.
+     *
+     * Loading is CLOSED, not additive — the adapter passes `--no-extensions` plus one
+     * `--extension` per entry, so exactly these load and nothing discovered does.
+     * (Measured on pi 0.83.0: that flag pair isolates even under `-a` project-local
+     * trust.) Without it, whatever the developer happened to have installed would
+     * silently become part of the test.
+     */
+    extensions?: string[];
+    /**
+     * `covers`: instruction sections this scenario is declared to exercise, e.g.
+     * `SKILL.md#core-principle`.
+     *
+     * METADATA. It stales nothing — see `sources.ts`, where it is deliberately in
+     * no digest. A `covers` edit changes which tests `--affected` selects, not what
+     * any past run measured, so charging a re-run for it would be the exact
+     * "pay tokens to fix a label" trap the facet split exists to remove.
+     */
+    covers?: string[];
     reps?: number;
     passThreshold?: number;
 }

package/dist/spec.js

@@ -1,5 +1,6 @@
 import { readFileSync } from "node:fs";
 import yaml from "js-yaml";
+import { parseTraceAssert } from "./trace-gates.js";
 /** Thrown on any validation failure. Message always carries the spec file path. */
 export class SpecError extends Error {
     constructor(message, file) {
@@ -54,6 +55,34 @@ function resolveWorkspace(env, mode, fixture, id, file) {
     }
     throw new SpecError(`scenario \`${id}\` env.workspace must be none | empty-git | fixture:<path>`, file);
 }
+/**
+ * Resolve `env.extensions` into a list of paths.
+ *
+ * Incompatible with `system_prompt_file` by construction: that flag REPLACES the
+ * system prompt to test a subagent definition in isolation, while an
+ * orchestration scenario tests the PARENT that delegates to one. Allowing both
+ * would silently test neither — the parent's instructions would be gone.
+ */
+function resolveExtensions(env, hasSystemPrompt, id, file) {
+    const raw = env && typeof env === "object" ? env.extensions : undefined;
+    if (raw === undefined)
+        return undefined;
+    if (!Array.isArray(raw) || raw.length === 0) {
+        throw new SpecError(`scenario \`${id}\` env.extensions must be a non-empty list of paths`, file);
+    }
+    const paths = raw.map((p, i) => {
+        if (typeof p !== "string" || p.trim() === "") {
+            throw new SpecError(`scenario \`${id}\` env.extensions[${i}] must be a non-empty path`, file);
+        }
+        return p.trim();
+    });
+    if (hasSystemPrompt) {
+        throw new SpecError(`scenario \`${id}\` sets both env.extensions and system_prompt_file — ` +
+            `system_prompt_file replaces the system prompt to test a subagent in isolation, ` +
+            `while env.extensions tests the parent that delegates to one. Pick one.`, file);
+    }
+    return paths;
+}
 /**
  * Resolve `env.remote`. A remote needs a repo to attach to, so it is only meaningful
  * with empty-git or a fixture — asking for one on a bare cwd is an authoring mistake,
@@ -144,6 +173,12 @@ export function parseSpec(text, file) {
             workspace: "none",
             remote: false,
         };
+        // `assert.trace` is legal for inline AND seeded scenarios — it reads the
+        // execution trace, which every run produces, not a staged diff.
+        const rawAssert = s.assert;
+        if (rawAssert?.trace !== undefined) {
+            scenario.traceAssert = parseTraceAssert(rawAssert.trace, `${file}: scenario \`${id}\``);
+        }
         if (mode === "seeded") {
             if (typeof s.fixture !== "string" || s.fixture.length === 0) {
                 throw new SpecError(`seeded scenario \`${id}\` requires a \`fixture\` path`, file);
@@ -208,6 +243,26 @@ export function parseSpec(text, file) {
             }
             scenario.systemPromptFile = s.system_prompt_file.trim();
         }
+        if (s.covers !== undefined) {
+            if (!isStringArray(s.covers) || s.covers.length === 0) {
+                throw new SpecError(`scenario \`${id}\` \`covers\` must be a non-empty list of strings`, file);
+            }
+            const bad = s.covers.find((c) => c.trim() === "");
+            if (bad !== undefined)
+                throw new SpecError(`scenario \`${id}\` \`covers\` has an empty entry`, file);
+            scenario.covers = s.covers.map((c) => c.trim());
+        }
+        // `unchanged_paths` is checked against the workspace's git state, so a scenario
+        // with no repo has nothing to observe. Refused here — free and offline — rather
+        // than at run time, because the alternative shipped for a while: the assertion
+        // silently passed against an empty change list and reported a green safety gate.
+        if (scenario.traceAssert?.unchanged_paths?.length && scenario.workspace === "none") {
+            throw new SpecError(`scenario \`${id}\` declares \`assert.trace.unchanged_paths\` but has no workspace to observe — ` +
+                `set \`env.workspace: empty-git\` or \`fixture:<path>\`, or drop the assertion. ` +
+                `A path policy with nothing to compare against would pass unconditionally.`, file);
+        }
+        // After system_prompt_file, so the incompatibility check sees the resolved value.
+        scenario.extensions = resolveExtensions(s.env, scenario.systemPromptFile !== undefined, id, file);
         if (s.reps !== undefined) {
             if (typeof s.reps !== "number" || !Number.isInteger(s.reps) || s.reps < 1) {
                 throw new SpecError(`scenario \`${id}\` \`reps\` must be a positive integer`, file);

package/dist/trace-gates.d.ts

@@ -0,0 +1,133 @@
+import type { ExecutionTraceV1 } from "./capture-trace-types.js";
+/**
+ * The objective gate layer: assertions evaluated against a saved execution
+ * trace, before any judge is asked anything.
+ *
+ * The DSL is deliberately tiny and entirely declarative — no expressions, no
+ * callbacks, no executable predicates. A spec is data that arrives from a
+ * repository; giving it a code path would make "add a test" and "run arbitrary
+ * code in CI" the same act. Everything here is a comparison between a value the
+ * trace recorded and a literal the spec wrote down.
+ *
+ * What these assertions can and cannot prove is a hard boundary, restated here
+ * because it is easy to over-claim: a trace proves **a registered tool was
+ * called with given arguments**. It proves nothing about what that tool then did
+ * to the machine. A `bash` command string is not a filesystem audit.
+ */
+export interface ArgPredicate {
+    equals?: unknown;
+    contains?: string;
+    starts_with?: string;
+    ends_with?: string;
+    matches?: string;
+    exists?: boolean;
+    /** For array-valued arguments: at least one element satisfies the inner predicate. */
+    any?: ArgPredicate;
+}
+export declare const PREDICATE_KEYS: readonly ["equals", "contains", "starts_with", "ends_with", "matches", "exists", "any"];
+export interface CountConstraint {
+    min?: number;
+    max?: number;
+}
+export interface RequireCall {
+    tool: string;
+    count?: CountConstraint;
+    args?: Record<string, ArgPredicate>;
+}
+export interface ForbidCall {
+    tool: string;
+    /** When present, only calls whose arguments match are forbidden. */
+    args?: Record<string, ArgPredicate>;
+}
+/**
+ * Convenience syntax for the orchestration case: "the parent delegated to
+ * `plan`, and the handoff carried X but not Y".
+ *
+ * Sugar over `require_calls`, not a second mechanism — it normalizes the known
+ * subagent argument shapes and then evaluates through the same path. There is
+ * deliberately no universal subagent extension assumed: an unknown extension can
+ * still be asserted on with plain `require_calls`, which is why this stays
+ * optional sugar rather than the only way in.
+ */
+export interface RequireSubagent {
+    /** The registered tool name — declared by the spec, since pi has no standard one. */
+    tool: string;
+    /** Which subagent the parent should have selected. */
+    agent: string;
+    count?: CountConstraint;
+    /** Substrings the handoff MUST carry. */
+    task_contains?: string[];
+    /** Substrings the handoff must NOT carry — the leak check. */
+    task_excludes?: string[];
+}
+export interface TraceAssert {
+    require_calls?: RequireCall[];
+    require_subagents?: RequireSubagent[];
+    forbid_calls?: ForbidCall[];
+    unchanged_paths?: string[];
+}
+/**
+ * Subagent invocations extracted from one tool call.
+ *
+ * A single call can carry several: `{tasks: [...]}` fans out and `{chain: [...]}`
+ * sequences. Normalizing to a flat list means a `count` constraint means the same
+ * thing — how many subagent invocations happened — whichever shape the extension
+ * uses to express them.
+ */
+export interface SubagentInvocation {
+    agent: string;
+    task: string;
+}
+/**
+ * Recognize the known subagent argument shapes.
+ *
+ * Three are supported because three exist in the wild; anything else yields an
+ * empty list, and the scenario should use plain `require_calls` instead. It
+ * deliberately does NOT guess: inventing an `agent` from an unrecognized shape
+ * would produce a confident assertion about a field nobody wrote.
+ */
+export declare function normalizeSubagentCall(args: Record<string, unknown>): SubagentInvocation[];
+/**
+ * ERROR is a third outcome, not a shade of FAIL: it means the assertion could
+ * not be evaluated because the evidence is absent. The two call for different
+ * fixes — a FAIL means change the skill, an ERROR means the harness could not
+ * look — and only one of them is a finding about the model.
+ */
+export type AssertionStatus = "PASS" | "FAIL" | "ERROR";
+export interface AssertionResult {
+    kind: "require_call" | "require_subagent" | "forbid_call" | "unchanged_path";
+    status: AssertionStatus;
+    detail: string;
+}
+export interface TraceGateResult {
+    status: AssertionStatus;
+    assertions: AssertionResult[];
+}
+/**
+ * Evaluate every assertion. All of them run even after the first failure — a
+ * scorecard that reports one problem per run makes the author re-run to find the
+ * second, and re-running is the expensive thing this whole layer exists to avoid.
+ *
+ * (One deliberate exception, marked inline in the `require_subagents` loop: the
+ * three sub-questions there are reported separately, and a later one is skipped
+ * when an earlier one already established there is nothing to ask it about.)
+ */
+export declare function evaluateTraceGates(assert: TraceAssert, trace: ExecutionTraceV1): TraceGateResult;
+export declare function testPredicate(value: unknown, p: ArgPredicate): boolean;
+/**
+ * Minimal glob over workspace-relative paths: `**` any depth, `*` one segment.
+ *
+ * Paths are normalized to forward slashes and stripped of a leading `./` first,
+ * so `./src/a.ts` and `src/a.ts` are the same path — otherwise an assertion
+ * would pass or fail on how the runner happened to spell it.
+ */
+export declare function matchesGlob(pattern: string, path: string): boolean;
+/**
+ * Validate an `assert.trace` block from a spec.
+ *
+ * Strict on purpose: an unknown key is an error, not something ignored. A
+ * silently-ignored `forbid_call` (singular, say) would read in review as a gate
+ * that is protecting something while asserting nothing at all — the worst
+ * possible failure for a safety check.
+ */
+export declare function parseTraceAssert(raw: unknown, ctx: string): TraceAssert;