@skill-harness/core 0.29.0 → 0.31.0

package/dist/sources.js

@@ -1,8 +1,10 @@
 import { createHash } from "node:crypto";
-import { readFileSync, readdirSync } from "node:fs";
-import { isAbsolute, join, resolve } from "node:path";
+import { readFileSync } from "node:fs";
+import { isAbsolute, resolve } from "node:path";
 import yaml from "js-yaml";
+import { resolveJudgeReferences } from "./judge-references.js";
 import { PROMPT_NORMALIZATION_SOURCE_DIGEST, PROMPT_NORMALIZATION_SOURCE_KEY } from "./prompt-normalization.js";
+import { fixtureInventory, hashFixtureFile } from "./util/fixture-inventory.js";
 export { PROMPT_NORMALIZATION_SOURCE_DIGEST, PROMPT_NORMALIZATION_SOURCE_KEY } from "./prompt-normalization.js";
 /**
  * What a run measured, as a map of `key -> sha256`, recorded in results.yaml so
@@ -113,42 +115,20 @@ export function fileSha256(path) {
  * normalised so a Linux-recorded hash still matches on Windows.
  */
 export function dirSha256(dir) {
-    let files;
     try {
-        files = walk(dir).sort();
-    }
-    catch {
-        return null;
-    }
-    const h = createHash("sha256");
-    for (const rel of files) {
-        h.update(rel);
-        h.update("\0");
-        try {
-            h.update(readFileSync(join(dir, rel)));
-        }
-        catch {
-            return null; // a file that vanished mid-walk makes the whole digest untrustworthy
+        const files = fixtureInventory(dir);
+        const h = createHash("sha256");
+        for (const file of files) {
+            h.update(file.relative);
+            h.update("\0");
+            hashFixtureFile(h, file);
+            h.update("\0");
         }
-        h.update("\0");
+        return h.digest("hex");
     }
-    return h.digest("hex");
-}
-/** Relative POSIX paths of every file under `dir`, recursively. Throws if `dir` is unreadable. */
-function walk(dir, prefix = "") {
-    const out = [];
-    for (const e of readdirSync(dir, { withFileTypes: true })) {
-        const rel = prefix ? `${prefix}/${e.name}` : e.name;
-        if (e.isDirectory()) {
-            out.push(...walk(join(dir, e.name), rel));
-        }
-        else if (e.isFile()) {
-            out.push(rel);
-        }
-        // symlinks/sockets/etc. are deliberately skipped: a fixture is plain files,
-        // and following links could hash something outside the fixture entirely.
+    catch {
+        return null; // Missing, oversized or changing input is unavailable evidence.
     }
-    return out;
 }
 /**
  * A scenario's semantic digest: everything that changes what the scenario
@@ -170,7 +150,7 @@ function walk(dir, prefix = "") {
  * transcripts are actually invalid.
  */
 function facets(s) {
-    const { id, title, critical, mode, turns, checklist, fixture, assert, traceAssert, workspace, remote, systemPromptFile, extensions, reps, passThreshold, covers: _coversIsMetadata, ...restScenario } = s;
+    const { id, title, critical, mode, turns, checklist, judgeContext, judgeEvidence, judgeReferences, fixture, assert, traceAssert, workspace, remote, systemPromptFile, extensions, tools, reps, passThreshold, covers: _coversIsMetadata, ...restScenario } = s;
     const _scenarioExhaustive = restScenario;
     void _scenarioExhaustive;
     // `covers` is destructured into a discard on purpose, and this comment is the
@@ -206,8 +186,15 @@ function facets(s) {
             id, mode, turns, workspace, remote, systemPromptFile ?? null,
             fixture ?? null, vitest ?? null, post_test ?? null,
             ...(extensions ? [extensions] : []),
+            ...(tools !== undefined ? [["tools", tools]] : []),
+        ]),
+        // Judge-only reference text changes grading, not what the subject received.
+        // Preserve historical hashes when the optional field is absent.
+        rubric: JSON.stringify([id, title, checklist,
+            ...(judgeContext !== undefined ? [["judge_context", judgeContext]] : []),
+            ...(judgeEvidence !== undefined ? [["judge_evidence", judgeEvidence]] : []),
+            ...(judgeReferences !== undefined ? [["judge_references", judgeReferences]] : []),
         ]),
-        rubric: JSON.stringify([id, title, checklist]),
         policy: JSON.stringify([id, critical, reps ?? null, passThreshold ?? null]),
         // Same rule as `stimulus` above: conditional, so a needle-gated scenario that
         // declares no trace assertions keeps the digest it was published with.
@@ -227,8 +214,15 @@ function sha(canonical) {
 export function stimulusDigest(s) {
     return sha(facets(s).stimulus);
 }
-export function rubricDigest(s) {
-    return sha(facets(s).rubric);
+export function rubricDigest(s, specDir, references) {
+    if (!s.judgeReferences?.length)
+        return sha(facets(s).rubric);
+    if (!references && !specDir)
+        throw new Error("A specification directory is required to digest selected judge references");
+    const selected = references ?? resolveJudgeReferences(s, specDir);
+    if (selected.length !== s.judgeReferences.length || selected.some((ref, index) => ref.file !== s.judgeReferences[index]))
+        throw new Error("Resolved judge references do not match the scenario selection");
+    return sha(JSON.stringify([facets(s).rubric, selected.map(({ file, sha256 }) => ({ file, sha256 }))]));
 }
 export function policyDigest(s) {
     return sha(facets(s).policy);
@@ -312,7 +306,12 @@ export function sourceHashes(ctx) {
         // single key could only ever name the most expensive one. The legacy
         // `scenario:<id>` key is deliberately NOT written any more — see scenarioDigest.
         hashes[STIMULUS_PREFIX + s.id] = stimulusDigest(s);
-        hashes[RUBRIC_PREFIX + s.id] = rubricDigest(s);
+        try {
+            hashes[RUBRIC_PREFIX + s.id] = rubricDigest(s, ctx.specDir, ctx.judgeReferences?.[s.id]);
+        }
+        catch {
+            hashes[RUBRIC_PREFIX + s.id] = UNREADABLE;
+        }
         hashes[POLICY_PREFIX + s.id] = policyDigest(s);
         const gates = gatesDigest(s);
         if (gates !== null)
@@ -384,7 +383,12 @@ export function currentHashFor(key, ctx) {
     // scenario is a reshape, not staleness, on every kind.
     const facetResolvers = [
         [STIMULUS_PREFIX, stimulusDigest],
-        [RUBRIC_PREFIX, rubricDigest],
+        [RUBRIC_PREFIX, s => { try {
+                return rubricDigest(s, ctx.specDir);
+            }
+            catch {
+                return null;
+            } }],
         [POLICY_PREFIX, policyDigest],
         [GATES_PREFIX, gatesDigest],
         [SCENARIO_PREFIX, scenarioDigest],

package/dist/spec.d.ts

@@ -31,6 +31,12 @@ export interface Scenario {
     mode: ScenarioMode;
     turns: string[];
     checklist: string[];
+    /** Explicit author-supplied reference text for the judge only; never a subject prompt or file path. */
+    judgeContext?: string;
+    /** Judge evidence is independent of whether a fixture workspace is used. */
+    judgeEvidence?: "response" | "code";
+    /** Explicit judge-only files, relative to the specification directory. */
+    judgeReferences?: string[];
     fixture?: string;
     assert?: ScenarioAssert;
     /**
@@ -54,6 +60,8 @@ export interface Scenario {
      * silently become part of the test.
      */
     extensions?: string[];
+    /** Explicit Pi tool filter; an empty list disables tools. */
+    tools?: string[];
     /**
      * `covers`: instruction sections this scenario is declared to exercise, e.g.
      * `SKILL.md#core-principle`.

package/dist/spec.js

@@ -191,6 +191,33 @@ export function parseSpec(text, file) {
             workspace: "none",
             remote: false,
         };
+        const envTools = s.env && typeof s.env === "object" ? s.env.tools : undefined;
+        if (envTools !== undefined) {
+            if (!Array.isArray(envTools) || envTools.some(tool => typeof tool !== "string" || !/^[A-Za-z][A-Za-z0-9_:/.-]*$/.test(tool)) || new Set(envTools).size !== envTools.length) {
+                throw new SpecError(`scenario \`${id}\` env.tools must be a list of unique Pi tool names`, file);
+            }
+            scenario.tools = envTools;
+        }
+        if (s.judge_context !== undefined) {
+            if (typeof s.judge_context !== "string" || s.judge_context.trim() === "" || Buffer.byteLength(s.judge_context, "utf8") > 65_536) {
+                throw new SpecError(`scenario \`${id}\` \`judge_context\` must be a non-empty string of at most 65536 UTF-8 bytes`, file);
+            }
+            scenario.judgeContext = s.judge_context;
+        }
+        if (s.judge_evidence !== undefined) {
+            if (s.judge_evidence !== "response" && s.judge_evidence !== "code") {
+                throw new SpecError(`scenario \`${id}\` \`judge_evidence\` must be response or code`, file);
+            }
+            scenario.judgeEvidence = s.judge_evidence;
+        }
+        if (s.judge_references !== undefined) {
+            assertStringList(s.judge_references, id, "judge_references", file);
+            if (s.judge_references.length > 16 || new Set(s.judge_references).size !== s.judge_references.length ||
+                s.judge_references.some(path => !path || /[\\\\:\x00-\x1f]/.test(path) || path.startsWith("/") || path.split("/").some(part => !part || part === "." || part === ".."))) {
+                throw new SpecError(`scenario \`${id}\` \`judge_references\` requires at most 16 unique relative POSIX file paths without traversal`, file);
+            }
+            scenario.judgeReferences = s.judge_references;
+        }
         // Output and trace assertions are legal for inline AND seeded scenarios.
         if (s.assert !== undefined && (s.assert === null || typeof s.assert !== "object" || Array.isArray(s.assert))) {
             throw new SpecError(`scenario \`${id}\` \`assert\` must be a mapping`, file);

package/dist/trace-gates.d.ts

@@ -33,6 +33,8 @@ export interface RequireCall {
     tool: string;
     count?: CountConstraint;
     args?: Record<string, ArgPredicate>;
+    /** Optional tool-reported result; omission preserves invocation-only matching. */
+    outcome?: "success" | "error";
 }
 export interface ForbidCall {
     tool: string;

package/dist/trace-gates.js

@@ -43,10 +43,15 @@ export function evaluateTraceGates(assert, trace) {
         assertions.push({ kind: "trace_evidence", status: "ERROR", detail: error });
     }
     for (const req of assert.require_calls ?? []) {
-        const matched = trace.tool_calls.filter((c) => c.name === req.tool && argsMatch(c, req.args));
+        const candidates = trace.tool_calls.filter((c) => c.name === req.tool && argsMatch(c, req.args));
+        if (req.outcome && candidates.some(c => !Number.isInteger(c.completionIndex) || c.completionIndex < 0 || typeof c.isError !== "boolean")) {
+            assertions.push({ kind: "require_call", status: "ERROR", detail: `cannot establish ${req.outcome} outcome for \`${req.tool}\`: completion evidence is missing` });
+            continue;
+        }
+        const matched = req.outcome ? candidates.filter(c => c.isError === (req.outcome === "error")) : candidates;
         const min = req.count?.min ?? 1;
         const max = req.count?.max;
-        const described = describeArgs(req.args);
+        const described = describeArgs(req.args) + (req.outcome ? ` with tool-reported ${req.outcome}` : "");
         if (matched.length < min) {
             assertions.push({
                 kind: "require_call",
@@ -354,8 +359,13 @@ export function parseTraceAssert(raw, ctx) {
                 req.count = parseCount(entry.count, `${ctx}: \`require_calls[${i}].count\``);
             if (entry.args !== undefined)
                 req.args = parseArgs(entry.args, `${ctx}: \`require_calls[${i}].args\``);
+            if (entry.outcome !== undefined) {
+                if (entry.outcome !== "success" && entry.outcome !== "error")
+                    throw new Error(`${ctx}: \`require_calls[${i}].outcome\` must be success or error`);
+                req.outcome = entry.outcome;
+            }
             for (const key of Object.keys(entry)) {
-                if (!["tool", "count", "args"].includes(key)) {
+                if (!["tool", "count", "args", "outcome"].includes(key)) {
                     throw new Error(`${ctx}: unknown key \`${key}\` in \`require_calls[${i}]\``);
                 }
             }

package/dist/util/exec.d.ts

@@ -1,12 +1,17 @@
+import { type ProcessObservation } from "./process.js";
 export interface ExecResult {
     stdout: string;
     stderr: string;
     code: number | null;
+    observation?: ProcessObservation;
 }
 export interface ExecOpts {
     cwd?: string;
     timeoutMs?: number;
     env?: NodeJS.ProcessEnv;
+    signal?: AbortSignal;
+    maxStdoutBytes?: number;
+    maxStderrBytes?: number;
 }
 /** Spawn a command, capture stdout/stderr. Never throws on non-zero exit; returns the code. */
 export declare function exec(cmd: string, args: string[], opts?: ExecOpts): Promise<ExecResult>;

package/dist/util/exec.js

@@ -1,38 +1,22 @@
-import { spawn } from "node:child_process";
 import { existsSync } from "node:fs";
 import { join, delimiter } from "node:path";
+import { BoundedCapture, ProcessOutputLimitError, runProcess } from "./process.js";
 /** Spawn a command, capture stdout/stderr. Never throws on non-zero exit; returns the code. */
-export function exec(cmd, args, opts = {}) {
-    return new Promise((resolve, reject) => {
-        // stdin must be ignored (not inherited): pi blocks reading stdin even in
-        // --print mode if its stdin is an open pipe/tty.
-        const child = spawn(cmd, args, {
-            cwd: opts.cwd,
-            env: opts.env ?? process.env,
-            stdio: ["ignore", "pipe", "pipe"],
-        });
-        let stdout = "";
-        let stderr = "";
-        let timer;
-        if (opts.timeoutMs) {
-            timer = setTimeout(() => {
-                child.kill("SIGKILL");
-                stderr += `\n[skill-harness] killed after ${opts.timeoutMs}ms timeout`;
-            }, opts.timeoutMs);
-        }
-        child.stdout.on("data", (d) => (stdout += d.toString()));
-        child.stderr.on("data", (d) => (stderr += d.toString()));
-        child.on("error", (e) => {
-            if (timer)
-                clearTimeout(timer);
-            reject(e);
-        });
-        child.on("close", (code) => {
-            if (timer)
-                clearTimeout(timer);
-            resolve({ stdout, stderr, code });
-        });
-    });
+export async function exec(cmd, args, opts = {}) {
+    const stdout = new BoundedCapture(opts.maxStdoutBytes ?? 8 * 1024 * 1024);
+    const stderr = new BoundedCapture(opts.maxStderrBytes ?? 128 * 1024);
+    const capture = (sink, name) => (chunk) => {
+        sink.push(chunk);
+        if (sink.truncated)
+            throw new ProcessOutputLimitError(`${name} capture exceeded ${sink.limit} bytes`);
+    };
+    const result = await runProcess(cmd, args, { ...opts,
+        onStdout: capture(stdout, "stdout"), onStderr: capture(stderr, "stderr") });
+    // A partial command output must never masquerade as a successful judge answer,
+    // git diff, or tool lookup. Ordinary non-zero exits remain the caller's decision.
+    if (result.error)
+        throw result.error;
+    return { stdout: stdout.text(), stderr: stderr.text(), code: result.code, observation: result.observation };
 }
 /** True if a binary is resolvable on PATH. */
 export function onPath(bin) {

package/dist/util/fixture-inventory.d.ts

@@ -0,0 +1,19 @@
+import type { Hash } from "node:crypto";
+export declare const FIXTURE_LIMITS: Readonly<{
+    entries: 4096;
+    depth: 32;
+    fileBytes: number;
+    totalBytes: number;
+}>;
+export declare class FixtureLimitError extends Error {
+    constructor(message: string);
+}
+export interface FixtureFile {
+    path: string;
+    relative: string;
+    size: number;
+}
+/** Bounded metadata walk of one explicitly selected tree. Never follows child symlinks. */
+export declare function fixtureInventory(root: string, skip?: (name: string, relative: string) => boolean): FixtureFile[];
+/** Update the existing hash format without allocating a whole file or accepting a growing file. */
+export declare function hashFixtureFile(hash: Hash, file: FixtureFile): void;

package/dist/util/fixture-inventory.js

@@ -0,0 +1,68 @@
+import { closeSync, constants, fstatSync, lstatSync, openSync, opendirSync, readSync } from "node:fs";
+import { join } from "node:path";
+export const FIXTURE_LIMITS = Object.freeze({ entries: 4096, depth: 32, fileBytes: 64 * 1024 * 1024, totalBytes: 256 * 1024 * 1024 });
+export class FixtureLimitError extends Error {
+    constructor(message) { super(`Selected fixture limit exceeded: ${message}`); this.name = "FixtureLimitError"; }
+}
+/** Bounded metadata walk of one explicitly selected tree. Never follows child symlinks. */
+export function fixtureInventory(root, skip) {
+    const files = [];
+    let entries = 0, total = 0;
+    const walk = (dir, prefix, depth) => {
+        if (depth > FIXTURE_LIMITS.depth)
+            throw new FixtureLimitError(`depth exceeds ${FIXTURE_LIMITS.depth}`);
+        const opened = opendirSync(dir);
+        try {
+            for (let entry = opened.readSync(); entry; entry = opened.readSync()) {
+                if (++entries > FIXTURE_LIMITS.entries)
+                    throw new FixtureLimitError(`more than ${FIXTURE_LIMITS.entries} entries`);
+                const relative = prefix ? `${prefix}/${entry.name}` : entry.name;
+                if (skip?.(entry.name, relative))
+                    continue;
+                const path = join(dir, entry.name);
+                if (entry.isDirectory())
+                    walk(path, relative, depth + 1);
+                else if (entry.isFile()) {
+                    const stat = lstatSync(path);
+                    if (!stat.isFile())
+                        throw new Error(`Selected fixture changed during inventory: ${relative}`);
+                    if (stat.size > FIXTURE_LIMITS.fileBytes)
+                        throw new FixtureLimitError(`${relative} exceeds ${FIXTURE_LIMITS.fileBytes} bytes`);
+                    total += stat.size;
+                    if (total > FIXTURE_LIMITS.totalBytes)
+                        throw new FixtureLimitError(`total exceeds ${FIXTURE_LIMITS.totalBytes} bytes`);
+                    files.push({ path, relative, size: stat.size });
+                }
+            }
+        }
+        finally {
+            opened.closeSync();
+        }
+    };
+    walk(root, "", 0);
+    return files.sort((a, b) => a.relative < b.relative ? -1 : a.relative > b.relative ? 1 : 0);
+}
+/** Update the existing hash format without allocating a whole file or accepting a growing file. */
+export function hashFixtureFile(hash, file) {
+    const fd = openSync(file.path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+    try {
+        const stat = fstatSync(fd);
+        if (!stat.isFile() || stat.size !== file.size || stat.size > FIXTURE_LIMITS.fileBytes)
+            throw new Error(`Selected fixture changed before hashing: ${file.relative}`);
+        const buffer = Buffer.allocUnsafe(64 * 1024);
+        let offset = 0;
+        while (offset < file.size) {
+            const count = readSync(fd, buffer, 0, Math.min(buffer.length, file.size - offset), offset);
+            if (!count)
+                throw new Error(`Selected fixture changed while hashing: ${file.relative}`);
+            hash.update(buffer.subarray(0, count));
+            offset += count;
+        }
+        if (readSync(fd, buffer, 0, 1, offset) !== 0 || fstatSync(fd).size !== file.size)
+            throw new Error(`Selected fixture changed while hashing: ${file.relative}`);
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+//# sourceMappingURL=fixture-inventory.js.map

package/dist/util/process.d.ts

@@ -0,0 +1,78 @@
+export interface ProcessObservation {
+    schema: 1;
+    platform: string;
+    scope: "owned-process-group" | "child-process";
+    pid: number | null;
+    reason: "running" | "exited" | "timeout" | "aborted" | "output-limit" | "resource-limit" | "spawn-error";
+    code: number | null;
+    durationMs: number;
+    stdoutBytes: number;
+    stderrBytes: number;
+    samples: number;
+    processesObserved: number;
+    peakRssBytes: number | null;
+    cpuUserTicks: number | null;
+    cpuSystemTicks: number | null;
+    readBytes: number | null;
+    writeBytes: number | null;
+    gaps: string[];
+    cleanup: {
+        termSent: boolean;
+        killSent: boolean;
+        groupGone: boolean | null;
+        pipesClosed: boolean;
+    };
+}
+export interface ProcessScope {
+    signal?: AbortSignal;
+    /** One metadata-only record on settlement; never receives argv, environment or output. */
+    onObservation?: (record: ProcessObservation) => void;
+    /** Latest active metadata, initially and at most every five seconds; never raw logs. */
+    onSample?: (record: ProcessObservation) => void;
+    /** Optional owned-tree sampled limits; these are not host quotas or sandbox boundaries. */
+    maxRssBytes?: number;
+    maxWriteBytes?: number;
+}
+/** Scope all subprocesses in an evaluation, including concurrent subject/judge calls. */
+export declare function withProcessScope<T>(scope: ProcessScope, work: () => T): T;
+export declare class ProcessControlError extends Error {
+    readonly reason: "aborted" | "output-limit" | "resource-limit";
+    constructor(reason: "aborted" | "output-limit" | "resource-limit", message: string);
+}
+export declare class ProcessOutputLimitError extends ProcessControlError {
+    constructor(message: string);
+}
+export declare function isProcessControlError(error: unknown): error is ProcessControlError;
+export declare function getProcessSignal(): AbortSignal | undefined;
+export declare function throwIfProcessAborted(): void;
+/** Retains at most `limit` bytes, including a single oversized chunk. */
+export declare class BoundedCapture {
+    readonly limit: number;
+    private chunks;
+    private tailUsed;
+    bytes: number;
+    truncated: boolean;
+    constructor(limit: number);
+    push(chunk: Buffer): void;
+    text(): string;
+}
+export interface ProcessOptions {
+    cwd?: string;
+    env?: NodeJS.ProcessEnv;
+    timeoutMs?: number;
+    signal?: AbortSignal;
+    /** Total stream traffic ceiling, independent of how much a consumer retains. */
+    maxOutputBytes?: number;
+    onStdout?: (chunk: Buffer) => void;
+    onStderr?: (chunk: Buffer) => void;
+    onStdoutEnd?: () => void;
+    /** Internal/test seam; not a model execution budget. */
+    killGraceMs?: number;
+}
+export interface ProcessResult {
+    code: number | null;
+    observation: ProcessObservation;
+    error: Error | null;
+}
+/** Shared owned-process lifecycle. Callers retain only their bounded stream representation. */
+export declare function runProcess(command: string, args: string[], options?: ProcessOptions): Promise<ProcessResult>;