@skill-harness/core 0.28.0 → 0.30.0

package/dist/sources.js

@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
 import { readFileSync, readdirSync } from "node:fs";
 import { isAbsolute, join, resolve } from "node:path";
 import yaml from "js-yaml";
+import { resolveJudgeReferences } from "./judge-references.js";
 import { PROMPT_NORMALIZATION_SOURCE_DIGEST, PROMPT_NORMALIZATION_SOURCE_KEY } from "./prompt-normalization.js";
 export { PROMPT_NORMALIZATION_SOURCE_DIGEST, PROMPT_NORMALIZATION_SOURCE_KEY } from "./prompt-normalization.js";
 /**
@@ -170,7 +171,7 @@ function walk(dir, prefix = "") {
  * transcripts are actually invalid.
  */
 function facets(s) {
-    const { id, title, critical, mode, turns, checklist, fixture, assert, traceAssert, workspace, remote, systemPromptFile, extensions, reps, passThreshold, covers: _coversIsMetadata, ...restScenario } = s;
+    const { id, title, critical, mode, turns, checklist, judgeContext, judgeEvidence, judgeReferences, fixture, assert, traceAssert, workspace, remote, systemPromptFile, extensions, tools, reps, passThreshold, covers: _coversIsMetadata, ...restScenario } = s;
     const _scenarioExhaustive = restScenario;
     void _scenarioExhaustive;
     // `covers` is destructured into a discard on purpose, and this comment is the
@@ -206,8 +207,15 @@ function facets(s) {
             id, mode, turns, workspace, remote, systemPromptFile ?? null,
             fixture ?? null, vitest ?? null, post_test ?? null,
             ...(extensions ? [extensions] : []),
+            ...(tools !== undefined ? [["tools", tools]] : []),
+        ]),
+        // Judge-only reference text changes grading, not what the subject received.
+        // Preserve historical hashes when the optional field is absent.
+        rubric: JSON.stringify([id, title, checklist,
+            ...(judgeContext !== undefined ? [["judge_context", judgeContext]] : []),
+            ...(judgeEvidence !== undefined ? [["judge_evidence", judgeEvidence]] : []),
+            ...(judgeReferences !== undefined ? [["judge_references", judgeReferences]] : []),
         ]),
-        rubric: JSON.stringify([id, title, checklist]),
         policy: JSON.stringify([id, critical, reps ?? null, passThreshold ?? null]),
         // Same rule as `stimulus` above: conditional, so a needle-gated scenario that
         // declares no trace assertions keeps the digest it was published with.
@@ -227,8 +235,15 @@ function sha(canonical) {
 export function stimulusDigest(s) {
     return sha(facets(s).stimulus);
 }
-export function rubricDigest(s) {
-    return sha(facets(s).rubric);
+export function rubricDigest(s, specDir, references) {
+    if (!s.judgeReferences?.length)
+        return sha(facets(s).rubric);
+    if (!references && !specDir)
+        throw new Error("A specification directory is required to digest selected judge references");
+    const selected = references ?? resolveJudgeReferences(s, specDir);
+    if (selected.length !== s.judgeReferences.length || selected.some((ref, index) => ref.file !== s.judgeReferences[index]))
+        throw new Error("Resolved judge references do not match the scenario selection");
+    return sha(JSON.stringify([facets(s).rubric, selected.map(({ file, sha256 }) => ({ file, sha256 }))]));
 }
 export function policyDigest(s) {
     return sha(facets(s).policy);
@@ -312,7 +327,12 @@ export function sourceHashes(ctx) {
         // single key could only ever name the most expensive one. The legacy
         // `scenario:<id>` key is deliberately NOT written any more — see scenarioDigest.
         hashes[STIMULUS_PREFIX + s.id] = stimulusDigest(s);
-        hashes[RUBRIC_PREFIX + s.id] = rubricDigest(s);
+        try {
+            hashes[RUBRIC_PREFIX + s.id] = rubricDigest(s, ctx.specDir, ctx.judgeReferences?.[s.id]);
+        }
+        catch {
+            hashes[RUBRIC_PREFIX + s.id] = UNREADABLE;
+        }
         hashes[POLICY_PREFIX + s.id] = policyDigest(s);
         const gates = gatesDigest(s);
         if (gates !== null)
@@ -384,7 +404,12 @@ export function currentHashFor(key, ctx) {
     // scenario is a reshape, not staleness, on every kind.
     const facetResolvers = [
         [STIMULUS_PREFIX, stimulusDigest],
-        [RUBRIC_PREFIX, rubricDigest],
+        [RUBRIC_PREFIX, s => { try {
+                return rubricDigest(s, ctx.specDir);
+            }
+            catch {
+                return null;
+            } }],
         [POLICY_PREFIX, policyDigest],
         [GATES_PREFIX, gatesDigest],
         [SCENARIO_PREFIX, scenarioDigest],

package/dist/spec.d.ts

@@ -31,6 +31,12 @@ export interface Scenario {
     mode: ScenarioMode;
     turns: string[];
     checklist: string[];
+    /** Explicit author-supplied reference text for the judge only; never a subject prompt or file path. */
+    judgeContext?: string;
+    /** Judge evidence is independent of whether a fixture workspace is used. */
+    judgeEvidence?: "response" | "code";
+    /** Explicit judge-only files, relative to the specification directory. */
+    judgeReferences?: string[];
     fixture?: string;
     assert?: ScenarioAssert;
     /**
@@ -54,6 +60,8 @@ export interface Scenario {
      * silently become part of the test.
      */
     extensions?: string[];
+    /** Explicit Pi tool filter; an empty list disables tools. */
+    tools?: string[];
     /**
      * `covers`: instruction sections this scenario is declared to exercise, e.g.
      * `SKILL.md#core-principle`.

package/dist/spec.js

@@ -191,6 +191,33 @@ export function parseSpec(text, file) {
             workspace: "none",
             remote: false,
         };
+        const envTools = s.env && typeof s.env === "object" ? s.env.tools : undefined;
+        if (envTools !== undefined) {
+            if (!Array.isArray(envTools) || envTools.some(tool => typeof tool !== "string" || !/^[A-Za-z][A-Za-z0-9_:/.-]*$/.test(tool)) || new Set(envTools).size !== envTools.length) {
+                throw new SpecError(`scenario \`${id}\` env.tools must be a list of unique Pi tool names`, file);
+            }
+            scenario.tools = envTools;
+        }
+        if (s.judge_context !== undefined) {
+            if (typeof s.judge_context !== "string" || s.judge_context.trim() === "" || Buffer.byteLength(s.judge_context, "utf8") > 65_536) {
+                throw new SpecError(`scenario \`${id}\` \`judge_context\` must be a non-empty string of at most 65536 UTF-8 bytes`, file);
+            }
+            scenario.judgeContext = s.judge_context;
+        }
+        if (s.judge_evidence !== undefined) {
+            if (s.judge_evidence !== "response" && s.judge_evidence !== "code") {
+                throw new SpecError(`scenario \`${id}\` \`judge_evidence\` must be response or code`, file);
+            }
+            scenario.judgeEvidence = s.judge_evidence;
+        }
+        if (s.judge_references !== undefined) {
+            assertStringList(s.judge_references, id, "judge_references", file);
+            if (s.judge_references.length > 16 || new Set(s.judge_references).size !== s.judge_references.length ||
+                s.judge_references.some(path => !path || /[\\\\:\x00-\x1f]/.test(path) || path.startsWith("/") || path.split("/").some(part => !part || part === "." || part === ".."))) {
+                throw new SpecError(`scenario \`${id}\` \`judge_references\` requires at most 16 unique relative POSIX file paths without traversal`, file);
+            }
+            scenario.judgeReferences = s.judge_references;
+        }
         // Output and trace assertions are legal for inline AND seeded scenarios.
         if (s.assert !== undefined && (s.assert === null || typeof s.assert !== "object" || Array.isArray(s.assert))) {
             throw new SpecError(`scenario \`${id}\` \`assert\` must be a mapping`, file);

package/dist/trace-gates.d.ts

@@ -33,6 +33,8 @@ export interface RequireCall {
     tool: string;
     count?: CountConstraint;
     args?: Record<string, ArgPredicate>;
+    /** Optional tool-reported result; omission preserves invocation-only matching. */
+    outcome?: "success" | "error";
 }
 export interface ForbidCall {
     tool: string;

package/dist/trace-gates.js

@@ -43,10 +43,15 @@ export function evaluateTraceGates(assert, trace) {
         assertions.push({ kind: "trace_evidence", status: "ERROR", detail: error });
     }
     for (const req of assert.require_calls ?? []) {
-        const matched = trace.tool_calls.filter((c) => c.name === req.tool && argsMatch(c, req.args));
+        const candidates = trace.tool_calls.filter((c) => c.name === req.tool && argsMatch(c, req.args));
+        if (req.outcome && candidates.some(c => !Number.isInteger(c.completionIndex) || c.completionIndex < 0 || typeof c.isError !== "boolean")) {
+            assertions.push({ kind: "require_call", status: "ERROR", detail: `cannot establish ${req.outcome} outcome for \`${req.tool}\`: completion evidence is missing` });
+            continue;
+        }
+        const matched = req.outcome ? candidates.filter(c => c.isError === (req.outcome === "error")) : candidates;
         const min = req.count?.min ?? 1;
         const max = req.count?.max;
-        const described = describeArgs(req.args);
+        const described = describeArgs(req.args) + (req.outcome ? ` with tool-reported ${req.outcome}` : "");
         if (matched.length < min) {
             assertions.push({
                 kind: "require_call",
@@ -354,8 +359,13 @@ export function parseTraceAssert(raw, ctx) {
                 req.count = parseCount(entry.count, `${ctx}: \`require_calls[${i}].count\``);
             if (entry.args !== undefined)
                 req.args = parseArgs(entry.args, `${ctx}: \`require_calls[${i}].args\``);
+            if (entry.outcome !== undefined) {
+                if (entry.outcome !== "success" && entry.outcome !== "error")
+                    throw new Error(`${ctx}: \`require_calls[${i}].outcome\` must be success or error`);
+                req.outcome = entry.outcome;
+            }
             for (const key of Object.keys(entry)) {
-                if (!["tool", "count", "args"].includes(key)) {
+                if (!["tool", "count", "args", "outcome"].includes(key)) {
                     throw new Error(`${ctx}: unknown key \`${key}\` in \`require_calls[${i}]\``);
                 }
             }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.28.0",
+    "version": "0.30.0",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",

package/schemas/results-v2.schema.json

@@ -235,6 +235,16 @@
       "properties": {
         "judgeFormat": { "enum": ["json", "prose"] },
         "judgeRetries": { "type": "integer", "const": 1 },
+        "input": {
+          "type": "object", "additionalProperties": false,
+          "required": ["artifact", "sha256", "execution"],
+          "properties": {
+            "artifact": { "type": "string", "pattern": "^judge-input\\.[a-f0-9]{64}\\.txt$" },
+            "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
+            "execution": { "enum": ["verified", "unknown"] },
+            "trace_sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }
+          }
+        },
         "ordinal": {
           "type": "integer",
           "minimum": 1,

package/schemas/results-v3.schema.json

@@ -247,6 +247,16 @@
       "properties": {
         "judgeFormat": { "enum": ["json", "prose"] },
         "judgeRetries": { "type": "integer", "const": 1 },
+        "input": {
+          "type": "object", "additionalProperties": false,
+          "required": ["artifact", "sha256", "execution"],
+          "properties": {
+            "artifact": { "type": "string", "pattern": "^judge-input\\.[a-f0-9]{64}\\.txt$" },
+            "sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
+            "execution": { "enum": ["verified", "unknown"] },
+            "trace_sha256": { "type": "string", "pattern": "^[a-f0-9]{64}$" }
+          }
+        },
         "ordinal": {
           "type": "integer",
           "minimum": 1,

package/schemas/specification-v1.schema.json

@@ -102,6 +102,22 @@
           },
           "minItems": 1
         },
+        "judge_context": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 65536,
+          "pattern": "\\S",
+          "description": "Explicit author-supplied reference text for the judge only. The runtime also enforces at most 65536 UTF-8 bytes. Not a file path; no files are loaded. Changes stale the rubric only."
+        },
+        "judge_evidence": {
+          "enum": ["response", "code"],
+          "description": "Evidence emphasis, independent of fixture workspace. Response tasks never receive code-diff priority guidance."
+        },
+        "judge_references": {
+          "type": "array", "minItems": 1, "maxItems": 16, "uniqueItems": true,
+          "items": { "type": "string", "minLength": 1 },
+          "description": "Explicit relative POSIX files below the specification directory, judge-only, collectively at most 65536 UTF-8 bytes. Runtime rejects traversal, escaping symlinks and binary data."
+        },
         "covers": {
           "type": "array",
           "items": {
@@ -133,6 +149,11 @@
               "items": {
                 "type": "string"
               }
+            },
+            "tools": {
+              "type": "array", "uniqueItems": true,
+              "items": { "type": "string", "pattern": "^[A-Za-z][A-Za-z0-9_:/.-]*$" },
+              "description": "Explicit Pi tool filter. Empty list disables all tools. Not an OS sandbox."
             }
           }
         },