@skill-harness/core 0.29.0 → 0.31.0

package/dist/judge-evidence.js

@@ -0,0 +1,120 @@
+import { existsSync, readFileSync } from "node:fs";
+import { EXECUTION_TRACE_VERSION } from "./capture-trace-types.js";
+import { deserializeTrace, mergeTraces, traceSha256 } from "./execution-trace.js";
+import { redactText } from "./redaction.js";
+import { finalAssistantMessage } from "./seeded.js";
+const HASH = /^[a-f0-9]{64}$/;
+const MAX_CALLS = 32;
+const object = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
+const index = (value) => Number.isSafeInteger(value) && value >= 0;
+/** Validate the fields consumed by merging and the evidence projection before using them. */
+function validTrace(trace) {
+    if (!object(trace) || trace.trace_version !== EXECUTION_TRACE_VERSION || !object(trace.subject) || typeof trace.subject.provider !== "string" || typeof trace.subject.model !== "string" ||
+        !(trace.pi_version === null || typeof trace.pi_version === "string") || typeof trace.final_text !== "string" ||
+        !index(trace.turn) || !index(trace.rep) || !Array.isArray(trace.tool_calls) ||
+        !(trace.changed_paths === null || Array.isArray(trace.changed_paths) && trace.changed_paths.every(p => typeof p === "string")) ||
+        !(trace.cost_usd === null || typeof trace.cost_usd === "number" && Number.isFinite(trace.cost_usd)) ||
+        !(trace.capture_errors === undefined || Array.isArray(trace.capture_errors) && trace.capture_errors.every(e => typeof e === "string")))
+        return false;
+    if (trace.metrics !== undefined && (!object(trace.metrics) ||
+        ["input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens", "cost_usd"].some(key => {
+            const value = trace.metrics[key];
+            return value !== null && !(typeof value === "number" && Number.isFinite(value) && value >= 0);
+        }) || [trace.metrics.tool_calls, trace.metrics.delegated_children, trace.metrics.max_concurrency].some(value => !index(value))))
+        return false;
+    const ids = new Set();
+    const completions = new Set();
+    for (const [position, call] of trace.tool_calls.entries()) {
+        if (!object(call) || typeof call.id !== "string" || !call.id || ids.has(call.id) ||
+            typeof call.name !== "string" || !call.name || !object(call.args) || call.issueIndex !== position ||
+            !(call.completionIndex === -1 || index(call.completionIndex)) || typeof call.isError !== "boolean" ||
+            !object(call.result) || !index(call.result.bytes) || typeof call.result.sha256 !== "string" || !HASH.test(call.result.sha256))
+            return false;
+        ids.add(call.id);
+        if (call.completionIndex >= 0) {
+            if (completions.has(call.completionIndex))
+                return false;
+            completions.add(call.completionIndex);
+        }
+    }
+    return [...completions].sort((a, b) => a - b).every((value, position) => value === position);
+}
+export function unavailableExecution(reason = "No bound execution trace was supplied.", status = "unknown") {
+    return { schema: 1, status, coverage: "unknown", reason, observedCalls: null, omittedCalls: 0, calls: [] };
+}
+/** Tool inputs are untrusted labels. Keep only bounded local read paths, never commands, URLs, tasks or results. */
+function readPath(call) {
+    if (call.name !== "read")
+        return undefined;
+    const path = call.args.path ?? call.args.file_path;
+    if (typeof path !== "string")
+        return undefined;
+    if (path.length > 256 || /[\x00-\x1f\x7f?#]/.test(path) || /^[a-z][a-z0-9+.-]*:\/\//i.test(path))
+        return "[path withheld]";
+    return redactText(path);
+}
+/** Digest binding establishes artifact continuity, not a signature or proof of file contents. */
+export function executionFacts(traces, identity, binding) {
+    if (binding.source === "saved" && !binding.expectedSha256)
+        return unavailableExecution("Legacy trace has no recorded digest binding; execution is unverified.");
+    const invalid = (reason) => unavailableExecution(reason, "invalid");
+    if (!traces.length)
+        return binding.source === "live" ? unavailableExecution() : invalid("Recorded execution trace is missing or unreadable.");
+    if (traces.some(trace => !validTrace(trace)))
+        return invalid("Execution trace has malformed structure.");
+    if (traces.some((trace, turn) => trace.scenario_id !== identity.scenarioId || trace.mode !== identity.mode || trace.rep !== identity.repetition || trace.turn !== turn ||
+        identity.subject && (trace.subject.provider !== identity.subject.provider || trace.subject.model !== identity.subject.model) ||
+        identity.piVersion !== undefined && trace.pi_version !== identity.piVersion ||
+        trace.subject.provider !== traces[0].subject.provider || trace.subject.model !== traces[0].subject.model || trace.pi_version !== traces[0].pi_version)) {
+        return invalid("Execution trace identity or turn sequence does not match this repetition.");
+    }
+    if (identity.responses !== undefined && (identity.responses === null || identity.responses.length !== traces.length || traces.some((trace, turn) => trace.final_text.trim() !== identity.responses[turn]))) {
+        return invalid("Execution trace final answers do not match the judged transcript.");
+    }
+    if (traces.some(trace => trace.trace_sha256 !== undefined && trace.trace_sha256 !== traceSha256(trace)))
+        return invalid("Execution trace has a mismatching turn digest.");
+    // Use the existing canonical merge/digest; only the projection below reaches the judge.
+    const merged = mergeTraces(traces);
+    const digest = merged?.trace_sha256;
+    if (!merged || !digest)
+        return invalid("Execution trace could not be merged.");
+    if (binding.source === "saved" && binding.expectedSha256 !== digest)
+        return invalid("Saved trace no longer matches the digest recorded by the run.");
+    const complete = traces.length === identity.turns && traces.every(trace => trace.final_status === "complete" && !trace.capture_errors?.length && trace.tool_calls.every(call => call.completionIndex >= 0));
+    const calls = merged.tool_calls.slice(0, MAX_CALLS).map(call => ({
+        tool: /^[A-Za-z0-9_.:/-]{1,80}$/.test(call.name) ? call.name : "[tool name withheld]",
+        state: call.completionIndex < 0 ? "unfinished" : call.isError ? "error" : "completed",
+        ...(readPath(call) === undefined ? {} : { path: readPath(call) }),
+        ...(call.completionIndex < 0 ? {} : { resultBytes: call.result.bytes, resultSha256: call.result.sha256 }),
+    }));
+    const omittedCalls = merged.tool_calls.length - calls.length;
+    return {
+        schema: 1, status: "verified", coverage: complete && !omittedCalls ? "complete" : "partial",
+        reason: complete ? omittedCalls ? "Call list is bounded; omitted calls may include relevant actions." : "All expected turns settled; all recorded calls are listed." : "Turn settlement or capture coverage is incomplete; do not infer that unlisted actions did not occur.",
+        traceSha256: digest, observedCalls: merged.tool_calls.length, omittedCalls, calls,
+    };
+}
+/** Reuse the transcript ambiguity check, then compare every assistant turn, excluding harness-owned trailers. */
+export function transcriptResponses(transcript, scenario) {
+    if (finalAssistantMessage(transcript, scenario) === null)
+        return null;
+    return transcript.split(/^<<< ASSISTANT:[ \t]*$/m).slice(1).map(section => section.split(/^(?:>>> USER(?: \(turn \d+\/\d+\))?:[ \t]*|=== SEEDED GATES ===[ \t]*)$/m)[0].trim());
+}
+/** Read every JSONL turn or decline the whole artifact; never silently drop malformed lines. */
+export function loadExecutionFacts(path, identity, expectedSha256) {
+    if (!expectedSha256)
+        return unavailableExecution("Legacy run has no recorded trace digest; execution is unverified.");
+    if (!existsSync(path))
+        return unavailableExecution("Recorded execution trace is missing.", "invalid");
+    try {
+        const lines = readFileSync(path, "utf8").split("\n").filter(line => line.trim());
+        const traces = lines.map(deserializeTrace);
+        if (traces.some(trace => trace === null))
+            return unavailableExecution("Execution trace contains unreadable turns.", "invalid");
+        return executionFacts(traces, identity, { source: "saved", expectedSha256 });
+    }
+    catch {
+        return unavailableExecution("Execution trace could not be read or validated.", "invalid");
+    }
+}
+//# sourceMappingURL=judge-evidence.js.map

package/dist/judge-references.d.ts

@@ -0,0 +1,8 @@
+import type { Scenario } from "./spec.js";
+export interface JudgeReference {
+    file: string;
+    sha256: string;
+    text: string;
+}
+/** Only explicitly selected local UTF-8 files are read. Never discover sibling files. */
+export declare function resolveJudgeReferences(scenario: Scenario, specDir: string): JudgeReference[];

package/dist/judge-references.js

@@ -0,0 +1,34 @@
+import { createHash } from "node:crypto";
+import { readFileSync, realpathSync, statSync } from "node:fs";
+import { isAbsolute, relative, resolve } from "node:path";
+const LIMIT = 65_536;
+/** Only explicitly selected local UTF-8 files are read. Never discover sibling files. */
+export function resolveJudgeReferences(scenario, specDir) {
+    const selected = scenario.judgeReferences ?? [];
+    if (!selected.length)
+        return [];
+    if (selected.length > 16 || new Set(selected).size !== selected.length)
+        throw new Error("Invalid judge reference selection");
+    const root = realpathSync(specDir);
+    let total = 0;
+    return selected.map(file => {
+        if (!file || /[\\:\x00-\x1f]/.test(file) || file.startsWith("/") || file.split("/").some(part => !part || part === "." || part === ".."))
+            throw new Error(`Unsafe judge reference: ${file}`);
+        const path = realpathSync(resolve(root, file));
+        const rel = relative(root, path);
+        if (isAbsolute(rel) || rel === ".." || rel.startsWith("../") || rel.startsWith("..\\"))
+            throw new Error(`Judge reference escapes specification directory: ${file}`);
+        const info = statSync(path);
+        if (!info.isFile() || info.size > LIMIT)
+            throw new Error(`Judge reference must be a regular file of at most ${LIMIT} bytes: ${file}`);
+        const bytes = readFileSync(path);
+        total += bytes.length;
+        if (bytes.length > LIMIT || total > LIMIT)
+            throw new Error(`Selected judge references exceed ${LIMIT} bytes`);
+        const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
+        if (text.includes("\0"))
+            throw new Error(`Judge reference contains binary content: ${file}`);
+        return { file, sha256: createHash("sha256").update(bytes).digest("hex"), text };
+    });
+}
+//# sourceMappingURL=judge-references.js.map

package/dist/lint.d.ts

@@ -1,4 +1,4 @@
-export type LintCode = "spec" | "ship_bar" | "critical" | "fixture" | "fixture-marker" | "consistency" | "stale" | "stability" | "covers" | "lint-error";
+export type LintCode = "spec" | "ship_bar" | "critical" | "fixture" | "fixture-marker" | "consistency" | "stale" | "stability" | "covers" | "lint-error" | "authoring";
 /**
  * How much a finding means.
  *

package/dist/lint.js

@@ -9,6 +9,8 @@ import { downgradeWarning } from "./downgrade.js";
 import { collectScoredRuns } from "./trends.js";
 import { boundaryCells, stabilityFrom, stabilityNote } from "./stability.js";
 import { MARKERS, unknownMarkerDirs, suggestMarker } from "./workspace.js";
+import { fixtureInventory, inspectEvalAuthoring, authoringEvidenceProfile } from "./authoring-preflight.js";
+import { resolveJudgeReferences } from "./judge-references.js";
 /** True when a finding fails the gate. The single place the exit-code rule lives. */
 export function failsGate(f) {
     return (f.severity ?? "error") === "error";
@@ -49,6 +51,20 @@ export function lintSkill(skillDir, testsDir = join(skillDir, "tests")) {
         return [{ skill: basename(skillDir), code: "spec", message }];
     }
     const skill = spec.skill;
+    for (const hint of inspectEvalAuthoring(spec, { ...authoringEvidenceProfile(testsDir), fixtureFiles: scenario => {
+            const fixture = effectiveFixture(scenario);
+            if (!fixture)
+                return [];
+            // Unreadable/missing fixtures are reported by the fixture checks below.
+            try {
+                return fixtureInventory(resolve(testsDir, fixture));
+            }
+            catch {
+                return [];
+            }
+        } })) {
+        findings.push({ skill, scenario: hint.scenario, code: "authoring", severity: hint.severity ?? "info", message: `${hint.code}: ${hint.message}` });
+    }
     // A lint from an older tool than the records it is checking cannot be trusted to
     // check them: a 0.1.0 binary reported 38 spurious `consistency` findings against
     // partial runs that 0.3.x handles correctly, and it cannot read source-hash key
@@ -91,6 +107,12 @@ export function lintSkill(skillDir, testsDir = join(skillDir, "tests")) {
     // relative to the spec's dir, matching workspace.ts resolve(specDir, fixture) where specDir = <skillDir>/tests.
     const specDir = dirname(specPath);
     for (const s of spec.scenarios) {
+        try {
+            resolveJudgeReferences(s, specDir);
+        }
+        catch (error) {
+            findings.push({ skill, scenario: s.id, code: "fixture", message: String(error) });
+        }
         const fx = effectiveFixture(s); // shared with sources.ts so hashing and linting can't drift
         if (fx) {
             const abs = isAbsolute(fx) ? fx : resolve(specDir, fx);

package/dist/regate.js

@@ -1,10 +1,12 @@
 import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs";
 import { basename, join } from "node:path";
-import { parseVerdict, detectMisfire } from "./grade.js";
+import { parseModelRef } from "./adapters/types.js";
+import { parseVerdict, detectMisfire, resolveJudgeReferences } from "./grade.js";
 import { evaluateNeedleGates, evaluateOutputGates, hasNeedleGates, hasOutputGates } from "./seeded.js";
 import { evaluateTraceGates } from "./trace-gates.js";
 import { mergeTraces, deserializeTrace } from "./execution-trace.js";
 import { judgeOneRep } from "./regrade.js";
+import { loadExecutionFacts, transcriptResponses, unavailableExecution } from "./judge-evidence.js";
 import { readResults, writeResults, diffPath, transcriptPath, judgeRawPath, findDiffFiles, findTranscriptFiles, findTraceFiles, tracePath, effectiveThreshold, scoreContextFor, rebuildScenarioResult, mergeScenarioMetrics, deliveryStatusForObservations, } from "./results.js";
 import { outcomesToResult } from "./reps.js";
 import { appendJournal } from "./journal.js";
@@ -150,6 +152,7 @@ export async function regateRun(opts) {
         throw new Error(`nothing to regate in ${opts.runDir}` +
             (blocked.length > 0 ? `:\n  ${blocked.join("\n  ")}` : " — no scenario declares diff/output assertions or assert.trace"));
     }
+    const references = Object.fromEntries(targets.map(s => [s.id, resolveJudgeReferences(s, opts.specDir)]));
     const changes = [];
     let judgeCalls = 0;
     const scenarios = [];
@@ -176,6 +179,16 @@ export async function regateRun(opts) {
             const tPath = transcriptPath(opts.runDir, scenario.id, mode, rep);
             const before = existsSync(tPath) ? readFileSync(tPath, "utf8") : "";
             const outputGate = evaluateOutputGates(scenario, before);
+            const retainedPanel = rec.rep_judgments?.find(panel => panel.repetition === (rep ?? 0));
+            const recordedTraceHashes = [retainedPanel?.judgments[0]?.input?.trace_sha256, retainedPanel?.objective?.trace_sha256,
+                rec.objective?.rep_trace_sha256?.[rep ?? 0], rec.reps === undefined ? rec.objective?.trace_sha256 : undefined].filter((hash) => hash !== undefined);
+            const execution = new Set(recordedTraceHashes).size > 1
+                ? unavailableExecution("Conflicting recorded trace digests.", "invalid")
+                : loadExecutionFacts(tracePath(opts.runDir, scenario.id, mode, rep), {
+                    scenarioId: scenario.id, mode, repetition: rep ?? 0, turns: scenario.turns.length,
+                    ...(prev.model.includes(":") ? { subject: parseModelRef(prev.model) } : {}), piVersion: prev.harness_cli_version,
+                    responses: transcriptResponses(before, scenario),
+                }, recordedTraceHashes[0]);
             // Output and trace gates are re-decided from saved artifacts. Free: no model, no judge.
             let traceFailure = null;
             const expectedOutputHash = rec.objective?.rep_output_sha256?.[rep ?? 0]
@@ -191,7 +204,11 @@ export async function regateRun(opts) {
                     assertions: [...outputIntegrityAssertion, ...outputGate.assertions],
                 }
                 : undefined;
-            if (scenario.traceAssert) {
+            if (execution.status === "invalid") {
+                traceFailure = `objective: ${execution.reason}`;
+                objective = { ...(objective ?? {}), status: "ERROR", assertions: [...(objective?.assertions ?? []), { kind: "trace_evidence", status: "ERROR", detail: execution.reason }] };
+            }
+            else if (scenario.traceAssert) {
                 const tp = tracePath(opts.runDir, scenario.id, mode, rep);
                 // A PARTIAL read is refused, not graded. `deserializeTrace` returns null
                 // for a malformed line and for a version it declines, and dropping those
@@ -303,7 +320,7 @@ export async function regateRun(opts) {
             const judged = await judgeOneRep({
                 runDir: opts.runDir, spec: opts.spec, scenario, transcript,
                 adapter: opts.adapter, judge: opts.judge, specDir: opts.specDir,
-                mode, rep, now, rejudge: true,
+                mode, rep, now, rejudge: true, execution, objective, references: references[scenario.id],
             });
             outcomes.push({ ...judged, objective });
             judgeCalls += judged.metrics?.judge_calls ?? 0;
@@ -347,6 +364,7 @@ export async function regateRun(opts) {
         skill: prev.skill, harness: prev.harness, model: prev.model,
         // Carried verbatim: a regate re-reads saved diffs, it never re-runs the harness.
         harness_cli_version: prev.harness_cli_version, delivery_canary: prev.delivery_canary,
+        delivery_observations: prev.delivery_observations,
         // The arm is provenance of the MEASUREMENT, not of this rewrite, and it is the
         // only record that a `+<arm>` run actually delegated: rebuilding the draft
         // field-by-field without it silently deleted `definitions`/`ledger_events`

package/dist/regrade.d.ts

@@ -1,7 +1,9 @@
 import type { Spec, Scenario } from "./spec.js";
 import type { HarnessAdapter, ModelRef } from "./adapters/types.js";
+import { type JudgeReference } from "./grade.js";
 import { type ScenarioResult, type ResultsFile } from "./results.js";
 import { type RepOutcome } from "./reps.js";
+import { type JudgeExecutionFacts } from "./judge-evidence.js";
 /**
  * Carry a run's recorded hashes forward, refreshing only the `rubric:` keys this
  * re-grade actually judged under (plus the persona, which applies to all of them).
@@ -14,8 +16,9 @@ import { type RepOutcome } from "./reps.js";
  * No hashes recorded (a pre-`source_hashes` run) stays that way: inventing hashes for
  * a run that never recorded any would claim a coverage it does not have.
  */
-export declare function refreshRubricHashes(recorded: Record<string, string> | undefined, spec: Spec, judgedIds: string[]): Record<string, string> | undefined;
+export declare function refreshRubricHashes(recorded: Record<string, string> | undefined, spec: Spec, judgedIds: string[], specDir?: string, references?: Record<string, JudgeReference[]>, refreshPersona?: boolean): Record<string, string> | undefined;
 export interface RegradeOptions {
+    references?: JudgeReference[];
     runDir: string;
     spec: Spec;
     scenario: Scenario;
@@ -37,6 +40,8 @@ export interface RegradeOptions {
     /** Select only repetitions with retained ERROR criterion votes. */
     onlyUnparsed?: boolean;
     prior?: ScenarioResult;
+    subject?: ModelRef;
+    piVersion?: string;
     now?: () => string;
 }
 /** Judge one saved transcript: writes the judge-raw artifact, emits a `judge-verdict` journal event (plus `misfire-flag` when the verdict is suspect), and returns the outcome. */
@@ -55,6 +60,8 @@ export declare function judgeOneRep(opts: {
     rejudge?: boolean;
     /** Retained objective evidence must precede missing-vote normalization. */
     objective?: RepOutcome["objective"];
+    execution?: JudgeExecutionFacts;
+    references?: JudgeReference[];
 }): Promise<RepOutcome>;
 /**
  * Re-judge a scenario's saved transcript(s) for the run's mode with `judge` — no
@@ -77,6 +84,8 @@ export interface RegradeRunOptions {
      * cleanly, so spending judge calls on them buys nothing.
      */
     onlySuspect?: boolean;
+    /** Rejudge only these recorded scenarios; untouched verdicts remain historical. */
+    only?: string[];
     /** Rejudge only retained repetitions containing ERROR criterion votes, with the recorded judge. */
     onlyUnparsed?: boolean;
 }