@skill-harness/core 0.6.0 → 0.8.0

package/dist/reps.js

@@ -1,3 +1,25 @@
+/**
+ * Collapse per-rep objective results.
+ *
+ * Strict on purpose, and deliberately NOT the same policy as the judge's
+ * pass-threshold aggregation: an objective assertion is a statement about what
+ * the model DID, so one rep that called a forbidden tool is a real finding, not
+ * a minority draw to be voted away. ERROR dominates (missing evidence is never a
+ * pass), then FAIL, then PASS. The retained assertion detail comes from the
+ * first non-passing rep, since that is the one worth reading.
+ */
+export function aggregateObjective(outcomes) {
+    const present = outcomes.map((o) => o.objective).filter((o) => o !== undefined);
+    if (present.length === 0)
+        return undefined;
+    const errored = present.find((o) => o.status === "ERROR");
+    if (errored)
+        return errored;
+    const failed = present.find((o) => o.status === "FAIL");
+    if (failed)
+        return failed;
+    return present[0];
+}
 /**
  * Collapse N rep outcomes into one scenario verdict. A rep is "clean" when its
  * judge did not misfire. If fewer than half the reps are clean the scenario is
@@ -31,15 +53,19 @@ export function aggregateReps(outcomes, threshold) {
  * caller to merge.
  */
 export function outcomesToResult(id, outcomes, repCount, threshold) {
+    // Spread rather than always-set: a scenario with no trace assertions must
+    // produce a result byte-identical to one from before this field existed.
+    const objective = aggregateObjective(outcomes);
+    const objectiveField = objective ? { objective } : {};
     if (repCount === 1) {
         const o = outcomes[0];
-        return { id, judge_verdict: o.verdict, judge_reason: o.reason, suspect: o.suspect, override: null, note: "" };
+        return { id, judge_verdict: o.verdict, judge_reason: o.reason, suspect: o.suspect, override: null, note: "", ...objectiveField };
     }
     const agg = aggregateReps(outcomes, threshold);
     return {
         id, judge_verdict: agg.verdict, judge_reason: agg.reason, suspect: agg.suspect,
         reps: agg.reps, passes: agg.passes, clean: agg.clean, flakiness: agg.flakiness,
-        pass_threshold: threshold, override: null, note: "",
+        pass_threshold: threshold, override: null, note: "", ...objectiveField,
     };
 }
 //# sourceMappingURL=reps.js.map

package/dist/rescore.js

@@ -1,6 +1,6 @@
 import { existsSync } from "node:fs";
 import { join } from "node:path";
-import { readResults, writeResults, scoreContextFor } from "./results.js";
+import { readResults, writeResults, scoreContextFor, rebuildScenarioResult } from "./results.js";
 import { appendJournal } from "./journal.js";
 import { policyDigest, POLICY_PREFIX } from "./sources.js";
 /**
@@ -65,7 +65,16 @@ export function rescoreRun(opts) {
         if (verdict !== s.judge_verdict) {
             changes.push({ id: s.id, from: s.judge_verdict, to: verdict, passes: s.passes, clean: s.clean, fromThreshold, toThreshold });
         }
-        return { ...s, judge_verdict: verdict, pass_threshold: toThreshold };
+        // Through the choke point, not a spread. This was the FIFTH rewriter of a
+        // `ScenarioResult` and the only one still using `{ ...s }` — so it inherited
+        // none of the invariants the others get, and adding `objective` and
+        // `adjudication` to the type did not fail the build here. Concretely: a cell
+        // that adjudication settled FAIL reverted to PASS when a threshold change
+        // recomputed it from rep counters that adjudication never updated.
+        //
+        // Both blocks are CARRIED: a rescore re-applies a threshold to reps that were
+        // already measured. It re-measures nothing, so it may discard nothing.
+        return rebuildScenarioResult({ ...s, judge_verdict: verdict, pass_threshold: toThreshold }, s, { objective: "carry", adjudication: "carry" });
     });
     const ctx = scoreContextFor(prev, opts.spec);
     const results = writeResults(opts.runDir, {

package/dist/restamp.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Upgrade already-committed runs to the model-visible digests, without re-running anything.
+ *
+ * ## Why a command and not a rule inside lint
+ *
+ * A stored hash is one-way. A run recorded before 0.8.0 carries only sha256 of SKILL.md's
+ * RAW BYTES, so once that file has been edited there is nothing in the record — and
+ * nothing saved beside it; run dirs hold transcripts, not a copy of the skill — that can
+ * say whether the edit touched the body or only the frontmatter. Lint therefore cannot
+ * retroactively forgive a frontmatter-only edit on a legacy record, and pretending
+ * otherwise would mean dropping the check that catches real body edits across the whole
+ * published corpus.
+ *
+ * What CAN be established, and is the entire proof this command rests on: when the
+ * recorded raw-bytes hash still equals the file's current raw-bytes hash, that run
+ * measured the exact bytes on disk right now. The model-visible digest of those same
+ * bytes is then a true statement about what that run measured, and recording it is
+ * bookkeeping rather than a claim. Any record that fails the check is left untouched —
+ * it stays honestly stale, and `run` is the only thing that can clear it.
+ *
+ * So the migration is: upgrade while the board is green, and every later
+ * frontmatter-only edit is free. Skipping it costs nothing either — a legacy record with
+ * an unedited skill keeps matching on the raw-bytes key exactly as before.
+ *
+ * ## `--from <ref>`, for when the edit already landed
+ *
+ * Nobody discovers a gate over-fires until it has over-fired, so the green worktree the
+ * paragraph above assumes is usually already gone. The pre-edit bytes are still in git,
+ * and they carry the same proof in two steps: the recorded hash matches `<ref>`'s bytes
+ * (so the run measured THOSE), and `<ref>`'s model-visible digest equals today's (so the
+ * text the model receives has not moved since). Only then is the digest recorded.
+ *
+ * It is the same proof, not a weaker one — with no `--from`, "the ref" is just the
+ * working tree, which is why both go through one code path.
+ *
+ * Free and offline: no model, no judge, no network. It rewrites only `source_hashes`.
+ */
+/**
+ * Three exclusive buckets that SUM to `runs`, plus `partial` as a subset of `upgraded`.
+ *
+ * Summing is the point. These counters are the only thing an operator can judge a
+ * migration by, and "18 upgraded, 20 left alone" out of 140 leaves 102 records
+ * unaccounted for — which reads like the command quietly skipped them rather than like
+ * the 102 having nothing to upgrade.
+ */
+export interface RestampReport {
+    /** Run records examined. `upgraded + unprovable + unchanged`. */
+    runs: number;
+    /** Records that gained at least one model-visible digest, and were rewritten. */
+    upgraded: number;
+    /**
+     * Records that gained NOTHING because a document they measured has already moved —
+     * honestly stale, and `run` is the only remedy.
+     */
+    unprovable: number;
+    /** Records with nothing to do: no `source_hashes`, unreadable, or already upgraded. */
+    unchanged: number;
+    /** Of `upgraded`, those still carrying a document that could not be proven. */
+    partial: number;
+    /** `<runDir>: <key>` for each digest added, for the CLI to print. */
+    added: string[];
+}
+export interface RestampOptions {
+    /** Git ref holding the bytes the runs measured, for a tree that has already been edited. */
+    from?: string;
+}
+/** Upgrade every provable record under one skill's tests/results. */
+export declare function restampSkill(skillDir: string, opts?: RestampOptions): RestampReport;

package/dist/restamp.js

@@ -0,0 +1,194 @@
+import { createHash } from "node:crypto";
+import { execFileSync } from "node:child_process";
+import { readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join, relative, resolve } from "node:path";
+import yaml from "js-yaml";
+import { loadSpec } from "./spec.js";
+import { resultsPath } from "./results.js";
+import { enumerateRunDirs } from "./lint.js";
+import { promptDocDigest, SKILL_KEY, SKILL_PROMPT_KEY, PROMPT_PREFIX, UNREADABLE, } from "./sources.js";
+/**
+ * The `source_hashes` map of a raw-loaded results doc, or null when the doc has none.
+ * Read from the RAW yaml rather than through `readResults` on purpose: this rewrites the
+ * file, and `readResults` migrates schema 1 → 2 in memory, so writing its output back
+ * would silently upgrade the schema of records nobody asked to touch.
+ */
+function rawHashes(doc) {
+    if (!doc || typeof doc !== "object")
+        return null;
+    const h = doc.source_hashes;
+    if (!h || typeof h !== "object" || Array.isArray(h))
+        return null;
+    return h;
+}
+/**
+ * `git show <ref>:<path>` as BYTES, or null when the ref, the repo, or the path isn't there.
+ *
+ * `execFileSync` with a buffer encoding rather than the shared `exec` helper: that one
+ * accumulates stdout as decoded strings, so a multi-byte character landing on a chunk
+ * boundary would come back with a replacement character and hash to something the run
+ * never recorded. This comparison is byte-exact by definition, so it reads bytes.
+ */
+function bytesAtRef(abs, ref) {
+    const cwd = dirname(abs);
+    try {
+        const top = execFileSync("git", ["rev-parse", "--show-toplevel"], { cwd, encoding: "utf8" }).trim();
+        const rel = relative(top, abs).split("\\").join("/");
+        return execFileSync("git", ["show", `${ref}:${rel}`], { cwd, maxBuffer: 64 * 1024 * 1024 });
+    }
+    catch {
+        return null; // no git, no repo, no such ref, or the file did not exist there
+    }
+}
+/**
+ * The bytes a run is claimed to have measured: `<ref>`'s copy, or the working tree's when
+ * no ref was given. Null when they cannot be read at all, which proves nothing and so
+ * upgrades nothing.
+ */
+function measuredBytes(abs, from) {
+    if (from !== undefined)
+        return bytesAtRef(abs, from);
+    try {
+        return readFileSync(abs);
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Fail loudly when `--from` names something git cannot resolve.
+ *
+ * Without this the whole migration degrades into "0 upgraded, everything left alone",
+ * which is indistinguishable from a corpus that genuinely cannot be proven — measured on
+ * a clone whose only local branch was the feature branch, where `--from main` silently
+ * did nothing and read as the tool being broken. A ref the operator got wrong is their
+ * mistake to see, not one to absorb.
+ */
+function assertRefResolves(dir, ref) {
+    try {
+        execFileSync("git", ["rev-parse", "--verify", "--quiet", `${ref}^{commit}`], { cwd: dir, stdio: "ignore" });
+    }
+    catch {
+        throw new Error(`--from \`${ref}\` does not resolve to a commit in the repo holding ${dir} — ` +
+            `check the ref exists locally (a fresh clone often has no local \`main\`; try \`origin/main\`)`);
+    }
+}
+/**
+ * Write via a temp file in the same directory, then rename.
+ *
+ * `rename(2)` is atomic within a filesystem, so a reader sees the old file or the new one
+ * and never a half-written one. It matters more here than in the writers that produce a
+ * run: this loops over an entire published corpus rewriting records that already hold
+ * measurements nobody can reproduce without spending model tokens, so an interrupt or a
+ * full disk partway through would truncate exactly the artifacts the command exists to
+ * preserve. The temp file is `.tmp`-suffixed beside the target rather than in the system
+ * temp dir, because a cross-device rename is not atomic — and is not even permitted.
+ */
+function writeAtomically(path, text) {
+    const tmp = `${path}.restamp.tmp`;
+    writeFileSync(tmp, text, "utf8");
+    try {
+        renameSync(tmp, path);
+    }
+    catch (e) {
+        try {
+            rmSync(tmp, { force: true });
+        }
+        catch { /* the rename failure is what matters */ }
+        throw e;
+    }
+}
+/** Upgrade every provable record under one skill's tests/results. */
+export function restampSkill(skillDir, opts = {}) {
+    const report = { runs: 0, upgraded: 0, unprovable: 0, unchanged: 0, partial: 0, added: [] };
+    const specPath = join(skillDir, "tests", "specification.yaml");
+    const spec = loadSpec(specPath);
+    const specDir = dirname(specPath);
+    // Only `system_prompt_file` paths, resolved from the CURRENT spec. The other
+    // path-shaped keys (`post_test`, `env.extensions`) name code, not prompt documents:
+    // every byte of those reaches the run, so they keep hashing raw and are left alone.
+    const promptDocs = new Map([[SKILL_KEY, resolve(skillDir, "SKILL.md")]]);
+    for (const s of spec.scenarios) {
+        if (s.systemPromptFile)
+            promptDocs.set(s.systemPromptFile, resolve(specDir, s.systemPromptFile));
+    }
+    if (opts.from !== undefined)
+        assertRefResolves(skillDir, opts.from);
+    // Resolved once per document, not once per run: with `--from`, each resolution is two
+    // git invocations, and a corpus of 140 runs would otherwise pay for them 140 times over.
+    const provable = new Map();
+    for (const [legacyKey, abs] of promptDocs) {
+        const measured = measuredBytes(abs, opts.from);
+        let currentPrompt = null;
+        try {
+            currentPrompt = promptDocDigest(readFileSync(abs, "utf8"));
+        }
+        catch {
+            currentPrompt = null;
+        }
+        provable.set(legacyKey, {
+            measuredRaw: measured === null ? null : createHash("sha256").update(measured).digest("hex"),
+            measuredPrompt: measured === null ? null : promptDocDigest(measured.toString("utf8")),
+            currentPrompt,
+        });
+    }
+    for (const runDir of enumerateRunDirs(join(skillDir, "tests", "results"))) {
+        report.runs++;
+        let doc;
+        try {
+            doc = yaml.load(readFileSync(resultsPath(runDir), "utf8"));
+        }
+        catch {
+            // Unreadable/malformed → lint already reports it; never rewrite what we can't read.
+            // Still bucketed: every record counted in `runs` has to be accounted for somewhere.
+            report.unchanged++;
+            continue;
+        }
+        const hashes = rawHashes(doc);
+        if (!hashes) {
+            report.unchanged++; // predates source_hashes → nothing to upgrade from
+            continue;
+        }
+        let blocked = false;
+        const additions = [];
+        for (const legacyKey of promptDocs.keys()) {
+            const recorded = hashes[legacyKey];
+            if (recorded === undefined || recorded === UNREADABLE)
+                continue; // never measured → nothing proven
+            const upgradedKey = legacyKey === SKILL_KEY ? SKILL_PROMPT_KEY : PROMPT_PREFIX + legacyKey;
+            if (hashes[upgradedKey] !== undefined)
+                continue; // already upgraded
+            const p = provable.get(legacyKey);
+            if (p.currentPrompt === null)
+                continue; // the file is gone; lint reports that itself
+            // Two steps, and both are required. First: this run measured the reference bytes —
+            // otherwise the reference says nothing about THIS run. Second: those bytes and
+            // today's carry the same model-visible text — otherwise what the model reads has
+            // genuinely moved and only a re-run can speak for it.
+            if (p.measuredRaw === null || recorded !== p.measuredRaw || p.measuredPrompt !== p.currentPrompt) {
+                blocked = true;
+                continue;
+            }
+            additions.push([upgradedKey, p.currentPrompt]);
+        }
+        if (additions.length === 0) {
+            // Exclusive: a record that gained nothing is `unprovable` only if something
+            // actually blocked it, and `unchanged` otherwise.
+            if (blocked)
+                report.unprovable++;
+            else
+                report.unchanged++;
+            continue;
+        }
+        for (const [k, v] of additions) {
+            hashes[k] = v;
+            report.added.push(`${runDir}: ${k}`);
+        }
+        writeAtomically(resultsPath(runDir), yaml.dump(doc, { lineWidth: 100 }));
+        report.upgraded++;
+        if (blocked)
+            report.partial++;
+    }
+    return report;
+}
+//# sourceMappingURL=restamp.js.map

package/dist/results.d.ts

@@ -14,6 +14,56 @@ export interface ScenarioResult {
     clean?: number;
     flakiness?: number;
     pass_threshold?: number;
+    /**
+     * Objective trace-gate evidence. ADDITIVE and optional.
+     *
+     * Absent means "the scenario declared no trace assertions" — NOT an objective
+     * pass. Anything reading this must treat the two as different; collapsing them
+     * would silently upgrade every legacy result to "objectively verified".
+     */
+    objective?: ObjectiveResult;
+    /**
+     * Confidence-aware adjudication. ADDITIVE and optional.
+     *
+     * Absent means historical single-judge behavior — NOT that judges agreed. An
+     * unresolved adjudication additionally sets `suspect: true`, which is what
+     * actually blocks SHIP; this field is the audit trail behind that flag.
+     */
+    adjudication?: AdjudicationResult;
+}
+/** One judge's answer for a cell, kept verbatim however the collapse turned out. */
+export interface Judgment {
+    /** 1-based: judgment 1 is the first-wave judge, 2 the secondary, 3 the tie-break. */
+    ordinal: number;
+    judge: {
+        provider: string;
+        model: string;
+    };
+    verdict: Verdict;
+    reason: string;
+    /** The judge misfired — recorded, never counted as a clean vote. */
+    suspect: boolean;
+}
+export interface AdjudicationResult {
+    state: "confirmed" | "tie_broken" | "unresolved";
+    /** Why the cell was re-judged. */
+    trigger: "ambiguous" | "contradictory" | "non_unanimous" | "ship_deciding";
+    /** Every judgment, in order. Never pruned — an author resolving this needs all of them. */
+    judgments: Judgment[];
+    /** The collapsed answer. Absent when unresolved. */
+    verdict?: "PASS" | "FAIL";
+}
+/** Objective gate outcome for one scenario cell. */
+export interface ObjectiveResult {
+    /** ERROR means the evidence was missing or malformed — never a pass. */
+    status: "PASS" | "FAIL" | "ERROR";
+    trace_version?: number;
+    trace_sha256?: string;
+    assertions: {
+        kind: string;
+        status: "PASS" | "FAIL" | "ERROR";
+        detail: string;
+    }[];
 }
 export interface GradeSummary {
     passed: number;
@@ -56,16 +106,23 @@ export interface ResultsFile {
     harness_cli_version?: string;
     /**
      * `pass` when this run proved, before spending the wave, that the skill body was
-     * reachable in the model's context (see canary.ts). Absent means the probe was
-     * not asked for — never that it failed, because a failed canary aborts the run
-     * and no results.yaml is written.
+     * reachable in the model's context (see canary.ts). `skipped` when the probe was
+     * asked for but could not be performed — SKILL.md has no `## ` heading to quote
+     * back, so no reply could prove anything. Absent means the probe was not asked
+     * for — never that it failed, because a failed canary aborts the run and no
+     * results.yaml is written.
+     *
+     * `skipped` exists because absent and skipped were the same value: a user who
+     * passed `--canary` precisely because pi ≥ 0.83.0 delivery is unreliable got a
+     * silently degraded probe, a fully billed wave, and a committed results.yaml
+     * byte-identical to a run where delivery was never checked at all.
      *
      * Only green runs can carry it: red delivers nothing by design and force delivers
      * through the system prompt. It is provenance for the *validity* of a green run,
      * which is why it lives here rather than only in the journal — `journal.jsonl` is
      * gitignored, and this claim has to survive a commit.
      */
-    delivery_canary?: "pass";
+    delivery_canary?: "pass" | "skipped";
     skill: string;
     harness: string;
     model: string;
@@ -152,12 +209,35 @@ export declare function runDirFor(skillDir: string, harness: string, model: Mode
 export declare function transcriptPath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
 export declare function reportPath(runDir: string): string;
 export declare function resultsPath(runDir: string): string;
-/** The verdict that counts: author override when present, else the judge's. */
+/**
+ * The verdict that counts: author override when present, else the objective
+ * gate, else the judge's.
+ *
+ * **The objective gate outranks the judge.** `assert.trace` is a mechanical
+ * statement about what the model DID — it called `write`, it touched `.env`.
+ * The judge is an LLM reading prose. When they disagree, the measurement wins.
+ *
+ * This is the only place that ordering is enforced, and it has to be here.
+ * `objective` used to reach the ship decision solely through `gatePrefix` in
+ * `run.ts`, which forces a single rep's verdict — so every path that recomputed
+ * a verdict afterwards silently dropped the gate while keeping the `objective`
+ * block that claimed it was enforced. Three of them did: `--reps N` out-voted an
+ * objective FAIL 2-to-1 (100%, grade A, SHIP, on a CRITICAL scenario that called
+ * a forbidden tool), `regrade` re-judged from a transcript the tool calls are
+ * absent from, and `regate` recomputed it. `reps.ts` already states the policy —
+ * "one rep that called a forbidden tool is a real finding, not a minority draw
+ * to be voted away" — and nothing enforced it.
+ *
+ * An author override still wins, exactly as it does over `suspect`. Overriding a
+ * deterministic assertion is a deliberate, recorded human act — and the failure
+ * this guards against was never a human deciding, it was nobody deciding.
+ */
 export declare function effectiveVerdicts(scenarios: ScenarioResult[]): ScenarioVerdict[];
 /**
  * The ONLY place effective_grade is computed. Every writer goes through here,
  * so a persisted grade can never disagree with verdicts + overrides.
- * ctx is null for unscored (red/force) runs.
+ * ctx is null for unscored runs — `red` only, since 0.5.0: `force` is a real
+ * deployment and is scored (see SCORED_MODES directly above).
  */
 export declare function finalizeResults(draft: ResultsDraft, ctx: ScoreContext | null): ResultsFile;
 /** Finalize + persist results.yaml (creating the run dir). Returns what was written. */
@@ -207,8 +287,50 @@ export declare function findJudgeRawFiles(runDir: string, scenarioId: string, mo
  * generated evidence, ignored like transcripts, not committed like results.yaml.
  */
 export declare function diffPath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
+/**
+ * What happens to one piece of recorded evidence when a command rebuilds a result.
+ *
+ *  - `carry` — the command did not re-measure this, so the prior value still
+ *    describes the run and must survive.
+ *  - `fresh` — the command re-measured it; take the new value.
+ *  - `drop`  — the command invalidated it; a stale value would misinform.
+ */
+export type EvidencePolicy = "carry" | "fresh" | "drop";
+export interface RebuildPolicy {
+    objective: EvidencePolicy;
+    adjudication: EvidencePolicy;
+}
+/**
+ * Rebuild a `ScenarioResult` after a command re-measured part of it.
+ *
+ * **The single choke point for every rewriter**, and exhaustive by construction:
+ * every field is destructured below, so adding one to `ScenarioResult` fails the
+ * build HERE until someone decides whether it is carried, taken fresh, or
+ * dropped. That guard is the whole point of the function.
+ *
+ * It exists because the ad-hoc version — `{ ...fresh, override: prior.override,
+ * note: prior.note }`, written independently in three places — silently dropped
+ * `objective` from `grade` and `adjudication` from `regate`. Both failures ran in
+ * the dangerous direction: a trace-gated scenario re-read as "no assertions
+ * declared", and an unresolved judge disagreement as a settled verdict. 1,036
+ * tests passed through it; a real smoke run caught it.
+ *
+ * The author's `override` and `note` are always carried and are not policy —
+ * no command re-measures a human's judgement.
+ */
+export declare function rebuildScenarioResult(fresh: ScenarioResult, prior: ScenarioResult | undefined, policy: RebuildPolicy): ScenarioResult;
+/**
+ * Where a rep's execution trace is saved: `<id>.<mode>[.rep<k>].trace.jsonl`.
+ *
+ * `.jsonl` rather than `.txt` so it is distinguishable at a glance from a
+ * transcript, and one JSON object per line so a multi-turn scenario's per-turn
+ * traces append without a wrapper.
+ */
+export declare function tracePath(runDir: string, scenarioId: string, mode: string, rep?: number): string;
 /** A scenario's staged-diff files, sorted (plain first, then numeric rep). Mode-scoped when given. */
 export declare function findDiffFiles(runDir: string, scenarioId: string, mode?: string): string[];
+/** A scenario's execution-trace files, sorted (plain first, then numeric rep). Mode-scoped when given. */
+export declare function findTraceFiles(runDir: string, scenarioId: string, mode?: string): string[];
 /** A single representative transcript file for a scenario in a run dir. Null if none. */
 export declare function findTranscriptFile(runDir: string, scenarioId: string): string | null;
 /**