@vibe-agent-toolkit/agent-skills 0.1.39-rc.9 → 0.1.39

package/dist/skill-test/experimenter-prompt.d.ts

@@ -6,6 +6,7 @@ export interface BuildPromptOptions {
     evalsPath: string;
     gradingOut: string;
     frictionOut: string;
+    workspacesRoot: string;
     baseline: boolean;
 }
 /**
@@ -16,5 +17,26 @@ export interface BuildPromptOptions {
  */
 export declare const DEFAULT_EXPERIMENTER_PROMPT: string;
 export declare function buildExperimenterPrompt(opts: BuildPromptOptions): string;
+/**
+ * Sentinel the persisted (audit) copy of the prompt shows in place of the real
+ * nonce, so the on-disk artifact never leaks the secret to skill code.
+ */
+export declare const REDACTED_NONCE_PLACEHOLDER = "<run-nonce redacted>";
+/**
+ * Append the per-run integrity-nonce directive to the effective prompt. The
+ * experimenter must copy `nonce` verbatim into grading.json's top-level
+ * `runNonce` field; the harness then rejects any grading.json whose nonce is
+ * absent or wrong (see run-harness.ts). This is defense-in-depth against
+ * untrusted skill code forging a passing grading.json in the shared sandbox —
+ * without the secret nonce (delivered only via stdin, never written to disk) a
+ * forged grading cannot be authenticated.
+ *
+ * Appended AFTER assertPromptInvariants and AFTER any user `experimenterPrompt`
+ * override, so the nonce requirement is ALWAYS enforced and a committed config
+ * cannot opt out of it.
+ */
+export declare function appendIntegrityNonceDirective(prompt: string, nonce: string): string;
+/** Redact the per-run nonce from a prompt for the on-disk audit copy. */
+export declare function redactNonce(prompt: string, nonce: string): string;
 export declare function assertPromptInvariants(prompt: string): void;
 //# sourceMappingURL=experimenter-prompt.d.ts.map

package/dist/skill-test/experimenter-prompt.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"experimenter-prompt.d.ts","sourceRoot":"","sources":["../../src/skill-test/experimenter-prompt.ts"],"names":[],"mappings":"AAAA,qBAAa,oBAAqB,SAAQ,KAAK;gBACjC,OAAO,EAAE,MAAM;CAI5B;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B,QAuB5B,CAAC;AAKb,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,kBAAkB,GAAG,MAAM,CAOxE;AAmBD,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAI3D"}
+{"version":3,"file":"experimenter-prompt.d.ts","sourceRoot":"","sources":["../../src/skill-test/experimenter-prompt.ts"],"names":[],"mappings":"AAAA,qBAAa,oBAAqB,SAAQ,KAAK;gBACjC,OAAO,EAAE,MAAM;CAI5B;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B,QA2B5B,CAAC;AAKb,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,kBAAkB,GAAG,MAAM,CAQxE;AAED;;;GAGG;AACH,eAAO,MAAM,0BAA0B,yBAAyB,CAAC;AAEjE;;;;;;;;;;;;GAYG;AACH,wBAAgB,6BAA6B,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAQnF;AAED,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEjE;AAmBD,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAI3D"}

package/dist/skill-test/experimenter-prompt.js

@@ -15,11 +15,15 @@ export const DEFAULT_EXPERIMENTER_PROMPT = [
     '',
     'For each eval in {{EVALS_PATH}}:',
     '  1. Dispatch ONE executor subagent. Tell it ONLY the task prompt and the staged subject path {{SUBJECT_PATH}}.',
-    '     Never tell the executor it is being tested.',
+    '     When the eval declares input `files`, ALSO tell the executor its working directory is {{WORKSPACES_ROOT}}/<id>',
+    '     (substitute the eval\'s own id) — the project it should operate on. Never tell the executor it is being tested.',
     '  2. Grade the executor output against the eval\'s `expectations` using skill-creator\'s grader.md rubric.',
+    '     If the eval includes an `expected_output`, treat it as the author\'s prose description of a',
+    '     correct result and use it as CONTEXT when judging — but the pass/fail verdict is still decided',
+    '     per `expectations` entry; `expected_output` informs your judgment, it is not itself a checklist item.',
     '  3. Append each graded expectation to the SINGLE top-level `expectations` array in {{GRADING_OUT}} IMMEDIATELY',
     '     (incremental flush — a mid-run kill must leave partial results).',
-    '  4. Record any packaging-fidelity friction to {{FRICTION_OUT}} using the vat friction schema.',
+    '  4. Record any packaging-fidelity friction to {{FRICTION_OUT}} using the vat friction.json schema.',
     '     If a file referenced by the skill is absent from the staged tree, record a `missing-bundled-file` friction entry.',
     '',
     '{{GRADING_OUT}} MUST be ONE flat JSON object in skill-creator\'s grading.json shape (references/schemas.md):',
@@ -39,10 +43,42 @@ export function buildExperimenterPrompt(opts) {
     return DEFAULT_EXPERIMENTER_PROMPT
         .replace('{{EVALS_PATH}}', opts.evalsPath)
         .replace('{{SUBJECT_PATH}}', opts.subjectPath)
+        .replaceAll('{{WORKSPACES_ROOT}}', opts.workspacesRoot)
         .replaceAll('{{GRADING_OUT}}', opts.gradingOut)
         .replace('{{FRICTION_OUT}}', opts.frictionOut)
         .replace('{{BASELINE_BLOCK}}', opts.baseline ? BASELINE_BLOCK : '');
 }
+/**
+ * Sentinel the persisted (audit) copy of the prompt shows in place of the real
+ * nonce, so the on-disk artifact never leaks the secret to skill code.
+ */
+export const REDACTED_NONCE_PLACEHOLDER = '<run-nonce redacted>';
+/**
+ * Append the per-run integrity-nonce directive to the effective prompt. The
+ * experimenter must copy `nonce` verbatim into grading.json's top-level
+ * `runNonce` field; the harness then rejects any grading.json whose nonce is
+ * absent or wrong (see run-harness.ts). This is defense-in-depth against
+ * untrusted skill code forging a passing grading.json in the shared sandbox —
+ * without the secret nonce (delivered only via stdin, never written to disk) a
+ * forged grading cannot be authenticated.
+ *
+ * Appended AFTER assertPromptInvariants and AFTER any user `experimenterPrompt`
+ * override, so the nonce requirement is ALWAYS enforced and a committed config
+ * cannot opt out of it.
+ */
+export function appendIntegrityNonceDirective(prompt, nonce) {
+    return [
+        prompt,
+        '',
+        'INTEGRITY (required, do this LAST): the grading.json you write MUST include a',
+        `top-level string field "runNonce" whose value is EXACTLY: ${nonce}`,
+        'Copy it verbatim. vat rejects a grading.json whose runNonce is missing or wrong.',
+    ].join('\n');
+}
+/** Redact the per-run nonce from a prompt for the on-disk audit copy. */
+export function redactNonce(prompt, nonce) {
+    return prompt.replaceAll(nonce, REDACTED_NONCE_PLACEHOLDER);
+}
 const REQUIRED_PATTERNS = [
     { test: /\bSTOP\b/, label: 'must instruct the experimenter to STOP' },
     { test: /grading\.json|\{\{GRADING_OUT\}\}/i, label: 'must write grading.json' },

package/dist/skill-test/experimenter-prompt.js.map

@@ -1 +1 @@
-{"version":3,"file":"experimenter-prompt.js","sourceRoot":"","sources":["../../src/skill-test/experimenter-prompt.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC7C,YAAY,OAAe;QACzB,KAAK,CAAC,2CAA2C,OAAO,EAAE,CAAC,CAAC;QAC5D,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAUD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG;IACzC,yGAAyG;IACzG,EAAE;IACF,kCAAkC;IAClC,iHAAiH;IACjH,kDAAkD;IAClD,4GAA4G;IAC5G,iHAAiH;IACjH,uEAAuE;IACvE,gGAAgG;IAChG,wHAAwH;IACxH,EAAE;IACF,8GAA8G;IAC9G,8GAA8G;IAC9G,gHAAgH;IAChH,wEAAwE;IACxE,qGAAqG;IACrG,EAAE;IACF,mFAAmF;IACnF,EAAE;IACF,6GAA6G;IAC7G,4FAA4F;IAC5F,oBAAoB;CACrB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,MAAM,cAAc,GAClB,sIAAsI,CAAC;AAEzI,MAAM,UAAU,uBAAuB,CAAC,IAAwB;IAC9D,OAAO,2BAA2B;SAC/B,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC;SACzC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,WAAW,CAAC;SAC7C,UAAU,CAAC,iBAAiB,EAAE,IAAI,CAAC,UAAU,CAAC;SAC9C,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,WAAW,CAAC;SAC7C,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,iBAAiB,GAAsC;IAC3D,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,wCAAwC,EAAE;IACrE,EAAE,IAAI,EAAE,oCAAoC,EAAE,KAAK,EAAE,yBAAyB,EAAE;IAChF,EAAE,IAAI,EAAE,sCAAsC,EAAE,KAAK,EAAE,0BAA0B,EAAE;IACnF,EAAE,IAAI,EAAE,yBAAyB,EAAE,KAAK,EAAE,oDAAoD,EAAE;IAChG,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,iDAAiD,EAAE;IACrF,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,yBAAyB,EAAE;IACxD;QACE,IAAI,EAAE,+BAA+B;QACrC,KAAK,EAAE,4EAA4E;KACpF;IACD;QACE,IAAI,EAAE,qCAAqC;QAC3C,KAAK,EAAE,0DAA0D;KAClE;CACF,CAAC;AAEF,MAAM,UAAU,sBAAsB,CAAC,MAAc;IACnD,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,iBAAiB,EAAE,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAChE,CAAC;AACH,CAAC"}
+{"version":3,"file":"experimenter-prompt.js","sourceRoot":"","sources":["../../src/skill-test/experimenter-prompt.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC7C,YAAY,OAAe;QACzB,KAAK,CAAC,2CAA2C,OAAO,EAAE,CAAC,CAAC;QAC5D,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAWD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG;IACzC,yGAAyG;IACzG,EAAE;IACF,kCAAkC;IAClC,iHAAiH;IACjH,qHAAqH;IACrH,sHAAsH;IACtH,4GAA4G;IAC5G,kGAAkG;IAClG,qGAAqG;IACrG,4GAA4G;IAC5G,iHAAiH;IACjH,uEAAuE;IACvE,qGAAqG;IACrG,wHAAwH;IACxH,EAAE;IACF,8GAA8G;IAC9G,8GAA8G;IAC9G,gHAAgH;IAChH,wEAAwE;IACxE,qGAAqG;IACrG,EAAE;IACF,mFAAmF;IACnF,EAAE;IACF,6GAA6G;IAC7G,4FAA4F;IAC5F,oBAAoB;CACrB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,MAAM,cAAc,GAClB,sIAAsI,CAAC;AAEzI,MAAM,UAAU,uBAAuB,CAAC,IAAwB;IAC9D,OAAO,2BAA2B;SAC/B,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC;SACzC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,WAAW,CAAC;SAC7C,UAAU,CAAC,qBAAqB,EAAE,IAAI,CAAC,cAAc,CAAC;SACtD,UAAU,CAAC,iBAAiB,EAAE,IAAI,CAAC,UAAU,CAAC;SAC9C,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,WAAW,CAAC;SAC7C,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACxE,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,sBAAsB,CAAC;AAEjE;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,6BAA6B,CAAC,MAAc,EAAE,KAAa;IACzE,OAAO;QACL,MAAM;QACN,EAAE;QACF,+EAA+E;QAC/E,6DAA6D,KAAK,EAAE;QACpE,kFAAkF;KACnF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,WAAW,CAAC,MAAc,EAAE,KAAa;IACvD,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,0BAA0B,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,iBAAiB,GAAsC;IAC3D,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,wCAAwC,EAAE;IACrE,EAAE,IAAI,EAAE,oCAAoC,EAAE,KAAK,EAAE,yBAAyB,EAAE;IAChF,EAAE,IAAI,EAAE,sCAAsC,EAAE,KAAK,EAAE,0BAA0B,EAAE;IACnF,EAAE,IAAI,EAAE,yBAAyB,EAAE,KAAK,EAAE,oDAAoD,EAAE;IAChG,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,iDAAiD,EAAE;IACrF,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,yBAAyB,EAAE;IACxD;QACE,IAAI,EAAE,+BAA+B;QACrC,KAAK,EAAE,4EAA4E;KACpF;IACD;QACE,IAAI,EAAE,qCAAqC;QAC3C,KAAK,EAAE,0DAA0D;KAClE;CACF,CAAC;AAEF,MAAM,UAAU,sBAAsB,CAAC,MAAc;IACnD,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,iBAAiB,EAAE,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAChE,CAAC;AACH,CAAC"}

package/dist/skill-test/grading-adapter.d.ts

@@ -17,6 +17,48 @@ export interface NormalizedGrading {
         passed: boolean;
         evidence?: string;
     }[];
+    /** Per-run integrity nonce the experimenter copied from its prompt (if present). */
+    runNonce?: string;
 }
+/**
+ * Thrown when grading.json's per-run integrity nonce is absent or does not match
+ * the nonce the harness stamped into the experimenter prompt for THIS run. A
+ * missing/wrong nonce means the grading was not produced by the experimenter we
+ * prompted — most likely a grading.json forged or left behind by untrusted skill
+ * code in the shared sandbox — so the verdict cannot be trusted.
+ */
+export declare class GradingNonceError extends Error {
+    constructor(message: string);
+}
+/**
+ * Assert grading.json carries the exact per-run nonce the harness expected.
+ * Called before the verdict is trusted. Throws {@link GradingNonceError} on a
+ * missing or mismatched nonce.
+ */
+export declare function assertGradingNonce(actual: string | undefined, expected: string): void;
 export declare function parseGradingJson(raw: unknown): NormalizedGrading;
+/** The authoritative eval verdict, recomputed from per-expectation `passed` flags. */
+export interface GradingVerdict {
+    passed: number;
+    total: number;
+    allPassed: boolean;
+}
+/**
+ * Reconcile the grader's self-reported `summary` against the authoritative
+ * per-expectation `passed` flags and return the recomputed verdict.
+ *
+ * The grader emits BOTH an aggregate `summary.{passed,total}` AND one
+ * `expectations[]` entry per graded expectation (each with its own `passed`).
+ * The schema (docs/skill-test-grading-schema.md) defines `summary` as the exact
+ * aggregate of `expectations`, so the two MUST agree. We never trust the
+ * self-reported summary alone — a grader that emits `summary {5,5}` alongside a
+ * failing expectation would otherwise be a false green.
+ *
+ * Throws {@link GradingSkewError} when:
+ *  - there are zero expectations (the grader graded NOTHING — an error, never a
+ *    pass); or
+ *  - the summary disagrees with the recomputed counts (a grader bug, surfaced
+ *    loudly rather than silently flowing a wrong verdict downstream).
+ */
+export declare function reconcileGrading(report: NormalizedGrading): GradingVerdict;
 //# sourceMappingURL=grading-adapter.d.ts.map

package/dist/skill-test/grading-adapter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"grading-adapter.d.ts","sourceRoot":"","sources":["../../src/skill-test/grading-adapter.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,KAAK;gBAC7B,OAAO,EAAE,MAAM;CAQ5B;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,YAAY,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACtE;AAaD,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,iBAAiB,CAuBhE"}
+{"version":3,"file":"grading-adapter.d.ts","sourceRoot":"","sources":["../../src/skill-test/grading-adapter.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,KAAK;gBAC7B,OAAO,EAAE,MAAM;CAQ5B;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,YAAY,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACrE,oFAAoF;IACpF,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;GAMG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC9B,OAAO,EAAE,MAAM;CAS5B;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAOrF;AAaD,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,iBAAiB,CAwBhE;AAED,sFAAsF;AACtF,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,iBAAiB,GAAG,cAAc,CAqB1E"}

package/dist/skill-test/grading-adapter.js

@@ -13,6 +13,35 @@ export class GradingSkewError extends Error {
         this.name = 'GradingSkewError';
     }
 }
+/**
+ * Thrown when grading.json's per-run integrity nonce is absent or does not match
+ * the nonce the harness stamped into the experimenter prompt for THIS run. A
+ * missing/wrong nonce means the grading was not produced by the experimenter we
+ * prompted — most likely a grading.json forged or left behind by untrusted skill
+ * code in the shared sandbox — so the verdict cannot be trusted.
+ */
+export class GradingNonceError extends Error {
+    constructor(message) {
+        super(`grading.json integrity check failed: ${message}. The harness stamps a secret ` +
+            'per-run nonce into the experimenter prompt (never written to disk) and requires ' +
+            'grading.json to echo it; a missing or wrong nonce means the grading was not ' +
+            'produced by the prompted experimenter and is rejected.');
+        this.name = 'GradingNonceError';
+    }
+}
+/**
+ * Assert grading.json carries the exact per-run nonce the harness expected.
+ * Called before the verdict is trusted. Throws {@link GradingNonceError} on a
+ * missing or mismatched nonce.
+ */
+export function assertGradingNonce(actual, expected) {
+    if (actual === undefined) {
+        throw new GradingNonceError('grading.json has no top-level `runNonce`');
+    }
+    if (actual !== expected) {
+        throw new GradingNonceError('grading.json `runNonce` does not match this run');
+    }
+}
 /**
  * Detect the common per-eval nested mistake: `{ evals: [ { expectations, ... } ] }`
  * with no top-level `expectations`. The grader (an LLM) reaches for this when the
@@ -36,7 +65,7 @@ export function parseGradingJson(raw) {
         const path = firstIssue?.path.join('.') ?? '(root)';
         throw new GradingSkewError(`missing/invalid field at "${path}" (${firstIssue?.message ?? 'unknown'})`);
     }
-    const { summary, expectations } = result.data;
+    const { summary, expectations, runNonce } = result.data;
     return {
         summary: { passed: summary.passed, total: summary.total },
         expectations: expectations.map(e => ({
@@ -44,6 +73,39 @@ export function parseGradingJson(raw) {
             passed: e.passed,
             ...(e.evidence === undefined ? {} : { evidence: e.evidence }),
         })),
+        ...(runNonce === undefined ? {} : { runNonce }),
     };
 }
+/**
+ * Reconcile the grader's self-reported `summary` against the authoritative
+ * per-expectation `passed` flags and return the recomputed verdict.
+ *
+ * The grader emits BOTH an aggregate `summary.{passed,total}` AND one
+ * `expectations[]` entry per graded expectation (each with its own `passed`).
+ * The schema (docs/skill-test-grading-schema.md) defines `summary` as the exact
+ * aggregate of `expectations`, so the two MUST agree. We never trust the
+ * self-reported summary alone — a grader that emits `summary {5,5}` alongside a
+ * failing expectation would otherwise be a false green.
+ *
+ * Throws {@link GradingSkewError} when:
+ *  - there are zero expectations (the grader graded NOTHING — an error, never a
+ *    pass); or
+ *  - the summary disagrees with the recomputed counts (a grader bug, surfaced
+ *    loudly rather than silently flowing a wrong verdict downstream).
+ */
+export function reconcileGrading(report) {
+    const computedPassed = report.expectations.filter(e => e.passed).length;
+    const computedTotal = report.expectations.length;
+    if (computedTotal === 0) {
+        throw new GradingSkewError('the grader recorded zero expectations — nothing was graded, so this can never be a pass. ' +
+            'A valid grading.json lists every graded expectation in top-level `expectations`');
+    }
+    const { passed, total } = report.summary;
+    if (passed !== computedPassed || total !== computedTotal) {
+        throw new GradingSkewError(`the grader's summary disagrees with its expectations: summary={passed:${passed},total:${total}} ` +
+            `but expectations recompute to {passed:${computedPassed},total:${computedTotal}}. ` +
+            '`summary` must be the exact aggregate of `expectations` (one entry per graded expectation)');
+    }
+    return { passed: computedPassed, total: computedTotal, allPassed: computedPassed === computedTotal };
+}
 //# sourceMappingURL=grading-adapter.js.map

package/dist/skill-test/grading-adapter.js.map

@@ -1 +1 @@
-{"version":3,"file":"grading-adapter.js","sourceRoot":"","sources":["../../src/skill-test/grading-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D;;;;;GAKG;AACH,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACzC,YAAY,OAAe;QACzB,KAAK,CACH,4BAA4B,OAAO,gDAAgD;YACjF,0EAA0E;YAC1E,yFAAyF,CAC5F,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAOD;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,GAAY;IACtC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC1D,MAAM,GAAG,GAAG,GAA8B,CAAC;IAC3C,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,IAAI,GAAG,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAY;IAC3C,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAClD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,IAAI,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,gBAAgB,CACxB,oFAAoF;gBAClF,kFAAkF;gBAClF,qCAAqC,CACxC,CAAC;QACJ,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC;QACpD,MAAM,IAAI,gBAAgB,CAAC,6BAA6B,IAAI,MAAM,UAAU,EAAE,OAAO,IAAI,SAAS,GAAG,CAAC,CAAC;IACzG,CAAC;IACD,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;IAC9C,OAAO;QACL,OAAO,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE;QACzD,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;SAC9D,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC"}
+{"version":3,"file":"grading-adapter.js","sourceRoot":"","sources":["../../src/skill-test/grading-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D;;;;;GAKG;AACH,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACzC,YAAY,OAAe;QACzB,KAAK,CACH,4BAA4B,OAAO,gDAAgD;YACjF,0EAA0E;YAC1E,yFAAyF,CAC5F,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AASD;;;;;;GAMG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAC1C,YAAY,OAAe;QACzB,KAAK,CACH,wCAAwC,OAAO,gCAAgC;YAC7E,kFAAkF;YAClF,8EAA8E;YAC9E,wDAAwD,CAC3D,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;IAClC,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAA0B,EAAE,QAAgB;IAC7E,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,MAAM,IAAI,iBAAiB,CAAC,0CAA0C,CAAC,CAAC;IAC1E,CAAC;IACD,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QACxB,MAAM,IAAI,iBAAiB,CAAC,iDAAiD,CAAC,CAAC;IACjF,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,GAAY;IACtC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC1D,MAAM,GAAG,GAAG,GAA8B,CAAC;IAC3C,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,IAAI,GAAG,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAY;IAC3C,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAClD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,IAAI,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,gBAAgB,CACxB,oFAAoF;gBAClF,kFAAkF;gBAClF,qCAAqC,CACxC,CAAC;QACJ,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC;QACpD,MAAM,IAAI,gBAAgB,CAAC,6BAA6B,IAAI,MAAM,UAAU,EAAE,OAAO,IAAI,SAAS,GAAG,CAAC,CAAC;IACzG,CAAC;IACD,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;IACxD,OAAO;QACL,OAAO,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE;QACzD,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;SAC9D,CAAC,CAAC;QACH,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;KAChD,CAAC;AACJ,CAAC;AASD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAyB;IACxD,MAAM,cAAc,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,aAAa,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC;IAEjD,IAAI,aAAa,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,gBAAgB,CACxB,2FAA2F;YACzF,iFAAiF,CACpF,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;IACzC,IAAI,MAAM,KAAK,cAAc,IAAI,KAAK,KAAK,aAAa,EAAE,CAAC;QACzD,MAAM,IAAI,gBAAgB,CACxB,yEAAyE,MAAM,UAAU,KAAK,IAAI;YAChG,yCAAyC,cAAc,UAAU,aAAa,KAAK;YACnF,4FAA4F,CAC/F,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,cAAc,KAAK,aAAa,EAAE,CAAC;AACvG,CAAC"}

package/dist/skill-test/grading-schema.d.ts

@@ -41,8 +41,12 @@ export declare const GradedExpectationSchema: z.ZodObject<{
     evidence: z.ZodOptional<z.ZodString>;
 }, z.ZodTypeAny, "passthrough">>;
 export type GradedExpectation = z.infer<typeof GradedExpectationSchema>;
-/** Aggregate pass/fail counts. `failed`/`pass_rate` are documented but optional. */
-export declare const GradingSummarySchema: z.ZodObject<{
+/**
+ * Aggregate pass/fail counts. `failed`/`pass_rate` are documented but optional.
+ * Counts are non-negative integers (a float or negative count is a grader bug),
+ * and `passed` can never exceed `total`.
+ */
+export declare const GradingSummarySchema: z.ZodEffects<z.ZodObject<{
     passed: z.ZodNumber;
     total: z.ZodNumber;
     failed: z.ZodOptional<z.ZodNumber>;
@@ -57,6 +61,16 @@ export declare const GradingSummarySchema: z.ZodObject<{
     total: z.ZodNumber;
     failed: z.ZodOptional<z.ZodNumber>;
     pass_rate: z.ZodOptional<z.ZodNumber>;
+}, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{
+    passed: z.ZodNumber;
+    total: z.ZodNumber;
+    failed: z.ZodOptional<z.ZodNumber>;
+    pass_rate: z.ZodOptional<z.ZodNumber>;
+}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+    passed: z.ZodNumber;
+    total: z.ZodNumber;
+    failed: z.ZodOptional<z.ZodNumber>;
+    pass_rate: z.ZodOptional<z.ZodNumber>;
 }, z.ZodTypeAny, "passthrough">>;
 export type GradingSummary = z.infer<typeof GradingSummarySchema>;
 /**
@@ -78,7 +92,7 @@ export declare const GradingReportSchema: z.ZodObject<{
         passed: z.ZodBoolean;
         evidence: z.ZodOptional<z.ZodString>;
     }, z.ZodTypeAny, "passthrough">>, "many">;
-    summary: z.ZodObject<{
+    summary: z.ZodEffects<z.ZodObject<{
         passed: z.ZodNumber;
         total: z.ZodNumber;
         failed: z.ZodOptional<z.ZodNumber>;
@@ -93,7 +107,25 @@ export declare const GradingReportSchema: z.ZodObject<{
         total: z.ZodNumber;
         failed: z.ZodOptional<z.ZodNumber>;
         pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
     }, z.ZodTypeAny, "passthrough">>;
+    /**
+     * Per-run integrity nonce the experimenter is told to copy verbatim from its
+     * prompt (see experimenter-prompt.ts). Optional in the schema (a grading.json
+     * validated by external tooling need not carry it), but the harness REQUIRES
+     * it to match the run's secret nonce before trusting the verdict — this is how
+     * a forged grading.json written by untrusted skill code is rejected.
+     */
+    runNonce: z.ZodOptional<z.ZodString>;
 }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
     expectations: z.ZodArray<z.ZodObject<{
         text: z.ZodString;
@@ -108,7 +140,7 @@ export declare const GradingReportSchema: z.ZodObject<{
         passed: z.ZodBoolean;
         evidence: z.ZodOptional<z.ZodString>;
     }, z.ZodTypeAny, "passthrough">>, "many">;
-    summary: z.ZodObject<{
+    summary: z.ZodEffects<z.ZodObject<{
         passed: z.ZodNumber;
         total: z.ZodNumber;
         failed: z.ZodOptional<z.ZodNumber>;
@@ -123,7 +155,25 @@ export declare const GradingReportSchema: z.ZodObject<{
         total: z.ZodNumber;
         failed: z.ZodOptional<z.ZodNumber>;
         pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
     }, z.ZodTypeAny, "passthrough">>;
+    /**
+     * Per-run integrity nonce the experimenter is told to copy verbatim from its
+     * prompt (see experimenter-prompt.ts). Optional in the schema (a grading.json
+     * validated by external tooling need not carry it), but the harness REQUIRES
+     * it to match the run's secret nonce before trusting the verdict — this is how
+     * a forged grading.json written by untrusted skill code is rejected.
+     */
+    runNonce: z.ZodOptional<z.ZodString>;
 }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
     expectations: z.ZodArray<z.ZodObject<{
         text: z.ZodString;
@@ -138,7 +188,7 @@ export declare const GradingReportSchema: z.ZodObject<{
         passed: z.ZodBoolean;
         evidence: z.ZodOptional<z.ZodString>;
     }, z.ZodTypeAny, "passthrough">>, "many">;
-    summary: z.ZodObject<{
+    summary: z.ZodEffects<z.ZodObject<{
         passed: z.ZodNumber;
         total: z.ZodNumber;
         failed: z.ZodOptional<z.ZodNumber>;
@@ -153,7 +203,25 @@ export declare const GradingReportSchema: z.ZodObject<{
         total: z.ZodNumber;
         failed: z.ZodOptional<z.ZodNumber>;
         pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
     }, z.ZodTypeAny, "passthrough">>;
+    /**
+     * Per-run integrity nonce the experimenter is told to copy verbatim from its
+     * prompt (see experimenter-prompt.ts). Optional in the schema (a grading.json
+     * validated by external tooling need not carry it), but the harness REQUIRES
+     * it to match the run's secret nonce before trusting the verdict — this is how
+     * a forged grading.json written by untrusted skill code is rejected.
+     */
+    runNonce: z.ZodOptional<z.ZodString>;
 }, z.ZodTypeAny, "passthrough">>;
 export type GradingReport = z.infer<typeof GradingReportSchema>;
 /**

package/dist/skill-test/grading-schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"grading-schema.d.ts","sourceRoot":"","sources":["../../src/skill-test/grading-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,sFAAsF;AACtF,eAAO,MAAM,uBAAuB;;;;;;;;;;;;gCAMpB,CAAC;AAEjB,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,oFAAoF;AACpF,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;gCAOjB,CAAC;AAEjB,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE;;;;GAIG;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAKhB,CAAC;AAEjB,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB;;;;;CAAyD,CAAC"}
+{"version":3,"file":"grading-schema.d.ts","sourceRoot":"","sources":["../../src/skill-test/grading-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,sFAAsF;AACtF,eAAO,MAAM,uBAAuB;;;;;;;;;;;;gCAMpB,CAAC;AAEjB,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE;;;;GAIG;AACH,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;gCAU7B,CAAC;AAEL,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE;;;;GAIG;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAI5B;;;;;;OAMG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IANH;;;;;;OAMG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IANH;;;;;;OAMG;;gCAGS,CAAC;AAEjB,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB;;;;;CAAyD,CAAC"}

package/dist/skill-test/grading-schema.js

@@ -35,15 +35,22 @@ export const GradedExpectationSchema = z
     evidence: z.string().optional(),
 })
     .passthrough();
-/** Aggregate pass/fail counts. `failed`/`pass_rate` are documented but optional. */
+/**
+ * Aggregate pass/fail counts. `failed`/`pass_rate` are documented but optional.
+ * Counts are non-negative integers (a float or negative count is a grader bug),
+ * and `passed` can never exceed `total`.
+ */
 export const GradingSummarySchema = z
     .object({
-    passed: z.number(),
-    total: z.number(),
-    failed: z.number().optional(),
+    passed: z.number().int().nonnegative(),
+    total: z.number().int().nonnegative(),
+    failed: z.number().int().nonnegative().optional(),
     pass_rate: z.number().optional(),
 })
-    .passthrough();
+    .passthrough()
+    .refine(s => s.passed <= s.total, {
+    message: 'summary.passed must not exceed summary.total',
+});
 /**
  * The full grading.json contract. Required: top-level `expectations[]` and
  * `summary`. Everything else passes through untouched (forward-compatible with
@@ -53,6 +60,14 @@ export const GradingReportSchema = z
     .object({
     expectations: z.array(GradedExpectationSchema),
     summary: GradingSummarySchema,
+    /**
+     * Per-run integrity nonce the experimenter is told to copy verbatim from its
+     * prompt (see experimenter-prompt.ts). Optional in the schema (a grading.json
+     * validated by external tooling need not carry it), but the harness REQUIRES
+     * it to match the run's secret nonce before trusting the verdict — this is how
+     * a forged grading.json written by untrusted skill code is rejected.
+     */
+    runNonce: z.string().optional(),
 })
     .passthrough();
 /**

package/dist/skill-test/grading-schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"grading-schema.js","sourceRoot":"","sources":["../../src/skill-test/grading-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,sFAAsF;AACtF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC;KACrC,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC;KACD,WAAW,EAAE,CAAC;AAIjB,oFAAoF;AACpF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC;KAClC,MAAM,CAAC;IACN,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACjC,CAAC;KACD,WAAW,EAAE,CAAC;AAIjB;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC;KACjC,MAAM,CAAC;IACN,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,uBAAuB,CAAC;IAC9C,OAAO,EAAE,oBAAoB;CAC9B,CAAC;KACD,WAAW,EAAE,CAAC;AAIjB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,eAAe,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,CAAC"}
+{"version":3,"file":"grading-schema.js","sourceRoot":"","sources":["../../src/skill-test/grading-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,sFAAsF;AACtF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC;KACrC,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC;KACD,WAAW,EAAE,CAAC;AAIjB;;;;GAIG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC;KAClC,MAAM,CAAC;IACN,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IACtC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IACrC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;IACjD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACjC,CAAC;KACD,WAAW,EAAE;KACb,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,EAAE;IAChC,OAAO,EAAE,8CAA8C;CACxD,CAAC,CAAC;AAIL;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC;KACjC,MAAM,CAAC;IACN,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,uBAAuB,CAAC;IAC9C,OAAO,EAAE,oBAAoB;IAC7B;;;;;;OAMG;IACH,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC;KACD,WAAW,EAAE,CAAC;AAIjB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,eAAe,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,CAAC"}

package/dist/skill-test/harness-location.d.ts

@@ -15,8 +15,16 @@ export declare function resolveHarnessRoot(skillNames: string[], tmpRoot?: strin
  * Refuse a --workdir whose ancestry contains CLAUDE.md or .claude/ — cwd
  * discovery would re-pollute the run (spec §7). Defense in depth with
  * --setting-sources "".
+ *
+ * `stopAt` (optional) bounds the ancestry walk: when the walk reaches that
+ * directory it stops *before* inspecting it (exclusive boundary), mirroring
+ * `assertSafeHarnessRoot`'s `trustedRoot`. Default (undefined) walks to the
+ * filesystem root, preserving the strictest production behavior. Tests use it
+ * to scope the walk to a sandbox — necessary on Windows, where the OS tmp dir
+ * lives inside the user's home and would otherwise surface an ambient
+ * ~/.claude during the walk.
  */
-export declare function assertSafeWorkdir(dir: string): void;
+export declare function assertSafeWorkdir(dir: string, stopAt?: string): void;
 /**
  * Prepare the harness root directory so that `assertSafeHarnessRoot` will
  * pass on the next call. If the path does not exist, this is a no-op (the
@@ -31,10 +39,20 @@ export declare function assertSafeWorkdir(dir: string): void;
  */
 export declare function prepareHarnessRoot(dir: string): void;
 /**
- * FS-bound hardening for the shared-tmp harness root (spec §7): the root must
- * be 0700 and owned by the current uid, and no path component may be a symlink.
+ * FS-bound hardening for the shared-tmp harness root (spec §7). Validates EVERY
+ * path component from the leaf up to (but excluding) `trustedRoot`: no component
+ * may be a symlink, and on POSIX each must be owned by the current uid. This
+ * closes the shared-/tmp TOCTOU where the recursively-created intermediate parent
+ * (`<tmp>/vat-skill-test`) — not just the leaf — could be pre-created as a symlink
+ * or under another user's ownership. The leaf must additionally be 0700.
+ *
+ * `trustedRoot` (default: the OS tmp dir) is the boundary: it is system-owned
+ * (sticky-bit /tmp) so ownership/mode checks are not applied to it. When the leaf
+ * is not a descendant of `trustedRoot` (an explicit --out elsewhere), validation
+ * degrades to the leaf alone.
+ *
  * Integration-tested (requires real lstat/stat). On Windows, uid checks are
  * skipped (process.getuid is undefined) but the symlink refusal still applies.
  */
-export declare function assertSafeHarnessRoot(dir: string, currentUid: number): void;
+export declare function assertSafeHarnessRoot(dir: string, currentUid: number, trustedRoot?: string): void;
 //# sourceMappingURL=harness-location.d.ts.map

package/dist/skill-test/harness-location.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"harness-location.d.ts","sourceRoot":"","sources":["../../src/skill-test/harness-location.ts"],"names":[],"mappings":"AAKA,sDAAsD;AACtD,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAOD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,MAAM,CAQ7D;AAED,mEAAmE;AACnE,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAGjF;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAenD;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAkBpD;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,CAiB3E"}
+{"version":3,"file":"harness-location.d.ts","sourceRoot":"","sources":["../../src/skill-test/harness-location.ts"],"names":[],"mappings":"AAKA,sDAAsD;AACtD,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAOD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,MAAM,CAQ7D;AAED,mEAAmE;AACnE,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAGjF;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAgBpE;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAkBpD;AAkDD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,qBAAqB,CACnC,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,EAClB,WAAW,GAAE,MAA2B,GACvC,IAAI,CAkBN"}

package/dist/skill-test/harness-location.js

@@ -1,6 +1,6 @@
 import { createHash } from 'node:crypto';
 import { chmodSync, existsSync, lstatSync, statSync } from 'node:fs';
-import { normalizedTmpdir, safePath } from '@vibe-agent-toolkit/utils';
+import { isAbsolutePath, normalizedTmpdir, safePath } from '@vibe-agent-toolkit/utils';
 /** Harness-location failure — maps to exit code 2. */
 export class HarnessLocationError extends Error {
     exitCode = 2;
@@ -36,11 +36,20 @@ export function resolveHarnessRoot(skillNames, tmpRoot) {
  * Refuse a --workdir whose ancestry contains CLAUDE.md or .claude/ — cwd
  * discovery would re-pollute the run (spec §7). Defense in depth with
  * --setting-sources "".
+ *
+ * `stopAt` (optional) bounds the ancestry walk: when the walk reaches that
+ * directory it stops *before* inspecting it (exclusive boundary), mirroring
+ * `assertSafeHarnessRoot`'s `trustedRoot`. Default (undefined) walks to the
+ * filesystem root, preserving the strictest production behavior. Tests use it
+ * to scope the walk to a sandbox — necessary on Windows, where the OS tmp dir
+ * lives inside the user's home and would otherwise surface an ambient
+ * ~/.claude during the walk.
  */
-export function assertSafeWorkdir(dir) {
+export function assertSafeWorkdir(dir, stopAt) {
+    const boundary = stopAt === undefined ? undefined : safePath.resolve(stopAt);
     let current = safePath.resolve(dir);
     let previous = '';
-    while (current !== previous) {
+    while (current !== previous && current !== boundary) {
         // eslint-disable-next-line security/detect-non-literal-fs-filename -- ancestry walk over a user-supplied workdir
         if (existsSync(safePath.join(current, 'CLAUDE.md'))) {
             throw new HarnessLocationError(`--workdir is inside a project: CLAUDE.md found at ${current}. Use an OS-tmp location.`);
@@ -83,29 +92,85 @@ export function prepareHarnessRoot(dir) {
         }
     }
 }
+/** True when `child` is a strict descendant of `root` (neither equal nor escaping). */
+function isStrictlyUnder(root, child) {
+    if (child === root)
+        return false;
+    const rel = safePath.relative(root, child);
+    return rel !== '' && rel !== '..' && !rel.startsWith('../') && !isAbsolutePath(rel);
+}
+/**
+ * The harness path components to validate, leaf-first: the leaf and every
+ * ancestor strictly between it and `trustedRoot` (exclusive). When the leaf is
+ * not a descendant of `trustedRoot` (e.g. an explicit --out elsewhere), only the
+ * leaf is returned — we cannot bound the walk without a trusted boundary, so we
+ * fall back to leaf-only validation.
+ */
+function harnessAncestry(leaf, trustedRoot) {
+    if (!isStrictlyUnder(trustedRoot, leaf))
+        return [leaf];
+    const components = [];
+    let current = leaf;
+    while (current !== trustedRoot) {
+        const parent = safePath.join(current, '..');
+        if (parent === current)
+            break; // reached the filesystem root defensively
+        components.push(current);
+        current = parent;
+    }
+    return components;
+}
 /**
- * FS-bound hardening for the shared-tmp harness root (spec §7): the root must
- * be 0700 and owned by the current uid, and no path component may be a symlink.
+ * Reject a single harness path component that is a symlink or not owned by the
+ * current user. A non-existent component is skipped (an absent ancestor cannot
+ * be a live attacker-controlled symlink). On Windows the ownership check is
+ * skipped (no real uids); the symlink refusal still applies.
+ */
+function assertComponentSafe(component, currentUid) {
+    // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived harness path component
+    if (!existsSync(component))
+        return;
+    // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived harness path component
+    const ls = lstatSync(component);
+    if (ls.isSymbolicLink()) {
+        throw new HarnessLocationError(`Refusing to use a symlinked harness path component: ${component}.`);
+    }
+    if (process.platform !== 'win32' && typeof ls.uid === 'number' && ls.uid !== currentUid) {
+        throw new HarnessLocationError(`Harness path component ${component} is not owned by the current user (uid ${ls.uid} != ${currentUid}).`);
+    }
+}
+/**
+ * FS-bound hardening for the shared-tmp harness root (spec §7). Validates EVERY
+ * path component from the leaf up to (but excluding) `trustedRoot`: no component
+ * may be a symlink, and on POSIX each must be owned by the current uid. This
+ * closes the shared-/tmp TOCTOU where the recursively-created intermediate parent
+ * (`<tmp>/vat-skill-test`) — not just the leaf — could be pre-created as a symlink
+ * or under another user's ownership. The leaf must additionally be 0700.
+ *
+ * `trustedRoot` (default: the OS tmp dir) is the boundary: it is system-owned
+ * (sticky-bit /tmp) so ownership/mode checks are not applied to it. When the leaf
+ * is not a descendant of `trustedRoot` (an explicit --out elsewhere), validation
+ * degrades to the leaf alone.
+ *
  * Integration-tested (requires real lstat/stat). On Windows, uid checks are
  * skipped (process.getuid is undefined) but the symlink refusal still applies.
  */
-export function assertSafeHarnessRoot(dir, currentUid) {
+export function assertSafeHarnessRoot(dir, currentUid, trustedRoot = normalizedTmpdir()) {
+    const resolved = safePath.resolve(dir);
+    // eslint-disable-next-line local/no-unsafe-root-join -- single-arg normalization of the trusted boundary itself (no caller-controlled segment is joined here); it is only used as an equality boundary for the ancestry walk.
+    const root = safePath.resolve(trustedRoot);
+    for (const component of harnessAncestry(resolved, root)) {
+        assertComponentSafe(component, currentUid);
+    }
+    // Leaf-only 0700 check (intermediates are created 0700 by recursive mkdir and
+    // need only the symlink/ownership guarantees above). Skipped on win32.
     // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived harness root
-    if (!existsSync(dir))
+    if (!existsSync(resolved))
         return; // not yet created — caller creates it 0700
     // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived harness root
-    const ls = lstatSync(dir);
-    if (ls.isSymbolicLink()) {
-        throw new HarnessLocationError(`Refusing to use a symlinked harness root: ${dir}.`);
-    }
-    // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived harness root
-    const st = statSync(dir);
-    if (typeof st.uid === 'number' && st.uid !== currentUid) {
-        throw new HarnessLocationError(`Harness root ${dir} is not owned by the current user (uid ${st.uid} != ${currentUid}).`);
-    }
-    const mode = st.mode & 0o777;
+    const mode = statSync(resolved).mode & 0o777;
     if (mode !== 0o700 && process.platform !== 'win32') {
-        throw new HarnessLocationError(`Harness root ${dir} must be 0700 (found ${mode.toString(8)}).`);
+        throw new HarnessLocationError(`Harness root ${resolved} must be 0700 (found ${mode.toString(8)}).`);
     }
 }
 //# sourceMappingURL=harness-location.js.map