@vibe-agent-toolkit/agent-skills 0.1.39-rc.8 → 0.1.39

package/dist/skill-test/exit-codes.js

@@ -1,6 +1,8 @@
 import { AuthPreflightError } from '@vibe-agent-toolkit/utils';
+import { BuildHookError } from './build-hook.js';
+import { UnknownEnvTokenError } from './declared-env.js';
 import { PromptInvariantError } from './experimenter-prompt.js';
-import { GradingSkewError } from './grading-adapter.js';
+import { GradingNonceError, GradingSkewError } from './grading-adapter.js';
 import { HarnessLocationError } from './harness-location.js';
 /** Exit codes for `vat skill test` (spec §6d). */
 export const SkillTestExitCode = {
@@ -8,21 +10,58 @@ export const SkillTestExitCode = {
     Internal: 1,
     Preflight: 2,
     Bootstrap: 3,
+    /**
+     * At least one eval FAILED and the caller opted into eval-gating via
+     * `--fail-on-eval-failure`. Distinct from the harness-broke codes (1/2):
+     * the harness ran to completion and produced a valid grading.json — the
+     * skill's expectations simply did not all pass. Returned DIRECTLY by the
+     * harness verdict (see verdictExitCode in run-harness.ts), not via
+     * mapErrorToExitCode, because it is an outcome, not a thrown error.
+     */
+    EvalFailure: 4,
 };
 /**
  * A required input is absent in a way vat can scaffold (missing evals.json).
- * Exit 3, NOT a failure. `expectedPath` is the real, persistent location where
- * the harness wrote the annotated starter template for the user to fill in.
+ * Exit 3, NOT a failure. `expectedPath` is the persistent location of the
+ * annotated starter template.
+ *
+ * In a real run the harness has already written that template, and the message
+ * says so. Under `--dry-run` nothing is written — a dry run must never touch the
+ * filesystem — so the message instead describes what a real run *would* scaffold
+ * and where, while surfacing the same exit-3 "bootstrap needed" signal.
  */
 export class BootstrapNeededError extends Error {
     expectedPath;
     exitCode = 3;
-    constructor(expectedPath) {
-        super(`Wrote an evals.json template at ${expectedPath} — fill it in and re-run.`);
+    constructor(expectedPath, opts) {
+        super(opts?.dryRun === true
+            ? `[dry-run] No evals.json found. A real run would scaffold an annotated template at ${expectedPath} — fill it in and re-run. (dry-run: nothing was written.)`
+            : `Wrote an evals.json template at ${expectedPath} — fill it in and re-run.`);
         this.expectedPath = expectedPath;
         this.name = 'BootstrapNeededError';
     }
 }
+/** Thrown when building a declared skill (pool packageSkill or plugin build) fails. Exit 2 (preflight class). */
+export class SkillBuildError extends Error {
+    exitCode = 2;
+    constructor(message) {
+        super(message);
+        this.name = 'SkillBuildError';
+    }
+}
+/**
+ * The §12 security acknowledgment is required but absent, thrown BEFORE any
+ * build/pre-stage command (which executes untrusted repo code) runs for a
+ * buildable subject. Exit 2 (preflight class). Mirrors the harness Step-6 ack
+ * message wording so the two enforcement points read identically.
+ */
+export class SecurityAckError extends Error {
+    exitCode = 2;
+    constructor() {
+        super('Security acknowledgment required. Pass --i-understand-this-runs-skill-code to proceed.');
+        this.name = 'SecurityAckError';
+    }
+}
 /** Internal harness failure (incl. experimenter exiting without valid grading.json). Exit 1. */
 export class InternalHarnessError extends Error {
     exitCode = 1;
@@ -35,19 +74,34 @@ export class InternalHarnessError extends Error {
  * Map any thrown error to the process exit code. Errors that carry their own
  * `exitCode` (Bootstrap/Auth/HarnessLocation/Internal) are authoritative;
  * a PromptInvariantError is a user-correctable preflight problem (a supplied
- * prompt override is missing a required safety instruction) → 2; GradingSkewError
- * is a parse failure → 1; everything unknown → 1.
+ * prompt override is missing a required safety instruction) → 2; a BuildHookError
+ * is a pre-stage build failure → 2; a SkillBuildError is a declared-skill build
+ * failure → 2; a SecurityAckError is a missing security ack before a build → 2;
+ * an UnknownEnvTokenError is a bad ${token} in a
+ * declared env value → 2; GradingSkewError (parse failure) and GradingNonceError
+ * (forged/mismatched grading nonce) are both → 1; everything unknown → 1.
+ *
+ * Note: SkillTestExitCode.EvalFailure (4) is NOT produced here — it is an
+ * outcome of a completed run (an eval failed under `--fail-on-eval-failure`),
+ * not a thrown error, so the harness returns it directly (see verdictExitCode).
  */
 export function mapErrorToExitCode(err) {
     if (err instanceof BootstrapNeededError)
         return SkillTestExitCode.Bootstrap;
     if (err instanceof AuthPreflightError ||
+        err instanceof BuildHookError ||
         err instanceof HarnessLocationError ||
-        err instanceof PromptInvariantError) {
+        err instanceof PromptInvariantError ||
+        err instanceof SecurityAckError ||
+        err instanceof SkillBuildError ||
+        err instanceof UnknownEnvTokenError) {
         return SkillTestExitCode.Preflight;
     }
-    if (err instanceof GradingSkewError || err instanceof InternalHarnessError)
+    if (err instanceof GradingNonceError ||
+        err instanceof GradingSkewError ||
+        err instanceof InternalHarnessError) {
         return SkillTestExitCode.Internal;
+    }
     return SkillTestExitCode.Internal;
 }
 //# sourceMappingURL=exit-codes.js.map

package/dist/skill-test/exit-codes.js.map

@@ -1 +1 @@
-{"version":3,"file":"exit-codes.js","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAE7D,kDAAkD;AAClD,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,EAAE,EAAE,CAAC;IACL,QAAQ,EAAE,CAAC;IACX,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,CAAC;CACJ,CAAC;AAIX;;;;GAIG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAEjB;IADnB,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAA4B,YAAoB;QAC9C,KAAK,CAAC,mCAAmC,YAAY,2BAA2B,CAAC,CAAC;QADxD,iBAAY,GAAZ,YAAY,CAAQ;QAE9C,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,gGAAgG;AAChG,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACpC,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAY;IAC7C,IAAI,GAAG,YAAY,oBAAoB;QAAE,OAAO,iBAAiB,CAAC,SAAS,CAAC;IAC5E,IACE,GAAG,YAAY,kBAAkB;QACjC,GAAG,YAAY,oBAAoB;QACnC,GAAG,YAAY,oBAAoB,EACnC,CAAC;QACD,OAAO,iBAAiB,CAAC,SAAS,CAAC;IACrC,CAAC;IACD,IAAI,GAAG,YAAY,gBAAgB,IAAI,GAAG,YAAY,oBAAoB;QAAE,OAAO,iBAAiB,CAAC,QAAQ,CAAC;IAC9G,OAAO,iBAAiB,CAAC,QAAQ,CAAC;AACpC,CAAC"}
+{"version":3,"file":"exit-codes.js","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAE7D,kDAAkD;AAClD,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,EAAE,EAAE,CAAC;IACL,QAAQ,EAAE,CAAC;IACX,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,CAAC;IACZ;;;;;;;OAOG;IACH,WAAW,EAAE,CAAC;CACN,CAAC;AAIX;;;;;;;;;GASG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAEjB;IADnB,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAA4B,YAAoB,EAAE,IAA2B;QAC3E,KAAK,CACH,IAAI,EAAE,MAAM,KAAK,IAAI;YACnB,CAAC,CAAC,qFAAqF,YAAY,2DAA2D;YAC9J,CAAC,CAAC,mCAAmC,YAAY,2BAA2B,CAC/E,CAAC;QALwB,iBAAY,GAAZ,YAAY,CAAQ;QAM9C,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,iHAAiH;AACjH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAC/B,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IAChC,QAAQ,GAAG,CAAU,CAAC;IAC/B;QACE,KAAK,CAAC,wFAAwF,CAAC,CAAC;QAChG,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAED,gGAAgG;AAChG,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACpC,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAY;IAC7C,IAAI,GAAG,YAAY,oBAAoB;QAAE,OAAO,iBAAiB,CAAC,SAAS,CAAC;IAC5E,IACE,GAAG,YAAY,kBAAkB;QACjC,GAAG,YAAY,cAAc;QAC7B,GAAG,YAAY,oBAAoB;QACnC,GAAG,YAAY,oBAAoB;QACnC,GAAG,YAAY,gBAAgB;QAC/B,GAAG,YAAY,eAAe;QAC9B,GAAG,YAAY,oBAAoB,EACnC,CAAC;QACD,OAAO,iBAAiB,CAAC,SAAS,CAAC;IACrC,CAAC;IACD,IACE,GAAG,YAAY,iBAAiB;QAChC,GAAG,YAAY,gBAAgB;QAC/B,GAAG,YAAY,oBAAoB,EACnC,CAAC;QACD,OAAO,iBAAiB,CAAC,QAAQ,CAAC;IACpC,CAAC;IACD,OAAO,iBAAiB,CAAC,QAAQ,CAAC;AACpC,CAAC"}

package/dist/skill-test/experimenter-prompt.d.ts

@@ -6,6 +6,7 @@ export interface BuildPromptOptions {
     evalsPath: string;
     gradingOut: string;
     frictionOut: string;
+    workspacesRoot: string;
     baseline: boolean;
 }
 /**
@@ -16,5 +17,26 @@ export interface BuildPromptOptions {
  */
 export declare const DEFAULT_EXPERIMENTER_PROMPT: string;
 export declare function buildExperimenterPrompt(opts: BuildPromptOptions): string;
+/**
+ * Sentinel the persisted (audit) copy of the prompt shows in place of the real
+ * nonce, so the on-disk artifact never leaks the secret to skill code.
+ */
+export declare const REDACTED_NONCE_PLACEHOLDER = "<run-nonce redacted>";
+/**
+ * Append the per-run integrity-nonce directive to the effective prompt. The
+ * experimenter must copy `nonce` verbatim into grading.json's top-level
+ * `runNonce` field; the harness then rejects any grading.json whose nonce is
+ * absent or wrong (see run-harness.ts). This is defense-in-depth against
+ * untrusted skill code forging a passing grading.json in the shared sandbox —
+ * without the secret nonce (delivered only via stdin, never written to disk) a
+ * forged grading cannot be authenticated.
+ *
+ * Appended AFTER assertPromptInvariants and AFTER any user `experimenterPrompt`
+ * override, so the nonce requirement is ALWAYS enforced and a committed config
+ * cannot opt out of it.
+ */
+export declare function appendIntegrityNonceDirective(prompt: string, nonce: string): string;
+/** Redact the per-run nonce from a prompt for the on-disk audit copy. */
+export declare function redactNonce(prompt: string, nonce: string): string;
 export declare function assertPromptInvariants(prompt: string): void;
 //# sourceMappingURL=experimenter-prompt.d.ts.map

package/dist/skill-test/experimenter-prompt.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"experimenter-prompt.d.ts","sourceRoot":"","sources":["../../src/skill-test/experimenter-prompt.ts"],"names":[],"mappings":"AAAA,qBAAa,oBAAqB,SAAQ,KAAK;gBACjC,OAAO,EAAE,MAAM;CAI5B;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B,QAsB5B,CAAC;AAKb,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,kBAAkB,GAAG,MAAM,CAOxE;AAmBD,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAI3D"}
+{"version":3,"file":"experimenter-prompt.d.ts","sourceRoot":"","sources":["../../src/skill-test/experimenter-prompt.ts"],"names":[],"mappings":"AAAA,qBAAa,oBAAqB,SAAQ,KAAK;gBACjC,OAAO,EAAE,MAAM;CAI5B;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B,QA2B5B,CAAC;AAKb,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,kBAAkB,GAAG,MAAM,CAQxE;AAED;;;GAGG;AACH,eAAO,MAAM,0BAA0B,yBAAyB,CAAC;AAEjE;;;;;;;;;;;;GAYG;AACH,wBAAgB,6BAA6B,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAQnF;AAED,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAEjE;AAmBD,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAI3D"}

package/dist/skill-test/experimenter-prompt.js

@@ -15,11 +15,16 @@ export const DEFAULT_EXPERIMENTER_PROMPT = [
     '',
     'For each eval in {{EVALS_PATH}}:',
     '  1. Dispatch ONE executor subagent. Tell it ONLY the task prompt and the staged subject path {{SUBJECT_PATH}}.',
-    '     Never tell the executor it is being tested.',
+    '     When the eval declares input `files`, ALSO tell the executor its working directory is {{WORKSPACES_ROOT}}/<id>',
+    '     (substitute the eval\'s own id) — the project it should operate on. Never tell the executor it is being tested.',
     '  2. Grade the executor output against the eval\'s `expectations` using skill-creator\'s grader.md rubric.',
+    '     If the eval includes an `expected_output`, treat it as the author\'s prose description of a',
+    '     correct result and use it as CONTEXT when judging — but the pass/fail verdict is still decided',
+    '     per `expectations` entry; `expected_output` informs your judgment, it is not itself a checklist item.',
     '  3. Append each graded expectation to the SINGLE top-level `expectations` array in {{GRADING_OUT}} IMMEDIATELY',
     '     (incremental flush — a mid-run kill must leave partial results).',
-    '  4. Record any packaging-fidelity friction to {{FRICTION_OUT}} using the vat friction schema.',
+    '  4. Record any packaging-fidelity friction to {{FRICTION_OUT}} using the vat friction.json schema.',
+    '     If a file referenced by the skill is absent from the staged tree, record a `missing-bundled-file` friction entry.',
     '',
     '{{GRADING_OUT}} MUST be ONE flat JSON object in skill-creator\'s grading.json shape (references/schemas.md):',
     'a top-level `expectations` array — one entry {"text","passed","evidence"} per expectation across ALL evals —',
@@ -38,10 +43,42 @@ export function buildExperimenterPrompt(opts) {
     return DEFAULT_EXPERIMENTER_PROMPT
         .replace('{{EVALS_PATH}}', opts.evalsPath)
         .replace('{{SUBJECT_PATH}}', opts.subjectPath)
+        .replaceAll('{{WORKSPACES_ROOT}}', opts.workspacesRoot)
         .replaceAll('{{GRADING_OUT}}', opts.gradingOut)
         .replace('{{FRICTION_OUT}}', opts.frictionOut)
         .replace('{{BASELINE_BLOCK}}', opts.baseline ? BASELINE_BLOCK : '');
 }
+/**
+ * Sentinel the persisted (audit) copy of the prompt shows in place of the real
+ * nonce, so the on-disk artifact never leaks the secret to skill code.
+ */
+export const REDACTED_NONCE_PLACEHOLDER = '<run-nonce redacted>';
+/**
+ * Append the per-run integrity-nonce directive to the effective prompt. The
+ * experimenter must copy `nonce` verbatim into grading.json's top-level
+ * `runNonce` field; the harness then rejects any grading.json whose nonce is
+ * absent or wrong (see run-harness.ts). This is defense-in-depth against
+ * untrusted skill code forging a passing grading.json in the shared sandbox —
+ * without the secret nonce (delivered only via stdin, never written to disk) a
+ * forged grading cannot be authenticated.
+ *
+ * Appended AFTER assertPromptInvariants and AFTER any user `experimenterPrompt`
+ * override, so the nonce requirement is ALWAYS enforced and a committed config
+ * cannot opt out of it.
+ */
+export function appendIntegrityNonceDirective(prompt, nonce) {
+    return [
+        prompt,
+        '',
+        'INTEGRITY (required, do this LAST): the grading.json you write MUST include a',
+        `top-level string field "runNonce" whose value is EXACTLY: ${nonce}`,
+        'Copy it verbatim. vat rejects a grading.json whose runNonce is missing or wrong.',
+    ].join('\n');
+}
+/** Redact the per-run nonce from a prompt for the on-disk audit copy. */
+export function redactNonce(prompt, nonce) {
+    return prompt.replaceAll(nonce, REDACTED_NONCE_PLACEHOLDER);
+}
 const REQUIRED_PATTERNS = [
     { test: /\bSTOP\b/, label: 'must instruct the experimenter to STOP' },
     { test: /grading\.json|\{\{GRADING_OUT\}\}/i, label: 'must write grading.json' },

package/dist/skill-test/experimenter-prompt.js.map

@@ -1 +1 @@
-{"version":3,"file":"experimenter-prompt.js","sourceRoot":"","sources":["../../src/skill-test/experimenter-prompt.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC7C,YAAY,OAAe;QACzB,KAAK,CAAC,2CAA2C,OAAO,EAAE,CAAC,CAAC;QAC5D,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAUD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG;IACzC,yGAAyG;IACzG,EAAE;IACF,kCAAkC;IAClC,iHAAiH;IACjH,kDAAkD;IAClD,4GAA4G;IAC5G,iHAAiH;IACjH,uEAAuE;IACvE,gGAAgG;IAChG,EAAE;IACF,8GAA8G;IAC9G,8GAA8G;IAC9G,gHAAgH;IAChH,wEAAwE;IACxE,qGAAqG;IACrG,EAAE;IACF,mFAAmF;IACnF,EAAE;IACF,6GAA6G;IAC7G,4FAA4F;IAC5F,oBAAoB;CACrB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,MAAM,cAAc,GAClB,sIAAsI,CAAC;AAEzI,MAAM,UAAU,uBAAuB,CAAC,IAAwB;IAC9D,OAAO,2BAA2B;SAC/B,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC;SACzC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,WAAW,CAAC;SAC7C,UAAU,CAAC,iBAAiB,EAAE,IAAI,CAAC,UAAU,CAAC;SAC9C,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,WAAW,CAAC;SAC7C,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,iBAAiB,GAAsC;IAC3D,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,wCAAwC,EAAE;IACrE,EAAE,IAAI,EAAE,oCAAoC,EAAE,KAAK,EAAE,yBAAyB,EAAE;IAChF,EAAE,IAAI,EAAE,sCAAsC,EAAE,KAAK,EAAE,0BAA0B,EAAE;IACnF,EAAE,IAAI,EAAE,yBAAyB,EAAE,KAAK,EAAE,oDAAoD,EAAE;IAChG,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,iDAAiD,EAAE;IACrF,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,yBAAyB,EAAE;IACxD;QACE,IAAI,EAAE,+BAA+B;QACrC,KAAK,EAAE,4EAA4E;KACpF;IACD;QACE,IAAI,EAAE,qCAAqC;QAC3C,KAAK,EAAE,0DAA0D;KAClE;CACF,CAAC;AAEF,MAAM,UAAU,sBAAsB,CAAC,MAAc;IACnD,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,iBAAiB,EAAE,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAChE,CAAC;AACH,CAAC"}
+{"version":3,"file":"experimenter-prompt.js","sourceRoot":"","sources":["../../src/skill-test/experimenter-prompt.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC7C,YAAY,OAAe;QACzB,KAAK,CAAC,2CAA2C,OAAO,EAAE,CAAC,CAAC;QAC5D,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAWD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG;IACzC,yGAAyG;IACzG,EAAE;IACF,kCAAkC;IAClC,iHAAiH;IACjH,qHAAqH;IACrH,sHAAsH;IACtH,4GAA4G;IAC5G,kGAAkG;IAClG,qGAAqG;IACrG,4GAA4G;IAC5G,iHAAiH;IACjH,uEAAuE;IACvE,qGAAqG;IACrG,wHAAwH;IACxH,EAAE;IACF,8GAA8G;IAC9G,8GAA8G;IAC9G,gHAAgH;IAChH,wEAAwE;IACxE,qGAAqG;IACrG,EAAE;IACF,mFAAmF;IACnF,EAAE;IACF,6GAA6G;IAC7G,4FAA4F;IAC5F,oBAAoB;CACrB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,MAAM,cAAc,GAClB,sIAAsI,CAAC;AAEzI,MAAM,UAAU,uBAAuB,CAAC,IAAwB;IAC9D,OAAO,2BAA2B;SAC/B,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC;SACzC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,WAAW,CAAC;SAC7C,UAAU,CAAC,qBAAqB,EAAE,IAAI,CAAC,cAAc,CAAC;SACtD,UAAU,CAAC,iBAAiB,EAAE,IAAI,CAAC,UAAU,CAAC;SAC9C,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,WAAW,CAAC;SAC7C,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACxE,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,sBAAsB,CAAC;AAEjE;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,6BAA6B,CAAC,MAAc,EAAE,KAAa;IACzE,OAAO;QACL,MAAM;QACN,EAAE;QACF,+EAA+E;QAC/E,6DAA6D,KAAK,EAAE;QACpE,kFAAkF;KACnF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,WAAW,CAAC,MAAc,EAAE,KAAa;IACvD,OAAO,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,0BAA0B,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,iBAAiB,GAAsC;IAC3D,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,wCAAwC,EAAE;IACrE,EAAE,IAAI,EAAE,oCAAoC,EAAE,KAAK,EAAE,yBAAyB,EAAE;IAChF,EAAE,IAAI,EAAE,sCAAsC,EAAE,KAAK,EAAE,0BAA0B,EAAE;IACnF,EAAE,IAAI,EAAE,yBAAyB,EAAE,KAAK,EAAE,oDAAoD,EAAE;IAChG,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,iDAAiD,EAAE;IACrF,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,yBAAyB,EAAE;IACxD;QACE,IAAI,EAAE,+BAA+B;QACrC,KAAK,EAAE,4EAA4E;KACpF;IACD;QACE,IAAI,EAAE,qCAAqC;QAC3C,KAAK,EAAE,0DAA0D;KAClE;CACF,CAAC;AAEF,MAAM,UAAU,sBAAsB,CAAC,MAAc;IACnD,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,iBAAiB,EAAE,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAChE,CAAC;AACH,CAAC"}

package/dist/skill-test/grading-adapter.d.ts

@@ -17,6 +17,48 @@ export interface NormalizedGrading {
         passed: boolean;
         evidence?: string;
     }[];
+    /** Per-run integrity nonce the experimenter copied from its prompt (if present). */
+    runNonce?: string;
 }
+/**
+ * Thrown when grading.json's per-run integrity nonce is absent or does not match
+ * the nonce the harness stamped into the experimenter prompt for THIS run. A
+ * missing/wrong nonce means the grading was not produced by the experimenter we
+ * prompted — most likely a grading.json forged or left behind by untrusted skill
+ * code in the shared sandbox — so the verdict cannot be trusted.
+ */
+export declare class GradingNonceError extends Error {
+    constructor(message: string);
+}
+/**
+ * Assert grading.json carries the exact per-run nonce the harness expected.
+ * Called before the verdict is trusted. Throws {@link GradingNonceError} on a
+ * missing or mismatched nonce.
+ */
+export declare function assertGradingNonce(actual: string | undefined, expected: string): void;
 export declare function parseGradingJson(raw: unknown): NormalizedGrading;
+/** The authoritative eval verdict, recomputed from per-expectation `passed` flags. */
+export interface GradingVerdict {
+    passed: number;
+    total: number;
+    allPassed: boolean;
+}
+/**
+ * Reconcile the grader's self-reported `summary` against the authoritative
+ * per-expectation `passed` flags and return the recomputed verdict.
+ *
+ * The grader emits BOTH an aggregate `summary.{passed,total}` AND one
+ * `expectations[]` entry per graded expectation (each with its own `passed`).
+ * The schema (docs/skill-test-grading-schema.md) defines `summary` as the exact
+ * aggregate of `expectations`, so the two MUST agree. We never trust the
+ * self-reported summary alone — a grader that emits `summary {5,5}` alongside a
+ * failing expectation would otherwise be a false green.
+ *
+ * Throws {@link GradingSkewError} when:
+ *  - there are zero expectations (the grader graded NOTHING — an error, never a
+ *    pass); or
+ *  - the summary disagrees with the recomputed counts (a grader bug, surfaced
+ *    loudly rather than silently flowing a wrong verdict downstream).
+ */
+export declare function reconcileGrading(report: NormalizedGrading): GradingVerdict;
 //# sourceMappingURL=grading-adapter.d.ts.map

package/dist/skill-test/grading-adapter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"grading-adapter.d.ts","sourceRoot":"","sources":["../../src/skill-test/grading-adapter.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,KAAK;gBAC7B,OAAO,EAAE,MAAM;CAQ5B;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,YAAY,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACtE;AAaD,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,iBAAiB,CAuBhE"}
+{"version":3,"file":"grading-adapter.d.ts","sourceRoot":"","sources":["../../src/skill-test/grading-adapter.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,KAAK;gBAC7B,OAAO,EAAE,MAAM;CAQ5B;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,YAAY,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACrE,oFAAoF;IACpF,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;GAMG;AACH,qBAAa,iBAAkB,SAAQ,KAAK;gBAC9B,OAAO,EAAE,MAAM;CAS5B;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAOrF;AAaD,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,iBAAiB,CAwBhE;AAED,sFAAsF;AACtF,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,iBAAiB,GAAG,cAAc,CAqB1E"}

package/dist/skill-test/grading-adapter.js

@@ -13,6 +13,35 @@ export class GradingSkewError extends Error {
         this.name = 'GradingSkewError';
     }
 }
+/**
+ * Thrown when grading.json's per-run integrity nonce is absent or does not match
+ * the nonce the harness stamped into the experimenter prompt for THIS run. A
+ * missing/wrong nonce means the grading was not produced by the experimenter we
+ * prompted — most likely a grading.json forged or left behind by untrusted skill
+ * code in the shared sandbox — so the verdict cannot be trusted.
+ */
+export class GradingNonceError extends Error {
+    constructor(message) {
+        super(`grading.json integrity check failed: ${message}. The harness stamps a secret ` +
+            'per-run nonce into the experimenter prompt (never written to disk) and requires ' +
+            'grading.json to echo it; a missing or wrong nonce means the grading was not ' +
+            'produced by the prompted experimenter and is rejected.');
+        this.name = 'GradingNonceError';
+    }
+}
+/**
+ * Assert grading.json carries the exact per-run nonce the harness expected.
+ * Called before the verdict is trusted. Throws {@link GradingNonceError} on a
+ * missing or mismatched nonce.
+ */
+export function assertGradingNonce(actual, expected) {
+    if (actual === undefined) {
+        throw new GradingNonceError('grading.json has no top-level `runNonce`');
+    }
+    if (actual !== expected) {
+        throw new GradingNonceError('grading.json `runNonce` does not match this run');
+    }
+}
 /**
  * Detect the common per-eval nested mistake: `{ evals: [ { expectations, ... } ] }`
  * with no top-level `expectations`. The grader (an LLM) reaches for this when the
@@ -36,7 +65,7 @@ export function parseGradingJson(raw) {
         const path = firstIssue?.path.join('.') ?? '(root)';
         throw new GradingSkewError(`missing/invalid field at "${path}" (${firstIssue?.message ?? 'unknown'})`);
     }
-    const { summary, expectations } = result.data;
+    const { summary, expectations, runNonce } = result.data;
     return {
         summary: { passed: summary.passed, total: summary.total },
         expectations: expectations.map(e => ({
@@ -44,6 +73,39 @@ export function parseGradingJson(raw) {
             passed: e.passed,
             ...(e.evidence === undefined ? {} : { evidence: e.evidence }),
         })),
+        ...(runNonce === undefined ? {} : { runNonce }),
     };
 }
+/**
+ * Reconcile the grader's self-reported `summary` against the authoritative
+ * per-expectation `passed` flags and return the recomputed verdict.
+ *
+ * The grader emits BOTH an aggregate `summary.{passed,total}` AND one
+ * `expectations[]` entry per graded expectation (each with its own `passed`).
+ * The schema (docs/skill-test-grading-schema.md) defines `summary` as the exact
+ * aggregate of `expectations`, so the two MUST agree. We never trust the
+ * self-reported summary alone — a grader that emits `summary {5,5}` alongside a
+ * failing expectation would otherwise be a false green.
+ *
+ * Throws {@link GradingSkewError} when:
+ *  - there are zero expectations (the grader graded NOTHING — an error, never a
+ *    pass); or
+ *  - the summary disagrees with the recomputed counts (a grader bug, surfaced
+ *    loudly rather than silently flowing a wrong verdict downstream).
+ */
+export function reconcileGrading(report) {
+    const computedPassed = report.expectations.filter(e => e.passed).length;
+    const computedTotal = report.expectations.length;
+    if (computedTotal === 0) {
+        throw new GradingSkewError('the grader recorded zero expectations — nothing was graded, so this can never be a pass. ' +
+            'A valid grading.json lists every graded expectation in top-level `expectations`');
+    }
+    const { passed, total } = report.summary;
+    if (passed !== computedPassed || total !== computedTotal) {
+        throw new GradingSkewError(`the grader's summary disagrees with its expectations: summary={passed:${passed},total:${total}} ` +
+            `but expectations recompute to {passed:${computedPassed},total:${computedTotal}}. ` +
+            '`summary` must be the exact aggregate of `expectations` (one entry per graded expectation)');
+    }
+    return { passed: computedPassed, total: computedTotal, allPassed: computedPassed === computedTotal };
+}
 //# sourceMappingURL=grading-adapter.js.map

package/dist/skill-test/grading-adapter.js.map

@@ -1 +1 @@
-{"version":3,"file":"grading-adapter.js","sourceRoot":"","sources":["../../src/skill-test/grading-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D;;;;;GAKG;AACH,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACzC,YAAY,OAAe;QACzB,KAAK,CACH,4BAA4B,OAAO,gDAAgD;YACjF,0EAA0E;YAC1E,yFAAyF,CAC5F,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAOD;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,GAAY;IACtC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC1D,MAAM,GAAG,GAAG,GAA8B,CAAC;IAC3C,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,IAAI,GAAG,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAY;IAC3C,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAClD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,IAAI,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,gBAAgB,CACxB,oFAAoF;gBAClF,kFAAkF;gBAClF,qCAAqC,CACxC,CAAC;QACJ,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC;QACpD,MAAM,IAAI,gBAAgB,CAAC,6BAA6B,IAAI,MAAM,UAAU,EAAE,OAAO,IAAI,SAAS,GAAG,CAAC,CAAC;IACzG,CAAC;IACD,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;IAC9C,OAAO;QACL,OAAO,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE;QACzD,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;SAC9D,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC"}
+{"version":3,"file":"grading-adapter.js","sourceRoot":"","sources":["../../src/skill-test/grading-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D;;;;;GAKG;AACH,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACzC,YAAY,OAAe;QACzB,KAAK,CACH,4BAA4B,OAAO,gDAAgD;YACjF,0EAA0E;YAC1E,yFAAyF,CAC5F,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AASD;;;;;;GAMG;AACH,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAC1C,YAAY,OAAe;QACzB,KAAK,CACH,wCAAwC,OAAO,gCAAgC;YAC7E,kFAAkF;YAClF,8EAA8E;YAC9E,wDAAwD,CAC3D,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;IAClC,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAA0B,EAAE,QAAgB;IAC7E,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,MAAM,IAAI,iBAAiB,CAAC,0CAA0C,CAAC,CAAC;IAC1E,CAAC;IACD,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QACxB,MAAM,IAAI,iBAAiB,CAAC,iDAAiD,CAAC,CAAC;IACjF,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,GAAY;IACtC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC1D,MAAM,GAAG,GAAG,GAA8B,CAAC;IAC3C,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,IAAI,GAAG,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAY;IAC3C,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAClD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,IAAI,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,gBAAgB,CACxB,oFAAoF;gBAClF,kFAAkF;gBAClF,qCAAqC,CACxC,CAAC;QACJ,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC;QACpD,MAAM,IAAI,gBAAgB,CAAC,6BAA6B,IAAI,MAAM,UAAU,EAAE,OAAO,IAAI,SAAS,GAAG,CAAC,CAAC;IACzG,CAAC;IACD,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;IACxD,OAAO;QACL,OAAO,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE;QACzD,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;SAC9D,CAAC,CAAC;QACH,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;KAChD,CAAC;AACJ,CAAC;AASD;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAyB;IACxD,MAAM,cAAc,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACxE,MAAM,aAAa,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC;IAEjD,IAAI,aAAa,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,gBAAgB,CACxB,2FAA2F;YACzF,iFAAiF,CACpF,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;IACzC,IAAI,MAAM,KAAK,cAAc,IAAI,KAAK,KAAK,aAAa,EAAE,CAAC;QACzD,MAAM,IAAI,gBAAgB,CACxB,yEAAyE,MAAM,UAAU,KAAK,IAAI;YAChG,yCAAyC,cAAc,UAAU,aAAa,KAAK;YACnF,4FAA4F,CAC/F,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,cAAc,KAAK,aAAa,EAAE,CAAC;AACvG,CAAC"}

package/dist/skill-test/grading-schema.d.ts

@@ -41,8 +41,12 @@ export declare const GradedExpectationSchema: z.ZodObject<{
     evidence: z.ZodOptional<z.ZodString>;
 }, z.ZodTypeAny, "passthrough">>;
 export type GradedExpectation = z.infer<typeof GradedExpectationSchema>;
-/** Aggregate pass/fail counts. `failed`/`pass_rate` are documented but optional. */
-export declare const GradingSummarySchema: z.ZodObject<{
+/**
+ * Aggregate pass/fail counts. `failed`/`pass_rate` are documented but optional.
+ * Counts are non-negative integers (a float or negative count is a grader bug),
+ * and `passed` can never exceed `total`.
+ */
+export declare const GradingSummarySchema: z.ZodEffects<z.ZodObject<{
     passed: z.ZodNumber;
     total: z.ZodNumber;
     failed: z.ZodOptional<z.ZodNumber>;
@@ -57,6 +61,16 @@ export declare const GradingSummarySchema: z.ZodObject<{
     total: z.ZodNumber;
     failed: z.ZodOptional<z.ZodNumber>;
     pass_rate: z.ZodOptional<z.ZodNumber>;
+}, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{
+    passed: z.ZodNumber;
+    total: z.ZodNumber;
+    failed: z.ZodOptional<z.ZodNumber>;
+    pass_rate: z.ZodOptional<z.ZodNumber>;
+}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+    passed: z.ZodNumber;
+    total: z.ZodNumber;
+    failed: z.ZodOptional<z.ZodNumber>;
+    pass_rate: z.ZodOptional<z.ZodNumber>;
 }, z.ZodTypeAny, "passthrough">>;
 export type GradingSummary = z.infer<typeof GradingSummarySchema>;
 /**
@@ -78,7 +92,7 @@ export declare const GradingReportSchema: z.ZodObject<{
         passed: z.ZodBoolean;
         evidence: z.ZodOptional<z.ZodString>;
     }, z.ZodTypeAny, "passthrough">>, "many">;
-    summary: z.ZodObject<{
+    summary: z.ZodEffects<z.ZodObject<{
         passed: z.ZodNumber;
         total: z.ZodNumber;
         failed: z.ZodOptional<z.ZodNumber>;
@@ -93,7 +107,25 @@ export declare const GradingReportSchema: z.ZodObject<{
         total: z.ZodNumber;
         failed: z.ZodOptional<z.ZodNumber>;
         pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
     }, z.ZodTypeAny, "passthrough">>;
+    /**
+     * Per-run integrity nonce the experimenter is told to copy verbatim from its
+     * prompt (see experimenter-prompt.ts). Optional in the schema (a grading.json
+     * validated by external tooling need not carry it), but the harness REQUIRES
+     * it to match the run's secret nonce before trusting the verdict — this is how
+     * a forged grading.json written by untrusted skill code is rejected.
+     */
+    runNonce: z.ZodOptional<z.ZodString>;
 }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
     expectations: z.ZodArray<z.ZodObject<{
         text: z.ZodString;
@@ -108,7 +140,7 @@ export declare const GradingReportSchema: z.ZodObject<{
         passed: z.ZodBoolean;
         evidence: z.ZodOptional<z.ZodString>;
     }, z.ZodTypeAny, "passthrough">>, "many">;
-    summary: z.ZodObject<{
+    summary: z.ZodEffects<z.ZodObject<{
         passed: z.ZodNumber;
         total: z.ZodNumber;
         failed: z.ZodOptional<z.ZodNumber>;
@@ -123,7 +155,25 @@ export declare const GradingReportSchema: z.ZodObject<{
         total: z.ZodNumber;
         failed: z.ZodOptional<z.ZodNumber>;
         pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
     }, z.ZodTypeAny, "passthrough">>;
+    /**
+     * Per-run integrity nonce the experimenter is told to copy verbatim from its
+     * prompt (see experimenter-prompt.ts). Optional in the schema (a grading.json
+     * validated by external tooling need not carry it), but the harness REQUIRES
+     * it to match the run's secret nonce before trusting the verdict — this is how
+     * a forged grading.json written by untrusted skill code is rejected.
+     */
+    runNonce: z.ZodOptional<z.ZodString>;
 }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
     expectations: z.ZodArray<z.ZodObject<{
         text: z.ZodString;
@@ -138,7 +188,7 @@ export declare const GradingReportSchema: z.ZodObject<{
         passed: z.ZodBoolean;
         evidence: z.ZodOptional<z.ZodString>;
     }, z.ZodTypeAny, "passthrough">>, "many">;
-    summary: z.ZodObject<{
+    summary: z.ZodEffects<z.ZodObject<{
         passed: z.ZodNumber;
         total: z.ZodNumber;
         failed: z.ZodOptional<z.ZodNumber>;
@@ -153,7 +203,25 @@ export declare const GradingReportSchema: z.ZodObject<{
         total: z.ZodNumber;
         failed: z.ZodOptional<z.ZodNumber>;
         pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
     }, z.ZodTypeAny, "passthrough">>;
+    /**
+     * Per-run integrity nonce the experimenter is told to copy verbatim from its
+     * prompt (see experimenter-prompt.ts). Optional in the schema (a grading.json
+     * validated by external tooling need not carry it), but the harness REQUIRES
+     * it to match the run's secret nonce before trusting the verdict — this is how
+     * a forged grading.json written by untrusted skill code is rejected.
+     */
+    runNonce: z.ZodOptional<z.ZodString>;
 }, z.ZodTypeAny, "passthrough">>;
 export type GradingReport = z.infer<typeof GradingReportSchema>;
 /**

package/dist/skill-test/grading-schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"grading-schema.d.ts","sourceRoot":"","sources":["../../src/skill-test/grading-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,sFAAsF;AACtF,eAAO,MAAM,uBAAuB;;;;;;;;;;;;gCAMpB,CAAC;AAEjB,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,oFAAoF;AACpF,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;gCAOjB,CAAC;AAEjB,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE;;;;GAIG;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAKhB,CAAC;AAEjB,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB;;;;;CAAyD,CAAC"}
+{"version":3,"file":"grading-schema.d.ts","sourceRoot":"","sources":["../../src/skill-test/grading-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,sFAAsF;AACtF,eAAO,MAAM,uBAAuB;;;;;;;;;;;;gCAMpB,CAAC;AAEjB,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE;;;;GAIG;AACH,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;gCAU7B,CAAC;AAEL,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE;;;;GAIG;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAI5B;;;;;;OAMG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IANH;;;;;;OAMG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IANH;;;;;;OAMG;;gCAGS,CAAC;AAEjB,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB;;;;;CAAyD,CAAC"}

package/dist/skill-test/grading-schema.js

@@ -35,15 +35,22 @@ export const GradedExpectationSchema = z
     evidence: z.string().optional(),
 })
     .passthrough();
-/** Aggregate pass/fail counts. `failed`/`pass_rate` are documented but optional. */
+/**
+ * Aggregate pass/fail counts. `failed`/`pass_rate` are documented but optional.
+ * Counts are non-negative integers (a float or negative count is a grader bug),
+ * and `passed` can never exceed `total`.
+ */
 export const GradingSummarySchema = z
     .object({
-    passed: z.number(),
-    total: z.number(),
-    failed: z.number().optional(),
+    passed: z.number().int().nonnegative(),
+    total: z.number().int().nonnegative(),
+    failed: z.number().int().nonnegative().optional(),
     pass_rate: z.number().optional(),
 })
-    .passthrough();
+    .passthrough()
+    .refine(s => s.passed <= s.total, {
+    message: 'summary.passed must not exceed summary.total',
+});
 /**
  * The full grading.json contract. Required: top-level `expectations[]` and
  * `summary`. Everything else passes through untouched (forward-compatible with
@@ -53,6 +60,14 @@ export const GradingReportSchema = z
     .object({
     expectations: z.array(GradedExpectationSchema),
     summary: GradingSummarySchema,
+    /**
+     * Per-run integrity nonce the experimenter is told to copy verbatim from its
+     * prompt (see experimenter-prompt.ts). Optional in the schema (a grading.json
+     * validated by external tooling need not carry it), but the harness REQUIRES
+     * it to match the run's secret nonce before trusting the verdict — this is how
+     * a forged grading.json written by untrusted skill code is rejected.
+     */
+    runNonce: z.string().optional(),
 })
     .passthrough();
 /**

package/dist/skill-test/grading-schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"grading-schema.js","sourceRoot":"","sources":["../../src/skill-test/grading-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,sFAAsF;AACtF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC;KACrC,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC;KACD,WAAW,EAAE,CAAC;AAIjB,oFAAoF;AACpF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC;KAClC,MAAM,CAAC;IACN,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACjC,CAAC;KACD,WAAW,EAAE,CAAC;AAIjB;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC;KACjC,MAAM,CAAC;IACN,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,uBAAuB,CAAC;IAC9C,OAAO,EAAE,oBAAoB;CAC9B,CAAC;KACD,WAAW,EAAE,CAAC;AAIjB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,eAAe,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,CAAC"}
+{"version":3,"file":"grading-schema.js","sourceRoot":"","sources":["../../src/skill-test/grading-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,sFAAsF;AACtF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC;KACrC,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC;KACD,WAAW,EAAE,CAAC;AAIjB;;;;GAIG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC;KAClC,MAAM,CAAC;IACN,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IACtC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IACrC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;IACjD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACjC,CAAC;KACD,WAAW,EAAE;KACb,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,EAAE;IAChC,OAAO,EAAE,8CAA8C;CACxD,CAAC,CAAC;AAIL;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC;KACjC,MAAM,CAAC;IACN,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,uBAAuB,CAAC;IAC9C,OAAO,EAAE,oBAAoB;IAC7B;;;;;;OAMG;IACH,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC;KACD,WAAW,EAAE,CAAC;AAIjB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,eAAe,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,CAAC"}

package/dist/skill-test/harness-location.d.ts

@@ -15,13 +15,44 @@ export declare function resolveHarnessRoot(skillNames: string[], tmpRoot?: strin
  * Refuse a --workdir whose ancestry contains CLAUDE.md or .claude/ — cwd
  * discovery would re-pollute the run (spec §7). Defense in depth with
  * --setting-sources "".
+ *
+ * `stopAt` (optional) bounds the ancestry walk: when the walk reaches that
+ * directory it stops *before* inspecting it (exclusive boundary), mirroring
+ * `assertSafeHarnessRoot`'s `trustedRoot`. Default (undefined) walks to the
+ * filesystem root, preserving the strictest production behavior. Tests use it
+ * to scope the walk to a sandbox — necessary on Windows, where the OS tmp dir
+ * lives inside the user's home and would otherwise surface an ambient
+ * ~/.claude during the walk.
  */
-export declare function assertSafeWorkdir(dir: string): void;
+export declare function assertSafeWorkdir(dir: string, stopAt?: string): void;
 /**
- * FS-bound hardening for the shared-tmp harness root (spec §7): the root must
- * be 0700 and owned by the current uid, and no path component may be a symlink.
+ * Prepare the harness root directory so that `assertSafeHarnessRoot` will
+ * pass on the next call. If the path does not exist, this is a no-op (the
+ * caller creates it at 0700 via mkdirSyncReal). If it exists:
+ *
+ * - Symlink → throw HarnessLocationError (security gate; never relax).
+ * - Real directory whose mode != 0700 → chmod to 0700. Removing group/other
+ *   access is strictly safer, never a relaxation.
+ *
+ * Mode checks/changes are only performed on non-win32 (matching
+ * assertSafeHarnessRoot's platform guard).
+ */
+export declare function prepareHarnessRoot(dir: string): void;
+/**
+ * FS-bound hardening for the shared-tmp harness root (spec §7). Validates EVERY
+ * path component from the leaf up to (but excluding) `trustedRoot`: no component
+ * may be a symlink, and on POSIX each must be owned by the current uid. This
+ * closes the shared-/tmp TOCTOU where the recursively-created intermediate parent
+ * (`<tmp>/vat-skill-test`) — not just the leaf — could be pre-created as a symlink
+ * or under another user's ownership. The leaf must additionally be 0700.
+ *
+ * `trustedRoot` (default: the OS tmp dir) is the boundary: it is system-owned
+ * (sticky-bit /tmp) so ownership/mode checks are not applied to it. When the leaf
+ * is not a descendant of `trustedRoot` (an explicit --out elsewhere), validation
+ * degrades to the leaf alone.
+ *
  * Integration-tested (requires real lstat/stat). On Windows, uid checks are
  * skipped (process.getuid is undefined) but the symlink refusal still applies.
  */
-export declare function assertSafeHarnessRoot(dir: string, currentUid: number): void;
+export declare function assertSafeHarnessRoot(dir: string, currentUid: number, trustedRoot?: string): void;
 //# sourceMappingURL=harness-location.d.ts.map