@vibe-agent-toolkit/agent-skills 0.1.39-rc.7 → 0.1.39-rc.9

package/dist/skill-test/build-hook.d.ts

@@ -0,0 +1,58 @@
+/**
+ * build-hook.ts — optional pre-stage build step for `vat skill test run`.
+ *
+ * When the test config includes a `build:` field, this module runs that shell
+ * command ONCE before staging so that generated artifacts (e.g. bundled scripts
+ * not committed to source) are present in the source tree for staging to copy.
+ *
+ * The command runs with cwd = the CONFIG ROOT (the directory containing
+ * vibe-agent-toolkit.config.yaml), because real build commands are root-level
+ * package scripts (e.g. `pnpm bundle:report`).
+ *
+ * Security note: the `build:` field is a developer-authored value from the
+ * project's own vibe-agent-toolkit.config.yaml — a trusted source under the
+ * adopter's source control. The command is passed directly to the OS shell
+ * (shell: true) intentionally, because build commands frequently include shell
+ * syntax (npm script chaining, env vars, etc.). This is equivalent to running
+ * `npm run build` or `pnpm bundle:report` from the terminal; it is NOT arbitrary
+ * user input. The adopter is already executing skill code via this command
+ * (`vat skill test run` requires --i-understand-this-runs-skill-code).
+ */
+/**
+ * Options for the pre-stage build hook.
+ *
+ * `spawnFn` is injectable for unit testing — production code uses the default
+ * (node:child_process spawnSync). Tests inject a vi.fn() mock.
+ */
+export interface BuildHookOptions {
+    /** Shell command to run (from `test.build` in vibe-agent-toolkit.config.yaml). */
+    buildCommand: string | undefined;
+    /** Absolute path to the config root (cwd for the build command). */
+    configRoot: string;
+    /**
+     * Injectable spawn function for unit testing.
+     * Defaults to node:child_process spawnSync when not provided.
+     */
+    spawnFn?: (cmd: string, opts: {
+        shell: boolean;
+        cwd: string;
+        stdio: 'inherit';
+    }) => {
+        status: number | null;
+    };
+}
+/** Thrown when the pre-stage build command exits with a non-zero code. Maps to preflight (exit 2). */
+export declare class BuildHookError extends Error {
+    readonly buildExitCode: number;
+    readonly exitCode: 2;
+    constructor(message: string, buildExitCode: number);
+}
+/**
+ * Run the pre-stage build hook if configured.
+ *
+ * Runs `buildCommand` in a shell with `cwd = configRoot`. On non-zero exit,
+ * throws `BuildHookError` with a clear message naming the command and exit code.
+ * When `buildCommand` is undefined, this is a no-op (behavior unchanged).
+ */
+export declare function runPreStageBuild(opts: BuildHookOptions): void;
+//# sourceMappingURL=build-hook.d.ts.map

package/dist/skill-test/build-hook.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"build-hook.d.ts","sourceRoot":"","sources":["../../src/skill-test/build-hook.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAIH;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC/B,kFAAkF;IAClF,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,oEAAoE;IACpE,UAAU,EAAE,MAAM,CAAC;IACnB;;;OAGG;IACH,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,KAAK,EAAE,OAAO,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,SAAS,CAAA;KAAE,KAAK;QAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;CAC/G;AAED,sGAAsG;AACtG,qBAAa,cAAe,SAAQ,KAAK;aAEM,aAAa,EAAE,MAAM;IADlE,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM,EAAkB,aAAa,EAAE,MAAM;CAInE;AAeD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,gBAAgB,GAAG,IAAI,CAe7D"}

package/dist/skill-test/build-hook.js

@@ -0,0 +1,63 @@
+/**
+ * build-hook.ts — optional pre-stage build step for `vat skill test run`.
+ *
+ * When the test config includes a `build:` field, this module runs that shell
+ * command ONCE before staging so that generated artifacts (e.g. bundled scripts
+ * not committed to source) are present in the source tree for staging to copy.
+ *
+ * The command runs with cwd = the CONFIG ROOT (the directory containing
+ * vibe-agent-toolkit.config.yaml), because real build commands are root-level
+ * package scripts (e.g. `pnpm bundle:report`).
+ *
+ * Security note: the `build:` field is a developer-authored value from the
+ * project's own vibe-agent-toolkit.config.yaml — a trusted source under the
+ * adopter's source control. The command is passed directly to the OS shell
+ * (shell: true) intentionally, because build commands frequently include shell
+ * syntax (npm script chaining, env vars, etc.). This is equivalent to running
+ * `npm run build` or `pnpm bundle:report` from the terminal; it is NOT arbitrary
+ * user input. The adopter is already executing skill code via this command
+ * (`vat skill test run` requires --i-understand-this-runs-skill-code).
+ */
+import { spawnSync } from 'node:child_process';
+/** Thrown when the pre-stage build command exits with a non-zero code. Maps to preflight (exit 2). */
+export class BuildHookError extends Error {
+    buildExitCode;
+    exitCode = 2;
+    constructor(message, buildExitCode) {
+        super(message);
+        this.buildExitCode = buildExitCode;
+        this.name = 'BuildHookError';
+    }
+}
+/**
+ * Default spawn implementation: runs the command in a shell with stdio inherited.
+ *
+ * `build:` is a developer-authored shell command from the adopter's own config
+ * (vibe-agent-toolkit.config.yaml), equivalent to running `pnpm bundle:report` at the
+ * terminal. It is NOT arbitrary user input. The adopter already acknowledges running
+ * skill code via --i-understand-this-runs-skill-code.
+ */
+function defaultSpawn(cmd, opts) {
+    // eslint-disable-next-line sonarjs/os-command -- developer-authored build command from trusted project config; equivalent to running pnpm/npm build manually
+    return spawnSync(cmd, { ...opts, shell: true });
+}
+/**
+ * Run the pre-stage build hook if configured.
+ *
+ * Runs `buildCommand` in a shell with `cwd = configRoot`. On non-zero exit,
+ * throws `BuildHookError` with a clear message naming the command and exit code.
+ * When `buildCommand` is undefined, this is a no-op (behavior unchanged).
+ */
+export function runPreStageBuild(opts) {
+    const { buildCommand, configRoot } = opts;
+    if (buildCommand === undefined)
+        return;
+    const spawn = opts.spawnFn ?? defaultSpawn;
+    const result = spawn(buildCommand, { shell: true, cwd: configRoot, stdio: 'inherit' });
+    const status = result.status ?? -1;
+    if (status !== 0) {
+        throw new BuildHookError(`Pre-stage build hook failed: command "${buildCommand}" exited with code ${status}. ` +
+            `Resolve the build error before running vat skill test run.`, status);
+    }
+}
+//# sourceMappingURL=build-hook.js.map

package/dist/skill-test/build-hook.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"build-hook.js","sourceRoot":"","sources":["../../src/skill-test/build-hook.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAoB/C,sGAAsG;AACtG,MAAM,OAAO,cAAe,SAAQ,KAAK;IAEM;IADpC,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe,EAAkB,aAAqB;QAChE,KAAK,CAAC,OAAO,CAAC,CAAC;QAD4B,kBAAa,GAAb,aAAa,CAAQ;QAEhE,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED;;;;;;;GAOG;AACH,SAAS,YAAY,CAAC,GAAW,EAAE,IAAuD;IACxF,6JAA6J;IAC7J,OAAO,SAAS,CAAC,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAClD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAsB;IACrD,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;IAC1C,IAAI,YAAY,KAAK,SAAS;QAAE,OAAO;IAEvC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,IAAI,YAAY,CAAC;IAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;IACvF,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;IAEnC,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,MAAM,IAAI,cAAc,CACtB,yCAAyC,YAAY,sBAAsB,MAAM,IAAI;YACnF,4DAA4D,EAC9D,MAAM,CACP,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/skill-test/exit-codes.d.ts

@@ -25,8 +25,9 @@ export declare class InternalHarnessError extends Error {
  * Map any thrown error to the process exit code. Errors that carry their own
  * `exitCode` (Bootstrap/Auth/HarnessLocation/Internal) are authoritative;
  * a PromptInvariantError is a user-correctable preflight problem (a supplied
- * prompt override is missing a required safety instruction) → 2; GradingSkewError
- * is a parse failure → 1; everything unknown → 1.
+ * prompt override is missing a required safety instruction) → 2; a BuildHookError
+ * is a pre-stage build failure → 2; GradingSkewError is a parse failure → 1;
+ * everything unknown → 1.
  */
 export declare function mapErrorToExitCode(err: unknown): number;
 //# sourceMappingURL=exit-codes.d.ts.map

package/dist/skill-test/exit-codes.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"exit-codes.d.ts","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AAMA,kDAAkD;AAClD,eAAO,MAAM,iBAAiB;;;;;CAKpB,CAAC;AAEX,MAAM,MAAM,sBAAsB,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,OAAO,iBAAiB,CAAC,CAAC;AAEhG;;;;GAIG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;aAEjB,YAAY,EAAE,MAAM;IADhD,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACH,YAAY,EAAE,MAAM;CAIjD;AAED,gGAAgG;AAChG,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAWvD"}
+{"version":3,"file":"exit-codes.d.ts","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AAOA,kDAAkD;AAClD,eAAO,MAAM,iBAAiB;;;;;CAKpB,CAAC;AAEX,MAAM,MAAM,sBAAsB,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,OAAO,iBAAiB,CAAC,CAAC;AAEhG;;;;GAIG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;aAEjB,YAAY,EAAE,MAAM;IADhD,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACH,YAAY,EAAE,MAAM;CAIjD;AAED,gGAAgG;AAChG,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAED;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAYvD"}

package/dist/skill-test/exit-codes.js

@@ -1,4 +1,5 @@
 import { AuthPreflightError } from '@vibe-agent-toolkit/utils';
+import { BuildHookError } from './build-hook.js';
 import { PromptInvariantError } from './experimenter-prompt.js';
 import { GradingSkewError } from './grading-adapter.js';
 import { HarnessLocationError } from './harness-location.js';
@@ -35,13 +36,15 @@ export class InternalHarnessError extends Error {
  * Map any thrown error to the process exit code. Errors that carry their own
  * `exitCode` (Bootstrap/Auth/HarnessLocation/Internal) are authoritative;
  * a PromptInvariantError is a user-correctable preflight problem (a supplied
- * prompt override is missing a required safety instruction) → 2; GradingSkewError
- * is a parse failure → 1; everything unknown → 1.
+ * prompt override is missing a required safety instruction) → 2; a BuildHookError
+ * is a pre-stage build failure → 2; GradingSkewError is a parse failure → 1;
+ * everything unknown → 1.
  */
 export function mapErrorToExitCode(err) {
     if (err instanceof BootstrapNeededError)
         return SkillTestExitCode.Bootstrap;
     if (err instanceof AuthPreflightError ||
+        err instanceof BuildHookError ||
         err instanceof HarnessLocationError ||
         err instanceof PromptInvariantError) {
         return SkillTestExitCode.Preflight;

package/dist/skill-test/exit-codes.js.map

@@ -1 +1 @@
-{"version":3,"file":"exit-codes.js","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAE7D,kDAAkD;AAClD,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,EAAE,EAAE,CAAC;IACL,QAAQ,EAAE,CAAC;IACX,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,CAAC;CACJ,CAAC;AAIX;;;;GAIG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAEjB;IADnB,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAA4B,YAAoB;QAC9C,KAAK,CAAC,mCAAmC,YAAY,2BAA2B,CAAC,CAAC;QADxD,iBAAY,GAAZ,YAAY,CAAQ;QAE9C,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,gGAAgG;AAChG,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACpC,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAY;IAC7C,IAAI,GAAG,YAAY,oBAAoB;QAAE,OAAO,iBAAiB,CAAC,SAAS,CAAC;IAC5E,IACE,GAAG,YAAY,kBAAkB;QACjC,GAAG,YAAY,oBAAoB;QACnC,GAAG,YAAY,oBAAoB,EACnC,CAAC;QACD,OAAO,iBAAiB,CAAC,SAAS,CAAC;IACrC,CAAC;IACD,IAAI,GAAG,YAAY,gBAAgB,IAAI,GAAG,YAAY,oBAAoB;QAAE,OAAO,iBAAiB,CAAC,QAAQ,CAAC;IAC9G,OAAO,iBAAiB,CAAC,QAAQ,CAAC;AACpC,CAAC"}
+{"version":3,"file":"exit-codes.js","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAE7D,kDAAkD;AAClD,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,EAAE,EAAE,CAAC;IACL,QAAQ,EAAE,CAAC;IACX,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,CAAC;CACJ,CAAC;AAIX;;;;GAIG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAEjB;IADnB,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAA4B,YAAoB;QAC9C,KAAK,CAAC,mCAAmC,YAAY,2BAA2B,CAAC,CAAC;QADxD,iBAAY,GAAZ,YAAY,CAAQ;QAE9C,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,gGAAgG;AAChG,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACpC,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAY;IAC7C,IAAI,GAAG,YAAY,oBAAoB;QAAE,OAAO,iBAAiB,CAAC,SAAS,CAAC;IAC5E,IACE,GAAG,YAAY,kBAAkB;QACjC,GAAG,YAAY,cAAc;QAC7B,GAAG,YAAY,oBAAoB;QACnC,GAAG,YAAY,oBAAoB,EACnC,CAAC;QACD,OAAO,iBAAiB,CAAC,SAAS,CAAC;IACrC,CAAC;IACD,IAAI,GAAG,YAAY,gBAAgB,IAAI,GAAG,YAAY,oBAAoB;QAAE,OAAO,iBAAiB,CAAC,QAAQ,CAAC;IAC9G,OAAO,iBAAiB,CAAC,QAAQ,CAAC;AACpC,CAAC"}

package/dist/skill-test/experimenter-prompt.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"experimenter-prompt.d.ts","sourceRoot":"","sources":["../../src/skill-test/experimenter-prompt.ts"],"names":[],"mappings":"AAAA,qBAAa,oBAAqB,SAAQ,KAAK;gBACjC,OAAO,EAAE,MAAM;CAI5B;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B,QAe5B,CAAC;AAKb,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,kBAAkB,GAAG,MAAM,CAOxE;AAWD,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAI3D"}
+{"version":3,"file":"experimenter-prompt.d.ts","sourceRoot":"","sources":["../../src/skill-test/experimenter-prompt.ts"],"names":[],"mappings":"AAAA,qBAAa,oBAAqB,SAAQ,KAAK;gBACjC,OAAO,EAAE,MAAM;CAI5B;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED;;;;;GAKG;AACH,eAAO,MAAM,2BAA2B,QAuB5B,CAAC;AAKb,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,kBAAkB,GAAG,MAAM,CAOxE;AAmBD,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAI3D"}

package/dist/skill-test/experimenter-prompt.js

@@ -17,10 +17,18 @@ export const DEFAULT_EXPERIMENTER_PROMPT = [
     '  1. Dispatch ONE executor subagent. Tell it ONLY the task prompt and the staged subject path {{SUBJECT_PATH}}.',
     '     Never tell the executor it is being tested.',
     '  2. Grade the executor output against the eval\'s `expectations` using skill-creator\'s grader.md rubric.',
-    '  3. Append the per-eval result to {{GRADING_OUT}} IMMEDIATELY (incremental flush — a mid-run kill must leave partial results).',
+    '  3. Append each graded expectation to the SINGLE top-level `expectations` array in {{GRADING_OUT}} IMMEDIATELY',
+    '     (incremental flush — a mid-run kill must leave partial results).',
     '  4. Record any packaging-fidelity friction to {{FRICTION_OUT}} using the vat friction schema.',
+    '     If a file referenced by the skill is absent from the staged tree, record a `missing-bundled-file` friction entry.',
     '',
-    'When all evals are graded, write the final summary to {{GRADING_OUT}} and STOP.',
+    '{{GRADING_OUT}} MUST be ONE flat JSON object in skill-creator\'s grading.json shape (references/schemas.md):',
+    'a top-level `expectations` array — one entry {"text","passed","evidence"} per expectation across ALL evals —',
+    'and a top-level `summary` {"passed","total"}. Do NOT wrap results in an `evals` array or any per-eval nesting;',
+    'vat reads the flat top-level shape and rejects anything else. Example:',
+    '  {"expectations":[{"text":"...","passed":true,"evidence":"..."}],"summary":{"passed":1,"total":1}}',
+    '',
+    'When all evals are graded, write the final `summary` to {{GRADING_OUT}} and STOP.',
     '',
     'You are FORBIDDEN to: open a browser or viewer; run aggregation/optimizer scripts; wait for human feedback;',
     'or iterate/improve the skill. This is a downstream packaging check, not an authoring loop.',
@@ -42,6 +50,14 @@ const REQUIRED_PATTERNS = [
     { test: /forbidden|do not|never/i, label: 'must forbid browser/aggregation/feedback/iteration' },
     { test: /browser|viewer/i, label: 'must explicitly forbid opening a browser/viewer' },
     { test: /increment/i, label: 'must emit incrementally' },
+    {
+        test: /top-level\s+`?expectations`?/i,
+        label: 'must pin grading.json to the flat top-level `expectations`/`summary` shape',
+    },
+    {
+        test: /`?evals`?\s+array|per-eval nesting/i,
+        label: 'must forbid wrapping grading results in an `evals` array',
+    },
 ];
 export function assertPromptInvariants(prompt) {
     for (const { test, label } of REQUIRED_PATTERNS) {

package/dist/skill-test/experimenter-prompt.js.map

@@ -1 +1 @@
-{"version":3,"file":"experimenter-prompt.js","sourceRoot":"","sources":["../../src/skill-test/experimenter-prompt.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC7C,YAAY,OAAe;QACzB,KAAK,CAAC,2CAA2C,OAAO,EAAE,CAAC,CAAC;QAC5D,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAUD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG;IACzC,yGAAyG;IACzG,EAAE;IACF,kCAAkC;IAClC,iHAAiH;IACjH,kDAAkD;IAClD,4GAA4G;IAC5G,iIAAiI;IACjI,gGAAgG;IAChG,EAAE;IACF,iFAAiF;IACjF,EAAE;IACF,6GAA6G;IAC7G,4FAA4F;IAC5F,oBAAoB;CACrB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,MAAM,cAAc,GAClB,sIAAsI,CAAC;AAEzI,MAAM,UAAU,uBAAuB,CAAC,IAAwB;IAC9D,OAAO,2BAA2B;SAC/B,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC;SACzC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,WAAW,CAAC;SAC7C,UAAU,CAAC,iBAAiB,EAAE,IAAI,CAAC,UAAU,CAAC;SAC9C,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,WAAW,CAAC;SAC7C,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,iBAAiB,GAAsC;IAC3D,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,wCAAwC,EAAE;IACrE,EAAE,IAAI,EAAE,oCAAoC,EAAE,KAAK,EAAE,yBAAyB,EAAE;IAChF,EAAE,IAAI,EAAE,sCAAsC,EAAE,KAAK,EAAE,0BAA0B,EAAE;IACnF,EAAE,IAAI,EAAE,yBAAyB,EAAE,KAAK,EAAE,oDAAoD,EAAE;IAChG,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,iDAAiD,EAAE;IACrF,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,yBAAyB,EAAE;CACzD,CAAC;AAEF,MAAM,UAAU,sBAAsB,CAAC,MAAc;IACnD,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,iBAAiB,EAAE,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAChE,CAAC;AACH,CAAC"}
+{"version":3,"file":"experimenter-prompt.js","sourceRoot":"","sources":["../../src/skill-test/experimenter-prompt.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC7C,YAAY,OAAe;QACzB,KAAK,CAAC,2CAA2C,OAAO,EAAE,CAAC,CAAC;QAC5D,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAUD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG;IACzC,yGAAyG;IACzG,EAAE;IACF,kCAAkC;IAClC,iHAAiH;IACjH,kDAAkD;IAClD,4GAA4G;IAC5G,iHAAiH;IACjH,uEAAuE;IACvE,gGAAgG;IAChG,wHAAwH;IACxH,EAAE;IACF,8GAA8G;IAC9G,8GAA8G;IAC9G,gHAAgH;IAChH,wEAAwE;IACxE,qGAAqG;IACrG,EAAE;IACF,mFAAmF;IACnF,EAAE;IACF,6GAA6G;IAC7G,4FAA4F;IAC5F,oBAAoB;CACrB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,MAAM,cAAc,GAClB,sIAAsI,CAAC;AAEzI,MAAM,UAAU,uBAAuB,CAAC,IAAwB;IAC9D,OAAO,2BAA2B;SAC/B,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC;SACzC,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,WAAW,CAAC;SAC7C,UAAU,CAAC,iBAAiB,EAAE,IAAI,CAAC,UAAU,CAAC;SAC9C,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC,WAAW,CAAC;SAC7C,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,iBAAiB,GAAsC;IAC3D,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,wCAAwC,EAAE;IACrE,EAAE,IAAI,EAAE,oCAAoC,EAAE,KAAK,EAAE,yBAAyB,EAAE;IAChF,EAAE,IAAI,EAAE,sCAAsC,EAAE,KAAK,EAAE,0BAA0B,EAAE;IACnF,EAAE,IAAI,EAAE,yBAAyB,EAAE,KAAK,EAAE,oDAAoD,EAAE;IAChG,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,iDAAiD,EAAE;IACrF,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,yBAAyB,EAAE;IACxD;QACE,IAAI,EAAE,+BAA+B;QACrC,KAAK,EAAE,4EAA4E;KACpF;IACD;QACE,IAAI,EAAE,qCAAqC;QAC3C,KAAK,EAAE,0DAA0D;KAClE;CACF,CAAC;AAEF,MAAM,UAAU,sBAAsB,CAAC,MAAc;IACnD,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,iBAAiB,EAAE,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAChE,CAAC;AACH,CAAC"}

package/dist/skill-test/grading-adapter.d.ts

@@ -1,4 +1,9 @@
-/** Thrown when skill-creator's grading.json shape has drifted from what vat reads. */
+/**
+ * Thrown when grading.json does not match the canonical skill-creator shape
+ * (see grading-schema.ts / skill-creator references/schemas.md). vat refuses to
+ * limp along on malformed grading data — a wrong shape silently flowing
+ * downstream causes confusing failures far from the real cause.
+ */
 export declare class GradingSkewError extends Error {
     constructor(message: string);
 }

package/dist/skill-test/grading-adapter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"grading-adapter.d.ts","sourceRoot":"","sources":["../../src/skill-test/grading-adapter.ts"],"names":[],"mappings":"AAEA,sFAAsF;AACtF,qBAAa,gBAAiB,SAAQ,KAAK;gBAC7B,OAAO,EAAE,MAAM;CAI5B;AAqBD,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,YAAY,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACtE;AAED,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,iBAAiB,CAgBhE"}
+{"version":3,"file":"grading-adapter.d.ts","sourceRoot":"","sources":["../../src/skill-test/grading-adapter.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,KAAK;gBAC7B,OAAO,EAAE,MAAM;CAQ5B;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,YAAY,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CACtE;AAaD,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,iBAAiB,CAuBhE"}

package/dist/skill-test/grading-adapter.js

@@ -1,30 +1,37 @@
-import { z } from 'zod';
-/** Thrown when skill-creator's grading.json shape has drifted from what vat reads. */
+import { GradingReportSchema } from './grading-schema.js';
+/**
+ * Thrown when grading.json does not match the canonical skill-creator shape
+ * (see grading-schema.ts / skill-creator references/schemas.md). vat refuses to
+ * limp along on malformed grading data — a wrong shape silently flowing
+ * downstream causes confusing failures far from the real cause.
+ */
 export class GradingSkewError extends Error {
     constructor(message) {
-        super(`grading.json shape skew: ${message}. Re-sync the vendored skill-creator / adopted shapes.`);
+        super(`grading.json shape skew: ${message}. Expected skill-creator's grading.json shape ` +
+            '(a single flat object with top-level `expectations` and `summary`); see ' +
+            'docs/skill-test-grading-schema.md. Re-sync the vendored skill-creator / adopted shapes.');
         this.name = 'GradingSkewError';
     }
 }
 /**
- * Liberal read of skill-creator's external grading.json. `.passthrough()` keeps
- * unknown fields (viewer urls, etc.) — we validate only what we understand
- * (Postel). Field names mirror skill-creator's grader output.
+ * Detect the common per-eval nested mistake: `{ evals: [ { expectations, ... } ] }`
+ * with no top-level `expectations`. The grader (an LLM) reaches for this when the
+ * top-level shape is under-specified; we name it explicitly so the fix is obvious.
  */
-const ExternalGradingSchema = z.object({
-    summary: z.object({
-        passed: z.number(),
-        total: z.number(),
-    }).passthrough(),
-    expectations: z.array(z.object({
-        text: z.string(),
-        passed: z.boolean(),
-        evidence: z.string().optional(),
-    }).passthrough()),
-}).passthrough();
+function looksPerEvalNested(raw) {
+    if (typeof raw !== 'object' || raw === null)
+        return false;
+    const obj = raw;
+    return Array.isArray(obj['evals']) && !('expectations' in obj);
+}
 export function parseGradingJson(raw) {
-    const result = ExternalGradingSchema.safeParse(raw);
+    const result = GradingReportSchema.safeParse(raw);
     if (!result.success) {
+        if (looksPerEvalNested(raw)) {
+            throw new GradingSkewError('top-level `expectations` is missing — results were nested under an `evals` array. ' +
+                'grading.json must be ONE flat object whose top-level `expectations` lists every ' +
+                'graded expectation across all evals');
+        }
         const firstIssue = result.error.issues[0];
         const path = firstIssue?.path.join('.') ?? '(root)';
         throw new GradingSkewError(`missing/invalid field at "${path}" (${firstIssue?.message ?? 'unknown'})`);

package/dist/skill-test/grading-adapter.js.map

@@ -1 +1 @@
-{"version":3,"file":"grading-adapter.js","sourceRoot":"","sources":["../../src/skill-test/grading-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,sFAAsF;AACtF,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACzC,YAAY,OAAe;QACzB,KAAK,CAAC,4BAA4B,OAAO,wDAAwD,CAAC,CAAC;QACnG,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QAChB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;QAClB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;KAClB,CAAC,CAAC,WAAW,EAAE;IAChB,YAAY,EAAE,CAAC,CAAC,KAAK,CACnB,CAAC,CAAC,MAAM,CAAC;QACP,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;QAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;QACnB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAChC,CAAC,CAAC,WAAW,EAAE,CACjB;CACF,CAAC,CAAC,WAAW,EAAE,CAAC;AAOjB,MAAM,UAAU,gBAAgB,CAAC,GAAY;IAC3C,MAAM,MAAM,GAAG,qBAAqB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACpD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC;QACpD,MAAM,IAAI,gBAAgB,CAAC,6BAA6B,IAAI,MAAM,UAAU,EAAE,OAAO,IAAI,SAAS,GAAG,CAAC,CAAC;IACzG,CAAC;IACD,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;IAC9C,OAAO;QACL,OAAO,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE;QACzD,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;SAC9D,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC"}
+{"version":3,"file":"grading-adapter.js","sourceRoot":"","sources":["../../src/skill-test/grading-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D;;;;;GAKG;AACH,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACzC,YAAY,OAAe;QACzB,KAAK,CACH,4BAA4B,OAAO,gDAAgD;YACjF,0EAA0E;YAC1E,yFAAyF,CAC5F,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAOD;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,GAAY;IACtC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC1D,MAAM,GAAG,GAAG,GAA8B,CAAC;IAC3C,OAAO,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,IAAI,GAAG,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAY;IAC3C,MAAM,MAAM,GAAG,mBAAmB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAClD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,IAAI,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,gBAAgB,CACxB,oFAAoF;gBAClF,kFAAkF;gBAClF,qCAAqC,CACxC,CAAC;QACJ,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC;QACpD,MAAM,IAAI,gBAAgB,CAAC,6BAA6B,IAAI,MAAM,UAAU,EAAE,OAAO,IAAI,SAAS,GAAG,CAAC,CAAC;IACzG,CAAC;IACD,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;IAC9C,OAAO;QACL,OAAO,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE;QACzD,YAAY,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,GAAG,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;SAC9D,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC"}

package/dist/skill-test/grading-schema.d.ts

@@ -0,0 +1,171 @@
+import { z } from 'zod';
+/**
+ * Canonical schema for the grader agent's `grading.json` output.
+ *
+ * SOURCE OF TRUTH: skill-creator's `references/schemas.md` (the `grading.json`
+ * section). vat consumes that exact shape; this module is the machine-checkable
+ * encoding of it (skill-creator ships prose + an example, but no JSON Schema).
+ * The published JSON Schema is {@link GradingReportJsonSchema}.
+ *
+ * SHAPE: a SINGLE flat JSON object with two load-bearing top-level fields —
+ *
+ *   {
+ *     "expectations": [ { "text": string, "passed": boolean, "evidence"?: string }, ... ],
+ *     "summary":      { "passed": number, "total": number, "failed"?: number, "pass_rate"?: number }
+ *   }
+ *
+ * `expectations` holds ONE entry per graded expectation across ALL evals — it is
+ * NOT grouped per-eval and is NEVER wrapped in an `evals` array. A per-eval
+ * nested shape (`{ evals: [ { expectations, summary } ] }`) is a contract
+ * violation and is rejected loudly (see grading-adapter.ts); tolerating it would
+ * push malformed data downstream and create confusion.
+ *
+ * LIBERAL ON EXTRAS (Postel): the grader legitimately emits additional documented
+ * sections — `execution_metrics`, `timing`, `claims`, `user_notes_summary`,
+ * `eval_feedback` — plus viewer URLs and other adornments. We `.passthrough()`
+ * those: validate the two fields we depend on, carry the rest untouched. Extra
+ * fields are NOT "bad JSON"; a wrong top-level STRUCTURE is.
+ */
+/** One graded expectation. `evidence` is recommended but not load-bearing for vat. */
+export declare const GradedExpectationSchema: z.ZodObject<{
+    text: z.ZodString;
+    passed: z.ZodBoolean;
+    evidence: z.ZodOptional<z.ZodString>;
+}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+    text: z.ZodString;
+    passed: z.ZodBoolean;
+    evidence: z.ZodOptional<z.ZodString>;
+}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+    text: z.ZodString;
+    passed: z.ZodBoolean;
+    evidence: z.ZodOptional<z.ZodString>;
+}, z.ZodTypeAny, "passthrough">>;
+export type GradedExpectation = z.infer<typeof GradedExpectationSchema>;
+/** Aggregate pass/fail counts. `failed`/`pass_rate` are documented but optional. */
+export declare const GradingSummarySchema: z.ZodObject<{
+    passed: z.ZodNumber;
+    total: z.ZodNumber;
+    failed: z.ZodOptional<z.ZodNumber>;
+    pass_rate: z.ZodOptional<z.ZodNumber>;
+}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+    passed: z.ZodNumber;
+    total: z.ZodNumber;
+    failed: z.ZodOptional<z.ZodNumber>;
+    pass_rate: z.ZodOptional<z.ZodNumber>;
+}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+    passed: z.ZodNumber;
+    total: z.ZodNumber;
+    failed: z.ZodOptional<z.ZodNumber>;
+    pass_rate: z.ZodOptional<z.ZodNumber>;
+}, z.ZodTypeAny, "passthrough">>;
+export type GradingSummary = z.infer<typeof GradingSummarySchema>;
+/**
+ * The full grading.json contract. Required: top-level `expectations[]` and
+ * `summary`. Everything else passes through untouched (forward-compatible with
+ * skill-creator additions).
+ */
+export declare const GradingReportSchema: z.ZodObject<{
+    expectations: z.ZodArray<z.ZodObject<{
+        text: z.ZodString;
+        passed: z.ZodBoolean;
+        evidence: z.ZodOptional<z.ZodString>;
+    }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+        text: z.ZodString;
+        passed: z.ZodBoolean;
+        evidence: z.ZodOptional<z.ZodString>;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        text: z.ZodString;
+        passed: z.ZodBoolean;
+        evidence: z.ZodOptional<z.ZodString>;
+    }, z.ZodTypeAny, "passthrough">>, "many">;
+    summary: z.ZodObject<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">>;
+}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+    expectations: z.ZodArray<z.ZodObject<{
+        text: z.ZodString;
+        passed: z.ZodBoolean;
+        evidence: z.ZodOptional<z.ZodString>;
+    }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+        text: z.ZodString;
+        passed: z.ZodBoolean;
+        evidence: z.ZodOptional<z.ZodString>;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        text: z.ZodString;
+        passed: z.ZodBoolean;
+        evidence: z.ZodOptional<z.ZodString>;
+    }, z.ZodTypeAny, "passthrough">>, "many">;
+    summary: z.ZodObject<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">>;
+}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+    expectations: z.ZodArray<z.ZodObject<{
+        text: z.ZodString;
+        passed: z.ZodBoolean;
+        evidence: z.ZodOptional<z.ZodString>;
+    }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+        text: z.ZodString;
+        passed: z.ZodBoolean;
+        evidence: z.ZodOptional<z.ZodString>;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        text: z.ZodString;
+        passed: z.ZodBoolean;
+        evidence: z.ZodOptional<z.ZodString>;
+    }, z.ZodTypeAny, "passthrough">>, "many">;
+    summary: z.ZodObject<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        passed: z.ZodNumber;
+        total: z.ZodNumber;
+        failed: z.ZodOptional<z.ZodNumber>;
+        pass_rate: z.ZodOptional<z.ZodNumber>;
+    }, z.ZodTypeAny, "passthrough">>;
+}, z.ZodTypeAny, "passthrough">>;
+export type GradingReport = z.infer<typeof GradingReportSchema>;
+/**
+ * Published JSON Schema for grading.json — generated from {@link GradingReportSchema}
+ * so the two never drift. Importable by external tooling that wants to validate a
+ * grading.json without depending on Zod. Documented in
+ * docs/skill-test-grading-schema.md.
+ */
+export declare const GradingReportJsonSchema: import("zod-to-json-schema").JsonSchema7Type & {
+    $schema?: string | undefined;
+    definitions?: {
+        [key: string]: import("zod-to-json-schema").JsonSchema7Type;
+    } | undefined;
+};
+//# sourceMappingURL=grading-schema.d.ts.map

package/dist/skill-test/grading-schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"grading-schema.d.ts","sourceRoot":"","sources":["../../src/skill-test/grading-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,sFAAsF;AACtF,eAAO,MAAM,uBAAuB;;;;;;;;;;;;gCAMpB,CAAC;AAEjB,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,oFAAoF;AACpF,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;gCAOjB,CAAC;AAEjB,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE;;;;GAIG;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAKhB,CAAC;AAEjB,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE;;;;;GAKG;AACH,eAAO,MAAM,uBAAuB;;;;;CAAyD,CAAC"}

package/dist/skill-test/grading-schema.js

@@ -0,0 +1,65 @@
+import { z } from 'zod';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+/**
+ * Canonical schema for the grader agent's `grading.json` output.
+ *
+ * SOURCE OF TRUTH: skill-creator's `references/schemas.md` (the `grading.json`
+ * section). vat consumes that exact shape; this module is the machine-checkable
+ * encoding of it (skill-creator ships prose + an example, but no JSON Schema).
+ * The published JSON Schema is {@link GradingReportJsonSchema}.
+ *
+ * SHAPE: a SINGLE flat JSON object with two load-bearing top-level fields —
+ *
+ *   {
+ *     "expectations": [ { "text": string, "passed": boolean, "evidence"?: string }, ... ],
+ *     "summary":      { "passed": number, "total": number, "failed"?: number, "pass_rate"?: number }
+ *   }
+ *
+ * `expectations` holds ONE entry per graded expectation across ALL evals — it is
+ * NOT grouped per-eval and is NEVER wrapped in an `evals` array. A per-eval
+ * nested shape (`{ evals: [ { expectations, summary } ] }`) is a contract
+ * violation and is rejected loudly (see grading-adapter.ts); tolerating it would
+ * push malformed data downstream and create confusion.
+ *
+ * LIBERAL ON EXTRAS (Postel): the grader legitimately emits additional documented
+ * sections — `execution_metrics`, `timing`, `claims`, `user_notes_summary`,
+ * `eval_feedback` — plus viewer URLs and other adornments. We `.passthrough()`
+ * those: validate the two fields we depend on, carry the rest untouched. Extra
+ * fields are NOT "bad JSON"; a wrong top-level STRUCTURE is.
+ */
+/** One graded expectation. `evidence` is recommended but not load-bearing for vat. */
+export const GradedExpectationSchema = z
+    .object({
+    text: z.string(),
+    passed: z.boolean(),
+    evidence: z.string().optional(),
+})
+    .passthrough();
+/** Aggregate pass/fail counts. `failed`/`pass_rate` are documented but optional. */
+export const GradingSummarySchema = z
+    .object({
+    passed: z.number(),
+    total: z.number(),
+    failed: z.number().optional(),
+    pass_rate: z.number().optional(),
+})
+    .passthrough();
+/**
+ * The full grading.json contract. Required: top-level `expectations[]` and
+ * `summary`. Everything else passes through untouched (forward-compatible with
+ * skill-creator additions).
+ */
+export const GradingReportSchema = z
+    .object({
+    expectations: z.array(GradedExpectationSchema),
+    summary: GradingSummarySchema,
+})
+    .passthrough();
+/**
+ * Published JSON Schema for grading.json — generated from {@link GradingReportSchema}
+ * so the two never drift. Importable by external tooling that wants to validate a
+ * grading.json without depending on Zod. Documented in
+ * docs/skill-test-grading-schema.md.
+ */
+export const GradingReportJsonSchema = zodToJsonSchema(GradingReportSchema, 'grading-report');
+//# sourceMappingURL=grading-schema.js.map

package/dist/skill-test/grading-schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grading-schema.js","sourceRoot":"","sources":["../../src/skill-test/grading-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,sFAAsF;AACtF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC;KACrC,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC;KACD,WAAW,EAAE,CAAC;AAIjB,oFAAoF;AACpF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC;KAClC,MAAM,CAAC;IACN,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACjC,CAAC;KACD,WAAW,EAAE,CAAC;AAIjB;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC;KACjC,MAAM,CAAC;IACN,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,uBAAuB,CAAC;IAC9C,OAAO,EAAE,oBAAoB;CAC9B,CAAC;KACD,WAAW,EAAE,CAAC;AAIjB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,eAAe,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,CAAC"}

package/dist/skill-test/harness-location.d.ts

@@ -17,6 +17,19 @@ export declare function resolveHarnessRoot(skillNames: string[], tmpRoot?: strin
  * --setting-sources "".
  */
 export declare function assertSafeWorkdir(dir: string): void;
+/**
+ * Prepare the harness root directory so that `assertSafeHarnessRoot` will
+ * pass on the next call. If the path does not exist, this is a no-op (the
+ * caller creates it at 0700 via mkdirSyncReal). If it exists:
+ *
+ * - Symlink → throw HarnessLocationError (security gate; never relax).
+ * - Real directory whose mode != 0700 → chmod to 0700. Removing group/other
+ *   access is strictly safer, never a relaxation.
+ *
+ * Mode checks/changes are only performed on non-win32 (matching
+ * assertSafeHarnessRoot's platform guard).
+ */
+export declare function prepareHarnessRoot(dir: string): void;
 /**
  * FS-bound hardening for the shared-tmp harness root (spec §7): the root must
  * be 0700 and owned by the current uid, and no path component may be a symlink.

package/dist/skill-test/harness-location.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"harness-location.d.ts","sourceRoot":"","sources":["../../src/skill-test/harness-location.ts"],"names":[],"mappings":"AAKA,sDAAsD;AACtD,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAOD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,MAAM,CAQ7D;AAED,mEAAmE;AACnE,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAGjF;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAenD;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,CAiB3E"}
+{"version":3,"file":"harness-location.d.ts","sourceRoot":"","sources":["../../src/skill-test/harness-location.ts"],"names":[],"mappings":"AAKA,sDAAsD;AACtD,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAOD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,MAAM,CAQ7D;AAED,mEAAmE;AACnE,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAGjF;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAenD;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAkBpD;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,CAiB3E"}