@vibe-agent-toolkit/agent-skills 0.1.39-rc.8 → 0.1.39

package/dist/skill-test/declared-env.js

@@ -0,0 +1,85 @@
+/**
+ * declared-env.ts — resolve and assemble the declared test environment
+ * (Features A + B) for the experimenter spawn.
+ *
+ * Feature A (passEnv): forward named host env vars if present.
+ * Feature B (env): inject explicit key→value pairs whose values support
+ * stage-time interpolation tokens. An unknown token is a hard preflight error.
+ *
+ * The security-critical union (protected-key collisions, redaction) lives in
+ * `applyDeclaredEnv`/`formatForwardedEnvLine` (utils); this module owns the
+ * agent-skills-specific concerns: token computation, interpolation, and gluing
+ * the result onto the plugin-root env.
+ */
+import { dirname } from 'node:path';
+import { applyDeclaredEnv, formatForwardedEnvLine, safePath, toForwardSlash } from '@vibe-agent-toolkit/utils';
+import { withPluginRootEnv } from './plugin-env.js';
+/**
+ * A declared `env` value referenced an unknown `${token}`. Exit 2 (preflight):
+ * fail loud naming the offending token rather than forwarding a literal `${x}`.
+ */
+export class UnknownEnvTokenError extends Error {
+    token;
+    key;
+    exitCode = 2;
+    constructor(token, key) {
+        super(`Unknown interpolation token \${${token}} in env value for "${key}". ` +
+            `Known tokens: fixturesDir, stagedSkillDir, harnessRoot, resultsDir.`);
+        this.token = token;
+        this.key = key;
+        this.name = 'UnknownEnvTokenError';
+    }
+}
+/**
+ * Compute the interpolation tokens from the known staged dirs. `fixturesDir`
+ * tracks the eval suite's directory (`<evalsDir>/fixtures`), so the default
+ * `evals/evals.json` yields `<staged>/evals/fixtures`.
+ */
+export function computeEnvTokens(inputs) {
+    const evalsDir = dirname(toForwardSlash(inputs.evalsSubpath));
+    return {
+        fixturesDir: safePath.join(inputs.subjectStagedDir, evalsDir, 'fixtures'),
+        stagedSkillDir: inputs.subjectStagedDir,
+        harnessRoot: inputs.harnessRoot,
+        resultsDir: inputs.resultsDir,
+    };
+}
+function lookupToken(name, key, tokens) {
+    switch (name) {
+        case 'fixturesDir': return tokens.fixturesDir;
+        case 'stagedSkillDir': return tokens.stagedSkillDir;
+        case 'harnessRoot': return tokens.harnessRoot;
+        case 'resultsDir': return tokens.resultsDir;
+        default: throw new UnknownEnvTokenError(name, key);
+    }
+}
+/** Interpolate every `${token}` in `value`. Throws UnknownEnvTokenError on an unknown token. */
+export function interpolateEnvValue(value, key, tokens) {
+    return value.replaceAll(/\$\{([^}]*)\}/g, (_match, name) => lookupToken(name, key, tokens));
+}
+/** Resolve every value in a declared `env` map. undefined in → undefined out. */
+export function resolveInjectEnv(injectEnv, tokens) {
+    if (injectEnv === undefined)
+        return undefined;
+    const out = {};
+    for (const [key, value] of Object.entries(injectEnv)) {
+        out[key] = interpolateEnvValue(value, key, tokens);
+    }
+    return out;
+}
+/**
+ * Union the declared test env onto `base`, add the plugin-root var, and render
+ * the transparency line. Protected-key collisions surface as warnings; secrets
+ * and pass-through values are redacted in the line.
+ */
+export function assembleChildEnv(input) {
+    const declared = applyDeclaredEnv(input.base, {
+        source: input.source,
+        ...(input.passEnv ? { passEnv: input.passEnv } : {}),
+        ...(input.injectEnv ? { injectEnv: input.injectEnv } : {}),
+    });
+    const env = withPluginRootEnv(declared.env, input.subjectPluginRoot);
+    const line = formatForwardedEnvLine(env, declared);
+    return { env, warnings: declared.warnings, line };
+}
+//# sourceMappingURL=declared-env.js.map

package/dist/skill-test/declared-env.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"declared-env.js","sourceRoot":"","sources":["../../src/skill-test/declared-env.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAE/G,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAUpD;;;GAGG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAEjB;IAA+B;IADlD,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAA4B,KAAa,EAAkB,GAAW;QACpE,KAAK,CACH,kCAAkC,KAAK,uBAAuB,GAAG,KAAK;YACpE,qEAAqE,CACxE,CAAC;QAJwB,UAAK,GAAL,KAAK,CAAQ;QAAkB,QAAG,GAAH,GAAG,CAAQ;QAKpE,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAUD;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAsB;IACrD,MAAM,QAAQ,GAAG,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;IAC9D,OAAO;QACL,WAAW,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,QAAQ,EAAE,UAAU,CAAC;QACzE,cAAc,EAAE,MAAM,CAAC,gBAAgB;QACvC,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,UAAU,EAAE,MAAM,CAAC,UAAU;KAC9B,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,GAAW,EAAE,MAA8B;IAC5E,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,aAAa,CAAC,CAAC,OAAO,MAAM,CAAC,WAAW,CAAC;QAC9C,KAAK,gBAAgB,CAAC,CAAC,OAAO,MAAM,CAAC,cAAc,CAAC;QACpD,KAAK,aAAa,CAAC,CAAC,OAAO,MAAM,CAAC,WAAW,CAAC;QAC9C,KAAK,YAAY,CAAC,CAAC,OAAO,MAAM,CAAC,UAAU,CAAC;QAC5C,OAAO,CAAC,CAAC,MAAM,IAAI,oBAAoB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACrD,CAAC;AACH,CAAC;AAED,gGAAgG;AAChG,MAAM,UAAU,mBAAmB,CAAC,KAAa,EAAE,GAAW,EAAE,MAA8B;IAC5F,OAAO,KAAK,CAAC,UAAU,CAAC,gBAAgB,EAAE,CAAC,MAAM,EAAE,IAAY,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;AACtG,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,gBAAgB,CAC9B,SAA6C,EAC7C,MAA8B;IAE9B,IAAI,SAAS,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC9C,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QACrD,GAAG,CAAC,GAAG,CAAC,GAAG,mBAAmB,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAsBD;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAA4B;IAC3D,MAAM,QAAQ,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,EAAE;QAC5C,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3D,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,iBAAiB,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC;IACrE,MAAM,IAAI,GAAG,sBAAsB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACnD,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC;AACpD,CAAC"}

package/dist/skill-test/eval-inputs.d.ts

@@ -0,0 +1,159 @@
+import { z } from 'zod';
+/** Raised for any eval-input problem (bad JSON, schema failure, missing input file). Maps to exit 2. */
+export declare class EvalInputError extends Error {
+    constructor(message: string);
+}
+export declare const EvalEntrySchema: z.ZodEffects<z.ZodObject<{
+    id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+    prompt: z.ZodString;
+    expected_output: z.ZodOptional<z.ZodString>;
+    files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    expectations: z.ZodArray<z.ZodString, "many">;
+}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+    id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+    prompt: z.ZodString;
+    expected_output: z.ZodOptional<z.ZodString>;
+    files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    expectations: z.ZodArray<z.ZodString, "many">;
+}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+    id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+    prompt: z.ZodString;
+    expected_output: z.ZodOptional<z.ZodString>;
+    files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    expectations: z.ZodArray<z.ZodString, "many">;
+}, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{
+    id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+    prompt: z.ZodString;
+    expected_output: z.ZodOptional<z.ZodString>;
+    files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    expectations: z.ZodArray<z.ZodString, "many">;
+}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+    id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+    prompt: z.ZodString;
+    expected_output: z.ZodOptional<z.ZodString>;
+    files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    expectations: z.ZodArray<z.ZodString, "many">;
+}, z.ZodTypeAny, "passthrough">>;
+export declare const EvalSuiteSchema: z.ZodObject<{
+    _comment: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    skill_name: z.ZodString;
+    evals: z.ZodArray<z.ZodEffects<z.ZodObject<{
+        id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+        prompt: z.ZodString;
+        expected_output: z.ZodOptional<z.ZodString>;
+        files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        expectations: z.ZodArray<z.ZodString, "many">;
+    }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+        id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+        prompt: z.ZodString;
+        expected_output: z.ZodOptional<z.ZodString>;
+        files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        expectations: z.ZodArray<z.ZodString, "many">;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+        prompt: z.ZodString;
+        expected_output: z.ZodOptional<z.ZodString>;
+        files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        expectations: z.ZodArray<z.ZodString, "many">;
+    }, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{
+        id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+        prompt: z.ZodString;
+        expected_output: z.ZodOptional<z.ZodString>;
+        files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        expectations: z.ZodArray<z.ZodString, "many">;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+        prompt: z.ZodString;
+        expected_output: z.ZodOptional<z.ZodString>;
+        files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        expectations: z.ZodArray<z.ZodString, "many">;
+    }, z.ZodTypeAny, "passthrough">>, "many">;
+}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+    _comment: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    skill_name: z.ZodString;
+    evals: z.ZodArray<z.ZodEffects<z.ZodObject<{
+        id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+        prompt: z.ZodString;
+        expected_output: z.ZodOptional<z.ZodString>;
+        files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        expectations: z.ZodArray<z.ZodString, "many">;
+    }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+        id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+        prompt: z.ZodString;
+        expected_output: z.ZodOptional<z.ZodString>;
+        files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        expectations: z.ZodArray<z.ZodString, "many">;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+        prompt: z.ZodString;
+        expected_output: z.ZodOptional<z.ZodString>;
+        files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        expectations: z.ZodArray<z.ZodString, "many">;
+    }, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{
+        id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+        prompt: z.ZodString;
+        expected_output: z.ZodOptional<z.ZodString>;
+        files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        expectations: z.ZodArray<z.ZodString, "many">;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+        prompt: z.ZodString;
+        expected_output: z.ZodOptional<z.ZodString>;
+        files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        expectations: z.ZodArray<z.ZodString, "many">;
+    }, z.ZodTypeAny, "passthrough">>, "many">;
+}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+    _comment: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    skill_name: z.ZodString;
+    evals: z.ZodArray<z.ZodEffects<z.ZodObject<{
+        id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+        prompt: z.ZodString;
+        expected_output: z.ZodOptional<z.ZodString>;
+        files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        expectations: z.ZodArray<z.ZodString, "many">;
+    }, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+        id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+        prompt: z.ZodString;
+        expected_output: z.ZodOptional<z.ZodString>;
+        files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        expectations: z.ZodArray<z.ZodString, "many">;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+        prompt: z.ZodString;
+        expected_output: z.ZodOptional<z.ZodString>;
+        files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        expectations: z.ZodArray<z.ZodString, "many">;
+    }, z.ZodTypeAny, "passthrough">>, z.objectOutputType<{
+        id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+        prompt: z.ZodString;
+        expected_output: z.ZodOptional<z.ZodString>;
+        files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        expectations: z.ZodArray<z.ZodString, "many">;
+    }, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+        id: z.ZodUnion<[z.ZodNumber, z.ZodString]>;
+        prompt: z.ZodString;
+        expected_output: z.ZodOptional<z.ZodString>;
+        files: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        expectations: z.ZodArray<z.ZodString, "many">;
+    }, z.ZodTypeAny, "passthrough">>, "many">;
+}, z.ZodTypeAny, "passthrough">>;
+export type EvalEntry = z.infer<typeof EvalEntrySchema>;
+export type EvalSuite = z.infer<typeof EvalSuiteSchema>;
+/** Parse + validate a skill-test eval suite. Throws {@link EvalInputError} on any problem. */
+export declare function parseEvalSuite(jsonText: string): EvalSuite;
+export interface StageEvalWorkspacesInput {
+    /** Parsed suite (Task 1). */
+    suite: EvalSuite;
+    /** Directory containing evals.json — the base for each eval's relative `files`. */
+    evalsDir: string;
+    /** `<harnessRoot>/workspaces` — per-eval dirs are created beneath it. */
+    workspacesRoot: string;
+}
+/**
+ * Materialize each eval's declared input `files` into `<workspacesRoot>/<id>/<relpath>`,
+ * preserving relative structure. Evals without `files` are skipped. Throws
+ * {@link EvalInputError} if a listed file does not exist (the eval cannot run without it).
+ * Returns `workspacesRoot`.
+ */
+export declare function stageEvalWorkspaces(input: StageEvalWorkspacesInput): string;
+//# sourceMappingURL=eval-inputs.d.ts.map

package/dist/skill-test/eval-inputs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-inputs.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wGAAwG;AACxG,qBAAa,cAAe,SAAQ,KAAK;gBAC3B,OAAO,EAAE,MAAM;CAI5B;AA4CD,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAmCxB,CAAC;AAEL,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAMZ,CAAC;AAEjB,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD,8FAA8F;AAC9F,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAmB1D;AAED,MAAM,WAAW,wBAAwB;IACvC,6BAA6B;IAC7B,KAAK,EAAE,SAAS,CAAC;IACjB,mFAAmF;IACnF,QAAQ,EAAE,MAAM,CAAC;IACjB,yEAAyE;IACzE,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,wBAAwB,GAAG,MAAM,CAkC3E"}

package/dist/skill-test/eval-inputs.js

@@ -0,0 +1,158 @@
+import { cpSync, existsSync } from 'node:fs';
+import { mkdirSyncReal, safePath } from '@vibe-agent-toolkit/utils';
+import { z } from 'zod';
+/** Raised for any eval-input problem (bad JSON, schema failure, missing input file). Maps to exit 2. */
+export class EvalInputError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'EvalInputError';
+    }
+}
+/**
+ * String eval ids name a per-eval working directory ({@link stageEvalWorkspaces}),
+ * so they must be safe path segments on every platform. Letters, digits, hyphen,
+ * and underscore only — this rejects `/`, `\`, `:`, spaces, `..`, and other
+ * filesystem-illegal characters that would otherwise fail (or behave
+ * inconsistently) on Windows. Descriptive adopter ids like `dollar-quote-recovery`
+ * pass unchanged; `year:extraction` is rejected at parse with a clear message
+ * instead of failing later as an opaque copy/escape error.
+ */
+const EVAL_ID_PATTERN = /^[A-Za-z0-9_-]+$/;
+/** Fields VAT recognizes per eval. Unknown fields are allowed (passthrough); a
+ *  near-miss of one of these is flagged as a likely typo (see superRefine). */
+const RECOGNIZED_EVAL_FIELDS = ['id', 'prompt', 'expected_output', 'files', 'expectations'];
+/**
+ * True when `key` is exactly one edit (insert/delete/substitute one char) from
+ * `target`. Used only to catch typos of recognized fields — deliberately a tiny
+ * single-edit check, not a general edit-distance routine, so it stays cheap and
+ * never fires on legitimately distinct adopter keys (`name`, `category`, `notes`).
+ */
+function isSingleEditAway(key, target) {
+    if (key === target)
+        return false;
+    const lk = key.length;
+    const lt = target.length;
+    if (Math.abs(lk - lt) > 1)
+        return false;
+    let i = 0;
+    while (i < lk && i < lt && key[i] === target[i])
+        i++;
+    if (lk === lt)
+        return key.slice(i + 1) === target.slice(i + 1); // substitution
+    if (lk > lt)
+        return key.slice(i + 1) === target.slice(i); // deletion from key
+    return key.slice(i) === target.slice(i + 1); // insertion into key
+}
+// evals.json is adopter-authored input that VAT *reads* — so per the project's
+// Postel's Law (read the outside world liberally), we validate only the fields
+// VAT actually consumes and pass everything else through untouched. `id` accepts
+// a string OR an int: skill-creator's methodology encourages *descriptive* eval
+// identifiers, and real adopter suites (e.g. dxa) use descriptive string ids plus
+// adopter-owned metadata like `category` / top-level `_category_note`. The
+// load-bearing fields stay required, so a typo in a REQUIRED field is caught by
+// its absence; a near-miss typo of the OPTIONAL `files` field (which would
+// otherwise be silently swallowed by passthrough) is caught by the superRefine.
+export const EvalEntrySchema = z
+    .object({
+    id: z.union([
+        z.number().int(),
+        z
+            .string()
+            .min(1)
+            .regex(EVAL_ID_PATTERN, 'string eval id must contain only letters, digits, hyphen, or underscore (it names a working directory)'),
+    ]),
+    prompt: z.string().min(1),
+    // Optional: a human-readable success description. The pass/fail verdict is
+    // always decided per `expectations` entry, so this is not load-bearing and
+    // (per Postel's Law) is not required — real adopter suites (e.g. dxa-consumption)
+    // grade with `expectations` alone. When present, the experimenter prompt feeds it
+    // to the grader as prose CONTEXT informing judgment (see experimenter-prompt.ts).
+    expected_output: z.string().min(1).optional(),
+    files: z.array(z.string().min(1)).optional(),
+    expectations: z.array(z.string().min(1)).min(1),
+})
+    .passthrough()
+    .superRefine((entry, ctx) => {
+    for (const key of Object.keys(entry)) {
+        if (RECOGNIZED_EVAL_FIELDS.includes(key))
+            continue;
+        const near = RECOGNIZED_EVAL_FIELDS.find((field) => isSingleEditAway(key.toLowerCase(), field));
+        if (near !== undefined) {
+            ctx.addIssue({
+                code: z.ZodIssueCode.custom,
+                message: `unknown eval field "${key}" — did you mean "${near}"? (other custom fields are allowed and ignored)`,
+                path: [key],
+            });
+        }
+    }
+});
+export const EvalSuiteSchema = z
+    .object({
+    _comment: z.array(z.string()).optional(),
+    skill_name: z.string().min(1),
+    evals: z.array(EvalEntrySchema).min(1),
+})
+    .passthrough();
+/** Parse + validate a skill-test eval suite. Throws {@link EvalInputError} on any problem. */
+export function parseEvalSuite(jsonText) {
+    let raw;
+    try {
+        raw = JSON.parse(jsonText);
+    }
+    catch (e) {
+        throw new EvalInputError(`evals.json is not valid JSON: ${e.message}`);
+    }
+    const result = EvalSuiteSchema.safeParse(raw);
+    if (!result.success) {
+        throw new EvalInputError(`evals.json failed schema validation: ${result.error.message}`);
+    }
+    // Compare ids as strings: each id names a working directory via String(id),
+    // so a numeric `1` and a string `"1"` would collide on disk even though they
+    // are distinct JS values. Dedup on the stringified form to catch that.
+    const ids = result.data.evals.map((e) => String(e.id));
+    if (new Set(ids).size !== ids.length) {
+        throw new EvalInputError('eval ids must be unique within a suite (ids are compared as strings, so 1 and "1" collide)');
+    }
+    return result.data;
+}
+/**
+ * Materialize each eval's declared input `files` into `<workspacesRoot>/<id>/<relpath>`,
+ * preserving relative structure. Evals without `files` are skipped. Throws
+ * {@link EvalInputError} if a listed file does not exist (the eval cannot run without it).
+ * Returns `workspacesRoot`.
+ */
+export function stageEvalWorkspaces(input) {
+    for (const entry of input.suite.evals) {
+        if (entry.files === undefined || entry.files.length === 0)
+            continue;
+        const evalWorkspace = safePath.joinUnderRoot(input.workspacesRoot, String(entry.id));
+        for (const rel of entry.files) {
+            // Containment first: a `rel` that escapes evalsDir or the workspace is a
+            // genuine "escapes the eval directory" problem and is reported as such.
+            let src;
+            let dest;
+            try {
+                src = safePath.joinUnderRoot(input.evalsDir, rel);
+                dest = safePath.joinUnderRoot(evalWorkspace, rel);
+            }
+            catch (err) {
+                throw new EvalInputError(`eval ${entry.id} declares input file "${rel}" that escapes the eval directory: ${err.message}`);
+            }
+            // eslint-disable-next-line security/detect-non-literal-fs-filename -- src is contained under evalsDir via joinUnderRoot; suite is developer-authored
+            if (!existsSync(src)) {
+                throw new EvalInputError(`eval ${entry.id} declares input file "${rel}" but it is absent at ${src}`);
+            }
+            // Copy failures (permissions, illegal filename on the host, disk) are
+            // reported accurately rather than mislabeled as a containment escape.
+            try {
+                mkdirSyncReal(safePath.join(dest, '..'), { recursive: true });
+                cpSync(src, dest, { recursive: true });
+            }
+            catch (err) {
+                throw new EvalInputError(`eval ${entry.id} failed to stage input file "${rel}" into the workspace: ${err.message}`);
+            }
+        }
+    }
+    return input.workspacesRoot;
+}
+//# sourceMappingURL=eval-inputs.js.map

package/dist/skill-test/eval-inputs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-inputs.js","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE7C,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wGAAwG;AACxG,MAAM,OAAO,cAAe,SAAQ,KAAK;IACvC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED;;;;;;;;GAQG;AACH,MAAM,eAAe,GAAG,kBAAkB,CAAC;AAE3C;+EAC+E;AAC/E,MAAM,sBAAsB,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,iBAAiB,EAAE,OAAO,EAAE,cAAc,CAAU,CAAC;AAErG;;;;;GAKG;AACH,SAAS,gBAAgB,CAAC,GAAW,EAAE,MAAc;IACnD,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC;IACjC,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;IACtB,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;IACzB,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;QAAE,CAAC,EAAE,CAAC;IACrD,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe;IAC/E,IAAI,EAAE,GAAG,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,oBAAoB;IAC9E,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB;AACpE,CAAC;AAED,+EAA+E;AAC/E,+EAA+E;AAC/E,iFAAiF;AACjF,gFAAgF;AAChF,kFAAkF;AAClF,2EAA2E;AAC3E,gFAAgF;AAChF,2EAA2E;AAC3E,gFAAgF;AAChF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;QACV,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;QAChB,CAAC;aACE,MAAM,EAAE;aACR,GAAG,CAAC,CAAC,CAAC;aACN,KAAK,CACJ,eAAe,EACf,wGAAwG,CACzG;KACJ,CAAC;IACF,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,2EAA2E;IAC3E,2EAA2E;IAC3E,kFAAkF;IAClF,kFAAkF;IAClF,kFAAkF;IAClF,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC7C,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5C,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CAChD,CAAC;KACD,WAAW,EAAE;KACb,WAAW,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;IAC1B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAK,sBAA4C,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,SAAS;QAC1E,MAAM,IAAI,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;QAChG,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,CAAC,QAAQ,CAAC;gBACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;gBAC3B,OAAO,EAAE,uBAAuB,GAAG,qBAAqB,IAAI,kDAAkD;gBAC9G,IAAI,EAAE,CAAC,GAAG,CAAC;aACZ,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACxC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACvC,CAAC;KACD,WAAW,EAAE,CAAC;AAKjB,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,cAAc,CAAC,iCAAkC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,cAAc,CAAC,wCAAwC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3F,CAAC;IACD,4EAA4E;IAC5E,6EAA6E;IAC7E,uEAAuE;IACvE,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,IAAI,cAAc,CAAC,4FAA4F,CAAC,CAAC;IACzH,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAWD;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA+B;IACjE,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACpE,MAAM,aAAa,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QACrF,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAC9B,yEAAyE;YACzE,wEAAwE;YACxE,IAAI,GAAW,CAAC;YAChB,IAAI,IAAY,CAAC;YACjB,IAAI,CAAC;gBACH,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAClD,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,yBAAyB,GAAG,sCAAuC,GAAa,CAAC,OAAO,EAAE,CAC3G,CAAC;YACJ,CAAC;YACD,qJAAqJ;YACrJ,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,cAAc,CAAC,QAAQ,KAAK,CAAC,EAAE,yBAAyB,GAAG,yBAAyB,GAAG,EAAE,CAAC,CAAC;YACvG,CAAC;YACD,sEAAsE;YACtE,sEAAsE;YACtE,IAAI,CAAC;gBACH,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC9D,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,gCAAgC,GAAG,yBAA0B,GAAa,CAAC,OAAO,EAAE,CACrG,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,cAAc,CAAC;AAC9B,CAAC"}

package/dist/skill-test/evals-template.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evals-template.d.ts","sourceRoot":"","sources":["../../src/skill-test/evals-template.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,2EAA2E;AAC3E,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CA0B5D;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAM/E"}
+{"version":3,"file":"evals-template.d.ts","sourceRoot":"","sources":["../../src/skill-test/evals-template.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,2EAA2E;AAC3E,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAsC5D;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAW/E"}

package/dist/skill-test/evals-template.js

@@ -12,7 +12,7 @@
  * Both are stripped by the user as they fill in real evals; the harness does not
  * read `_comment`.
  */
-import { writeFileSync } from 'node:fs';
+import { existsSync, writeFileSync } from 'node:fs';
 import { mkdirSyncReal, safePath } from '@vibe-agent-toolkit/utils';
 /** Build the annotated evals.json template text for a given skill name. */
 export function buildEvalsTemplate(skillName) {
@@ -21,21 +21,33 @@ export function buildEvalsTemplate(skillName) {
             'This is a starter evals.json scaffolded by `vat skill test`.',
             'Replace the placeholder eval below with one entry per behavior you want to verify.',
             'Fields per eval:',
-            '  id              — unique integer identifier',
-            '  prompt          — the task prompt handed to the executor (it is NOT told it is being tested)',
-            '  expected_output — human-readable description of what success looks like',
-            '  files           — OPTIONAL list of input file paths relative to the skill root',
-            '  expectations    — list of verifiable statements the grader checks against the output',
-            'Delete this _comment field once you have filled in real evals, then re-run.',
+            '  id              — unique identifier: an integer (1) or a short descriptive',
+            '                    string ("year-extraction"). Descriptive ids read better in results.',
+            '                    A string id names a working directory, so use only letters,',
+            '                    digits, hyphen, or underscore (no spaces, slashes, or colons).',
+            '  prompt          — the task, written the way a REAL user would phrase it. The executor',
+            '                    is NOT told it is being tested, so do not mention testing/evals here.',
+            '  expected_output — OPTIONAL human-readable description of what a correct result looks like',
+            '  files           — OPTIONAL input files (paths relative to THIS evals.json dir),',
+            '                    staged into a per-eval working directory the executor operates on',
+            '  expectations    — verifiable statements the grader checks. Make them DISCRIMINATING:',
+            '                    a wrong or hallucinated output should FAIL them (presence != correctness).',
+            '                    Include negative assertions too, e.g. "does NOT claim every row reconciles".',
+            '  category        — OPTIONAL label you choose to group evals (e.g. "recognition",',
+            '                    "guidance", "recovery"). VAT ignores it; it is for your own organization.',
+            'Aim for at least 3 evals covering real scenarios. Delete this _comment field once filled in, then re-run.',
         ],
         skill_name: skillName,
         evals: [
             {
                 id: 1,
-                prompt: 'TODO: describe the task a user would ask this skill to perform.',
+                category: 'TODO: optional grouping label, or remove this field',
+                prompt: 'TODO: a realistic task a user would actually ask this skill to perform.',
                 expected_output: 'TODO: describe what a correct result looks like.',
+                files: [],
                 expectations: [
-                    'TODO: a verifiable statement about the output, e.g. "The output includes X".',
+                    'TODO: a verifiable statement a correct output satisfies, e.g. "The output reports a total of 29500".',
+                    'TODO: a negative assertion a wrong output would trip, e.g. "The output does NOT claim the file is empty".',
                 ],
             },
         ],
@@ -49,8 +61,13 @@ export function buildEvalsTemplate(skillName) {
 export function writeEvalsTemplate(evalsPath, skillName) {
     const parent = safePath.join(evalsPath, '..');
     mkdirSyncReal(parent, { recursive: true });
+    // Defensive: never clobber an authored eval suite. Callers only reach here when
+    // the suite is absent, but guard so a stray call can never destroy real evals.
     // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own resolved scaffold path
-    writeFileSync(evalsPath, buildEvalsTemplate(skillName), 'utf8');
+    if (!existsSync(evalsPath)) {
+        // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own resolved scaffold path
+        writeFileSync(evalsPath, buildEvalsTemplate(skillName), 'utf8');
+    }
     return evalsPath;
 }
 //# sourceMappingURL=evals-template.js.map

package/dist/skill-test/evals-template.js.map

@@ -1 +1 @@
-{"version":3,"file":"evals-template.js","sourceRoot":"","sources":["../../src/skill-test/evals-template.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAEpE,2EAA2E;AAC3E,MAAM,UAAU,kBAAkB,CAAC,SAAiB;IAClD,MAAM,QAAQ,GAAG;QACf,QAAQ,EAAE;YACR,8DAA8D;YAC9D,oFAAoF;YACpF,kBAAkB;YAClB,+CAA+C;YAC/C,gGAAgG;YAChG,2EAA2E;YAC3E,kFAAkF;YAClF,wFAAwF;YACxF,6EAA6E;SAC9E;QACD,UAAU,EAAE,SAAS;QACrB,KAAK,EAAE;YACL;gBACE,EAAE,EAAE,CAAC;gBACL,MAAM,EAAE,iEAAiE;gBACzE,eAAe,EAAE,kDAAkD;gBACnE,YAAY,EAAE;oBACZ,8EAA8E;iBAC/E;aACF;SACF;KACF,CAAC;IACF,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;AAClD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAAiB,EAAE,SAAiB;IACrE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC9C,aAAa,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,qGAAqG;IACrG,aAAa,CAAC,SAAS,EAAE,kBAAkB,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC;IAChE,OAAO,SAAS,CAAC;AACnB,CAAC"}
+{"version":3,"file":"evals-template.js","sourceRoot":"","sources":["../../src/skill-test/evals-template.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEpD,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAEpE,2EAA2E;AAC3E,MAAM,UAAU,kBAAkB,CAAC,SAAiB;IAClD,MAAM,QAAQ,GAAG;QACf,QAAQ,EAAE;YACR,8DAA8D;YAC9D,oFAAoF;YACpF,kBAAkB;YAClB,8EAA8E;YAC9E,yFAAyF;YACzF,iFAAiF;YACjF,oFAAoF;YACpF,yFAAyF;YACzF,2FAA2F;YAC3F,6FAA6F;YAC7F,mFAAmF;YACnF,uFAAuF;YACvF,wFAAwF;YACxF,gGAAgG;YAChG,kGAAkG;YAClG,mFAAmF;YACnF,+FAA+F;YAC/F,2GAA2G;SAC5G;QACD,UAAU,EAAE,SAAS;QACrB,KAAK,EAAE;YACL;gBACE,EAAE,EAAE,CAAC;gBACL,QAAQ,EAAE,qDAAqD;gBAC/D,MAAM,EAAE,yEAAyE;gBACjF,eAAe,EAAE,kDAAkD;gBACnE,KAAK,EAAE,EAAE;gBACT,YAAY,EAAE;oBACZ,sGAAsG;oBACtG,2GAA2G;iBAC5G;aACF;SACF;KACF,CAAC;IACF,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;AAClD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAAiB,EAAE,SAAiB;IACrE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC9C,aAAa,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,gFAAgF;IAChF,+EAA+E;IAC/E,qGAAqG;IACrG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,qGAAqG;QACrG,aAAa,CAAC,SAAS,EAAE,kBAAkB,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/dist/skill-test/exit-codes.d.ts

@@ -4,17 +4,48 @@ export declare const SkillTestExitCode: {
     readonly Internal: 1;
     readonly Preflight: 2;
     readonly Bootstrap: 3;
+    /**
+     * At least one eval FAILED and the caller opted into eval-gating via
+     * `--fail-on-eval-failure`. Distinct from the harness-broke codes (1/2):
+     * the harness ran to completion and produced a valid grading.json — the
+     * skill's expectations simply did not all pass. Returned DIRECTLY by the
+     * harness verdict (see verdictExitCode in run-harness.ts), not via
+     * mapErrorToExitCode, because it is an outcome, not a thrown error.
+     */
+    readonly EvalFailure: 4;
 };
 export type SkillTestExitCodeValue = (typeof SkillTestExitCode)[keyof typeof SkillTestExitCode];
 /**
  * A required input is absent in a way vat can scaffold (missing evals.json).
- * Exit 3, NOT a failure. `expectedPath` is the real, persistent location where
- * the harness wrote the annotated starter template for the user to fill in.
+ * Exit 3, NOT a failure. `expectedPath` is the persistent location of the
+ * annotated starter template.
+ *
+ * In a real run the harness has already written that template, and the message
+ * says so. Under `--dry-run` nothing is written — a dry run must never touch the
+ * filesystem — so the message instead describes what a real run *would* scaffold
+ * and where, while surfacing the same exit-3 "bootstrap needed" signal.
  */
 export declare class BootstrapNeededError extends Error {
     readonly expectedPath: string;
     readonly exitCode: 3;
-    constructor(expectedPath: string);
+    constructor(expectedPath: string, opts?: {
+        dryRun?: boolean;
+    });
+}
+/** Thrown when building a declared skill (pool packageSkill or plugin build) fails. Exit 2 (preflight class). */
+export declare class SkillBuildError extends Error {
+    readonly exitCode: 2;
+    constructor(message: string);
+}
+/**
+ * The §12 security acknowledgment is required but absent, thrown BEFORE any
+ * build/pre-stage command (which executes untrusted repo code) runs for a
+ * buildable subject. Exit 2 (preflight class). Mirrors the harness Step-6 ack
+ * message wording so the two enforcement points read identically.
+ */
+export declare class SecurityAckError extends Error {
+    readonly exitCode: 2;
+    constructor();
 }
 /** Internal harness failure (incl. experimenter exiting without valid grading.json). Exit 1. */
 export declare class InternalHarnessError extends Error {
@@ -25,8 +56,16 @@ export declare class InternalHarnessError extends Error {
  * Map any thrown error to the process exit code. Errors that carry their own
  * `exitCode` (Bootstrap/Auth/HarnessLocation/Internal) are authoritative;
  * a PromptInvariantError is a user-correctable preflight problem (a supplied
- * prompt override is missing a required safety instruction) → 2; GradingSkewError
- * is a parse failure → 1; everything unknown → 1.
+ * prompt override is missing a required safety instruction) → 2; a BuildHookError
+ * is a pre-stage build failure → 2; a SkillBuildError is a declared-skill build
+ * failure → 2; a SecurityAckError is a missing security ack before a build → 2;
+ * an UnknownEnvTokenError is a bad ${token} in a
+ * declared env value → 2; GradingSkewError (parse failure) and GradingNonceError
+ * (forged/mismatched grading nonce) are both → 1; everything unknown → 1.
+ *
+ * Note: SkillTestExitCode.EvalFailure (4) is NOT produced here — it is an
+ * outcome of a completed run (an eval failed under `--fail-on-eval-failure`),
+ * not a thrown error, so the harness returns it directly (see verdictExitCode).
  */
 export declare function mapErrorToExitCode(err: unknown): number;
 //# sourceMappingURL=exit-codes.d.ts.map

package/dist/skill-test/exit-codes.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"exit-codes.d.ts","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AAMA,kDAAkD;AAClD,eAAO,MAAM,iBAAiB;;;;;CAKpB,CAAC;AAEX,MAAM,MAAM,sBAAsB,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,OAAO,iBAAiB,CAAC,CAAC;AAEhG;;;;GAIG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;aAEjB,YAAY,EAAE,MAAM;IADhD,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACH,YAAY,EAAE,MAAM;CAIjD;AAED,gGAAgG;AAChG,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAWvD"}
+{"version":3,"file":"exit-codes.d.ts","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AAQA,kDAAkD;AAClD,eAAO,MAAM,iBAAiB;;;;;IAK5B;;;;;;;OAOG;;CAEK,CAAC;AAEX,MAAM,MAAM,sBAAsB,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,OAAO,iBAAiB,CAAC,CAAC;AAEhG;;;;;;;;;GASG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;aAEjB,YAAY,EAAE,MAAM;IADhD,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACH,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE;CAQ9E;AAED,iHAAiH;AACjH,qBAAa,eAAgB,SAAQ,KAAK;IACxC,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAED;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;;CAKhC;AAED,gGAAgG;AAChG,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAqBvD"}