@vibe-agent-toolkit/agent-skills 0.1.40 → 0.1.41-rc.2

package/dist/skill-test/eval-lint.js

@@ -0,0 +1,157 @@
+/**
+ * Advisory (never-fatal) lint over an eval suite's `expectations` prose (issue #145
+ * follow-up). An adopter reported that WEAK expectations — pure "mentions X" /
+ * "includes Y" presence checks with no discriminating or negative counterpart —
+ * pass even for a hallucinated or right-for-the-wrong-reason answer, giving false
+ * confidence in a green run. This module flags that pattern so authors can
+ * strengthen the eval; it never blocks a run or changes an exit code.
+ *
+ * Deliberately conservative: this is a heuristic over free-text prose, not a
+ * grammar. When in doubt, do NOT warn — a false "your eval is weak" nag is worse
+ * than a missed one, since the grader (an LLM judging the transcript) is the real
+ * backstop for eval quality.
+ */
+import { isSingleEditAway } from './eval-inputs.js';
+// An expectation "looks like" a pure presence/mention check when it uses one of
+// these verbs — the vocabulary authors reach for when asserting "X shows up
+// somewhere in the output" without saying what a WRONG output would look like.
+const PRESENCE_ONLY_PATTERN = /\b(mentions?|includes?|contains?|references?|present|appears?|states?)\b/i;
+// Any of these cues make an expectation discriminating, regardless of whether it
+// also uses a presence verb above: negation ("not"/"never"/"without"/"must not"),
+// an explicit alternative ("instead of"/"rather than"), a precision bound
+// ("only"/"exactly"), or a named failure mode ("wrong"/"incorrect"/"fails"/
+// "error"). ONE such cue on an expectation is enough to exempt it from the
+// presence-only bucket — a single negative check already rules out a hallucinated
+// or wrong-for-the-right-reason answer for that expectation.
+const DISCRIMINATING_PATTERN = /\b(not|never|without|must not|should not|instead of|rather than|wrong|incorrect|only|exactly|fails?|error)\b/i;
+/** True when a single expectation string is a presence-only check (matches the presence vocabulary and carries no discriminating cue). */
+function isPresenceOnlyExpectation(expectation) {
+    return PRESENCE_ONLY_PATTERN.test(expectation) && !DISCRIMINATING_PATTERN.test(expectation);
+}
+/**
+ * Lint a parsed eval suite's `expectations` for the weak-presence-only pattern.
+ * Flags an eval only when ALL of these hold (conservative — minimize false
+ * positives):
+ *   (a) it has at least one expectation;
+ *   (b) EVERY expectation string is presence-only (see {@link isPresenceOnlyExpectation});
+ *   (c) the eval declares no `toolExpectations` — a `mustRun`/`mustNotRun`/`sequence`
+ *       assertion is itself discriminating (it can fail a transcript that never
+ *       invoked the right tool), so its presence already covers the concern this
+ *       lint exists to catch.
+ *
+ * Pure and side-effect free — callers decide how/where to surface the warnings
+ * (this never throws and never affects exit code).
+ */
+export function lintEvalExpectations(evals) {
+    const warnings = [];
+    for (const entry of evals) {
+        if (entry.expectations.length === 0)
+            continue;
+        if (entry.toolExpectations !== undefined)
+            continue;
+        if (!entry.expectations.every(isPresenceOnlyExpectation))
+            continue;
+        const evalId = String(entry.id);
+        warnings.push({
+            evalId,
+            message: `eval "${evalId}": all expectations are presence-only ("mentions/includes …") with no discriminating ` +
+                'or negative check — a hallucinated or wrong-for-the-right-reason answer may still pass. Add a negative ' +
+                'expectation or a toolExpectations assertion.',
+        });
+    }
+    return warnings;
+}
+// ---------------------------------------------------------------------------
+// Undeclared-executable lint (issue #145 adopter follow-up)
+// ---------------------------------------------------------------------------
+// A trailing script/binary extension we strip before comparing an executable
+// name to a declared one, so `dxa.py`/`./dxa`/`dxa` all normalize to `dxa` (the
+// grader already recognizes those launch forms — see deriveDeclaredExecutableNames,
+// which strips the extension from a declared name too). Matches only well-known
+// script suffixes so a name that legitimately ENDS in one of these words is left
+// alone.
+const SCRIPT_EXTENSION = /\.(?:py|js|mjs|cjs|ts|sh|rb|go|pl|php)$/i;
+/** Normalize an executable name for comparison: trim, drop a leading `./` and a trailing script extension, lowercase. */
+function normalizeExecutableName(name) {
+    return name.trim().replace(/^\.\//, '').replace(SCRIPT_EXTENSION, '').toLowerCase();
+}
+/** Every executable named across an eval's `toolExpectations` arrays (mustRun/mustNotRun/mustSucceed/sequence). */
+function collectToolExpectationExecutables(entry) {
+    const te = entry.toolExpectations;
+    if (te === undefined)
+        return [];
+    return [...(te.mustRun ?? []), ...(te.mustNotRun ?? []), ...(te.mustSucceed ?? []), ...(te.sequence ?? [])];
+}
+/**
+ * True when `decorated` is `stem` plus a SEPARATOR-delimited suffix — i.e. `stem`
+ * is a ≥3-char prefix of `decorated` and the next char is a non-alphanumeric
+ * separator (`-`, `_`, `.`, space). Catches a declared `dxa` typo'd as `dxa-py`
+ * (a `-py` decoration) while NOT firing on a genuinely different word that merely
+ * shares a prefix (`git` vs `github` — the boundary char `h` is alphanumeric).
+ */
+function isDecoratedStem(decorated, stem) {
+    if (stem.length < 3 || decorated.length <= stem.length || !decorated.startsWith(stem))
+        return false;
+    return /[^a-z0-9]/.test(decorated.charAt(stem.length));
+}
+/**
+ * The declared executable a referenced `name` is PROBABLY a typo of, or undefined
+ * when it matches a declared name exactly (recognized — no warning) or is not
+ * close to any declared name (a legit reference to a built-in/system tool like
+ * `Bash`/`git` — also no warning, staying conservative like {@link lintEvalExpectations}).
+ * "Close" = a single-edit typo of, or a separator-decorated form of, a declared name.
+ */
+function probableDeclaredTypoTarget(name, declared) {
+    const normalized = normalizeExecutableName(name);
+    if (declared.some((d) => d.normalized === normalized))
+        return undefined; // exact / recognized launch form
+    for (const d of declared) {
+        if (isSingleEditAway(normalized, d.normalized))
+            return d.original;
+        if (isDecoratedStem(normalized, d.normalized) || isDecoratedStem(d.normalized, normalized))
+            return d.original;
+    }
+    return undefined;
+}
+/**
+ * Advisory (never-fatal) lint: flag a `toolExpectations` entry that names an
+ * executable which looks like a TYPO of one of the skill's `declaredExecutables`
+ * (issue #145 adopter follow-up). A name that never matches a real tool is a quiet
+ * footgun — `mustRun`/`mustSucceed`/`sequence` then fail for the wrong reason, and
+ * `mustNotRun` passes vacuously (the tool "never ran" only because the name is
+ * wrong). This is zero-token static plumbing, run before any spend.
+ *
+ * Conservative, like {@link lintEvalExpectations}: it only fires when there is a
+ * SPECIFIC declared name the reference is probably a typo of (so it can suggest
+ * "did you mean X?"). A reference that matches a declared name exactly, or that is
+ * not close to any declared name (a deliberate built-in/system tool reference), is
+ * never flagged. When the skill declares no executables there is nothing to compare
+ * against, so it returns no warnings. Pure + side-effect free.
+ */
+export function lintToolExpectationExecutables(evals, declaredExecutableNames) {
+    const warnings = [];
+    const declared = declaredExecutableNames.map((original) => ({ original, normalized: normalizeExecutableName(original) }));
+    if (declared.length === 0)
+        return warnings;
+    const declaredList = declared.map((d) => d.original).join(', ');
+    for (const entry of evals) {
+        const evalId = String(entry.id);
+        const seen = new Set();
+        for (const name of collectToolExpectationExecutables(entry)) {
+            if (seen.has(name))
+                continue;
+            seen.add(name);
+            const target = probableDeclaredTypoTarget(name, declared);
+            if (target === undefined)
+                continue;
+            warnings.push({
+                evalId,
+                message: `eval "${evalId}": toolExpectation references executable "${name}", which no declared executable matches — ` +
+                    `did you mean "${target}"? A name that never matches a real tool makes mustRun/mustSucceed/sequence fail ` +
+                    `for the wrong reason (or mustNotRun pass vacuously). Declared executables: ${declaredList}.`,
+            });
+        }
+    }
+    return warnings;
+}
+//# sourceMappingURL=eval-lint.js.map

package/dist/skill-test/eval-lint.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-lint.js","sourceRoot":"","sources":["../../src/skill-test/eval-lint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,gBAAgB,EAAkB,MAAM,kBAAkB,CAAC;AAQpE,gFAAgF;AAChF,4EAA4E;AAC5E,+EAA+E;AAC/E,MAAM,qBAAqB,GAAG,2EAA2E,CAAC;AAE1G,iFAAiF;AACjF,kFAAkF;AAClF,0EAA0E;AAC1E,4EAA4E;AAC5E,2EAA2E;AAC3E,kFAAkF;AAClF,6DAA6D;AAC7D,MAAM,sBAAsB,GAC1B,+GAA+G,CAAC;AAElH,0IAA0I;AAC1I,SAAS,yBAAyB,CAAC,WAAmB;IACpD,OAAO,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AAC9F,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAkB;IACrD,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC9C,IAAI,KAAK,CAAC,gBAAgB,KAAK,SAAS;YAAE,SAAS;QACnD,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,yBAAyB,CAAC;YAAE,SAAS;QACnE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM;YACN,OAAO,EACL,SAAS,MAAM,uFAAuF;gBACtG,yGAAyG;gBACzG,8CAA8C;SACjD,CAAC,CAAC;IACL,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,8EAA8E;AAC9E,4DAA4D;AAC5D,8EAA8E;AAE9E,6EAA6E;AAC7E,gFAAgF;AAChF,oFAAoF;AACpF,gFAAgF;AAChF,iFAAiF;AACjF,SAAS;AACT,MAAM,gBAAgB,GAAG,0CAA0C,CAAC;AAEpE,yHAAyH;AACzH,SAAS,uBAAuB,CAAC,IAAY;IAC3C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AACtF,CAAC;AAED,mHAAmH;AACnH,SAAS,iCAAiC,CAAC,KAAgB;IACzD,MAAM,EAAE,GAAG,KAAK,CAAC,gBAAgB,CAAC;IAClC,IAAI,EAAE,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAChC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC;AAC9G,CAAC;AAED;;;;;;GAMG;AACH,SAAS,eAAe,CAAC,SAAiB,EAAE,IAAY;IACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACpG,OAAO,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;AACzD,CAAC;AAED;;;;;;GAMG;AACH,SAAS,0BAA0B,CACjC,IAAY,EACZ,QAAoD;IAEpD,MAAM,UAAU,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IACjD,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,UAAU,CAAC;QAAE,OAAO,SAAS,CAAC,CAAC,iCAAiC;IAC1G,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,gBAAgB,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;QAClE,IAAI,eAAe,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,IAAI,eAAe,CAAC,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;IAChH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,8BAA8B,CAC5C,KAAkB,EAClB,uBAAiC;IAEjC,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,uBAAuB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1H,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC3C,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChE,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,iCAAiC,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5D,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACf,MAAM,MAAM,GAAG,0BAA0B,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAC1D,IAAI,MAAM,KAAK,SAAS;gBAAE,SAAS;YACnC,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM;gBACN,OAAO,EACL,SAAS,MAAM,6CAA6C,IAAI,4CAA4C;oBAC5G,iBAAiB,MAAM,mFAAmF;oBAC1G,8EAA8E,YAAY,GAAG;aAChG,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}

package/dist/skill-test/evals-template.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evals-template.d.ts","sourceRoot":"","sources":["../../src/skill-test/evals-template.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,2EAA2E;AAC3E,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAsC5D;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAW/E"}
+{"version":3,"file":"evals-template.d.ts","sourceRoot":"","sources":["../../src/skill-test/evals-template.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,2EAA2E;AAC3E,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CA6C5D;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAW/E"}

package/dist/skill-test/evals-template.js

@@ -35,6 +35,13 @@ export function buildEvalsTemplate(skillName) {
             '                    Include negative assertions too, e.g. "does NOT claim every row reconciles".',
             '  category        — OPTIONAL label you choose to group evals (e.g. "recognition",',
             '                    "guidance", "recovery"). VAT ignores it; it is for your own organization.',
+            '  tier            — OPTIONAL non-negative integer cost/foundational tier, ascending',
+            '                    (0 = cheapest/first). Used for fail-fast gating: cheaper tiers run',
+            '                    first and can short-circuit later, more expensive tiers. Default: 0.',
+            '  toolExpectations — OPTIONAL { mustRun, mustNotRun, sequence }, each an array of',
+            '                    executable tool NAMES. The grader checks these against what the',
+            '                    executor actually ran in the transcript (mustRun: all present;',
+            '                    mustNotRun: none present; sequence: appear in this relative order).',
             'Aim for at least 3 evals covering real scenarios. Delete this _comment field once filled in, then re-run.',
         ],
         skill_name: skillName,

package/dist/skill-test/evals-template.js.map

@@ -1 +1 @@
-{"version":3,"file":"evals-template.js","sourceRoot":"","sources":["../../src/skill-test/evals-template.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEpD,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAEpE,2EAA2E;AAC3E,MAAM,UAAU,kBAAkB,CAAC,SAAiB;IAClD,MAAM,QAAQ,GAAG;QACf,QAAQ,EAAE;YACR,8DAA8D;YAC9D,oFAAoF;YACpF,kBAAkB;YAClB,8EAA8E;YAC9E,yFAAyF;YACzF,iFAAiF;YACjF,oFAAoF;YACpF,yFAAyF;YACzF,2FAA2F;YAC3F,6FAA6F;YAC7F,mFAAmF;YACnF,uFAAuF;YACvF,wFAAwF;YACxF,gGAAgG;YAChG,kGAAkG;YAClG,mFAAmF;YACnF,+FAA+F;YAC/F,2GAA2G;SAC5G;QACD,UAAU,EAAE,SAAS;QACrB,KAAK,EAAE;YACL;gBACE,EAAE,EAAE,CAAC;gBACL,QAAQ,EAAE,qDAAqD;gBAC/D,MAAM,EAAE,yEAAyE;gBACjF,eAAe,EAAE,kDAAkD;gBACnE,KAAK,EAAE,EAAE;gBACT,YAAY,EAAE;oBACZ,sGAAsG;oBACtG,2GAA2G;iBAC5G;aACF;SACF;KACF,CAAC;IACF,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;AAClD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAAiB,EAAE,SAAiB;IACrE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC9C,aAAa,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,gFAAgF;IAChF,+EAA+E;IAC/E,qGAAqG;IACrG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,qGAAqG;QACrG,aAAa,CAAC,SAAS,EAAE,kBAAkB,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
+{"version":3,"file":"evals-template.js","sourceRoot":"","sources":["../../src/skill-test/evals-template.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEpD,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAEpE,2EAA2E;AAC3E,MAAM,UAAU,kBAAkB,CAAC,SAAiB;IAClD,MAAM,QAAQ,GAAG;QACf,QAAQ,EAAE;YACR,8DAA8D;YAC9D,oFAAoF;YACpF,kBAAkB;YAClB,8EAA8E;YAC9E,yFAAyF;YACzF,iFAAiF;YACjF,oFAAoF;YACpF,yFAAyF;YACzF,2FAA2F;YAC3F,6FAA6F;YAC7F,mFAAmF;YACnF,uFAAuF;YACvF,wFAAwF;YACxF,gGAAgG;YAChG,kGAAkG;YAClG,mFAAmF;YACnF,+FAA+F;YAC/F,qFAAqF;YACrF,wFAAwF;YACxF,0FAA0F;YAC1F,mFAAmF;YACnF,qFAAqF;YACrF,oFAAoF;YACpF,yFAAyF;YACzF,2GAA2G;SAC5G;QACD,UAAU,EAAE,SAAS;QACrB,KAAK,EAAE;YACL;gBACE,EAAE,EAAE,CAAC;gBACL,QAAQ,EAAE,qDAAqD;gBAC/D,MAAM,EAAE,yEAAyE;gBACjF,eAAe,EAAE,kDAAkD;gBACnE,KAAK,EAAE,EAAE;gBACT,YAAY,EAAE;oBACZ,sGAAsG;oBACtG,2GAA2G;iBAC5G;aACF;SACF;KACF,CAAC;IACF,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;AAClD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAAiB,EAAE,SAAiB;IACrE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC9C,aAAa,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,gFAAgF;IAChF,+EAA+E;IAC/E,qGAAqG;IACrG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,qGAAqG;QACrG,aAAa,CAAC,SAAS,EAAE,kBAAkB,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/dist/skill-test/executor-prompt.d.ts

@@ -0,0 +1,41 @@
+export interface BuildExecutorPromptOptions {
+    /** The eval's `prompt` field, verbatim — the task the executor must perform. */
+    task: string;
+    /** Path to the staged subject (skill/plugin under test). */
+    subjectPath: string;
+    /** `<workspacesRoot>/<id>` when the eval declares input `files` — the project the executor should operate on. */
+    workspaceDir?: string;
+}
+/**
+ * Build the prompt handed to the blind executor subagent (issue #145 —
+ * transcript-grounded eval rewrite). The executor is a real worker, not a
+ * participant in an eval: it receives the task and its working context ONLY,
+ * phrased exactly as a genuine user request. It must NEVER be told (directly
+ * or by implication) that it is being tested, evaluated, or graded — that
+ * knowledge would change its behavior and invalidate the eval.
+ *
+ * `subjectPath` is where the skill/plugin under test is staged (context the
+ * executor's environment provides); `workspaceDir`, when the eval declares
+ * input `files`, is the project the executor is actually meant to operate on.
+ *
+ * Pure function: same inputs always produce the same prompt.
+ */
+export declare function buildExecutorPrompt(opts: BuildExecutorPromptOptions): string;
+/**
+ * Invariants for the executor prompt. Two guarantees:
+ *
+ * 1. **Task-present**: `task` must be non-empty and appear verbatim in
+ *    `prompt` — the executor is a real worker, so the adopter's task text
+ *    must actually reach it.
+ * 2. **No blinding-breaker in OUR scaffolding**: the denylist
+ *    ({@link BLINDING_BREAKERS}) is scanned ONLY against the prompt's
+ *    scaffolding — the prompt with the adopter's `task` text removed. The
+ *    adopter's task is arbitrary and may legitimately contain words like
+ *    "grading" or "you are being evaluated" (e.g. an essay-grading skill);
+ *    policing it would be a false positive. The invariant's real job is to
+ *    catch OUR builder accidentally telling the executor it is under test.
+ *
+ * Throws `PromptInvariantError` on violation.
+ */
+export declare function assertExecutorPromptInvariants(prompt: string, task: string): void;
+//# sourceMappingURL=executor-prompt.d.ts.map

package/dist/skill-test/executor-prompt.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"executor-prompt.d.ts","sourceRoot":"","sources":["../../src/skill-test/executor-prompt.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,0BAA0B;IACzC,gFAAgF;IAChF,IAAI,EAAE,MAAM,CAAC;IACb,4DAA4D;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB,iHAAiH;IACjH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,0BAA0B,GAAG,MAAM,CAM5E;AAWD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,8BAA8B,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAiBjF"}

package/dist/skill-test/executor-prompt.js

@@ -0,0 +1,63 @@
+import { PromptInvariantError } from './prompt-invariants.js';
+/**
+ * Build the prompt handed to the blind executor subagent (issue #145 —
+ * transcript-grounded eval rewrite). The executor is a real worker, not a
+ * participant in an eval: it receives the task and its working context ONLY,
+ * phrased exactly as a genuine user request. It must NEVER be told (directly
+ * or by implication) that it is being tested, evaluated, or graded — that
+ * knowledge would change its behavior and invalidate the eval.
+ *
+ * `subjectPath` is where the skill/plugin under test is staged (context the
+ * executor's environment provides); `workspaceDir`, when the eval declares
+ * input `files`, is the project the executor is actually meant to operate on.
+ *
+ * Pure function: same inputs always produce the same prompt.
+ */
+export function buildExecutorPrompt(opts) {
+    const lines = [opts.task, '', `The relevant files are located at ${opts.subjectPath}.`];
+    if (opts.workspaceDir !== undefined) {
+        lines.push(`Your working directory is ${opts.workspaceDir} — the project to operate on.`);
+    }
+    return lines.join('\n');
+}
+/**
+ * Phrases that, if present in an executor prompt, would break the blind
+ * setup by revealing to the executor that it is participating in an eval.
+ * Kept intentionally tight: this is a denylist of known blinding-breakers,
+ * not a general "sounds suspicious" heuristic, so it stays predictable for
+ * prompt authors overriding the default.
+ */
+const BLINDING_BREAKERS = ['being tested', 'this is an eval', 'you are being evaluated', 'grading'];
+/**
+ * Invariants for the executor prompt. Two guarantees:
+ *
+ * 1. **Task-present**: `task` must be non-empty and appear verbatim in
+ *    `prompt` — the executor is a real worker, so the adopter's task text
+ *    must actually reach it.
+ * 2. **No blinding-breaker in OUR scaffolding**: the denylist
+ *    ({@link BLINDING_BREAKERS}) is scanned ONLY against the prompt's
+ *    scaffolding — the prompt with the adopter's `task` text removed. The
+ *    adopter's task is arbitrary and may legitimately contain words like
+ *    "grading" or "you are being evaluated" (e.g. an essay-grading skill);
+ *    policing it would be a false positive. The invariant's real job is to
+ *    catch OUR builder accidentally telling the executor it is under test.
+ *
+ * Throws `PromptInvariantError` on violation.
+ */
+export function assertExecutorPromptInvariants(prompt, task) {
+    if (task.trim() === '') {
+        throw new PromptInvariantError('must include the task text (task is empty)');
+    }
+    if (!prompt.includes(task)) {
+        throw new PromptInvariantError('must include the task text (task not present in prompt)');
+    }
+    // Exclude the adopter task from the scan: only OUR scaffolding is policed.
+    // A space join avoids fusing the surrounding words across the excised task.
+    const scaffolding = prompt.split(task).join(' ').toLowerCase();
+    for (const breaker of BLINDING_BREAKERS) {
+        if (scaffolding.includes(breaker)) {
+            throw new PromptInvariantError(`must not reveal to the executor that it is being tested/evaluated/graded (found "${breaker}")`);
+        }
+    }
+}
+//# sourceMappingURL=executor-prompt.js.map

package/dist/skill-test/executor-prompt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"executor-prompt.js","sourceRoot":"","sources":["../../src/skill-test/executor-prompt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAW9D;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAgC;IAClE,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,qCAAqC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;IACxF,IAAI,IAAI,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,6BAA6B,IAAI,CAAC,YAAY,+BAA+B,CAAC,CAAC;IAC5F,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;;GAMG;AACH,MAAM,iBAAiB,GAAG,CAAC,cAAc,EAAE,iBAAiB,EAAE,yBAAyB,EAAE,SAAS,CAAC,CAAC;AAEpG;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,8BAA8B,CAAC,MAAc,EAAE,IAAY;IACzE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACvB,MAAM,IAAI,oBAAoB,CAAC,4CAA4C,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,oBAAoB,CAAC,yDAAyD,CAAC,CAAC;IAC5F,CAAC;IACD,2EAA2E;IAC3E,4EAA4E;IAC5E,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;IAC/D,KAAK,MAAM,OAAO,IAAI,iBAAiB,EAAE,CAAC;QACxC,IAAI,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,oBAAoB,CAC5B,oFAAoF,OAAO,IAAI,CAChG,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/skill-test/exit-codes.d.ts

@@ -52,7 +52,7 @@ export declare class SecurityAckError extends Error {
     readonly exitCode: 2;
     constructor();
 }
-/** Internal harness failure (incl. experimenter exiting without valid grading.json). Exit 1. */
+/** Internal harness failure (an executor/grader spawn error, watchdog timeout/stall, or a missing grader fragment). Exit 1. */
 export declare class InternalHarnessError extends Error {
     readonly exitCode: 1;
     constructor(message: string);
@@ -60,13 +60,16 @@ export declare class InternalHarnessError extends Error {
 /**
  * Map any thrown error to the process exit code. Errors that carry their own
  * `exitCode` (Bootstrap/Auth/HarnessLocation/Internal) are authoritative;
- * a PromptInvariantError is a user-correctable preflight problem (a supplied
- * prompt override is missing a required safety instruction) → 2; a BuildHookError
+ * a PromptInvariantError is a defense-in-depth failure over VAT's OWN generated
+ * executor/grader prompts (a required safety directive missing from a prompt VAT
+ * built) — surfaced as a user-visible preflight-class problem → 2; a BuildHookError
  * is a pre-stage build failure → 2; a SkillBuildError is a declared-skill build
  * failure → 2; a SecurityAckError is a missing security ack before a build → 2;
  * an UnknownEnvTokenError is a bad ${token} in a
- * declared env value → 2; GradingSkewError (parse failure) and GradingNonceError
- * (forged/mismatched grading nonce) are both → 1; everything unknown → 1.
+ * declared env value → 2; GradingSkewError (aggregate grading.json shape skew),
+ * EvalFragmentError (per-eval grader fragment parse failure), and
+ * GradingNonceError (forged/mismatched per-fragment grader nonce) are all → 1;
+ * everything unknown → 1.
  *
  * Note: SkillTestExitCode.EvalFailure (4) is NOT produced here — it is an
  * outcome of a completed run (an eval failed, the fail-closed default unless

package/dist/skill-test/exit-codes.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"exit-codes.d.ts","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AAQA,kDAAkD;AAClD,eAAO,MAAM,iBAAiB;;;;;IAK5B;;;;;;;;;;;;OAYG;;CAEK,CAAC;AAEX,MAAM,MAAM,sBAAsB,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,OAAO,iBAAiB,CAAC,CAAC;AAEhG;;;;;;;;;GASG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;aAEjB,YAAY,EAAE,MAAM;IADhD,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACH,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE;CAQ9E;AAED,iHAAiH;AACjH,qBAAa,eAAgB,SAAQ,KAAK;IACxC,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAED;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;;CAKhC;AAED,gGAAgG;AAChG,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAqBvD"}
+{"version":3,"file":"exit-codes.d.ts","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AASA,kDAAkD;AAClD,eAAO,MAAM,iBAAiB;;;;;IAK5B;;;;;;;;;;;;OAYG;;CAEK,CAAC;AAEX,MAAM,MAAM,sBAAsB,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,OAAO,iBAAiB,CAAC,CAAC;AAEhG;;;;;;;;;GASG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;aAEjB,YAAY,EAAE,MAAM;IADhD,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACH,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE;CAQ9E;AAED,iHAAiH;AACjH,qBAAa,eAAgB,SAAQ,KAAK;IACxC,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAED;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;;CAKhC;AAED,+HAA+H;AAC/H,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAsBvD"}

package/dist/skill-test/exit-codes.js

@@ -1,9 +1,10 @@
 import { AuthPreflightError } from '@vibe-agent-toolkit/utils';
 import { BuildHookError } from './build-hook.js';
 import { UnknownEnvTokenError } from './declared-env.js';
-import { PromptInvariantError } from './experimenter-prompt.js';
+import { EvalFragmentError } from './eval-fragment.js';
 import { GradingNonceError, GradingSkewError } from './grading-adapter.js';
 import { HarnessLocationError } from './harness-location.js';
+import { PromptInvariantError } from './prompt-invariants.js';
 /** Exit codes for `vat skill test` (spec §6d). */
 export const SkillTestExitCode = {
     Ok: 0,
@@ -67,7 +68,7 @@ export class SecurityAckError extends Error {
         this.name = 'SecurityAckError';
     }
 }
-/** Internal harness failure (incl. experimenter exiting without valid grading.json). Exit 1. */
+/** Internal harness failure (an executor/grader spawn error, watchdog timeout/stall, or a missing grader fragment). Exit 1. */
 export class InternalHarnessError extends Error {
     exitCode = 1;
     constructor(message) {
@@ -78,13 +79,16 @@ export class InternalHarnessError extends Error {
 /**
  * Map any thrown error to the process exit code. Errors that carry their own
  * `exitCode` (Bootstrap/Auth/HarnessLocation/Internal) are authoritative;
- * a PromptInvariantError is a user-correctable preflight problem (a supplied
- * prompt override is missing a required safety instruction) → 2; a BuildHookError
+ * a PromptInvariantError is a defense-in-depth failure over VAT's OWN generated
+ * executor/grader prompts (a required safety directive missing from a prompt VAT
+ * built) — surfaced as a user-visible preflight-class problem → 2; a BuildHookError
  * is a pre-stage build failure → 2; a SkillBuildError is a declared-skill build
  * failure → 2; a SecurityAckError is a missing security ack before a build → 2;
  * an UnknownEnvTokenError is a bad ${token} in a
- * declared env value → 2; GradingSkewError (parse failure) and GradingNonceError
- * (forged/mismatched grading nonce) are both → 1; everything unknown → 1.
+ * declared env value → 2; GradingSkewError (aggregate grading.json shape skew),
+ * EvalFragmentError (per-eval grader fragment parse failure), and
+ * GradingNonceError (forged/mismatched per-fragment grader nonce) are all → 1;
+ * everything unknown → 1.
  *
  * Note: SkillTestExitCode.EvalFailure (4) is NOT produced here — it is an
  * outcome of a completed run (an eval failed, the fail-closed default unless
@@ -103,7 +107,8 @@ export function mapErrorToExitCode(err) {
         err instanceof UnknownEnvTokenError) {
         return SkillTestExitCode.Preflight;
     }
-    if (err instanceof GradingNonceError ||
+    if (err instanceof EvalFragmentError ||
+        err instanceof GradingNonceError ||
         err instanceof GradingSkewError ||
         err instanceof InternalHarnessError) {
         return SkillTestExitCode.Internal;

package/dist/skill-test/exit-codes.js.map

@@ -1 +1 @@
-{"version":3,"file":"exit-codes.js","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAE7D,kDAAkD;AAClD,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,EAAE,EAAE,CAAC;IACL,QAAQ,EAAE,CAAC;IACX,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,CAAC;IACZ;;;;;;;;;;;;OAYG;IACH,WAAW,EAAE,CAAC;CACN,CAAC;AAIX;;;;;;;;;GASG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAEjB;IADnB,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAA4B,YAAoB,EAAE,IAA2B;QAC3E,KAAK,CACH,IAAI,EAAE,MAAM,KAAK,IAAI;YACnB,CAAC,CAAC,qFAAqF,YAAY,2DAA2D;YAC9J,CAAC,CAAC,mCAAmC,YAAY,2BAA2B,CAC/E,CAAC;QALwB,iBAAY,GAAZ,YAAY,CAAQ;QAM9C,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,iHAAiH;AACjH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAC/B,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IAChC,QAAQ,GAAG,CAAU,CAAC;IAC/B;QACE,KAAK,CAAC,wFAAwF,CAAC,CAAC;QAChG,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAED,gGAAgG;AAChG,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACpC,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAY;IAC7C,IAAI,GAAG,YAAY,oBAAoB;QAAE,OAAO,iBAAiB,CAAC,SAAS,CAAC;IAC5E,IACE,GAAG,YAAY,kBAAkB;QACjC,GAAG,YAAY,cAAc;QAC7B,GAAG,YAAY,oBAAoB;QACnC,GAAG,YAAY,oBAAoB;QACnC,GAAG,YAAY,gBAAgB;QAC/B,GAAG,YAAY,eAAe;QAC9B,GAAG,YAAY,oBAAoB,EACnC,CAAC;QACD,OAAO,iBAAiB,CAAC,SAAS,CAAC;IACrC,CAAC;IACD,IACE,GAAG,YAAY,iBAAiB;QAChC,GAAG,YAAY,gBAAgB;QAC/B,GAAG,YAAY,oBAAoB,EACnC,CAAC;QACD,OAAO,iBAAiB,CAAC,QAAQ,CAAC;IACpC,CAAC;IACD,OAAO,iBAAiB,CAAC,QAAQ,CAAC;AACpC,CAAC"}
+{"version":3,"file":"exit-codes.js","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAE9D,kDAAkD;AAClD,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,EAAE,EAAE,CAAC;IACL,QAAQ,EAAE,CAAC;IACX,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,CAAC;IACZ;;;;;;;;;;;;OAYG;IACH,WAAW,EAAE,CAAC;CACN,CAAC;AAIX;;;;;;;;;GASG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAEjB;IADnB,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAA4B,YAAoB,EAAE,IAA2B;QAC3E,KAAK,CACH,IAAI,EAAE,MAAM,KAAK,IAAI;YACnB,CAAC,CAAC,qFAAqF,YAAY,2DAA2D;YAC9J,CAAC,CAAC,mCAAmC,YAAY,2BAA2B,CAC/E,CAAC;QALwB,iBAAY,GAAZ,YAAY,CAAQ;QAM9C,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,iHAAiH;AACjH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAC/B,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IAChC,QAAQ,GAAG,CAAU,CAAC;IAC/B;QACE,KAAK,CAAC,wFAAwF,CAAC,CAAC;QAChG,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAED,+HAA+H;AAC/H,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACpC,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAY;IAC7C,IAAI,GAAG,YAAY,oBAAoB;QAAE,OAAO,iBAAiB,CAAC,SAAS,CAAC;IAC5E,IACE,GAAG,YAAY,kBAAkB;QACjC,GAAG,YAAY,cAAc;QAC7B,GAAG,YAAY,oBAAoB;QACnC,GAAG,YAAY,oBAAoB;QACnC,GAAG,YAAY,gBAAgB;QAC/B,GAAG,YAAY,eAAe;QAC9B,GAAG,YAAY,oBAAoB,EACnC,CAAC;QACD,OAAO,iBAAiB,CAAC,SAAS,CAAC;IACrC,CAAC;IACD,IACE,GAAG,YAAY,iBAAiB;QAChC,GAAG,YAAY,iBAAiB;QAChC,GAAG,YAAY,gBAAgB;QAC/B,GAAG,YAAY,oBAAoB,EACnC,CAAC;QACD,OAAO,iBAAiB,CAAC,QAAQ,CAAC;IACpC,CAAC;IACD,OAAO,iBAAiB,CAAC,QAAQ,CAAC;AACpC,CAAC"}

package/dist/skill-test/fragment-merge.d.ts

@@ -0,0 +1,46 @@
+import type { EvalFragment } from './eval-fragment.js';
+import { type FrictionReport } from './friction-schema.js';
+import { type NormalizedGrading } from './grading-adapter.js';
+import { type ToolEvalReport } from './tool-eval-schema.js';
+/**
+ * Assemble the run's aggregate grading result (skill-creator's flat
+ * `grading.json` shape) from every eval's per-eval grader fragment (issue #145
+ * Task 4/5). vat is the SOLE writer of the aggregate; this is that assembly
+ * step — the result feeds {@link import('./grading-adapter.js').reconcileGrading}
+ * unchanged.
+ *
+ * Every fragment must carry the SAME `runNonce` the harness stamped for this
+ * run — a mismatch means a fragment was not produced for this run (forged,
+ * stale, or left behind by untrusted skill code in the shared sandbox) and is
+ * rejected via {@link GradingNonceError}, never silently dropped.
+ *
+ * Zero fragments in → zero expectations out. This is deliberate: the existing
+ * "graded nothing" guard in `reconcileGrading` already throws
+ * `GradingSkewError` on an empty `expectations[]`, so that case is NOT
+ * special-cased here.
+ */
+export declare function mergeFragmentsToGrading(fragments: EvalFragment[], runNonce: string): NormalizedGrading;
+/**
+ * Assemble the run's aggregate friction report from every eval's fragment
+ * (vat-owned; see friction-schema.ts). Concatenates each fragment's
+ * `friction` items (fragments without any are treated as empty), de-dups
+ * byte-identical items, then validates the result against
+ * {@link FrictionReportSchema} — vat's own output contract, strict, no
+ * passthrough.
+ */
+export declare function mergeFragmentsToFriction(fragments: EvalFragment[]): FrictionReport;
+/**
+ * Assemble vat's `tool-eval.json` from every eval's fragment (issue #145
+ * Phase T; see tool-eval-schema.ts). A SEPARATE channel from `grading.json`
+ * and from friction (C2) — tool verdicts judge declared `toolExpectations`
+ * against the transcript, not prose expectations or packaging friction.
+ *
+ * Fragments carry only the `tool` BODY (no `evalId` — see
+ * {@link import('./tool-eval-schema.js').ToolVerdictBodySchema}); this merge
+ * re-attaches each fragment's own `evalId` to produce a full
+ * {@link import('./tool-eval-schema.js').ToolVerdict}. Fragments with no
+ * `tool` body are OMITTED — not every eval declares `toolExpectations`, so
+ * "no tool block" is a legitimate, non-error case, not a zero-filled verdict.
+ */
+export declare function mergeFragmentsToToolEval(fragments: EvalFragment[]): ToolEvalReport;
+//# sourceMappingURL=fragment-merge.d.ts.map

package/dist/skill-test/fragment-merge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fragment-merge.d.ts","sourceRoot":"","sources":["../../src/skill-test/fragment-merge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAqB,KAAK,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAElF;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,YAAY,EAAE,EAAE,QAAQ,EAAE,MAAM,GAAG,iBAAiB,CAoBtG;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,YAAY,EAAE,GAAG,cAAc,CAYlF;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,YAAY,EAAE,GAAG,cAAc,CAMlF"}

package/dist/skill-test/fragment-merge.js

@@ -0,0 +1,77 @@
+import { FrictionReportSchema } from './friction-schema.js';
+import { GradingNonceError } from './grading-adapter.js';
+import { ToolEvalReportSchema } from './tool-eval-schema.js';
+/**
+ * Assemble the run's aggregate grading result (skill-creator's flat
+ * `grading.json` shape) from every eval's per-eval grader fragment (issue #145
+ * Task 4/5). vat is the SOLE writer of the aggregate; this is that assembly
+ * step — the result feeds {@link import('./grading-adapter.js').reconcileGrading}
+ * unchanged.
+ *
+ * Every fragment must carry the SAME `runNonce` the harness stamped for this
+ * run — a mismatch means a fragment was not produced for this run (forged,
+ * stale, or left behind by untrusted skill code in the shared sandbox) and is
+ * rejected via {@link GradingNonceError}, never silently dropped.
+ *
+ * Zero fragments in → zero expectations out. This is deliberate: the existing
+ * "graded nothing" guard in `reconcileGrading` already throws
+ * `GradingSkewError` on an empty `expectations[]`, so that case is NOT
+ * special-cased here.
+ */
+export function mergeFragmentsToGrading(fragments, runNonce) {
+    const expectations = [];
+    for (const fragment of fragments) {
+        if (fragment.runNonce !== runNonce) {
+            throw new GradingNonceError(`fragment for eval "${fragment.evalId}" carries runNonce "${fragment.runNonce}", expected "${runNonce}"`);
+        }
+        for (const expectation of fragment.expectations) {
+            expectations.push({
+                text: expectation.text,
+                passed: expectation.passed,
+                ...(expectation.evidence === undefined ? {} : { evidence: expectation.evidence }),
+            });
+        }
+    }
+    const passed = expectations.filter(e => e.passed).length;
+    return { summary: { passed, total: expectations.length }, expectations, runNonce };
+}
+/**
+ * Assemble the run's aggregate friction report from every eval's fragment
+ * (vat-owned; see friction-schema.ts). Concatenates each fragment's
+ * `friction` items (fragments without any are treated as empty), de-dups
+ * byte-identical items, then validates the result against
+ * {@link FrictionReportSchema} — vat's own output contract, strict, no
+ * passthrough.
+ */
+export function mergeFragmentsToFriction(fragments) {
+    const allItems = fragments.flatMap(fragment => fragment.friction ?? []);
+    const seen = new Set();
+    const deduped = allItems.filter(item => {
+        const key = JSON.stringify(item);
+        if (seen.has(key))
+            return false;
+        seen.add(key);
+        return true;
+    });
+    return FrictionReportSchema.parse({ items: deduped });
+}
+/**
+ * Assemble vat's `tool-eval.json` from every eval's fragment (issue #145
+ * Phase T; see tool-eval-schema.ts). A SEPARATE channel from `grading.json`
+ * and from friction (C2) — tool verdicts judge declared `toolExpectations`
+ * against the transcript, not prose expectations or packaging friction.
+ *
+ * Fragments carry only the `tool` BODY (no `evalId` — see
+ * {@link import('./tool-eval-schema.js').ToolVerdictBodySchema}); this merge
+ * re-attaches each fragment's own `evalId` to produce a full
+ * {@link import('./tool-eval-schema.js').ToolVerdict}. Fragments with no
+ * `tool` body are OMITTED — not every eval declares `toolExpectations`, so
+ * "no tool block" is a legitimate, non-error case, not a zero-filled verdict.
+ */
+export function mergeFragmentsToToolEval(fragments) {
+    const evals = fragments
+        .filter((fragment) => fragment.tool !== undefined)
+        .map(fragment => ({ evalId: fragment.evalId, ...fragment.tool }));
+    return ToolEvalReportSchema.parse({ evals });
+}
+//# sourceMappingURL=fragment-merge.js.map

package/dist/skill-test/fragment-merge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fragment-merge.js","sourceRoot":"","sources":["../../src/skill-test/fragment-merge.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAuB,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAE,iBAAiB,EAA0B,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAE,oBAAoB,EAAuB,MAAM,uBAAuB,CAAC;AAElF;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,uBAAuB,CAAC,SAAyB,EAAE,QAAgB;IACjF,MAAM,YAAY,GAAsC,EAAE,CAAC;IAE3D,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACnC,MAAM,IAAI,iBAAiB,CACzB,sBAAsB,QAAQ,CAAC,MAAM,uBAAuB,QAAQ,CAAC,QAAQ,gBAAgB,QAAQ,GAAG,CACzG,CAAC;QACJ,CAAC;QACD,KAAK,MAAM,WAAW,IAAI,QAAQ,CAAC,YAAY,EAAE,CAAC;YAChD,YAAY,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,MAAM,EAAE,WAAW,CAAC,MAAM;gBAC1B,GAAG,CAAC,WAAW,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,WAAW,CAAC,QAAQ,EAAE,CAAC;aAClF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACzD,OAAO,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC;AACrF,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,wBAAwB,CAAC,SAAyB;IAChE,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IAExE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAChC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,OAAO,oBAAoB,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;AACxD,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,wBAAwB,CAAC,SAAyB;IAChE,MAAM,KAAK,GAAG,SAAS;SACpB,MAAM,CAAC,CAAC,QAAQ,EAA0E,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC;SACzH,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAEpE,OAAO,oBAAoB,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AAC/C,CAAC"}

package/dist/skill-test/friction-schema.d.ts

@@ -3,25 +3,25 @@ import { z } from 'zod';
 export declare const FrictionSeveritySchema: z.ZodEnum<["high", "medium", "low"]>;
 /**
  * Closed set of packaging-fidelity friction categories (spec §18). vat owns
- * this enum; the experimenter must emit one of these.
+ * this enum; the grader must emit one of these.
  */
-export declare const FrictionCategorySchema: z.ZodEnum<["path-assumption", "undeclared-dependency", "ambient-propping", "doc-engine-drift", "missing-bundled-file"]>;
+export declare const FrictionCategorySchema: z.ZodEnum<["path-assumption", "undeclared-dependency", "ambient-propping", "doc-engine-drift", "missing-bundled-file", "tool-expectation"]>;
 export declare const FrictionItemSchema: z.ZodObject<{
     severity: z.ZodEnum<["high", "medium", "low"]>;
-    category: z.ZodEnum<["path-assumption", "undeclared-dependency", "ambient-propping", "doc-engine-drift", "missing-bundled-file"]>;
+    category: z.ZodEnum<["path-assumption", "undeclared-dependency", "ambient-propping", "doc-engine-drift", "missing-bundled-file", "tool-expectation"]>;
     message: z.ZodString;
     subjectFile: z.ZodOptional<z.ZodString>;
     evidence: z.ZodOptional<z.ZodString>;
 }, "strict", z.ZodTypeAny, {
     message: string;
     severity: "high" | "medium" | "low";
-    category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file";
+    category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file" | "tool-expectation";
     evidence?: string | undefined;
     subjectFile?: string | undefined;
 }, {
     message: string;
     severity: "high" | "medium" | "low";
-    category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file";
+    category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file" | "tool-expectation";
     evidence?: string | undefined;
     subjectFile?: string | undefined;
 }>;
@@ -30,20 +30,20 @@ export type FrictionItem = z.infer<typeof FrictionItemSchema>;
 export declare const FrictionReportSchema: z.ZodObject<{
     items: z.ZodArray<z.ZodObject<{
         severity: z.ZodEnum<["high", "medium", "low"]>;
-        category: z.ZodEnum<["path-assumption", "undeclared-dependency", "ambient-propping", "doc-engine-drift", "missing-bundled-file"]>;
+        category: z.ZodEnum<["path-assumption", "undeclared-dependency", "ambient-propping", "doc-engine-drift", "missing-bundled-file", "tool-expectation"]>;
         message: z.ZodString;
         subjectFile: z.ZodOptional<z.ZodString>;
         evidence: z.ZodOptional<z.ZodString>;
     }, "strict", z.ZodTypeAny, {
         message: string;
         severity: "high" | "medium" | "low";
-        category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file";
+        category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file" | "tool-expectation";
         evidence?: string | undefined;
         subjectFile?: string | undefined;
     }, {
         message: string;
         severity: "high" | "medium" | "low";
-        category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file";
+        category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file" | "tool-expectation";
         evidence?: string | undefined;
         subjectFile?: string | undefined;
     }>, "many">;
@@ -51,7 +51,7 @@ export declare const FrictionReportSchema: z.ZodObject<{
     items: {
         message: string;
         severity: "high" | "medium" | "low";
-        category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file";
+        category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file" | "tool-expectation";
         evidence?: string | undefined;
         subjectFile?: string | undefined;
     }[];
@@ -59,7 +59,7 @@ export declare const FrictionReportSchema: z.ZodObject<{
     items: {
         message: string;
         severity: "high" | "medium" | "low";
-        category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file";
+        category: "path-assumption" | "undeclared-dependency" | "ambient-propping" | "doc-engine-drift" | "missing-bundled-file" | "tool-expectation";
         evidence?: string | undefined;
         subjectFile?: string | undefined;
     }[];