@vibe-agent-toolkit/agent-skills 0.1.39-rc.8 → 0.1.39

package/dist/skill-test/run-harness.js

@@ -8,17 +8,21 @@
  *   → build effective prompt + assertPromptInvariants → dry-run short-circuit
  *   → spawnHeadlessClaude → parseGradingJson → release lock → return result
  */
-import { existsSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { randomBytes } from 'node:crypto';
+import { cpSync, existsSync, lstatSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
 import { basename, dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { getToolVersion, mkdirSyncReal, normalizedTmpdir, probeAuthStatus, safeExecResult, safePath, spawnHeadlessClaude, toForwardSlash, } from '@vibe-agent-toolkit/utils';
+import { getToolVersion, mkdirSyncReal, normalizedTmpdir, probeAuthStatus, resolveAssetReference, safeExecResult, safePath, spawnHeadlessClaude, toForwardSlash, } from '@vibe-agent-toolkit/utils';
 import { resolveSkillSource } from '../skill-source/resolve-skill-source.js';
+import { assembleChildEnv, computeEnvTokens, resolveInjectEnv } from './declared-env.js';
+import { EvalInputError, parseEvalSuite, stageEvalWorkspaces } from './eval-inputs.js';
 import { writeEvalsTemplate } from './evals-template.js';
-import { BootstrapNeededError, InternalHarnessError, SkillTestExitCode } from './exit-codes.js';
-import { assertPromptInvariants, buildExperimenterPrompt, } from './experimenter-prompt.js';
-import { parseGradingJson } from './grading-adapter.js';
-import { assertSafeHarnessRoot, assertSafeWorkdir, resolveHarnessRoot } from './harness-location.js';
-import { acquireHarnessLock } from './lock.js';
+import { BootstrapNeededError, InternalHarnessError, SkillTestExitCode, } from './exit-codes.js';
+import { appendIntegrityNonceDirective, assertPromptInvariants, buildExperimenterPrompt, redactNonce, } from './experimenter-prompt.js';
+import { assertGradingNonce, parseGradingJson, reconcileGrading } from './grading-adapter.js';
+import { assertSafeHarnessRoot, assertSafeWorkdir, prepareHarnessRoot, resolveHarnessRoot } from './harness-location.js';
+import { acquireHarnessLock, installSignalCleanup } from './lock.js';
+import { detectPluginLayout } from './plugin-layout.js';
 import { runPreflight } from './preflight.js';
 import { descriptorToSource, stageHarness } from './staging.js';
 import { verifyVendoredManifest } from './vendor-manifest.js';
@@ -38,13 +42,42 @@ const VENDORED_SKILL_CREATOR_DIR = safePath.resolve(dirname(fileURLToPath(import
 const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes
 const DEFAULT_MAX_TURNS = 50;
 const DEFAULT_MAX_BUDGET_USD = 5;
-function resolveTimeoutMs(opts) {
+/**
+ * Built-in cost/runtime safety ceilings for the experimenter spawn. These are the
+ * SAME values the harness applies as defaults, but exported as an explicit cap so
+ * the CLI precedence layer (run.ts) can enforce a critical asymmetry:
+ *
+ *   - a CLI flag (explicit operator intent, typed at the terminal for THIS run)
+ *     may RAISE a knob above the built-in ceiling;
+ *   - a value sourced from a committed `test.*` config (which rides along in an
+ *     untrusted subject repo you may only be testing) may only LOWER a ceiling,
+ *     never raise it — so cloning + testing a hostile skill can't silently
+ *     escalate the $5 / 50-turn / 5-minute budget the run bills against.
+ *
+ * `timeoutSeconds` is expressed in seconds to match the `--timeout`/`test.timeout`
+ * unit (the harness multiplies by 1000 internally; see resolveTimeoutMs).
+ */
+export const SKILL_TEST_BUILTIN_CAPS = {
+    maxTurns: DEFAULT_MAX_TURNS,
+    maxBudgetUsd: DEFAULT_MAX_BUDGET_USD,
+    timeoutSeconds: DEFAULT_TIMEOUT_MS / 1000,
+};
+export function resolveTimeoutMs(opts) {
     return opts.timeout === undefined ? DEFAULT_TIMEOUT_MS : opts.timeout * 1000;
 }
-function resolveStallMs(opts) {
+/**
+ * Map an eval verdict to a process exit code. Default behavior: a completed run
+ * exits Ok regardless of pass/fail (the count lives in the summary). When
+ * `failOnEvalFailure` is set, a failing verdict escalates to EvalFailure (4) so
+ * CI can gate on eval outcomes without conflating them with harness breakage.
+ */
+export function verdictExitCode(allPassed, failOnEvalFailure) {
+    return failOnEvalFailure && !allPassed ? SkillTestExitCode.EvalFailure : SkillTestExitCode.Ok;
+}
+export function resolveStallMs(opts) {
     return opts.stall === undefined ? undefined : opts.stall * 1000;
 }
-function resolveKnobs(opts) {
+export function resolveKnobs(opts) {
     const result = {
         maxTurns: opts.maxTurns ?? DEFAULT_MAX_TURNS,
         maxBudgetUsd: opts.maxBudgetUsd ?? DEFAULT_MAX_BUDGET_USD,
@@ -56,32 +89,60 @@ function resolveKnobs(opts) {
         result.stallMs = stallMs;
     return result;
 }
-function buildStageItems(opts) {
+/**
+ * Detect the plugin-root layout for a `{ path }` source. The resolver later COPIES
+ * the source into a temp dir (losing its plugin ancestry), so we must detect here
+ * — while the true on-disk source dir is still known — by resolving the path spec
+ * against repoRoot and walking up for `.claude-plugin/plugin.json`. Non-`{path}`
+ * sources (npm/url/vendored/workspace) have no local source tree to walk, so they
+ * are always staged flat (returns undefined). undefined → flat staging.
+ */
+export function detectItemPluginLayout(source, repoRoot) {
+    if (!('path' in source))
+        return undefined;
+    const sourceDir = resolveAssetReference(source.path, repoRoot);
+    return detectPluginLayout(sourceDir, existsSync) ?? undefined;
+}
+export function makeStageItem(name, source, repoRoot, role) {
+    const pluginLayout = detectItemPluginLayout(source, repoRoot);
+    return {
+        name,
+        source,
+        ...(role === undefined ? {} : { role }),
+        ...(pluginLayout === undefined ? {} : { pluginLayout }),
+    };
+}
+export function buildStageItems(opts, repoRoot) {
     const items = [];
     // The FIRST positional skill is the subject under test; the rest of the
     // primary set and any `--with`/`--with-optional` deps are supporting context.
     const subjectName = opts.skills[0];
     for (const name of opts.skills) {
         const override = opts.withSources?.[name];
-        const source = override
-            ? descriptorToSource(override)
-            : { path: name };
-        items.push(name === subjectName ? { name, source, role: 'subject' } : { name, source });
+        let source;
+        if (name === subjectName && opts.subjectSource !== undefined) {
+            source = opts.subjectSource;
+        }
+        else if (override) {
+            source = descriptorToSource(override);
+        }
+        else {
+            source = { path: name };
+        }
+        items.push(makeStageItem(name, source, repoRoot, name === subjectName ? 'subject' : undefined));
     }
     if (opts.withOptional !== undefined) {
         for (const [name, spec] of Object.entries(opts.withOptional)) {
-            items.push({
-                name,
-                source: descriptorToSource(spec),
-            });
+            const source = descriptorToSource(spec);
+            items.push(makeStageItem(name, source, repoRoot, undefined));
         }
     }
     return items;
 }
-function buildResolveCtx(harnessRoot, repoRoot) {
+export function buildResolveCtx(harnessRoot, repoRoot) {
     return {
         repoRoot,
-        stagingRoot: safePath.join(harnessRoot, 'staged'),
+        stagingRoot: safePath.joinUnderRoot(harnessRoot, 'staged'),
         fetchCacheDir: safePath.join(normalizedTmpdir(), 'vat-fetch-cache'),
     };
 }
@@ -104,7 +165,7 @@ const FLAG_DUMMY_VALUES = {
  * Return a sensible dummy value for a given CLI flag so that value-validation
  * doesn't reject the argument before `--help` can short-circuit the session.
  */
-function flagDummyValueFor(flag) {
+export function flagDummyValueFor(flag) {
     return FLAG_DUMMY_VALUES[flag] ?? '1';
 }
 /**
@@ -132,7 +193,7 @@ function buildFlagParseProbe() {
         return result.success;
     };
 }
-function buildPreflightInput(evalsPath, pluginDirs, opts, knobs) {
+export function buildPreflightInput(evalsPath, pluginDirs, opts, knobs) {
     const costEstimate = {
         evalCount: 1,
         configurations: 1,
@@ -161,13 +222,20 @@ function buildPreflightInput(evalsPath, pluginDirs, opts, knobs) {
         preflightOpts.requireAuth = opts.requireAuth;
     return preflightOpts;
 }
-function renderPreflightSummary(checks) {
+export function renderPreflightSummary(checks) {
     const failed = checks.filter(c => !c.passed);
     return failed.length > 0
         ? failed.map(c => `  [FAIL] ${c.name}: ${c.message}`).join('\n')
         : '  All preflight checks passed.';
 }
-function isAcknowledged(opts) {
+/**
+ * The single source of truth for "is this run acknowledged?": a dry-run never
+ * executes skill code (so it is implicitly acknowledged), otherwise the caller
+ * must pass --i-understand-this-runs-skill-code. Narrowed to the two fields it
+ * reads so run.ts can reuse the SAME predicate to gate the pre-build ack check
+ * (the harness Step-6 check and the run.ts pre-build check cannot diverge).
+ */
+export function isAcknowledged(opts) {
     return opts.dryRun === true || opts.acknowledgedRunsSkillCode === true;
 }
 /**
@@ -175,7 +243,7 @@ function isAcknowledged(opts) {
  * A stall, a timeout, OR a non-zero exit are each authoritative — a non-zero exit
  * is never laundered into a PASS even if a grading.json happens to be on disk.
  */
-function assertExperimenterSucceeded(spawnResult, stallMs, timeoutMs) {
+export function assertExperimenterSucceeded(spawnResult, stallMs, timeoutMs) {
     if (spawnResult.stalled) {
         throw new InternalHarnessError(`Experimenter stalled (no output for ${stallMs ?? 0}ms).`);
     }
@@ -193,19 +261,197 @@ function assertExperimenterSucceeded(spawnResult, stallMs, timeoutMs) {
  * that real source dir; otherwise we anchor under the repo root by skill name.
  */
 /** The subject skill's display name (trailing segment of the positional skill arg). */
-function subjectSkillName(opts) {
+export function subjectSkillName(opts) {
     const subject = opts.skills[0] ?? 'skill';
     return basename(toForwardSlash(subject)) || subject;
 }
-function resolveScaffoldEvalsPath(opts, repoRoot, evalsSubpath) {
+export function resolveScaffoldEvalsPath(opts, repoRoot, evalsSubpath) {
+    // Prefer the explicit authored source dir resolved by run.ts (the staged/built
+    // tree is ephemeral; this is where the user can edit the scaffolded template).
+    if (opts.subjectScaffoldDir !== undefined) {
+        return safePath.join(opts.subjectScaffoldDir, evalsSubpath);
+    }
     const subjectName = opts.skills[0] ?? '';
     const override = opts.withSources?.[subjectName];
-    const overridePath = override && typeof override.path === 'string' ? override.path : undefined;
+    const overridePath = override && 'path' in override ? override.path : undefined;
     // Default (no override) resolution treats the positional name as a path.
     const sourcePath = overridePath ?? subjectName;
+    // eslint-disable-next-line local/no-unsafe-root-join -- the positional skill source may be an absolute path; resolving it against repoRoot (which returns an absolute sourcePath unchanged) is intentional, documented behavior, not a containment bug.
     const sourceDir = safePath.resolve(repoRoot, sourcePath);
     return safePath.join(sourceDir, evalsSubpath);
 }
+/**
+ * Bootstrap (exit 3) when the eval suite is absent everywhere. A real run writes
+ * a persistent template at the source scaffold location and reports it; a dry run
+ * must never touch the filesystem, so it reports where a real run *would* scaffold
+ * and writes nothing. Both surface the same exit-3 BootstrapNeededError. No-op when
+ * `evalsPath` already exists.
+ */
+function bootstrapEvalSuiteIfMissing(opts, repoRoot, evalsSubpath, evalsPath) {
+    // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own staged-subject path
+    if (existsSync(evalsPath)) {
+        return;
+    }
+    const scaffoldPath = resolveScaffoldEvalsPath(opts, repoRoot, evalsSubpath);
+    if (opts.dryRun === true) {
+        throw new BootstrapNeededError(scaffoldPath, { dryRun: true });
+    }
+    throw new BootstrapNeededError(writeEvalsTemplate(scaffoldPath, subjectSkillName(opts)));
+}
+/**
+ * The eval suite (`evals/`, incl. `fixtures/`) is authored TEST INPUT, not a
+ * shipped artifact — a built/dist subject won't carry it (packageSkill bundles
+ * only link-reachable resources + `files:`). Since the harness reads evals and
+ * fixtures relative to the staged subject, overlay the authored suite from the
+ * scaffold (source) dir onto the staged subject when it lacks it. No-op when there
+ * is no scaffold dir, the suite subpath has no directory, or the staged subject
+ * already carries the suite — so authored evals are never overwritten.
+ */
+function overlayAuthoredEvalSuite(opts, subjectStagedDir, evalsSubpath) {
+    const evalsDir = dirname(evalsSubpath);
+    if (evalsDir === '.' || opts.subjectScaffoldDir === undefined)
+        return;
+    const scaffoldEvalsDir = safePath.join(opts.subjectScaffoldDir, evalsDir);
+    const stagedEvalsDir = safePath.join(subjectStagedDir, evalsDir);
+    // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own staged/scaffold paths
+    if (!existsSync(stagedEvalsDir) && existsSync(scaffoldEvalsDir)) {
+        cpSync(scaffoldEvalsDir, stagedEvalsDir, { recursive: true });
+    }
+}
+/** Parse the staged eval suite and materialize each eval's input `files` into
+ * `<harnessRoot>/workspaces/<id>/`. Returns the workspaces root. The dir is wiped
+ * first so a reused harness root cannot leak a prior run's inputs. Throws
+ * {@link EvalInputError} (mapped by the caller to exit 2) on a bad suite or a
+ * missing input file. */
+export function stageWorkspacesForRun(evalsPath, harnessRoot) {
+    const workspacesRoot = safePath.joinUnderRoot(harnessRoot, 'workspaces');
+    rmSync(workspacesRoot, { recursive: true, force: true });
+    mkdirSyncReal(workspacesRoot, { recursive: true });
+    // eslint-disable-next-line security/detect-non-literal-fs-filename -- evalsPath is our staged-subject path
+    const suite = parseEvalSuite(readFileSync(evalsPath, 'utf-8'));
+    return stageEvalWorkspaces({ suite, evalsDir: dirname(evalsPath), workspacesRoot });
+}
+/** Format the model flag string for logging and dry-run summary output. */
+function buildModelFlag(model) {
+    return model === undefined ? '(no --model; claude default)' : `--model ${model}`;
+}
+/**
+ * Wrap {@link stageWorkspacesForRun} for the orchestrator: returns the
+ * `workspacesRoot` string on success, or a {@link RunHarnessResult} early-return
+ * value when the suite is invalid or a declared input file is missing. Any other
+ * error is re-thrown so it propagates as an InternalHarnessError upstream.
+ * Keeping the try/catch in a private helper avoids inflating the orchestrator's
+ * cognitive complexity.
+ */
+function attemptStageWorkspaces(evalsPath, harnessRoot) {
+    try {
+        return { workspacesRoot: stageWorkspacesForRun(evalsPath, harnessRoot) };
+    }
+    catch (e) {
+        if (e instanceof EvalInputError) {
+            return {
+                harnessPath: harnessRoot,
+                exitCode: SkillTestExitCode.Preflight,
+                summary: `Eval input error:\n  ${e.message}`,
+            };
+        }
+        throw e;
+    }
+}
+/**
+ * Assemble the experimenter's child env: the scrubbed forwarded env unioned with
+ * the declared test env (Features A + B) and CLAUDE_PLUGIN_ROOT, then emit the
+ * transparency line and any protected-key collision warnings to stderr.
+ *
+ * Refuses to assemble without a resolved auth — a null here is an internal
+ * invariant violation (preflight passed), never a reason to fall back to
+ * process.env, which would hand untrusted skill code every secret it contains.
+ */
+function resolveDeclaredChildEnv(input) {
+    if (input.resolvedAuth === null) {
+        throw new InternalHarnessError('Internal: preflight passed but resolvedAuth is null — refusing to spawn with an unscrubbed environment.');
+    }
+    const envTokens = computeEnvTokens({
+        subjectStagedDir: input.subjectStagedDir,
+        harnessRoot: input.harnessRoot,
+        resultsDir: input.resultsDir,
+        evalsSubpath: input.evalsSubpath,
+    });
+    const injectEnv = resolveInjectEnv(input.opts.env, envTokens);
+    const assembled = assembleChildEnv({
+        base: input.resolvedAuth.forwardedEnv,
+        source: process.env,
+        ...(input.opts.passEnv ? { passEnv: input.opts.passEnv } : {}),
+        ...(injectEnv ? { injectEnv } : {}),
+        subjectPluginRoot: input.subjectPluginRoot,
+    });
+    for (const warning of assembled.warnings)
+        process.stderr.write(`warning: ${warning}\n`);
+    process.stderr.write(assembled.line + '\n');
+    return assembled;
+}
+/**
+ * Build the dry-run summary string. Pure function so it can be unit-tested
+ * without running the full harness.
+ *
+ * The summary covers three scenarios:
+ *   1. Declared (buildable) subject — no dist existed, fell back to source dir.
+ *   2. Declared (buildable) subject — existing dist was staged WITHOUT rebuilding
+ *      (potentially stale).
+ *   3. Plain source subject — staged as-is, no build step.
+ *
+ * Always includes the assembled spawn command, the staged-manifest entry count +
+ * fingerprint, and the provenance.json path so a stale tree is visible at a glance.
+ */
+export function buildDryRunSummary(input) {
+    const lines = [];
+    if (input.wouldBuild) {
+        lines.push('[dry-run] A real run would: build + stage the declared skill, then spawn claude.');
+        if (input.dryRunStagedExistingDist === true) {
+            lines.push('[dry-run] WARNING: This preview used the EXISTING built dist WITHOUT rebuilding — it may be STALE.', '[dry-run] Run `vat build` before testing to ensure the preview reflects current source.');
+        }
+        else if (input.dryRunStagedExistingDist === false) {
+            lines.push('[dry-run] No built dist exists yet; this preview fell back to the source dir.');
+        }
+    }
+    else {
+        lines.push('[dry-run] A real run would: stage the source dir as-is, then spawn claude.');
+    }
+    const count = input.provenanceEntryCount;
+    lines.push(`[dry-run] Would spawn: claude -p ${input.modelFlag} (prompt via stdin)`, `[dry-run] Staged manifest: ${count} entr${count === 1 ? 'y' : 'ies'} | fingerprint: ${input.provenanceFingerprint}`, `[dry-run] Provenance: ${input.provenancePath}`);
+    return lines.join('\n');
+}
+/**
+ * Remove the harness directory after a run so staged untrusted skill bytes and
+ * prompts do not accumulate in OS tmp. No-op when the user asked to keep it, when
+ * the dir is a user-supplied location (`--out`/`--workdir`), or when it is already
+ * gone. Idempotent and never throws — it runs from a `finally`, so it must not
+ * mask the run's real outcome.
+ *
+ * SAFETY: re-asserts the root is not a symlink immediately before removal (via
+ * `lstat`, which does NOT follow the link). A root swapped to a symlink between
+ * the run and cleanup is left in place rather than followed — `rmSync(recursive)`
+ * could otherwise delete the symlink's target outside tmp.
+ */
+export function cleanupHarness(harnessRoot, opts) {
+    if (opts.keep || !opts.created)
+        return;
+    // Best-effort: cleanup runs from a `finally`, so a TOCTOU race (the dir is
+    // reaped between checks) or a permission error must never throw out and mask
+    // the run's real outcome. Worst case is a leftover 0700 tmp dir, not a failure.
+    try {
+        // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived harness root
+        if (!existsSync(harnessRoot))
+            return;
+        // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived harness root
+        if (lstatSync(harnessRoot).isSymbolicLink())
+            return;
+        rmSync(harnessRoot, { recursive: true, force: true });
+    }
+    catch {
+        // Swallow: a failed cleanup is not a run failure.
+    }
+}
 // ---------------------------------------------------------------------------
 // Main orchestrator
 // ---------------------------------------------------------------------------
@@ -220,42 +466,65 @@ export async function runSkillTestHarness(opts) {
         assertSafeWorkdir(opts.workdir);
     }
     const harnessRoot = opts.out ?? resolveHarnessRoot(opts.skills, opts.workdir);
+    // Only auto-remove the dir the harness itself created under OS tmp. An explicit
+    // --out (exact dir) or --workdir (user-chosen base) is a location the user owns;
+    // treat it like --keep and never delete it.
+    const harnessCreated = opts.out === undefined && opts.workdir === undefined;
     const repoRoot = opts.repoRoot ?? harnessRoot;
     const evalsSubpath = opts.evalsSubpath ?? DEFAULT_EVALS_SUBPATH;
     const currentUid = typeof process.getuid === 'function' ? process.getuid() : 0;
+    // Tighten an existing directory to 0700 (if present) BEFORE mkdir — adopter
+    // may have created the --out dir with default umask (0755). This is strictly
+    // safer: we only remove access, never grant it. Symlink still throws.
+    prepareHarnessRoot(harnessRoot);
     // Ensure the harness root directory exists before validating it.
     mkdirSyncReal(harnessRoot, { recursive: true, mode: 0o700 });
     // Step 1: Assert safe harness root (symlink/ownership/mode checks) BEFORE
     // acquiring the lock — never write a lockfile into a directory we have not yet
-    // confirmed is non-symlinked, owned by the current uid, and 0700.
-    assertSafeHarnessRoot(harnessRoot, currentUid);
+    // confirmed is non-symlinked, owned by the current uid, and 0700. The trusted
+    // boundary is the tmp/workdir base the harness root was derived from; every
+    // component beneath it (incl. the recursively-created `vat-skill-test` parent)
+    // is validated, closing the shared-/tmp TOCTOU on intermediate components.
+    const trustedTmpRoot = opts.workdir ?? normalizedTmpdir();
+    assertSafeHarnessRoot(harnessRoot, currentUid, trustedTmpRoot);
     // Step 2: Acquire exclusive harness lock.
     const lock = acquireHarnessLock(harnessRoot);
+    // Release the lock and remove the harness dir if the run is interrupted by
+    // SIGINT/SIGTERM — a try/finally alone does not run on a signal, so without
+    // this the lockfile (and staged bytes) would leak and break the next run.
+    const cleanup = () => {
+        lock.release();
+        cleanupHarness(harnessRoot, { keep: opts.keep === true, created: harnessCreated });
+    };
+    const removeSignalCleanup = installSignalCleanup({ onSignal: cleanup });
     try {
         // Step 3: Stage the harness FIRST — the subject's own evals/evals.json lands
-        // inside its staged dir, so we must stage before we can locate it.
+        // inside its staged dir, so we must stage before we can locate it. The subject's
+        // source is pre-resolved (and, for declared skills, pre-built) by run.ts and
+        // arrives via opts.subjectSource; build no longer happens in-domain here.
         const resolveCtx = buildResolveCtx(harnessRoot, repoRoot);
-        const items = buildStageItems(opts);
-        const { pluginDirs, subjectStagedDir } = await stageHarness({
+        const items = buildStageItems(opts, repoRoot);
+        const stageResult = await stageHarness({
             harnessRoot,
             items,
             resolve: (source, ctx) => resolveSkillSource(source, ctx),
             ctx: resolveCtx,
             currentUid,
         });
+        const { pluginDirs, subjectStagedDir, subjectPluginRoot } = stageResult;
         if (subjectStagedDir === null) {
             throw new InternalHarnessError('Staging did not yield a subject directory (no item tagged role:subject).');
         }
-        // Step 4: Resolve the subject's eval path inside its staged dir. Bootstrap
-        // (exit 3) fires when it is absent — scaffold a persistent template the user
-        // can edit, then throw pointing at that real location.
+        // Step 4: Resolve the subject's eval path inside its staged dir, overlaying the
+        // authored suite onto a built/dist subject that doesn't carry it (see
+        // overlayAuthoredEvalSuite). The bootstrap below then fires — and writes a
+        // template to the source scaffold — ONLY when the suite genuinely doesn't exist
+        // anywhere, so authored evals are never overwritten.
+        overlayAuthoredEvalSuite(opts, subjectStagedDir, evalsSubpath);
+        // Bootstrap (exit 3) fires when the suite is absent everywhere — scaffold a
+        // persistent template at the source location the user can edit, then throw.
         const evalsPath = safePath.join(subjectStagedDir, evalsSubpath);
-        // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own staged-subject path
-        if (!existsSync(evalsPath)) {
-            const scaffoldPath = resolveScaffoldEvalsPath(opts, repoRoot, evalsSubpath);
-            const written = writeEvalsTemplate(scaffoldPath, subjectSkillName(opts));
-            throw new BootstrapNeededError(written);
-        }
+        bootstrapEvalSuiteIfMissing(opts, repoRoot, evalsSubpath, evalsPath);
         // Step 5: Preflight checks.
         const knobs = resolveKnobs(opts);
         const preflightInput = buildPreflightInput(evalsPath, pluginDirs, opts, knobs);
@@ -268,6 +537,11 @@ export async function runSkillTestHarness(opts) {
                 summary: `Preflight failed:\n${summary}`,
             };
         }
+        // Step 5.5: Parse the eval suite and stage per-eval input workspaces.
+        const workspaceStageResult = attemptStageWorkspaces(evalsPath, harnessRoot);
+        if ('exitCode' in workspaceStageResult)
+            return workspaceStageResult;
+        const { workspacesRoot } = workspaceStageResult;
         // Step 6: Enforce the §12 security ack (must pass --i-understand-this-runs-skill-code).
         // Only enforced when not a dry-run and not already acknowledged.
         if (!isAcknowledged(opts)) {
@@ -278,8 +552,21 @@ export async function runSkillTestHarness(opts) {
             };
         }
         // Step 7: Build effective experimenter prompt and validate invariants.
-        const resultsDir = safePath.join(harnessRoot, 'results');
+        const resultsDir = safePath.joinUnderRoot(harnessRoot, 'results');
         mkdirSyncReal(resultsDir, { recursive: true });
+        // Record what was actually staged & tested (the subject identity + the
+        // staged manifest fingerprint + per-entry content hashes) so a run is
+        // auditable: which source was resolved, and whether it was rebuilt by run.ts.
+        const provenance = {
+            subject: subjectSkillName(opts),
+            fingerprint: stageResult.manifest.fingerprint,
+            entries: stageResult.manifest.entries,
+            rebuilt: opts.rebuilt === true,
+        };
+        const provenancePath = safePath.join(resultsDir, 'provenance.json');
+        // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
+        writeFileSync(provenancePath, JSON.stringify(provenance, null, 2) + '\n', 'utf-8');
+        process.stderr.write(`Provenance: ${provenance.fingerprint}\n`);
         const gradingOut = safePath.join(resultsDir, 'grading.json');
         const frictionOut = safePath.join(resultsDir, 'friction.json');
         const effectivePrompt = opts.promptOverride ??
@@ -288,39 +575,77 @@ export async function runSkillTestHarness(opts) {
                 evalsPath,
                 gradingOut,
                 frictionOut,
+                workspacesRoot,
                 baseline: opts.baseline ?? false,
             });
         assertPromptInvariants(effectivePrompt);
-        // Step 8: Write the experimenter-prompt.txt.
+        // Integrity nonce (Harness B): stamp a secret per-run nonce the experimenter
+        // must echo into grading.json, and verify it after the run. This is what
+        // distinguishes a grading produced by the experimenter WE prompted from one
+        // forged/left behind by untrusted skill code in the shared sandbox. The nonce
+        // is appended AFTER any user prompt override so a committed config can't opt
+        // out. The full prompt (with the nonce) reaches claude only via stdin (see
+        // spawnHeadlessClaude) and is NEVER written to disk; the persisted audit copy
+        // below is redacted so skill code can't read the nonce back and forge a match.
+        const runNonce = randomBytes(16).toString('hex');
+        const noncedPrompt = appendIntegrityNonceDirective(effectivePrompt, runNonce);
+        // Step 8: Write the REDACTED experimenter prompt as the audit artifact. The
+        // real (nonce-bearing) prompt is passed to the spawn in memory, not from here.
         const promptFile = safePath.join(resultsDir, 'experimenter-prompt.txt');
         // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived path
-        writeFileSync(promptFile, effectivePrompt + '\n', 'utf-8');
+        writeFileSync(promptFile, redactNonce(noncedPrompt, runNonce) + '\n', 'utf-8');
+        // Step 7.5: Resolve the declared test env (Features A + B). Token resolution
+        // can hard-fail (exit 2) on an unknown ${token}; do it before the dry-run
+        // short-circuit so a dry run validates interpolation too.
+        const assembledEnv = resolveDeclaredChildEnv({
+            opts,
+            resolvedAuth: preflightResult.resolvedAuth,
+            subjectStagedDir,
+            harnessRoot,
+            resultsDir,
+            evalsSubpath,
+            subjectPluginRoot,
+        });
+        // Echo the selected model on EVERY run (regular + dry-run) so it is
+        // unambiguous which model the experimenter spawn uses. The id is forwarded
+        // verbatim to `claude --model`; with none set, no flag is passed and claude
+        // uses its own default.
+        const modelFlag = buildModelFlag(knobs.model);
+        process.stderr.write(`Model: ${knobs.model ?? '(claude default)'}\n`);
         // Step 9: Dry-run short-circuit — return assembled info without spawning.
         if (opts.dryRun === true) {
             return {
                 harnessPath: harnessRoot,
                 exitCode: SkillTestExitCode.Ok,
-                summary: `[dry-run] Would spawn: claude -p (prompt via stdin from ${promptFile})`,
+                summary: buildDryRunSummary({
+                    wouldBuild: opts.wouldBuild === true,
+                    ...(opts.dryRunStagedExistingDist === undefined
+                        ? {}
+                        : { dryRunStagedExistingDist: opts.dryRunStagedExistingDist }),
+                    provenancePath,
+                    provenanceFingerprint: provenance.fingerprint,
+                    provenanceEntryCount: provenance.entries.length,
+                    modelFlag,
+                }),
             };
         }
         // Step 10: Spawn headless Claude.
-        // Refuse to spawn with an unscrubbed environment. When preflight passes,
-        // resolvedAuth is always non-null; a null here is an internal invariant
-        // violation — NOT a reason to fall back to the full parent `process.env`,
-        // which would hand the untrusted skill code every secret it contains.
-        if (preflightResult.resolvedAuth === null) {
-            throw new InternalHarnessError('Internal: preflight passed but resolvedAuth is null — refusing to spawn with an unscrubbed environment.');
-        }
-        // A reused harness root (--out / --keep) may carry a grading.json from an
-        // earlier run. Remove it so a post-spawn read can only reflect THIS run.
+        // A reused harness root (--out / --keep) may carry a grading.json OR a
+        // friction.json from an earlier run. Remove both so a post-spawn read can
+        // only reflect THIS run (a stale friction.json must not leak across runs).
         rmSync(gradingOut, { force: true });
+        rmSync(frictionOut, { force: true });
         const timeoutMs = resolveTimeoutMs(opts);
         const spawnOpts = {
-            promptFile,
+            // In-memory, nonce-bearing prompt — streamed to stdin, never written to a
+            // file the skill could read (see the integrity-nonce note above).
+            prompt: noncedPrompt + '\n',
             pluginDirs,
             sandboxDir: harnessRoot,
             cwd: harnessRoot,
-            env: preflightResult.resolvedAuth.forwardedEnv,
+            // The scrubbed forwarded env, unioned with the declared test env (Features
+            // A + B) and CLAUDE_PLUGIN_ROOT when the subject is plugin-distributed.
+            env: assembledEnv.env,
             timeoutMs,
             onStdout: (chunk) => { process.stderr.write(chunk); },
             onStderr: (chunk) => { process.stderr.write(chunk); },
@@ -345,20 +670,31 @@ export async function runSkillTestHarness(opts) {
             throw new InternalHarnessError(`grading.json is not valid JSON: ${String(e)}`);
         }
         const grading = parseGradingJson(gradingRaw);
-        const { passed, total } = grading.summary;
-        const allPassed = passed === total;
+        // Integrity gate (Harness B): reject a grading.json that doesn't echo THIS
+        // run's secret nonce BEFORE trusting any verdict — a missing/wrong nonce means
+        // the grading was not produced by the experimenter we prompted (e.g. forged by
+        // skill code in the sandbox). Thrown GradingNonceError maps to exit 1.
+        assertGradingNonce(grading.runNonce, runNonce);
+        // Verdict comes from the authoritative per-expectation `passed` flags, NOT
+        // the grader's self-reported summary. reconcileGrading throws GradingSkewError
+        // if the grader graded nothing or its summary disagrees with the expectations.
+        const { passed, total, allPassed } = reconcileGrading(grading);
         const summary = `${allPassed ? 'PASS' : 'FAIL'} ${passed}/${total}`;
-        // Exit 0 = harness ran to completion and produced a valid grading.json.
-        // Pass/fail counts are reported in the summary string and grading.json;
-        // callers should not use the exit code to distinguish eval pass from fail.
+        // Default: exit Ok whenever the harness ran to completion and produced a
+        // valid grading.json — pass/fail lives in the summary string and grading.json,
+        // and callers should not read the exit code to distinguish eval pass from fail.
+        // Opt-in: with failOnEvalFailure, a failing verdict escalates to EvalFailure (4).
         return {
             harnessPath: harnessRoot,
-            exitCode: SkillTestExitCode.Ok,
+            exitCode: verdictExitCode(allPassed, opts.failOnEvalFailure === true),
             summary,
         };
     }
     finally {
-        lock.release();
+        // Remove the signal handlers first (no listener leak across runs), then run
+        // the same cleanup: release the lock, then remove the harness dir.
+        removeSignalCleanup();
+        cleanup();
     }
 }
 //# sourceMappingURL=run-harness.js.map