@vibe-agent-toolkit/cli 0.1.42 → 0.2.0-rc.10

package/dist/commands/skill/test/run.js

@@ -7,17 +7,21 @@
  */
 import { existsSync } from 'node:fs';
 import { basename, dirname, extname } from 'node:path';
-import { BootstrapNeededError, buildStaleDistWarningLines, DuplicateStagedSkillError, isAcknowledged, mapErrorToExitCode, packageSkill, packagingConfigToPackageOptions, runPreStageBuild, runSkillTestHarness, SecurityAckError, SKILL_TEST_BUILTIN_CAPS, SkillBuildError, SkillTestExitCode, } from '@vibe-agent-toolkit/agent-skills';
-import { findProjectRoot, resolveAssetReference, safePath, toForwardSlash } from '@vibe-agent-toolkit/utils';
+import { BootstrapNeededError, buildStaleDistWarningLines, DuplicateStagedSkillError, isAcknowledged, conventionalSuiteProbe, packageSkill, packagingConfigToPackageOptions, runPreStageBuild, runSkillTestHarness, SecurityAckError, SKILL_TEST_BUILTIN_CAPS, SkillBuildError, skillTestFailureReason, } from '@vibe-agent-toolkit/agent-skills';
+import { ExitCode } from '@vibe-agent-toolkit/schema';
+import { findProjectRoot, prefixMessageOnce, resolveAssetReference, safePath, toForwardSlashAnyPlatform } from '@vibe-agent-toolkit/utils';
+import { DirectoryListingRefusedError } from '@vibe-agent-toolkit/utils/crawl';
 import { Command } from 'commander';
 import { parseSourceSpec } from '../../../skill-resolution/classify.js';
 import { findDeclaredSkillForPath, findDeclaredSkillForSourceDir, resolveProjectDeclaredEvalSuites, resolveSkillReference, } from '../../../skill-resolution/index.js';
 import { ConfigLoadError, loadConfig, loadConfigCached } from '../../../utils/config-loader.js';
+import { collectRepeated } from '../../../utils/repeatable-option.js';
+import { isSkillPublished } from '../../../utils/skill-packaging-config.js';
 import { runClaudePluginBuild } from '../../claude/plugin/build.js';
 import { assertValidAuth, assertValidRequireAuth } from './auth-flags.js';
 /** Extract the trailing path segment (cross-platform) from a path-like string. */
 function lastPathSegment(p) {
-    return basename(toForwardSlash(p));
+    return basename(toForwardSlashAnyPlatform(p));
 }
 // ---------------------------------------------------------------------------
 // Security warning (§12)
@@ -82,20 +86,28 @@ function coerceKnobs(options) {
 }
 /**
  * Map a resolved skill's packaging-config `executables` (SkillExecutableEntry[]) to
- * the grader's recognition-aid shape (issue #145 Phase T): each entry's stable NAME
+ * the grader's recognition-aid shape: each entry's stable NAME
  * is its `path` basename with the extension stripped (`scripts/csvsum.py` → `csvsum`),
- * carried alongside its `howInvoked` + `kind`. Returns undefined for absent/empty
- * input so the harness omits the aid entirely (the grader still matches tools by the
- * commands in the transcript). Pure + unit-testable.
+ * carried alongside its `howInvoked`, `kind` and the declared `path` itself. Returns
+ * undefined for absent/empty input so the harness omits the aid entirely (the grader
+ * still matches tools by the commands in the transcript). Pure + unit-testable.
+ *
+ * ⛔ THE `path` USED TO BE READ AND DISCARDED. It is the ONLY thing that separates
+ * an ambient copy of the skill — which is a copy, so it reproduces
+ * `…/scripts/csvsum.py` under whatever root it sits at — from the control arm's own
+ * scratch file of the same name. Forward-slashed at the derivation rather than at the
+ * consumer, because every needle and every resolved reach inside the detector is
+ * `/`-normalized and a `\` would silently match nothing.
  */
 export function deriveDeclaredExecutableNames(executables) {
     if (executables === undefined || executables.length === 0)
         return undefined;
     return executables.map((e) => {
-        const base = basename(toForwardSlash(e.path));
+        const path = toForwardSlashAnyPlatform(e.path);
+        const base = basename(path);
         const ext = extname(base);
         const name = ext === '' ? base : base.slice(0, -ext.length);
-        return { name, howInvoked: e.howInvoked, kind: e.kind };
+        return { name, howInvoked: e.howInvoked, kind: e.kind, path };
     });
 }
 /**
@@ -126,7 +138,7 @@ export function parseWithFlags(pairs) {
     for (const pair of pairs) {
         const [name, spec] = parseWithPair(pair);
         // Fail closed on a repeated companion name rather than silently overwriting
-        // (last-wins would drop a companion the user asked for — the #153 no-op class).
+        // (last-wins would drop a companion the user asked for — a silent no-op).
         if (Object.hasOwn(record, name))
             throw new DuplicateStagedSkillError(name);
         record[name] = spec;
@@ -153,8 +165,8 @@ export function descriptorsToRecord(list) {
     for (const d of list) {
         const name = descriptorName(d);
         // Two config descriptors deriving the same name (e.g. two `path:` entries with
-        // the same basename) would silently overwrite — the exact silent-drop #153
-        // targets, one layer above buildStageItems. Fail closed here instead.
+        // the same basename) would silently overwrite — the exact silent drop
+        // buildStageItems refuses, one layer above it. Fail closed here instead.
         if (Object.hasOwn(record, name))
             throw new DuplicateStagedSkillError(name);
         record[name] = d;
@@ -233,7 +245,7 @@ function resolveRepoRoot() {
  * {@link loadTestConfig}, which reads the PER-SKILL `skills.config.<skill>.test`
  * block: graderModel/concurrency are global judge/pipeline settings that apply
  * across every skill's test run, not something a single skill's config should
- * override (issue #145).
+ * override.
  *
  * Undefined when there's no project root or no `test:` node. A broken config
  * throws {@link ConfigLoadError}, propagated the same way as loadTestConfig's
@@ -272,7 +284,7 @@ function applyScalarMerges(opts, options, config) {
     const authRequirement = options.requireAuth ?? config?.requireAuth;
     if (authRequirement !== undefined)
         opts.requireAuth = authRequirement;
-    const baseline = options.baseline ?? config?.baseline;
+    const baseline = resolveBaseline(options.baseline, config?.baseline);
     if (baseline !== undefined)
         opts.baseline = baseline;
     const model = options.model ?? config?.model;
@@ -287,6 +299,36 @@ function applyScalarMerges(opts, options, config) {
     if (evalsRef !== undefined)
         opts.evalsSubpath = evalsRef;
 }
+/**
+ * Resolve `--baseline` with flag > config precedence, ANNOUNCING a config-sourced
+ * enable on stderr.
+ *
+ * Same threat model as {@link resolveCappedKnob} next door — a value from a
+ * COMMITTED config rides along in an untrusted subject repo — applied to the one
+ * cost knob that has no cap to clamp. `baseline` is the LARGEST cost multiplier in
+ * this command: it runs every eval TWICE (skill declared vs withheld), and each arm
+ * is an executor spawn AND a grader spawn, so `skills.config.<skill>.test.baseline:
+ * true` roughly doubles the spend of every `vat skill test run` an operator types.
+ * `--max-budget-usd` is a PER-SPAWN cap, so the worst-case ceiling doubles with it.
+ *
+ * A boolean cannot be clamped to a ceiling, so it is announced instead, and
+ * `--no-baseline` exists so the operator can turn it back off for THIS run without
+ * hand-editing someone else's YAML. Precedence runs both directions: an explicit
+ * flag wins whether it says true or false (hence the `undefined` check rather than
+ * `??`-style falsiness, which would let a config `true` override `--no-baseline`).
+ *
+ * Only an ENABLE is announced. A config `false` costs nothing and needs no note.
+ */
+export function resolveBaseline(flagValue, configValue) {
+    if (flagValue !== undefined)
+        return flagValue;
+    if (configValue !== true)
+        return configValue;
+    process.stderr.write("Note: baseline is ON from this project's committed config ('skills.config.<skill>.test.baseline'), " +
+        'not from a flag; it runs every eval TWICE (an executor AND a grader spawn per arm), roughly doubling ' +
+        'the spawns and the spend. Pass --no-baseline to turn it off for this run.\n');
+    return true;
+}
 /**
  * Resolve one cost/runtime knob with flag > config > default precedence AND a
  * built-in safety cap, enforcing an asymmetry between the two sources:
@@ -401,15 +443,16 @@ function applyGraderMerges(opts, options, knobs, globalTest) {
  * precedence. CLI flags win; config (`skills.config.<skill>.test`) fills gaps;
  * built-in defaults (inside the domain) are the final fallback.
  */
-function buildHarnessOpts(subject, options, knobs, config, globalTest) {
+function buildHarnessOpts(subject, options, knobs, config, globalTest, resolvedSubject) {
     const repoRoot = resolveRepoRoot();
-    const opts = { subject, repoRoot };
+    const opts = { subject, repoRoot, subjectInPlace: resolvedSubject.inPlace };
     applyFlagOnlyOptions(opts, options);
     applyScalarMerges(opts, options, config);
     applyKnobMerges(opts, knobs, config);
     applyDepMerges(opts, options, config);
     applyEnvMerges(opts, options, config);
     applyGraderMerges(opts, options, knobs, globalTest);
+    applyResolvedSubject(opts, resolvedSubject);
     return opts;
 }
 /**
@@ -432,7 +475,7 @@ function sourceScaffoldFields(source, declaredSkill, cwd) {
  * Project-aware subject resolution for `vat skill test run`. Resolves the subject
  * reference; for a declared skill, builds it (real entry points) and returns the
  * dist dir to stage; everything else is staged as-is. Throws SkillBuildError
- * (exit 2) for name-miss / not-found / --no-build-without-dist / build failure.
+ * (reason `preflight`) for name-miss / not-found / --no-build-without-dist / build failure.
  *
  * `memo` is the per-run build memo (see {@link BuildMemo}) — pass the SAME map used
  * for companion resolution so a skill that is both subject and companion builds once.
@@ -449,6 +492,7 @@ export async function resolveSubjectForTest(ref, cwd, flags, memo, buildCommand)
             return {
                 subjectSource: resolved.source,
                 rebuilt: false,
+                inPlace: false,
                 wouldBuild: false,
                 ...sourceScaffoldFields(resolved.source, resolved.declaredSkill, cwd),
             };
@@ -466,6 +510,16 @@ export async function resolveSubjectForTest(ref, cwd, flags, memo, buildCommand)
         }
     }
 }
+/**
+ * An IN-PLACE declared skill: a POOL skill whose merged config says `publish: false`.
+ * `vat build` never bundles it (see `isSkillPublished`), so a test must not invent the
+ * bundle either — it stages the authored directory, which is what is used in place. A
+ * plugin-local skill is never in-place: `publish` scopes the pool only, and it ships
+ * with its plugin whatever the flag says.
+ */
+function isInPlaceSkill(ref) {
+    return ref.distribution.kind === 'pool' && !isSkillPublished(ref.packagingConfig);
+}
 /**
  * --no-build / --dry-run branch of {@link buildDeclaredSkill}: never builds. Stage
  * the existing dist if present; for a dry-run with no dist yet, fall back to the
@@ -473,7 +527,6 @@ export async function resolveSubjectForTest(ref, cwd, flags, memo, buildCommand)
  * when --no-build is set but no dist exists.
  */
 function resolveExistingDistOrThrow(ref, flags) {
-    // eslint-disable-next-line security/detect-non-literal-fs-filename -- expectedDistDir derived from project config
     if (existsSync(ref.expectedDistDir)) {
         if (flags.noBuild) {
             process.stderr.write(`Using existing dist (NOT rebuilt): ${ref.expectedDistDir}\n`);
@@ -513,7 +566,7 @@ export function buildMemoKey(ref) {
  * writes nothing (or writes an empty shell with no `SKILL.md`) would otherwise hand
  * back a path a required companion then fails opaquely at staging, or an OPTIONAL
  * one silently skips — the exact non-functional-companion-with-no-diagnostic
- * symptom of issue #158. Two distinct checks, two distinct messages, so the error
+ * symptom adopters reported. Two distinct checks, two distinct messages, so the error
  * tells the author which failure mode they hit:
  *   1. `expectedDistDir` itself is missing — "no output at all".
  *   2. `expectedDistDir` exists but has no `SKILL.md` — an empty/incomplete shell.
@@ -523,13 +576,11 @@ export function buildMemoKey(ref) {
  * and is always checked.
  */
 function verifyBuiltDist(ref) {
-    // eslint-disable-next-line security/detect-non-literal-fs-filename -- expectedDistDir derived from project config
     if (!existsSync(ref.expectedDistDir)) {
         throw new SkillBuildError(`Skill build for '${ref.name}' reported success but produced no output at ${ref.expectedDistDir}. ` +
             `Check the skill's packaging config (\`vat build\` should create this directory).`);
     }
     const skillMdPath = safePath.join(ref.expectedDistDir, 'SKILL.md');
-    // eslint-disable-next-line security/detect-non-literal-fs-filename -- skillMdPath derived from project config
     if (!existsSync(skillMdPath)) {
         throw new SkillBuildError(`Skill build for '${ref.name}' reported success but produced no SKILL.md at ${ref.expectedDistDir}. ` +
             `Check the skill's packaging config (\`vat build\` should create this file).`);
@@ -580,7 +631,20 @@ async function runDeclaredSkillBuild(ref) {
         // excludes every skill's test input. Getting this wrong would hand the executor
         // under test another skill's answer key — the exact signal the harness exists to
         // protect. Memoized per config root, so a multi-skill run discovers once.
-        await packageSkill(ref.sourcePath, packagingConfigToPackageOptions(ref.packagingConfig, { skillPath: ref.sourcePath, outputPath: ref.expectedDistDir }, await resolveProjectDeclaredEvalSuites(ref.sourcePath)));
+        await packageSkill(ref.sourcePath, packagingConfigToPackageOptions(ref.packagingConfig, { skillPath: ref.sourcePath, outputPath: ref.expectedDistDir }, 
+        // `'refuse'`: staging a dist around a directory discovery could not
+        // list may hand the executor another skill's answer key. The throw is
+        // wrapped as `SkillBuildError` by the caller, message intact.
+        await resolveProjectDeclaredEvalSuites(ref.sourcePath, 'refuse'), 
+        // Per CALL, deliberately, and this is parity rather than a regression: the
+        // probe was built inside `resolveTestInputDirs` on every path before it was
+        // threaded, so this lane pays exactly what it paid before. It is also the
+        // wrong lane to optimize — `runDeclaredSkillBuild` runs per buildable ref for
+        // a test run's handful of skills, not over a whole project, and there is no
+        // run-scoped seam threaded this far down to hang one on. The lane that made
+        // the O(S²) matter (`vat resources validate`, measured at 10,815 probes over
+        // 103 paths) hoists its probe; see the comment there.
+        conventionalSuiteProbe()));
         return;
     }
     await runClaudePluginBuild(ref.configRoot, { marketplace: ref.distribution.marketplaceName });
@@ -590,7 +654,7 @@ async function runDeclaredSkillBuild(ref) {
  * the existing dist under --no-build/--dry-run. Shared by SUBJECT resolution
  * ({@link resolveBuildableSubject}) and `--with`/`--with-optional` COMPANION
  * resolution ({@link resolveCompanionSpec}) so a companion that maps to a declared
- * skill gets the exact same build treatment as the subject (issue #158) — its
+ * skill gets the exact same build treatment as the subject — its
  * `files:` injection runs before staging, instead of a raw, possibly
  * build-artifact-incomplete, source-tree copy.
  *
@@ -619,8 +683,11 @@ async function buildDeclaredSkill(ref, flags, memo, buildCommand) {
     // dry run stays the one mode that is safe to point at an untrusted clone. That
     // preview falls back to an existing dist and says so (stale warning) — the honest
     // trade, since accuracy requires consent to execute the repo's own build.
+    // An in-place skill has no dist to reuse or build: its authored directory IS what
+    // ships (in place), under every flag combination.
+    const inPlaceResult = { distDir: dirname(ref.sourcePath), rebuilt: false };
     if (flags.noBuild || (flags.dryRun && !flags.explicitAck)) {
-        return resolveExistingDistOrThrow(ref, flags);
+        return isInPlaceSkill(ref) ? inPlaceResult : resolveExistingDistOrThrow(ref, flags);
     }
     // SECURITY (§12): reaching here means a real build WOULD run — runPreStageBuild
     // executes an ARBITRARY shell command from the repo's committed `test.build`, and
@@ -630,6 +697,12 @@ async function buildDeclaredSkill(ref, flags, memo, buildCommand) {
     if (!flags.acknowledged) {
         throw new SecurityAckError();
     }
+    // The `test.build` hook still runs for an in-place skill — it prepares what the
+    // test needs (the declared contract: once, before staging) — but nothing is bundled.
+    if (isInPlaceSkill(ref)) {
+        runPreStageBuildOnce(buildCommand, ref.configRoot, memo);
+        return inPlaceResult;
+    }
     const key = buildMemoKey(ref);
     // Memo HIT: this operation already ran in THIS process — skip the rebuild, but
     // still verify this ref's own dist (the hit may be a marketplace sibling's).
@@ -660,10 +733,22 @@ async function buildDeclaredSkill(ref, flags, memo, buildCommand) {
 async function resolveBuildableSubject(ref, flags, memo, buildCommand) {
     const scaffoldDir = dirname(ref.sourcePath);
     const build = await buildDeclaredSkill(ref, flags, memo, buildCommand);
+    if (isInPlaceSkill(ref)) {
+        // Staged as-is from source, but still the declared skill: its `test:` config applies.
+        return {
+            subjectSource: { path: build.distDir },
+            subjectScaffoldDir: scaffoldDir,
+            rebuilt: false,
+            inPlace: true,
+            wouldBuild: false,
+            linkedToDeclaredSkill: true,
+        };
+    }
     return {
         subjectSource: { path: build.distDir },
         subjectScaffoldDir: scaffoldDir,
         rebuilt: build.rebuilt,
+        inPlace: false,
         wouldBuild: true,
         ...(build.dryRunStagedExistingDist === undefined
             ? {}
@@ -713,11 +798,12 @@ function isSurvivableCompanionFailure(err, declared, optional, flags) {
  * with several companions leaves the user unable to tell which flag to fix.
  *
  * The message is prefixed IN PLACE so the error's CLASS (and therefore its
- * `exitCode` / {@link mapErrorToExitCode} mapping) is unchanged — no new error class,
- * no re-wrapping. Prefixing is skipped when the message is ALREADY prefixed (starts
- * with `companion '`) — this makes the function idempotent against repeat rethrows of
- * the SAME error object, structurally, rather than relying on a hand-maintained
- * argument that no caught error class is ever cached/reused across companions.
+ * `reason` / {@link skillTestFailureReason} mapping) is unchanged — no new error class,
+ * no re-wrapping. Prefixing happens ONCE per error object (`prefixMessageOnce`
+ * remembers the object, never re-reads the message) — this makes the function
+ * idempotent against repeat rethrows of the SAME error object, structurally, rather
+ * than relying on a hand-maintained argument that no caught error class is ever
+ * cached/reused across companions.
  * {@link ConfigLoadError} instances ARE cached and re-thrown per config root by
  * `loadConfigCached`, so mutating one in place would otherwise corrupt the cache; the
  * already-prefixed check covers this case too (in practice `ConfigLoadError` is
@@ -728,18 +814,16 @@ function isSurvivableCompanionFailure(err, declared, optional, flags) {
  * Debug output that prints `.stack` can disagree with the prefixed text on stderr.
  */
 function rethrowNamingCompanion(err, alias, declaredName) {
-    if (err instanceof Error && !err.message.startsWith("companion '")) {
-        err.message = `companion '${alias}' (declared skill '${declaredName}'): ${err.message}`;
-    }
+    prefixMessageOnce(err, `companion '${alias}' (declared skill '${declaredName}'): `);
     throw err;
 }
 /**
- * Companion analog of {@link resolveBuildableSubject} (issue #158): resolve every
+ * Companion analog of {@link resolveBuildableSubject}: resolve every
  * `--with`/`--with-optional` companion whose spec is a `{ path }` pointing at a
  * declared skill's SOURCE directory to that skill's BUILT dist, via the exact same
  * {@link buildDeclaredSkill} the subject uses. Anything else — workspace:/npm:/
- * url:/vendored specs, or a path outside this project's config — is "a different
- * story" (per the issue's own framing): left untouched, staged as today.
+ * url:/vendored specs, or a path outside this project's config — is a different
+ * story: left untouched, staged as today.
  *
  * The `test.build` hook used is the COMPANION's own — read from the config that
  * governs IT ({@link declaredSkillTestConfig}), not the subject's. The subject's hook
@@ -821,7 +905,7 @@ function warnIfPathTargetBypassesConfig(subject) {
 /**
  * Validate usage-level flags (auth values, numeric knobs) and load the subject's
  * persisted test config. Runs before the async harness work, so a bad flag exits
- * with a clean message + preflight code (2) instead of surfacing as an unhandled
+ * with a clean message + `Reason: preflight` instead of surfacing as an unhandled
  * promise rejection (raw stack trace, exit 1).
  */
 async function preflightKnobsAndConfig(subject, options, cwd) {
@@ -836,21 +920,49 @@ async function preflightKnobsAndConfig(subject, options, cwd) {
     }
     catch (err) {
         process.stderr.write(`Error: ${err instanceof Error ? err.message : String(err)}\n`);
-        process.exit(SkillTestExitCode.Preflight);
+        exitWithReason('preflight');
     }
 }
 /**
- * Exit code for an error escaping EITHER phase of {@link runSkillTestRun}.
+ * End the run on `ERROR`, saying WHY on stderr.
+ *
+ * The reason used to be three exit codes (1 internal, 2 preflight, 3
+ * bootstrap). Under the one exit-code contract every verb shares, every way the
+ * harness could not run is `ERROR`, and the `Reason:` line is where a CI
+ * author reads "harness broke" apart from "my environment is wrong".
+ */
+function exitWithReason(reason) {
+    process.stderr.write(`Reason: ${reason}\n`);
+    process.exit(ExitCode.ERROR);
+}
+/**
+ * End the run on the harness's own verdict: the `Summary:` line on stdout (the
+ * one channel kept machine-readable), the `Reason:` line on stderr when the
+ * harness could not run, and the exit code the harness decided.
+ */
+function exitWithResult(result) {
+    process.stdout.write(`Summary: ${result.summary}\n`);
+    if (result.reason !== undefined)
+        process.stderr.write(`Reason: ${result.reason}\n`);
+    process.exit(result.exitCode);
+}
+/**
+ * The reason for an error escaping EITHER phase of {@link runSkillTestRun}.
  *
  * A broken governing config ({@link ConfigLoadError}) is a user-fixable PREFLIGHT
- * problem (exit 2), not an internal harness failure — {@link mapErrorToExitCode} has
- * no case for it and falls through to Internal (1). ONE helper shared by BOTH catch
+ * problem, not an internal harness failure — {@link skillTestFailureReason} has
+ * no case for it and falls through to `internal`. ONE helper shared by BOTH catch
  * blocks so the SUBJECT arm and the COMPANION arm can never drift: a companion's
  * governing config root can differ from the subject's, so a broken config is
- * reachable from either arm and must report the same code from both.
+ * reachable from either arm and must report the same reason from both.
  */
-function exitCodeForRunError(err) {
-    return err instanceof ConfigLoadError ? SkillTestExitCode.Preflight : mapErrorToExitCode(err);
+function reasonForRunError(err) {
+    // A directory the governing `skills.include` reaches and the crawl cannot list
+    // is the same class: discovery refuses it by name, and the remedy is `chmod` or
+    // a narrower include pattern — the operator's, not the harness's.
+    return err instanceof ConfigLoadError || err instanceof DirectoryListingRefusedError
+        ? 'preflight'
+        : skillTestFailureReason(err);
 }
 /**
  * Copy the resolved-subject fields onto the harness options. Optional fields are
@@ -892,7 +1004,7 @@ export async function runSkillTestRun(subject, options) {
         dryRun: options.dryRun === true,
         acknowledgedRunsSkillCode: options.iUnderstandThisRunsSkillCode === true,
     });
-    // Shared by subject AND companion resolution (issue #158) so a --with/--with-optional
+    // Shared by subject AND companion resolution so a --with/--with-optional
     // companion that maps to a declared skill gets the exact same build GATING (no-build/
     // dry-run/security-ack) as the subject. The `test.build` hook is deliberately NOT in
     // here: it is per-skill (see BuildFlags), so the subject's own command is passed only
@@ -913,20 +1025,19 @@ export async function runSkillTestRun(subject, options) {
     }
     catch (err) {
         // A broken governing config surfacing during subject resolution is a preflight
-        // problem the user must fix (exit 2), not an internal harness failure (exit 1).
-        const exitCode = exitCodeForRunError(err);
+        // problem the user must fix, not an internal harness failure.
         process.stderr.write(`Error: ${err instanceof Error ? err.message : String(err)}\n`);
-        process.exit(exitCode);
+        // 🪤 `return` after a `never`: under test `process.exit` is a spy that RETURNS.
+        exitWithReason(reasonForRunError(err));
         return;
     }
     warnIfPathTargetBypassesConfig(resolvedSubject);
     try {
         // buildHarnessOpts assembles the companion records (--with/--with-optional and
         // config with:/optional:) and can throw DuplicateStagedSkillError on a repeated
-        // name — inside the try so it maps to exit 2 like every other preflight error.
-        const harnessOpts = buildHarnessOpts(subject, options, knobs, config, globalTest);
-        applyResolvedSubject(harnessOpts, resolvedSubject);
-        // Companion build resolution (issue #158): a --with/--with-optional companion
+        // name — inside the try so it reads as preflight like every other preflight error.
+        const harnessOpts = buildHarnessOpts(subject, options, knobs, config, globalTest, resolvedSubject);
+        // Companion build resolution: a --with/--with-optional companion
         // whose source is a path into a declared skill gets built (its `files:`
         // injection runs) exactly like the subject, instead of a raw source-tree copy.
         const repoRoot = resolveRepoRoot();
@@ -937,25 +1048,56 @@ export async function runSkillTestRun(subject, options) {
         if (withOptional !== undefined)
             harnessOpts.withOptional = withOptional;
         const result = await runSkillTestHarness(harnessOpts);
-        process.stderr.write(`Harness: ${result.harnessPath}\n`);
-        process.stdout.write(`Summary: ${result.summary}\n`);
-        process.exit(result.exitCode);
+        // Same rule as the `Workspaces:` line below — "a populated value is a promise
+        // that the path is still there" — applied to the sibling it was written next to
+        // and not applied to. Harness cleanup runs inside the harness's own `finally`,
+        // BEFORE this result reaches the CLI: a run that returns before Step 7 creates
+        // `results/` (the §12 security-ack refusal, and the preflight-failure branch
+        // reachable from a `--require-auth` mismatch) leaves the harness root with no
+        // `results/` child, and cleanup removes the ROOT outright. Printing the path
+        // anyway hands the operator a directory that no longer exists.
+        //
+        // Asked of the PATH rather than re-derived from --keep/--out/--workdir on
+        // purpose: the retention rule has ONE author (cleanupHarness), and a second copy
+        // of its predicate here would drift from it. The filesystem cannot.
+        if (existsSync(result.harnessPath)) {
+            process.stderr.write(`Harness: ${result.harnessPath}\n`);
+        }
+        // The artifacts this command's help text tells the operator to read
+        // (grading.json / friction.json / baseline.json). Reported separately from the
+        // harness path because on a default run — no --out, no --workdir, no --keep —
+        // results/ is the ONLY thing left under it: everything else is staged untrusted
+        // bytes that cleanup evicts. Pointing at the harness root alone made the
+        // operator guess which of its children still existed.
+        if (result.resultsPath !== undefined) {
+            process.stderr.write(`Results: ${result.resultsPath}\n`);
+        }
+        // The executor's working directories live OUTSIDE the harness root under an
+        // unguessable token, so the harness path no longer leads an operator to them.
+        // Under --keep they survive holding everything the evals produced; unreported,
+        // they would be an orphan the operator cannot find to inspect or reap. On every
+        // other run cleanup deletes them, and the harness answers with no
+        // `workspacesPath` at all — the retention rule has ONE author, over there, so
+        // this stays a plain presence check and never a second copy of it.
+        if (result.workspacesPath !== undefined) {
+            process.stderr.write(`Workspaces: ${result.workspacesPath}\n`);
+        }
+        exitWithResult(result);
         return;
     }
     catch (err) {
-        // Same rule as the subject arm above (see {@link exitCodeForRunError}): companion
+        // Same rule as the subject arm above (see {@link reasonForRunError}): companion
         // resolution can surface a ConfigLoadError from a DIFFERENT config root than the
-        // subject's, and that must exit 2 (preflight), not 1 (internal).
-        const exitCode = exitCodeForRunError(err);
-        // BootstrapNeededError (exit 3) is the happy "wrote a template, fill it in
-        // and re-run" path — surface its message plainly, not as a hard `Error:`.
+        // subject's, and that must read as preflight, not internal.
+        // BootstrapNeededError is the happy "wrote a template, fill it in and
+        // re-run" path — surface its message plainly, not as a hard `Error:`.
         if (err instanceof BootstrapNeededError) {
             process.stderr.write(`${err.message}\n`);
         }
         else {
             process.stderr.write(`Error: ${err instanceof Error ? err.message : String(err)}\n`);
         }
-        process.exit(exitCode);
+        exitWithReason(reasonForRunError(err));
         return;
     }
 }
@@ -967,11 +1109,15 @@ export function createSkillTestRunCommand() {
     command
         .description('Execute a packaged skill\'s eval suite in a headless, context-isolated Claude session (runs the skill\'s code; not an OS sandbox)')
         .argument('<skill>', 'The skill to test')
-        .option('--with <pair...>', 'Stage a REQUIRED companion skill the subject can invoke, as name=<src> (repeatable). <src> is workspace:<pkg> | npm:<spec> | url:<u> | path:<dir> | vendored (e.g. helper=npm:@scope/s@1.2.3). A path:<dir> that maps to a declared skill is BUILT like the subject (files: build artifacts included), not tree-copied as raw source. The run fails if a source cannot be resolved or its build fails.')
-        .option('--with-optional <pair...>', 'Stage an OPTIONAL companion skill, as name=<src> (same syntax, repeatable). Staged from its raw (unbuilt) source with a warning if its source cannot be resolved, or if a non-destructive build fails; the run continues.')
-        .option('--env <pair...>', 'Inject an env var into the executor spawn as KEY=VALUE (repeatable). Values support ${fixturesDir}, ${stagedSkillDir}, ${harnessRoot}, ${resultsDir}. ${fixturesDir} is per-eval and requires that eval to declare input `files`. CLI overrides config for the same key.')
-        .option('--pass-env <key...>', 'Forward a host env var by NAME to the executor spawn if present (repeatable). Protected names (PATH, auth, model) are ignored.')
-        .option('--refresh', 'Force a full re-stage (ignore existing staged content)')
+        // All four are repeatable single-value options, NOT variadic. A variadic
+        // option consumes every token up to the next flag, so it ate the `<skill>`
+        // positional and the run died reporting the subject missing. See
+        // collectRepeated. Their help text always said "(repeatable)"; now it is true.
+        .option('--with <pair>', 'Stage a REQUIRED companion skill the subject can invoke, as name=<src> (repeatable). <src> is workspace:<pkg> | npm:<spec> | url:<u> | path:<dir> | vendored (e.g. helper=npm:@scope/s@1.2.3). A path:<dir> that maps to a declared skill is BUILT like the subject (files: build artifacts included), not tree-copied as raw source. The run fails if a source cannot be resolved or its build fails.', collectRepeated)
+        .option('--with-optional <pair>', 'Stage an OPTIONAL companion skill, as name=<src> (same syntax, repeatable). Staged from its raw (unbuilt) source with a warning if its source cannot be resolved, or if a non-destructive build fails; the run continues.', collectRepeated)
+        .option('--env <pair>', 'Inject an env var into the executor spawn as KEY=VALUE (repeatable). Values support ${fixturesDir}, ${stagedSkillDir}, ${harnessRoot}, ${resultsDir}. ${fixturesDir} is per-eval and requires that eval to declare input `files`. CLI overrides config for the same key.', collectRepeated)
+        .option('--pass-env <key>', 'Forward a host env var by NAME to the executor spawn if present (repeatable). Protected names (PATH, auth, model) are ignored.', collectRepeated)
+        .option('--refresh', 'No-op today, accepted for forward compatibility: staging already does a full re-stage on every run, so there is no retained staged content for this flag to ignore. Nothing reads it — passing it changes nothing.')
         .option('--no-build', 'Skip building declared skills (subject and any --with/--with-optional companion); stage existing dist instead. Errors if absent for the subject or a REQUIRED companion; an OPTIONAL companion falls back to raw source with a warning.')
         .option('--workdir <dir>', 'Override the harness working directory')
         .option('--out <dir>', 'Override the harness output directory')
@@ -979,15 +1125,19 @@ export function createSkillTestRunCommand() {
         .option('--dry-run', 'Build and stage exactly as a real run would, then stop without spawning Claude (no tokens spent). Combine with --no-build to skip the build too.')
         .option('--auth <mode>', 'Auth mechanism: inherit | subscription | api-key | auto')
         .option('--require-auth <mech>', 'Require a specific auth mechanism: subscription | api-key')
-        .option('--baseline', 'Enable A/B baseline run (with/without skill)')
-        .option('--allow-eval-failure', 'Opt out of the fail-closed default: exit 0 even when an eval fails (for interactive use). By DEFAULT a failing eval exits 4, distinct from the harness-broke codes (1/2/3) so CI can gate on it.')
+        // `--baseline` MUST be declared before `--no-baseline`: Commander only leaves the
+        // value undefined-when-untyped if a positive option already exists, and that
+        // tri-state is what lets a flag beat config in BOTH directions (see resolveBaseline).
+        .option('--baseline', "A/B the skill's INSTRUCTIONS (declared vs withheld) and report the lift on stderr. Runs every eval TWICE, so it roughly DOUBLES the spawns and the spend (--max-budget-usd is per-spawn). Both arms share a filesystem — not a capability control; see baselineDelta and baselineIntegrity in baseline.json")
+        .option('--no-baseline', 'Force the baseline A/B OFF for this run, overriding a committed `test.baseline: true` in the project config (which is announced on stderr when it applies).')
+        .option('--allow-eval-failure', 'Opt out of the fail-closed default: exit 0 even when an eval fails (for interactive use). By DEFAULT a failing eval exits 1, distinct from the harness-could-not-run exit 2 (with a Reason: line) so CI can gate on it.')
         .option('--allow-unverified-skill-source', 'Skip the vendored manifest integrity check')
         .option('--i-understand-this-runs-skill-code', 'Acknowledge this command executes skill code (required)')
         .option('--model <id>', "Model ID passed VERBATIM to `claude --model <id>` for the model UNDER TEST (the executor). VAT does no mapping/validation (e.g. opus, sonnet, claude-opus-4-8). Omit to use claude's own default model. Independent of --grader-model.")
         .option('--grader-model <id>', 'Model for the fixed grader/judge, passed VERBATIM to `claude --model` (default claude-sonnet-5). GLOBAL (top-level `test:` config), independent of --model (the model UNDER TEST).')
         .option('--evals <path>', "Eval suite to grade against: a path to an evals.json (resolved against the CURRENT DIRECTORY, so it may point outside the skill's tree) or an npm bare specifier honoring that package's exports map. Overrides `test.evals`, which is resolved against the skill source instead. Use this to test a skill you did not author — a correctly packaged skill ships no evals, because the suite is the answer key.")
         .option('--max-turns <n>', 'Per-spawn cap on executor/grader turns (positive integer)')
-        .option('--max-budget-usd <n>', 'Hard USD budget cap (positive number)')
+        .option('--max-budget-usd <n>', 'Per-spawn USD budget cap, applied to EACH executor and grader spawn (positive number). Not a whole-run ceiling: a --baseline run has twice the spawns, so twice the worst-case spend.')
         .option('--timeout <s>', 'Wall-clock timeout in seconds (positive integer)')
         .option('--stall <s>', 'Stall-watchdog in seconds (positive integer)')
         .option('--concurrency <n>', 'Max evals graded in parallel (positive integer; default 4)')
@@ -1004,11 +1154,41 @@ Description:
   transcript against the skill's expectations. VAT merges the grader results and
   writes grading.json.
 
+  A declared skill is built before staging -- except an IN-PLACE one (merged
+  config publish: false), which 'vat build' never bundles: it is staged from its
+  source directory, and friction.json says links leaving that directory are absent.
+
   IMPORTANT: This command EXECUTES the skill's code with your user account's
   full privileges (filesystem, network, shell) and a reachable auth credential.
   Only run skills you trust. You MUST pass --i-understand-this-runs-skill-code
   to acknowledge this and proceed.
 
+Artifacts:
+  grading.json (the verdict), friction.json, tool-eval.json, provenance.json,
+  and -- with --baseline -- baseline.json are written to a results/ directory
+  whose path is echoed to stderr ("Results: <path>"). On a DEFAULT run (no --out,
+  --workdir or --keep) that directory SURVIVES and the staged skill bytes around
+  it are removed.
+
+  --out or --workdir spares only the harness ROOT: it is a location you own, so
+  vat never deletes it, and the staged (untrusted) skill bytes inside it stay
+  until you delete them yourself. It is NOT a blanket "keep everything". Three
+  vat-owned directories live OUTSIDE that root -- the grader's output dir, the
+  held eval suite, and the per-eval executor WORKSPACES (every eval's working
+  directory and everything the evals wrote) -- and they are reaped on a rule that
+  ignores --out and --workdir entirely. --keep is the only flag that retains the
+  workspaces; without it they are gone before this command prints, which is why
+  the "Workspaces: <path>" line appears only under --keep. Pass --keep when you
+  intend to inspect what the evals produced.
+
+  A --baseline run also echoes its lift to stderr ("Baseline delta: +2 (with
+  skill: 3/3, without skill: 1/3)."), and stamps the same numbers into
+  baseline.json as baselineDelta, run-level and per eval. A delta of 0 means the
+  skill lifted nothing; a delta of null means the two arms were graded against a
+  different number of expectations and cannot be subtracted at all -- see
+  baselineIntegrity.skew for which evals, and prefer null over a number that
+  would read as "100% without the skill".
+
 Model:
   --model <id> selects the model UNDER TEST (the executor spawn): passed
   straight through as \`claude --model <id>\` (verbatim -- VAT does not map or
@@ -1027,23 +1207,24 @@ Model:
 
 Exit Codes:
   0 - Harness ran to completion and every eval passed (or --allow-eval-failure suppressed a failing verdict)
-  1 - Internal error (grader fragment absent/invalid, summary/expectations skew, executor/grader crash, stall/timeout)
-  2 - Preflight failed (missing binary, auth error, eval inputs absent, unsafe workdir, ack missing, broken project config, a required skill -- subject or --with companion -- failed to build, --no-build with no existing dist for one of them, or an OPTIONAL --with-optional companion hitting a non-survivable failure: a destructive plugin-local build failure, missing security ack, or broken config)
-  3 - Bootstrap needed: evals.json was absent, so VAT wrote a starter template next to the skill source. Fill it in and re-run.
-  4 - An eval FAILED (the harness completed and produced a valid grading.json; expectations did not all pass). This is the fail-closed DEFAULT -- suppress with --allow-eval-failure.
+  1 - An eval FAILED (the harness completed and produced a valid grading.json; expectations did not all pass). This is the fail-closed DEFAULT -- suppress with --allow-eval-failure.
+  2 - The harness could not run. A 'Reason: <reason>' line on stderr says why:
+        internal  - the harness broke (grader fragment absent or off-schema, summary/expectations skew, executor/grader crash, stall/timeout)
+        preflight - the environment or inputs need fixing (missing binary, auth error, eval inputs absent, unsafe workdir, ack missing, broken project config, a required skill -- subject or --with companion -- failed to build, --no-build with no existing dist for one of them, or an OPTIONAL --with-optional companion hitting a non-survivable failure: a destructive plugin-local build failure, missing security ack, or broken config)
+        bootstrap - evals.json was absent, so VAT wrote a starter template next to the skill source. Fill it in and re-run.
 
-  The taxonomy is designed so a CI consumer can tolerate eval failures while
-  failing closed on every other (harness-broke) outcome:
+  The same three-way contract as every other vat command, so a CI consumer
+  can tolerate eval failures while failing closed on a harness that broke:
 
     vat skill test run my-skill --i-understand-this-runs-skill-code
     case $? in
       0) ;;              # all evals passed
-      4) ;;              # evals failed but the harness is healthy -- tolerate/warn
-      *) exit 1 ;;       # 1/2/3/unknown -- harness broke, fail the build
+      1) ;;              # evals failed but the harness is healthy -- tolerate/warn
+      *) exit 1 ;;       # harness could not run -- fail the build; read Reason:
     esac
 
   Which specific evals failed lives in grading.json, never in the exit code.
-  For interactive iteration, --allow-eval-failure downgrades 4 to 0.
+  For interactive iteration, --allow-eval-failure downgrades 1 to 0.
 
 Example:
   $ vat skill test run my-skill --i-understand-this-runs-skill-code