@vibe-agent-toolkit/cli 0.1.41-rc.4 → 0.1.41-rc.6

package/dist/commands/skill/test/run.js

@@ -7,12 +7,12 @@
  */
 import { existsSync } from 'node:fs';
 import { basename, dirname, extname } from 'node:path';
-import { BootstrapNeededError, DuplicateStagedSkillError, isAcknowledged, mapErrorToExitCode, packageSkill, packagingConfigToPackageOptions, runPreStageBuild, runSkillTestHarness, SecurityAckError, SKILL_TEST_BUILTIN_CAPS, SkillBuildError, SkillTestExitCode, } from '@vibe-agent-toolkit/agent-skills';
-import { findProjectRoot, safePath, toForwardSlash } from '@vibe-agent-toolkit/utils';
+import { BootstrapNeededError, buildStaleDistWarningLines, DuplicateStagedSkillError, isAcknowledged, mapErrorToExitCode, packageSkill, packagingConfigToPackageOptions, runPreStageBuild, runSkillTestHarness, SecurityAckError, SKILL_TEST_BUILTIN_CAPS, SkillBuildError, SkillTestExitCode, } from '@vibe-agent-toolkit/agent-skills';
+import { findProjectRoot, resolveAssetReference, safePath, toForwardSlash } from '@vibe-agent-toolkit/utils';
 import { Command } from 'commander';
 import { parseSourceSpec } from '../../../skill-resolution/classify.js';
 import { findDeclaredSkillForPath, findDeclaredSkillForSourceDir, resolveSkillReference, } from '../../../skill-resolution/index.js';
-import { ConfigLoadError, loadConfig } from '../../../utils/config-loader.js';
+import { ConfigLoadError, loadConfig, loadConfigCached } from '../../../utils/config-loader.js';
 import { runClaudePluginBuild } from '../../claude/plugin/build.js';
 import { assertValidAuth, assertValidRequireAuth } from './auth-flags.js';
 /** Extract the trailing path segment (cross-platform) from a path-like string. */
@@ -164,34 +164,62 @@ export function descriptorsToRecord(list) {
 // ---------------------------------------------------------------------------
 // Precedence resolution: flag > config > default
 // ---------------------------------------------------------------------------
+/**
+ * Read a DECLARED skill's own `skills.config.<name>.test` block from the config that
+ * governs IT — keyed by the declared name against that skill's `configRoot`, never by
+ * cwd. The one lookup used wherever "what did THIS skill declare?" is the question:
+ * {@link loadTestConfig}'s path-target arm and companion `test.build` resolution
+ * ({@link resolveCompanionSpec}), so a companion in a nested config never inherits the
+ * subject's block.
+ */
+function declaredSkillTestConfig(link) {
+    return loadConfigCached(link.configRoot)?.skills?.config?.[link.name]?.test;
+}
 /**
  * Load the persisted `skills.config.<skill>.test` block for the subject skill,
- * PROJECT-AWARE (mirrors the resolver — not a parallel cwd+basename resolver).
+ * PROJECT-AWARE (mirrors the resolver's ladder ORDER — not a parallel cwd+basename
+ * resolver).
+ *
+ * A PATH target is mapped back to its declared skill through the SAME two lookups
+ * {@link resolveDefinitePath} (in resolve-skill-reference.ts) uses, tried in the SAME
+ * order: first {@link findDeclaredSkillForSourceDir} (a path AT a declared skill's
+ * SOURCE dir — rung 2a, `buildable`), then {@link findDeclaredSkillForPath} (a path AT
+ * its already-built DIST dir — the `source` + `declaredSkill` back-link rung). Either
+ * hit is keyed by the DECLARED name against ITS OWN governing config via
+ * {@link declaredSkillTestConfig} — cwd-independent, so a path target honors the same
+ * model/evals/timeout/`test.build` as its name.
  *
- * A PATH target (`./dist/skills/my-skill/`) is mapped back to its declared skill
- * via {@link findDeclaredSkillForPath} (walks up from the path, config-first) and
- * keyed by the DECLARED name against the GOVERNING config — so a path target honors
- * the same model/evals/timeout as its name, regardless of cwd. A NAME target (or a
- * path that maps to no declared skill) falls back to the cwd-anchored governing
- * config keyed by basename. Missing config / missing file → undefined (defaults apply).
+ * The basename fallback below is a LAST RESORT, reached only when `subject` carries
+ * NEITHER link — a bare name (matched by literal string), or a path outside every
+ * declared skill's source/dist dir. It must never be the path a LINKED subject's
+ * config resolves through: a directory basename that happens to differ from its
+ * declared skill's name would otherwise silently miss that skill's `test:` block
+ * (including a required `test.build` hook) with no error — the exact failure mode
+ * this ordering exists to prevent. Missing config / missing file → undefined
+ * (defaults apply).
  *
  * A broken config throws {@link ConfigLoadError}; we let it propagate to the preflight
  * guard in runSkillTestRun, which surfaces it as a clean exit-2 error rather than
  * silently applying defaults against a config the author clearly intended.
  */
 async function loadTestConfig(subject, cwd) {
-    // Path target → resolve the declared skill it materializes, and read its test
-    // block from that skill's own governing config root (cwd-independent).
+    // Rung 2a: a path AT a declared skill's SOURCE dir — tried FIRST, matching
+    // resolveDefinitePath's own precedence.
+    const sourceLink = await findDeclaredSkillForSourceDir(subject, cwd);
+    if (sourceLink !== undefined)
+        return declaredSkillTestConfig(sourceLink);
+    // Reverse rung: a path AT a declared skill's already-built DIST dir.
     const link = await findDeclaredSkillForPath(subject, cwd);
-    if (link !== undefined) {
-        return loadConfig(link.configRoot)?.skills?.config?.[link.name]?.test;
-    }
+    if (link !== undefined)
+        return declaredSkillTestConfig(link);
     const projectRoot = findProjectRoot(cwd);
     if (projectRoot === null)
         return undefined;
     const perSkill = loadConfig(projectRoot)?.skills?.config;
     if (perSkill === undefined)
         return undefined;
+    // Last-resort basename fallback (see doc comment above): only reached when neither
+    // lookup above found a declared-skill link for `subject`.
     const base = lastPathSegment(subject) || subject;
     return perSkill[subject]?.test ?? perSkill[base]?.test;
 }
@@ -250,8 +278,14 @@ function applyScalarMerges(opts, options, config) {
     const model = options.model ?? config?.model;
     if (model !== undefined)
         opts.model = model;
-    if (config?.evals !== undefined)
-        opts.evalsSubpath = config.evals;
+    // `--evals` is resolved HERE, against the process cwd, so the operator's own
+    // shell path means what they typed; `test.evals` is left alone and resolves
+    // against the skill source, because config travels with the skill. Both then
+    // reach the harness as one field, and both accept a path or an npm bare
+    // specifier (a suite published as a shared corpus).
+    const evalsRef = options.evals === undefined ? config?.evals : resolveAssetReference(options.evals, process.cwd());
+    if (evalsRef !== undefined)
+        opts.evalsSubpath = evalsRef;
 }
 /**
  * Resolve one cost/runtime knob with flag > config > default precedence AND a
@@ -399,8 +433,16 @@ function sourceScaffoldFields(source, declaredSkill, cwd) {
  * reference; for a declared skill, builds it (real entry points) and returns the
  * dist dir to stage; everything else is staged as-is. Throws SkillBuildError
  * (exit 2) for name-miss / not-found / --no-build-without-dist / build failure.
+ *
+ * `memo` is the per-run build memo (see {@link BuildMemo}) — pass the SAME map used
+ * for companion resolution so a skill that is both subject and companion builds once.
+ *
+ * `buildCommand` is the SUBJECT's own `test.build` hook, resolved by
+ * {@link loadTestConfig} (which carries a bare-name/basename fallback a raw config
+ * lookup lacks). It applies to the subject's build ONLY — companions resolve theirs
+ * from their own governing config.
  */
-export async function resolveSubjectForTest(ref, cwd, flags) {
+export async function resolveSubjectForTest(ref, cwd, flags, memo, buildCommand) {
     const resolved = await resolveSkillReference(ref, cwd);
     switch (resolved.kind) {
         case 'source':
@@ -418,7 +460,7 @@ export async function resolveSubjectForTest(ref, cwd, flags) {
             // A buildable ref carries `packagingConfig` — the ONE cleanly-reachable place
             // the subject's declared executables live — so attach the grader recognition
             // aid here (a plain source has no packaging config and leaves it undefined).
-            const subject = await resolveBuildableSubject(resolved, flags);
+            const subject = await resolveBuildableSubject(resolved, flags, memo, buildCommand);
             const declaredExecutables = deriveDeclaredExecutableNames(resolved.packagingConfig.executables);
             return declaredExecutables === undefined ? subject : { ...subject, declaredExecutables };
         }
@@ -449,6 +491,98 @@ function resolveExistingDistOrThrow(ref, flags) {
     }
     throw new SkillBuildError(`--no-build: no built dist at ${ref.expectedDistDir} for '${ref.name}'. Run \`vat build\` first, or point at a built path.`);
 }
+/**
+ * Identity of the BUILD OPERATION a {@link BuildableReference} triggers — the memo key.
+ *
+ * A `pool` skill is built by `packageSkill` into its own `expectedDistDir`, so that
+ * dir IS the operation's identity. A `plugin-local` skill is built by
+ * `runClaudePluginBuild`, which `rm -rf`s and rebuilds EVERY plugin in the whole
+ * marketplace — so the operation's identity is the (configRoot, marketplace) pair,
+ * NOT the skill. Keying plugin-local per skill would make a second companion in the
+ * same marketplace trigger a full marketplace wipe-and-rebuild that reproduces work
+ * the first call already did.
+ */
+export function buildMemoKey(ref) {
+    return ref.distribution.kind === 'pool'
+        ? `pool:${ref.expectedDistDir}`
+        : `plugin:${ref.configRoot}:${ref.distribution.marketplaceName}`;
+}
+/**
+ * Assert a completed build actually MATERIALIZED `ref.expectedDistDir` with a real
+ * skill in it, and return the result to stage. A build that reports success but
+ * writes nothing (or writes an empty shell with no `SKILL.md`) would otherwise hand
+ * back a path a required companion then fails opaquely at staging, or an OPTIONAL
+ * one silently skips — the exact non-functional-companion-with-no-diagnostic
+ * symptom of issue #158. Two distinct checks, two distinct messages, so the error
+ * tells the author which failure mode they hit:
+ *   1. `expectedDistDir` itself is missing — "no output at all".
+ *   2. `expectedDistDir` exists but has no `SKILL.md` — an empty/incomplete shell.
+ *
+ * Also runs on a memo HIT, because a plugin-local hit may have been recorded by a
+ * DIFFERENT skill in the same marketplace: the dist dir always comes from THIS ref
+ * and is always checked.
+ */
+function verifyBuiltDist(ref) {
+    // eslint-disable-next-line security/detect-non-literal-fs-filename -- expectedDistDir derived from project config
+    if (!existsSync(ref.expectedDistDir)) {
+        throw new SkillBuildError(`Skill build for '${ref.name}' reported success but produced no output at ${ref.expectedDistDir}. ` +
+            `Check the skill's packaging config (\`vat build\` should create this directory).`);
+    }
+    const skillMdPath = safePath.join(ref.expectedDistDir, 'SKILL.md');
+    // eslint-disable-next-line security/detect-non-literal-fs-filename -- skillMdPath derived from project config
+    if (!existsSync(skillMdPath)) {
+        throw new SkillBuildError(`Skill build for '${ref.name}' reported success but produced no SKILL.md at ${ref.expectedDistDir}. ` +
+            `Check the skill's packaging config (\`vat build\` should create this file).`);
+    }
+    return { distDir: ref.expectedDistDir, rebuilt: true };
+}
+/**
+ * Identity of a `test.build` PRE-STAGE HOOK invocation — the (cwd, command) pair, which
+ * is everything that distinguishes one shell invocation from another. Namespaced `hook:`
+ * so it can never collide with a {@link buildMemoKey} entry in the shared {@link BuildMemo}.
+ */
+function preStageHookMemoKey(configRoot, buildCommand) {
+    return `hook:${configRoot}:${buildCommand}`;
+}
+/**
+ * Run a skill's `test.build` pre-stage hook, at most ONCE per (configRoot, command)
+ * within a run — the "runs that shell command ONCE before staging" contract documented
+ * on {@link runPreStageBuild}, preserved now that a run may build several skills.
+ *
+ * The command ALWAYS runs with the config root of the skill that DECLARED it as cwd
+ * (both values come from the same ref), so a subject's hook can never execute against
+ * a companion's package root. Memoized only on SUCCESS: a hook that failed and was
+ * tolerated (optional companion) is retried rather than treated as done.
+ *
+ * KNOWN LIMITATION: for `plugin-local` skills, the memo key this call is gated behind
+ * ({@link buildMemoKey} in {@link buildDeclaredSkill}) is the shared `plugin:<configRoot>:
+ * <marketplace>` key, not a per-skill one — because one marketplace build serves every
+ * participant. When two declared skills share a marketplace, only the FIRST participant's
+ * `test.build` hook runs; the second skill's own hook is silently skipped, since by the
+ * time its build call is reached the memo already reports the marketplace as built and
+ * this function is never invoked for it. Fixing that requires gathering the hook set for
+ * every marketplace participant BEFORE the one marketplace build — out of scope here.
+ */
+function runPreStageBuildOnce(buildCommand, configRoot, memo) {
+    if (buildCommand === undefined)
+        return;
+    const key = preStageHookMemoKey(configRoot, buildCommand);
+    if (memo.has(key))
+        return;
+    runPreStageBuild({ buildCommand, configRoot });
+    memo.add(key);
+}
+/** Dispatch the real build for a declared skill, by how it ships. Throws the raw build error. */
+async function runDeclaredSkillBuild(ref) {
+    if (ref.distribution.kind === 'pool') {
+        await packageSkill(ref.sourcePath, packagingConfigToPackageOptions(ref.packagingConfig, {
+            skillPath: ref.sourcePath,
+            outputPath: ref.expectedDistDir,
+        }));
+        return;
+    }
+    await runClaudePluginBuild(ref.configRoot, { marketplace: ref.distribution.marketplaceName });
+}
 /**
  * Build a declared skill (BuildableReference) into its `expectedDistDir`, or reuse
  * the existing dist under --no-build/--dry-run. Shared by SUBJECT resolution
@@ -457,10 +591,33 @@ function resolveExistingDistOrThrow(ref, flags) {
  * skill gets the exact same build treatment as the subject (issue #158) — its
  * `files:` injection runs before staging, instead of a raw, possibly
  * build-artifact-incomplete, source-tree copy.
+ *
+ * `buildCommand` is the `test.build` pre-stage hook DECLARED BY THIS REF's skill —
+ * never a shared per-run flag. It runs with `ref.configRoot` as cwd, so command and
+ * cwd always come from the same skill's config (see {@link runPreStageBuildOnce}).
+ * GUARANTEED to run (once, on the first call for this ref) for a `pool`-distributed
+ * skill, whose memo key is per-skill. KNOWN LIMITATION for `plugin-local` skills: the
+ * memo key is the shared marketplace build, so when two declared skills share a
+ * marketplace, only the FIRST participant reaching this function gets its hook run —
+ * the second skill's build is served by the memo hit at the check below, and its own
+ * `test.build` hook is silently skipped (see {@link runPreStageBuildOnce}).
+ *
+ * `memo` collapses repeat work within ONE run (see {@link buildMemoKey}). It is
+ * consulted only AFTER the no-build/dry-run short-circuit and AFTER the security-ack
+ * gate: a cached build result must never be handed to a caller that has not passed
+ * the ack, and the no-build branch performs no build to memoize.
  */
-async function buildDeclaredSkill(ref, flags) {
-    // --no-build and --dry-run never build — delegate to the dedicated branch helper.
-    if (flags.noBuild || flags.dryRun) {
+async function buildDeclaredSkill(ref, flags, memo, buildCommand) {
+    // --dry-run DOES build once acknowledged: the "dry" part is the skill TESTING (no
+    // Claude session, no tokens), and a preview assembled from a stale dist previews
+    // something a real run would not test — which is the one job a preview has.
+    //
+    // Without the acknowledgement it still does NOT build. Building runs the repo's
+    // committed `test.build` hook, an arbitrary shell command, so an unacknowledged
+    // dry run stays the one mode that is safe to point at an untrusted clone. That
+    // preview falls back to an existing dist and says so (stale warning) — the honest
+    // trade, since accuracy requires consent to execute the repo's own build.
+    if (flags.noBuild || (flags.dryRun && !flags.explicitAck)) {
         return resolveExistingDistOrThrow(ref, flags);
     }
     // SECURITY (§12): reaching here means a real build WOULD run — runPreStageBuild
@@ -471,30 +628,36 @@ async function buildDeclaredSkill(ref, flags) {
     if (!flags.acknowledged) {
         throw new SecurityAckError();
     }
+    const key = buildMemoKey(ref);
+    // Memo HIT: this operation already ran in THIS process — skip the rebuild, but
+    // still verify this ref's own dist (the hit may be a marketplace sibling's).
+    if (memo.has(key))
+        return verifyBuiltDist(ref);
     // test.build hook (upstream artifacts) BEFORE the skill build -- ordering matters.
-    if (flags.build !== undefined) {
-        runPreStageBuild({ buildCommand: flags.build, configRoot: ref.configRoot });
-    }
+    runPreStageBuildOnce(buildCommand, ref.configRoot, memo);
+    // ANY error thrown here is wrapped as `SkillBuildError` below, not just a narrowly-
+    // scoped build-tool failure — see {@link isSurvivableCompanionFailure} for why that
+    // matters to companion degradation. `err.stack` is also captured at construction,
+    // so it keeps reflecting THIS un-prefixed message even after a caller mutates
+    // `.message` in place (see {@link rethrowNamingCompanion}).
     try {
-        if (ref.distribution.kind === 'pool') {
-            await packageSkill(ref.sourcePath, packagingConfigToPackageOptions(ref.packagingConfig, {
-                skillPath: ref.sourcePath,
-                outputPath: ref.expectedDistDir,
-            }));
-        }
-        else {
-            await runClaudePluginBuild(ref.configRoot, { marketplace: ref.distribution.marketplaceName });
-        }
+        await runDeclaredSkillBuild(ref);
     }
     catch (e) {
         throw new SkillBuildError(`Skill build failed for '${ref.name}': ${e instanceof Error ? e.message : String(e)}`);
     }
-    return { distDir: ref.expectedDistDir, rebuilt: true };
+    // Record the operation as soon as the build COMMAND succeeded — before verifying
+    // this ref's output. The build did run; a marketplace sibling must not re-run it
+    // just because this skill's own dist turned out to be missing. Verification is
+    // deliberately OUTSIDE the try/catch so its SkillBuildError is not re-wrapped into
+    // a doubled "Skill build failed for 'x': Skill build for 'x' reported success…".
+    memo.add(key);
+    return verifyBuiltDist(ref);
 }
 /** Build a declared skill (or stage its existing dist under --no-build/--dry-run), then return the dist to stage. */
-async function resolveBuildableSubject(ref, flags) {
+async function resolveBuildableSubject(ref, flags, memo, buildCommand) {
     const scaffoldDir = dirname(ref.sourcePath);
-    const build = await buildDeclaredSkill(ref, flags);
+    const build = await buildDeclaredSkill(ref, flags, memo, buildCommand);
     return {
         subjectSource: { path: build.distDir },
         subjectScaffoldDir: scaffoldDir,
@@ -505,6 +668,69 @@ async function resolveBuildableSubject(ref, flags) {
             : { dryRunStagedExistingDist: build.dryRunStagedExistingDist }),
     };
 }
+/**
+ * May an OPTIONAL companion's failure be DEGRADED to "stage its raw source instead"
+ * rather than failing the run? True for a genuine build failure ({@link SkillBuildError})
+ * that could not have been DESTRUCTIVE. Both narrowings matter:
+ *
+ *   - **Error class.** Everything else reaching the catch is a user-fixable PREFLIGHT
+ *     problem, not a degraded companion: a {@link SecurityAckError} (the fix is
+ *     `--i-understand-this-runs-skill-code`, not "your companion didn't build"), a
+ *     `ConfigLoadError` (a broken config reported as a build failure), or a
+ *     `BuildHookError` (a genuinely misconfigured `test.build`). Swallowing those
+ *     replaces the actionable message with a misleading one. Note that
+ *     {@link buildDeclaredSkill} wraps ANY error `runDeclaredSkillBuild` throws —
+ *     including a malformed packaging config or a stray `TypeError` deep inside
+ *     `packageSkill` — into `SkillBuildError`, so "genuine build failure" here really
+ *     means "anything the build call raised", not just a narrowly-scoped build error.
+ *   - **Destructiveness.** A `plugin-local` build is DESTRUCTIVE once it reaches
+ *     `runClaudePluginBuild`: that function `rm -rf`s the whole marketplace output
+ *     tree BEFORE building, so a failure THERE may leave the user's
+ *     `dist/.claude/plugins/marketplaces/<mp>/` already gone. Continuing would stage
+ *     from an inconsistent dist and finish SUCCESS with the build output deleted — a
+ *     test command must never destroy build output and report only a "Note:". A
+ *     `pool` build writes only its own `expectedDistDir` and destroys nothing, so
+ *     falling back to the raw source tree is always safe there. And under
+ *     `--no-build`/`--dry-run`, `resolveExistingDistOrThrow` throws BEFORE
+ *     `buildDeclaredSkill` reaches any build path at all — no build was attempted,
+ *     plugin-local or not, so nothing was wiped either way and degrading is safe.
+ *
+ * A real (non `--no-build`/`--dry-run`) `plugin-local` failure stays fail-closed
+ * because it MAY have failed mid-wipe; `runClaudePluginBuild` can also throw BEFORE
+ * the `rm` (broken config, colliding plugin names), but those are indistinguishable
+ * from outside once wrapped in `SkillBuildError`, so fail-closed is deliberate there.
+ */
+function isSurvivableCompanionFailure(err, declared, optional, flags) {
+    if (!optional || !(err instanceof SkillBuildError))
+        return false;
+    return declared.distribution.kind === 'pool' || flags.noBuild;
+}
+/**
+ * Rethrow a companion failure with the `--with`/`--with-optional` ALIAS the user
+ * TYPED prefixed onto its message. Build errors name only the DECLARED skill, which
+ * with several companions leaves the user unable to tell which flag to fix.
+ *
+ * The message is prefixed IN PLACE so the error's CLASS (and therefore its
+ * `exitCode` / {@link mapErrorToExitCode} mapping) is unchanged — no new error class,
+ * no re-wrapping. Prefixing is skipped when the message is ALREADY prefixed (starts
+ * with `companion '`) — this makes the function idempotent against repeat rethrows of
+ * the SAME error object, structurally, rather than relying on a hand-maintained
+ * argument that no caught error class is ever cached/reused across companions.
+ * {@link ConfigLoadError} instances ARE cached and re-thrown per config root by
+ * `loadConfigCached`, so mutating one in place would otherwise corrupt the cache; the
+ * already-prefixed check covers this case too (in practice `ConfigLoadError` is
+ * thrown by the lookup ABOVE this try and never reaches here — belt-and-braces).
+ *
+ * NOTE: `err.stack` was captured at construction time and therefore still reflects
+ * the UN-PREFIXED message — mutating `.message` afterward does not update `.stack`.
+ * Debug output that prints `.stack` can disagree with the prefixed text on stderr.
+ */
+function rethrowNamingCompanion(err, alias, declaredName) {
+    if (err instanceof Error && !err.message.startsWith("companion '")) {
+        err.message = `companion '${alias}' (declared skill '${declaredName}'): ${err.message}`;
+    }
+    throw err;
+}
 /**
  * Companion analog of {@link resolveBuildableSubject} (issue #158): resolve every
  * `--with`/`--with-optional` companion whose spec is a `{ path }` pointing at a
@@ -513,37 +739,53 @@ async function resolveBuildableSubject(ref, flags) {
  * url:/vendored specs, or a path outside this project's config — is "a different
  * story" (per the issue's own framing): left untouched, staged as today.
  *
- * A REQUIRED companion's build failure propagates (matches the existing "--with
- * fails if a source cannot be resolved" contract, extended to "cannot be built").
- * An OPTIONAL companion's build failure falls back to the ORIGINAL (unbuilt) path
- * spec — this fix must never make an already-degraded optional companion WORSE;
- * staging's existing skip-with-warning fallback still applies if that raw copy
- * also fails to stage.
+ * The `test.build` hook used is the COMPANION's own — read from the config that
+ * governs IT ({@link declaredSkillTestConfig}), not the subject's. The subject's hook
+ * is a different skill's command in a (possibly) different package, and running it
+ * here would execute it with the companion's config root as cwd.
+ *
+ * FAILURE HANDLING. A REQUIRED companion's failure ALWAYS propagates (the existing
+ * "--with fails if a source cannot be resolved" contract, extended to "cannot be
+ * built"), re-messaged to name the alias the user typed. An OPTIONAL companion
+ * degrades to the ORIGINAL (unbuilt) path spec — with a mandatory stderr note — only
+ * for the narrow, non-destructive case {@link isSurvivableCompanionFailure} allows;
+ * every other failure propagates for it too. Staging's existing skip-with-warning
+ * fallback still applies if the raw copy also fails to stage.
+ *
+ * DRY-RUN STALENESS: a `--dry-run` preview that staged an EXISTING
+ * companion dist WITHOUT rebuilding it warns on stderr, naming the companion +
+ * its declared skill, using {@link buildStaleDistWarningLines} — the SAME
+ * warning-construction the subject's own dry-run summary uses (never a copied
+ * string), so the identical fact ("this preview may be stale") warns for either
+ * role instead of only the subject.
  */
-export async function resolveCompanionSpec(name, spec, repoRoot, flags, optional) {
+export async function resolveCompanionSpec(name, spec, repoRoot, flags, optional, memo) {
     if (!('path' in spec))
         return spec;
     const declared = await findDeclaredSkillForSourceDir(spec.path, repoRoot);
     if (declared === undefined)
         return spec;
     try {
-        const build = await buildDeclaredSkill(declared, flags);
+        const build = await buildDeclaredSkill(declared, flags, memo, declaredSkillTestConfig(declared)?.build);
+        if (build.dryRunStagedExistingDist === true) {
+            process.stderr.write(buildStaleDistWarningLines(`companion '${name}' (declared skill '${declared.name}')`).join('\n') + '\n');
+        }
         return { path: build.distDir };
     }
     catch (e) {
-        if (!optional)
-            throw e;
+        if (!isSurvivableCompanionFailure(e, declared, optional, flags))
+            rethrowNamingCompanion(e, name, declared.name);
         process.stderr.write(`Note: companion '${name}' (declared skill '${declared.name}') failed to build; staging its raw source instead: ${e instanceof Error ? e.message : String(e)}\n`);
         return spec;
     }
 }
 /** Resolve every entry of a `--with`/`--with-optional` companion record (see {@link resolveCompanionSpec}). */
-export async function resolveCompanionSources(sources, repoRoot, flags, optional) {
+export async function resolveCompanionSources(sources, repoRoot, flags, optional, memo) {
     if (sources === undefined)
         return undefined;
     const resolved = {};
     for (const [name, spec] of Object.entries(sources)) {
-        resolved[name] = await resolveCompanionSpec(name, spec, repoRoot, flags, optional);
+        resolved[name] = await resolveCompanionSpec(name, spec, repoRoot, flags, optional, memo);
     }
     return resolved;
 }
@@ -595,6 +837,19 @@ async function preflightKnobsAndConfig(subject, options, cwd) {
         process.exit(SkillTestExitCode.Preflight);
     }
 }
+/**
+ * Exit code for an error escaping EITHER phase of {@link runSkillTestRun}.
+ *
+ * A broken governing config ({@link ConfigLoadError}) is a user-fixable PREFLIGHT
+ * problem (exit 2), not an internal harness failure — {@link mapErrorToExitCode} has
+ * no case for it and falls through to Internal (1). ONE helper shared by BOTH catch
+ * blocks so the SUBJECT arm and the COMPANION arm can never drift: a companion's
+ * governing config root can differ from the subject's, so a broken config is
+ * reachable from either arm and must report the same code from both.
+ */
+function exitCodeForRunError(err) {
+    return err instanceof ConfigLoadError ? SkillTestExitCode.Preflight : mapErrorToExitCode(err);
+}
 /**
  * Copy the resolved-subject fields onto the harness options. Optional fields are
  * only assigned when present (exactOptionalPropertyTypes). Extracted so
@@ -636,22 +891,28 @@ export async function runSkillTestRun(subject, options) {
         acknowledgedRunsSkillCode: options.iUnderstandThisRunsSkillCode === true,
     });
     // Shared by subject AND companion resolution (issue #158) so a --with/--with-optional
-    // companion that maps to a declared skill gets the exact same build gating (no-build/
-    // dry-run/security-ack) as the subject.
+    // companion that maps to a declared skill gets the exact same build GATING (no-build/
+    // dry-run/security-ack) as the subject. The `test.build` hook is deliberately NOT in
+    // here: it is per-skill (see BuildFlags), so the subject's own command is passed only
+    // to subject resolution and each companion resolves its own.
     const buildFlags = {
         noBuild,
         dryRun: options.dryRun === true,
         acknowledged,
-        ...(config?.build === undefined ? {} : { build: config.build }),
+        explicitAck: options.iUnderstandThisRunsSkillCode === true,
     };
+    // ONE memo for the whole run, shared by subject AND companion resolution: a skill
+    // that is both the subject and a companion builds exactly once, and N companions
+    // in one marketplace trigger ONE marketplace build, not N.
+    const buildMemo = new Set();
     let resolvedSubject;
     try {
-        resolvedSubject = await resolveSubjectForTest(subject, process.cwd(), buildFlags);
+        resolvedSubject = await resolveSubjectForTest(subject, process.cwd(), buildFlags, buildMemo, config?.build);
     }
     catch (err) {
         // A broken governing config surfacing during subject resolution is a preflight
         // problem the user must fix (exit 2), not an internal harness failure (exit 1).
-        const exitCode = err instanceof ConfigLoadError ? SkillTestExitCode.Preflight : mapErrorToExitCode(err);
+        const exitCode = exitCodeForRunError(err);
         process.stderr.write(`Error: ${err instanceof Error ? err.message : String(err)}\n`);
         process.exit(exitCode);
         return;
@@ -667,10 +928,10 @@ export async function runSkillTestRun(subject, options) {
         // whose source is a path into a declared skill gets built (its `files:`
         // injection runs) exactly like the subject, instead of a raw source-tree copy.
         const repoRoot = resolveRepoRoot();
-        const withSources = await resolveCompanionSources(harnessOpts.withSources, repoRoot, buildFlags, false);
+        const withSources = await resolveCompanionSources(harnessOpts.withSources, repoRoot, buildFlags, false, buildMemo);
         if (withSources !== undefined)
             harnessOpts.withSources = withSources;
-        const withOptional = await resolveCompanionSources(harnessOpts.withOptional, repoRoot, buildFlags, true);
+        const withOptional = await resolveCompanionSources(harnessOpts.withOptional, repoRoot, buildFlags, true, buildMemo);
         if (withOptional !== undefined)
             harnessOpts.withOptional = withOptional;
         const result = await runSkillTestHarness(harnessOpts);
@@ -680,7 +941,10 @@ export async function runSkillTestRun(subject, options) {
         return;
     }
     catch (err) {
-        const exitCode = mapErrorToExitCode(err);
+        // Same rule as the subject arm above (see {@link exitCodeForRunError}): companion
+        // resolution can surface a ConfigLoadError from a DIFFERENT config root than the
+        // subject's, and that must exit 2 (preflight), not 1 (internal).
+        const exitCode = exitCodeForRunError(err);
         // BootstrapNeededError (exit 3) is the happy "wrote a template, fill it in
         // and re-run" path — surface its message plainly, not as a hard `Error:`.
         if (err instanceof BootstrapNeededError) {
@@ -701,16 +965,16 @@ export function createSkillTestRunCommand() {
     command
         .description('Execute a packaged skill\'s eval suite in a headless, context-isolated Claude session (runs the skill\'s code; not an OS sandbox)')
         .argument('<skill>', 'The skill to test')
-        .option('--with <pair...>', 'Stage a REQUIRED companion skill the subject can invoke, as name=<src> (repeatable). <src> is workspace:<pkg> | npm:<spec> | url:<u> | path:<dir> | vendored (e.g. helper=npm:@scope/s@1.2.3). The run fails if a source cannot be resolved.')
-        .option('--with-optional <pair...>', 'Stage an OPTIONAL companion skill, as name=<src> (same syntax, repeatable). Skipped with a warning if its source cannot be resolved; the run continues.')
-        .option('--env <pair...>', 'Inject an env var into the executor spawn as KEY=VALUE (repeatable). Values support ${fixturesDir}, ${stagedSkillDir}, ${harnessRoot}, ${resultsDir}. CLI overrides config for the same key.')
+        .option('--with <pair...>', 'Stage a REQUIRED companion skill the subject can invoke, as name=<src> (repeatable). <src> is workspace:<pkg> | npm:<spec> | url:<u> | path:<dir> | vendored (e.g. helper=npm:@scope/s@1.2.3). A path:<dir> that maps to a declared skill is BUILT like the subject (files: build artifacts included), not tree-copied as raw source. The run fails if a source cannot be resolved or its build fails.')
+        .option('--with-optional <pair...>', 'Stage an OPTIONAL companion skill, as name=<src> (same syntax, repeatable). Staged from its raw (unbuilt) source with a warning if its source cannot be resolved, or if a non-destructive build fails; the run continues.')
+        .option('--env <pair...>', 'Inject an env var into the executor spawn as KEY=VALUE (repeatable). Values support ${fixturesDir}, ${stagedSkillDir}, ${harnessRoot}, ${resultsDir}. ${fixturesDir} is per-eval and requires that eval to declare input `files`. CLI overrides config for the same key.')
         .option('--pass-env <key...>', 'Forward a host env var by NAME to the executor spawn if present (repeatable). Protected names (PATH, auth, model) are ignored.')
         .option('--refresh', 'Force a full re-stage (ignore existing staged content)')
-        .option('--no-build', 'Skip building a declared skill; stage its existing dist instead (errors if absent)')
+        .option('--no-build', 'Skip building declared skills (subject and any --with/--with-optional companion); stage existing dist instead. Errors if absent for the subject or a REQUIRED companion; an OPTIONAL companion falls back to raw source with a warning.')
         .option('--workdir <dir>', 'Override the harness working directory')
         .option('--out <dir>', 'Override the harness output directory')
         .option('--keep', 'Keep the harness directory after the run')
-        .option('--dry-run', 'Assemble the command without spawning Claude')
+        .option('--dry-run', 'Build and stage exactly as a real run would, then stop without spawning Claude (no tokens spent). Combine with --no-build to skip the build too.')
         .option('--auth <mode>', 'Auth mechanism: inherit | subscription | api-key | auto')
         .option('--require-auth <mech>', 'Require a specific auth mechanism: subscription | api-key')
         .option('--baseline', 'Enable A/B baseline run (with/without skill)')
@@ -719,6 +983,7 @@ export function createSkillTestRunCommand() {
         .option('--i-understand-this-runs-skill-code', 'Acknowledge this command executes skill code (required)')
         .option('--model <id>', "Model ID passed VERBATIM to `claude --model <id>` for the model UNDER TEST (the executor). VAT does no mapping/validation (e.g. opus, sonnet, claude-opus-4-8). Omit to use claude's own default model. Independent of --grader-model.")
         .option('--grader-model <id>', 'Model for the fixed grader/judge, passed VERBATIM to `claude --model` (default claude-sonnet-5). GLOBAL (top-level `test:` config), independent of --model (the model UNDER TEST).')
+        .option('--evals <path>', "Eval suite to grade against: a path to an evals.json (resolved against the CURRENT DIRECTORY, so it may point outside the skill's tree) or an npm bare specifier honoring that package's exports map. Overrides `test.evals`, which is resolved against the skill source instead. Use this to test a skill you did not author — a correctly packaged skill ships no evals, because the suite is the answer key.")
         .option('--max-turns <n>', 'Per-spawn cap on executor/grader turns (positive integer)')
         .option('--max-budget-usd <n>', 'Hard USD budget cap (positive number)')
         .option('--timeout <s>', 'Wall-clock timeout in seconds (positive integer)')
@@ -761,7 +1026,7 @@ Model:
 Exit Codes:
   0 - Harness ran to completion and every eval passed (or --allow-eval-failure suppressed a failing verdict)
   1 - Internal error (grader fragment absent/invalid, summary/expectations skew, executor/grader crash, stall/timeout)
-  2 - Preflight failed (missing binary, auth error, eval inputs absent, unsafe workdir, ack missing, skill build failed, or --no-build with no existing dist)
+  2 - Preflight failed (missing binary, auth error, eval inputs absent, unsafe workdir, ack missing, broken project config, a required skill -- subject or --with companion -- failed to build, --no-build with no existing dist for one of them, or an OPTIONAL --with-optional companion hitting a non-survivable failure: a destructive plugin-local build failure, missing security ack, or broken config)
   3 - Bootstrap needed: evals.json was absent, so VAT wrote a starter template next to the skill source. Fill it in and re-run.
   4 - An eval FAILED (the harness completed and produced a valid grading.json; expectations did not all pass). This is the fail-closed DEFAULT -- suppress with --allow-eval-failure.