@vibe-agent-toolkit/agent-skills 0.2.0-rc.2 → 0.2.0-rc.3

package/dist/skill-test/run-harness.d.ts

@@ -11,13 +11,16 @@
  *   → release lock → return result
  */
 import type { SkillSourceDescriptor } from '@vibe-agent-toolkit/resources';
-import { type spawnHeadlessClaude } from '@vibe-agent-toolkit/utils';
+import { type spawnHeadlessClaude, type ResolvedAuth } from '@vibe-agent-toolkit/utils';
 import type { ResolveSkillSourceContext, SkillSource } from '../skill-source/types.js';
+import { type BaselineDelta } from './baseline-delta.js';
+import { type ArmEvalGrade, type BaselineControlArmFailure, type BaselineIntegrity, type DetectBaselineContaminationInput } from './baseline-integrity.js';
 import type { EvalFragment } from './eval-fragment.js';
-import { type EvalEntry, type EvalSuite } from './eval-inputs.js';
+import { type ArmWorkspaceDirs, type EvalArm, type EvalEntry, type EvalSuite } from './eval-inputs.js';
 import { type SkillTestExitCodeValue } from './exit-codes.js';
 import { type FrictionItem } from './friction-schema.js';
 import { type GradingVerdict } from './grading-adapter.js';
+import { RateLimitSignal } from './pipeline.js';
 import { type PreflightInput } from './preflight.js';
 import { type StageItem } from './staging.js';
 import { type SkippedEvalsSummary } from './tier-plan.js';
@@ -86,6 +89,15 @@ export interface RunHarnessOptions {
      * install. Production callers leave it undefined (the real spawn is used).
      */
     spawn?: typeof spawnHeadlessClaude;
+    /**
+     * Injectable rate-limit backoff (tests only) — the pipeline's `onRateLimit`,
+     * given the 1-based retry attempt and returning milliseconds. Returning `0`
+     * collapses the retry budget to nothing, which is what makes the EXHAUSTED
+     * control-arm path testable at all: the real backoff is exponential and spends
+     * 31 seconds per item before the budget runs out. Production callers leave it
+     * undefined (the real exponential backoff is used).
+     */
+    rateLimitBackoffMs?: (attempt: number) => number;
     /** Enable A/B baseline run (with/without skill). */
     baseline?: boolean;
     /** Allow unverified skill source (skip manifest check). */
@@ -144,11 +156,18 @@ export interface RunHarnessOptions {
      * does not — issue #145 Phase T). Passed to the grader on the WITH arm ONLY as
      * a recognition aid alongside each eval's `toolExpectations`. Absent → the
      * grader still matches tools by the commands it sees in the transcript.
+     *
+     * `path` serves a SECOND consumer with a different need: `detectBaselineContamination`
+     * matches an escaping reach against the skill-relative PATH, because an ambient copy
+     * of the skill reproduces `…/scripts/csvsum.py` under whatever root it sits at, while
+     * the control arm's own `/tmp/summary.txt` does not. Matching the extension-stripped
+     * `name` instead convicted clean control arms of reading their own scratch files.
      */
     declaredExecutables?: Array<{
         name: string;
         howInvoked: string;
         kind: string;
+        path: string;
     }>;
     /**
      * Opt-OUT of eval gating (for interactive use). By DEFAULT (false/absent) a
@@ -163,6 +182,27 @@ export interface RunHarnessResult {
     harnessPath: string;
     exitCode: number;
     summary: string;
+    /**
+     * Where the executor's per-eval working directories were materialized. Lives
+     * OUTSIDE `harnessPath` on purpose (see {@link resolveWorkspacesRoot}), so it
+     * cannot be derived from the harness path and has to be reported.
+     *
+     * Present ONLY when those directories SURVIVE the run — i.e. under `--keep`,
+     * the sole thing that retains them (NOT `--out`/`--workdir`, unlike the harness
+     * root). Cleanup runs in the harness's own `finally`, before this result reaches
+     * any caller, so a populated value is a promise that the path is still there:
+     * anything that reports it (the CLI's `Workspaces:` line) may print it as-is.
+     * Also absent when the run ended before workspaces were staged.
+     */
+    workspacesPath?: string;
+    /**
+     * Where this run's artifacts (`grading.json`, `friction.json`, `tool-eval.json`,
+     * and `baseline.json` on a `--baseline` run) were written. Reported rather than
+     * left to be derived from `harnessPath`, because it is the ONLY part of the
+     * harness root that survives cleanup on a default run — see
+     * {@link cleanupHarness}. Absent when the run ended before Step 7 created it.
+     */
+    resultsPath?: string;
 }
 /**
  * Built-in cost/runtime safety ceilings applied PER executor/grader spawn. These
@@ -231,10 +271,61 @@ export declare function makeStageItem(name: string, source: SkillSource, repoRoo
 export declare function buildStageItems(opts: RunHarnessOptions, repoRoot: string): StageItem[];
 export declare function buildResolveCtx(harnessRoot: string, repoRoot: string): ResolveSkillSourceContext;
 /**
- * Return a sensible dummy value for a given CLI flag so that value-validation
- * doesn't reject the argument before `--help` can short-circuit the session.
+ * Match a flag name as a whole token in `claude --help` output.
+ *
+ * The boundary matters: a bare `includes('--plugin-dir')` also matches
+ * `--plugin-dirs` or `--plugin-dir-cache`, and vat's own flag names are prefixes
+ * of each other (`--max-turns` / `--max-turns-per-eval` would collide). Neither
+ * side of the token may be a letter or a dash.
+ */
+export declare function helpTextDeclaresFlag(helpText: string, flag: string): boolean;
+/**
+ * A flag name no `claude` will ever accept. It is probed alongside the real ones
+ * as a NEGATIVE CONTROL: if the probe reports this as supported, the probe is
+ * broken and its answer about every other flag is worthless.
+ */
+export declare const FLAG_PROBE_SENTINEL = "--vat-probe-flag-that-cannot-exist";
+/**
+ * Build a token-free flag-support probe from `claude --help`.
+ *
+ * ⚠️ This REPLACES an exit-code probe (`claude <flag> <dummy> --help`, exit 0 ⇒
+ * supported) that could not discriminate at all. Verified against claude 2.x:
+ * `claude --no-such-flag-xyz 1 --help` exits **0**, because `--help` short-circuits
+ * before argument validation. Every one of preflight's `flag <name>` checks
+ * therefore reported "supported" unconditionally — including for flags that do not
+ * exist — so the gate that is supposed to stop vat spawning with an unsupported
+ * flag was decorative. (The same command without `--help` exits 1, but running it
+ * would start a real session and bill tokens, which is why `--help` was there.)
+ *
+ * Help-text matching does discriminate, and it is still token-free: one `--help`
+ * invocation for the whole run instead of one spawn per flag.
+ *
+ * The probe carries its own negative control ({@link FLAG_PROBE_SENTINEL}). If a
+ * future `claude` prints something that matches the sentinel — or `--help` fails
+ * outright — the probe reports EVERY flag unsupported rather than every flag
+ * supported. A probe that cannot tell must fail closed and be seen; the defect it
+ * replaces failed open and was invisible.
+ */
+export declare function buildFlagParseProbe(runHelp?: () => string | null): (flag: string) => boolean;
+/**
+ * ⚠️ KNOWN OPEN — `costEstimate` here is DEAD, in three independent ways, and none
+ * of them is fixable from inside this function:
+ *
+ * 1. `evalCount: 1` is a placeholder. The suite is not parsed until Step 5.5, which
+ *    runs AFTER preflight, so the real count does not exist yet. Making it real means
+ *    either moving preflight after the suite parse (it currently gates that parse) or
+ *    adding a second reader of `evals.json` — and two readers of the answer key is
+ *    precisely the drift this module refuses elsewhere.
+ * 2. `renderPreflightSummary` returns only FAILING checks, and it is called only
+ *    inside the `if (!preflightResult.passed)` branch — so on a passing run the
+ *    estimate never reaches a terminal at all.
+ * 3. Preflight runs after the `--dry-run` short-circuit, so a dry run never gets here.
+ *
+ * The consequence is that {@link buildDryRunSummary}'s spawn-count line is the ONLY
+ * pre-spend number an operator ever sees. Fix the count before making the check
+ * reachable: a reachable estimate that says "1 evals × 2 config" is worse than a
+ * silent one, because it is quotable.
  */
-export declare function flagDummyValueFor(flag: string): string;
 export declare function buildPreflightInput(evalsPath: string, pluginDirs: string[], opts: RunHarnessOptions, knobs: {
     maxBudgetUsd: number;
 }): PreflightInput;
@@ -255,6 +346,13 @@ export declare function isAcknowledged(opts: Pick<RunHarnessOptions, 'dryRun' |
  * Format a friction report for human consumption — one line per entry as
  * `[<severity>] <category>: <message>`. Pure; returns the empty string for no
  * entries so the caller can skip emitting anything.
+ *
+ * `severity` and `category` are closed enums vat owns, so they are safe to
+ * interpolate. `message` is GRADER text and is sanitized here even though
+ * {@link parseEvalFragment} already sanitized it on the way in — this is a
+ * second, independent boundary, not a redundant one: {@link emitFrictionReport}
+ * re-reads `friction.json` from the harness results dir, which same-uid skill
+ * code can reach and rewrite after vat wrote it.
  */
 export declare function formatFrictionReport(items: readonly FrictionItem[]): string;
 /**
@@ -322,17 +420,65 @@ export declare function resolveEvalSuitePath(input: {
     holdDir: string;
     subjectEvalSuiteHeld: boolean;
 }): string | undefined;
+/**
+ * The per-run executor workspace root: `<tmp>/vat-skill-test-ws-<dirToken>/`.
+ *
+ * Deliberately OUTSIDE `harnessRoot`, alongside {@link resolveGraderOutDir} and
+ * {@link resolveEvalSuiteHoldDir}. The executor's cwd lives under here, and the
+ * harness root is where vat stages runnable copies of the skill (`staged/` and
+ * the assembled plugin dir). While the two were parent and child, the skill-absent
+ * arm of a `--baseline` run sat one `ls ..` away from the treatment it was
+ * supposed to be denied — the control could reach vat's own copy without ever
+ * leaving its working directory.
+ *
+ * Named by an unpredictable token, like the other vat-only dirs, and removed by
+ * the same cleanup. Pure (derives a path only).
+ */
+export declare function resolveWorkspacesRoot(dirToken: string): string;
 /** Parse the staged eval suite and materialize each eval's input `files` into
- * `<harnessRoot>/workspaces/<id>/`. Returns the workspaces root, the parsed
- * {@link EvalSuite} (so the eval loop has the entries without re-reading), and the
- * declared eval count (derived from the suite). The dir is wiped first so a reused
- * harness root cannot leak a prior run's inputs. Throws {@link EvalInputError}
- * (mapped by the caller to exit 2) on a bad suite or a missing input file. */
-export declare function stageWorkspacesForRun(evalsPath: string, harnessRoot: string): {
+ * `<workspacesRoot>/<arm>/<id>/`, once per arm this run will execute. Returns the
+ * workspaces root, the parsed {@link EvalSuite} (so the eval loop has the entries
+ * without re-reading), and the declared eval count (derived from the suite). The
+ * dir is wiped first so a reused root cannot leak a prior run's inputs. Throws
+ * {@link EvalInputError} (mapped by the caller to exit 2) on a bad suite or a
+ * missing input file. */
+/**
+ * Mint this run's opaque per-arm workspace directory segments.
+ *
+ * Independent random tokens, NOT the arm names and not derived from one another:
+ * the segment ends up in the executor's cwd, which the prompt states verbatim, so
+ * a name like `without` unblinds the control arm outright — and a name like `with`
+ * hands it a guessable path to the treatment arm's live output one directory over.
+ * 8 bytes is ample for "not guessable within a run that lasts minutes"; these are
+ * blinding/anti-enumeration tokens, not secrets protecting data at rest.
+ */
+export declare function mintArmWorkspaceDirs(baseline: boolean): ArmWorkspaceDirs;
+export declare function stageWorkspacesForRun(evalsPath: string, workspacesRoot: string, armDirs: ArmWorkspaceDirs): {
     workspacesRoot: string;
     declaredEvalCount: number;
     suite: EvalSuite;
 };
+interface ResolveDeclaredChildEnvInput {
+    opts: RunHarnessOptions;
+    resolvedAuth: ResolvedAuth;
+    subjectStagedDir: string;
+    harnessRoot: string;
+    resultsDir: string;
+    evalsSubpath: string;
+    subjectPluginRoot: string | null;
+    /** The eval's staged input workspace — what `${fixturesDir}` resolves under. */
+    workspaceDir?: string;
+    /** Suppress the stderr transparency line (emitted once, by the first eval). */
+    quiet?: boolean;
+}
+/** Everything needed to re-assemble the executor env once an eval's workspace is known. */
+interface PerEvalEnvAssembly {
+    input: Omit<ResolveDeclaredChildEnvInput, 'workspaceDir' | 'quiet'>;
+    /** Mutable latch so exactly one eval emits the stderr transparency line. */
+    logged: {
+        done: boolean;
+    };
+}
 /** Inputs for the dry-run summary string. */
 export interface DryRunSummaryInput {
     /** True when the resolved subject is buildable (a real run would build + stage it). */
@@ -342,7 +488,14 @@ export interface DryRunSummaryInput {
      * rebuilding (may be stale); false = no dist existed, fell back to source dir.
      */
     dryRunStagedExistingDist?: boolean;
-    /** Absolute path to the written provenance.json (already on disk). */
+    /**
+     * Absolute path where a REAL run would write provenance.json.
+     *
+     * Not written by a dry run, and the summary line says so. It used to be written —
+     * from the same block that wiped the previous run's grading/friction/baseline
+     * artifacts — which made "preview the cost" a destructive operation against the
+     * results the operator was about to read.
+     */
     provenancePath: string;
     /** Content fingerprint from the staged manifest. */
     provenanceFingerprint: string;
@@ -352,10 +505,29 @@ export interface DryRunSummaryInput {
     modelFlag: string;
     /** Number of declared evals a real run would execute. */
     evalCount: number;
+    /**
+     * Whether `--baseline` is set, i.e. whether every eval runs TWICE.
+     *
+     * Without this the preview reported the SUITE SIZE and called it the spawn count,
+     * which under `--baseline` under-reports the run by exactly 2x: a 3-eval suite
+     * previewed as "3 evals as executor→grader spawn pairs" and then launched 6 pairs
+     * — twelve `claude` sessions. This is the ONLY pre-spend number an operator ever
+     * sees (see {@link buildPreflightInput} for why the other one never reaches a
+     * terminal), so a 2x understatement here is the whole cost estimate being wrong.
+     */
+    baseline: boolean;
     /** Bounded-parallel executor→grader pipeline width a real run would use. */
     concurrency: number;
     /** Pinned grader/judge model a real run would grade with. */
     graderModel: string;
+    /**
+     * The PER-SPAWN budget ceiling (`--max-budget-usd`), not a run total. Reported
+     * because the two facts only mean something together: the worst case is
+     * `pairs × 2 × maxBudgetUsd`, and an operator who reads the flag name as a run
+     * budget under-provisions by the session count — which under `--baseline` is
+     * four times the eval count.
+     */
+    maxBudgetUsd: number;
 }
 /**
  * Build the "stale dist" warning lines for a `--dry-run` preview that staged an
@@ -372,19 +544,6 @@ export interface DryRunSummaryInput {
  * than copying the string.
  */
 export declare function buildStaleDistWarningLines(roleLabel?: string): string[];
-/**
- * Build the dry-run summary string. Pure function so it can be unit-tested
- * without running the full harness.
- *
- * The summary covers three scenarios:
- *   1. Declared (buildable) subject — no dist existed, fell back to source dir.
- *   2. Declared (buildable) subject — existing dist was staged WITHOUT rebuilding
- *      (potentially stale).
- *   3. Plain source subject — staged as-is, no build step.
- *
- * Always includes the assembled spawn command, the staged-manifest entry count +
- * fingerprint, and the provenance.json path so a stale tree is visible at a glance.
- */
 export declare function buildDryRunSummary(input: DryRunSummaryInput): string;
 export interface CleanupHarnessOptions {
     /** User asked to retain the harness dir (`--keep`) — never remove. */
@@ -397,11 +556,32 @@ export interface CleanupHarnessOptions {
     created: boolean;
 }
 /**
- * Remove the harness directory after a run so staged untrusted skill bytes and
- * prompts do not accumulate in OS tmp. No-op when the user asked to keep it, when
- * the dir is a user-supplied location (`--out`/`--workdir`), or when it is already
- * gone. Idempotent and never throws — it runs from a `finally`, so it must not
- * mask the run's real outcome.
+ * The one child of the harness root that cleanup NEVER removes. Cleanup exists to
+ * evict staged untrusted skill bytes and prompts from OS tmp; `results/` is the
+ * opposite — it is the run's product, written solely by vat.
+ */
+export declare const RETAINED_RESULTS_DIRNAME = "results";
+/**
+ * Remove the harness directory's staged contents after a run so untrusted skill
+ * bytes and prompts do not accumulate in OS tmp. No-op when the user asked to keep
+ * it, when the dir is a user-supplied location (`--out`/`--workdir`), or when it is
+ * already gone. Idempotent and never throws — it runs from a `finally`, so it must
+ * not mask the run's real outcome.
+ *
+ * `results/` SURVIVES. It holds `grading.json`, `friction.json`, `tool-eval.json`
+ * and — for a `--baseline` run — `baseline.json`, i.e. every artifact the command's
+ * own help text tells the operator to go read. Removing the whole root on the
+ * DEFAULT invocation (no `--out`, no `--workdir`, no `--keep`) deleted those files
+ * inside this function's own `finally`, before `run.ts` had printed a single line:
+ * the operator was handed a `Harness:` path to a directory that no longer existed.
+ * That is the invocation `vat-skill-testing.md`'s copy-paste example uses.
+ *
+ * The retention is deliberately UNCONDITIONAL rather than `--baseline`-only: every
+ * run's verdict detail lives in the same directory, and a rule that depends on a
+ * flag is a rule that will drift away from the flag. The cost is bounded — the
+ * harness root is a deterministic function of the subject set, so a re-run reuses
+ * (and {@link wipeStaleArtifacts} clears) the same `results/`, rather than
+ * accumulating one per run.
  *
  * SAFETY: re-asserts the root is not a symlink immediately before removal (via
  * `lstat`, which does NOT follow the link). A root swapped to a symlink between
@@ -412,7 +592,7 @@ export declare function cleanupHarness(harnessRoot: string, opts: CleanupHarness
 /** One unit of executor→grader work: an eval + which arm (with/without skill). */
 export interface EvalWorkItem {
     entry: EvalEntry;
-    arm: 'with' | 'without';
+    arm: EvalArm;
 }
 /**
  * Build the per-eval work items for a set of evals (one tier's worth, or a whole
@@ -455,18 +635,264 @@ export declare function partitionFragmentsByArm(fragments: EvalFragment[]): {
  */
 export declare function resolveGraderOutDir(dirToken: string): string;
 /**
- * The executor working directory for one eval: its staged input workspace
- * `<workspacesRoot>/<id>` when the eval declares input `files`, else undefined
- * (the executor then defaults to the staged subject dir). Pure + unit-testable.
+ * The executor working directory for one eval ON ONE ARM: its staged input
+ * workspace `<workspacesRoot>/<arm>/<id>`, ALWAYS — empty when the eval declares
+ * no input `files` (see {@link stageEvalWorkspaces}).
+ *
+ * This used to return undefined for a file-less eval, which made the executor
+ * fall back to running inside the staged subject dir. That fallback was the
+ * quiet half of the `--baseline` control defect: the skill-absent arm's cwd
+ * was the skill.
+ *
+ * The per-arm segment closes the OTHER half, found a round later: without it both
+ * arms of one eval shared a single writable directory AND ran concurrently, so
+ * the control arm could read the treatment arm's output files and answer from
+ * them. Keyed on the eval id alone this is a pure function of the eval; the arm
+ * is what makes the two runs independent.
+ *
+ * The segment is an OPAQUE per-run token, never the arm's name — this path is
+ * quoted verbatim to the executor as its working directory, so `…/without/e1`
+ * told the control arm it was the control, and `../with/e1` told it where to look.
+ * See {@link mintArmWorkspaceDirs}. Pure + unit-testable.
  */
-export declare function resolvePerEvalWorkspaceDir(entry: EvalEntry, workspacesRoot: string): string | undefined;
+export declare function resolvePerEvalWorkspaceDir(entry: EvalEntry, workspacesRoot: string, arm: EvalArm, armDirs: ArmWorkspaceDirs): string;
 /**
  * Best-effort removal of a vat-only tmp dir that lives OUTSIDE the harness root —
  * the grader fragment dir and the held eval suite. Never throws (it runs from
  * cleanup, where masking the run's real outcome would be worse than a leftover 0700
  * tmp dir) and refuses to follow a symlinked root.
  */
+/**
+ * Run one cleanup step, swallowing whatever it throws.
+ *
+ * The discipline every other step in the harness `finally` already follows, stated
+ * once: a cleanup failure is not a run failure, and letting one out REPLACES an
+ * already-good result — verdict computed, artifacts written, summary composed — with
+ * an error and exit 1.
+ *
+ * ⚠️ A CALL-SITE GUARD IS THE WEAK FORM, and this helper is a last resort rather
+ * than the pattern to copy. It is invisible to the compiler and to the test suite:
+ * reverting either of its two original call sites to a bare call left the unit suite
+ * AND the integration suite green, so nothing but review stood between the fix and
+ * its own silent removal. `lock.release()` was therefore made non-throwing at its own
+ * definition instead (see {@link acquireHarnessLock}), which is a contract the type
+ * carries and no edit here can undo. That leaves ONE caller —
+ * `killAllActiveClaudeChildren`, which lives in `@vibe-agent-toolkit/utils` and is
+ * only contractually non-throwing as long as that package says so.
+ *
+ * The failure is reported on stderr rather than silently dropped: a lockfile that
+ * could not be removed will break the NEXT run with a "busy" error, and an operator
+ * who saw nothing here has no way to connect the two.
+ */
+export declare function swallowCleanupFailure(step: () => void): void;
 export declare function removeVatOnlyDir(dir: string | undefined): void;
+/** Everything one executor→grader pipeline worker needs, built once per run. */
+interface EvalRunContext {
+    subjectStagedDir: string;
+    workspacesRoot: string;
+    /** This run's opaque per-arm workspace segments (see {@link mintArmWorkspaceDirs}). */
+    armDirs: ArmWorkspaceDirs;
+    pluginDirs: string[];
+    graderOutDir: string;
+    /**
+     * The vat-only dir a harvested eval suite is held in — the ANSWER KEY for a
+     * fetched-artifact subject. Carried on the context solely so the WITHOUT arm's
+     * contamination scan can look for it; nothing in the eval loop reads from it.
+     */
+    evalSuiteHoldDir: string;
+    /**
+     * Verbatim lines of the staged SKILL.md, for the skill-absent arm's scan. The
+     * only contamination signal that does not need the arm to name a path or an
+     * executable, and so the only cover an instruction-only skill has.
+     */
+    skillContentNeedles: readonly string[];
+    runNonce: string;
+    graderModel: string;
+    model?: string;
+    maxTurns: number;
+    maxBudgetUsd: number;
+    timeoutMs: number;
+    stallMs?: number;
+    /**
+     * Full assembled env (skill secrets included) for the executor spawn, WITHOUT
+     * the declared `env` injections — those carry `${fixturesDir}`, which names the
+     * eval's own workspace and so can only be resolved per eval (see `perEvalEnv`).
+     */
+    executorEnv: NodeJS.ProcessEnv;
+    /**
+     * Present only when the subject declares an `env` block. Re-assembles the child
+     * env per eval so `${fixturesDir}` resolves under THAT eval's workspace. The
+     * transparency line is emitted once, by whichever eval assembles first.
+     */
+    perEvalEnv: PerEvalEnvAssembly | undefined;
+    /** AUTH-ONLY env for the grader spawn (trusted vat infra loads no skill). */
+    graderEnv: NodeJS.ProcessEnv;
+    /** Declared executables (WITH-arm grader aid + the detector's path needles); absent when unreachable. */
+    declaredExecutables?: Array<{
+        name: string;
+        howInvoked: string;
+        kind: string;
+        path: string;
+    }>;
+    spawn?: typeof spawnHeadlessClaude;
+    /** Shared spend accumulator — each worker folds in its executor + grader session cost. */
+    costAccumulator: RunCostSummary;
+    /**
+     * The harness root, where vat stages runnable copies of the subject. Used ONLY
+     * as a contamination needle for the skill-absent arm (see baseline-integrity.ts):
+     * nothing points the control arm here any more, so a mention of this path in its
+     * transcript means it went looking and found vat's own copy.
+     */
+    harnessRoot: string;
+}
+/**
+ * The control arm ran out of rate-limit retries — record it and let the run live.
+ *
+ * The retryable case is handled correctly by {@link runEvalWorker} and must stay that
+ * way; this is the EXHAUSTED case, which used to be indistinguishable from it. A
+ * control executor rate-limited on every attempt threw after five retries, taking
+ * down a run in which both treatment executors and both treatment graders had already
+ * completed and been billed, leaving `results/` holding `provenance.json` and nothing
+ * else — the exact failure {@link runEvalWorker} exists to prevent, reached by the one
+ * error class it had to let through.
+ *
+ * The treatment arm still hard-fails: there is no treatment result to salvage, which
+ * is the same asymmetry {@link EvalWorkOutcome} documents.
+ *
+ * Wired as the pipeline's `onRetriesExhausted`, and `--baseline` — which doubles the
+ * spawn count — is the run most likely to need it.
+ */
+export declare function rateLimitExhaustedOutcome(item: EvalWorkItem, error: RateLimitSignal): EvalWorkOutcome;
+/**
+ * Strip the fields VAT attaches itself, before VAT attaches them.
+ *
+ * The grader's only input is the executor transcript, which untrusted skill code
+ * controls — so a prompt injection there could otherwise fabricate a
+ * `BASELINE CONTAMINATED` verdict carrying attacker-chosen strings into an
+ * operator-facing artifact, or (via `degraded`) claim a scan degraded that did not.
+ * Spreading VAT's value last is NOT sufficient: it is absent on a clean run, so it
+ * overwrites nothing — which is exactly why the omission direction matters more
+ * than the invention one for `degraded`. A grader that omits it cannot re-hide a
+ * degradation VAT detected, because VAT's value is spread after this strip; a
+ * grader that INVENTS it would otherwise stamp a blind-scan warning on a run that
+ * scanned properly, and an operator who learns to ignore that warning stops reading
+ * the real ones.
+ */
+export declare function withoutGraderContamination(fragment: EvalFragment): EvalFragment;
+/**
+ * What one work item produced: a graded fragment, or — CONTROL ARM ONLY — a
+ * recorded failure that the run carries on past.
+ *
+ * The asymmetry is the whole point and it is not a subjective call. A treatment-arm
+ * failure leaves nothing to salvage: `grading.json` IS the treatment arm, so a run
+ * that cannot produce it has produced nothing and may as well stop. A control-arm
+ * failure destroys only the second half of a comparison whose first half is already
+ * bought and paid for — and the artifact that reports that comparison already has a
+ * way to say "I do not exist here" (`delta: null`). Failing the whole run on a
+ * control-arm timeout threw away every treatment executor and grader spawn that had
+ * already completed, wrote no `grading.json` at all, and left `results/` holding
+ * only `provenance.json`.
+ */
+type EvalWorkOutcome = {
+    readonly fragment: EvalFragment;
+    readonly controlFailure?: undefined;
+} | {
+    readonly fragment?: undefined;
+    readonly controlFailure: BaselineControlArmFailure;
+};
+/**
+ * Lift the run's content needles off the STAGED SKILL.md.
+ *
+ * Staged, not source: the staged copy is byte-identical to what the treatment arm
+ * is given, so a needle taken from it is exactly the text an ambient copy of the
+ * same build would carry.
+ *
+ * The exclusion set is EVERY channel through which vat itself hands the arm text:
+ * the suite's prompts, its `expected_output`, its expectations, and the CONTENTS
+ * of the input `files` fixtures vat stages into the arm's workspace and instructs
+ * it to operate on. An adopter who quotes a sentence of their SKILL.md through any
+ * of the four would otherwise stamp every run contaminated, including the arm that
+ * merely read the input vat gave it — the skill's words arriving THROUGH vat are
+ * not the arm reaching the skill. The fixture channel is the one that was missed,
+ * and it is the worst of the four to get wrong: the summary attached to that
+ * verdict tells the operator to go uninstall an ambient copy of the plugin that
+ * does not exist.
+ *
+ * Fixtures are read from the STAGED workspace rather than the suite directory, for
+ * the same reason the SKILL.md is: that is byte-for-byte what the arm was handed.
+ *
+ * Returns `[]` when the file cannot be read. A missing SKILL.md is not this
+ * function's error to raise (staging has already validated the subject), and an
+ * unarmed signal is reported as unarmed rather than claimed as clean. An
+ * unreadable FIXTURE is skipped for the same reason, and errs the safe way: a
+ * fixture we could not exclude can only make the signal noisier, never blinder.
+ */
+export declare function resolveSkillContentNeedles(subjectStagedDir: string, evals: readonly EvalEntry[], 
+/**
+ * Where the staged fixtures live, as the two inputs an arm root is DERIVED from
+ * rather than as the pre-joined path itself — see {@link armWorkspaceRoot}. Taking
+ * a bare `string` here let the call site pass `workspacesRoot` (one level too high,
+ * so every `<id>/` fixture lookup missed) with the whole unit suite still green.
+ *
+ * The `with` arm, because the fixture BYTES are what matter and both arms are
+ * staged from the same source — and `with` is the only arm every run mints.
+ */
+workspaces: {
+    workspacesRoot: string;
+    armDirs: ArmWorkspaceDirs;
+}): string[];
+/**
+ * Baseline integrity for ONE skill-absent eval: vat has removed its OWN copies
+ * from the control arm's reach, but an ambient copy in the adopter's repo or
+ * installed plugin cache is not vat's to remove — so the control arm can still,
+ * in principle, find and run the skill. Detect it from the transcript and return
+ * the fragment patch that carries the evidence to the merge step, which turns it
+ * into a loud caveat on baseline.json. A wrong number that announces itself is
+ * recoverable; a silent one gets believed.
+ *
+ * Returns an empty patch when clean, so the caller spreads it unconditionally.
+ */
+/**
+ * Exactly the per-run values {@link buildContaminationInput} reads. A `Pick`, not a
+ * fresh interface, so it cannot drift from the context the run actually assembles.
+ */
+export type ContaminationCtx = Pick<EvalRunContext, 'armDirs' | 'workspacesRoot' | 'harnessRoot' | 'evalSuiteHoldDir' | 'graderOutDir' | 'skillContentNeedles' | 'declaredExecutables'>;
+/**
+ * Build the detector input for the skill-absent arm. ONE builder, so the hits
+ * written per eval and the `signals` list stamped on the run-level integrity
+ * block are derived from the same values — a second construction site is how the
+ * block would come to claim a detector the scan never actually ran.
+ *
+ * `transcript` and `evalId` are supplied per call; everything else is per-run.
+ *
+ * Takes a NARROWED context rather than the whole {@link EvalRunContext} so the
+ * builder is directly unit-testable — the detector honours `armCwd` (dropping it
+ * there kills three tests) while the wiring that SUPPLIES it was droppable green,
+ * and a dropped `armCwd` anchors every relative climb one directory too high, where
+ * no needle can see it.
+ *
+ * `evalId` is what turns the arm's workspace ROOT into the arm's actual starting
+ * CWD (`<armWorkspaceDir>/<evalId>` — see {@link resolvePerEvalWorkspaceDir}).
+ * The detector needs both and they are one directory apart: containment is judged
+ * against the root (the arm's whole tree is its own), while the cwd walk must
+ * start exactly where the executor did, or every relative climb in the transcript
+ * resolves one level too high and lands where no needle can see it.
+ *
+ * `evalId` is REQUIRED here, and the signals-only caller gets its own entry point
+ * ({@link buildContaminationSignalsInput}) rather than sharing an optional third
+ * argument. An optional parameter is a pin the compiler cannot hold: dropping the
+ * argument at the SCAN call site left all 2270 unit tests green and was caught only
+ * by the integration suite — which this repo's fast implementer loop explicitly says
+ * not to run. That is the same shape as the dropped `armCwd` that anchored the whole
+ * walk one directory too high, also green.
+ */
+export declare function buildContaminationInput(transcript: string, ctx: ContaminationCtx, evalId: string): DetectBaselineContaminationInput;
+/**
+ * The same input built for the "which detectors are ARMED?" question, which is asked
+ * once per run with no transcript and no eval — so it can honestly supply neither,
+ * and `armCwd` is correctly absent rather than accidentally dropped.
+ */
+export declare function buildContaminationSignalsInput(ctx: ContaminationCtx): DetectBaselineContaminationInput;
 /** The results/ artifacts vat is the SOLE writer of, resolved for one run. */
 export interface ArtifactPaths {
     gradingOut: string;
@@ -488,6 +914,85 @@ export declare function resolveArtifactPaths(resultsDir: string): ArtifactPaths;
  * (`force: true`) — a missing file is fine.
  */
 export declare function wipeStaleArtifacts(paths: ArtifactPaths): void;
+/**
+ * Where a results artifact goes when it fails the fail-closed gate.
+ *
+ * The gate throws and `results/` is deliberately RETAINED, so the file that failed
+ * used to sit there under its authoritative name — the one artifact whose schema
+ * violation is BY DEFINITION a harness bug, indistinguishable in a CI archive from a
+ * good one until the next run's wipe. Renamed rather than deleted: the bytes are what
+ * a maintainer needs in order to see which refine tripped, and a name ending in
+ * `.rejected` cannot be mistaken for the artifact a reader came for.
+ */
+export declare function rejectedArtifactPath(path: string): string;
+/**
+ * Merge the run's grader fragments and WRITE the run artifacts (vat is the SOLE
+ * writer): grading.json from the WITH-arm fragments, friction.json from ALL
+ * fragments, tool-eval.json from the WITH-arm fragments' `tool` verdicts (a
+ * SEPARATE channel — C2; the WITHOUT arm carries no toolExpectations), and — only
+ * when a WITHOUT arm ran (baseline) — baseline.json from the WITHOUT-arm fragments
+ * (informational A/B, never part of the verdict). Stale copies from a reused harness
+ * were already removed by {@link wipeStaleArtifacts} pre-pipeline. Returns the
+ * reconciled prose-expectation verdict AND the merged tool-eval report so the caller
+ * can compute the COMPOSITE verdict. Every fragment's per-run nonce is re-verified
+ * inside {@link mergeFragmentsToGrading}.
+ */
+/**
+ * Collect per-eval baseline-integrity findings from the WITHOUT-arm fragments.
+ * Fragments with no `contamination` are clean and contribute nothing. Pure.
+ */
+/**
+ * What one arm graded, per eval — the SINGLE derivation behind both the parity check
+ * ({@link armExpectationSkew}, which reads `total`) and the A/B delta (which reads
+ * `passed`). Deriving those separately would let the two blocks in `baseline.json`
+ * disagree about the same arm. Pure.
+ *
+ * Exported for its own unit test, not for reuse. Setting `passed` to `total` here
+ * failed zero unit tests and three integration ones: every unit fixture graded
+ * everything it declared, so the two fields were numerically identical in all of
+ * them and the ONE thing this function exists to keep distinct was unpinned.
+ */
+export declare function gradedCounts(fragments: EvalFragment[]): ArmEvalGrade[];
+/**
+ * The stderr report for a `--baseline` run: the delta line, then the ⚠️ banner when
+ * the run is unusable. Pure — the caller decides WHEN it goes out, which is the
+ * whole point (see the call site).
+ *
+ * Composed on EVERY baseline run, including a withheld one: "the delta is
+ * unavailable, and here is why" is the report an operator needs, and a run that
+ * silently prints nothing is indistinguishable from a vat that still does not
+ * compute a delta at all. The delta line comes FIRST so an unusable run reads as the
+ * number followed by the caveat that explains it — the banner is the only line that
+ * says the number means nothing, so it must never be the one scrolled away.
+ *
+ * ⚠️ IT IS NOT LITERALLY LAST ON THE SCREEN, and this docblock used to say it was.
+ * The CLI prints `Harness:`, `Results:`, `Workspaces:` and `Summary:` AFTER the
+ * harness returns, so 2–4 lines always follow the banner. What this ordering buys is
+ * that the banner is not separated from its delta by up to ~1250 rows of friction —
+ * a bounded, short tail after it, not none. Moving those CLI lines above the harness
+ * report is the CLI's call, not this module's.
+ *
+ * THREE conditions have to hold before the caveat is dropped, not two. A DEGRADED
+ * scan is neither contaminated nor incomparable, so a two-clause gate printed
+ * `Baseline delta: +0 (with skill: 2/2, without skill: 2/2).` and exited 0 while
+ * `baseline.json` carried `degraded: [...]` and an `integrity.summary` ending in a
+ * `⚠️ DEGRADED SCAN: …` sentence that was composed, written to the artifact, and
+ * never emitted anywhere. Everything this module says about telling "checked and clean"
+ * apart from "checked with the blunt instrument" was true of the artifact and false
+ * of the terminal — and the terminal is the only surface an operator reads.
+ *
+ * A CONTAMINATED run still reports its number, deliberately. Contamination does not
+ * make the subtraction illegal — both arms were graded to the same depth, so the
+ * arithmetic is sound; what it invalidates is INTERPRETING that number as skill
+ * lift, which is precisely what the ⚠️ line says ("The reported delta is NOT a
+ * measure of skill lift"), and what `vat-skill-testing.md` turns into the
+ * instruction "discard the delta". Withholding it here would (a) leave that sentence
+ * referring to a number that was never reported, and (b) spend `null` — which means
+ * "these arms cannot be subtracted", a different failure with a different remedy —
+ * on a run where they can be. A contaminated delta is also diagnostic in its own
+ * right: it collapses toward zero, which is corroboration, not noise.
+ */
+export declare function formatBaselineReport(delta: BaselineDelta, integrity: BaselineIntegrity): string;
 /**
  * The COMPOSITE run verdict (issue #145 Phase T): the run passes only when BOTH
  * the prose-expectation grading passed AND every tool-expectation verdict passed.
@@ -540,9 +1045,47 @@ export declare function buildRunSummary(verdict: GradingVerdict, toolEval: ToolE
  * the call site), so the base line is already FAIL when the note is present.
  */
 export declare function buildRunSummaryWithSkips(verdict: GradingVerdict, toolEval: ToolEvalReport, compositeAllPassed: boolean, skipped: SkippedEvalsSummary | undefined, cost?: RunCostSummary): string;
+/**
+ * Post-merge D2 gate over the vat-written results artifacts (vat is SOLE writer):
+ * grading.json, friction.json, tool-eval.json — and, on a baseline run, baseline.json
+ * — must each exist + parse + validate. A separate, explicit check (NOT folded into
+ * any prompt-invariant linter) — fail-closed and safe because vat wrote every one of
+ * these itself just above.
+ *
+ * `baseline.json` was EXEMPT from this gate for as long as the gate has existed, and
+ * it is the artifact the exemption fits worst: on a `--baseline` run it is the only
+ * durable record of the thing the command sells, and this branch has already been
+ * bitten once by a default run deleting `results/` out from under it. Its validation
+ * is also the strictest of the four — {@link BaselineArtifactSchema} carries the
+ * `passed <= total` and `delta === with − without` refines, so a merge that produced
+ * a self-contradicting number fails the run here instead of being written down.
+ *
+ * `baselineRun` gates it rather than "does the file exist", deliberately: an
+ * existence check would pass on the one failure that matters (vat asked for a
+ * baseline and wrote nothing), which is exactly what a fail-CLOSED gate must not do.
+ * A non-baseline run writes no baseline.json at all and is not asked about one.
+ */
+export declare function assertVatWroteArtifacts(paths: ArtifactPaths, baselineRun: boolean): void;
 /**
  * Domain orchestrator for `vat skill test run`. Pure of process.exit — all
  * exit-code decisions live in the caller (run.ts).
  */
+/**
+ * Resolve WHERE this run lives, and whether the harness owns that location.
+ *
+ * `--out` names the harness root exactly; `--workdir` names the BASE the root is
+ * derived under. They are mutually exclusive: passing both used to silently
+ * discard `--workdir` — the run went entirely to `--out` and the `--workdir` path
+ * was never even created — so an operator using the pair to separate the
+ * executor's cwd from the staged trees got no separation and no warning.
+ *
+ * `harnessCreated` is true only for the derived-under-OS-tmp case; an explicit
+ * `--out` or `--workdir` is a location the user owns and is never auto-removed.
+ */
+export declare function resolveHarnessLocation(opts: Pick<RunHarnessOptions, 'out' | 'workdir' | 'subject'>): {
+    harnessRoot: string;
+    harnessCreated: boolean;
+};
 export declare function runSkillTestHarness(opts: RunHarnessOptions): Promise<RunHarnessResult>;
+export {};
 //# sourceMappingURL=run-harness.d.ts.map