@vzn/vx 0.0.16 → 0.0.21

package/src/orchestrator/run.ts

@@ -2,17 +2,14 @@
 // run with caching. Each step delegates to a single-purpose sibling file
 // so the layers can be swapped without touching the others.
 
-import os from 'node:os'
-import {
-  type CacheLayer,
-  type CachePolicy,
-  FULL_CACHE_POLICY,
-  type InvocationRecord,
-  LayeredCache,
-  type RunRecord,
-} from '../cache/index.js'
+import type { ProjectEntry } from '../workspace/index.js'
+import path from 'node:path'
+import { type CacheLayer, type CachePolicy, FULL_CACHE_POLICY } from '../cache/index.js'
 import { VERSION } from '../version.js'
-import { initSandbox, probeSandbox, resetSandbox, signalExitCode } from '../exec/index.js'
+import { resetSandbox, VX_RUN_TASK_ENV, VX_RUN_WORKSPACE_ENV } from '../exec/index.js'
+import { DeferredOutputs } from './deferred-outputs.js'
+import { resolveDownloadModes } from './download-policy.js'
+import type { TaskExecutor } from '../exec/index.js'
 import {
   isGroupTask,
   markSurfacedDeps,
@@ -20,24 +17,40 @@ import {
   type TaskNode,
   type TaskOutcome,
 } from '../graph/index.js'
-import { ulid, UserError } from '../util/index.js'
-import { executeTask } from './execute-task.js'
-import { resolveResourceCosts } from './resources.js'
-import { computeTaskHash } from './task-hash.js'
+import {
+  mark,
+  MAX_TIMEOUT_MS,
+  printTimings,
+  ulid,
+  nearest,
+  UserError,
+  machineParallelism,
+} from '../util/index.js'
+import { prepareSandbox } from './sandbox-request.js'
+import type { OutputDirSnapshot } from './miss-save.js'
+import { admitTasks, taintTracker } from './admission.js'
 import { busLogger, createEventBus, terminalSubscriber } from './events.js'
 import { installPlugins } from './plugin.js'
-import type { VxPlugin } from './plugin.js'
-import { subscribeEventSinks, teardownPlugins } from './plugin-host.js'
-import type { SubscribedEventSinks } from './plugin-host.js'
+import { buildAdmission, resolveExecutors, teardownPlugins } from './plugin-host.js'
 import { subscribeTelemetry, type TelemetryHandle } from './telemetry-host.js'
-import { deriveCacheSource, TELEMETRY_SCHEMA_VERSION } from './telemetry.js'
-import type { RunContextRecord, RunSummaryRecord, TaskTelemetry } from './telemetry.js'
+import { assembleRunSummary, isPassStatus } from './telemetry.js'
+import type { RunContextRecord } from './telemetry.js'
 import { defaultLogger, resolveOutputView } from './logger.js'
 import { detectColors } from './colors.js'
 import { formatPersistentList } from './framed-output.js'
+import { LocalHistoryProvider } from './history.js'
 import { plan, type RunPlan } from './plan.js'
 import { prepareRun } from './prepare.js'
+import { forwardSignals, terminateChildren } from './signals.js'
 import {
+  hasPooledExecutor,
+  placeTasks,
+  planExecutorOf,
+  poolOfPlacement,
+  UNPLACED_EXECUTOR,
+} from './placement.js'
+import {
+  captureDefaultBranch,
   captureGitContext,
   captureHostContext,
   captureWorkspaceIdentity,
@@ -45,24 +58,39 @@ import {
 } from './run-context.js'
 import { startRemotePrefetch } from './remote-prefetch.js'
 import { startLocalShortCircuit, type ShortCircuit } from './local-shortcircuit.js'
-import { formatVerifySection } from './verify.js'
 
-const EMPTY_SHORT_CIRCUIT: ShortCircuit = { preProbed: new Map(), restoreTier: new Set() }
+import { assembleRunRecords } from './run-records.js'
+import { selectKeepAlive, shutdownPersistent } from './persistent.js'
 import { writeRunProfile, writeRunSummary } from './run-artifacts.js'
-import { formatRunSummary } from './summary.js'
+import { createSaveLane } from './save-lane.js'
+import { formatAbortedSection, formatFlakySection, formatRunSummary } from './summary.js'
+import { detectFlaky, type FlakyCandidate } from './failure-mode.js'
 import type { RunOptions, RunSummary } from './options.js'
 
+// Per run, never shared: a `vx watch` process runs many, and a shared map
+// is one `preProbed.set` away from leaking a hit across cycles.
+const emptyShortCircuit = (): ShortCircuit => ({ preProbed: new Map(), restoreTier: new Set() })
+
 /**
  * Parse the `VX_TASK_TIMEOUT` env var (ms) — the "global" run-level task
  * timeout default. A missing/empty/non-positive-integer value yields
  * `undefined` (ignored), so a typo never silently disables a task's own
- * `exec.timeout`.
+ * `exec.timeout`. A value past `MAX_TIMEOUT_MS` is clamped to it — see below.
+ *
+ * Exported for `tests/options-resolve.test.ts`, which pins every accepted
+ * and ignored form of this rung; it has no other caller outside this file.
  */
-function readTaskTimeoutEnv(): number | undefined {
+export function readTaskTimeoutEnv(): number | undefined {
   const raw = process.env['VX_TASK_TIMEOUT']
   if (raw === undefined || raw === '') return undefined
   const n = Number(raw)
-  return Number.isInteger(n) && n > 0 ? n : undefined
+  if (!Number.isInteger(n) || n <= 0) return undefined
+  // CLAMPED, not refused — unlike `exec.timeout` and `--timeout`, this rung's
+  // contract is already "unusable value → fall back", so it never throws. But
+  // falling back is wrong here: someone who typed a huge number wants a long
+  // timeout, and the largest expressible one IS long (~24.8 days). Handing it
+  // to `setTimeout` unbounded would mean 1 ms — killing every task instantly.
+  return Math.min(n, MAX_TIMEOUT_MS)
 }
 
 /**
@@ -79,32 +107,49 @@ function compactCachePolicy(p: CachePolicy): string {
   return parts.join(',')
 }
 
+/**
+ * The policy that will actually govern the run. The remote axes are inert
+ * without a remote layer to serve them: a `remote:w` policy over a bare
+ * local cache writes NOWHERE, yet the write axis reads as on. Normalising
+ * here — the ONE place both `run()` and `planRun()` derive it from — is
+ * what keeps `--dry` describing the run you are about to get: reading the
+ * raw request made the plan label `--cache=local:,remote:rw` "cache miss —
+ * would exec" (a result that will be stored) for a run in which caching is
+ * entirely off.
+ */
+function effectiveCachePolicy(requested: CachePolicy, hasRemoteLayer: boolean): CachePolicy {
+  return hasRemoteLayer ? requested : { ...requested, remoteRead: false, remoteWrite: false }
+}
+
 /**
  * The perf firewall for the local short-circuit. Always-on; the only
  * gates are correctness/no-op gates:
- *   - LOCAL-ONLY cache. Under a LayeredCache, `cache.get` is a remote
+ *   - LOCAL-ONLY cache. Behind a remote layer, `cache.get` is a remote
  *     READ-THROUGH — the up-front classify is awaited before scheduling,
  *     so N remote GETs would land on the critical path before any task
  *     starts. Remote runs are owned by `startRemotePrefetch`, which
  *     overlaps the GETs with execution instead (fire-and-forget) and
- *     ingests hits into local for execute-task's lazy probe.
+ *     ingests hits into local for execute-task's lazy probe. The gate asks
+ *     the layer (`hasRemote`), not `instanceof LayeredCache` — a
+ *     third-party remote layer stalls on the classify just as hard.
  *   - the policy reads locally (a `--no-cache`/`--force`/`--cache=local:`
  *     run reads nothing locally → nothing to restore → skip);
- *   - the graph has at least one dependency edge (a flat graph has no
- *     ordering to bypass — restoring is what execute() already does, so
- *     there's no upside and we avoid the upfront probe pass).
- * The per-task correctness gates (stable key, no workspace-outputs in
- * graph) live in `startLocalShortCircuit`.
+ *   - there is at least one task.
+ * A flat graph has no ordering to bypass, but the classify pass is worth
+ * paying for anyway: it probes every stable key through ONE batched
+ * `getMany` instead of a `cache.get` per task inside the run, which
+ * measured 84 → 78 ms on 100 dep-free tasks and 249 → 237 ms on 1000
+ * (2026-09-03, interleaved arms). The per-task correctness gates (stable
+ * key, no workspace-outputs in graph) live in `startLocalShortCircuit`.
  */
 export function shouldShortCircuit(
   nodes: Map<string, TaskNode>,
   policy: CachePolicy,
   cache: CacheLayer,
 ): boolean {
-  if (cache instanceof LayeredCache) return false
+  if (cache.hasRemote === true) return false
   if (!policy.localRead) return false
-  for (const node of nodes.values()) if (node.deps.length > 0) return true
-  return false
+  return nodes.size > 0
 }
 
 export async function run(options: RunOptions): Promise<RunSummary> {
@@ -130,6 +175,34 @@ export async function run(options: RunOptions): Promise<RunSummary> {
   const log = busLogger(bus)
 
   const prepared = await prepareRun(options, log)
+  mark('plugin stages')
+  // A task whose command re-enters `vx run` in the workspace running it
+  // is refused. When the inner run reaches this task again it forks a run
+  // per run until the machine gives out; when it does not (`ci` shelling
+  // out to `vx run lint`) it is a nested run the outer graph cannot see —
+  // its tasks escape the schedule, the concurrency budget and this task's
+  // cache key. The markers `taskEnv` sets on every child (exec/env.ts)
+  // name the task; the check is on the root so both shapes are caught.
+  const outerRoot = process.env[VX_RUN_WORKSPACE_ENV]
+  if (outerRoot !== undefined && path.resolve(outerRoot) === path.resolve(prepared.workspaceRoot)) {
+    prepared.cache.close()
+    throw new UserError(
+      `task ${process.env[VX_RUN_TASK_ENV] ?? '<unknown>'} runs \`vx run\` inside its own workspace: a nested run is invisible to the outer graph (its tasks escape the schedule, the concurrency budget and the cache key) and a loop back to this task forks without bound. Declare what it needs with dependsOn instead.`,
+    )
+  }
+  // A requested name that matched no project is a typo (or a stray
+  // positional from an `=`-only flag written with a space). Failing the
+  // whole run — even when OTHER requested tasks resolved — is the point:
+  // a CI job that renames a task must go red, not silently stop running
+  // it. When EVERY name is unresolved this is the `no-tasks-declared`
+  // case too; the message is identical, so that branch stays below.
+  if (prepared.unresolvedTasks.length > 0) {
+    log.status(
+      `No projects declare task(s): ${prepared.unresolvedTasks.join(', ')}.${didYouMean(prepared.unresolvedTasks, prepared.projects)}${initHint(prepared)}`,
+    )
+    prepared.cache.close()
+    return { ok: false, outcomes: [] }
+  }
   if (prepared.empty !== null) {
     // `no-tasks-declared` is almost always a typo in CI; we surface
     // a clear message and return NOT-ok so the script exits 1.
@@ -137,7 +210,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     // buildTaskGraph semantics but logged just in case.
     const msg =
       prepared.empty === 'no-tasks-declared'
-        ? `No projects declare task(s): ${options.tasks.join(', ')}.`
+        ? `No projects declare task(s): ${options.tasks.join(', ')}.${initHint(prepared)}`
         : 'No tasks to run.'
     log.status(msg)
     prepared.cache.close()
@@ -145,35 +218,22 @@ export async function run(options: RunOptions): Promise<RunSummary> {
   }
   // Install user plugins as additional bus subscribers BEFORE the run
   // starts emitting events. `installPlugins` runs each plugin's optional
-  // `setup` hook (the old observe-only path); `subscribeEventSinks` wires
-  // each plugin's `eventSink` capability onto the bus via wireForwarder.
-  // Both fail-fast on a setup() throw with a clean UserError naming the
-  // plugin; eventSink init failures are isolated (observability never
-  // breaks a run). With no plugin, both are no-ops.
+  // `setup` hook and fails fast on a throw with a clean UserError naming
+  // the plugin. A plugin without `setup` subscribes nothing.
   let disposePlugins: (() => void) | undefined
-  let eventSinks: SubscribedEventSinks | undefined
   let telemetry: TelemetryHandle | undefined
-  if (prepared.workspaceConfig?.plugins && prepared.workspaceConfig.plugins.length > 0) {
-    const plugins = prepared.workspaceConfig.plugins as readonly VxPlugin[]
-    const pluginCtx = {
+  try {
+    disposePlugins = await installPlugins({
+      plugins: prepared.plugins as never,
+      bus,
       workspaceRoot: prepared.workspaceRoot,
       cacheDir: prepared.cacheDir,
-      warn: (m: string) => log.status(m),
-    }
-    try {
-      disposePlugins = await installPlugins({
-        plugins: prepared.workspaceConfig.plugins as never,
-        bus,
-        workspaceRoot: prepared.workspaceRoot,
-        cacheDir: prepared.cacheDir,
-        warn: (m) => log.status(m),
-      })
-      eventSinks = await subscribeEventSinks(plugins, bus, pluginCtx)
-    } catch (err) {
-      disposePlugins?.()
-      prepared.cache.close()
-      throw err
-    }
+      warn: (m) => log.status(m),
+    })
+  } catch (err) {
+    disposePlugins?.()
+    prepared.cache.close()
+    throw err
   }
   const {
     workspaceRoot,
@@ -187,10 +247,81 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     hashCache,
     workspaceProjectCount,
   } = prepared
-  const concurrency =
-    options.concurrency ??
-    workspaceConfig?.concurrency ??
-    Math.max(1, navigator.hardwareConcurrency)
+  const concurrency = options.concurrency ?? workspaceConfig?.concurrency ?? machineParallelism()
+
+  // Resolved ONCE per run, in declaration order, the local executor last.
+  // A broken factory aborts here, before any task starts.
+  let executors: readonly TaskExecutor[]
+  try {
+    executors = await resolveExecutors(prepared.plugins, {
+      workspaceRoot: prepared.workspaceRoot,
+      cacheDir: prepared.cacheDir,
+      warn: (m: string) => log.status(m),
+      concurrency,
+    })
+  } catch (err) {
+    disposePlugins?.()
+    prepared.cache.close()
+    throw err
+  }
+  // Placement: decided ONCE per task, before scheduling, so the scheduler
+  // can admit a remote-pooled task against its pool instead of a local
+  // worker slot. Group tasks run nothing; persistent tasks never reach an
+  // executor (local by construction) — both stay off the map.
+  const placements = placeTasks(nodes, executors, false)
+  // A `remote: 'only'` task nobody takes succeeds WITHOUT running. That is
+  // deliberate — on a machine with no remote pool the ambient state already
+  // is what the task would have produced — but it must not be SILENT: a task
+  // reporting success while doing nothing is indistinguishable from one that
+  // worked, and the usual cause (no `cache` block, so no executor can
+  // describe its inputs to a worker) cannot be read off the outcome.
+  if (placements.remoteOnlyNoop.size > 0) {
+    const anyRemote = executors.some((e) => e.remote === true)
+    for (const id of placements.remoteOnlyNoop) {
+      const why = !anyRemote
+        ? 'no remote executor is declared'
+        : nodes.get(id)?.config.cache === undefined
+          ? 'no remote executor accepted it — the task declares no `cache` block, so there is no input set to ship'
+          : 'no remote executor accepted it'
+      log.status(`[vx] ${id}: exec.remote is 'only' and ${why} — nothing ran`)
+    }
+  }
+  // Demand: what each executor still has left to run. An executor that
+  // provisions per task (a container, an allocation, a pod) otherwise has to
+  // guess from `capacity` alone and hold everything until teardown. Built
+  // ONLY for executors that asked — with none declared there is no map, no
+  // set, and no call in the completion path.
+  const demandOf = new Map<TaskExecutor, Set<string>>()
+  for (const [id, executor] of placements.executors) {
+    if (executor.demand === undefined) continue
+    let remaining = demandOf.get(executor)
+    if (remaining === undefined) demandOf.set(executor, (remaining = new Set()))
+    remaining.add(id)
+  }
+  // The SAME set is handed back every time, narrowed in place: a consumer
+  // reading it later sees the truth rather than a stale snapshot.
+  for (const [executor, remaining] of demandOf) executor.demand!(remaining)
+  const narrowDemand =
+    demandOf.size === 0
+      ? (): void => {}
+      : (id: string): void => {
+          for (const [executor, remaining] of demandOf) {
+            if (remaining.delete(id)) executor.demand!(remaining)
+          }
+        }
+
+  // `--download` (default `all`) decides ONCE per task, here, whether a
+  // remote execution's outputs come home.
+  const downloadPolicy = options.download ?? 'all'
+  const localPlaced = new Set(
+    [...nodes.keys()].filter((id) => placements.executors.get(id)?.remote !== true),
+  )
+  const download = resolveDownloadModes({
+    nodes,
+    policy: downloadPolicy,
+    localPlaced,
+    remoteOnly: placements.remoteOnly,
+  })
 
   // Run-level default task timeout (ms), applied to any task WITHOUT its own
   // `exec.timeout`. Precedence, highest first: `--timeout`/RunOptions.timeout
@@ -208,38 +339,39 @@ export async function run(options: RunOptions): Promise<RunSummary> {
   //     exit; ownership moves here so the orchestrator can SIGTERM
   //     them once the rest of the graph finishes.
   //
-  // A SIGINT/SIGTERM mid-run forwards SIGTERM to everything live,
-  // closes the cache handle, and exits 128+signo (130/143). Without
-  // this, a programmatic signal to the vx process alone (CI
-  // cancellation, `kill <pid>`) orphans every running child —
-  // terminal Ctrl-C only worked via process-group propagation. The
-  // handlers are removed in the finally below so repeated run()
-  // calls (test suites) never stack listeners.
+  // A SIGINT/SIGTERM mid-run forwards SIGTERM to everything in both
+  // (`signals.ts`); the handlers are removed in the finally below so
+  // repeated run() calls (test suites) never stack listeners.
   const liveChildren = new Set<ReturnType<typeof Bun.spawn>>()
   const persistentRegistry = new Map<string, ReturnType<typeof Bun.spawn>>()
-  const onSignal = (signal: 'SIGINT' | 'SIGTERM'): void => {
-    // Clear the live worker/status region BEFORE exiting so a TTY isn't
-    // left with a frozen region frozen in the scrollback (the documented
-    // KNOWN-OPEN). runEnd is idempotent and a no-op for non-TTY loggers.
-    try {
-      log.runEnd?.()
-    } catch {
-      // teardown must not throw on the way out
-    }
-    for (const child of liveChildren) child.kill('SIGTERM')
-    for (const child of persistentRegistry.values()) child.kill('SIGTERM')
-    try {
-      cache.close()
-    } catch {
-      // double-close race with the normal path; we're exiting anyway
-    }
-    process.exit(signalExitCode(signal))
+  const signals = forwardSignals({
+    enabled: options.handleSignals ?? true,
+    log,
+    cache,
+    liveChildren,
+    persistentRegistry,
+  })
+  // `RunOptions.signal`: the same teardown the process handler runs, minus
+  // the exit — the scheduler stops dispatching (it reads the signal) and
+  // run() returns to its caller. Detached in the finally below.
+  const onAbort = (): void => {
+    void terminateChildren(() => [...liveChildren, ...persistentRegistry.values()])
   }
-  const onSigint = (): void => onSignal('SIGINT')
-  const onSigterm = (): void => onSignal('SIGTERM')
-  if (options.handleSignals ?? true) {
-    process.on('SIGINT', onSigint)
-    process.on('SIGTERM', onSigterm)
+  options.signal?.addEventListener('abort', onAbort, { once: true })
+  // The cache handle must be released on EVERY exit path, not just the
+  // happy one: `close()` is also where the run's deferred `accessed_at`
+  // bumps are flushed, so a throw between opening the cache and the
+  // normal close leaked the SQLite handle (it matters in a long-lived
+  // host) AND lost this run's touch record, after which an LRU
+  // `vx cache prune` can evict entries the run just hit. Once-only
+  // because the normal path closes before the persistent-task wait —
+  // holding the handle open for a dev server's whole lifetime would be
+  // worse — and because close() re-runs its retention DELETEs.
+  let cacheClosed = false
+  const closeCache = (): void => {
+    if (cacheClosed) return
+    cacheClosed = true
+    cache.close()
   }
   try {
     // One run-id per `vx run` invocation. Every task in the resulting
@@ -247,22 +379,29 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     const runId = ulid()
     const runStartHrTimeNs = process.hrtime.bigint()
     const endedAtMsAtStart = Date.now()
-    const remoteCacheEnabled = cache instanceof LayeredCache
-    const policy: CachePolicy = options.cache ?? FULL_CACHE_POLICY
-
-    // `--verify` observes the miss-then-save path, so a policy with NO write
-    // axis (`--no-cache`, `--cache=local:,remote:`) verifies nothing — every
-    // task would come back green with zero verdicts. The user asked for a
-    // proof; silently skipping it is the one failure mode verification must
-    // never have (same platform-honesty rule as the sandbox-unavailable
-    // error). `--force --verify` is the supported re-verify-warm recipe.
-    if (options.verify !== undefined && !policy.localWrite && !policy.remoteWrite) {
-      prepared.cache.close()
-      throw new UserError(
-        '--verify needs cache writes to prove anything (it verifies the save path); ' +
-          'drop --no-cache, or use --force --verify to re-execute and verify a warm graph',
+    const remoteCacheEnabled = prepared.hasRemoteLayer
+    // Normalised ONCE so every consumer — execute-task, the dedup
+    // predicate, the recorded invocation row — reads the policy that
+    // actually governed the run. Reading the raw request here made tasks
+    // clean their outputs before every exec for a save that never
+    // happened.
+    const policy: CachePolicy = effectiveCachePolicy(
+      options.cache ?? FULL_CACHE_POLICY,
+      prepared.hasRemoteLayer,
+    )
+    if (options.remoteRequested === true && !prepared.hasRemoteLayer) {
+      log.status(
+        'vx: --cache named the remote cache, but no cache plugin in vx.workspace.* supplies one — the remote axes are off for this run',
       )
     }
+    const deferredOutputs = new DeferredOutputs({
+      nodes,
+      cache,
+      workspaceRoot,
+      nestedDirsByProject,
+      ...(gitFilesCache !== undefined ? { gitFilesCache } : {}),
+      localWrite: policy.localWrite,
+    })
 
     // Per-run context for the Tier-3 `invocations` header row. Captured
     // ONCE (git is ONE spawn for commit+branch, behind try/catch; never
@@ -275,16 +414,21 @@ export async function run(options: RunOptions): Promise<RunSummary> {
 
     // The canonical run-context record — the same git/CI/host data the
     // invocation header uses, shaped as the telemetry export contract.
-    // Built + consulted ONLY when plugins are declared (an explicit opt-in),
-    // and BEFORE run:start is emitted so a sink catches the whole stream.
+    // Built + consulted ONLY when a plugin CONTRIBUTES `telemetry`, and
+    // BEFORE run:start is emitted so a sink
+    // catches the whole stream.
     // subscribeTelemetry returns undefined when no sink is contributed (no
     // telemetry plugin, or all declined — e.g. otel() with no OTLP endpoint),
     // so a plain run does ZERO extra work: no record allocation, no bus
     // subscriber, no summary building. The hot path stays off-limits.
     let runContextRecord: RunContextRecord | undefined
-    const hasPlugins =
-      prepared.workspaceConfig?.plugins !== undefined && prepared.workspaceConfig.plugins.length > 0
-    if (hasPlugins || options.telemetrySinks !== undefined) {
+    // Gate on a plugin that can actually CONTRIBUTE a sink, not on declaring
+    // any plugin at all: a cache-only plugin has no telemetry hook to
+    // consult, so paying for the record (2 git spawns + a `.vx/workspace-id`
+    // write on a remote-less repo) buys nothing. A plugin that has the hook
+    // but declines still pays — its answer is only knowable by asking.
+    const hasTelemetryPlugin = prepared.plugins.some((p) => p.telemetry !== undefined)
+    if (hasTelemetryPlugin || options.telemetrySinks !== undefined) {
       // Workspace identity (telemetry v2): one git spawn, paid only when a
       // telemetry consumer can exist — a plain run never reaches here.
       const wsIdentity = captureWorkspaceIdentity(workspaceRoot)
@@ -298,6 +442,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
         flow: options.flow ?? null,
         commitSha: gitContext.commitSha,
         branch: gitContext.branch,
+        defaultBranch: captureDefaultBranch(process.env, workspaceRoot),
         dirty: gitContext.dirty,
         ci: ciContext.ci,
         ciProvider: ciContext.provider,
@@ -309,7 +454,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
         tags: options.tags ?? {},
       }
       telemetry = await subscribeTelemetry(
-        (prepared.workspaceConfig?.plugins ?? []) as readonly VxPlugin[],
+        prepared.plugins,
         bus,
         { workspaceRoot, cacheDir, warn: (m: string) => log.status(m) },
         runContextRecord,
@@ -317,27 +462,15 @@ export async function run(options: RunOptions): Promise<RunSummary> {
       )
     }
 
-    // Lazy SRT init: fire it up if at least one task opts into sandboxing
-    // via its `sandbox: {...}` block, OR `--verify=inputs`/`=all` is on (it
-    // forces the declared-input baseline sandbox onto every cacheable task to
-    // prove input-completeness). Tasks that need sandboxing on an unsupported
-    // platform get a hard error so they don't silently run unsandboxed —
-    // `--verify=inputs` in particular must fail loud, never falsely "pass".
-    const verifyInputs = options.verify?.inputs === true
-    const anySandboxed =
-      verifyInputs || [...nodes.values()].some((n) => n.config.sandbox !== undefined)
-    if (anySandboxed) {
-      const avail = await probeSandbox()
-      if (!avail.available) {
-        prepared.cache.close()
-        throw new UserError(
-          verifyInputs
-            ? `--verify=inputs needs the sandbox, which is not available: ${avail.reason}`
-            : `sandbox not available: ${avail.reason}`,
-        )
-      }
-      await initSandbox()
-    }
+    const sandboxArmer = prepareSandbox(nodes.values())
+    const outputDirSnapshots: OutputDirSnapshot[] = []
+    // Saves run off the execution slot, twice the cap at once (memory:
+    // each pack holds an artifact's bytes); a failed save is a miss next
+    // time, said once — the task's work ran.
+    const saveLane = createSaveLane(2 * concurrency, (err) =>
+      log.status(`[vx] cache save failed: ${err instanceof Error ? err.message : String(err)}`),
+    )
+    const deferredSaves = new Map<string, Promise<void>>()
 
     // Focused flow: a requested GROUP has no output of its own, so
     // surface the same-project, non-group tasks it chains (one level)
@@ -361,16 +494,10 @@ export async function run(options: RunOptions): Promise<RunSummary> {
         if (node.requested || node.surfaced === true) requestedCount++
       }
     }
-    // Resource-aware admission: resolve every task's `exec.resources`
-    // into absolute costs ONCE, up front (percent forms against the
-    // budgets), so the scheduler's inner loop is a plain Map.get. The
-    // CPU budget is the run's concurrency; the memory budget is
-    // os.totalmem() unless `--memory` overrides it (pass `--memory` in
-    // cgroup-limited containers — totalmem() reports the HOST's RAM).
-    // Nothing declared → empty map → fields omitted from the scheduler
-    // AND the footer → byte-identical legacy path.
-    const memBudget = options.memory ?? os.totalmem()
-    const resourceCosts = resolveResourceCosts(nodes, concurrency, memBudget)
+    // Admission over the worker count is a plugin's (`admit`): built once
+    // into the predicate the scheduler asks at every local dispatch. No
+    // plugin answers → undefined → the count-only path, byte for byte.
+    const admit = buildAdmission(prepared.plugins, nodes, concurrency, (m) => log.status(m))
 
     // Run context for the footer. The top-of-run header is gone — the
     // banner now lives in the summary, where the eye lands at the end.
@@ -380,22 +507,27 @@ export async function run(options: RunOptions): Promise<RunSummary> {
       remoteCacheEnabled,
       concurrency,
       workspaceProjectCount,
-      ...(resourceCosts.size > 0 ? { cpuBudget: concurrency, memBudget } : {}),
     }
 
     // Lifecycle hooks drive the default logger's dynamic status line
     // (TTY-only); custom loggers may ignore them.
-    log.runStart?.({ total: taskCount, concurrency, requestedCount, context: runContext })
+    log.runStart?.({
+      total: taskCount,
+      concurrency,
+      requestedCount,
+      context: runContext,
+      startedAtMs: endedAtMsAtStart,
+    })
 
     // Remote-only: kick off background prefetches so remote-GET latency
     // overlaps execution. Fire-and-forget — execution starts on the next
-    // line; LayeredCache ingests hits into local and de-dups so
+    // line; the layer ingests hits into local and de-dups so
     // execute-task's cache.get awaits the in-flight promise (one remote
     // GET per key). Gated entirely on a remote layer being configured;
     // local-only runs never reach here, so their behavior + perf is
     // unchanged (no upfront key pass, no local probing).
     let prefetchDone: Promise<void> = Promise.resolve()
-    if (cache instanceof LayeredCache) {
+    if (prepared.hasRemoteLayer) {
       prefetchDone = startRemotePrefetch({
         nodes,
         cache,
@@ -416,9 +548,9 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     // restore-tier the scheduler runs ahead of their deps but only as
     // worker-slot backfill (misses own the pool) — and execute reuses
     // each probe, so there is no second cache.get. Gated by
-    // shouldShortCircuit (localRead + has-deps); when off, both maps are
-    // empty and the run is byte-identical.
-    let shortCircuit: ShortCircuit = EMPTY_SHORT_CIRCUIT
+    // shouldShortCircuit (local reads on, no remote layer); when off, both
+    // maps are empty and the run is byte-identical.
+    let shortCircuit: ShortCircuit = emptyShortCircuit()
     if (shouldShortCircuit(nodes, policy, cache)) {
       shortCircuit = await startLocalShortCircuit({
         nodes,
@@ -433,8 +565,14 @@ export async function run(options: RunOptions): Promise<RunSummary> {
       })
     }
 
+    // Whether this task runs behind a failure (`continueMode: 'always'`
+    // only) — its save is withheld and the taint propagates; see
+    // admission.ts.
+    const isTainted = taintTracker(options.continueMode === 'always')
+
     const buildExecuteArgs = (node: TaskNode, upstream: TaskOutcome[], reuseProbe = true) => {
       const probe = reuseProbe ? shortCircuit.preProbed.get(node.id) : undefined
+      const taint = isTainted(node, upstream)
       return {
         node,
         upstream,
@@ -445,8 +583,12 @@ export async function run(options: RunOptions): Promise<RunSummary> {
         forwardArgs: options.forwardArgs,
         ...(options.retries !== undefined ? { retries: options.retries } : {}),
         ...(taskTimeoutDefault !== undefined ? { timeout: taskTimeoutDefault } : {}),
-        ...(options.verify !== undefined ? { verify: options.verify } : {}),
         log,
+        executor: placements.executors.get(node.id) ?? UNPLACED_EXECUTOR,
+        ...(download.modeOf.get(node.id) === 'deferred' ? { download: 'deferred' as const } : {}),
+        deferred: deferredOutputs,
+        ...(placements.remoteOnlyNoop.has(node.id) ? { remoteOnlyNoop: true } : {}),
+        ...(placements.remoteOnly.has(node.id) ? { remoteOnly: true } : {}),
         nestedProjectDirs: nestedDirsByProject.get(node.projectName) ?? [],
         runStartHrTimeNs,
         persistentRegistry,
@@ -454,153 +596,72 @@ export async function run(options: RunOptions): Promise<RunSummary> {
         gitFilesCache,
         hashCache,
         ...(probe !== undefined ? { preProbed: probe } : {}),
+        ...(taint ? { taintedUpstream: true } : {}),
+        ...(sandboxArmer !== null ? { armSandbox: () => sandboxArmer.arm() } : {}),
+        outputDirSnapshots,
+        deferSave: saveLane.defer,
+        deferredSaves,
       }
     }
 
-    // In-flight dedup. Only when a service supplies a shared `inflight`
-    // registry (concurrent runs in one `vx serve`); a stateless `vx run`
-    // passes none and takes the untouched path. Gated to cacheable tasks —
-    // the join works by waiting for the sibling to populate the cache, then
-    // letting executeTask cache-hit on it. executeTask stays unchanged.
-    const inflight = options.inflight
-    const executeWithDedup = async (
-      node: TaskNode,
-      upstream: TaskOutcome[],
-    ): Promise<TaskOutcome> => {
-      // Dedup only helps when the sibling will WRITE the artifact and
-      // this task can READ it back — i.e. both axes effectively on.
-      const canRead = policy.localRead || policy.remoteRead
-      const canWrite = policy.localWrite || policy.remoteWrite
-      const cacheable =
-        !isGroupTask(node) &&
-        node.config.exec?.persistent === undefined &&
-        node.config.cache !== undefined &&
-        canRead &&
-        canWrite
-      // A restore-tier task (confirmed local hit, may run before its
-      // deps) needs no dedup — it's a restore, not an executor, and its
-      // live `upstream` is incomplete, so the dedup hash recompute would
-      // be wrong. Route it straight to executeTask, which reuses the
-      // up-front probe.
-      const restorable = shortCircuit.restoreTier.has(node.id)
-      if (inflight === undefined || !cacheable || restorable) {
-        return executeTask(buildExecuteArgs(node, upstream))
-      }
-      const hash = await computeTaskHash({
-        node,
-        upstream,
+    const executeWithDedup = admitTasks({
+      inflight: options.inflight,
+      policy,
+      shortCircuit,
+      hashArgs: {
         workspaceRoot,
         workspaceFingerprint,
         cache,
         forwardArgs: options.forwardArgs,
-        nestedProjectDirs: nestedDirsByProject.get(node.projectName) ?? [],
+        nestedDirsByProject,
         gitFilesCache,
         hashCache,
-      })
-      const existing = inflight.get(hash)
-      if (existing !== undefined) {
-        // Join a sibling already computing this exact task: wait, then
-        // executeTask cache-hits on the artifact it just saved. The
-        // up-front probe (preProbed) predates the sibling's save — its
-        // "confirmed stable miss" would skip the lazy cache.get and
-        // re-execute, defeating the dedup — so the join path drops it
-        // and lets executeTask probe fresh.
-        await existing.catch(() => {})
-        return executeTask(buildExecuteArgs(node, upstream, false))
-      }
-      // Become the executor: register a barrier siblings await. get→set has
-      // no await between, so registration is atomic — at most one executor
-      // per hash. Released on every exit (success / failure / throw).
-      let release!: () => void
-      inflight.set(
-        hash,
-        new Promise<void>((resolve) => {
-          release = resolve
-        }),
-      )
-      try {
-        return await executeTask(buildExecuteArgs(node, upstream))
-      } finally {
-        inflight.delete(hash)
-        release()
-      }
-    }
+      },
+      buildExecuteArgs,
+    })
 
+    mark('classify + probe')
     const outcomes = await runGraph({
       nodes,
       concurrency,
-      ...(resourceCosts.size > 0 ? { resourceCosts, cpuBudget: concurrency, memBudget } : {}),
+      settledOf: (o) => deferredSaves.get(o.node.id),
+      ...(hasPooledExecutor(executors) ? { poolOf: poolOfPlacement(placements) } : {}),
+      ...(admit !== undefined ? { admit } : {}),
       ...(options.continueMode !== undefined ? { continueMode: options.continueMode } : {}),
+      ...(options.signal !== undefined ? { signal: options.signal } : {}),
       onStart: (node) => {
         log.taskStart?.(node)
       },
       onFinish: (o) => {
         log.taskComplete(o.node, o)
+        narrowDemand(o.node.id)
       },
       execute: executeWithDedup,
-      // Predictive scheduling: empty map when not opted in, in which
-      // case the scheduler keeps the static baseline behavior.
-      priorities: prepared.priorities,
+      // A `schedule` plugin's weights; the scheduler keeps its structural
+      // baseline as the tie-break. Empty map → baseline only.
+      ...(prepared.priorities.size > 0 ? { priorities: prepared.priorities } : {}),
       // Local short-circuit: confirmed stable local hits the scheduler
       // runs ahead of their deps as low-priority worker-slot backfill.
       // Empty when the short-circuit didn't fire → byte-identical.
       restoreTier: shortCircuit.restoreTier,
     })
 
-    // A persistent task the user REQUESTED (a dev server / watcher) is
-    // the run's whole purpose — don't tear it down the instant it's
-    // ready. We leave those running and block on them at the very end
-    // (after the normal summary prints); everything else (persistent
-    // tasks pulled in only as dependencies of now-finished work) is
-    // SIGTERMed here as before. Scoped to the real CLI foreground:
-    // `options.log === undefined` means the default logger (a `vx run`
-    // invocation), and `handleSignals` excludes watch mode (own signal
-    // loop) and embedders that manage lifecycle themselves — both expect
-    // run() to return, not block on a server.
+    // Which persistent children outlive the graph, and the bounded SIGTERM
+    // of the rest, before the summary prints. Scoped to the real CLI
+    // foreground: `options.log === undefined` means the default logger (a
+    // `vx run` invocation), and `handleSignals` excludes watch mode (own
+    // signal loop) and embedders that manage lifecycle themselves — both
+    // expect run() to return, not block on a server.
     const foreground = options.log === undefined && (options.handleSignals ?? true)
-    const keepAliveNodes: TaskNode[] = []
-    const keepAlive: ReturnType<typeof Bun.spawn>[] = []
-    if (foreground) {
-      for (const [id, child] of persistentRegistry) {
-        const n = nodes.get(id)
-        if (n !== undefined && (n.requested || n.surfaced === true)) {
-          keepAliveNodes.push(n)
-          keepAlive.push(child)
-        }
-      }
-    }
-    const keepAliveSet = new Set(keepAlive)
-
-    // Shut down the dependency-only persistent tasks before reporting the
-    // final summary. SIGTERM gives well-behaved servers (vite, next,
-    // esbuild --watch) a moment to clean up; we don't escalate to SIGKILL
-    // — process-group propagation on Ctrl-C handles the unhappy case.
-    // Bun's Subprocess.kill is idempotent on an already-exited child.
-    for (const child of persistentRegistry.values()) {
-      if (!keepAliveSet.has(child)) child.kill('SIGTERM')
-    }
-    await Promise.allSettled(
-      [...persistentRegistry.values()].filter((c) => !keepAliveSet.has(c)).map((c) => c.exited),
-    )
+    const keepAlive = selectKeepAlive(persistentRegistry, nodes, foreground)
+    await shutdownPersistent(persistentRegistry, keepAlive.children)
 
+    mark('run graph')
     // Clear the status line for good before the summary prints.
     log.runEnd?.()
 
     const list = [...outcomes.values()]
-    const ok =
-      list.every(
-        (o) =>
-          o.status === 'success' || o.status === 'cache-hit' || o.status === 'cache-hit-remote',
-      ) &&
-      // `--verify`: a provably-unsafe cache entry (non-deterministic outputs, a
-      // re-run that failed, or a read of undeclared inputs) turns the run red so
-      // CI catches it.
-      !list.some(
-        (o) =>
-          o.verify?.kind === 'nondeterministic' ||
-          o.verify?.kind === 'rerun-failed' ||
-          o.verify?.kind === 'undeclared-inputs',
-      )
+    const ok = list.every((o) => isPassStatus(o.status))
 
     // The summary + artifact writers + recordRun pass all exclude group
     // tasks via the shared tallyOutcomes helper. We pass the full
@@ -609,27 +670,27 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     const totalMs = Number(process.hrtime.bigint() - runStartHrTimeNs) / 1_000_000
     // Foreground dev mode: between the task frame and the footer, list
     // the persistent tasks still running (see the keep-alive block below).
-    if (keepAliveNodes.length > 0) {
-      for (const line of formatPersistentList(keepAliveNodes, colors)) log.status(line)
+    if (keepAlive.nodes.length > 0) {
+      for (const line of formatPersistentList(keepAlive.nodes, colors)) log.status(line)
     }
     for (const line of formatRunSummary(list, totalMs, colors, runContext)) log.status(line)
-    if (options.verify !== undefined) {
-      for (const line of formatVerifySection(list)) log.status(line)
-      // A fingerprint-only run attaches no verdicts, so the verdict-driven
-      // section above prints nothing — report what actually happened.
-      if (options.verify.fingerprint && !options.verify.determinism && !options.verify.inputs) {
-        const n = list.filter((o) => o.outputFp !== undefined).length
-        log.status('')
-        log.status(
-          `  Verify:   fingerprinted ${n} task output trees (cross-machine diff via a connected serve)`,
-        )
-        // Only EXECUTED tasks fingerprint — a warm all-hit run reports 0.
-        // A per-platform matrix wired without `--force` produces nothing
-        // forever, so name the cause instead of a bare 0.
-        if (n === 0) {
-          log.status('            (0 executed — cache hits do not fingerprint; pair with --force)')
-        }
-      }
+    // A task killed by a shutdown signal is in no bucket above, yet it makes
+    // `ok` false — name it, or the red exit is undiagnosable.
+    for (const line of formatAbortedSection(list)) log.status(line)
+    // Judged against the history BEFORE this run's rows land, so the query
+    // is one scan over the executed tasks' keys and nothing at all on a run
+    // that executed none (every hit, every skip).
+    const flaky = detectFlaky(prepared.localCache.dbHandle(), flakyCandidates(list))
+    for (const line of formatFlakySection(flaky)) log.status(line)
+    // Outputs that never came home are not an error, but a silent `dist/`
+    // that is empty-or-stale would be: name every task whose bytes are
+    // still remote.
+    const stillDeferred = deferredOutputs.pending()
+    if (stillDeferred.length > 0) {
+      log.status('')
+      log.status(
+        `  Deferred: ${stillDeferred.length} task(s) left outputs remote (--download=none): ${stillDeferred.join(', ')}`,
+      )
     }
 
     // Optional artifacts. Errors are surfaced to the user but don't
@@ -644,7 +705,9 @@ export async function run(options: RunOptions): Promise<RunSummary> {
           startedAtMs: endedAtMsAtStart,
           endedAtMs,
           totalMs,
+          ok,
           outcomes: list,
+          flaky,
         })
         log.status(`vx: summary written to ${wrote}`)
       } catch (err) {
@@ -667,142 +730,52 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     }
 
     // Record each task to the run history in a single SQLite transaction
-    // (one fsync instead of N). Group tasks (no `exec`) are skipped —
-    // they aren't real runs and the `runs` table is analytics-focused.
-    // One invocation header row is written alongside, atomically via
-    // recordRunBundle. The Tier-3 input-fingerprint rows (entry_inputs)
-    // are NOT built here — they're persisted inside each entry's save
-    // transaction (miss path only), so a warm all-cache-hit run does no
-    // extra recording work.
-    const now = endedAtMs
-    const toRecord: RunRecord[] = []
-    // Per-task telemetry mirrors `toRecord` 1:1 — built only when a
-    // telemetry sink is active, so a no-telemetry run allocates nothing.
-    const summaryTasks: TaskTelemetry[] = []
-    let failedCount = 0
-    let hitLocalCount = 0
-    let hitRemoteCount = 0
-    for (const o of list) {
-      if (!o.hash) continue
-      if (isGroupTask(o.node)) continue
-      // aborted (killed by a shutdown signal) isn't a real run.
-      if (o.status === 'aborted') continue
-      if (telemetry !== undefined) {
-        const t: TaskTelemetry = {
-          taskId: o.node.id,
-          project: o.node.projectName,
-          task: o.node.taskName,
-          status: o.status,
-          cacheSource: deriveCacheSource(o.status),
-          exitCode: o.exitCode,
-          durationMs: o.durationMs,
-        }
-        if (o.hash !== undefined) t.hash = o.hash
-        if (o.cpuMs !== undefined) t.cpuMs = o.cpuMs
-        if (o.peakRssBytes !== undefined) t.peakRssBytes = o.peakRssBytes
-        if (o.attempts !== undefined) t.attempts = o.attempts
-        if (o.verify !== undefined) t.verify = o.verify
-        if (o.outputFp !== undefined) t.outputFp = o.outputFp
-        if (o.wallclockStartNs !== undefined) t.wallclockStartNs = o.wallclockStartNs.toString()
-        if (o.wallclockEndNs !== undefined) t.wallclockEndNs = o.wallclockEndNs.toString()
-        summaryTasks.push(t)
-      }
-      toRecord.push({
-        hash: o.hash,
-        project: o.node.projectName,
-        task: o.node.taskName,
-        status: o.status,
-        exitCode: o.exitCode,
-        durationMs: o.durationMs,
-        ...(options.forwardArgs !== undefined ? { forwardArgs: options.forwardArgs } : {}),
-        // Anchor to the REAL per-task wall-clock window: run-start wall time +
-        // the task's ns offset (captured for hits and executed tasks alike).
-        // The `now - duration` fallback only applies to outcomes without an
-        // offset (none today). Using run-end-minus-duration for everything was
-        // the old bug that piled every task at the right edge of the timeline.
-        startedAt:
-          o.wallclockStartNs !== undefined
-            ? endedAtMsAtStart + Math.round(Number(o.wallclockStartNs) / 1e6)
-            : now - o.durationMs,
-        endedAt:
-          o.wallclockEndNs !== undefined
-            ? endedAtMsAtStart + Math.round(Number(o.wallclockEndNs) / 1e6)
-            : now,
-        runId,
-        ...(o.cpuMs !== undefined ? { cpuMs: o.cpuMs } : {}),
-        ...(o.peakRssBytes !== undefined ? { peakRssBytes: o.peakRssBytes } : {}),
-        ...(o.wallclockStartNs !== undefined ? { wallclockStartNs: o.wallclockStartNs } : {}),
-        ...(o.wallclockEndNs !== undefined ? { wallclockEndNs: o.wallclockEndNs } : {}),
-        cacheHit: o.status === 'cache-hit' || o.status === 'cache-hit-remote',
-        ...(o.attempts !== undefined ? { attempts: o.attempts } : {}),
-      })
-      if (o.status === 'failed') failedCount++
-      if (o.status === 'cache-hit') hitLocalCount++
-      if (o.status === 'cache-hit-remote') hitRemoteCount++
-    }
-    const invocation: InvocationRecord = {
+    // (one fsync instead of N), with the invocation header row alongside,
+    // atomically via recordRunBundle. The Tier-3 input-fingerprint rows
+    // (entry_inputs) are NOT built here — they're persisted inside each
+    // entry's save transaction (miss path only), so a warm all-cache-hit
+    // run does no extra recording work. The telemetry mirror is built in
+    // the same pass, only when a sink is active.
+    const records = assembleRunRecords({
+      outcomes: list,
       runId,
+      startedAtMs: endedAtMsAtStart,
+      endedAtMs,
+      totalMs,
+      ok,
       command: options.command ?? process.argv.slice(1).join(' '),
-      requestedTasks: JSON.stringify([...options.tasks]),
+      requestedTasks: options.tasks,
       cachePolicy: compactCachePolicy(policy),
       concurrency,
       flow: options.flow ?? null,
-      startedAt: endedAtMsAtStart,
-      endedAt: endedAtMs,
-      totalDurationMs: Math.round(totalMs),
-      taskCount: toRecord.length,
-      failedCount,
-      hitCount: hitLocalCount + hitRemoteCount,
-      hitLocalCount,
-      hitRemoteCount,
-      exitOk: ok,
-      commitSha: gitContext.commitSha,
-      branch: gitContext.branch,
-      dirty: gitContext.dirty,
-      ci: ciContext.ci,
-      ciProvider: ciContext.provider,
-      host: hostContext.host,
-      os: hostContext.os,
-      arch: hostContext.arch,
-      vxVersion: VERSION,
-      tags: JSON.stringify(options.tags ?? {}),
-    }
-    cache.recordRunBundle({ runs: toRecord, invocation })
+      forwardArgs: options.forwardArgs,
+      tags: options.tags ?? {},
+      git: gitContext,
+      ci: ciContext,
+      host: hostContext,
+      withTelemetry: telemetry !== undefined,
+    })
+    cache.recordRunBundle(records)
+    mark('record history')
     // Hand the per-run summary to the telemetry sinks + drain them. Only
     // when a sink is active (telemetry !== undefined) — otherwise this
     // whole block is skipped and the run is byte-identical to before.
     // emitSummary/flush are crash-isolated, so a faulty sink can't fail
     // the run; flush is the sink's last chance to ship buffered records.
     if (telemetry !== undefined && runContextRecord !== undefined) {
-      const summary: RunSummaryRecord = {
-        v: TELEMETRY_SCHEMA_VERSION,
-        run: runContextRecord,
+      const summary = assembleRunSummary(runContextRecord, records.telemetryTasks, {
         startedAt: endedAtMsAtStart,
         endedAt: endedAtMs,
         totalDurationMs: Math.round(totalMs),
-        taskCount: toRecord.length,
-        failedCount,
-        hitCount: hitLocalCount + hitRemoteCount,
-        hitLocalCount,
-        hitRemoteCount,
         exitOk: ok,
-        tasks: summaryTasks,
-      }
+      })
       telemetry.emitSummary(summary)
       await telemetry.flush()
     }
-    // End-of-run plugin lifecycle: each event sink's flush() (its last
-    // chance to ship buffered records) and each plugin's teardown().
-    // Crash-isolated + time-bounded inside teardownPlugins, so a faulty
-    // plugin can neither fail nor hang the run. Normal completion path
-    // only — the finally below just unsubscribes.
-    if (prepared.workspaceConfig?.plugins && prepared.workspaceConfig.plugins.length > 0) {
-      await teardownPlugins(
-        prepared.workspaceConfig.plugins as readonly VxPlugin[],
-        eventSinks?.sinks ?? [],
-        (m) => log.status(m),
-      )
-    }
+    // End-of-run plugin lifecycle: each plugin's teardown(). Crash-isolated
+    // + time-bounded inside teardownPlugins, so a faulty plugin can neither
+    // fail nor hang the run. Normal completion path only — the finally
+    // below just unsubscribes.
     // Drain any still-in-flight background prefetches before closing the
     // cache handle — a prefetch ingesting into a closed SQLite DB would
     // throw. Tasks that resolved as local hits never awaited their
@@ -811,14 +784,41 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     // before scheduling, so nothing of its is in flight here.) The
     // background write-through uploads queued by LayeredCache.save
     // settle here for the same reason.
+    //
+    // BEFORE teardownPlugins, and that order is load-bearing: the drain
+    // pushes bytes through a layer a PLUGIN provided, so a plugin that
+    // releases its client in `teardown()` would otherwise have the
+    // channel shut from under the upload — silently losing every remote
+    // write with nothing but a warning. The seam's contract is that a
+    // cache layer stays usable until the run's uploads have settled.
     await prefetchDone
-    if (cache instanceof LayeredCache) await cache.drainUploads()
-    cache.close()
+    // Every deferred save settles first: the upload drain below carries
+    // the entries the saves queued, and the snapshot loop after it reads
+    // what the saves pushed.
+    await saveLane.drain()
+    mark('save lane')
+    await cache.drainUploads?.()
+    // The miss path's output-directory snapshots, taken now that the
+    // directories are old enough for the snapshot's racy window (see
+    // miss-save.ts). A few at a time: each is an lstat + readdir per
+    // prefix and one index transaction.
+    for (let i = 0; i < outputDirSnapshots.length; i += 32) {
+      await Promise.all(
+        outputDirSnapshots
+          .slice(i, i + 32)
+          .map((s) => cache.recordOutputDirs?.(s.hash, s.projectDir, s.prefixes)),
+      )
+    }
+    mark('output dir snapshots')
+    await teardownPlugins(prepared.plugins, (m) => log.status(m))
+    closeCache()
+    mark('close')
+    printTimings()
 
     // Tear down SRT's network bridge + (on macOS) log monitor. No-op if
     // no task was sandboxed; otherwise SRT keeps proxy servers alive and
     // the next vx run would init on top of stale state.
-    if (anySandboxed) {
+    if (sandboxArmer?.armed) {
       try {
         await resetSandbox()
       } catch (err) {
@@ -831,12 +831,30 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     // persistent task (dev server / watcher). The run is "done" in every
     // bookkeeping sense — summary printed, history recorded — but the
     // server is still up and that's the point. Stay in the foreground
-    // until it exits: Ctrl-C hits the whole process group (the server
-    // dies; our SIGINT handler also exits 130), and a crash resolves the
-    // wait so the run returns. Nothing here prints — the UI is unchanged.
-    if (keepAlive.length > 0) {
-      await Promise.allSettled(keepAlive.map((c) => c.exited))
-      for (const child of keepAlive) child.kill('SIGTERM')
+    // until ONE of them exits: Ctrl-C hits the whole process group (the
+    // server dies; our SIGINT handler also exits 130), and a crash ends
+    // the session — the others are torn down (SIGTERM, grace, SIGKILL)
+    // and a non-zero exit makes the run not ok, so `vx run dev` in a
+    // script fails when the server it started fell over. Until
+    // 2026-09-10 this waited for EVERY server, so the SIGTERM after it
+    // was dead code and a crashed server left the rest running under a
+    // run that never returned. One status line names the server that
+    // ended the session and its code — the summary above said `success`
+    // for it, and an exit 1 with no word about why is a mystery in a log.
+    if (keepAlive.children.length > 0) {
+      const first = await Promise.race(
+        keepAlive.children.map((c, i) => c.exited.then((code) => ({ code, i }))),
+      )
+      const node = keepAlive.nodes[first.i]!
+      const others = keepAlive.nodes.length - 1
+      log.status(
+        `vx: ${node.id} exited with code ${first.code}` +
+          (others > 0
+            ? `; stopping ${others} other persistent task${others === 1 ? '' : 's'}`
+            : ''),
+      )
+      await terminateChildren(() => keepAlive.children)
+      return { ok: ok && first.code === 0, outcomes: list }
     }
 
     return { ok, outcomes: list }
@@ -844,13 +862,22 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     // Idempotent; also reached on mid-run throws, so a crashed cycle
     // can't leave a live status-line ticker behind.
     log.runEnd?.()
-    process.off('SIGINT', onSigint)
-    process.off('SIGTERM', onSigterm)
+    signals.remove()
+    options.signal?.removeEventListener('abort', onAbort)
     // Plugins installed at the top of run() get their bus subscriptions
     // released here. Idempotent; safe even if installPlugins threw.
     disposePlugins?.()
-    eventSinks?.dispose()
     telemetry?.dispose()
+    // No-op after the normal path's close. On a throw this is the only
+    // close there is, and it must not itself throw — that would replace
+    // the run's real error with a teardown one. Background uploads are
+    // NOT drained here: they hold no database state, and awaiting a
+    // wedged remote would turn a failing run into a hanging one.
+    try {
+      closeCache()
+    } catch {
+      // teardown must not throw on the way out
+    }
   }
 }
 
@@ -869,19 +896,109 @@ export async function planRun(options: RunOptions): Promise<RunPlan> {
   const log = options.log ?? defaultLogger()
   const prepared = await prepareRun(options, log)
   try {
+    if (prepared.unresolvedTasks.length > 0) {
+      return { tasks: [], unresolvedTasks: prepared.unresolvedTasks }
+    }
     if (prepared.empty !== null) return { tasks: [] }
     return await plan({
       nodes: prepared.nodes,
       workspaceRoot: prepared.workspaceRoot,
       workspaceFingerprint: prepared.workspaceFingerprint,
       cache: prepared.cache,
-      cachePolicy: options.cache ?? FULL_CACHE_POLICY,
+      cachePolicy: effectiveCachePolicy(
+        options.cache ?? FULL_CACHE_POLICY,
+        prepared.hasRemoteLayer,
+      ),
       forwardArgs: options.forwardArgs,
       nestedDirsByProject: prepared.nestedDirsByProject,
       gitFilesCache: prepared.gitFilesCache,
       hashCache: prepared.hashCache,
+      // Plan-time duration prediction (dev-scenarios S2): the same local
+      // history the predictive scheduler reads, folded into the plan as
+      // per-task p50s + a predicted wall-clock. `--dry` is an explicit
+      // inspection command, so the history read's cost is fine here even
+      // though it stays OFF the default run path.
+      history: new LocalHistoryProvider(prepared.localCache.dbHandle()),
+      // Placement, by the same rules the run applies. Only worth showing
+      // when there is a choice to show — with one declared executor every
+      // line would carry the same label. Resolving the executors here is
+      // the same plugin-factory call `prepareRun` already makes for the
+      // cache capability, so plan mode gains no new class of side effect.
+      ...(await planExecutorOf(prepared, log, options.download ?? 'all')),
     })
   } finally {
     prepared.cache.close()
+    // The same table a run prints: a dry run is how the prepare stages
+    // (discovery, config load, the git enumeration) get profiled on a
+    // real repo that has no install and no cache to run against.
+    mark('close')
+    printTimings()
+  }
+}
+
+/**
+ * The first-run case, told apart from a typo: no package in the workspace
+ * has a `vx.config.*` at all, so no name could have resolved. Reached only
+ * on the error path.
+ */
+function initHint(prepared: { anyProjectConfig: boolean }): string {
+  return prepared.anyProjectConfig
+    ? ''
+    : ' No package declares a vx.config — run `vx init` to write one per package from its package.json scripts.'
+}
+
+/**
+ * A typo's nearest declared name, when one is within two edits — the
+ * message names the fix instead of only the mistake. A bare name is
+ * matched against every declared task; `pkg#task` against the project
+ * names first (the task kept) and then against that project's tasks, so
+ * the hint is a spec the user can run.
+ */
+function didYouMean(
+  unresolved: readonly string[],
+  projects: ReadonlyMap<string, ProjectEntry>,
+): string {
+  const tasksOf = (p: ProjectEntry | undefined): string[] => Object.keys(p?.config.tasks ?? {})
+  const allTasks = new Set<string>()
+  for (const p of projects.values()) for (const t of tasksOf(p)) allTasks.add(t)
+  // A Set: two typos of the same task hint it once, not once per typo.
+  const hints = new Set<string>()
+  for (const spec of unresolved) {
+    const at = spec.indexOf('#')
+    if (at < 0) {
+      const t = nearest(spec, allTasks)
+      if (t !== undefined) hints.add(t)
+      continue
+    }
+    const [proj, task] = [spec.slice(0, at), spec.slice(at + 1)]
+    if (!projects.has(proj)) {
+      const p = nearest(proj, projects.keys())
+      if (p !== undefined && tasksOf(projects.get(p)).includes(task)) hints.add(`${p}#${task}`)
+      continue
+    }
+    const t = nearest(task, tasksOf(projects.get(proj)))
+    if (t !== undefined) hints.add(`${proj}#${t}`)
+  }
+  return hints.size === 0 ? '' : ` Did you mean ${[...hints].join(', ')}?`
+}
+
+/** The executed, keyed outcomes of a run — what flakiness is judged on. */
+function flakyCandidates(outcomes: readonly TaskOutcome[]): FlakyCandidate[] {
+  const out: FlakyCandidate[] = []
+  for (const o of outcomes) {
+    if (o.status !== 'success' && o.status !== 'failed') continue
+    // "Same inputs, different outcome" is a claim only a task with declared
+    // inputs can make: a task with no `cache` block keys on its config alone
+    // and runs every time, so one bad network day would read as a flake for
+    // thirty days. Groups do no work.
+    if (o.hash === undefined || o.node.config.cache === undefined || isGroupTask(o.node)) continue
+    out.push({
+      project: o.node.projectName,
+      task: o.node.taskName,
+      hash: o.hash,
+      status: o.status,
+      attempts: o.attempts ?? 1,
+    })
   }
+  return out
 }