@vzn/vx 0.0.652 → 0.0.653

package/src/orchestrator/run.ts

@@ -4,7 +4,15 @@
 
 import type { InputStability } from '../config.js'
 import type { ProjectEntry } from '../workspace/index.js'
-import { EMPTY_LOCK, inputStabilityEnv, LOCK_INVALID, lockEntryBroken } from '../workspace/index.js'
+import {
+  EMPTY_LOCK,
+  findProjectConfigs,
+  inputStabilityEnv,
+  LOCK_INVALID,
+  lockEntryBroken,
+  lockKey,
+  readLockfile,
+} from '../workspace/index.js'
 import { realpathSync } from 'node:fs'
 import path from 'node:path'
 import {
@@ -14,6 +22,7 @@ import {
   type CachePolicy,
   cachesNothing,
   formatCachePolicy,
+  holdCacheDir,
   HistoryStream,
   networkConcurrency,
   keepRunNotes,
@@ -29,6 +38,7 @@ import {
   VX_RUN_WORKSPACE_ENV,
 } from '../exec/index.js'
 import { DeferredOutputs } from './deferred-outputs.js'
+import { workspaceCleanOf } from './task-hash.js'
 import { resolveDownloadModes } from './download-policy.js'
 import type { MachineReport, TaskExecutor, WorkerAdvice } from '../exec/index.js'
 import {
@@ -59,6 +69,7 @@ import {
   isUserError,
   SecretBroker,
   thrownMessage,
+  relPosix,
 } from '../util/index.js'
 import * as ui from '../ui/index.js'
 
@@ -68,7 +79,7 @@ function printTimings(): void {
   if (lines !== undefined) ui.print('err', ui.trace(lines))
 }
 import { detectColors, type ColorSupport } from '../ui/index.js'
-import { keyedProjects, readableDeps } from './keyed-projects.js'
+import { keyedProjects, nestedDirsOf, readableDeps } from './keyed-projects.js'
 import { isDefaultBuild } from '../graph/index.js'
 import { commandWriteReach, prepareSandbox } from './sandbox-request.js'
 import { recordOutputDirSnapshots, type OutputDirSnapshot, type SaveFacts } from './miss-save.js'
@@ -89,13 +100,16 @@ import {
   formatPersistentList,
   formatRunSummary,
   type Logger,
+  neverStarted,
   plainOutput,
   resolveOutputView,
+  type RunEvent,
   terminalSubscriber,
   warnOf,
 } from '../report/index.js'
 import { installPlugins, PluginSetupError, type RunReport } from './plugin.js'
 import { keepsFailures, reportOf, RunKeeper } from './run-report.js'
+import { freezeDeep } from './read-only.js'
 import {
   buildAdmission,
   executorDemand,
@@ -112,9 +126,21 @@ import { isPassStatus } from '../graph/index.js'
 import type { RunContextRecord } from './telemetry.js'
 import { plan, type RunPlan } from './plan.js'
 import type { AffectedReason } from './affected-tasks.js'
-import { namesRemote, openRemotes, prepareRun, type PreparedRun } from './prepare.js'
-import { acquireRunLock } from './run-lock.js'
-import { terminateChildren } from './signals.js'
+import {
+  namesRemote,
+  openRemotes,
+  outputWallsOf,
+  prepareRunLocked,
+  type PreparedRun,
+} from './prepare.js'
+import { acquireRunLock, runLockPath, type RunLockRelease } from './run-lock.js'
+import {
+  exitOfSignal,
+  type HeldSignals,
+  holdSignals,
+  terminateChildren,
+  unlessStopped,
+} from './signals.js'
 import { CYCLE_INTERRUPT, runStop } from './run-stop.js'
 import {
   hasPooledExecutor,
@@ -153,7 +179,10 @@ import {
   failServer,
   holdServers,
   sessionEnd,
+  serverUsers,
+  stopServer,
   takeHeldServers,
+  unneededServers,
 } from './persistent.js'
 import { serverKey } from './persistent-task.js'
 import { writeRunProfile } from './run-artifacts.js'
@@ -389,11 +418,21 @@ export async function run(options: RunOptions): Promise<RunSummary> {
           unsubscribeTerminal = bus.subscribe(guardedOutput(claimed, render))
         }
   let held = false
+  const signals: { held?: HeldSignals } = {}
   try {
     // vx's own steps are timed only where something renders them.
     const showsSteps =
       terminal !== null ? terminal.showsSteps : options.log?.stepStart !== undefined
-    const summary = await runOnBus(options, bus, warned, colors, terminal, showsSteps, takeOutput)
+    const summary = await runOnBus(
+      options,
+      bus,
+      warned,
+      colors,
+      terminal,
+      showsSteps,
+      signals,
+      takeOutput,
+    )
     if (summary.persistent === undefined) return summary
     // Servers handed back still running still write: `vx watch dev` printed
     // none of its server's log while it idled, the renderer gone with this
@@ -409,6 +448,11 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     }
   } finally {
     if (!held) detach()
+    // A signal heard before the run's stop took over: the run left before
+    // its tasks (a refusal, a setup that threw), its plugins torn down and
+    // its lock released, and vx leaves as that signal says.
+    const heard = signals.held?.takeOver()
+    if (heard !== undefined) await exitOfSignal(heard)
   }
 }
 
@@ -419,6 +463,7 @@ async function runOnBus(
   colors: ColorSupport,
   terminal: DefaultLogger | null,
   showsSteps: boolean,
+  signals: { held?: HeldSignals },
   takeOutput?: (claimed: { logger: Logger; owner: string }) => void,
 ): Promise<RunSummary> {
   const log = busLogger(bus)
@@ -427,17 +472,82 @@ async function runOnBus(
   // A server's output goes through a cell of its own: a later watch cycle
   // that keeps the server points it at that cycle's bus (`RunOptions.keep`).
   const serverOuts = new Map<string, { bus: EventBus }>()
+  // What a server writes from ready until its outcome is logged: renderers
+  // keep output before the outcome for its frame, which a ready server's
+  // row never prints, so its first lines since ready were lost.
+  const sinceReady = new Map<string, RunEvent[]>()
   const serverLog = (id: string): Logger => {
     const out = { bus }
     serverOuts.set(id, out)
+    const emit = (event: RunEvent): void => {
+      const held = sinceReady.get(id)
+      if (held === undefined) out.bus.emit(event)
+      else held.push(event)
+    }
     return {
       ...log,
-      taskStdout: (node, chunk) => out.bus.emit({ kind: 'task:stdout', node, chunk }),
-      taskStderr: (node, chunk) => out.bus.emit({ kind: 'task:stderr', node, chunk }),
+      taskStdout: (node, chunk) => emit({ kind: 'task:stdout', node, chunk }),
+      taskStderr: (node, chunk) => emit({ kind: 'task:stderr', node, chunk }),
     }
   }
+  const releaseSinceReady = (id: string): void => {
+    const held = sinceReady.get(id)
+    if (held === undefined) return
+    sinceReady.delete(id)
+    const out = serverOuts.get(id) ?? { bus }
+    for (const event of held) out.bus.emit(event)
+  }
 
-  const prepared = await step('vx:load', () => prepareRun(options, log))
+  // Taken before the cache opens, never after: a run that held the store
+  // while it waited here, and a `vx cache clean` that holds this lock while
+  // it waits for every process with the store open, waited on each other
+  // for ever; and a clear deleted the store under a run opening it.
+  let heldLock: RunLockRelease | undefined
+  const takeRunLock = async (root: string, sharedCacheDir: string | undefined): Promise<void> => {
+    // A nested run is refused below with its own reason, holding nothing.
+    const outer = process.env[VX_RUN_WORKSPACE_ENV] || undefined
+    if (outer !== undefined && canonical(outer) === canonical(root)) return
+    // Heard from here, not only from the stop below: a signal between the
+    // lock and that stop killed vx with the lock held (`holdSignals`).
+    if (options.handleSignals ?? true) signals.held = holdSignals()
+    const held = signals.held?.signal
+    const signal =
+      held !== undefined && options.signal !== undefined
+        ? AbortSignal.any([options.signal, held])
+        : (held ?? options.signal)
+    const lock = await step('vx:run-lock', () => acquireRunLock(root, { log: warnOf(log), signal }))
+    heldLock = lock
+    // Another workspace's clean of a shared cache directory takes no lock of
+    // this one's: it waits for this hold, and this for it. Released with the
+    // run lock.
+    if (sharedCacheDir !== undefined) {
+      const dir = await holdCacheDir(sharedCacheDir, 'shared', warnOf(log), signal)
+      heldLock = Object.assign(
+        async (): Promise<void> => {
+          dir()
+          await lock()
+        },
+        { path: lock.path },
+      )
+    }
+    // A wait the signal ended holds no lock: opening the store now could
+    // meet a `vx cache clean` deleting it. run() exits of the signal.
+    if (held?.aborted === true) throw new Error(`stopped by ${String(held.reason)}`)
+  }
+  let prepared: PreparedRun
+  try {
+    prepared = await step('vx:load', () => prepareRunLocked(options, log, takeRunLock, true))
+  } catch (err) {
+    await heldLock?.()
+    throw err
+  }
+  const releaseRunLock: RunLockRelease =
+    heldLock ?? Object.assign(async () => {}, { path: runLockPath(prepared.workspaceRoot) })
+  // The cache and the lock go together on every exit before the run's own.
+  const closeUnrun = async (): Promise<void> => {
+    prepared.cache.close()
+    await releaseRunLock()
+  }
   mark('plugin stages')
   // Every exit from here tears the plugins down: their capability
   // factories ran in prepareRun, and teardown is what releases what those
@@ -465,12 +575,11 @@ async function runOnBus(
         ? { workspace: prepared.workspaceConfig.inputs.stability }
         : {}),
     }
-    cleanOutputs = resolveSetting('outputs.clean', prepared.workspaceConfig).value as
-      | boolean
-      | undefined
+    // Resolved once, beside the keys that fold it (prepare.ts).
+    cleanOutputs = workspaceCleanOf(prepared.hashCache)
   } catch (err) {
     await teardown()
-    prepared.cache.close()
+    await closeUnrun()
     throw err
   }
   // A task whose command re-enters `vx run` in the workspace running it
@@ -486,7 +595,7 @@ async function runOnBus(
   const outerRoot = process.env[VX_RUN_WORKSPACE_ENV] || undefined
   if (outerRoot !== undefined && canonical(outerRoot) === canonical(prepared.workspaceRoot)) {
     await teardown()
-    prepared.cache.close()
+    await closeUnrun()
     throw new UserError(
       `task ${process.env[VX_RUN_TASK_ENV] ?? '<unknown>'} runs \`vx run\` inside its own workspace: a nested run is invisible to the outer graph (its tasks escape the schedule, the concurrency budget and the cache key) and a loop back to this task forks without bound. Declare what it needs with dependsOn instead.`,
     )
@@ -502,13 +611,13 @@ async function runOnBus(
   if (prepared.unresolvedTasks.length > 0) {
     const refused = `no projects declare task(s): ${prepared.unresolvedTasks.join(', ')}.${await unresolvedHint(prepared)}`
     await teardown()
-    prepared.cache.close()
+    await closeUnrun()
     return { ok: false, outcomes: [], refused }
   }
   if (prepared.empty === 'none-affected') {
     log.status(noneAffected(prepared, options))
     await teardown()
-    prepared.cache.close()
+    await closeUnrun()
     return { ok: true, outcomes: [] }
   }
   if (prepared.empty !== null) {
@@ -521,7 +630,7 @@ async function runOnBus(
         ? `no projects declare task(s): ${options.tasks.join(', ')}.${elsewhereHint(prepared.declaredElsewhere)}${await initHint(prepared)}`
         : 'no tasks to run.'
     await teardown()
-    prepared.cache.close()
+    await closeUnrun()
     return { ok: false, outcomes: [], refused }
   }
   // Install user plugins as additional bus subscribers BEFORE the run
@@ -534,6 +643,7 @@ async function runOnBus(
   // carries it so analytics queries can group by invocation; plugins get it
   // in their teardown's `RunReport`.
   const runId = ulid()
+  prepared.localCache.runId = runId
   // A killed run's restore staging (`.vx/tmp/<pid>-0`), once per run.
   sweepStaging(prepared.workspaceRoot)
   const head = captureGitContext(prepared.workspaceRoot)
@@ -553,7 +663,7 @@ async function runOnBus(
       : new RunKeeper({
           runId,
           workspaceRoot: prepared.workspaceRoot,
-          nestedDirsByProject: prepared.nestedDirsByProject,
+          nestedDirsByProject: outputWallsOf(prepared.nestedDirsByProject),
           keep: prepared.keepRunEntry,
           failures: keepRun.failures,
           traffic: prepared.localCache.traffic,
@@ -567,6 +677,7 @@ async function runOnBus(
     terminal === null ? undefined : () => terminal.holdStatus(),
   )
   const remotesClosed = await openRemotes(prepared.remotes, ciContext.ci, secrets)
+  signals.held?.arm()
   try {
     disposePlugins = await installPlugins({
       plugins: prepared.plugins,
@@ -579,11 +690,12 @@ async function runOnBus(
       branch: prepared.branch(),
       secrets,
       step,
+      signal: signals.held?.signal,
     })
   } catch (err) {
     disposePlugins?.()
     await teardown(err instanceof PluginSetupError ? err.plugin : undefined)
-    prepared.cache.close()
+    await closeUnrun()
     throw err
   }
   // Until the run's own try below, a throw leaves through here.
@@ -593,7 +705,7 @@ async function runOnBus(
     } catch (err) {
       disposePlugins?.()
       await teardown()
-      prepared.cache.close()
+      await closeUnrun()
       throw err
     }
   }
@@ -621,22 +733,26 @@ async function runOnBus(
     if (options.signal?.aborted === true) {
       disposePlugins?.()
       await teardown()
-      prepared.cache.close()
+      await closeUnrun()
       return { ok: true, outcomes: [] }
     }
   }
 
   // Resolved ONCE per run, in declaration order, the local executor last.
-  // A broken factory aborts here, before any task starts.
+  // A broken factory aborts here, before any task starts, and a held
+  // signal ends the wait on one that never settles.
   const executors: readonly TaskExecutor[] = await abandoning(() =>
-    resolveExecutors(prepared.plugins, {
-      workspaceRoot: prepared.workspaceRoot,
-      cacheDir: prepared.cacheDir,
-      warn: warnOf(log),
-      concurrency,
-      artifacts: taskArtifacts(prepared.cache, prepared.workspaceRoot),
-      cacheFacts: () => cacheFacts(prepared.remotes, prepared.cacheRefs),
-    }),
+    unlessStopped(
+      resolveExecutors(prepared.plugins, {
+        workspaceRoot: prepared.workspaceRoot,
+        cacheDir: prepared.cacheDir,
+        warn: warnOf(log),
+        concurrency,
+        artifacts: taskArtifacts(prepared.cache, prepared.workspaceRoot),
+        cacheFacts: () => cacheFacts(prepared.remotes, prepared.cacheRefs),
+      }),
+      signals.held?.signal,
+    ),
   )
   // Placement: decided ONCE per task, before scheduling, so the scheduler
   // can admit a remote-pooled task against its pool instead of a local
@@ -688,26 +804,28 @@ async function runOnBus(
   // `cache.download` (default `all`) decides ONCE per task, here, whether a
   // remote execution's outputs come home.
   const downloadPolicy = options.download ?? 'all'
-  const localPlaced = locallyPlaced(placements, nodes.keys())
-  const download = resolveDownloadModes({
-    nodes,
-    policy: downloadPolicy,
-    localPlaced,
-    remoteOnly: placements.remoteOnly,
-  })
+  const localPlaced = locallyPlaced(placements)
+  // `all` keeps every task eager: no per-task map to say so, as `--dry`'s.
+  const download =
+    downloadPolicy === 'all'
+      ? undefined
+      : resolveDownloadModes({
+          nodes,
+          policy: downloadPolicy,
+          localPlaced,
+          remoteOnly: placements.remoteOnly,
+        })
 
   // `run.timeout` (ms), for any task WITHOUT its own `exec.timeout`.
   const taskTimeoutDefault = options.timeout
 
-  // The held servers this run took over, spared by a `CYCLE_INTERRUPT`.
-  const handed = new Set<ReturnType<typeof Bun.spawn>>()
   const stop = runStop({
-    handed: (child) => handed.has(child),
     log,
     cache,
     external: options.signal,
     handleSignals: options.handleSignals ?? true,
     handsTerminal: holders.size > 0,
+    held: signals.held,
     teardowns: prepared.plugins.filter((p) => p.teardown).length,
     onStop: () => {
       log.runStop?.()
@@ -731,6 +849,7 @@ async function runOnBus(
   const closeCache = async (): Promise<void> => {
     if (cacheClosed) return
     cacheClosed = true
+    await prepared.localCache.stopCheckpoints()
     cache.close()
     live?.unlisten()
     // Released with the handle: before a persistent task's wait, on every
@@ -742,16 +861,6 @@ async function runOnBus(
     // waited, but a directory per run is litter the box never sheds.
     await releaseRunLock()
   }
-  // Taken after the early exits above (nothing they do touches a tree) and
-  // before the schedule: from here on tasks clean, restore and write.
-  const releaseRunLock = await abandoning(() =>
-    step('vx:run-lock', () =>
-      acquireRunLock(prepared.workspaceRoot, {
-        log: warnOf(log),
-        signal: stop.signal,
-      }),
-    ),
-  )
   // Answers `vx last --live` while the lock is held (run-live.ts).
   live = serveLive({
     socketPath: liveSocketOf(releaseRunLock.path),
@@ -774,26 +883,17 @@ async function runOnBus(
     // clean their outputs before every exec for a save that never
     // happened.
     const policy: CachePolicy = effectiveCachePolicy(prepared.cachePolicy, remoteCacheEnabled)
-    // The workspace's `cache.policy` asks as a flag does: it was the one source never told.
-    const remoteNamed =
-      options.remoteRequested ??
-      (options.cache === undefined &&
-      options.cacheFlags?.noCache !== true &&
-      namesRemote(prepared.workspaceConfig?.cache?.policy)
-        ? 'cache.policy'
-        : undefined)
-    if (remoteNamed !== undefined && !prepared.hasRemoteLayer) {
-      warnOf(log)(
-        `vx: ${remoteNamed} named the remote cache, but no cache plugin in .vx/workspace.* supplies one — the remote axes are off for this run`,
-      )
-    }
+    warnNoRemote(prepared, options, log)
+    const windows = options.commandWindows ?? commandWindows()
     const deferredOutputs = new DeferredOutputs({
       nodes,
       cache,
       workspaceRoot,
-      nestedDirsByProject,
+      nestedDirsByProject: outputWallsOf(nestedDirsByProject),
       ...(gitFilesCache !== undefined ? { gitFilesCache } : {}),
       localWrite: policy.localWrite,
+      log,
+      commandWindows: windows,
     })
 
     // Per-run context for the Tier-3 `invocations` header row. Captured
@@ -874,7 +974,7 @@ async function runOnBus(
             (n) =>
               n.config.exec?.sandbox !== undefined &&
               (n.config.exec.persistent === undefined
-                ? localPlaced.has(n.id)
+                ? localPlaced(n.id)
                 : !placements.carried.has(n.id)),
           )
     let prewarming: Promise<void> | undefined
@@ -887,12 +987,19 @@ async function runOnBus(
     const outputDirSnapshots: OutputDirSnapshot[] = []
     // Saves run off the execution slot, twice the cap at once (memory:
     // each pack holds an artifact's bytes); a failed save is a miss next
-    // time, said once — the task's work ran.
-    const saveLane = createSaveLane(2 * concurrency, (err) =>
-      warnOf(log)(`[vx] cache save failed: ${err instanceof Error ? err.message : String(err)}`),
-    )
+    // time, said once — the task's work ran. A full disk fails every save
+    // after the first the same way: the first is said with its task, the
+    // rest named once the graph ends.
+    const unsaved: string[] = []
+    const saveLane = createSaveLane(2 * concurrency, (err, taskId) => {
+      if (unsaved.push(taskId) > 1) return
+      const msg = err instanceof Error ? err.message : String(err)
+      // The artifact ceiling's refusal names its task itself.
+      warnOf(log)(
+        `[vx] cache save failed: ${msg.startsWith(`${taskId} `) ? msg : `${taskId}: ${msg}`}`,
+      )
+    })
     const deferredSaves = new Map<string, Promise<SaveFacts>>()
-    const windows = commandWindows()
     // A server held from an earlier run (`vx watch`) runs through this one
     // too, so it may write a mutable input while it is described.
     for (const id of heldBefore.kept.keys()) {
@@ -934,9 +1041,9 @@ async function runOnBus(
     const footers = executors.filter((e) => e.footer !== undefined)
     const placers = executors.filter((e) => e.running !== undefined)
     // The scheduler's dependents count, and every row's `blocks` (the live
-    // list's and the results'): one closure over the graph, built on the
-    // first that asks.
-    let counted: ReadonlyMap<string, number> | undefined
+    // list's and the results'): one closure over the graph, counted by
+    // `prepareRun` beside git when it could, else on the first that asks.
+    let counted: ReadonlyMap<string, number> | undefined = prepared.dependents
     const blocksNow = (): ReadonlyMap<string, number> =>
       (counted ??= graphCounts(nodes, hashCache.folds, hashCache.graph))
     // Every row's `usual` and the forecast's timings: history read once, on
@@ -1149,9 +1256,16 @@ async function runOnBus(
     // dead runs on a failure the outcomes do not hold yet (the server's
     // says `success` until the run ends), so it is seeded here, at its
     // dispatch, and the tracker carries it to what is built on it.
+    // A server can also die under a task that is running (`serverDiedUnder`),
+    // which seeds it then, and so does a task that left a process running
+    // (`leftRunning`).
     const taintSeeds = new Set(excluded.seeds)
-    const taint = taintTracker(options.continueMode === 'always', taintSeeds, nodes)
-    // Each task that ran tainted, and the task at the root of it.
+    const hasServers = [...nodes.values()].some((n) => n.config.exec?.persistent !== undefined)
+    const continueAlways = options.continueMode === 'always'
+    let taint = taintTracker(continueAlways, taintSeeds, nodes, hasServers)
+    let taintOn = continueAlways || taintSeeds.size > 0 || hasServers
+    // Each task that ran tainted, and the task at the root of it: its
+    // outcome's `taintedBy` once it finishes.
     const taintedRan = new Map<string, string>()
     const depended = dependedOn(nodes.values())
 
@@ -1181,6 +1295,7 @@ async function runOnBus(
       fingerprintWatch,
       keyedProjects: keyed,
       readableDeps: readable,
+      nestedDirsOf: nestedDirsOf(nodes, prepared.keyOnly, nestedDirsByProject),
       outputDirSnapshots,
       deferSave: saveLane.defer,
       deferredSaves,
@@ -1188,6 +1303,40 @@ async function runOnBus(
       stopSignal: stop.signal,
       failFast: failFast.signal,
       inputStability,
+      serverReady: (id) => void sinceReady.set(id, []),
+    }
+    if (hasServers)
+      scope.serverDiedUnder = async (node) => {
+        const behind = (pick: (id: string) => boolean): string | undefined => {
+          for (const d of node.deps) {
+            const s = deadServerBehind(nodes, pick, d)
+            if (s !== undefined) return s
+          }
+          return undefined
+        }
+        let dead = behind(serverDied)
+        // A server that died just before the task ended may be heard after
+        // it: both exits can land in one turn of the event loop, the task's
+        // handled first and its save decided in that turn's microtasks. The
+        // next turn has handled the rest.
+        if (dead === undefined && behind((id) => persistentRegistry.has(id)) !== undefined) {
+          await new Promise<void>((resolve) => setImmediate(resolve))
+          dead = behind(serverDied)
+        }
+        if (dead === undefined) return false
+        taintSeeds.add(node.id)
+        taintedRan.set(node.id, dead)
+        return true
+      }
+    scope.leftRunning = (node) => {
+      // The check stays off until the first leftover: it is seeded before
+      // the task settles, so every outcome a dependent's judge reads is
+      // recorded by the tracker made here.
+      if (!taintOn) {
+        taint = taintTracker(continueAlways, taintSeeds, nodes, true)
+        taintOn = true
+      }
+      taintSeeds.add(node.id)
     }
     if (options.retries !== undefined) scope.retries = options.retries
     scope.knownFlaky = (node, hash) => {
@@ -1212,13 +1361,14 @@ async function runOnBus(
     }
     if (explainMiss !== undefined) scope.explainMiss = explainMiss
     if (keepRun !== undefined) scope.keepRun = keepRun
+    if (options.watch === true) scope.watch = true
     if (sandboxArmer !== null) scope.armSandbox = () => sandboxArmer.arm()
     if (demand.any) scope.executing = demand.executing
     const argsOf = executeArgsOf(scope, {
       log: (node) => (node.config.exec?.persistent !== undefined ? serverLog(node.id) : log),
       placements,
       nestedDirsByProject,
-      downloadModeOf: download.modeOf,
+      downloadModeOf: download?.modeOf,
       shortCircuit,
       dependedOn: depended,
       holders,
@@ -1287,7 +1437,7 @@ async function runOnBus(
     const historyDb = indexDb(prepared.localCache)
     const cacheOff = cachesNothing(policy)
     const judgeFlaky = (o: TaskOutcome): void => {
-      const candidates = flakyCandidates([o], taintedRan)
+      const candidates = flakyCandidates([o])
       if (candidates.length === 0) return
       try {
         const found = detectFlaky(historyDb, candidates)[0]
@@ -1303,7 +1453,6 @@ async function runOnBus(
       heldBefore.kept.delete(node.id)
       if (hasEnded(server.child)) return undefined
       server.handOver()
-      handed.add(server.child)
       server.out.bus = bus
       serverOuts.set(node.id, server.out)
       persistentRegistry.set(node.id, server.child)
@@ -1319,11 +1468,15 @@ async function runOnBus(
     }
     let lockBroken: boolean | undefined
     prepared.localCache.storeHeldBlobSizes()
+    prepared.localCache.checkpointOffThread()
     // A hit's row is whole when it finishes, so it is written while the
     // graph runs (history-stream.ts); every other row waits for the bundle.
     // A row with forward args is the bundle's too: only it digests them.
     const historyAt = historyFiles(prepared.localCache)
-    const stream = historyAt === undefined ? undefined : new HistoryStream<TaskOutcome>(historyAt)
+    let expectedHits = 0
+    for (const probe of shortCircuit.preProbed.values()) if (probe.hit !== null) expectedHits++
+    const stream =
+      historyAt === undefined ? undefined : new HistoryStream<TaskOutcome>(historyAt, expectedHits)
     const forwards = (options.forwardArgs?.length ?? 0) > 0
     const streamHit = (o: TaskOutcome): void => {
       if (stream === undefined || isGroupTask(o.node)) return
@@ -1333,7 +1486,6 @@ async function runOnBus(
       stream.add(
         o,
         runRecordOf(o as RecordedOutcome, {
-          tainted: taintedRan,
           runId,
           startedAtMs: endedAtMsAtStart,
           endedAtMs: endedAtMsAtStart,
@@ -1342,6 +1494,50 @@ async function runOnBus(
         }),
       )
     }
+    // A dependency-only server stops once no task left uses it, not at the
+    // end of the graph: it held a worker's cores for nothing. Not under a
+    // hold, whose next cycle may reuse it.
+    const unneeded =
+      options.holdPersistent === true
+        ? undefined
+        : unneededServers(nodes, options.log === undefined && (options.handleSignals ?? true))
+    const serving = executors.filter((e) => e.servers === true && e.unneeded !== undefined)
+    const released = new Set<string>()
+    const stopping: Promise<void>[] = []
+    // A server stops after the servers using it (`serverUsers`), which this
+    // same release reached first.
+    const usersOf = unneeded === undefined ? undefined : serverUsers(nodes)
+    const stops = new Map<string, Promise<void>>()
+    const release = (id: string): void => {
+      // A stop signals every server once and waits out its grace: a second
+      // signal here cut a server's cleanup short.
+      if (stop.signal.aborted) return
+      for (const s of unneeded?.(id) ?? []) {
+        // Copies an executor started beside the dependants it ran.
+        for (const e of serving)
+          try {
+            e.unneeded!(s)
+          } catch {
+            // Its copies stop at its teardown, as before.
+          }
+        const child = persistentRegistry.get(s)
+        // One that died on its own stays: the end of the graph reports it.
+        if (child === undefined || hasEnded(child)) continue
+        released.add(s)
+        persistentRegistry.delete(s)
+        const users: Promise<void>[] = []
+        for (const u of usersOf?.get(s) ?? []) {
+          const p = stops.get(u)
+          if (p !== undefined) users.push(p)
+        }
+        const p =
+          users.length === 0
+            ? stopServer(child)
+            : Promise.allSettled(users).then(() => stopServer(child))
+        stops.set(s, p)
+        stopping.push(p)
+      }
+    }
     const outcomes = await runGraph({
       nodes,
       concurrency,
@@ -1356,22 +1552,30 @@ async function runOnBus(
       onFailFast: () => failFast.abort(),
       signal: stop.signal,
       onStart: (node) => {
+        // A platform skip never starts, like a skip blocked upstream: its
+        // start opened a single task's frame that nothing closed.
+        if (placements.otherPlatform.has(node.id)) return
         log.taskStart?.(node)
       },
       onFinish: (o) => {
         if (cacheOff) o.cacheOff = true
+        const root = taintedRan.get(o.node.id)
+        if (root !== undefined && o.blockedBy === undefined) o.taintedBy = root
         taint.settled(o)
         judgeFlaky(o)
         keeper?.taskDone(o)
         log.taskComplete(o.node, o)
+        if (sinceReady.size > 0) releaseSinceReady(o.node.id)
         streamHit(o)
         demand.finished(o.node.id)
+        release(o.node.id)
         // A server that dies while the graph runs is no longer running:
         // its pin stayed `running` under the line naming its exit.
         const server = o.status === 'success' ? persistentRegistry.get(o.node.id) : undefined
         if (server !== undefined && o.serverExit === undefined)
           void server.exited.then((code) => {
-            if (code !== 0 && !graphOver && !stop.signal.aborted) o.serverExit = code
+            if (code !== 0 && !graphOver && !stop.signal.aborted && !released.has(o.node.id))
+              o.serverExit = code
           })
       },
       // A crash a hand-broken lock entry caused is named as that; asked
@@ -1443,6 +1647,14 @@ async function runOnBus(
     stop.graphDone()
     reprioritize = undefined
     graphOver = true
+    // Each task finishes once its save settles (`settledOf`): the list is whole.
+    if (unsaved.length > 1) {
+      const more = unsaved.slice(1)
+      const named = more.slice(0, 5).join(', ') + (more.length > 5 ? ', …' : '')
+      warnOf(log)(
+        `[vx] cache save failed for ${more.length} more task${more.length === 1 ? '' : 's'}: ${named}`,
+      )
+    }
 
     // Which persistent children outlive the graph, and the bounded SIGTERM
     // of the rest, before the summary prints. Scoped to the real CLI
@@ -1459,6 +1671,7 @@ async function runOnBus(
     // A stop already signalled every server once and waits out its grace:
     // a SIGTERM here, while a server ran its cleanup, cut it short.
     if (stop.signal.aborted) await stop.aborting?.catch(() => {})
+    await Promise.all(stopping)
     const { keepAlive, crashed: crashedPersistent } = await disposeServers({
       registry: persistentRegistry,
       nodes,
@@ -1551,7 +1764,6 @@ async function runOnBus(
     const recordsOf = (final: TaskOutcome[], runOk: boolean) =>
       assembleRunRecords({
         outcomes: final,
-        tainted: taintedRan,
         runId,
         startedAtMs: endedAtMsAtStart,
         endedAtMs,
@@ -1591,6 +1803,11 @@ async function runOnBus(
     // cache closes: its history is written once the wait has decided, or it
     // read `ok` over the exit 1 and fed the flaky list a pass (X-23).
     const historyAfterWait = keepAlive.children.length > 0 && !hold && stop.stoppedBy === undefined
+    // The servers this run is done with end now; the kept ones once the wait decides.
+    telemetry?.endServers(
+      list,
+      historyAfterWait ? new Set(keepAlive.nodes.map((n) => n.id)) : undefined,
+    )
     // A failed task's output, for `vx last` and an agent (run-failures.ts).
     // A cache directory this user cannot write keeps none of the run's
     // records: the run's one notice said so.
@@ -1611,7 +1828,7 @@ async function runOnBus(
       // measured, or the next run knows none of it (2026-10-21).
       const ended = list.filter((o) => o.status !== 'aborted' && o.status !== 'skipped')
       stepSync('vx:history', () => {
-        recordHistory(() => prepared.localCache.recordRuns(recordsOf(ended, false).runs))
+        recordHistory(() => prepared.localCache.recordRuns(recordsOf(ended, false).runs, head))
         recordFailures(ended)
       })
     }
@@ -1663,7 +1880,14 @@ async function runOnBus(
           runId,
         ),
       )
-    const report = reportOf(list, { runId, commit: head.commitSha, ciRun: ciContext, ok }, keeper)
+    // A stop passes none (`VxPlugin.teardown`): the run did not reach its end.
+    // Built and frozen only for a teardown that reads it.
+    const report =
+      stop.stoppedBy === undefined && prepared.plugins.some((p) => p.teardown !== undefined)
+        ? freezeDeep(
+            reportOf(list, { runId, commit: head.commitSha, ciRun: ciContext, ok }, keeper),
+          )
+        : undefined
     // Hand the per-run summary to the telemetry sinks + drain them. Only
     // when a sink is active (telemetry !== undefined) — otherwise this
     // whole block is skipped and the run is byte-identical to before.
@@ -1671,23 +1895,37 @@ async function runOnBus(
     // the run; flush is the sink's last chance to ship buffered records.
     // After the drain, so the summary says what the uploads moved; a live
     // sink's task records went out as each task ended.
-    if (telemetry !== undefined && runContextRecord !== undefined) {
-      const summary = assembleRunSummary(runContextRecord, records.telemetryTasks, {
+    const sinks = telemetry
+    const summarize = async (
+      final: TaskOutcome[],
+      runOk: boolean,
+      timed?: typeof step,
+      ended = { at: endedAtMs, totalMs },
+    ): Promise<void> => {
+      if (sinks === undefined || runContextRecord === undefined) return
+      sinks.endServers(final)
+      const tasks = final === list ? records.telemetryTasks : recordsOf(final, runOk).telemetryTasks
+      const summary = assembleRunSummary(runContextRecord, tasks, {
         startedAt: endedAtMsAtStart,
-        endedAt: endedAtMs,
-        totalDurationMs: Math.round(totalMs),
-        exitOk: ok,
-        abortedCount: list.filter((o) => o.status === 'aborted' && !isGroupTask(o.node)).length,
+        endedAt: ended.at,
+        totalDurationMs: Math.round(ended.totalMs),
+        exitOk: runOk,
+        // As the terminal counts them: `1 aborted · 1 not run`.
+        abortedCount: tasks.filter((t) => t.status === 'aborted').length,
+        notRunCount: final.filter((o) => neverStarted(o) && !isGroupTask(o.node)).length,
         stages: stageTimes(),
         uploads: prepared.localCache.uploads,
         machines: executors.flatMap((e) => executorMachines(e, fault)),
         idealMs: executors.map((e) => executorIdeal(e, fault)).find((ms) => ms !== undefined),
         advice: executors.flatMap((e) => executorAdvice(e, fault)),
       })
-      telemetry.emitSummary(summary)
-      const sinks = telemetry
-      await sinks.flush(step)
+      sinks.emitSummary(summary)
+      await sinks.flush(timed)
     }
+    // A kept server can still fail the run in the wait below: its summary
+    // goes out once the wait has decided, as its history does, or every
+    // sink (vx-ci's job summary) read a pass over the exit 1.
+    if (!historyAfterWait) await summarize(list, ok, step)
     // End-of-run plugin lifecycle: each plugin's teardown(). Crash-isolated
     // + time-bounded inside teardownPlugins, so a faulty plugin can neither
     // fail nor hang the run. Here on the normal path, after the drains
@@ -1798,6 +2036,28 @@ async function runOnBus(
             local.close()
           }
         })
+        // After the footer and the plugins' teardown: no step line there.
+        // The run ended with the session, not with the summary above. A stop
+        // reads as one, as a stop anywhere else does: the servers it took
+        // down are aborted there. The history keeps them as a dev session's
+        // usual end (item 854).
+        const stopped = stop.signal.aborted
+        const told = stopped
+          ? final.map((o) =>
+              keepAlive.nodes.some((n) => n.id === o.node.id) && isPassStatus(o.status)
+                ? {
+                    ...o,
+                    status: 'aborted' as const,
+                    exitCode:
+                      typeof first.code === 'number' ? first.code : signalExitCode(first.code),
+                  }
+                : o,
+            )
+          : final
+        await summarize(told, ok && !crashed && !stopped, undefined, {
+          at: Date.now(),
+          totalMs: Number(process.hrtime.bigint() - runStartHrTimeNs) / 1_000_000,
+        })
       }
       return { ok: ok && !crashed, outcomes: final }
     }
@@ -1853,16 +2113,38 @@ function noneAffected(prepared: PreparedRun, options: RunOptions): string {
     : `No affected project declares task(s): ${options.tasks.join(', ')}.`
 }
 
+/**
+ * One line when a policy names the remote cache and no plugin supplies
+ * one; a plan (`--dry`) gets it as the run does. The workspace's
+ * `cache.policy` asks as a flag does: it was the one source never told.
+ */
+function warnNoRemote(prepared: PreparedRun, options: RunOptions, log: Logger): void {
+  if (prepared.hasRemoteLayer) return
+  const remoteNamed =
+    options.remoteRequested ??
+    (options.cache === undefined &&
+    options.cacheFlags?.noCache !== true &&
+    namesRemote(prepared.workspaceConfig?.cache?.policy)
+      ? 'cache.policy'
+      : undefined)
+  if (remoteNamed === undefined) return
+  warnOf(log)(
+    `vx: ${remoteNamed} named the remote cache, but no cache plugin in .vx/workspace.* supplies one — the remote axes are off for this run`,
+  )
+}
+
 /**
  * Planning mode. Same setup as `run()` — workspace discovery, config
  * load, package graph, task graph — but stops short of execution.
  * Returns a `RunPlan` predicting the cache hit/miss outcome of every
  * task. Used by `--dry`.
  *
- * Side-effects are limited to opening + closing the local Cache handle
- * (and running `inputs.runtime` probe commands, which key
- * derivation requires). Cache probing is the byte-free `cache.has()`
- * existence check — no artifact download, no ingest, no accessed_at bump.
+ * Side-effects are limited to running `inputs.runtime` probe commands,
+ * which key derivation requires: with no remote the local cache opens to
+ * read only (`prepareRunLocked`'s `planning`); with one, the remote hits
+ * found are kept in the index for the run after. Probing is the
+ * byte-free `cache.has()` existence check — no artifact download, no
+ * ingest, no accessed_at bump.
  */
 export async function planRun(options: RunOptions): Promise<RunPlan> {
   refuseRunOptions(options)
@@ -1870,7 +2152,13 @@ export async function planRun(options: RunOptions): Promise<RunPlan> {
   // what a stage says on the way goes to stderr (C-6).
   const log = options.log ?? defaultLogger(undefined, undefined, process.stderr)
   const reasons = new Map<string, AffectedReason>()
-  const prepared = await prepareRun({ ...options, affectedReasons: reasons }, log)
+  const prepared = await prepareRunLocked(
+    { ...options, affectedReasons: reasons },
+    log,
+    undefined,
+    false,
+    true,
+  )
   try {
     options = withRunSettings(options, prepared.workspaceConfig)
     if (prepared.unresolvedTasks.length > 0) {
@@ -1890,6 +2178,7 @@ export async function planRun(options: RunOptions): Promise<RunPlan> {
     if (prepared.empty === 'none-affected')
       return { tasks: [], noneAffected: noneAffected(prepared, options) }
     if (prepared.empty !== null) return { tasks: [] }
+    warnNoRemote(prepared, options, log)
     // The plan orders nothing, but it refuses what the run would refuse.
     await prepared.priorities()
     terminalHolders(prepared.nodes, options.tty === true)
@@ -1980,7 +2269,7 @@ async function planRestorable(
 /** The names a project outside the run's selection declares, and how to reach them. */
 function elsewhereHint(names: readonly string[]): string {
   if (names.length === 0) return ''
-  return ` Only projects outside the selection declare ${names.join(', ')} — pass --all, or --filter to pick them.`
+  return ` Only projects outside the selection declare ${names.join(', ')} — pass --all, or a --filter that includes them.`
 }
 
 /**
@@ -1989,6 +2278,7 @@ function elsewhereHint(names: readonly string[]): string {
  * from: a task just added to a vx.config is not in it until `vx lock`.
  */
 async function unresolvedHint(prepared: {
+  workspaceRoot: string
   unresolvedTasks: readonly string[]
   hintProjects: ReadonlyMap<string, ProjectEntry>
   declaredElsewhere: readonly string[]
@@ -2002,10 +2292,18 @@ async function unresolvedHint(prepared: {
     prepared.hintProjects,
     prepared.declaredElsewhere,
     prepared.taskless,
+    prepared.workspaceRoot,
   )
   const init = await initHint(prepared)
+  // "No project declares a task yet" beside configs that declare one: the
+  // lock holds none of them (a `vx lock` that failed), so say that.
+  if (near.includes(NO_TASK)) {
+    const unlocked = await unlockedAnywhere(prepared.workspaceRoot)
+    if (unlocked !== undefined)
+      return `${elsewhere}${near.replace(NO_TASK, `The lock holds no project for ${unlocked}: run \`vx lock\` (did it fail?).`)}${init}`
+  }
   const lock =
-    init === '' && elsewhere === '' && !near.startsWith(' Did you mean')
+    init === '' && elsewhere === '' && !near.startsWith(' Did you mean') && !near.includes(NO_TASK)
       ? ' A run reads the lock: after editing a vx.config, run `vx lock`.'
       : ''
   return `${elsewhere}${near}${init}${lock}`
@@ -2028,6 +2326,19 @@ async function initHint(prepared: {
   return ` ${EMPTY_LOCK}.`
 }
 
+/** A project config the lock holds no project for, found by the walk `vx lock` makes. */
+async function unlockedAnywhere(root: string): Promise<string | undefined> {
+  const [{ configs }, lock] = await Promise.all([findProjectConfigs(root), readLockfile(root)])
+  const held = new Set(Object.keys(lock?.projects ?? {}))
+  const c = configs.find((x) => !held.has(lockKey(root, x.dir)))
+  return c === undefined ? undefined : relPosix(root, c.configPath)
+}
+
+// The state right after `vx init`: the lock is current, so the lock hint
+// alone sent a new user to a command that changed nothing.
+const NO_TASK =
+  'No project declares a task yet: add one under `tasks` in a vx.config, then run `vx lock`.'
+
 /**
  * A typo's nearest declared name, when one is within two edits — the
  * message names the fix instead of only the mistake. A bare name is
@@ -2040,6 +2351,7 @@ function didYouMean(
   projects: ReadonlyMap<string, ProjectEntry>,
   elsewhere: readonly string[],
   taskless: (name: string) => boolean,
+  root: string,
 ): string {
   // A default `build` is no declared task (X-102): neither a bare name's
   // match nor a menu item or hint for `p#typo`, which led to `p#build`
@@ -2068,7 +2380,7 @@ function didYouMean(
       // A name outside the selection is already said, with `--all`.
       else if (allTasks.size > 0 && !elsewhere.includes(spec)) {
         lists.add(`Tasks: ${listed(allTasks)}.`)
-      }
+      } else if (allTasks.size === 0 && elsewhere.length === 0) lists.add(NO_TASK)
       continue
     }
     const [proj, task] = [spec.slice(0, at), spec.slice(at + 1)]
@@ -2094,7 +2406,20 @@ function didYouMean(
       continue
     }
     if (!projects.has(proj)) {
-      const p = projectNamed(proj, projects) ?? nearest(proj, projects.keys())
+      // NFC typed, NFD on disk (or back): two edits apart, so no hint, and
+      // the list named the project exactly as typed.
+      const form = [...projects.keys()].find((n) => n.normalize('NFC') === proj.normalize('NFC'))
+      if (form !== undefined) {
+        lists.add(
+          `${proj} is not the project ${form}, though they look the same: the two are written in different Unicode forms (NFC and NFD). Set \`name\` in its vx.config to the form you type.`,
+        )
+        continue
+      }
+      // A project's directory first: by edit distance, `p/a` hinted `p/b`.
+      const p =
+        projectNamed(proj, projects) ??
+        projectAtDir(proj, root, projects.values()) ??
+        nearest(proj, projects.keys())
       if (p !== undefined) {
         const tasks = tasksOf(projects.get(p))
         const t = tasks.includes(task) ? task : nearest(task, tasks)
@@ -2150,11 +2475,22 @@ export function projectNamed(
   return scoped.length === 1 ? scoped[0] : undefined
 }
 
+/**
+ * The project whose directory a typed name is, relative to the workspace
+ * root, as `--filter p/a` reads it: `alpha` for `p/a`.
+ */
+export function projectAtDir(
+  typed: string,
+  root: string,
+  projects: Iterable<{ name: string; dir: string }>,
+): string | undefined {
+  const dir = path.resolve(root, typed)
+  for (const p of projects) if (path.resolve(p.dir) === dir) return p.name
+  return undefined
+}
+
 /** The executed, keyed outcomes of a run — what flakiness is judged on. */
-function flakyCandidates(
-  outcomes: readonly TaskOutcome[],
-  tainted: ReadonlyMap<string, string>,
-): FlakyCandidate[] {
+function flakyCandidates(outcomes: readonly TaskOutcome[]): FlakyCandidate[] {
   const out: FlakyCandidate[] = []
   for (const o of outcomes) {
     if (o.status !== 'success' && o.status !== 'failed') continue
@@ -2166,7 +2502,7 @@ function flakyCandidates(
     // Behind a failed dependency the outcome is not the key's: a failure is
     // the dependency's, and a pass built on bytes the key never named, so a
     // later failure relapsed from it and read as a flake for thirty days.
-    if (tainted.has(o.node.id)) continue
+    if (o.taintedBy !== undefined) continue
     out.push({
       project: o.node.projectName,
       task: o.node.taskName,