@vzn/vx 0.0.645 → 0.0.648

package/src/orchestrator/run.ts

@@ -2,8 +2,9 @@
 // run with caching. Each step delegates to a single-purpose sibling file
 // so the layers can be swapped without touching the others.
 
+import type { InputStability } from '../config.js'
 import type { ProjectEntry } from '../workspace/index.js'
-import { inputStabilityEnv, LOCK_INVALID, lockEntryBroken } from '../workspace/index.js'
+import { EMPTY_LOCK, inputStabilityEnv, LOCK_INVALID, lockEntryBroken } from '../workspace/index.js'
 import { realpathSync } from 'node:fs'
 import path from 'node:path'
 import {
@@ -17,6 +18,7 @@ import {
 } from '../cache/index.js'
 import { VERSION } from '../version.js'
 import {
+  hostPlatform,
   resetSandbox,
   signalExitCode,
   VX_RUN_TASK_ENV,
@@ -24,13 +26,14 @@ import {
 } from '../exec/index.js'
 import { DeferredOutputs } from './deferred-outputs.js'
 import { resolveDownloadModes } from './download-policy.js'
-import type { TaskExecutor } from '../exec/index.js'
+import type { MachineReport, TaskExecutor } from '../exec/index.js'
 import {
   deadServerBehind,
   declaredTask,
   isGroupTask,
   markSurfacedDeps,
   runGraph,
+  taskDefinitions,
   type TaskNode,
   type TaskOutcome,
 } from '../graph/index.js'
@@ -51,16 +54,18 @@ import {
   detectColors,
   type ColorSupport,
   SecretBroker,
+  thrownMessage,
+  type ui,
 } from '../util/index.js'
 import { keyedProjects } from './keyed-projects.js'
-import { isDefaultBuild } from './projects.js'
+import { isDefaultBuild } from '../graph/index.js'
 import { commandWriteReach, prepareSandbox } from './sandbox-request.js'
 import { recordOutputDirSnapshots, type OutputDirSnapshot, type SaveFacts } from './miss-save.js'
 import { admitTasks, taintTracker } from './admission.js'
-import { unkeyedGroupOutcome, type ExecuteArgs } from './execute-task.js'
+import { settleProbedHit, unkeyedGroupOutcome, type ExecuteArgs } from './execute-task.js'
 import { executeArgsOf, type RunScope } from './execute-args.js'
 import { excludedTaint } from './excluded-keys.js'
-import { keyUpstream } from './upstream.js'
+import { dependedOn, keyUpstream } from './upstream.js'
 import {
   busLogger,
   createEventBus,
@@ -79,9 +84,16 @@ import {
 } from '../report/index.js'
 import { installPlugins, PluginSetupError, type RunReport } from './plugin.js'
 import { keepsFailures, reportOf, RunKeeper } from './run-report.js'
-import { buildAdmission, executorDemand, resolveExecutors, teardownPlugins } from './plugin-host.js'
+import {
+  buildAdmission,
+  executorDemand,
+  executorLabel,
+  resolveExecutors,
+  teardownPlugins,
+} from './plugin-host.js'
 import { createSteps } from './steps.js'
-import { writeRunSteps } from './run-steps.js'
+import { liveSocketOf, serveLive } from './run-live.js'
+import { writeRunSteps, writeRunWarnings } from './run-steps.js'
 import { subscribeTelemetry, type TelemetryHandle } from './telemetry-host.js'
 import { claimOutput, guardedOutput } from './output-host.js'
 import { assembleRunSummary } from './telemetry.js'
@@ -89,14 +101,15 @@ import { isPassStatus } from '../graph/index.js'
 import type { RunContextRecord } from './telemetry.js'
 import { plan, type RunPlan } from './plan.js'
 import type { AffectedReason } from './affected-tasks.js'
-import { openRemotes, prepareRun, type PreparedRun } from './prepare.js'
+import { namesRemote, openRemotes, prepareRun, type PreparedRun } from './prepare.js'
 import { acquireRunLock } from './run-lock.js'
 import { terminateChildren } from './signals.js'
-import { runStop } from './run-stop.js'
+import { CYCLE_INTERRUPT, runStop } from './run-stop.js'
 import {
   hasPooledExecutor,
   placeTasks,
   planExecutorOf,
+  platformVerdict,
   poolOfPlacement,
   terminalHolders,
   locallyPlaced,
@@ -114,6 +127,7 @@ import {
   type ShortCircuit,
 } from './local-shortcircuit.js'
 import { deriveStableKeys, probesAfterWrites } from './stable-keys.js'
+import { askRemote, serversToHold } from './held-servers.js'
 
 import { writeRunFailures } from './run-failures.js'
 import { assembleRunRecords, recordLateRemoteErrors } from './run-records.js'
@@ -125,6 +139,7 @@ import {
   sessionEnd,
   takeHeldServers,
 } from './persistent.js'
+import { serverKey } from './persistent-task.js'
 import { writeRunProfile } from './run-artifacts.js'
 import { createSaveLane } from './save-lane.js'
 import { commandWindows } from './command-windows.js'
@@ -136,6 +151,7 @@ import {
 } from '../history/index.js'
 import type { RunOptions, RunSummary } from './options.js'
 import { withRunSettings } from './run-settings.js'
+import { cacheFacts, taskArtifacts } from './task-artifacts.js'
 import { resolveSetting } from '../settings.js'
 
 // Per run, never shared: a `vx watch` process runs many, and a shared map
@@ -145,6 +161,7 @@ const emptyShortCircuit = (): ShortCircuit => ({
   restoreTier: new Set(),
   uncachedKeys: new Map(),
   groupKeys: new Map(),
+  remoteAsk: new Map(),
 })
 
 /**
@@ -184,18 +201,37 @@ export function shouldShortCircuit(nodes: Map<string, TaskNode>, policy: CachePo
   return nodes.size > 0
 }
 
-function maskInvocation(command: string): string {
-  return secretMask([process.env])?.mask(command) ?? command
+/**
+ * The run's own secrets: vx's environment and every selected task's
+ * `define` and `exec.env.secret`, as each task's output masks them. With
+ * the environment's alone, a secret only a config names reached the stored
+ * line `vx last` prints in full.
+ */
+function invocationMask(nodes: Iterable<TaskNode>): (command: string) => string {
+  const sources: Array<Readonly<Record<string, string>>> = [process.env as Record<string, string>]
+  const named = new Set<string>()
+  for (const n of nodes) {
+    const env = n.config.exec?.env
+    if (env?.define !== undefined) sources.push(env.define)
+    for (const name of env?.secret ?? []) named.add(name)
+  }
+  const mask = secretMask(sources, [...named])
+  return (command) => mask?.mask(command) ?? command
 }
 
 /**
  * The command line a run hands to its telemetry sinks, with what follows
  * `--` counted, not quoted. Those are the task's arguments,
- * often a token (`-- --token=…`), and the line reached an OTLP span, the
- * GitHub job summary and a check-run posted over the API verbatim (item
- * 1057). Local history (`vx last`) keeps the whole line, on this machine.
+ * often a token (`-- --token=…`), and the line reached an OTLP span and
+ * the GitHub job summary verbatim (item 1057). Local history (`vx last`) keeps the whole line, on this machine.
  */
-export function invocationCommand(argv: readonly string[]): string {
+export function invocationCommand(line: readonly string[] | string): string {
+  // A line is shell-quoted (the CLI's): its words as written, so `'a b'`
+  // counts one argument, not the two a split at blanks made.
+  const argv =
+    typeof line === 'string'
+      ? (line.match(/(?:[^\s'"\\]|\\.|'[^']*'|"(?:[^"\\]|\\.)*")+/g) ?? [])
+      : line
   const sep = argv.indexOf('--')
   if (sep === -1) return argv.join(' ')
   const rest = argv.length - sep - 1
@@ -309,8 +345,14 @@ export async function run(options: RunOptions): Promise<RunSummary> {
   // bytes as written, as the cache holds them: it formats its own output.
   const render = terminalSubscriber(terminal === null || colors.enabled ? sink : plainOutput(sink))
   let unsubscribeTerminal = bus.subscribe(render)
+  // What the run warned of, no task's: the footer repeats it (`warn`).
+  const warned: string[] = []
+  const unsubscribeWarned = bus.subscribe((e) => {
+    if (e.kind === 'run:status' && 'warn' in e) warned.push(e.line)
+  })
   const detach = (): void => {
     unsubscribeTerminal()
+    unsubscribeWarned()
     terminal?.settle()
   }
   // A plugin's `output` renders the run from its start; core's renderer
@@ -329,7 +371,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
     // vx's own steps are timed only where something renders them.
     const showsSteps =
       terminal !== null ? terminal.showsSteps : options.log?.stepStart !== undefined
-    const summary = await runOnBus(options, bus, colors, terminal, showsSteps, takeOutput)
+    const summary = await runOnBus(options, bus, warned, colors, terminal, showsSteps, takeOutput)
     if (summary.persistent === undefined) return summary
     // Servers handed back still running still write: `vx watch dev` printed
     // none of its server's log while it idled, the renderer gone with this
@@ -351,6 +393,7 @@ export async function run(options: RunOptions): Promise<RunSummary> {
 async function runOnBus(
   options: RunOptions,
   bus: EventBus,
+  warned: readonly string[],
   colors: ColorSupport,
   terminal: DefaultLogger | null,
   showsSteps: boolean,
@@ -373,7 +416,6 @@ async function runOnBus(
   }
 
   const prepared = await step('vx:load', () => prepareRun(options, log))
-  options = withRunSettings(options, prepared.workspaceConfig)
   mark('plugin stages')
   // Every exit from here tears the plugins down: their capability
   // factories ran in prepareRun, and teardown is what releases what those
@@ -388,6 +430,27 @@ async function runOnBus(
     const plugins = prepared.plugins.filter((p) => p !== except)
     await teardownPlugins(plugins, warnOf(log), report, step)
   }
+  // The setting variables are read before any task, so a bad value is
+  // refused here, not mid-run; an embedder's (the CLI refuses it first)
+  // threw past the teardown and the cache close.
+  let inputStability: { run: InputStability; workspace?: InputStability }
+  let cleanOutputs: boolean | undefined
+  try {
+    options = withRunSettings(options, prepared.workspaceConfig)
+    inputStability = {
+      run: inputStabilityEnv(),
+      ...(prepared.workspaceConfig?.cache?.inputStability !== undefined
+        ? { workspace: prepared.workspaceConfig.cache.inputStability }
+        : {}),
+    }
+    cleanOutputs = resolveSetting('cache.outputs.clean', prepared.workspaceConfig).value as
+      | boolean
+      | undefined
+  } catch (err) {
+    await teardown()
+    prepared.cache.close()
+    throw err
+  }
   // A task whose command re-enters `vx run` in the workspace running it
   // is refused. When the inner run reaches this task again it forks a run
   // per run until the machine gives out; when it does not (`ci` shelling
@@ -397,7 +460,8 @@ async function runOnBus(
   // name the task; the check is on the root so both shapes are caught.
   // Compared canonical: a root reached through a symlink (`run({ cwd })`,
   // macOS's /var/folders) and the inner vx's realpath'd cwd are one tree.
-  const outerRoot = process.env[VX_RUN_WORKSPACE_ENV]
+  // Empty is unset: resolved, '' named the cwd.
+  const outerRoot = process.env[VX_RUN_WORKSPACE_ENV] || undefined
   if (outerRoot !== undefined && canonical(outerRoot) === canonical(prepared.workspaceRoot)) {
     await teardown()
     prepared.cache.close()
@@ -513,7 +577,6 @@ async function runOnBus(
   }
   const {
     workspaceRoot,
-    workspaceConfig,
     cacheDir,
     cache,
     nodes,
@@ -523,12 +586,14 @@ async function runOnBus(
     hashCache,
     workspaceProjectCount,
   } = prepared
+  const maskInvocation = invocationMask(nodes.values())
   const concurrency = options.concurrency ?? machineParallelism()
   // An earlier cycle's servers: the ones this graph starts again unchanged
   // stay up; the others go now, before anything binds their ports. A stop
   // that lands meanwhile ends the run here, having run nothing: an aborted
   // graph printed a `not run` row and a footer above `stopped` (WD-22).
   const heldBefore = takeHeldServers(options.keep?.servers, nodes, options.forwardArgs)
+  for (const server of heldBefore.kept.values()) server.out.bus = bus
   if (heldBefore.stale.length > 0) {
     await terminateChildren(() => heldBefore.stale)
     if (options.signal?.aborted === true) {
@@ -547,6 +612,8 @@ async function runOnBus(
       cacheDir: prepared.cacheDir,
       warn: warnOf(log),
       concurrency,
+      artifacts: taskArtifacts(prepared.cache, prepared.workspaceRoot),
+      cacheFacts: () => cacheFacts(prepared.remotes, prepared.cacheRefs),
     }),
   )
   // Placement: decided ONCE per task, before scheduling, so the scheduler
@@ -572,8 +639,29 @@ async function runOnBus(
       warnOf(log)(`[vx] ${id}: exec.remote is 'only' and ${why} — nothing ran`)
     }
   }
-  // Demand: what each executor still has left to run (`executorDemand`).
-  const narrowDemand = executorDemand(placements.executors, warnOf(log))
+  const platforms = platformVerdict(placements, ciContext.ci)
+  if (platforms !== undefined && 'refuse' in platforms) {
+    await abandoning(() => {
+      throw new UserError(platforms.refuse)
+    })
+  }
+  if (platforms !== undefined && 'warn' in platforms) warnOf(log)(platforms.warn)
+  /** A task its `exec.platforms` skip here: it never starts, and nor does what depends on it. */
+  const platformSkip = (node: TaskNode): TaskOutcome | undefined => {
+    const needs = placements.otherPlatform.get(node.id)
+    return needs === undefined
+      ? undefined
+      : {
+          node,
+          status: 'skipped',
+          exitCode: 0,
+          durationMs: 0,
+          otherPlatform: { needs, host: hostPlatform() },
+        }
+  }
+  // Demand: what each executor still has left to run (`executorDemand`),
+  // told once the cache has said what it answers.
+  const demand = executorDemand(placements.executors, warnOf(log))
 
   // `cache.download` (default `all`) decides ONCE per task, here, whether a
   // remote execution's outputs come home.
@@ -588,19 +676,11 @@ async function runOnBus(
 
   // `run.timeout` (ms), for any task WITHOUT its own `exec.timeout`.
   const taskTimeoutDefault = options.timeout
-  // Read once, before any task: a bad value is refused here, not mid-run.
-  const inputStability = {
-    run: inputStabilityEnv(),
-    ...(workspaceConfig?.cache?.inputStability !== undefined
-      ? { workspace: workspaceConfig.cache.inputStability }
-      : {}),
-  }
-
-  const cleanOutputs = resolveSetting('cache.outputs.clean', workspaceConfig).value as
-    | boolean
-    | undefined
 
+  // The held servers this run took over, spared by a `CYCLE_INTERRUPT`.
+  const handed = new Set<ReturnType<typeof Bun.spawn>>()
   const stop = runStop({
+    handed: (child) => handed.has(child),
     log,
     cache,
     external: options.signal,
@@ -624,11 +704,13 @@ async function runOnBus(
   // because the normal path closes before the persistent-task wait —
   // holding the handle open for a dev server's whole lifetime would be
   // worse.
+  let live: ReturnType<typeof serveLive> | undefined
   let cacheClosed = false
   const closeCache = async (): Promise<void> => {
     if (cacheClosed) return
     cacheClosed = true
     cache.close()
+    live?.unlisten()
     // Released with the handle: before a persistent task's wait, on every
     // exit path (see run-lock.ts for why a run holds one at all). AWAITED:
     // fired and forgotten, the release lost the race with the CLI's exit —
@@ -648,6 +730,16 @@ async function runOnBus(
       }),
     ),
   )
+  // Answers `vx last --live` while the lock is held (run-live.ts).
+  live = serveLive({
+    socketPath: liveSocketOf(releaseRunLock.path),
+    bus,
+    runId,
+    command: maskInvocation(invocationCommand(options.command ?? process.argv.slice(1))),
+    startedAt: Date.now(),
+    nodes: () => nodes.values(),
+    warnings: () => warned,
+  })
   try {
     const runStartHrTimeNs = process.hrtime.bigint()
     const endedAtMsAtStart = Date.now()
@@ -660,9 +752,17 @@ async function runOnBus(
     // clean their outputs before every exec for a save that never
     // happened.
     const policy: CachePolicy = effectiveCachePolicy(prepared.cachePolicy, remoteCacheEnabled)
-    if (options.remoteRequested === true && !prepared.hasRemoteLayer) {
+    // The workspace's `cache.policy` asks as a flag does: it was the one source never told.
+    const remoteNamed =
+      options.remoteRequested ??
+      (options.cache === undefined &&
+      options.cacheFlags?.noCache !== true &&
+      namesRemote(prepared.workspaceConfig?.cache?.policy)
+        ? 'cache.policy'
+        : undefined)
+    if (remoteNamed !== undefined && !prepared.hasRemoteLayer) {
       warnOf(log)(
-        'vx: --cache-policy named the remote cache, but no cache plugin in .vx/workspace.* supplies one — the remote axes are off for this run',
+        `vx: ${remoteNamed} named the remote cache, but no cache plugin in .vx/workspace.* supplies one — the remote axes are off for this run`,
       )
     }
     const deferredOutputs = new DeferredOutputs({
@@ -710,9 +810,7 @@ async function runOnBus(
         // token after `--` to every sink verbatim (C-67).
         // Masked as the stored line is: a flag's secret value sits before
         // the `--` the count covers (L-44).
-        command: maskInvocation(
-          invocationCommand(options.command?.split(' ') ?? process.argv.slice(1)),
-        ),
+        command: maskInvocation(invocationCommand(options.command ?? process.argv.slice(1))),
         requestedTasks: options.tasks,
         cachePolicy: formatCachePolicy(policy),
         concurrency,
@@ -744,10 +842,19 @@ async function runOnBus(
       n.config.exec?.persistent !== undefined ||
       n.config.cache === undefined ||
       !(policy.localRead || policy.remoteRead)
+    // Only what runs here arms this machine's sandbox: a `sandbox` executor
+    // enforces it where it runs the task, so a distributed run's
+    // orchestrator needs none (2026-10-22).
     const sandboxed =
       sandboxArmer === null
         ? []
-        : [...nodes.values()].filter((n) => n.config.exec?.sandbox !== undefined)
+        : [...nodes.values()].filter(
+            (n) =>
+              n.config.exec?.sandbox !== undefined &&
+              (n.config.exec.persistent === undefined
+                ? localPlaced.has(n.id)
+                : !placements.carried.has(n.id)),
+          )
     let prewarming: Promise<void> | undefined
     const prewarm = (): void => {
       prewarming ??= sandboxArmer?.arm().catch(() => {})
@@ -800,7 +907,19 @@ async function runOnBus(
 
     // Run context for the footer. The top-of-run header is gone — the
     // banner now lives in the summary, where the eye lands at the end.
+    const fault = executorFaults(warnOf(log))
+    const footers = executors.filter((e) => e.footer !== undefined)
+    const placers = executors.filter((e) => e.running !== undefined)
     const runContext = {
+      ...(footers.length > 0
+        ? { executorRows: () => footers.flatMap((e) => executorFooter(e, fault)) }
+        : {}),
+      ...(placers.length > 0
+        ? {
+            whereRunning: (id: string) =>
+              placers.map((e) => executorWhere(e, id, fault)).find((w) => w !== undefined),
+          }
+        : {}),
       version: VERSION,
       packageCount: packagesInScope.size,
       remoteCacheEnabled,
@@ -816,7 +935,9 @@ async function runOnBus(
       },
       concurrency,
       workspaceProjectCount,
-      runId: undefined as string | undefined,
+      // Live too: a long run is read while it runs (2026-10-21).
+      runId,
+      warnings: () => warned,
     }
 
     // Asked only when a plugin declares `output`: a plain run pays nothing.
@@ -854,6 +975,7 @@ async function runOnBus(
           prepared.localCache.dbHandle(),
           undefined,
           prepared.branch(),
+          taskDefinitions(nodes.values()),
         ).p50sFor(ids)
         return createForecast(nodes, p50s, concurrency)
       },
@@ -867,9 +989,20 @@ async function runOnBus(
     // local-only runs never reach here, so their behavior + perf is
     // unchanged (no upfront key pass, no local probing).
     let prefetchDone: Promise<void> = Promise.resolve()
+    /** `--profile`'s run events beside the tasks: the cache probe's. */
+    const runSpans: Record<string, unknown>[] = []
+    // A remote that is read answers which tasks it holds: demand settles
+    // on that answer, a failed prefetch settling it on none.
+    const prefetchSettles = prepared.hasRemoteLayer && policy.remoteRead
     if (prepared.hasRemoteLayer) {
       prefetchDone = steps.background('vx:prefetch', () =>
         startRemotePrefetch({
+          ...(demand.any
+            ? {
+                held: (ids: readonly string[], missed: readonly string[]) =>
+                  demand.settle(ids, missed),
+              }
+            : {}),
           nodes,
           cache,
           workspaceRoot,
@@ -881,6 +1014,7 @@ async function runOnBus(
           concurrency: options.downloadConcurrency ?? networkConcurrency(),
           remoteRead: policy.remoteRead,
           remoteWrite: policy.remoteWrite,
+          spans: options.profile === undefined ? undefined : runSpans,
         }),
       )
     }
@@ -902,10 +1036,19 @@ async function runOnBus(
       shouldShortCircuit(nodes, policy) && (remote || cache === prepared.localCache)
         ? prepared.localCache.holdsNothing()
         : false
+    // Behind a remote, a server's dependants need their keys up front even
+    // on an empty store: the remote may hold them all, and then the server
+    // never starts (`held-servers.ts`).
+    const askForServers =
+      remote &&
+      policy.remoteRead &&
+      [...nodes.values()].some(
+        (n) => n.config.exec?.persistent !== undefined && !n.requested && n.surfaced !== true,
+      )
     // A remote may still hold what an empty local store does not: the
     // prefetch asks it, and no confirmed miss warms the sandbox early.
-    if (coldLocal && !remote) prewarm()
-    else if (!coldLocal && shouldShortCircuit(nodes, policy)) {
+    if (coldLocal && !remote && sandboxed.length > 0) prewarm()
+    else if ((!coldLocal || askForServers) && shouldShortCircuit(nodes, policy)) {
       shortCircuit = await step('vx:keys', () =>
         startLocalShortCircuit({
           nodes,
@@ -926,6 +1069,19 @@ async function runOnBus(
       if (!remote && sandboxed.some((n) => shortCircuit.preProbed.get(n.id)?.hit === null))
         prewarm()
     }
+    if (demand.any) {
+      if (prefetchSettles) void prefetchDone.finally(() => demand.settle([]))
+      else {
+        const hits: string[] = []
+        const missed: string[] = []
+        for (const [id, probe] of shortCircuit.preProbed)
+          (probe.hit !== null ? hits : missed).push(id)
+        // An empty store was not asked: it holds none of them.
+        if (coldLocal && !remote)
+          for (const [id, n] of nodes) if (n.config.cache !== undefined) missed.push(id)
+        demand.settle(hits, missed)
+      }
+    }
 
     // Whether this task runs behind a failure (`continueMode: 'always'`
     // only) or on the key of a dependency `--exclude-dependencies` kept
@@ -951,8 +1107,7 @@ async function runOnBus(
     const taint = taintTracker(options.continueMode === 'always', taintSeeds, nodes)
     // Each task that ran tainted, and the task at the root of it.
     const taintedRan = new Map<string, string>()
-    const dependedOn = new Set<string>()
-    for (const n of nodes.values()) for (const d of n.deps) dependedOn.add(d)
+    const depended = dependedOn(nodes.values())
 
     // Why a task missed, for a sink: a read of the history per miss, so only
     // when a sink exists.
@@ -992,13 +1147,14 @@ async function runOnBus(
     if (explainMiss !== undefined) scope.explainMiss = explainMiss
     if (keepRun !== undefined) scope.keepRun = keepRun
     if (sandboxArmer !== null) scope.armSandbox = () => sandboxArmer.arm()
+    if (demand.any) scope.executing = demand.executing
     const argsOf = executeArgsOf(scope, {
       log: (node) => (node.config.exec?.persistent !== undefined ? serverLog(node.id) : log),
       placements,
       nestedDirsByProject,
       downloadModeOf: download.modeOf,
       shortCircuit,
-      dependedOn,
+      dependedOn: depended,
       holders,
     })
     const buildExecuteArgs = (
@@ -1078,6 +1234,7 @@ async function runOnBus(
       heldBefore.kept.delete(node.id)
       if (hasEnded(server.child)) return undefined
       server.handOver()
+      handed.add(server.child)
       server.out.bus = bus
       serverOuts.set(node.id, server.out)
       persistentRegistry.set(node.id, server.child)
@@ -1113,7 +1270,14 @@ async function runOnBus(
         judgeFlaky(o)
         keeper?.taskDone(o)
         log.taskComplete(o.node, o)
-        narrowDemand(o.node.id)
+        demand.finished(o.node.id)
+        // A server that dies while the graph runs is no longer running:
+        // its pin stayed `running` under the line naming its exit.
+        const server = o.status === 'success' ? persistentRegistry.get(o.node.id) : undefined
+        if (server !== undefined && o.serverExit === undefined)
+          void server.exited.then((code) => {
+            if (code !== 0 && !graphOver && !stop.signal.aborted) o.serverExit = code
+          })
       },
       // A crash a hand-broken lock entry caused is named as that; asked
       // only here, so a run that does not crash pays nothing.
@@ -1125,24 +1289,36 @@ async function runOnBus(
             : line,
         ),
       execute: (node, upstream) => {
+        const skip = platformSkip(node)
+        if (skip !== undefined) return Promise.resolve(skip)
         const kept = keepServer(node)
-        return kept !== undefined
-          ? Promise.resolve(kept)
-          : executeWithDedup(node, keyUpstream(node, upstream))
+        if (kept === undefined) return executeWithDedup(node, keyUpstream(node, upstream))
+        // Keyed as a started server is: a cached dependant folds its key,
+        // and without it an edit to the server's sources hit stale.
+        return Promise.resolve(
+          serverKey(argsOf(node, keyUpstream(node, upstream), true, false)),
+        ).then((hash) => (hash === undefined ? kept : { ...kept, hash }))
       },
-      // An unkeyed group runs nothing: settled in place, unless a taint
-      // may ride it, which `buildExecuteArgs` records on dispatch.
+      // An unkeyed group runs nothing, and a restore-tier hit whose tree is
+      // current restores nothing: settled in place, unless a taint may ride
+      // it, which `buildExecuteArgs` records on dispatch.
       ...(options.continueMode !== 'always' && taintSeeds.size === 0
         ? {
-            settleNow: (node: TaskNode, upstream: TaskOutcome[]) =>
-              isGroupTask(node) && node.config.inputs === undefined
-                ? unkeyedGroupOutcome(
+            settleNow: (node: TaskNode, upstream: TaskOutcome[]) => {
+              const skip = platformSkip(node)
+              if (skip !== undefined) return () => skip
+              if (isGroupTask(node) && node.config.inputs === undefined)
+                return () =>
+                  unkeyedGroupOutcome(
                     node,
                     keyUpstream(node, upstream),
                     shortCircuit.groupKeys.get(node.id),
                     runStartHrTimeNs,
                   )
-                : undefined,
+              if (!shortCircuit.restoreTier.has(node.id) || heldBefore.kept.has(node.id))
+                return undefined
+              return settleProbedHit(buildExecuteArgs(node, keyUpstream(node, upstream)))
+            },
           }
         : {}),
       // A `schedule` plugin's weights; the scheduler keeps its structural
@@ -1158,6 +1334,16 @@ async function runOnBus(
       // runs ahead of their deps as low-priority worker-slot backfill.
       // Empty when the short-circuit didn't fire → byte-identical.
       restoreTier: shortCircuit.restoreTier,
+      ...(askForServers
+        ? {
+            heldServers: askRemote(
+              serversToHold(nodes, shortCircuit.restoreTier, shortCircuit.remoteAsk),
+              nodes,
+              shortCircuit,
+              cache,
+            ),
+          }
+        : {}),
     })
     stop.graphDone()
     reprioritize = undefined
@@ -1170,8 +1356,11 @@ async function runOnBus(
     // signal loop) and embedders that manage lifecycle themselves — both
     // expect run() to return, not block on a server.
     const foreground = options.log === undefined && (options.handleSignals ?? true)
-    // An aborted run's children are already being torn down: nothing to hold.
-    const hold = options.holdPersistent === true && !stop.signal.aborted
+    // An aborted run's children are already being torn down: nothing to
+    // hold, but the servers a `CYCLE_INTERRUPT` spared.
+    const hold =
+      options.holdPersistent === true &&
+      (!stop.signal.aborted || stop.signal.reason === CYCLE_INTERRUPT)
     const { keepAlive, crashed: crashedPersistent } = await disposeServers({
       registry: persistentRegistry,
       nodes,
@@ -1187,7 +1376,14 @@ async function runOnBus(
     // Clear the status line for good before the summary prints.
     log.runEnd?.()
     const list = [...outcomes.values()]
-    const ok = list.every((o) => isPassStatus(o.status))
+    // A platform skip, and a skip it blocked, is no failure: the task is not this machine's.
+    const ok = list.every(
+      (o) =>
+        isPassStatus(o.status) ||
+        (o.status === 'skipped' &&
+          (o.otherPlatform !== undefined ||
+            (o.blockedBy !== undefined && placements.otherPlatform.has(o.blockedBy)))),
+    )
 
     // The summary + artifact writers + recordRun pass all exclude group
     // tasks via the shared tallyOutcomes helper. We pass the full
@@ -1217,7 +1413,7 @@ async function runOnBus(
     const stillDeferred = deferredOutputs.pending()
     if (stillDeferred.length > 0) {
       log.status(
-        `  Deferred: left outputs remote (--cache-download=none): ${stillDeferred.join(', ')}`,
+        `  Deferred: left outputs remote${downloadPolicy === 'all' ? '' : ` (--cache-download=${downloadPolicy})`}: ${stillDeferred.join(', ')}`,
       )
     }
 
@@ -1233,6 +1429,9 @@ async function runOnBus(
             target: profile,
             cwd: options.cwd,
             outcomes: list,
+            runStartNs: runStartHrTimeNs,
+            executors: executors.map((e) => executorProfile(e, fault)),
+            spans: runSpans,
           }),
         )
         log.status(`vx: profile written to ${wrote}`)
@@ -1290,16 +1489,27 @@ async function runOnBus(
     // read `ok` over the exit 1 and fed the flaky list a pass (X-23).
     const historyAfterWait = keepAlive.children.length > 0 && !hold && stop.stoppedBy === undefined
     // A failed task's output, for `vx last` and an agent (run-failures.ts).
-    const recordFailures = (final: readonly TaskOutcome[]): void =>
-      writeRunFailures(prepared.cacheDir, runId, final, warnOf(log))
+    // A cache directory this user cannot write keeps none of the run's
+    // records: the run's one notice said so.
+    const keeps = prepared.localCache.readOnlyNotice() === null
+    const recordFailures = (final: readonly TaskOutcome[]): void => {
+      if (keeps) writeRunFailures(prepared.cacheDir, runId, final, warnOf(log))
+    }
     if (stop.stoppedBy === undefined && !historyAfterWait) {
       stepSync('vx:history', () => {
         recordHistory(() => cache.recordRunBundle(records))
         recordFailures(list)
       })
+    } else if (stop.stoppedBy !== undefined) {
+      // Stopped: not a run `vx last` lists (item 854), but what finished is
+      // measured, or the next run knows none of it (2026-10-21).
+      const ended = list.filter((o) => o.status !== 'aborted' && o.status !== 'skipped')
+      stepSync('vx:history', () => {
+        recordHistory(() => prepared.localCache.recordRuns(recordsOf(ended, false).runs))
+        recordFailures(ended)
+      })
     }
     mark('record history')
-    runContext.runId = runId
     // Drain any still-in-flight background prefetches before closing the
     // cache handle — a prefetch ingesting into a closed SQLite DB would
     // throw. Tasks that resolved as local hits never awaited their
@@ -1324,7 +1534,10 @@ async function runOnBus(
       await prefetchDone
       await Promise.all([cache.drainUploads?.(), keeper?.settled()])
     })
-    writeRunSteps(prepared.cacheDir, runId, steps.ended, warnOf(log))
+    if (keeps) {
+      writeRunSteps(prepared.cacheDir, runId, steps.ended, warnOf(log))
+      writeRunWarnings(prepared.cacheDir, runId, warned, warnOf(log))
+    }
     // An upload that failed after the rows went in: its row says so too.
     if (prepared.localCache.uploads.failed > 0 && stop.stoppedBy === undefined && !historyAfterWait)
       recordHistory(() => recordLateRemoteErrors(prepared.localCache, records.runs, runId))
@@ -1345,6 +1558,8 @@ async function runOnBus(
         abortedCount: list.filter((o) => o.status === 'aborted' && !isGroupTask(o.node)).length,
         stages: stageTimes(),
         uploads: prepared.localCache.uploads,
+        machines: executors.flatMap((e) => executorMachines(e, fault)),
+        idealMs: executors.map((e) => executorIdeal(e, fault)).find((ms) => ms !== undefined),
       })
       telemetry.emitSummary(summary)
       const sinks = telemetry
@@ -1382,16 +1597,20 @@ async function runOnBus(
       }
     })
 
-    // The footer is the run's last word (owner, 2026-10-06): it prints once
+    // The footer is the run's last word (2026-10-06): it prints once
     // the run's own work is done, history, uploads, plugin teardown and the
     // sandbox reset included, so any warning they raise lands above it. A
     // task's own facts (flaky, blocked) ride its row. Only a kept server's
     // own output follows.
     // The footer replaces the live region in place.
     log.runDone?.()
-    if (options.summaryTable === true) for (const line of formatOutcomeTable(list)) log.status(line)
+    if (options.summaryTable === true)
+      for (const line of formatOutcomeTable(list, colors)) log.status(line)
+    // A run a signal stopped records nothing `vx last` opens: no id to copy.
+    const footerContext =
+      stop.stoppedBy === undefined ? runContext : { ...runContext, runId: undefined }
     if (options.footer !== false)
-      for (const line of formatRunSummary(list, totalMs, colors, runContext)) log.status(line)
+      for (const line of formatRunSummary(list, totalMs, colors, footerContext)) log.status(line)
 
     // Edge case the summary already reported: the user requested a
     // persistent task (dev server / watcher). The run is "done" in every
@@ -1414,7 +1633,6 @@ async function runOnBus(
         persistent: holdServers(keepAlive, {
           forwardArgs: options.forwardArgs,
           outOf: (id) => serverOuts.get(id) ?? { bus },
-          warn: warnOf(log),
         }),
       }
     }
@@ -1441,7 +1659,9 @@ async function runOnBus(
       if (stop.signal.aborted) await stop.aborting?.catch(() => {})
       else await terminateChildren(() => keepAlive.children)
       // The server that ended the session on its own, not cleanly, failed.
-      if (!stop.signal.aborted && first.code !== 0) failServer(outcomes, node.id, first.code)
+      // A stop's own kill is no crash: the user ended the session.
+      const crashed = !stop.signal.aborted && first.code !== 0
+      if (crashed) failServer(outcomes, node.id, first.code)
       const final = [...outcomes.values()]
       if (historyAfterWait) {
         recordHistory(() => {
@@ -1449,14 +1669,14 @@ async function runOnBus(
           // to its schema during the session, and a mismatch resets silently.
           const local = new Cache(prepared.cacheDir, undefined, prepared.workspaceRoot)
           try {
-            local.recordRunBundle(recordsOf(final, ok && first.code === 0))
+            local.recordRunBundle(recordsOf(final, ok && !crashed))
           } finally {
             local.close()
           }
         })
         recordFailures(final)
       }
-      return { ok: ok && first.code === 0, outcomes: final }
+      return { ok: ok && !crashed, outcomes: final }
     }
 
     return { ok, outcomes: list }
@@ -1470,6 +1690,7 @@ async function runOnBus(
     // floor for a throw a later change adds, not a path a row can drive.
     log.runEnd?.()
     log.runDone?.()
+    live?.dispose()
     stop.detach()
     // Plugins installed at the top of run() get their bus subscriptions
     // released here. Idempotent; safe even if installPlugins threw.
@@ -1527,8 +1748,8 @@ export async function planRun(options: RunOptions): Promise<RunPlan> {
   const log = options.log ?? defaultLogger(undefined, undefined, process.stderr)
   const reasons = new Map<string, AffectedReason>()
   const prepared = await prepareRun({ ...options, affectedReasons: reasons }, log)
-  options = withRunSettings(options, prepared.workspaceConfig)
   try {
+    options = withRunSettings(options, prepared.workspaceConfig)
     if (prepared.unresolvedTasks.length > 0) {
       return {
         tasks: [],
@@ -1548,6 +1769,7 @@ export async function planRun(options: RunOptions): Promise<RunPlan> {
     if (prepared.empty !== null) return { tasks: [] }
     // The plan orders nothing, but it refuses what the run would refuse.
     await prepared.priorities()
+    terminalHolders(prepared.nodes, options.tty === true)
     // Its own mark: a dry run's plan (every task's hash, the cache lookups,
     // the history p50s) was booked under `close`, the next mark, and read
     // as 105 ms of closing a cache at 1,000 projects (item 601).
@@ -1570,13 +1792,14 @@ export async function planRun(options: RunOptions): Promise<RunPlan> {
         prepared.localCache.dbHandle(),
         undefined,
         prepared.branch(),
+        taskDefinitions(prepared.nodes.values()),
       ),
       // Placement, by the same rules the run applies. Only worth showing
       // when there is a choice to show — with one declared executor every
       // line would carry the same label. Resolving the executors here is
       // the same plugin-factory call `prepareRun` already makes for the
       // cache capability, so plan mode gains no new class of side effect.
-      ...(await planExecutorOf(prepared, log, options.download ?? 'all')),
+      ...(await planExecutorOf(prepared, log, options.download ?? 'all', detectCi(process.env).ci)),
       ...(await planRestorable(prepared, policy, options.forwardArgs)),
     })
     mark('plan')
@@ -1647,12 +1870,15 @@ async function unresolvedHint(prepared: {
   hintProjects: ReadonlyMap<string, ProjectEntry>
   declaredElsewhere: readonly string[]
   anyProjectConfig: boolean
+  taskless: (name: string) => boolean
+  unlockedConfig: () => string | undefined
 }): Promise<string> {
   const elsewhere = elsewhereHint(prepared.declaredElsewhere)
   const near = didYouMean(
     prepared.unresolvedTasks,
     prepared.hintProjects,
     prepared.declaredElsewhere,
+    prepared.taskless,
   )
   const init = await initHint(prepared)
   const lock =
@@ -1663,13 +1889,20 @@ async function unresolvedHint(prepared: {
 }
 
 /**
- * The first-run case, told apart from a typo: no project in the workspace
- * has a `vx.config.*` at all, so no name could have resolved. Reached only
- * on the error path.
+ * The first-run case, told apart from a typo: the lock holds no project,
+ * so no name could have resolved. Said of the lock, the only thing a run
+ * reads: "no directory holds a vx.config" was false beside configs never
+ * locked. A config written after the lock is no first run: its fix is
+ * `vx lock`. Reached only on the error path.
  */
-async function initHint(prepared: { anyProjectConfig: boolean }): Promise<string> {
+async function initHint(prepared: {
+  anyProjectConfig: boolean
+  unlockedConfig: () => string | undefined
+}): Promise<string> {
   if (prepared.anyProjectConfig) return ''
-  return ' No directory holds a vx.config — run `vx init` to write them, then `vx lock`.'
+  const unlocked = prepared.unlockedConfig()
+  if (unlocked !== undefined) return ` ${unlocked} is not in the lock yet — run \`vx lock\`.`
+  return ` ${EMPTY_LOCK}.`
 }
 
 /**
@@ -1682,7 +1915,8 @@ async function initHint(prepared: { anyProjectConfig: boolean }): Promise<string
 function didYouMean(
   unresolved: readonly string[],
   projects: ReadonlyMap<string, ProjectEntry>,
-  elsewhere: readonly string[] = [],
+  elsewhere: readonly string[],
+  taskless: (name: string) => boolean,
 ): string {
   const tasksOf = (p: ProjectEntry | undefined): string[] => Object.keys(p?.config.tasks ?? {})
   // What a bare name can select: a default `build` is no match (X-102).
@@ -1727,10 +1961,20 @@ function didYouMean(
       )
       continue
     }
+    if (taskless(proj)) {
+      lists.add(
+        `${proj} has no vx config, and no configured project depends on it: it has no tasks.`,
+      )
+      continue
+    }
     if (!projects.has(proj)) {
       const p = projectNamed(proj, projects) ?? nearest(proj, projects.keys())
-      if (p !== undefined && tasksOf(projects.get(p)).includes(task)) hints.add(`${p}#${task}`)
-      else if (p === undefined && projects.size > 0) {
+      if (p !== undefined) {
+        const tasks = tasksOf(projects.get(p))
+        const t = tasks.includes(task) ? task : nearest(task, tasks)
+        if (t !== undefined) hints.add(`${p}#${t}`)
+        else lists.add(`No project is named ${proj}; ${p}'s tasks: ${listed(tasks)}.`)
+      } else if (projects.size > 0) {
         lists.add(`No project is named ${proj}; projects: ${listed(projects.keys())}.`)
       }
       continue
@@ -1816,3 +2060,89 @@ function canonical(dir: string): string {
     return path.resolve(dir)
   }
 }
+
+/** What an executor's report threw: said once by `executorFaults`, the report left out. */
+type ExecutorFault = (e: TaskExecutor, what: string, err: unknown) => void
+
+/**
+ * Each executor report that throws is said once and left out: isolated,
+ * it was dropped with nothing said, a summary short its machines.
+ */
+function executorFaults(warn: (line: string) => void): ExecutorFault {
+  const said = new Set<string>()
+  return (e, what, err) => {
+    const k = `${e.name}\0${what}`
+    if (said.has(k)) return
+    said.add(k)
+    warn(`[vx] ${executorLabel(e)}: ${what}() threw: ${thrownMessage(err)} — left out`)
+  }
+}
+
+/** An executor's footer rows; a plugin's throw or bad shape shows none. */
+function executorFooter(e: TaskExecutor, fault: ExecutorFault): (readonly [string, ui.Inline])[] {
+  try {
+    const rows = e.footer?.()
+    if (!Array.isArray(rows)) return []
+    return rows.filter(
+      (r): r is readonly [string, ui.Inline] =>
+        Array.isArray(r) &&
+        typeof r[0] === 'string' &&
+        (typeof r[1] === 'string' || (typeof r[1] === 'object' && r[1] !== null)),
+    )
+  } catch (err) {
+    fault(e, 'footer', err)
+    return []
+  }
+}
+
+/** An executor's `--profile` events; a throw or a bad shape adds none. */
+function executorProfile(
+  e: TaskExecutor,
+  fault: ExecutorFault,
+): readonly Record<string, unknown>[] {
+  try {
+    const events = e.profile?.()
+    return Array.isArray(events)
+      ? events.filter((x): x is Record<string, unknown> => typeof x === 'object' && x !== null)
+      : []
+  } catch (err) {
+    fault(e, 'profile', err)
+    return []
+  }
+}
+
+function executorMachines(e: TaskExecutor, fault: ExecutorFault): readonly MachineReport[] {
+  try {
+    const m = e.machines?.()
+    return Array.isArray(m) ? m : []
+  } catch (err) {
+    fault(e, 'machines', err)
+    return []
+  }
+}
+
+function executorIdeal(e: TaskExecutor, fault: ExecutorFault): number | undefined {
+  try {
+    const ms = e.idealMs?.()
+    return typeof ms === 'number' && Number.isFinite(ms) && ms >= 0 ? Math.round(ms) : undefined
+  } catch (err) {
+    fault(e, 'idealMs', err)
+    return undefined
+  }
+}
+
+/** Where `e` says a task runs, or `null` while it waits; a throw or anything else says nothing. */
+function executorWhere(
+  e: TaskExecutor,
+  id: string,
+  fault: ExecutorFault,
+): string | null | undefined {
+  try {
+    const w = e.running?.(id)
+    if (w === null) return null
+    return typeof w === 'string' && w !== '' ? w : undefined
+  } catch (err) {
+    fault(e, 'running', err)
+    return undefined
+  }
+}