@vzn/vx 0.0.645 → 0.0.648

package/src/graph/scheduler.ts

@@ -4,6 +4,8 @@ import {
   isOutOfFds,
   isUserError,
   OUT_OF_FDS_HINT,
+  thrownMessage,
+  ui,
 } from '../util/index.js'
 import {
   computeReverseDepCount,
@@ -98,6 +100,12 @@ export interface TaskOutcome {
   /** v11 analytics: CPU time + peak RSS for this task's child process. */
   cpuMs?: number
   peakRssBytes?: number
+  /**
+   * The most cores one sampling interval burned and the mean resident
+   * memory over the looks; only when a sink samples and a look landed.
+   */
+  peakCores?: number
+  avgRssBytes?: number
   /**
    * A GROUP task's own dependency outcomes — the tasks it stands for.
    * A group has no `exec` and therefore no outputs and no cache entry: its
@@ -116,11 +124,11 @@ export interface TaskOutcome {
    */
   unkeyed?: true
   /**
-   * This task's key folds a task with no `inputs` (or it is one): no run
-   * can derive that key again, so it neither read nor saved an entry, and a
-   * dependant that folds `hash` does neither.
+   * The task with no `inputs` this task's key folds (this task, or one
+   * upstream): no run can derive that key again, so it neither read nor
+   * saved an entry, and a dependant that folds `hash` does neither.
    */
-  keyless?: true
+  keylessBy?: string
   /**
    * The run's cache policy read and wrote nothing (`--no-cache`,
    * `--cache-policy=local:`): no cache answered for this task, so a run of it is
@@ -135,6 +143,12 @@ export interface TaskOutcome {
    * read this one field (item 267).
    */
   blockedBy?: string
+  /**
+   * On a `skipped` outcome: its `exec.platforms` leave out this machine.
+   * Its dependants are skipped with `blockedBy` naming it, and neither
+   * fails the run.
+   */
+  otherPlatform?: { readonly needs: readonly string[]; readonly host: string }
   /**
    * On a `failed` outcome: vx's own `timeout` killed the final attempt.
    * The exit is the shell's 143, and every label says "timed out" instead
@@ -159,6 +173,12 @@ export interface TaskOutcome {
    * the spawn itself failed. Every label reads this instead of the exit.
    */
   notReady?: 'timeout' | 'exited' | 'spawn'
+  /**
+   * On a ready persistent task: its child exited on its own while the run
+   * went on — a setup step's 0 at ready, a crash's code after it — so no
+   * label calls it running.
+   */
+  serverExit?: number
   /**
    * The input that changed while the command ran, failing the task under
    * `inputStability.severity: 'error'` (workspace-relative; '' when vx
@@ -166,8 +186,15 @@ export interface TaskOutcome {
    */
   inputMoved?: string
   /** Executor-reported placement label (`ExecuteResult.where`) — set only
-   *  when the task ran somewhere other than this host. Telemetry-only. */
+   *  when the task ran somewhere other than this host: telemetry and the task's row. */
   where?: string
+  /** A persistent task each dependant's executor starts where it runs: nothing ran here, and it is no execution of its own. */
+  carried?: true
+  /** The cores and memory of the machine `where` names (`ExecuteResult.machine`). */
+  whereCores?: number
+  whereMemoryBytes?: number
+  /** `ExecuteResult.machine.cpuShare`: the share of a core its threads got there. */
+  cpuShare?: number
   /** `'deferred'` when the task's outputs were left in the remote store
    *  (`--cache-download=none`) instead of landing on this machine. Absent for
    *  every ordinary outcome. Telemetry-only. */
@@ -201,7 +228,7 @@ export interface TaskOutcome {
   failedAttempts?: readonly { endedAt: number; exitCode: number; timedOut?: true }[]
   /**
    * Set when this run proved the task flaky: its key failed after it had
-   * passed (this run's included) or it needed a retry. Its row carries it.
+   * passed (this run's included) or it passed only on a retry. Its row carries it.
    */
   flaky?: { passes: number; failures: number }
   /**
@@ -294,14 +321,16 @@ export interface ScheduleOptions {
    */
   settledOf?: (outcome: TaskOutcome) => Promise<Partial<TaskOutcome> | void> | undefined
   /**
-   * The outcome of a task the caller settles in place, or undefined to
-   * dispatch it through `execute`. Asked of an exec-tier task the lane has
-   * room for, after the skip check; its outcome lands in the same tick, it
-   * holds no slot, and `onStart` / `onFinish` hear it as any other. An
-   * unkeyed group runs nothing, and the dispatch, the slot and the promise
-   * around it were ~10 µs of each of a warm run's 1,090 groups (X-192).
+   * A task the caller settles in place: a thunk returning its outcome, or
+   * undefined to dispatch it through `execute`. Asked of a task its lane
+   * has room for, after the skip check; `onStart` hears it, then the thunk
+   * runs (a hit replays its output there), and its outcome lands in the
+   * same tick, holding no slot. An unkeyed group runs nothing, and the
+   * dispatch, the slot and the promise around it were ~10 µs of each of a
+   * warm run's 1,090 groups (X-192); a hit whose tree is current does no
+   * more. A restore-tier task's `upstream` may hold undefined entries.
    */
-  settleNow?: (node: TaskNode, upstream: TaskOutcome[]) => TaskOutcome | undefined
+  settleNow?: (node: TaskNode, upstream: TaskOutcome[]) => (() => TaskOutcome) | undefined
   /**
    * Optional priority override: callers pass their own per-node weight
    * (e.g. `computePredictedPriorities` from the orchestrator's history
@@ -344,6 +373,14 @@ export interface ScheduleOptions {
    * second cache.get. When undefined/empty, behavior is byte-identical.
    */
   restoreTier?: ReadonlySet<string>
+  /**
+   * Servers held back until the caller knows whether their dependants are
+   * hits a remote holds. Each resolves to the dependants it has added to
+   * `restoreTier` since the run began (the caller's set, read live); a
+   * server whose every dependant is then in the tier is idle, as above,
+   * and any other is dispatched as usual.
+   */
+  heldServers?: ReadonlyMap<string, Promise<readonly string[]>>
   /**
    * Pool assignment for tasks placed on an executor with its own capacity
    * (a remote pool). Such a task occupies one of the pool's `capacity` slots
@@ -386,7 +423,12 @@ class ReadyHeap {
   // was two per heap level on every push and pop.
   private readonly prio: number[] = []
   private next = 0
-  constructor(private priority: ReadonlyMap<string, number>) {}
+  /** Queued ids `counts` holds true for. */
+  counted = 0
+  constructor(
+    private priority: ReadonlyMap<string, number>,
+    private readonly counts?: (id: string) => boolean,
+  ) {}
   get size(): number {
     return this.ids.length
   }
@@ -441,6 +483,7 @@ class ReadyHeap {
    * it holds without that argument.
    */
   push(id: string, seq: number = this.next++): void {
+    if (this.counts?.(id) === true) this.counted++
     this.ids.push(id)
     this.seq.push(seq)
     this.prio.push(this.priority.get(id) ?? 0)
@@ -460,6 +503,7 @@ class ReadyHeap {
     const n = this.ids.length
     if (n === 0) return undefined
     const top = this.ids[0]!
+    if (this.counts?.(top) === true) this.counted--
     const lastId = this.ids.pop()!
     const lastSeq = this.seq.pop()!
     const lastPrio = this.prio.pop()!
@@ -571,10 +615,50 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   // (`dormant`) until a dependant is demoted, which starts it, or until
   // every dependant has finished, which settles it unspawned. Settled
   // without an outcome: it is not part of the run, as a pruned task is not.
-  const idle = idleServers(nodes, dependents, restoreTier)
+  // A group between it and its hits is idle with it.
+  const idleGroups = new Set<string>()
+  const idle = idleServers(nodes, dependents, restoreTier, idleGroups)
+  for (const g of idleGroups) idle.add(g)
   const dormant = new Set<string>()
+  // The idle tasks below `id`, through idle groups: those its outcome may
+  // let settle, and those its demotion wakes.
+  const idleBelow = (id: string, each: (p: string) => void): void => {
+    const stack = [...(nodes.get(id) as TaskNode).deps]
+    const seen = new Set<string>()
+    while (stack.length > 0) {
+      const p = stack.pop()!
+      if (seen.has(p) || !idle.has(p)) continue
+      seen.add(p)
+      each(p)
+      if (idleGroups.has(p)) stack.push(...(nodes.get(p) as TaskNode).deps)
+    }
+  }
+  // Every task above an idle one, through idle groups, has finished.
+  const settledAbove = (id: string): boolean => {
+    const stack = [id]
+    const seen = new Set<string>()
+    while (stack.length > 0) {
+      for (const d of dependents.get(stack.pop()!) ?? []) {
+        if (outcomes.has(d)) continue
+        if (!idleGroups.has(d) || !idle.has(d)) return false
+        if (!seen.has(d)) {
+          seen.add(d)
+          stack.push(d)
+        }
+      }
+    }
+    return true
+  }
+  // Held servers not yet answered, and those of them that became ready.
+  const unanswered = new Set(options.heldServers?.keys() ?? [])
+  for (const id of idle) unanswered.delete(id)
+  const heldReady = new Set<string>()
   let settledIdle = 0
-  const execReady = new ReadyHeap(priority)
+  // Counting those no pool takes: with every pool full, only one of them can start.
+  const execReady = new ReadyHeap(
+    priority,
+    options.poolOf === undefined ? undefined : (id) => options.poolOf!(id) === undefined,
+  )
   const restoreReady = new ReadyHeap(priority)
   options.reprioritize?.((weights) => {
     const merged = mergePriorities(baseline, weights)
@@ -595,6 +679,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
     else continue
     readyAt.set(node.id, startedAt)
   }
+  const promoted = new Set<string>()
 
   let active = 0
   // Restore-tier work is disk I/O — a confirmed hit's tar extract is ~8
@@ -612,8 +697,9 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   const restoreRoom = (): boolean =>
     serial ? active + activeRestore < 1 : activeRestore < restoreConcurrency
   let resolved = false
+  let stopListening = (): void => {}
   // A refusal's text → the first task that said it. A corrupt cache index
-  // reaches every lookup as one UserError; a repeat names the first.
+  // reaches every lookup as one UserError; on bare stderr a repeat names the first.
   const refusedBy = new Map<string, string>()
 
   // Admission policy over the count limit (a plugin's `admit`). Inactive
@@ -633,6 +719,18 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   // against a real executor (and never parks).
   const poolOf = options.poolOf
   const poolActive = new Map<string, number>()
+  // Every pool a task of the run is placed in, by name: a task's pool is
+  // fixed for the run, so read once here.
+  const pools = new Map<string, number>()
+  if (poolOf !== undefined)
+    for (const id of nodes.keys()) {
+      const p = poolOf(id)
+      if (p !== undefined) pools.set(p.name, p.capacity)
+    }
+  const poolRoom = (): boolean => {
+    for (const [name, capacity] of pools) if ((poolActive.get(name) ?? 0) < capacity) return true
+    return false
+  }
   // A pooled task is admitted against ITS pool's capacity, a local one
   // against `concurrency`; pooled tasks reserve no local resources. A
   // restore is local disk work wherever its task was placed: on the pool's
@@ -701,7 +799,9 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
       // A dormant server is asked again: the dispatch settles it once its
       // last dependant is in.
       if (dormant.size > 0) {
-        for (const p of (nodes.get(id) as TaskNode).deps) if (dormant.delete(p)) execReady.push(p)
+        idleBelow(id, (p) => {
+          if (dormant.delete(p)) execReady.push(p)
+        })
       }
       // Observer hook (the logger's taskComplete). Crash-isolated: a
       // throwing observer must NOT break scheduling — it would otherwise
@@ -712,8 +812,8 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
       try {
         onFinish?.(outcome)
       } catch (err) {
-        const m = err instanceof Error ? err.message : String(err)
-        process.stderr.write(`[vx] onFinish observer threw for ${id}: ${m}\n`)
+        const m = thrownMessage(err)
+        ui.say('warn', `[vx] onFinish observer threw for ${id}: ${m}\n`)
       }
       // A restore-tier task runs before its deps, but its DEPENDENTS keep
       // the order `dependsOn` states: they are released once its own deps
@@ -734,16 +834,19 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
     // one carries.
     const blockedRestores = new Map<string, string | undefined>()
     const heldRelease = new Set<string>()
-    const release = (id: string): void => {
-      if (inRestoreTier(id) && continueMode !== 'always') {
+    const blockOf = (id: string): void => {
+      if (inRestoreTier(id)) {
         for (const dep of (nodes.get(id) as TaskNode).deps) {
           const u = outcomes.get(dep)
-          if (u?.status === 'failed' || u?.status === 'aborted') {
+          if (continueMode !== 'always' && (u?.status === 'failed' || u?.status === 'aborted')) {
             blockedRestores.set(id, dep)
             break
           }
           if (u?.status === 'skipped') {
-            blockedRestores.set(id, u.blockedBy)
+            blockedRestores.set(
+              id,
+              u.blockedBy ?? (u.otherPlatform !== undefined ? dep : undefined),
+            )
             break
           }
           if (blockedRestores.has(dep)) {
@@ -752,16 +855,33 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
           }
         }
       }
-      const ds = dependents.get(id)
-      if (!ds) return
-      for (const d of ds) {
+    }
+    // Its block, then its dependents: one whose pending hits 0 is enqueued,
+    // or, when it is a held hit, released in turn before the next sibling.
+    // A frame stack, not recursion: a chain of held hits is released in one
+    // walk, and recursion threw RangeError on a 30,000-deep one.
+    const release = (root: string): void => {
+      const frames: { ds: readonly string[]; i: number }[] = []
+      const enter = (id: string): void => {
+        blockOf(id)
+        const ds = dependents.get(id)
+        if (ds !== undefined) frames.push({ ds, i: 0 })
+      }
+      enter(root)
+      while (frames.length > 0) {
+        const f = frames[frames.length - 1]!
+        if (f.i === f.ds.length) {
+          frames.pop()
+          continue
+        }
+        const d = f.ds[f.i++]!
         const rem = (pending.get(d) ?? 0) - 1
         pending.set(d, rem)
         if (rem !== 0) continue
         // Restore-tier dependents were already enqueued at startup (they
         // don't wait on deps); one that finished early releases its own
         // dependents now. Only an exec-tier dependent is enqueued.
-        if (heldRelease.delete(d)) release(d)
+        if (heldRelease.delete(d)) enter(d)
         else if (!inRestoreTier(d)) {
           execReady.push(d)
           readyAt.set(d, Date.now())
@@ -802,8 +922,13 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
         return true
       }
       if (inRestoreTier(id)) return false
-      if (continueMode === 'always') return false
       const node = nodes.get(id) as TaskNode
+      // `always` runs past a failed upstream, never past one that did not
+      // run on this platform: under it, a skip has no other cause.
+      if (continueMode === 'always')
+        return node.deps.some(
+          (d) => outcomes.get(d)?.status === 'skipped' || blockedRestores.has(d),
+        )
       return node.deps.some((d) => {
         const u = outcomes.get(d)
         // `aborted` propagates even though it is counted nowhere: the
@@ -823,7 +948,10 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
     }
 
     // A throw falls back to the dispatch, whose `execute` reports it.
-    const settleInPlace = (node: TaskNode, upstream: TaskOutcome[]): TaskOutcome | undefined => {
+    const settleInPlace = (
+      node: TaskNode,
+      upstream: TaskOutcome[],
+    ): (() => TaskOutcome) | undefined => {
       if (options.settleNow === undefined) return undefined
       try {
         return options.settleNow(node, upstream)
@@ -831,6 +959,14 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
         return undefined
       }
     }
+    const started = (node: TaskNode): void => {
+      try {
+        onStart?.(node)
+      } catch (err) {
+        const m = thrownMessage(err)
+        ui.say('warn', `[vx] onStart observer threw for ${node.id}: ${m}\n`)
+      }
+    }
 
     const tick = (): void => {
       if (resolved) return
@@ -857,12 +993,19 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
         // with a wide frontier that is O(R) per tick, O(R²) per run (the
         // 6,000-task scale pin went 0.5 s → 28 s when the restore lane's
         // first cut let the scan run past a full exec lane, 2026-09-10).
-        // This is the legacy O(1) gate, kept per lane.
-        const execAdmissible = poolOf !== undefined || execRoom()
+        // This is the legacy O(1) gate, kept per lane. Every pool full is
+        // the same: a remote pool of 6 slots before 2000 ready tasks
+        // re-popped them all on each end (DTE, 2026-10-25).
+        const execAdmissible =
+          poolOf === undefined ? execRoom() : (execRoom() && execReady.counted > 0) || poolRoom()
         if (execAdmissible) {
           while (execReady.size > 0) {
             const seq = execReady.peekSeq()
             const id = execReady.pop() as string
+            if (unanswered.has(id) && !willSkip(id)) {
+              heldReady.add(id)
+              continue
+            }
             if (willSkip(id) || idle.has(id)) return id
             if (exclusive?.has(id) === true) {
               if (!busy() && (!admitActive || admits(id))) return id
@@ -892,35 +1035,60 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
         // nothing on the restore path may.
         const upstream = node.deps.map((d) => outcomes.get(d) as TaskOutcome)
 
+        // An idle task the stop or fail-fast would skip settles with no
+        // outcome, as it would have run: the run never meant to start it.
+        // Below a failed dep it is skipped as before, so the hits above it
+        // pass the block down.
+        const settlesIdle =
+          idle.has(id) &&
+          (!willSkip(id) ||
+            aborted() ||
+            (failFastTripped &&
+              !node.deps.some((d) => {
+                const s = outcomes.get(d)?.status
+                return (
+                  s === 'failed' || s === 'skipped' || s === 'aborted' || blockedRestores.has(d)
+                )
+              })))
+
         // If any upstream failed/skipped, propagate skip synchronously
         // without running. Skipped tasks still flow through this queue
         // because dependents are pushed when `pending` hits 0 regardless
         // of outcome — keeps the propagation logic in one place.
-        if (willSkip(id)) {
+        if (!settlesIdle && willSkip(id)) {
           // The root of the block: a failed or aborted upstream names
           // itself; a skipped one hands down its own root.
-          const blocker = upstream.find(
-            (u) => u !== undefined && (u.status === 'failed' || u.status === 'aborted'),
-          )
+          const blocker =
+            continueMode === 'always'
+              ? undefined
+              : upstream.find(
+                  (u) => u !== undefined && (u.status === 'failed' || u.status === 'aborted'),
+                )
           const viaSkip = upstream.find((u) => u !== undefined && u.status === 'skipped')
           const viaRestore = node.deps.find((d) => blockedRestores.has(d))
           const blockedBy =
             blocker?.node.id ??
             node.deps.map(deadServerVia).find((d) => d !== undefined) ??
             viaSkip?.blockedBy ??
+            (viaSkip?.otherPlatform !== undefined ? viaSkip.node.id : undefined) ??
             (viaRestore === undefined ? undefined : blockedRestores.get(viaRestore))
+          // Behind a platform skip is no failure: the run passes, so the exit does.
+          const notHere =
+            !aborted() &&
+            blockedBy !== undefined &&
+            outcomes.get(blockedBy)?.otherPlatform !== undefined
           finishOne(id, {
             node,
             status: aborted() ? 'aborted' : 'skipped',
-            exitCode: 1,
+            exitCode: notHere ? 0 : 1,
             durationMs: 0,
             ...(blockedBy !== undefined ? { blockedBy } : {}),
           })
           continue
         }
-        if (idle.has(id)) {
+        if (settlesIdle) {
           readyAt.delete(id)
-          if (dependents.get(id)!.every((d) => outcomes.has(d))) {
+          if (settledAbove(id)) {
             settledIdle++
             release(id)
           } else dormant.add(id)
@@ -929,14 +1097,17 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
 
         const queuedMs = Date.now() - (readyAt.get(id) ?? Date.now())
         readyAt.delete(id)
-        const now = inRestoreTier(id) ? undefined : settleInPlace(node, upstream)
-        if (now !== undefined) {
+        const settle = settleInPlace(node, upstream)
+        let now: TaskOutcome | undefined
+        if (settle !== undefined) {
+          if (!demoted.has(id)) started(node)
           try {
-            onStart?.(node)
-          } catch (err) {
-            const m = err instanceof Error ? err.message : String(err)
-            process.stderr.write(`[vx] onStart observer threw for ${id}: ${m}\n`)
+            now = settle()
+          } catch {
+            // Dispatched below, already started: `execute` reports it.
           }
+        }
+        if (now !== undefined) {
           finishOne(id, queuedMs === 0 ? now : { ...now, queuedMs })
           continue
         }
@@ -957,15 +1128,9 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
               }
         // Crash-isolated observer hook — a throwing onStart must not abort
         // the dispatch loop (it would strand the tick with the slot held).
-        // A demoted task's second dispatch is the same task, already started.
-        if (!demoted.has(id)) {
-          try {
-            onStart?.(node)
-          } catch (err) {
-            const m = err instanceof Error ? err.message : String(err)
-            process.stderr.write(`[vx] onStart observer threw for ${id}: ${m}\n`)
-          }
-        }
+        // A demoted task's second dispatch is the same task, already started,
+        // as is one whose settle threw.
+        if (!demoted.has(id) && settle === undefined) started(node)
 
         // `.then(onFulfilled, onRejected)` — NOT `.then(f).catch(g)`. The
         // rejection arm handles ONLY `execute()` rejecting; a throw from
@@ -1005,12 +1170,13 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
               leave()
               untrack()
               demoted.add(id)
-              for (const p of node.deps) {
-                if (idle.delete(p) && dormant.delete(p)) {
+              idleBelow(id, (p) => {
+                idle.delete(p)
+                if (dormant.delete(p)) {
                   execReady.push(p)
                   readyAt.set(p, Date.now())
                 }
-              }
+              })
               if (pending.get(id) === 0) {
                 execReady.push(id)
                 readyAt.set(id, Date.now())
@@ -1018,7 +1184,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
               tick()
               return
             }
-            const message = err instanceof Error ? err.message : String(err)
+            const message = thrownMessage(err)
             // After the run's stop a rejection is the stop's doing (a probe
             // it killed, C-65): aborted, as any task it kills, and unsaid.
             const stopped = aborted()
@@ -1045,8 +1211,11 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
               const text = isUserError(err)
                 ? message
                 : `${message} — ${isOutOfFds(err) ? OUT_OF_FDS_HINT : fsRefusalHint(err)}`
-              const first = refusedBy.get(text)
-              if (first === undefined) refusedBy.set(text, id)
+              // In a task's own output block, in full: that block is read
+              // alone, and "as <task> above" named one printed elsewhere or
+              // not at all (2026-10-22). On bare stderr the first is above.
+              const first = options.onError === undefined ? refusedBy.get(text) : undefined
+              if (!refusedBy.has(text)) refusedBy.set(text, id)
               line = `[vx] ${id}: ${first === undefined ? text : `as ${first} above`}\n`
             } else {
               const named = err instanceof Error && err.name !== 'Error' ? `${err.name}: ` : ''
@@ -1062,10 +1231,10 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
               line = `[vx] internal error in ${id}: ${named}${message}${because}\n`
             }
             try {
-              if (options.onError === undefined) process.stderr.write(line)
+              if (options.onError === undefined) ui.say('error', line)
               else options.onError(node, line, err)
             } catch {
-              process.stderr.write(line)
+              ui.say('error', line)
             }
             finishOne(id, outcome)
             tick()
@@ -1085,26 +1254,118 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
       // condition survived the whole core suite (item 644).
       if (outcomes.size + settledIdle === nodes.size) {
         resolved = true
+        stopListening()
         resolve(outcomes)
       }
     }
 
+    // A held server's answer: the dependants it found go on the restore
+    // lane, and the server is idle when they were all of them. One whose
+    // deps all settled is already on the exec queue, running, or done:
+    // pushed again it ran twice (its server skipped, the exec lane full).
+    const answer = (id: string, found: readonly string[]): void => {
+      if (resolved || !unanswered.delete(id)) return
+      for (const d of found) {
+        // In `readyAt`: already queued, released to the exec queue when its
+        // server was skipped before this answer. On both lanes it ran twice.
+        if (
+          promoted.has(d) ||
+          outcomes.has(d) ||
+          readyAt.has(d) ||
+          pending.get(d) === 0 ||
+          !inRestoreTier(d)
+        )
+          continue
+        promoted.add(d)
+        restoreReady.push(d)
+        readyAt.set(d, Date.now())
+      }
+      if (dependents.get(id)?.every((d) => inRestoreTier(d)) === true) idle.add(id)
+      if (heldReady.delete(id)) execReady.push(id)
+      tick()
+    }
+    for (const [id, found] of options.heldServers ?? []) {
+      if (unanswered.has(id))
+        void found.then(
+          (ids) => answer(id, ids),
+          () => answer(id, []),
+        )
+    }
+    // A stop reaches the scheduler through a completion, and a held server
+    // waiting on the remote has none: the run waited out the remote's answer.
+    const signal = options.signal
+    if (unanswered.size > 0 && signal !== undefined) {
+      const stop = (): void => {
+        for (const id of [...unanswered]) answer(id, [])
+      }
+      signal.addEventListener('abort', stop, { once: true })
+      stopListening = () => signal.removeEventListener('abort', stop)
+    }
+
     tick()
   })
 }
 
-/** Persistent tasks nobody asked for whose every dependant is a restore-tier hit. */
+/**
+ * Persistent tasks nobody asked for whose every dependant is a restore-tier
+ * hit, or a group nobody asked for that only such dependants stand behind:
+ * a group runs nothing, so `env: { dependsOn: ['dev'] }` between the server
+ * and its hits booted the server on every warm run. Those groups go in
+ * `groups`.
+ */
 export function idleServers(
   nodes: ReadonlyMap<string, TaskNode>,
   dependents: ReadonlyMap<string, string[]>,
   restoreTier: ReadonlySet<string> | undefined,
+  groups?: Set<string>,
 ): Set<string> {
   const out = new Set<string>()
   if (restoreTier === undefined || restoreTier.size === 0) return out
+  const unasked = (n: TaskNode): boolean => !n.requested && n.surfaced !== true
+  // Per group: whether only hits stand behind it. Post-order on a stack, as
+  // groups nest as deep as the graph.
+  const spare = new Map<string, boolean>()
+  const spareGroup = (root: string): boolean => {
+    const stack = [root]
+    while (stack.length > 0) {
+      const id = stack[stack.length - 1]!
+      if (spare.has(id)) {
+        stack.pop()
+        continue
+      }
+      const n = nodes.get(id)
+      const ds = dependents.get(id)
+      if (n === undefined || !isGroupTask(n) || !unasked(n) || ds === undefined) {
+        spare.set(id, false)
+        stack.pop()
+        continue
+      }
+      const next = ds.find((d) => !restoreTier.has(d) && !spare.has(d))
+      if (next !== undefined) {
+        stack.push(next)
+        continue
+      }
+      stack.pop()
+      spare.set(
+        id,
+        ds.every((d) => restoreTier.has(d) || spare.get(d) === true),
+      )
+    }
+    return spare.get(root)!
+  }
   for (const n of nodes.values()) {
-    if (n.config.exec?.persistent === undefined || n.requested || n.surfaced === true) continue
+    if (n.config.exec?.persistent === undefined || !unasked(n)) continue
     const ds = dependents.get(n.id)
-    if (ds !== undefined && ds.every((d) => restoreTier.has(d))) out.add(n.id)
+    if (ds === undefined || !ds.every((d) => restoreTier.has(d) || spareGroup(d))) continue
+    out.add(n.id)
+    if (groups === undefined) continue
+    const stack = ds.filter((d) => !restoreTier.has(d))
+    while (stack.length > 0) {
+      const g = stack.pop()!
+      if (groups.has(g)) continue
+      groups.add(g)
+      for (const d of dependents.get(g)!) if (!restoreTier.has(d)) stack.push(d)
+    }
   }
   return out
 }