@vzn/vx 0.0.16 → 0.0.21

package/src/graph/scheduler.ts

@@ -1,4 +1,4 @@
-import { UserError } from '../util/index.js'
+import { isUserError } from '../util/index.js'
 import type { TaskNode } from './task-graph.js'
 
 export type TaskStatus =
@@ -8,65 +8,12 @@ export type TaskStatus =
   | 'failed'
   | 'skipped'
   // A child killed by a shutdown signal (SIGINT/SIGTERM — e.g. Ctrl-C):
-  // the task never finished on its own terms, so it reverts to
-  // aborted — not counted, not shown (the run is tearing down).
+  // the task never finished on its own terms, so it reverts to aborted —
+  // excluded from the tally and the run history. It still PROPAGATES like
+  // a failure (see `willSkip`): the child died mid-write, so its outputs
+  // are partial.
   | 'aborted'
 
-/**
- * Cache-correctness verdict for a task under `vx run --verify` (Phase 1:
- * determinism; Phase 2: input-completeness). Declared here (structurally)
- * because `graph` can't import `orchestrator` where the verifier lives —
- * same pattern as `inputComponents`. A pure side-channel: never hashed,
- * never persisted in Phase 1/2.
- */
-export type VerifyVerdict =
-  | { kind: 'proven-deterministic' }
-  /** Phase 2: input-completeness proved — the task read nothing outside its
-   *  declared inputs (from `--verify=inputs` when determinism wasn't also
-   *  requested; an `--verify=all` pass reports the stronger deterministic). */
-  | { kind: 'proven-complete' }
-  /** Re-ran; outputs differ from the cached ones. `changed` names the rels. */
-  | { kind: 'nondeterministic'; changed: readonly string[] }
-  /** Phase 2: read a workspace path outside the declared inputs — the declared
-   *  `cache.inputs` are incomplete, so a hit could serve stale bytes. `paths`
-   *  names the undeclared reads (workspace-relative; empty when the sandbox
-   *  denied structurally but strace wasn't available to name them). */
-  | { kind: 'undeclared-inputs'; paths: readonly string[] }
-  /** Diverged but the task is on `--verify-allow` — reported, not failed. */
-  | { kind: 'allowed-nondeterministic'; changed: readonly string[] }
-  /** The verify re-run exited non-zero / timed out (nondeterministic by
-   *  definition — identical inputs, different outcome). */
-  | { kind: 'rerun-failed'; exitCode: number }
-  /** Cacheable + executed but declares no outputs — nothing to replay. */
-  | { kind: 'no-outputs' }
-  /** Didn't execute (cache hit) / not cacheable / group / persistent. */
-  | { kind: 'not-verified' }
-
-/**
- * Content fingerprint of a task's output tree, computed under a `--verify*`
- * mode on the executed (miss) path. Never hashed into any key; pure telemetry
- * side-channel for the cross-machine diff (a serve pairs fingerprints for the
- * SAME cache key across platforms and names diverging outputs). Declared here
- * (structurally) because `graph` can't import `orchestrator` — the
- * `VerifyVerdict` pattern. See docs/design/verify-cross-machine-2026-07.md.
- */
-export interface OutputFingerprint {
-  /** Roll-up: xxh3hex over the sorted (key, hash) pairs, folded as
-   *  `key \0 hash \n` (\0 boundaries — the v18 lesson). Always present;
-   *  divergence DETECTION never depends on the per-file map. */
-  tree: string
-  /** Total files in the tree (pre-truncation). */
-  fileCount: number
-  /** Per-file map as sorted [outputKey, xxh3hex] pairs, capped at
-   *  FP_MAX_FILES (500). Deterministic truncation — sorted by key,
-   *  first N — so two machines' truncated maps cover the same subset
-   *  and partial diffs still name real rels. */
-  files?: ReadonlyArray<readonly [string, string]>
-  /** Set when `files` was truncated to the cap (or dropped by the
-   *  sink's run budget). */
-  truncated?: boolean
-}
-
 export interface TaskOutcome {
   node: TaskNode
   status: TaskStatus
@@ -75,9 +22,51 @@ export interface TaskOutcome {
   /** Cache key hash, if one was computed. Folded into dependents' keys
    *  (pure-input transitive — the upstream's input key, not its output). */
   hash?: string
+  /**
+   * For cache-hit statuses: the exec time the ENTRY was stored with — i.e.
+   * the work this hit skipped. Distinct from `durationMs`, which is always
+   * what THIS run spent (a hit's restore cost). The two differ by whatever
+   * the exec:restore ratio happens to be, so a surface reporting "time
+   * saved" must read this one; a surface reporting "time spent" must not.
+   * Undefined on every non-hit outcome — nothing was skipped.
+   */
+  storedDurationMs?: number
+  /**
+   * For cache-hit statuses: what the PRODUCING execution used, read from
+   * the entry (it rides the artifact, so a remote hit on a fresh machine
+   * carries it too). The same split as `storedDurationMs`: `cpuMs` /
+   * `peakRssBytes` are what THIS run spent and stay absent on a hit.
+   */
+  storedCpuMs?: number
+  storedPeakRssBytes?: number
+  /**
+   * How long an `admit` policy held this task after it was ready with a
+   * free worker — the wait the plugin, not the count gate, imposed. Set
+   * only when a policy refused it at least once (so a run with no policy
+   * carries the field on no outcome); `--summarize` rows, the event
+   * stream and the summary footer show it.
+   */
+  admissionHeldMs?: number
   /** v11 analytics: CPU time + peak RSS for this task's child process. */
   cpuMs?: number
   peakRssBytes?: number
+  /**
+   * A GROUP task's own dependency outcomes — the tasks it stands for.
+   * A group has no `exec` and therefore no outputs and no cache entry: its
+   * hash is a synthetic roll-up (`computeGroupHash`), so asking the local
+   * index what it produced returns nothing. A dependent needs the real
+   * tasks beneath it to describe its own input set, and only the group
+   * itself is ever in a position to say which those are. Set on group
+   * outcomes only; never folded into any key.
+   */
+  groupUpstream?: readonly TaskOutcome[]
+  /** Executor-reported placement label (`ExecuteResult.where`) — set only
+   *  when the task ran somewhere other than this host. Telemetry-only. */
+  where?: string
+  /** `'deferred'` when the task's outputs were left in the remote store
+   *  (`--download=none`) instead of landing on this machine. Absent for
+   *  every ordinary outcome. Telemetry-only. */
+  outputs?: 'deferred'
   /**
    * v11 analytics: hrtime span relative to the parent run's t=0.
    * Lets downstream analytics reconstruct the actual parallel timeline
@@ -102,10 +91,10 @@ export interface TaskOutcome {
    */
   attempts?: number
   /**
-   * Count of sandbox violations captured during this task's exec.
-   * Populated only when `--sandbox` was set and the task is cached.
-   * Non-zero values mean the task read files outside its declared
-   * inputs; `cache.save()` was skipped so the result can't be replayed.
+   * Count of sandbox violations captured during this task's exec (a task
+   * declaring `exec.sandbox`). Non-zero means the task touched something
+   * it did not declare; execute-task fails the task on any violation, so
+   * nothing built that way is ever saved or replayed.
    */
   sandboxViolations?: number
   /**
@@ -114,58 +103,46 @@ export interface TaskOutcome {
    * inline in the task's block instead of as loose status output.
    */
   sandboxViolationLines?: string[]
-  /**
-   * Cache-correctness verdict under `vx run --verify`. Set only in verify
-   * mode; a plain run leaves it undefined. Pure side-channel (never hashed).
-   */
-  verify?: VerifyVerdict
-  /**
-   * Output-tree fingerprint under a fingerprinting `--verify*` mode
-   * (`--verify` / `=all` / `=fingerprint`), executed + cacheable +
-   * output-declaring tasks only. A plain run leaves it undefined.
-   */
-  outputFp?: OutputFingerprint
 }
 
 export type ContinueMode = 'never' | 'deps-ok' | 'always'
 
-/**
- * Resolved per-task resource reservation, in absolute units (cpu may be
- * fractional; mem is bytes). Declared here (structurally) because `graph`
- * can't import `orchestrator`, where the resolver lives — same pattern as
- * `VerifyVerdict`. A `0` axis means "reserve nothing, run freely": the
- * task is exempt from that axis entirely (needs no headroom, holds none).
- */
-export interface ResourceCost {
-  cpu: number
-  mem: number
-}
-
-export const ZERO_COST: ResourceCost = { cpu: 0, mem: 0 }
-
 export interface ScheduleOptions {
   nodes: Map<string, TaskNode>
   concurrency: number
+  /**
+   * An aborted run dispatches nothing further: every task not yet
+   * started completes `aborted` (the ones in flight are being killed by
+   * the orchestrator and report `aborted` through the runner's signal).
+   */
+  signal?: AbortSignal
   /**
    * Failure propagation (default 'deps-ok' — the historical behavior):
-   *   - 'deps-ok': a failed/skipped upstream skips its dependents;
-   *     independent siblings keep running.
+   *   - 'deps-ok': a failed/skipped/aborted upstream skips its
+   *     dependents; independent siblings keep running.
    *   - 'never': the first failure stops dispatch — in-flight tasks
    *     finish naturally, everything not yet started completes as
    *     skipped (restore-tier included: a fail-fast run stops
    *     restoring too).
-   *   - 'always': dependents run even when an upstream failed. Sound
-   *     under pure-input transitive hashing: failed outcomes carry the
-   *     upstream's INPUT key (computed before exec), so dependents'
-   *     keys fold exactly what a healthy run folds. An upstream that
-   *     died before hashing folds nothing — that key is derivable by
-   *     no healthy run, so the worst case is an unreachable cache
-   *     entry, never a stale hit.
+   *   - 'always': dependents run even when an upstream failed. Their
+   *     keys are the ones a healthy run derives (failed outcomes carry
+   *     the upstream's INPUT key, computed before exec), which is
+   *     exactly why the orchestrator withholds their SAVE: the bytes
+   *     were built on a partial tree, and a healthy key over them would
+   *     be the next clean run's stale hit (`ExecuteArgs.taintedUpstream`).
    */
   continueMode?: ContinueMode
   execute: (node: TaskNode, upstream: TaskOutcome[]) => Promise<TaskOutcome>
   onStart?: (node: TaskNode) => void
   onFinish?: (outcome: TaskOutcome) => void
+  /**
+   * What an outcome still owes before its dependents may start — a cache
+   * save that runs off the execution slot (the orchestrator's save lane):
+   * a dependent reads the upstream's entry (its output rows travel in the
+   * execute request), so it waits for the save to land, while the freed
+   * slot admits other work at once. Undefined: nothing owed.
+   */
+  settledOf?: (outcome: TaskOutcome) => Promise<void> | undefined
   /**
    * Optional priority override: callers pass their own per-node weight
    * (e.g. `computePredictedPriorities` from the orchestrator's history
@@ -193,17 +170,22 @@ export interface ScheduleOptions {
    */
   restoreTier?: ReadonlySet<string>
   /**
-   * Resolved per-task resource reservations (`exec.resources`, resolved
-   * by the orchestrator against the run's budgets). An absent id means
-   * zero cost; undefined/empty means no task opted in — the scheduler
-   * takes the legacy path byte-identically. Admission control only —
-   * nothing is enforced on the child process.
+   * Pool assignment for tasks placed on an executor with its own capacity
+   * (a remote pool). Such a task occupies one of the pool's `capacity` slots
+   * instead of a local worker slot and reserves no local resources, so a
+   * remote pool is never throttled by the local CPU count. Undefined (or
+   * a `undefined` result) = the local pool.
    */
-  resourceCosts?: ReadonlyMap<string, ResourceCost>
-  /** CPU budget reservations pack against. Defaults to `concurrency`. */
-  cpuBudget?: number
-  /** Memory budget (bytes). Defaults to Infinity (axis off). */
-  memBudget?: number
+  poolOf?: (id: string) => { name: string; capacity: number } | undefined
+  /**
+   * Admission over the worker count: asked for every exec-tier task about
+   * to start on this machine, with the ids of the exec-tier tasks running
+   * here right now (added on dispatch, so two asks in one tick see each
+   * other). `false` parks the task until something finishes. Undefined →
+   * count-only, the legacy path byte for byte. Restore-tier hits and
+   * pooled tasks hold no local resources and are never asked.
+   */
+  admit?: (id: string, running: ReadonlySet<string>) => boolean
 }
 
 /**
@@ -426,73 +408,76 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   }
 
   let active = 0
+  // Restore-tier work is disk I/O — a confirmed hit's tar extract is ~8
+  // filesystem round trips and no CPU — so it gets its own lane, twice
+  // the exec cap: on the 1,000-project bench with every task a restore,
+  // 4 → 8 workers cut the run-graph stage 683–754 → 556–595 ms and 16 was
+  // no better (2026-09-10). Exec-tier work keeps the CPU-shaped cap; the
+  // two lanes never wait on each other. `--concurrency 1` stays serial.
+  let activeRestore = 0
+  const restoreConcurrency = concurrency === 1 ? 1 : 2 * concurrency
   let resolved = false
 
-  // Resource admission (2-D bin packing over the count limit). Inactive
-  // (no task opted in) → the tick loop short-circuits before any of this
-  // and behaves byte-identically to the count-only scheduler.
-  const costs = options.resourceCosts
-  const resourcesActive = costs !== undefined && costs.size > 0
-  const cpuBudget = options.cpuBudget ?? concurrency
-  const memBudget = options.memBudget ?? Infinity
-  // Reservations are FLOAT sums (fractional cpus, and percent-of-budget
-  // resolves to non-representable values like `0.30000000000000004`), so
-  // add/release cycles leave ~1e-17 residue instead of an exact 0. Two
-  // guards keep that residue from corrupting admission:
-  //   - the solo-clamp gate ("is the axis idle?") reads INTEGER holder
-  //     counts, never the float sum === 0 — a residue would otherwise
-  //     park an over-budget task forever (active===0, no future tick =
-  //     a silent hang / exit-0-without-running);
-  //   - `reserved` snaps back to EXACT 0 whenever its holder count hits
-  //     0, so residue can't accumulate across busy periods.
-  // The within-budget comparison also carries a tiny relative epsilon so
-  // an exact-fill (`reserved + cost == budget`) can't mis-round into a
-  // spurious block. Admission is a hint, so the epsilon's sub-ulp
-  // over-admission is harmless.
-  let reservedCpu = 0
-  let reservedMem = 0
-  let holdersCpu = 0
-  let holdersMem = 0
+  // Admission policy over the count limit (a plugin's `admit`). Inactive
+  // → the tick loop short-circuits before any of this and behaves
+  // byte-identically to the count-only scheduler.
+  const admitPolicy = options.admit
+  const admitActive = admitPolicy !== undefined
+  const running = new Set<string>()
+  // When a policy first refused each parked task — so its outcome can say
+  // how long the policy, not the count gate, held it. Nothing with no
+  // policy: the map is never touched.
+  const heldSince = new Map<string, number>()
 
   // A restore-tier task is a confirmed local cache hit: its "execution"
   // is a cheap tar extract, not the task's real work — it reserves ZERO
   // regardless of what the config declares, so it never holds budget
   // against a real executor (and never parks).
-  const costOf = (id: string): ResourceCost =>
-    options.restoreTier?.has(id) ? ZERO_COST : (costs?.get(id) ?? ZERO_COST)
-
-  // Zero never blocks; a within-budget cost needs headroom (with an
-  // exact-fill epsilon); an over-budget cost can never have headroom, so
-  // it solo-clamps: admitted only when the axis is idle (no holders — an
-  // idle pool always admits at least one ready task, no deadlock).
-  const fitsAxis = (cost: number, reserved: number, holders: number, budget: number): boolean =>
-    cost === 0 ? true : cost <= budget ? reserved + cost <= budget + budget * 1e-9 : holders === 0
-
-  const fits = (id: string): boolean => {
-    const c = costOf(id)
-    return (
-      fitsAxis(c.cpu, reservedCpu, holdersCpu, cpuBudget) &&
-      fitsAxis(c.mem, reservedMem, holdersMem, memBudget)
-    )
+  const poolOf = options.poolOf
+  const poolActive = new Map<string, number>()
+  // A pooled task is admitted against ITS pool's capacity, a local one
+  // against `concurrency`; pooled tasks reserve no local resources.
+  const hasRoom = (id: string): boolean => {
+    const pool = poolOf?.(id)
+    if (pool === undefined) {
+      return options.restoreTier?.has(id)
+        ? activeRestore < restoreConcurrency
+        : active < concurrency
+    }
+    return (poolActive.get(pool.name) ?? 0) < pool.capacity
   }
-
-  // Reserve/release keep the float sum AND the integer holder count in
-  // lockstep; releasing the last holder on an axis snaps its sum to 0.
-  const reserve = (c: ResourceCost): void => {
-    if (c.cpu > 0) {
-      reservedCpu += c.cpu
-      holdersCpu++
+  const admit = (id: string): (() => void) => {
+    const pool = poolOf?.(id)
+    if (pool === undefined) {
+      if (options.restoreTier?.has(id)) {
+        activeRestore++
+        return () => {
+          activeRestore--
+        }
+      }
+      active++
+      return () => {
+        active--
+      }
     }
-    if (c.mem > 0) {
-      reservedMem += c.mem
-      holdersMem++
+    poolActive.set(pool.name, (poolActive.get(pool.name) ?? 0) + 1)
+    return () => {
+      poolActive.set(pool.name, (poolActive.get(pool.name) ?? 1) - 1)
     }
   }
-  const release = (c: ResourceCost): void => {
-    if (c.cpu > 0 && --holdersCpu === 0) reservedCpu = 0
-    else reservedCpu -= c.cpu
-    if (c.mem > 0 && --holdersMem === 0) reservedMem = 0
-    else reservedMem -= c.mem
+  // The policy sees exec-tier local tasks only; a restore is a tar
+  // extract and a pooled task runs on someone else's capacity.
+  const local = (id: string): boolean => !options.restoreTier?.has(id) && poolOf?.(id) === undefined
+  const admits = (id: string): boolean => !local(id) || admitPolicy!(id, running)
+  // With no policy nothing reads `running`, so nothing is tracked: the
+  // count-only dispatch allocates no closure and touches no set per task.
+  const untracked = (): void => {}
+  const track = (id: string): (() => void) => {
+    if (!admitActive || !local(id)) return untracked
+    running.add(id)
+    return () => {
+      running.delete(id)
+    }
   }
 
   return new Promise<Map<string, TaskOutcome>>((resolve) => {
@@ -502,7 +487,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
       // Observer hook (the logger's taskComplete). Crash-isolated: a
       // throwing observer must NOT break scheduling — it would otherwise
       // skip the dependent-enqueue + tick below and hang the run. Same
-      // "observability never breaks a run" rule the telemetry/eventSink
+      // "observability never breaks a run" rule the telemetry
       // paths hold. Isolated here (not the completion arm) so a throw
       // can't be mistaken for the task itself failing.
       try {
@@ -523,7 +508,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
     }
 
     // True when the task would be finished as `skipped` without running
-    // (fail-fast tripped, or a failed/skipped upstream under
+    // (fail-fast tripped, or a failed/skipped/aborted upstream under
     // continueMode !== 'always'). ONE predicate shared by the admission
     // parker and the dispatch loop's skip branch — a would-skip task
     // executes nothing, so it must never park on a resource fit (a
@@ -534,43 +519,66 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
     // cached output is reported `cache-hit` even if a dep failed — and
     // they're dep-independent, so they typically restore before a dep
     // could fail anyway.
+    const aborted = (): boolean => options.signal?.aborted === true
     const willSkip = (id: string): boolean => {
-      if (failFastTripped) return true
+      if (failFastTripped || aborted()) return true
       if (restoreTier?.has(id)) return false
       if (continueMode === 'always') return false
       const node = nodes.get(id) as TaskNode
       return node.deps.some((d) => {
         const u = outcomes.get(d)
-        return u?.status === 'failed' || u?.status === 'skipped'
+        // `aborted` propagates even though it is counted nowhere: the
+        // upstream was killed mid-write, so its declared outputs are
+        // partial. A dependent that ran anyway would CACHE what it built
+        // from them — under the key a healthy run derives, since the fold
+        // takes the upstream's INPUT key — so the next run replays those
+        // bytes as a green hit.
+        return u?.status === 'failed' || u?.status === 'skipped' || u?.status === 'aborted'
       })
     }
 
     const tick = (): void => {
       if (resolved) return
-      // Exec-tier tasks parked THIS tick on a failed resource fit.
-      // Within one synchronous tick `reserved` only increases (release
-      // happens in the async completion callbacks, which run a fresh
-      // tick), so a task that doesn't fit now cannot fit later in the
-      // same tick — parking it for the tick's remainder is exact, and
-      // each id pops at most once per tick (O(R log R)).
+      // Exec-tier tasks parked THIS tick on a refused admission. Within
+      // one synchronous tick `running` only grows (tasks leave it in the
+      // async completion callbacks, which run a fresh tick), so a task
+      // refused now cannot be admitted later in the same tick — parking
+      // it for the tick's remainder is exact, and each id pops at most
+      // once per tick (O(R log R)).
       const parked: Array<[string, number]> = []
 
       // Highest-priority admissible task: exec tier first (misses own
-      // the pool), then restore tier. With no reservations declared this
+      // the pool), then restore tier. With no admission policy this
       // short-circuits to exactly the legacy takeReady. A would-skip
-      // task returns without a fit check (finishing it is free); restore
-      // tasks cost zero by construction, so they never park.
+      // task returns without asking (finishing it is free); restore
+      // tasks are never asked, so they never park.
       const takeFitting = (): string | undefined => {
-        while (execReady.size > 0) {
-          const seq = execReady.peekSeq()
-          const id = execReady.pop() as string
-          if (!resourcesActive || willSkip(id) || fits(id)) return id
-          parked.push([id, seq])
+        // No pools, no policy and a full exec lane: nothing on the
+        // exec queue can be admitted, so it is not scanned — scanning it
+        // would pop and re-park every ready exec task on every tick, and
+        // with a wide frontier that is O(R) per tick, O(R²) per run (the
+        // 6,000-task scale pin went 0.5 s → 28 s when the restore lane's
+        // first cut let the scan run past a full exec lane, 2026-09-10).
+        // This is the legacy O(1) gate, kept per lane.
+        const execAdmissible = poolOf !== undefined || admitActive || active < concurrency
+        if (execAdmissible) {
+          while (execReady.size > 0) {
+            const seq = execReady.peekSeq()
+            const id = execReady.pop() as string
+            if (willSkip(id)) return id
+            if (hasRoom(id)) {
+              if (!admitActive || admits(id)) return id
+              // A free worker, refused by the policy: the hold starts now.
+              if (!heldSince.has(id)) heldSince.set(id, Date.now())
+            }
+            parked.push([id, seq])
+          }
         }
-        return restoreReady.pop()
+        // Restore-tier tasks are local disk work, on their own lane.
+        return activeRestore < restoreConcurrency ? restoreReady.pop() : undefined
       }
 
-      while (active < concurrency) {
+      for (;;) {
         const id = takeFitting()
         if (id === undefined) break // nothing ready is admissible right now
         const node = nodes.get(id) as TaskNode
@@ -586,17 +594,26 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
         // because dependents are pushed when `pending` hits 0 regardless
         // of outcome — keeps the propagation logic in one place.
         if (willSkip(id)) {
-          finishOne(id, { node, status: 'skipped', exitCode: 1, durationMs: 0 })
+          finishOne(id, {
+            node,
+            status: aborted() ? 'aborted' : 'skipped',
+            exitCode: 1,
+            durationMs: 0,
+          })
           continue
         }
 
-        active++
-        // Reserve on dispatch; capture the cost so the release in the
-        // completion callbacks is symmetric even if the maps change.
-        const cost = costOf(id)
-        reserve(cost)
+        const leave = admit(id)
+        // Listed as running on dispatch, so the policy's next ask in this
+        // tick sees it; the completion callbacks unlist it.
+        const untrack = track(id)
+        const since = admitActive ? heldSince.get(id) : undefined
+        const heldMs = since === undefined ? 0 : Math.max(1, Date.now() - since)
+        if (since !== undefined) heldSince.delete(id)
+        const withHold = (o: TaskOutcome): TaskOutcome =>
+          heldMs > 0 ? { ...o, admissionHeldMs: heldMs } : o
         // Crash-isolated observer hook — a throwing onStart must not abort
-        // the dispatch loop (it would strand the tick with reservations held).
+        // the dispatch loop (it would strand the tick with the slot held).
         try {
           onStart?.(node)
         } catch (err) {
@@ -607,33 +624,45 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
         // `.then(onFulfilled, onRejected)` — NOT `.then(f).catch(g)`. The
         // rejection arm handles ONLY `execute()` rejecting; a throw from
         // the fulfillment arm (finishOne / onFinish / tick) must NOT also
-        // run the rejection arm, or `active`/`reserved` release twice —
-        // and a double release drives `reserved` negative, permanently
-        // wedging the solo-clamp gate.
+        // run the rejection arm, or `active` releases twice — and a
+        // double release wedges the count gate.
         execute(node, upstream).then(
-          (outcome) => {
-            active--
-            release(cost)
-            finishOne(id, outcome)
+          (raw) => {
+            const outcome = withHold(raw)
+            leave()
+            untrack()
+            // The slot is free now; the dependents wait for what the
+            // outcome still owes (a save landing), if anything.
+            const settled = options.settledOf?.(outcome)
+            if (settled === undefined) {
+              finishOne(id, outcome)
+              tick()
+              return
+            }
             tick()
+            const done = (): void => {
+              finishOne(id, outcome)
+              tick()
+            }
+            void settled.then(done, done)
           },
           (err: unknown) => {
             const message = err instanceof Error ? err.message : String(err)
-            const outcome: TaskOutcome = {
+            const outcome: TaskOutcome = withHold({
               node,
               status: 'failed',
               exitCode: 1,
               durationMs: 0,
-            }
-            active--
-            release(cost)
+            })
+            leave()
+            untrack()
             finishOne(id, outcome)
             // Surface the error live; the outcome itself doesn't
             // carry captured stderr (that's the logger's job). A
             // UserError is a config/input failure (e.g. a failed
             // `cache.inputs.runtime` command), not a vx bug — report it
             // plainly, never as an "internal error".
-            if (err instanceof UserError) {
+            if (isUserError(err)) {
               process.stderr.write(`[vx] ${id}: ${message}\n`)
             } else {
               const named = err instanceof Error && err.name !== 'Error' ? `${err.name}: ` : ''
@@ -648,7 +677,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
       // is exactly preserved for the next tick's admission pass.
       for (const [id, seq] of parked) execReady.push(id, seq)
 
-      if (outcomes.size === nodes.size && active === 0) {
+      if (outcomes.size === nodes.size && active === 0 && activeRestore === 0) {
         resolved = true
         resolve(outcomes)
       }
@@ -658,7 +687,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   })
 }
 
-function mergePriorities(
+export function mergePriorities(
   baseline: ReadonlyMap<string, number>,
   overrides: ReadonlyMap<string, number>,
 ): ReadonlyMap<string, number> {