@vzn/vx 0.0.18 → 0.0.22

package/src/graph/scheduler.ts

@@ -1,6 +1,22 @@
-import { isUserError } from '../util/index.js'
+import { fsRefusalHint, isFsRefusal, isUserError } from '../util/index.js'
+import { computeReverseDepCount, mergePriorities, tieredReverseDepCount } from './priorities.js'
 import type { TaskNode } from './task-graph.js'
 
+/**
+ * Thrown by `execute` for a restore-tier task that turned out to have
+ * nothing to restore (its artifact vanished after the up-front probe). It
+ * may be running ahead of its deps, so it cannot run its command in that
+ * slot: the scheduler releases the slot and dispatches it again as an
+ * exec-tier task once its deps are done. It records no outcome, and its
+ * dependents wait for the second dispatch.
+ */
+export class RestoreDemoted extends Error {
+  constructor(readonly taskId: string) {
+    super(`${taskId}: restore demoted to a run`)
+    this.name = 'RestoreDemoted'
+  }
+}
+
 export type TaskStatus =
   | 'success'
   | 'cache-hit'
@@ -14,65 +30,6 @@ export type TaskStatus =
   // are partial.
   | 'aborted'
 
-/**
- * Cache-correctness verdict for a task under `vx run --verify` (Phase 1:
- * determinism; Phase 2: input-completeness). Declared here (structurally)
- * because `graph` can't import `orchestrator` where the verifier lives —
- * same pattern as `inputComponents`. A pure side-channel: never hashed,
- * never persisted in Phase 1/2.
- */
-export type VerifyVerdict =
-  | { kind: 'proven-deterministic' }
-  /** Phase 2: input-completeness proved — the task read nothing outside its
-   *  declared inputs (from `--verify=inputs` when determinism wasn't also
-   *  requested; an `--verify=all` pass reports the stronger deterministic). */
-  | { kind: 'proven-complete' }
-  /** Re-ran; outputs differ from the cached ones. `changed` names the rels. */
-  | { kind: 'nondeterministic'; changed: readonly string[] }
-  /** Phase 2: read a workspace path outside the declared inputs — the declared
-   *  `cache.inputs` are incomplete, so a hit could serve stale bytes. `paths`
-   *  names the undeclared reads (workspace-relative; empty when the sandbox
-   *  denied structurally but strace wasn't available to name them). */
-  | { kind: 'undeclared-inputs'; paths: readonly string[] }
-  /** Diverged but the task is on `--verify-allow` — reported, not failed. */
-  | { kind: 'allowed-nondeterministic'; changed: readonly string[] }
-  /** The verify re-run exited non-zero / timed out (nondeterministic by
-   *  definition — identical inputs, different outcome). */
-  | { kind: 'rerun-failed'; exitCode: number }
-  /** Cacheable + executed but declares no outputs — nothing to replay. */
-  | { kind: 'no-outputs' }
-  /** Didn't execute (cache hit) / not cacheable / group / persistent. */
-  | { kind: 'not-verified' }
-  /** A `remote: 'only'` task under `--verify=inputs`: verify pins placement
-   *  local, so the task no-ops — there is no execution to sandbox, locally
-   *  or anywhere. Reported, not silent: the proof does not cover it. */
-  | { kind: 'unverifiable-remote-only' }
-
-/**
- * Content fingerprint of a task's output tree, computed under a `--verify*`
- * mode on the executed (miss) path. Never hashed into any key; pure telemetry
- * side-channel for the cross-machine diff (a serve pairs fingerprints for the
- * SAME cache key across platforms and names diverging outputs). Declared here
- * (structurally) because `graph` can't import `orchestrator` — the
- * `VerifyVerdict` pattern. See docs/design/verify-cross-machine-2026-07.md.
- */
-export interface OutputFingerprint {
-  /** Roll-up: xxh3hex over the sorted (key, hash) pairs, folded as
-   *  `key \0 hash \n` (\0 boundaries — the v18 lesson). Always present;
-   *  divergence DETECTION never depends on the per-file map. */
-  tree: string
-  /** Total files in the tree (pre-truncation). */
-  fileCount: number
-  /** Per-file map as sorted [outputKey, xxh3hex] pairs, capped at
-   *  FP_MAX_FILES (500). Deterministic truncation — sorted by key,
-   *  first N — so two machines' truncated maps cover the same subset
-   *  and partial diffs still name real rels. */
-  files?: ReadonlyArray<readonly [string, string]>
-  /** Set when `files` was truncated to the cap (or dropped by the
-   *  sink's run budget). */
-  truncated?: boolean
-}
-
 export interface TaskOutcome {
   node: TaskNode
   status: TaskStatus
@@ -90,6 +47,22 @@ export interface TaskOutcome {
    * Undefined on every non-hit outcome — nothing was skipped.
    */
   storedDurationMs?: number
+  /**
+   * For cache-hit statuses: what the PRODUCING execution used, read from
+   * the entry (it rides the artifact, so a remote hit on a fresh machine
+   * carries it too). The same split as `storedDurationMs`: `cpuMs` /
+   * `peakRssBytes` are what THIS run spent and stay absent on a hit.
+   */
+  storedCpuMs?: number
+  storedPeakRssBytes?: number
+  /**
+   * How long an `admit` policy held this task after it was ready with a
+   * free worker — the wait the plugin, not the count gate, imposed. Set
+   * only when a policy refused it at least once (so a run with no policy
+   * carries the field on no outcome); `--summarize` rows, the event
+   * stream and the summary footer show it.
+   */
+  admissionHeldMs?: number
   /** v11 analytics: CPU time + peak RSS for this task's child process. */
   cpuMs?: number
   peakRssBytes?: number
@@ -103,6 +76,26 @@ export interface TaskOutcome {
    * outcomes only; never folded into any key.
    */
   groupUpstream?: readonly TaskOutcome[]
+  /**
+   * On a `skipped` outcome: the task at the ROOT of what blocked it — the
+   * failed (or aborted) upstream, followed through any chain of skips
+   * between. Absent when the skip was fail-fast's (no upstream failed).
+   * The footer, `--summarize`, the telemetry record and the run row all
+   * read this one field (item 267).
+   */
+  blockedBy?: string
+  /**
+   * On a `failed` outcome: vx's own `timeout` killed the final attempt.
+   * The exit is the shell's 143, and every label says "timed out" instead
+   * of reading it as a signal (item 268).
+   */
+  timedOut?: true
+  /**
+   * On a failed persistent task: why it never became ready — the readiness
+   * deadline fired, the child exited first (`exitCode` is then its own), or
+   * the spawn itself failed. Every label reads this instead of the exit.
+   */
+  notReady?: 'timeout' | 'exited' | 'spawn'
   /** Executor-reported placement label (`ExecuteResult.where`) — set only
    *  when the task ran somewhere other than this host. Telemetry-only. */
   where?: string
@@ -134,10 +127,10 @@ export interface TaskOutcome {
    */
   attempts?: number
   /**
-   * Count of sandbox violations captured during this task's exec.
-   * Populated only when `--sandbox` was set and the task is cached.
-   * Non-zero values mean the task read files outside its declared
-   * inputs; `cache.save()` was skipped so the result can't be replayed.
+   * Count of sandbox violations captured during this task's exec (a task
+   * declaring `exec.sandbox`). Non-zero means the task touched something
+   * it did not declare; execute-task fails the task on any violation, so
+   * nothing built that way is ever saved or replayed.
    */
   sandboxViolations?: number
   /**
@@ -146,38 +139,19 @@ export interface TaskOutcome {
    * inline in the task's block instead of as loose status output.
    */
   sandboxViolationLines?: string[]
-  /**
-   * Cache-correctness verdict under `vx run --verify`. Set only in verify
-   * mode; a plain run leaves it undefined. Pure side-channel (never hashed).
-   */
-  verify?: VerifyVerdict
-  /**
-   * Output-tree fingerprint under a fingerprinting `--verify*` mode
-   * (`--verify` / `=all` / `=fingerprint`), executed + cacheable +
-   * output-declaring tasks only. A plain run leaves it undefined.
-   */
-  outputFp?: OutputFingerprint
 }
 
 export type ContinueMode = 'never' | 'deps-ok' | 'always'
 
-/**
- * Resolved per-task resource reservation, in absolute units (cpu may be
- * fractional; mem is bytes). Declared here (structurally) because `graph`
- * can't import `orchestrator`, where the resolver lives — same pattern as
- * `VerifyVerdict`. A `0` axis means "reserve nothing, run freely": the
- * task is exempt from that axis entirely (needs no headroom, holds none).
- */
-export interface ResourceCost {
-  cpu: number
-  mem: number
-}
-
-const ZERO_COST: ResourceCost = { cpu: 0, mem: 0 }
-
 export interface ScheduleOptions {
   nodes: Map<string, TaskNode>
   concurrency: number
+  /**
+   * An aborted run dispatches nothing further: every task not yet
+   * started completes `aborted` (the ones in flight are being killed by
+   * the orchestrator and report `aborted` through the runner's signal).
+   */
+  signal?: AbortSignal
   /**
    * Failure propagation (default 'deps-ok' — the historical behavior):
    *   - 'deps-ok': a failed/skipped/aborted upstream skips its
@@ -186,18 +160,25 @@ export interface ScheduleOptions {
    *     finish naturally, everything not yet started completes as
    *     skipped (restore-tier included: a fail-fast run stops
    *     restoring too).
-   *   - 'always': dependents run even when an upstream failed. Sound
-   *     under pure-input transitive hashing: failed outcomes carry the
-   *     upstream's INPUT key (computed before exec), so dependents'
-   *     keys fold exactly what a healthy run folds. An upstream that
-   *     died before hashing folds nothing — that key is derivable by
-   *     no healthy run, so the worst case is an unreachable cache
-   *     entry, never a stale hit.
+   *   - 'always': dependents run even when an upstream failed. Their
+   *     keys are the ones a healthy run derives (failed outcomes carry
+   *     the upstream's INPUT key, computed before exec), which is
+   *     exactly why the orchestrator withholds their SAVE: the bytes
+   *     were built on a partial tree, and a healthy key over them would
+   *     be the next clean run's stale hit (`ExecuteArgs.taintedUpstream`).
    */
   continueMode?: ContinueMode
   execute: (node: TaskNode, upstream: TaskOutcome[]) => Promise<TaskOutcome>
   onStart?: (node: TaskNode) => void
   onFinish?: (outcome: TaskOutcome) => void
+  /**
+   * What an outcome still owes before its dependents may start — a cache
+   * save that runs off the execution slot (the orchestrator's save lane):
+   * a dependent reads the upstream's entry (its output rows travel in the
+   * execute request), so it waits for the save to land, while the freed
+   * slot admits other work at once. Undefined: nothing owed.
+   */
+  settledOf?: (outcome: TaskOutcome) => Promise<void> | undefined
   /**
    * Optional priority override: callers pass their own per-node weight
    * (e.g. `computePredictedPriorities` from the orchestrator's history
@@ -218,7 +199,9 @@ export interface ScheduleOptions {
    *     it);
    *   - bypasses the `failedDep`→`skipped` check (its key is independent
    *     of dep success — pure-input transitive hashing — so a valid cached
-   *     output is reported `cache-hit` regardless of a dep failing).
+   *     output is reported `cache-hit` regardless of a dep failing);
+   *   - leaves the tier when `execute` throws `RestoreDemoted`, and is
+   *     dispatched again once its deps are done, with the dep check.
    * It still runs through `execute()` (so the logger frame is unchanged);
    * the orchestrator's execute reuses the up-front probe, so there is no
    * second cache.get. When undefined/empty, behavior is byte-identical.
@@ -233,109 +216,16 @@ export interface ScheduleOptions {
    */
   poolOf?: (id: string) => { name: string; capacity: number } | undefined
   /**
-   * Resolved per-task resource reservations (`exec.resources`, resolved
-   * by the orchestrator against the run's budgets). An absent id means
-   * zero cost; undefined/empty means no task opted in — the scheduler
-   * takes the legacy path byte-identically. Admission control only —
-   * nothing is enforced on the child process.
+   * Admission over the worker count: asked for every exec-tier task about
+   * to start on this machine, with the ids of the exec-tier tasks running
+   * here right now (added on dispatch, so two asks in one tick see each
+   * other). `false` parks the task until something finishes. Undefined →
+   * count-only, the legacy path byte for byte. Restore-tier hits and
+   * pooled tasks hold no local resources and are never asked.
    */
-  resourceCosts?: ReadonlyMap<string, ResourceCost>
-  /** CPU budget reservations pack against. Defaults to `concurrency`. */
-  cpuBudget?: number
-  /** Memory budget (bytes). Defaults to Infinity (axis off). */
-  memBudget?: number
+  admit?: (id: string, running: ReadonlySet<string>) => boolean
 }
 
-/**
- * Compute, for each task in the graph, how many OTHER tasks are
- * transitively blocked on it. Tasks with the highest count are the
- * most valuable to schedule first — finishing them unlocks the most
- * downstream work and minimizes worker idle time at the end of the
- * run. Matches Nx's `calculateReverseDeps`-driven schedule sort
- * (`packages/nx/src/tasks-runner/tasks-schedule.ts:166-207`).
- */
-// Exported for the perf guard in tests/scheduler.test.ts, which times
-// this function in isolation — end-to-end runGraph timing drowns the
-// closure cost in task-promise overhead and flakes on slow CI.
-export function computeReverseDepCount(nodes: Map<string, TaskNode>): Map<string, number> {
-  // Exact transitive-dependent COUNTS need closure SETS (diamonds
-  // double-count under naive summing). Set-of-strings closures are
-  // O(N²) entries and took 8.5s on a 1090-package, 100-layer repo;
-  // bitsets make the same closure O(E·N/32) time and N²/8 bits of
-  // memory (3270 tasks ≈ 1.3 MB) — single-digit ms at that scale.
-  const ids = [...nodes.keys()]
-  const index = new Map<string, number>()
-  for (let i = 0; i < ids.length; i++) index.set(ids[i]!, i)
-  const n = ids.length
-  const words = (n + 31) >>> 5
-
-  // Direct dependents as index lists + in-degree for the topo pass.
-  const directReverse: number[][] = Array.from({ length: n }, () => [])
-  const indegree = new Uint32Array(n)
-  for (const node of nodes.values()) {
-    const ni = index.get(node.id)!
-    for (const dep of node.deps) {
-      const di = index.get(dep)
-      if (di === undefined) continue
-      directReverse[di]!.push(ni)
-      indegree[ni]!++
-    }
-  }
-
-  // Kahn topo order over dependency edges (deps before dependents).
-  // Insertion order is topo today, but the closure's correctness
-  // must not hinge on an unstated property of buildTaskGraph.
-  const topo = new Int32Array(n)
-  let head = 0
-  let tail = 0
-  for (let i = 0; i < n; i++) if (indegree[i] === 0) topo[tail++] = i
-  while (head < tail) {
-    const v = topo[head++]!
-    for (const r of directReverse[v]!) {
-      if (--indegree[r]! === 0) topo[tail++] = r
-    }
-  }
-
-  // Reverse-topo sweep: every direct dependent's closure is final
-  // before its dependency folds it in. closure[i] = bitset over node
-  // indices of i's transitive dependents.
-  const closure = new Uint32Array(n * words)
-  const counts = new Map<string, number>()
-  for (let t = tail - 1; t >= 0; t--) {
-    const i = topo[t]!
-    const base = i * words
-    for (const r of directReverse[i]!) {
-      closure[base + (r >>> 5)]! |= 1 << (r & 31)
-      const rbase = r * words
-      for (let w = 0; w < words; w++) closure[base + w]! |= closure[rbase + w]!
-    }
-    let count = 0
-    for (let w = 0; w < words; w++) {
-      let v = closure[base + w]!
-      v = v - ((v >>> 1) & 0x55555555)
-      v = (v & 0x33333333) + ((v >>> 2) & 0x33333333)
-      count += (((v + (v >>> 4)) & 0x0f0f0f0f) * 0x01010101) >>> 24
-    }
-    counts.set(ids[i]!, count)
-  }
-  // Cycle-stranded nodes (never topo-visited) can't occur — the graph
-  // builder rejects cycles — but a missing Map entry would silently
-  // sort as undefined, so default them defensively to 0.
-  for (const id of ids) if (!counts.has(id)) counts.set(id, 0)
-  return counts
-}
-
-/**
- * Run the task graph. Independent tasks run in parallel up to `concurrency`.
- * If a task fails, its dependents are marked `skipped` but unrelated tasks
- * keep running so the user gets maximum information per invocation.
- *
- * Scheduling: when more than one task is ready, the scheduler picks the
- * one that blocks the most downstream work (most transitive reverse
- * dependents). Ties break in graph-insertion order (which is the topo
- * order produced by `buildTaskGraph`). Minimizes worker idle at the
- * end of the run.
- */
 /**
  * A binary max-heap of ready task ids, ordered by (priority DESC, enqueue-seq
  * ASC) so `pop()` returns the highest-priority task and equal-priority ties
@@ -368,8 +258,18 @@ class ReadyHeap {
   }
   /**
    * `seq` defaults to a fresh monotonic counter. A parked-then-repushed
-   * task passes its ORIGINAL seq back in so FIFO-among-equals survives
-   * the round trip (a fresh seq would demote it behind later arrivals).
+   * task passes its ORIGINAL seq back in, which makes FIFO-among-equals a
+   * property of the heap rather than of the tick loop's shape.
+   *
+   * It is NOT the difference between correct and incorrect order today,
+   * and the comment here used to claim it was ("a fresh seq would demote
+   * it behind later arrivals"). It would not: `parked` is a subsequence
+   * of the tick's pop order, so it is already in ranking order, and every
+   * push that follows takes a later seq — so fresh seqs handed out in
+   * parked order reproduce the same ranking. Measured over 800 randomized
+   * scenarios (random DAGs, costs, budgets, concurrency, failures, all
+   * three continue modes): zero order differences, item 493. Kept because
+   * it holds without that argument.
    */
   push(id: string, seq: number = this.next++): void {
     this.ids.push(id)
@@ -411,6 +311,17 @@ class ReadyHeap {
   }
 }
 
+/**
+ * Run the task graph. Independent tasks run in parallel up to `concurrency`.
+ * If a task fails, its dependents are marked `skipped` but unrelated tasks
+ * keep running so the user gets maximum information per invocation.
+ *
+ * Scheduling: when more than one task is ready, the scheduler picks the
+ * one that blocks the most downstream work (most transitive reverse
+ * dependents). Ties break in graph-insertion order (which is the topo
+ * order produced by `buildTaskGraph`). Minimizes worker idle at the
+ * end of the run.
+ */
 export async function runGraph(options: ScheduleOptions): Promise<Map<string, TaskOutcome>> {
   const { nodes, concurrency, execute, onStart, onFinish } = options
   const continueMode = options.continueMode ?? 'deps-ok'
@@ -439,7 +350,10 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   // priorities map (e.g. from history-aware critical-path estimation),
   // use those weights. Falls back to the reverse-deps-count heuristic
   // for nodes the caller didn't score, so partial coverage works.
-  const baseline = computeReverseDepCount(nodes)
+  const baseline =
+    options.restoreTier !== undefined && options.restoreTier.size > 0
+      ? tieredReverseDepCount(nodes, options.restoreTier)
+      : computeReverseDepCount(nodes)
   const priority: ReadonlyMap<string, number> = options.priorities
     ? mergePriorities(baseline, options.priorities)
     : baseline
@@ -453,6 +367,10 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   // idle capacity (or run when an exec is blocked on a restorable dep and
   // nothing else is runnable).
   const restoreTier = options.restoreTier
+  // Restore-tier tasks whose execute threw `RestoreDemoted`: from then on
+  // they are exec-tier tasks, gated on their deps like any other.
+  const demoted = new Set<string>()
+  const inRestoreTier = (id: string): boolean => restoreTier?.has(id) === true && !demoted.has(id)
   const execReady = new ReadyHeap(priority)
   const restoreReady = new ReadyHeap(priority)
   // A restore-tier task is dep-independent (a stable hit's restore needs
@@ -466,33 +384,26 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   }
 
   let active = 0
+  // Restore-tier work is disk I/O — a confirmed hit's tar extract is ~8
+  // filesystem round trips and no CPU — so it gets its own lane, twice
+  // the exec cap: on the 1,000-project bench with every task a restore,
+  // 4 → 8 workers cut the run-graph stage 683–754 → 556–595 ms and 16 was
+  // no better (2026-09-10). Exec-tier work keeps the CPU-shaped cap; the
+  // two lanes never wait on each other. `--concurrency 1` stays serial.
+  let activeRestore = 0
+  const restoreConcurrency = concurrency === 1 ? 1 : 2 * concurrency
   let resolved = false
 
-  // Resource admission (2-D bin packing over the count limit). Inactive
-  // (no task opted in) → the tick loop short-circuits before any of this
-  // and behaves byte-identically to the count-only scheduler.
-  const costs = options.resourceCosts
-  const resourcesActive = costs !== undefined && costs.size > 0
-  const cpuBudget = options.cpuBudget ?? concurrency
-  const memBudget = options.memBudget ?? Infinity
-  // Reservations are FLOAT sums (fractional cpus, and percent-of-budget
-  // resolves to non-representable values like `0.30000000000000004`), so
-  // add/release cycles leave ~1e-17 residue instead of an exact 0. Two
-  // guards keep that residue from corrupting admission:
-  //   - the solo-clamp gate ("is the axis idle?") reads INTEGER holder
-  //     counts, never the float sum === 0 — a residue would otherwise
-  //     park an over-budget task forever (active===0, no future tick =
-  //     a silent hang / exit-0-without-running);
-  //   - `reserved` snaps back to EXACT 0 whenever its holder count hits
-  //     0, so residue can't accumulate across busy periods.
-  // The within-budget comparison also carries a tiny relative epsilon so
-  // an exact-fill (`reserved + cost == budget`) can't mis-round into a
-  // spurious block. Admission is a hint, so the epsilon's sub-ulp
-  // over-admission is harmless.
-  let reservedCpu = 0
-  let reservedMem = 0
-  let holdersCpu = 0
-  let holdersMem = 0
+  // Admission policy over the count limit (a plugin's `admit`). Inactive
+  // → the tick loop short-circuits before any of this and behaves
+  // byte-identically to the count-only scheduler.
+  const admitPolicy = options.admit
+  const admitActive = admitPolicy !== undefined
+  const running = new Set<string>()
+  // When a policy first refused each parked task — so its outcome can say
+  // how long the policy, not the count gate, held it. Nothing with no
+  // policy: the map is never touched.
+  const heldSince = new Map<string, number>()
 
   // A restore-tier task is a confirmed local cache hit: its "execution"
   // is a cheap tar extract, not the task's real work — it reserves ZERO
@@ -504,12 +415,20 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   // against `concurrency`; pooled tasks reserve no local resources.
   const hasRoom = (id: string): boolean => {
     const pool = poolOf?.(id)
-    if (pool === undefined) return active < concurrency
+    if (pool === undefined) {
+      return inRestoreTier(id) ? activeRestore < restoreConcurrency : active < concurrency
+    }
     return (poolActive.get(pool.name) ?? 0) < pool.capacity
   }
   const admit = (id: string): (() => void) => {
     const pool = poolOf?.(id)
     if (pool === undefined) {
+      if (inRestoreTier(id)) {
+        activeRestore++
+        return () => {
+          activeRestore--
+        }
+      }
       active++
       return () => {
         active--
@@ -520,44 +439,20 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
       poolActive.set(pool.name, (poolActive.get(pool.name) ?? 1) - 1)
     }
   }
-  const costOf = (id: string): ResourceCost =>
-    options.restoreTier?.has(id) || poolOf?.(id) !== undefined
-      ? ZERO_COST
-      : (costs?.get(id) ?? ZERO_COST)
-
-  // Zero never blocks; a within-budget cost needs headroom (with an
-  // exact-fill epsilon); an over-budget cost can never have headroom, so
-  // it solo-clamps: admitted only when the axis is idle (no holders — an
-  // idle pool always admits at least one ready task, no deadlock).
-  const fitsAxis = (cost: number, reserved: number, holders: number, budget: number): boolean =>
-    cost === 0 ? true : cost <= budget ? reserved + cost <= budget + budget * 1e-9 : holders === 0
-
-  const fits = (id: string): boolean => {
-    const c = costOf(id)
-    return (
-      fitsAxis(c.cpu, reservedCpu, holdersCpu, cpuBudget) &&
-      fitsAxis(c.mem, reservedMem, holdersMem, memBudget)
-    )
-  }
-
-  // Reserve/release keep the float sum AND the integer holder count in
-  // lockstep; releasing the last holder on an axis snaps its sum to 0.
-  const reserve = (c: ResourceCost): void => {
-    if (c.cpu > 0) {
-      reservedCpu += c.cpu
-      holdersCpu++
-    }
-    if (c.mem > 0) {
-      reservedMem += c.mem
-      holdersMem++
+  // The policy sees exec-tier local tasks only; a restore is a tar
+  // extract and a pooled task runs on someone else's capacity.
+  const local = (id: string): boolean => !inRestoreTier(id) && poolOf?.(id) === undefined
+  const admits = (id: string): boolean => !local(id) || admitPolicy!(id, running)
+  // With no policy nothing reads `running`, so nothing is tracked: the
+  // count-only dispatch allocates no closure and touches no set per task.
+  const untracked = (): void => {}
+  const track = (id: string): (() => void) => {
+    if (!admitActive || !local(id)) return untracked
+    running.add(id)
+    return () => {
+      running.delete(id)
     }
   }
-  const release = (c: ResourceCost): void => {
-    if (c.cpu > 0 && --holdersCpu === 0) reservedCpu = 0
-    else reservedCpu -= c.cpu
-    if (c.mem > 0 && --holdersMem === 0) reservedMem = 0
-    else reservedMem -= c.mem
-  }
 
   return new Promise<Map<string, TaskOutcome>>((resolve) => {
     const finishOne = (id: string, outcome: TaskOutcome): void => {
@@ -575,14 +470,54 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
         const m = err instanceof Error ? err.message : String(err)
         process.stderr.write(`[vx] onFinish observer threw for ${id}: ${m}\n`)
       }
+      // A restore-tier task runs before its deps, but its DEPENDENTS keep
+      // the order `dependsOn` states: they are released once its own deps
+      // have settled too. Released at its own finish, an `e2e` ran beside
+      // the `db:migrate` two edges up whenever the middle task was a hit,
+      // and ran green after that dep failed (item 963).
+      if (inRestoreTier(id) && (pending.get(id) ?? 0) > 0) {
+        heldRelease.add(id)
+        return
+      }
+      release(id)
+    }
+
+    // A restore-tier task whose deps did not all succeed is still a
+    // `cache-hit` (its key is theirs, not their outcome), but it passes the
+    // block down: its dependents are skipped as a failed dep's would be.
+    // Its root: a failed or aborted dep, or the root a skipped or blocked
+    // one carries.
+    const blockedRestores = new Map<string, string | undefined>()
+    const heldRelease = new Set<string>()
+    const release = (id: string): void => {
+      if (inRestoreTier(id) && continueMode !== 'always') {
+        for (const dep of (nodes.get(id) as TaskNode).deps) {
+          const u = outcomes.get(dep)
+          if (u?.status === 'failed' || u?.status === 'aborted') {
+            blockedRestores.set(id, dep)
+            break
+          }
+          if (u?.status === 'skipped') {
+            blockedRestores.set(id, u.blockedBy)
+            break
+          }
+          if (blockedRestores.has(dep)) {
+            blockedRestores.set(id, blockedRestores.get(dep))
+            break
+          }
+        }
+      }
       const ds = dependents.get(id)
       if (!ds) return
       for (const d of ds) {
         const rem = (pending.get(d) ?? 0) - 1
         pending.set(d, rem)
+        if (rem !== 0) continue
         // Restore-tier dependents were already enqueued at startup (they
-        // don't wait on deps); only re-enqueue an exec-tier dependent.
-        if (rem === 0 && !restoreTier?.has(d)) execReady.push(d)
+        // don't wait on deps); one that finished early releases its own
+        // dependents now. Only an exec-tier dependent is enqueued.
+        if (heldRelease.delete(d)) release(d)
+        else if (!inRestoreTier(d)) execReady.push(d)
       }
     }
 
@@ -598,9 +533,10 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
     // cached output is reported `cache-hit` even if a dep failed — and
     // they're dep-independent, so they typically restore before a dep
     // could fail anyway.
+    const aborted = (): boolean => options.signal?.aborted === true
     const willSkip = (id: string): boolean => {
-      if (failFastTripped) return true
-      if (restoreTier?.has(id)) return false
+      if (failFastTripped || aborted()) return true
+      if (inRestoreTier(id)) return false
       if (continueMode === 'always') return false
       const node = nodes.get(id) as TaskNode
       return node.deps.some((d) => {
@@ -611,37 +547,54 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
         // from them — under the key a healthy run derives, since the fold
         // takes the upstream's INPUT key — so the next run replays those
         // bytes as a green hit.
-        return u?.status === 'failed' || u?.status === 'skipped' || u?.status === 'aborted'
+        return (
+          u?.status === 'failed' ||
+          u?.status === 'skipped' ||
+          u?.status === 'aborted' ||
+          blockedRestores.has(d)
+        )
       })
     }
 
     const tick = (): void => {
       if (resolved) return
-      // Exec-tier tasks parked THIS tick on a failed resource fit.
-      // Within one synchronous tick `reserved` only increases (release
-      // happens in the async completion callbacks, which run a fresh
-      // tick), so a task that doesn't fit now cannot fit later in the
-      // same tick — parking it for the tick's remainder is exact, and
-      // each id pops at most once per tick (O(R log R)).
+      // Exec-tier tasks parked THIS tick on a refused admission. Within
+      // one synchronous tick `running` only grows (tasks leave it in the
+      // async completion callbacks, which run a fresh tick), so a task
+      // refused now cannot be admitted later in the same tick — parking
+      // it for the tick's remainder is exact, and each id pops at most
+      // once per tick (O(R log R)).
       const parked: Array<[string, number]> = []
 
       // Highest-priority admissible task: exec tier first (misses own
-      // the pool), then restore tier. With no reservations declared this
+      // the pool), then restore tier. With no admission policy this
       // short-circuits to exactly the legacy takeReady. A would-skip
-      // task returns without a fit check (finishing it is free); restore
-      // tasks cost zero by construction, so they never park.
+      // task returns without asking (finishing it is free); restore
+      // tasks are never asked, so they never park.
       const takeFitting = (): string | undefined => {
-        // No pools declared (the common case): a full local pool admits
-        // nothing, so do not scan — this is the legacy O(1) gate.
-        if (poolOf === undefined && active >= concurrency) return undefined
-        while (execReady.size > 0) {
-          const seq = execReady.peekSeq()
-          const id = execReady.pop() as string
-          if (willSkip(id) || ((!resourcesActive || fits(id)) && hasRoom(id))) return id
-          parked.push([id, seq])
+        // No pools, no policy and a full exec lane: nothing on the
+        // exec queue can be admitted, so it is not scanned — scanning it
+        // would pop and re-park every ready exec task on every tick, and
+        // with a wide frontier that is O(R) per tick, O(R²) per run (the
+        // 6,000-task scale pin went 0.5 s → 28 s when the restore lane's
+        // first cut let the scan run past a full exec lane, 2026-09-10).
+        // This is the legacy O(1) gate, kept per lane.
+        const execAdmissible = poolOf !== undefined || admitActive || active < concurrency
+        if (execAdmissible) {
+          while (execReady.size > 0) {
+            const seq = execReady.peekSeq()
+            const id = execReady.pop() as string
+            if (willSkip(id)) return id
+            if (hasRoom(id)) {
+              if (!admitActive || admits(id)) return id
+              // A free worker, refused by the policy: the hold starts now.
+              if (!heldSince.has(id)) heldSince.set(id, Date.now())
+            }
+            parked.push([id, seq])
+          }
         }
-        // Restore-tier tasks are local work (a cache restore on this disk).
-        return active < concurrency ? restoreReady.pop() : undefined
+        // Restore-tier tasks are local disk work, on their own lane.
+        return activeRestore < restoreConcurrency ? restoreReady.pop() : undefined
       }
 
       for (;;) {
@@ -660,47 +613,98 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
         // because dependents are pushed when `pending` hits 0 regardless
         // of outcome — keeps the propagation logic in one place.
         if (willSkip(id)) {
-          finishOne(id, { node, status: 'skipped', exitCode: 1, durationMs: 0 })
+          // The root of the block: a failed or aborted upstream names
+          // itself; a skipped one hands down its own root.
+          const blocker = upstream.find(
+            (u) => u !== undefined && (u.status === 'failed' || u.status === 'aborted'),
+          )
+          const viaSkip = upstream.find((u) => u !== undefined && u.status === 'skipped')
+          const viaRestore = node.deps.find((d) => blockedRestores.has(d))
+          const blockedBy =
+            blocker?.node.id ??
+            viaSkip?.blockedBy ??
+            (viaRestore === undefined ? undefined : blockedRestores.get(viaRestore))
+          finishOne(id, {
+            node,
+            status: aborted() ? 'aborted' : 'skipped',
+            exitCode: 1,
+            durationMs: 0,
+            ...(blockedBy !== undefined ? { blockedBy } : {}),
+          })
           continue
         }
 
         const leave = admit(id)
-        // Reserve on dispatch; capture the cost so the release in the
-        // completion callbacks is symmetric even if the maps change.
-        const cost = costOf(id)
-        reserve(cost)
+        // Listed as running on dispatch, so the policy's next ask in this
+        // tick sees it; the completion callbacks unlist it.
+        const untrack = track(id)
+        const since = admitActive ? heldSince.get(id) : undefined
+        const heldMs = since === undefined ? 0 : Math.max(1, Date.now() - since)
+        if (since !== undefined) heldSince.delete(id)
+        const withHold = (o: TaskOutcome): TaskOutcome =>
+          heldMs > 0 ? { ...o, admissionHeldMs: heldMs } : o
         // Crash-isolated observer hook — a throwing onStart must not abort
-        // the dispatch loop (it would strand the tick with reservations held).
-        try {
-          onStart?.(node)
-        } catch (err) {
-          const m = err instanceof Error ? err.message : String(err)
-          process.stderr.write(`[vx] onStart observer threw for ${id}: ${m}\n`)
+        // the dispatch loop (it would strand the tick with the slot held).
+        // A demoted task's second dispatch is the same task, already started.
+        if (!demoted.has(id)) {
+          try {
+            onStart?.(node)
+          } catch (err) {
+            const m = err instanceof Error ? err.message : String(err)
+            process.stderr.write(`[vx] onStart observer threw for ${id}: ${m}\n`)
+          }
         }
 
         // `.then(onFulfilled, onRejected)` — NOT `.then(f).catch(g)`. The
         // rejection arm handles ONLY `execute()` rejecting; a throw from
-        // the fulfillment arm (finishOne / onFinish / tick) must NOT also
-        // run the rejection arm, or `active`/`reserved` release twice —
-        // and a double release drives `reserved` negative, permanently
-        // wedging the solo-clamp gate.
+        // the fulfillment arm must NOT also run the rejection arm, or
+        // `active` releases twice — and a double release wedges the count
+        // gate. NOT `onFinish`, which this comment used to cite: it is
+        // isolated inside `finishOne`, so it never reaches here (which is
+        // why the row named for it cannot tell the two shapes apart —
+        // item 560). What is left unisolated is the caller's
+        // `settledOf`, and under THIS shape a throw there strands the
+        // task's outcome and hangs the run rather than double-releasing.
+        // The two-arm form is still the correct choice; there is simply
+        // no reachable throw that makes the difference observable.
         execute(node, upstream).then(
-          (outcome) => {
+          (raw) => {
+            const outcome = withHold(raw)
             leave()
-            release(cost)
-            finishOne(id, outcome)
+            untrack()
+            // The slot is free now; the dependents wait for what the
+            // outcome still owes (a save landing), if anything.
+            const settled = options.settledOf?.(outcome)
+            if (settled === undefined) {
+              finishOne(id, outcome)
+              tick()
+              return
+            }
             tick()
+            const done = (): void => {
+              finishOne(id, outcome)
+              tick()
+            }
+            void settled.then(done, done)
           },
           (err: unknown) => {
+            if (err instanceof RestoreDemoted) {
+              leave()
+              untrack()
+              demoted.add(id)
+              if (pending.get(id) === 0) execReady.push(id)
+              tick()
+              return
+            }
             const message = err instanceof Error ? err.message : String(err)
-            const outcome: TaskOutcome = {
+            const outcome: TaskOutcome = withHold({
               node,
               status: 'failed',
               exitCode: 1,
               durationMs: 0,
-            }
+            })
             leave()
-            release(cost)
+            untrack()
             finishOne(id, outcome)
             // Surface the error live; the outcome itself doesn't
             // carry captured stderr (that's the logger's job). A
@@ -709,20 +713,37 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
             // plainly, never as an "internal error".
             if (isUserError(err)) {
               process.stderr.write(`[vx] ${id}: ${message}\n`)
+            } else if (isFsRefusal(err)) {
+              process.stderr.write(`[vx] ${id}: ${message} — ${fsRefusalHint(err)}\n`)
             } else {
               const named = err instanceof Error && err.name !== 'Error' ? `${err.name}: ` : ''
-              process.stderr.write(`[vx] internal error in ${id}: ${named}${message}\n`)
+              // A wrapped error's cause is the fact the reader needs (a
+              // CorruptArtifactError over an ENOENT is a race, not a bad
+              // archive); print it, since the stack is not printed here.
+              const cause =
+                err instanceof Error && err.cause instanceof Error ? err.cause : undefined
+              const because =
+                cause === undefined
+                  ? ''
+                  : ` (cause: ${(cause as NodeJS.ErrnoException).code ?? cause.name}: ${cause.message})`
+              process.stderr.write(`[vx] internal error in ${id}: ${named}${message}${because}\n`)
             }
             tick()
           },
         )
       }
 
-      // Repush parked ids with their ORIGINAL seqs — FIFO-among-equals
-      // is exactly preserved for the next tick's admission pass.
+      // Repush parked ids with their ORIGINAL seqs, so the next tick's
+      // admission pass does not depend on `parked` having been built in
+      // ranking order (it is, but see `push`).
       for (const [id, seq] of parked) execReady.push(id, seq)
 
-      if (outcomes.size === nodes.size && active === 0) {
+      // The outcome count alone: a slot is released before its outcome
+      // lands (`leave()` precedes `finishOne` in both arms, and a skip
+      // takes none), so a task still holding one has no outcome yet, and
+      // the lane counters could add nothing — deleting both from this
+      // condition survived the whole core suite (item 644).
+      if (outcomes.size === nodes.size) {
         resolved = true
         resolve(outcomes)
       }
@@ -731,21 +752,3 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
     tick()
   })
 }
-
-export function mergePriorities(
-  baseline: ReadonlyMap<string, number>,
-  overrides: ReadonlyMap<string, number>,
-): ReadonlyMap<string, number> {
-  // The override caller scored "expected critical-path duration" in ms.
-  // Baseline reverse-deps counts are bounded by N; scale the override so
-  // it sorts above the baseline for any node it covers, and add the
-  // baseline as a tie-break for parity within the override set.
-  const SCALE = 1 << 20
-  const out = new Map<string, number>()
-  for (const [id, w] of baseline) out.set(id, w)
-  for (const [id, w] of overrides) {
-    const b = baseline.get(id) ?? 0
-    out.set(id, w * SCALE + b)
-  }
-  return out
-}