@vzn/vx 0.0.21 → 0.0.22

package/src/graph/scheduler.ts

@@ -1,6 +1,22 @@
-import { isUserError } from '../util/index.js'
+import { fsRefusalHint, isFsRefusal, isUserError } from '../util/index.js'
+import { computeReverseDepCount, mergePriorities, tieredReverseDepCount } from './priorities.js'
 import type { TaskNode } from './task-graph.js'
 
+/**
+ * Thrown by `execute` for a restore-tier task that turned out to have
+ * nothing to restore (its artifact vanished after the up-front probe). It
+ * may be running ahead of its deps, so it cannot run its command in that
+ * slot: the scheduler releases the slot and dispatches it again as an
+ * exec-tier task once its deps are done. It records no outcome, and its
+ * dependents wait for the second dispatch.
+ */
+export class RestoreDemoted extends Error {
+  constructor(readonly taskId: string) {
+    super(`${taskId}: restore demoted to a run`)
+    this.name = 'RestoreDemoted'
+  }
+}
+
 export type TaskStatus =
   | 'success'
   | 'cache-hit'
@@ -60,6 +76,26 @@ export interface TaskOutcome {
    * outcomes only; never folded into any key.
    */
   groupUpstream?: readonly TaskOutcome[]
+  /**
+   * On a `skipped` outcome: the task at the ROOT of what blocked it — the
+   * failed (or aborted) upstream, followed through any chain of skips
+   * between. Absent when the skip was fail-fast's (no upstream failed).
+   * The footer, `--summarize`, the telemetry record and the run row all
+   * read this one field (item 267).
+   */
+  blockedBy?: string
+  /**
+   * On a `failed` outcome: vx's own `timeout` killed the final attempt.
+   * The exit is the shell's 143, and every label says "timed out" instead
+   * of reading it as a signal (item 268).
+   */
+  timedOut?: true
+  /**
+   * On a failed persistent task: why it never became ready — the readiness
+   * deadline fired, the child exited first (`exitCode` is then its own), or
+   * the spawn itself failed. Every label reads this instead of the exit.
+   */
+  notReady?: 'timeout' | 'exited' | 'spawn'
   /** Executor-reported placement label (`ExecuteResult.where`) — set only
    *  when the task ran somewhere other than this host. Telemetry-only. */
   where?: string
@@ -163,7 +199,9 @@ export interface ScheduleOptions {
    *     it);
    *   - bypasses the `failedDep`→`skipped` check (its key is independent
    *     of dep success — pure-input transitive hashing — so a valid cached
-   *     output is reported `cache-hit` regardless of a dep failing).
+   *     output is reported `cache-hit` regardless of a dep failing);
+   *   - leaves the tier when `execute` throws `RestoreDemoted`, and is
+   *     dispatched again once its deps are done, with the dep check.
    * It still runs through `execute()` (so the logger frame is unchanged);
    * the orchestrator's execute reuses the up-front probe, so there is no
    * second cache.get. When undefined/empty, behavior is byte-identical.
@@ -188,96 +226,6 @@ export interface ScheduleOptions {
   admit?: (id: string, running: ReadonlySet<string>) => boolean
 }
 
-/**
- * Compute, for each task in the graph, how many OTHER tasks are
- * transitively blocked on it. Tasks with the highest count are the
- * most valuable to schedule first — finishing them unlocks the most
- * downstream work and minimizes worker idle time at the end of the
- * run. Matches Nx's `calculateReverseDeps`-driven schedule sort
- * (`packages/nx/src/tasks-runner/tasks-schedule.ts:166-207`).
- */
-// Exported for the perf guard in tests/scheduler.test.ts, which times
-// this function in isolation — end-to-end runGraph timing drowns the
-// closure cost in task-promise overhead and flakes on slow CI.
-export function computeReverseDepCount(nodes: Map<string, TaskNode>): Map<string, number> {
-  // Exact transitive-dependent COUNTS need closure SETS (diamonds
-  // double-count under naive summing). Set-of-strings closures are
-  // O(N²) entries and took 8.5s on a 1090-package, 100-layer repo;
-  // bitsets make the same closure O(E·N/32) time and N²/8 bits of
-  // memory (3270 tasks ≈ 1.3 MB) — single-digit ms at that scale.
-  const ids = [...nodes.keys()]
-  const index = new Map<string, number>()
-  for (let i = 0; i < ids.length; i++) index.set(ids[i]!, i)
-  const n = ids.length
-  const words = (n + 31) >>> 5
-
-  // Direct dependents as index lists + in-degree for the topo pass.
-  const directReverse: number[][] = Array.from({ length: n }, () => [])
-  const indegree = new Uint32Array(n)
-  for (const node of nodes.values()) {
-    const ni = index.get(node.id)!
-    for (const dep of node.deps) {
-      const di = index.get(dep)
-      if (di === undefined) continue
-      directReverse[di]!.push(ni)
-      indegree[ni]!++
-    }
-  }
-
-  // Kahn topo order over dependency edges (deps before dependents).
-  // Insertion order is topo today, but the closure's correctness
-  // must not hinge on an unstated property of buildTaskGraph.
-  const topo = new Int32Array(n)
-  let head = 0
-  let tail = 0
-  for (let i = 0; i < n; i++) if (indegree[i] === 0) topo[tail++] = i
-  while (head < tail) {
-    const v = topo[head++]!
-    for (const r of directReverse[v]!) {
-      if (--indegree[r]! === 0) topo[tail++] = r
-    }
-  }
-
-  // Reverse-topo sweep: every direct dependent's closure is final
-  // before its dependency folds it in. closure[i] = bitset over node
-  // indices of i's transitive dependents.
-  const closure = new Uint32Array(n * words)
-  const counts = new Map<string, number>()
-  for (let t = tail - 1; t >= 0; t--) {
-    const i = topo[t]!
-    const base = i * words
-    for (const r of directReverse[i]!) {
-      closure[base + (r >>> 5)]! |= 1 << (r & 31)
-      const rbase = r * words
-      for (let w = 0; w < words; w++) closure[base + w]! |= closure[rbase + w]!
-    }
-    let count = 0
-    for (let w = 0; w < words; w++) {
-      let v = closure[base + w]!
-      v = v - ((v >>> 1) & 0x55555555)
-      v = (v & 0x33333333) + ((v >>> 2) & 0x33333333)
-      count += (((v + (v >>> 4)) & 0x0f0f0f0f) * 0x01010101) >>> 24
-    }
-    counts.set(ids[i]!, count)
-  }
-  // Cycle-stranded nodes (never topo-visited) can't occur — the graph
-  // builder rejects cycles — but a missing Map entry would silently
-  // sort as undefined, so default them defensively to 0.
-  for (const id of ids) if (!counts.has(id)) counts.set(id, 0)
-  return counts
-}
-
-/**
- * Run the task graph. Independent tasks run in parallel up to `concurrency`.
- * If a task fails, its dependents are marked `skipped` but unrelated tasks
- * keep running so the user gets maximum information per invocation.
- *
- * Scheduling: when more than one task is ready, the scheduler picks the
- * one that blocks the most downstream work (most transitive reverse
- * dependents). Ties break in graph-insertion order (which is the topo
- * order produced by `buildTaskGraph`). Minimizes worker idle at the
- * end of the run.
- */
 /**
  * A binary max-heap of ready task ids, ordered by (priority DESC, enqueue-seq
  * ASC) so `pop()` returns the highest-priority task and equal-priority ties
@@ -310,8 +258,18 @@ class ReadyHeap {
   }
   /**
    * `seq` defaults to a fresh monotonic counter. A parked-then-repushed
-   * task passes its ORIGINAL seq back in so FIFO-among-equals survives
-   * the round trip (a fresh seq would demote it behind later arrivals).
+   * task passes its ORIGINAL seq back in, which makes FIFO-among-equals a
+   * property of the heap rather than of the tick loop's shape.
+   *
+   * It is NOT the difference between correct and incorrect order today,
+   * and the comment here used to claim it was ("a fresh seq would demote
+   * it behind later arrivals"). It would not: `parked` is a subsequence
+   * of the tick's pop order, so it is already in ranking order, and every
+   * push that follows takes a later seq — so fresh seqs handed out in
+   * parked order reproduce the same ranking. Measured over 800 randomized
+   * scenarios (random DAGs, costs, budgets, concurrency, failures, all
+   * three continue modes): zero order differences, item 493. Kept because
+   * it holds without that argument.
    */
   push(id: string, seq: number = this.next++): void {
     this.ids.push(id)
@@ -353,6 +311,17 @@ class ReadyHeap {
   }
 }
 
+/**
+ * Run the task graph. Independent tasks run in parallel up to `concurrency`.
+ * If a task fails, its dependents are marked `skipped` but unrelated tasks
+ * keep running so the user gets maximum information per invocation.
+ *
+ * Scheduling: when more than one task is ready, the scheduler picks the
+ * one that blocks the most downstream work (most transitive reverse
+ * dependents). Ties break in graph-insertion order (which is the topo
+ * order produced by `buildTaskGraph`). Minimizes worker idle at the
+ * end of the run.
+ */
 export async function runGraph(options: ScheduleOptions): Promise<Map<string, TaskOutcome>> {
   const { nodes, concurrency, execute, onStart, onFinish } = options
   const continueMode = options.continueMode ?? 'deps-ok'
@@ -381,7 +350,10 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   // priorities map (e.g. from history-aware critical-path estimation),
   // use those weights. Falls back to the reverse-deps-count heuristic
   // for nodes the caller didn't score, so partial coverage works.
-  const baseline = computeReverseDepCount(nodes)
+  const baseline =
+    options.restoreTier !== undefined && options.restoreTier.size > 0
+      ? tieredReverseDepCount(nodes, options.restoreTier)
+      : computeReverseDepCount(nodes)
   const priority: ReadonlyMap<string, number> = options.priorities
     ? mergePriorities(baseline, options.priorities)
     : baseline
@@ -395,6 +367,10 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   // idle capacity (or run when an exec is blocked on a restorable dep and
   // nothing else is runnable).
   const restoreTier = options.restoreTier
+  // Restore-tier tasks whose execute threw `RestoreDemoted`: from then on
+  // they are exec-tier tasks, gated on their deps like any other.
+  const demoted = new Set<string>()
+  const inRestoreTier = (id: string): boolean => restoreTier?.has(id) === true && !demoted.has(id)
   const execReady = new ReadyHeap(priority)
   const restoreReady = new ReadyHeap(priority)
   // A restore-tier task is dep-independent (a stable hit's restore needs
@@ -440,16 +416,14 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   const hasRoom = (id: string): boolean => {
     const pool = poolOf?.(id)
     if (pool === undefined) {
-      return options.restoreTier?.has(id)
-        ? activeRestore < restoreConcurrency
-        : active < concurrency
+      return inRestoreTier(id) ? activeRestore < restoreConcurrency : active < concurrency
     }
     return (poolActive.get(pool.name) ?? 0) < pool.capacity
   }
   const admit = (id: string): (() => void) => {
     const pool = poolOf?.(id)
     if (pool === undefined) {
-      if (options.restoreTier?.has(id)) {
+      if (inRestoreTier(id)) {
         activeRestore++
         return () => {
           activeRestore--
@@ -467,7 +441,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   }
   // The policy sees exec-tier local tasks only; a restore is a tar
   // extract and a pooled task runs on someone else's capacity.
-  const local = (id: string): boolean => !options.restoreTier?.has(id) && poolOf?.(id) === undefined
+  const local = (id: string): boolean => !inRestoreTier(id) && poolOf?.(id) === undefined
   const admits = (id: string): boolean => !local(id) || admitPolicy!(id, running)
   // With no policy nothing reads `running`, so nothing is tracked: the
   // count-only dispatch allocates no closure and touches no set per task.
@@ -496,14 +470,54 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
         const m = err instanceof Error ? err.message : String(err)
         process.stderr.write(`[vx] onFinish observer threw for ${id}: ${m}\n`)
       }
+      // A restore-tier task runs before its deps, but its DEPENDENTS keep
+      // the order `dependsOn` states: they are released once its own deps
+      // have settled too. Released at its own finish, an `e2e` ran beside
+      // the `db:migrate` two edges up whenever the middle task was a hit,
+      // and ran green after that dep failed (item 963).
+      if (inRestoreTier(id) && (pending.get(id) ?? 0) > 0) {
+        heldRelease.add(id)
+        return
+      }
+      release(id)
+    }
+
+    // A restore-tier task whose deps did not all succeed is still a
+    // `cache-hit` (its key is theirs, not their outcome), but it passes the
+    // block down: its dependents are skipped as a failed dep's would be.
+    // Its root: a failed or aborted dep, or the root a skipped or blocked
+    // one carries.
+    const blockedRestores = new Map<string, string | undefined>()
+    const heldRelease = new Set<string>()
+    const release = (id: string): void => {
+      if (inRestoreTier(id) && continueMode !== 'always') {
+        for (const dep of (nodes.get(id) as TaskNode).deps) {
+          const u = outcomes.get(dep)
+          if (u?.status === 'failed' || u?.status === 'aborted') {
+            blockedRestores.set(id, dep)
+            break
+          }
+          if (u?.status === 'skipped') {
+            blockedRestores.set(id, u.blockedBy)
+            break
+          }
+          if (blockedRestores.has(dep)) {
+            blockedRestores.set(id, blockedRestores.get(dep))
+            break
+          }
+        }
+      }
       const ds = dependents.get(id)
       if (!ds) return
       for (const d of ds) {
         const rem = (pending.get(d) ?? 0) - 1
         pending.set(d, rem)
+        if (rem !== 0) continue
         // Restore-tier dependents were already enqueued at startup (they
-        // don't wait on deps); only re-enqueue an exec-tier dependent.
-        if (rem === 0 && !restoreTier?.has(d)) execReady.push(d)
+        // don't wait on deps); one that finished early releases its own
+        // dependents now. Only an exec-tier dependent is enqueued.
+        if (heldRelease.delete(d)) release(d)
+        else if (!inRestoreTier(d)) execReady.push(d)
       }
     }
 
@@ -522,7 +536,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
     const aborted = (): boolean => options.signal?.aborted === true
     const willSkip = (id: string): boolean => {
       if (failFastTripped || aborted()) return true
-      if (restoreTier?.has(id)) return false
+      if (inRestoreTier(id)) return false
       if (continueMode === 'always') return false
       const node = nodes.get(id) as TaskNode
       return node.deps.some((d) => {
@@ -533,7 +547,12 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
         // from them — under the key a healthy run derives, since the fold
         // takes the upstream's INPUT key — so the next run replays those
         // bytes as a green hit.
-        return u?.status === 'failed' || u?.status === 'skipped' || u?.status === 'aborted'
+        return (
+          u?.status === 'failed' ||
+          u?.status === 'skipped' ||
+          u?.status === 'aborted' ||
+          blockedRestores.has(d)
+        )
       })
     }
 
@@ -594,11 +613,23 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
         // because dependents are pushed when `pending` hits 0 regardless
         // of outcome — keeps the propagation logic in one place.
         if (willSkip(id)) {
+          // The root of the block: a failed or aborted upstream names
+          // itself; a skipped one hands down its own root.
+          const blocker = upstream.find(
+            (u) => u !== undefined && (u.status === 'failed' || u.status === 'aborted'),
+          )
+          const viaSkip = upstream.find((u) => u !== undefined && u.status === 'skipped')
+          const viaRestore = node.deps.find((d) => blockedRestores.has(d))
+          const blockedBy =
+            blocker?.node.id ??
+            viaSkip?.blockedBy ??
+            (viaRestore === undefined ? undefined : blockedRestores.get(viaRestore))
           finishOne(id, {
             node,
             status: aborted() ? 'aborted' : 'skipped',
             exitCode: 1,
             durationMs: 0,
+            ...(blockedBy !== undefined ? { blockedBy } : {}),
           })
           continue
         }
@@ -614,18 +645,28 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
           heldMs > 0 ? { ...o, admissionHeldMs: heldMs } : o
         // Crash-isolated observer hook — a throwing onStart must not abort
         // the dispatch loop (it would strand the tick with the slot held).
-        try {
-          onStart?.(node)
-        } catch (err) {
-          const m = err instanceof Error ? err.message : String(err)
-          process.stderr.write(`[vx] onStart observer threw for ${id}: ${m}\n`)
+        // A demoted task's second dispatch is the same task, already started.
+        if (!demoted.has(id)) {
+          try {
+            onStart?.(node)
+          } catch (err) {
+            const m = err instanceof Error ? err.message : String(err)
+            process.stderr.write(`[vx] onStart observer threw for ${id}: ${m}\n`)
+          }
         }
 
         // `.then(onFulfilled, onRejected)` — NOT `.then(f).catch(g)`. The
         // rejection arm handles ONLY `execute()` rejecting; a throw from
-        // the fulfillment arm (finishOne / onFinish / tick) must NOT also
-        // run the rejection arm, or `active` releases twice — and a
-        // double release wedges the count gate.
+        // the fulfillment arm must NOT also run the rejection arm, or
+        // `active` releases twice — and a double release wedges the count
+        // gate. NOT `onFinish`, which this comment used to cite: it is
+        // isolated inside `finishOne`, so it never reaches here (which is
+        // why the row named for it cannot tell the two shapes apart —
+        // item 560). What is left unisolated is the caller's
+        // `settledOf`, and under THIS shape a throw there strands the
+        // task's outcome and hangs the run rather than double-releasing.
+        // The two-arm form is still the correct choice; there is simply
+        // no reachable throw that makes the difference observable.
         execute(node, upstream).then(
           (raw) => {
             const outcome = withHold(raw)
@@ -647,6 +688,14 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
             void settled.then(done, done)
           },
           (err: unknown) => {
+            if (err instanceof RestoreDemoted) {
+              leave()
+              untrack()
+              demoted.add(id)
+              if (pending.get(id) === 0) execReady.push(id)
+              tick()
+              return
+            }
             const message = err instanceof Error ? err.message : String(err)
             const outcome: TaskOutcome = withHold({
               node,
@@ -664,20 +713,37 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
             // plainly, never as an "internal error".
             if (isUserError(err)) {
               process.stderr.write(`[vx] ${id}: ${message}\n`)
+            } else if (isFsRefusal(err)) {
+              process.stderr.write(`[vx] ${id}: ${message} — ${fsRefusalHint(err)}\n`)
             } else {
               const named = err instanceof Error && err.name !== 'Error' ? `${err.name}: ` : ''
-              process.stderr.write(`[vx] internal error in ${id}: ${named}${message}\n`)
+              // A wrapped error's cause is the fact the reader needs (a
+              // CorruptArtifactError over an ENOENT is a race, not a bad
+              // archive); print it, since the stack is not printed here.
+              const cause =
+                err instanceof Error && err.cause instanceof Error ? err.cause : undefined
+              const because =
+                cause === undefined
+                  ? ''
+                  : ` (cause: ${(cause as NodeJS.ErrnoException).code ?? cause.name}: ${cause.message})`
+              process.stderr.write(`[vx] internal error in ${id}: ${named}${message}${because}\n`)
             }
             tick()
           },
         )
       }
 
-      // Repush parked ids with their ORIGINAL seqs — FIFO-among-equals
-      // is exactly preserved for the next tick's admission pass.
+      // Repush parked ids with their ORIGINAL seqs, so the next tick's
+      // admission pass does not depend on `parked` having been built in
+      // ranking order (it is, but see `push`).
       for (const [id, seq] of parked) execReady.push(id, seq)
 
-      if (outcomes.size === nodes.size && active === 0 && activeRestore === 0) {
+      // The outcome count alone: a slot is released before its outcome
+      // lands (`leave()` precedes `finishOne` in both arms, and a skip
+      // takes none), so a task still holding one has no outcome yet, and
+      // the lane counters could add nothing — deleting both from this
+      // condition survived the whole core suite (item 644).
+      if (outcomes.size === nodes.size) {
         resolved = true
         resolve(outcomes)
       }
@@ -686,21 +752,3 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
     tick()
   })
 }
-
-export function mergePriorities(
-  baseline: ReadonlyMap<string, number>,
-  overrides: ReadonlyMap<string, number>,
-): ReadonlyMap<string, number> {
-  // The override caller scored "expected critical-path duration" in ms.
-  // Baseline reverse-deps counts are bounded by N; scale the override so
-  // it sorts above the baseline for any node it covers, and add the
-  // baseline as a tie-break for parity within the override set.
-  const SCALE = 1 << 20
-  const out = new Map<string, number>()
-  for (const [id, w] of baseline) out.set(id, w)
-  for (const [id, w] of overrides) {
-    const b = baseline.get(id) ?? 0
-    out.set(id, w * SCALE + b)
-  }
-  return out
-}