@vzn/vx 0.0.651 → 0.0.653

package/src/graph/scheduler.ts

@@ -33,6 +33,81 @@ export class RestoreDemoted extends Error {
   }
 }
 
+/**
+ * Thrown by `execute` when restoring a cache hit threw (a full disk, a
+ * path in the way, a link out of the project). The run reports `error`
+ * as any throw, and the failed outcome says `restoreFailed`: a failed
+ * task, counted as neither a miss nor a hit.
+ */
+export class RestoreFailed extends Error {
+  constructor(readonly error: unknown) {
+    super(thrownMessage(error))
+    this.name = 'RestoreFailed'
+  }
+}
+
+/**
+ * Thrown by `execute` when deriving the task's key threw (a runtime probe
+ * that exited non-zero, a refused `inputs.env` value). The command never
+ * ran: the failed outcome is marked (`isKeyFailed`), and the line says so.
+ */
+export class KeyFailed extends Error {
+  constructor(readonly error: unknown) {
+    super(thrownMessage(error))
+    this.name = 'KeyFailed'
+  }
+}
+
+// A mark, not a field: `TaskOutcome` and `RunPlan` are public types. A symbol
+// rides a spread copy and never reaches JSON.
+const KEY_FAILED = Symbol('keyFailed')
+
+/** Mark a failed outcome (or a plan holding one) as having no derivable key. */
+export function markKeyFailed<T extends object>(o: T): T {
+  ;(o as Record<symbol, unknown>)[KEY_FAILED] = true
+  return o
+}
+
+/** Whether `markKeyFailed` marked it: no key, so no command ran and no cache was asked. */
+export function isKeyFailed(o: object): boolean {
+  return (o as Record<symbol, unknown>)[KEY_FAILED] === true
+}
+
+/**
+ * Thrown by `execute` when vx's own work on a miss's outputs threw: the
+ * clean before an attempt, or the save after an exit 0 (a name that is
+ * not UTF-8). The outcome keeps `exitCode` and `durationMs` (what the
+ * command returned and took so far) and is marked (`outputsFailedOf`),
+ * so no label names an exit the command never returned.
+ */
+export class OutputsFailed extends Error {
+  constructor(
+    readonly error: unknown,
+    readonly step: OutputsStep,
+    readonly exitCode: number,
+    readonly durationMs: number,
+  ) {
+    super(thrownMessage(error))
+    this.name = 'OutputsFailed'
+  }
+}
+
+type OutputsStep = 'clean' | 'save'
+
+// A mark, as KEY_FAILED: `TaskOutcome` is a public type.
+const OUTPUTS_FAILED = Symbol('outputsFailed')
+
+/** Mark a failed outcome (or its view) with the output step that failed it. */
+export function markOutputsFailed<T extends object>(o: T, step: OutputsStep): T {
+  ;(o as Record<symbol, unknown>)[OUTPUTS_FAILED] = step
+  return o
+}
+
+/** The output step `markOutputsFailed` marked, if any. */
+export function outputsFailedOf(o: object): OutputsStep | undefined {
+  return (o as Record<symbol, OutputsStep | undefined>)[OUTPUTS_FAILED]
+}
+
 /** What moved in a missed task's key since the last entry saved for it. */
 export interface InputChanges {
   /** How many components changed, appeared or went. 0: the same key, its entry gone. */
@@ -128,6 +203,13 @@ export interface TaskOutcome {
    * so a dependant that folds `hash` saves nothing either (A-12).
    */
   unkeyed?: true
+  /**
+   * The input this task's own re-check found moved while it ran (absolute),
+   * or null when the key re-derived before the command already differed.
+   * The command read a state of it no `vx watch` judgement saw, so the
+   * loop counts its next settled state as a change.
+   */
+  driftedInput?: string | null
   /**
    * The task with no `inputs` this task's key folds (this task, or one
    * upstream): no run can derive that key again, so it neither read nor
@@ -148,6 +230,14 @@ export interface TaskOutcome {
    * read this one field (item 267).
    */
   blockedBy?: string
+  /**
+   * On an outcome that ran behind a failed dependency (`continueMode:
+   * 'always'`), one `--exclude-dependencies` dropped, or a server that died
+   * while it ran: the task at the root of it. Its result is not its key's, so no flaky verdict reads it and
+   * its run row names the root as a skip's `blockedBy` does. Set by `run()`
+   * before any observer (a plugin's `onTaskComplete`) hears the outcome.
+   */
+  taintedBy?: string
   /**
    * On a `skipped` outcome: its `exec.platforms` leave out this machine.
    * Its dependants are skipped with `blockedBy` naming it, and neither
@@ -186,7 +276,7 @@ export interface TaskOutcome {
   serverExit?: number
   /**
    * The input that changed while the command ran, failing the task under
-   * `inputStability.severity: 'error'` (workspace-relative; '' when vx
+   * `inputs.stability.severity: 'error'` (workspace-relative; '' when vx
    * could not name it). The exit is vx's 1; the label reads this instead.
    */
   inputMoved?: string
@@ -206,6 +296,12 @@ export interface TaskOutcome {
   outputs?: 'deferred'
   /** The task ran here and its result went to the cache. The footer's `↑`. */
   saved?: true
+  /**
+   * On a `failed` outcome: a cache hit whose restore threw, so no command
+   * ran. It counts as failed only, in no cache column: not a miss, not
+   * its layer's hit.
+   */
+  restoreFailed?: true
   /**
    * v11 analytics: hrtime span relative to the parent run's t=0.
    * Lets downstream analytics reconstruct the actual parallel timeline
@@ -231,6 +327,8 @@ export interface TaskOutcome {
   attempts?: number
   /** The attempts that failed and were run again, in order: when each ended (epoch ms) and how. */
   failedAttempts?: readonly { endedAt: number; exitCode: number; timedOut?: true }[]
+  /** Each of `failedAttempts`' output, as `failedOutput` encodes it: what `vx last --log` shows. */
+  attemptOutputs?: readonly string[]
   /**
    * Set when this run proved the task flaky: its key failed after it had
    * passed (this run's included) or it passed only on a retry. Its row carries it.
@@ -334,6 +432,8 @@ export interface ScheduleOptions {
    * dispatch, the slot and the promise around it were ~10 µs of each of a
    * warm run's 1,090 groups (X-192); a hit whose tree is current does no
    * more. A restore-tier task's `upstream` may hold undefined entries.
+   * The thunk's outcome is the scheduler's: its queue time is set on it,
+   * not on a copy (a copy per hit, 2026-11-07).
    */
   settleNow?: (node: TaskNode, upstream: TaskOutcome[]) => (() => TaskOutcome) | undefined
   /**
@@ -429,23 +529,23 @@ export interface ScheduleOptions {
 class ReadyHeap {
   private readonly ids: string[] = []
   private readonly seq: number[] = []
-  // Each slot's priority, read once at push: a map lookup per comparison
+  // Each slot's priority, read once at push: a lookup per comparison
   // was two per heap level on every push and pop.
   private readonly prio: number[] = []
   private next = 0
   /** Queued ids `counts` holds true for. */
   counted = 0
   constructor(
-    private priority: ReadonlyMap<string, number>,
+    private priority: (id: string) => number,
     private readonly counts?: (id: string) => boolean,
   ) {}
   get size(): number {
     return this.ids.length
   }
   /** Re-rank every queued id under `priority`, and every later push. */
-  reprioritize(priority: ReadonlyMap<string, number>): void {
+  reprioritize(priority: (id: string) => number): void {
     this.priority = priority
-    for (let i = 0; i < this.ids.length; i++) this.prio[i] = priority.get(this.ids[i]!) ?? 0
+    for (let i = 0; i < this.ids.length; i++) this.prio[i] = priority(this.ids[i]!)
     for (let i = (this.ids.length >> 1) - 1; i >= 0; i--) this.down(i)
   }
   private down(i: number): void {
@@ -496,7 +596,7 @@ class ReadyHeap {
     if (this.counts?.(id) === true) this.counted++
     this.ids.push(id)
     this.seq.push(seq)
-    this.prio.push(this.priority.get(id) ?? 0)
+    this.prio.push(this.priority(id))
     let i = this.ids.length - 1
     while (i > 0) {
       const parent = (i - 1) >> 1
@@ -528,11 +628,13 @@ class ReadyHeap {
 }
 
 /**
- * The dead server a dependency stands for: itself, or one a group reaches.
- * A group is a name for its deps, and it finished the moment the server
- * was ready, so a task behind it ran against the dead one. Walked with a
- * stack and a seen set: recursion threw `RangeError` on a deep chain of
- * groups, and re-walked a group each path reached it by.
+ * The dead server a dependency stands for: itself, or one a group or a
+ * server reaches. A group is a name for its deps, and it finished the
+ * moment the server was ready, so a task behind it ran against the dead
+ * one; a server stays up on the servers it depends on, so a task behind
+ * `web` runs against the `api` under it too. Walked with a stack and a
+ * seen set: recursion threw `RangeError` on a deep chain of groups, and
+ * re-walked a group each path reached it by.
  */
 export function deadServerBehind(
   nodes: ReadonlyMap<string, TaskNode>,
@@ -547,7 +649,7 @@ export function deadServerBehind(
     seen.add(at)
     if (serverDied(at)) return at
     const n = nodes.get(at)
-    if (n === undefined || !isGroupTask(n)) continue
+    if (n === undefined || (!isGroupTask(n) && n.config.exec?.persistent === undefined)) continue
     // Reversed, so the first dep is walked first, as the order it names them.
     for (let i = n.deps.length - 1; i >= 0; i--) stack.push(n.deps[i]!)
   }
@@ -604,9 +706,9 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
       : options.restoreTier !== undefined && options.restoreTier.size > 0
         ? tieredReverseDepCount(nodes, options.restoreTier)
         : (options.reverseDeps?.() ?? computeReverseDepCount(nodes))
-  const priority: ReadonlyMap<string, number> = options.priorities
+  const priority: (id: string) => number = options.priorities
     ? mergePriorities(baseline, options.priorities)
-    : baseline
+    : (id) => baseline.get(id) ?? 0
 
   // Two ready queues — exec-tier (cache misses + unstable tasks) and
   // restore-tier (confirmed stable local hits). Each is a max-heap keyed by
@@ -625,7 +727,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   // (`dormant`) until a dependant is demoted, which starts it, or until
   // every dependant has finished, which settles it unspawned. Settled
   // without an outcome: it is not part of the run, as a pruned task is not.
-  // A group between it and its hits is idle with it.
+  // A group or a server between it and its hits is idle with it.
   const idleGroups = new Set<string>()
   const idle = idleServers(nodes, dependents, restoreTier, idleGroups)
   for (const g of idleGroups) idle.add(g)
@@ -682,7 +784,8 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   // complete.
   // When each queued id became ready; dispatch takes it out.
   const readyAt = new Map<string, number>()
-  const startedAt = Date.now()
+  // Elapsed-time stamps (queue and hold waits), immune to wall clock steps.
+  const startedAt = performance.now()
   for (const node of nodes.values()) {
     if (restoreTier?.has(node.id)) restoreReady.push(node.id)
     else if (node.deps.length === 0) execReady.push(node.id)
@@ -894,7 +997,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
         if (heldRelease.delete(d)) enter(d)
         else if (!inRestoreTier(d)) {
           execReady.push(d)
-          readyAt.set(d, Date.now())
+          readyAt.set(d, performance.now())
         }
       }
     }
@@ -1025,7 +1128,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
             if (hasRoom(id)) {
               if (!admitActive || admits(id)) return id
               // A free worker, refused by the policy: the hold starts now.
-              if (!heldSince.has(id)) heldSince.set(id, Date.now())
+              if (!heldSince.has(id)) heldSince.set(id, performance.now())
             }
             parked.push([id, seq])
           }
@@ -1105,7 +1208,8 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
           continue
         }
 
-        const queuedMs = Date.now() - (readyAt.get(id) ?? Date.now())
+        const readySince = readyAt.get(id)
+        const queuedMs = readySince === undefined ? 0 : Math.round(performance.now() - readySince)
         readyAt.delete(id)
         const settle = settleInPlace(node, upstream)
         let now: TaskOutcome | undefined
@@ -1118,7 +1222,8 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
           }
         }
         if (now !== undefined) {
-          finishOne(id, queuedMs === 0 ? now : { ...now, queuedMs })
+          if (queuedMs !== 0) now.queuedMs = queuedMs
+          finishOne(id, now)
           continue
         }
         const leave = admit(id)
@@ -1126,7 +1231,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
         // tick sees it; the completion callbacks unlist it.
         const untrack = track(id)
         const since = admitActive ? heldSince.get(id) : undefined
-        const heldMs = since === undefined ? 0 : Math.max(1, Date.now() - since)
+        const heldMs = since === undefined ? 0 : Math.max(1, Math.round(performance.now() - since))
         if (since !== undefined) heldSince.delete(id)
         const withHold = (o: TaskOutcome): TaskOutcome =>
           heldMs === 0 && queuedMs === 0
@@ -1175,8 +1280,8 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
             }
             void settled.then(done, () => done())
           },
-          (err: unknown) => {
-            if (err instanceof RestoreDemoted) {
+          (thrown: unknown) => {
+            if (thrown instanceof RestoreDemoted) {
               leave()
               untrack()
               demoted.add(id)
@@ -1184,16 +1289,22 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
                 idle.delete(p)
                 if (dormant.delete(p)) {
                   execReady.push(p)
-                  readyAt.set(p, Date.now())
+                  readyAt.set(p, performance.now())
                 }
               })
               if (pending.get(id) === 0) {
                 execReady.push(id)
-                readyAt.set(id, Date.now())
+                readyAt.set(id, performance.now())
               }
               tick()
               return
             }
+            const err =
+              thrown instanceof RestoreFailed ||
+              thrown instanceof KeyFailed ||
+              thrown instanceof OutputsFailed
+                ? thrown.error
+                : thrown
             const message = thrownMessage(err)
             // After the run's stop a rejection is the stop's doing (a probe
             // it killed, C-65): aborted, as any task it kills, and unsaid.
@@ -1201,9 +1312,12 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
             const outcome: TaskOutcome = withHold({
               node,
               status: stopped ? 'aborted' : 'failed',
-              exitCode: 1,
-              durationMs: 0,
+              exitCode: thrown instanceof OutputsFailed ? thrown.exitCode : 1,
+              durationMs: thrown instanceof OutputsFailed ? thrown.durationMs : 0,
+              ...(thrown instanceof RestoreFailed && !stopped ? { restoreFailed: true } : {}),
             })
+            if (thrown instanceof KeyFailed && !stopped) markKeyFailed(outcome)
+            if (thrown instanceof OutputsFailed && !stopped) markOutputsFailed(outcome, thrown.step)
             leave()
             untrack()
             if (stopped) {
@@ -1218,9 +1332,10 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
             // Out of descriptors is the environment's limit too: a stack
             // naming a file that was fine sent the user after the file.
             if (isUserError(err) || isFsRefusal(err) || isOutOfFds(err)) {
-              const text = isUserError(err)
+              const said = isUserError(err)
                 ? message
                 : `${message} — ${isOutOfFds(err) ? OUT_OF_FDS_HINT : fsRefusalHint(err)}`
+              const text = thrown instanceof KeyFailed ? `cache key not derived: ${said}` : said
               // In a task's own output block, in full: that block is read
               // alone, and "as <task> above" named one printed elsewhere or
               // not at all (2026-10-22). On bare stderr the first is above.
@@ -1288,9 +1403,25 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
           continue
         promoted.add(d)
         restoreReady.push(d)
-        readyAt.set(d, Date.now())
+        readyAt.set(d, performance.now())
+      }
+      // Idle as `idleServers` finds it: through the groups and servers
+      // nobody asked for above it. The servers beneath it are held on the
+      // same keys, so the walk idles them with it; a server already
+      // answered, or never held, keeps its course.
+      if (found.length > 0) {
+        const tier =
+          demoted.size === 0 ? restoreTier : new Set([...restoreTier!].filter(inRestoreTier))
+        const between = new Set<string>()
+        const holdsHere = (s: string): boolean =>
+          s === id || unanswered.has(s) || isGroupTask(nodes.get(s) as TaskNode)
+        for (const s of idleServers(nodes, dependents, tier, between)) if (holdsHere(s)) idle.add(s)
+        for (const g of between)
+          if (holdsHere(g) && !idleGroups.has(g)) {
+            idleGroups.add(g)
+            idle.add(g)
+          }
       }
-      if (dependents.get(id)?.every((d) => inRestoreTier(d)) === true) idle.add(id)
       if (heldReady.delete(id)) execReady.push(id)
       tick()
     }
@@ -1318,10 +1449,11 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
 
 /**
  * Persistent tasks nobody asked for whose every dependant is a restore-tier
- * hit, or a group nobody asked for that only such dependants stand behind:
- * a group runs nothing, so `env: { dependsOn: ['dev'] }` between the server
- * and its hits booted the server on every warm run. Those groups go in
- * `groups`.
+ * hit, or a group or server nobody asked for that only such dependants
+ * stand behind: a group runs nothing, so `env: { dependsOn: ['dev'] }`
+ * between the server and its hits booted the server on every warm run, and
+ * `api` under an idle `web` serves nobody either. Those groups and servers
+ * go in `groups`: the scheduler settles them as it settles a group.
  */
 export function idleServers(
   nodes: ReadonlyMap<string, TaskNode>,
@@ -1332,10 +1464,10 @@ export function idleServers(
   const out = new Set<string>()
   if (restoreTier === undefined || restoreTier.size === 0) return out
   const unasked = (n: TaskNode): boolean => !n.requested && n.surfaced !== true
-  // Per group: whether only hits stand behind it. Post-order on a stack, as
-  // groups nest as deep as the graph.
+  // Per group or server: whether only hits stand behind it. Post-order on a
+  // stack, as groups and servers nest as deep as the graph.
   const spare = new Map<string, boolean>()
-  const spareGroup = (root: string): boolean => {
+  const spareNode = (root: string): boolean => {
     const stack = [root]
     while (stack.length > 0) {
       const id = stack[stack.length - 1]!
@@ -1345,7 +1477,12 @@ export function idleServers(
       }
       const n = nodes.get(id)
       const ds = dependents.get(id)
-      if (n === undefined || !isGroupTask(n) || !unasked(n) || ds === undefined) {
+      if (
+        n === undefined ||
+        (!isGroupTask(n) && n.config.exec?.persistent === undefined) ||
+        !unasked(n) ||
+        ds === undefined
+      ) {
         spare.set(id, false)
         stack.pop()
         continue
@@ -1366,7 +1503,7 @@ export function idleServers(
   for (const n of nodes.values()) {
     if (n.config.exec?.persistent === undefined || !unasked(n)) continue
     const ds = dependents.get(n.id)
-    if (ds === undefined || !ds.every((d) => restoreTier.has(d) || spareGroup(d))) continue
+    if (ds === undefined || !ds.every((d) => restoreTier.has(d) || spareNode(d))) continue
     out.add(n.id)
     if (groups === undefined) continue
     const stack = ds.filter((d) => !restoreTier.has(d))

package/src/graph/stand-ins.ts

@@ -0,0 +1,36 @@
+// What an edge past a default build on a package cycle stands in for. Such
+// a build folds nothing of its cycle, so the walk takes edges past it
+// (`goPast`, `walkHolders` in task-graph.ts), and `inputs.tasks` selecting
+// it selects them (`selectFoldedDeps`).
+//
+// Kept off the public `TaskNode` shape under a symbol key: a spread copy
+// (the scheduler's restore view, an executor's `accepts` copy) carries it,
+// and `applyGraphStage` re-attaches it by id to a node a hook rebuilt.
+
+import type { TaskNode } from './task-graph.js'
+
+/** Each edge's id, to the ids of the edges it went past. */
+export type StandIns = Readonly<Record<string, readonly string[]>>
+
+const STAND_INS = Symbol('vx.standIns')
+type Carrier = TaskNode & { [STAND_INS]?: Record<string, string[]> }
+
+/** The node's stand-ins; undefined off a package cycle. */
+export function standInsOf(node: TaskNode): StandIns | undefined {
+  return (node as Carrier)[STAND_INS]
+}
+
+/** Give `node` its stand-ins: a graph read back, or a node a hook rebuilt. */
+export function setStandIns(node: TaskNode, standIns: StandIns): void {
+  const own: Record<string, string[]> = {}
+  for (const [edge, past] of Object.entries(standIns)) own[edge] = [...past]
+  ;(node as Carrier)[STAND_INS] = own
+}
+
+/** Record that `node`'s edge to `id` stands in for the edges in `past`. */
+export function addStandIn(node: TaskNode, id: string, past: readonly string[]): void {
+  const map = ((node as Carrier)[STAND_INS] ??= {})
+  const known = map[id]
+  if (known === undefined) map[id] = [...past]
+  else for (const p of past) if (!known.includes(p)) known.push(p)
+}