@vzn/vx 0.0.18 → 0.0.21

package/src/graph/scheduler.ts

@@ -14,65 +14,6 @@ export type TaskStatus =
   // are partial.
   | 'aborted'
 
-/**
- * Cache-correctness verdict for a task under `vx run --verify` (Phase 1:
- * determinism; Phase 2: input-completeness). Declared here (structurally)
- * because `graph` can't import `orchestrator` where the verifier lives —
- * same pattern as `inputComponents`. A pure side-channel: never hashed,
- * never persisted in Phase 1/2.
- */
-export type VerifyVerdict =
-  | { kind: 'proven-deterministic' }
-  /** Phase 2: input-completeness proved — the task read nothing outside its
-   *  declared inputs (from `--verify=inputs` when determinism wasn't also
-   *  requested; an `--verify=all` pass reports the stronger deterministic). */
-  | { kind: 'proven-complete' }
-  /** Re-ran; outputs differ from the cached ones. `changed` names the rels. */
-  | { kind: 'nondeterministic'; changed: readonly string[] }
-  /** Phase 2: read a workspace path outside the declared inputs — the declared
-   *  `cache.inputs` are incomplete, so a hit could serve stale bytes. `paths`
-   *  names the undeclared reads (workspace-relative; empty when the sandbox
-   *  denied structurally but strace wasn't available to name them). */
-  | { kind: 'undeclared-inputs'; paths: readonly string[] }
-  /** Diverged but the task is on `--verify-allow` — reported, not failed. */
-  | { kind: 'allowed-nondeterministic'; changed: readonly string[] }
-  /** The verify re-run exited non-zero / timed out (nondeterministic by
-   *  definition — identical inputs, different outcome). */
-  | { kind: 'rerun-failed'; exitCode: number }
-  /** Cacheable + executed but declares no outputs — nothing to replay. */
-  | { kind: 'no-outputs' }
-  /** Didn't execute (cache hit) / not cacheable / group / persistent. */
-  | { kind: 'not-verified' }
-  /** A `remote: 'only'` task under `--verify=inputs`: verify pins placement
-   *  local, so the task no-ops — there is no execution to sandbox, locally
-   *  or anywhere. Reported, not silent: the proof does not cover it. */
-  | { kind: 'unverifiable-remote-only' }
-
-/**
- * Content fingerprint of a task's output tree, computed under a `--verify*`
- * mode on the executed (miss) path. Never hashed into any key; pure telemetry
- * side-channel for the cross-machine diff (a serve pairs fingerprints for the
- * SAME cache key across platforms and names diverging outputs). Declared here
- * (structurally) because `graph` can't import `orchestrator` — the
- * `VerifyVerdict` pattern. See docs/design/verify-cross-machine-2026-07.md.
- */
-export interface OutputFingerprint {
-  /** Roll-up: xxh3hex over the sorted (key, hash) pairs, folded as
-   *  `key \0 hash \n` (\0 boundaries — the v18 lesson). Always present;
-   *  divergence DETECTION never depends on the per-file map. */
-  tree: string
-  /** Total files in the tree (pre-truncation). */
-  fileCount: number
-  /** Per-file map as sorted [outputKey, xxh3hex] pairs, capped at
-   *  FP_MAX_FILES (500). Deterministic truncation — sorted by key,
-   *  first N — so two machines' truncated maps cover the same subset
-   *  and partial diffs still name real rels. */
-  files?: ReadonlyArray<readonly [string, string]>
-  /** Set when `files` was truncated to the cap (or dropped by the
-   *  sink's run budget). */
-  truncated?: boolean
-}
-
 export interface TaskOutcome {
   node: TaskNode
   status: TaskStatus
@@ -90,6 +31,22 @@ export interface TaskOutcome {
    * Undefined on every non-hit outcome — nothing was skipped.
    */
   storedDurationMs?: number
+  /**
+   * For cache-hit statuses: what the PRODUCING execution used, read from
+   * the entry (it rides the artifact, so a remote hit on a fresh machine
+   * carries it too). The same split as `storedDurationMs`: `cpuMs` /
+   * `peakRssBytes` are what THIS run spent and stay absent on a hit.
+   */
+  storedCpuMs?: number
+  storedPeakRssBytes?: number
+  /**
+   * How long an `admit` policy held this task after it was ready with a
+   * free worker — the wait the plugin, not the count gate, imposed. Set
+   * only when a policy refused it at least once (so a run with no policy
+   * carries the field on no outcome); `--summarize` rows, the event
+   * stream and the summary footer show it.
+   */
+  admissionHeldMs?: number
   /** v11 analytics: CPU time + peak RSS for this task's child process. */
   cpuMs?: number
   peakRssBytes?: number
@@ -134,10 +91,10 @@ export interface TaskOutcome {
    */
   attempts?: number
   /**
-   * Count of sandbox violations captured during this task's exec.
-   * Populated only when `--sandbox` was set and the task is cached.
-   * Non-zero values mean the task read files outside its declared
-   * inputs; `cache.save()` was skipped so the result can't be replayed.
+   * Count of sandbox violations captured during this task's exec (a task
+   * declaring `exec.sandbox`). Non-zero means the task touched something
+   * it did not declare; execute-task fails the task on any violation, so
+   * nothing built that way is ever saved or replayed.
    */
   sandboxViolations?: number
   /**
@@ -146,38 +103,19 @@ export interface TaskOutcome {
    * inline in the task's block instead of as loose status output.
    */
   sandboxViolationLines?: string[]
-  /**
-   * Cache-correctness verdict under `vx run --verify`. Set only in verify
-   * mode; a plain run leaves it undefined. Pure side-channel (never hashed).
-   */
-  verify?: VerifyVerdict
-  /**
-   * Output-tree fingerprint under a fingerprinting `--verify*` mode
-   * (`--verify` / `=all` / `=fingerprint`), executed + cacheable +
-   * output-declaring tasks only. A plain run leaves it undefined.
-   */
-  outputFp?: OutputFingerprint
 }
 
 export type ContinueMode = 'never' | 'deps-ok' | 'always'
 
-/**
- * Resolved per-task resource reservation, in absolute units (cpu may be
- * fractional; mem is bytes). Declared here (structurally) because `graph`
- * can't import `orchestrator`, where the resolver lives — same pattern as
- * `VerifyVerdict`. A `0` axis means "reserve nothing, run freely": the
- * task is exempt from that axis entirely (needs no headroom, holds none).
- */
-export interface ResourceCost {
-  cpu: number
-  mem: number
-}
-
-const ZERO_COST: ResourceCost = { cpu: 0, mem: 0 }
-
 export interface ScheduleOptions {
   nodes: Map<string, TaskNode>
   concurrency: number
+  /**
+   * An aborted run dispatches nothing further: every task not yet
+   * started completes `aborted` (the ones in flight are being killed by
+   * the orchestrator and report `aborted` through the runner's signal).
+   */
+  signal?: AbortSignal
   /**
    * Failure propagation (default 'deps-ok' — the historical behavior):
    *   - 'deps-ok': a failed/skipped/aborted upstream skips its
@@ -186,18 +124,25 @@ export interface ScheduleOptions {
    *     finish naturally, everything not yet started completes as
    *     skipped (restore-tier included: a fail-fast run stops
    *     restoring too).
-   *   - 'always': dependents run even when an upstream failed. Sound
-   *     under pure-input transitive hashing: failed outcomes carry the
-   *     upstream's INPUT key (computed before exec), so dependents'
-   *     keys fold exactly what a healthy run folds. An upstream that
-   *     died before hashing folds nothing — that key is derivable by
-   *     no healthy run, so the worst case is an unreachable cache
-   *     entry, never a stale hit.
+   *   - 'always': dependents run even when an upstream failed. Their
+   *     keys are the ones a healthy run derives (failed outcomes carry
+   *     the upstream's INPUT key, computed before exec), which is
+   *     exactly why the orchestrator withholds their SAVE: the bytes
+   *     were built on a partial tree, and a healthy key over them would
+   *     be the next clean run's stale hit (`ExecuteArgs.taintedUpstream`).
    */
   continueMode?: ContinueMode
   execute: (node: TaskNode, upstream: TaskOutcome[]) => Promise<TaskOutcome>
   onStart?: (node: TaskNode) => void
   onFinish?: (outcome: TaskOutcome) => void
+  /**
+   * What an outcome still owes before its dependents may start — a cache
+   * save that runs off the execution slot (the orchestrator's save lane):
+   * a dependent reads the upstream's entry (its output rows travel in the
+   * execute request), so it waits for the save to land, while the freed
+   * slot admits other work at once. Undefined: nothing owed.
+   */
+  settledOf?: (outcome: TaskOutcome) => Promise<void> | undefined
   /**
    * Optional priority override: callers pass their own per-node weight
    * (e.g. `computePredictedPriorities` from the orchestrator's history
@@ -233,17 +178,14 @@ export interface ScheduleOptions {
    */
   poolOf?: (id: string) => { name: string; capacity: number } | undefined
   /**
-   * Resolved per-task resource reservations (`exec.resources`, resolved
-   * by the orchestrator against the run's budgets). An absent id means
-   * zero cost; undefined/empty means no task opted in — the scheduler
-   * takes the legacy path byte-identically. Admission control only —
-   * nothing is enforced on the child process.
+   * Admission over the worker count: asked for every exec-tier task about
+   * to start on this machine, with the ids of the exec-tier tasks running
+   * here right now (added on dispatch, so two asks in one tick see each
+   * other). `false` parks the task until something finishes. Undefined →
+   * count-only, the legacy path byte for byte. Restore-tier hits and
+   * pooled tasks hold no local resources and are never asked.
    */
-  resourceCosts?: ReadonlyMap<string, ResourceCost>
-  /** CPU budget reservations pack against. Defaults to `concurrency`. */
-  cpuBudget?: number
-  /** Memory budget (bytes). Defaults to Infinity (axis off). */
-  memBudget?: number
+  admit?: (id: string, running: ReadonlySet<string>) => boolean
 }
 
 /**
@@ -466,33 +408,26 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   }
 
   let active = 0
+  // Restore-tier work is disk I/O — a confirmed hit's tar extract is ~8
+  // filesystem round trips and no CPU — so it gets its own lane, twice
+  // the exec cap: on the 1,000-project bench with every task a restore,
+  // 4 → 8 workers cut the run-graph stage 683–754 → 556–595 ms and 16 was
+  // no better (2026-09-10). Exec-tier work keeps the CPU-shaped cap; the
+  // two lanes never wait on each other. `--concurrency 1` stays serial.
+  let activeRestore = 0
+  const restoreConcurrency = concurrency === 1 ? 1 : 2 * concurrency
   let resolved = false
 
-  // Resource admission (2-D bin packing over the count limit). Inactive
-  // (no task opted in) → the tick loop short-circuits before any of this
-  // and behaves byte-identically to the count-only scheduler.
-  const costs = options.resourceCosts
-  const resourcesActive = costs !== undefined && costs.size > 0
-  const cpuBudget = options.cpuBudget ?? concurrency
-  const memBudget = options.memBudget ?? Infinity
-  // Reservations are FLOAT sums (fractional cpus, and percent-of-budget
-  // resolves to non-representable values like `0.30000000000000004`), so
-  // add/release cycles leave ~1e-17 residue instead of an exact 0. Two
-  // guards keep that residue from corrupting admission:
-  //   - the solo-clamp gate ("is the axis idle?") reads INTEGER holder
-  //     counts, never the float sum === 0 — a residue would otherwise
-  //     park an over-budget task forever (active===0, no future tick =
-  //     a silent hang / exit-0-without-running);
-  //   - `reserved` snaps back to EXACT 0 whenever its holder count hits
-  //     0, so residue can't accumulate across busy periods.
-  // The within-budget comparison also carries a tiny relative epsilon so
-  // an exact-fill (`reserved + cost == budget`) can't mis-round into a
-  // spurious block. Admission is a hint, so the epsilon's sub-ulp
-  // over-admission is harmless.
-  let reservedCpu = 0
-  let reservedMem = 0
-  let holdersCpu = 0
-  let holdersMem = 0
+  // Admission policy over the count limit (a plugin's `admit`). Inactive
+  // → the tick loop short-circuits before any of this and behaves
+  // byte-identically to the count-only scheduler.
+  const admitPolicy = options.admit
+  const admitActive = admitPolicy !== undefined
+  const running = new Set<string>()
+  // When a policy first refused each parked task — so its outcome can say
+  // how long the policy, not the count gate, held it. Nothing with no
+  // policy: the map is never touched.
+  const heldSince = new Map<string, number>()
 
   // A restore-tier task is a confirmed local cache hit: its "execution"
   // is a cheap tar extract, not the task's real work — it reserves ZERO
@@ -504,12 +439,22 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   // against `concurrency`; pooled tasks reserve no local resources.
   const hasRoom = (id: string): boolean => {
     const pool = poolOf?.(id)
-    if (pool === undefined) return active < concurrency
+    if (pool === undefined) {
+      return options.restoreTier?.has(id)
+        ? activeRestore < restoreConcurrency
+        : active < concurrency
+    }
     return (poolActive.get(pool.name) ?? 0) < pool.capacity
   }
   const admit = (id: string): (() => void) => {
     const pool = poolOf?.(id)
     if (pool === undefined) {
+      if (options.restoreTier?.has(id)) {
+        activeRestore++
+        return () => {
+          activeRestore--
+        }
+      }
       active++
       return () => {
         active--
@@ -520,44 +465,20 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
       poolActive.set(pool.name, (poolActive.get(pool.name) ?? 1) - 1)
     }
   }
-  const costOf = (id: string): ResourceCost =>
-    options.restoreTier?.has(id) || poolOf?.(id) !== undefined
-      ? ZERO_COST
-      : (costs?.get(id) ?? ZERO_COST)
-
-  // Zero never blocks; a within-budget cost needs headroom (with an
-  // exact-fill epsilon); an over-budget cost can never have headroom, so
-  // it solo-clamps: admitted only when the axis is idle (no holders — an
-  // idle pool always admits at least one ready task, no deadlock).
-  const fitsAxis = (cost: number, reserved: number, holders: number, budget: number): boolean =>
-    cost === 0 ? true : cost <= budget ? reserved + cost <= budget + budget * 1e-9 : holders === 0
-
-  const fits = (id: string): boolean => {
-    const c = costOf(id)
-    return (
-      fitsAxis(c.cpu, reservedCpu, holdersCpu, cpuBudget) &&
-      fitsAxis(c.mem, reservedMem, holdersMem, memBudget)
-    )
-  }
-
-  // Reserve/release keep the float sum AND the integer holder count in
-  // lockstep; releasing the last holder on an axis snaps its sum to 0.
-  const reserve = (c: ResourceCost): void => {
-    if (c.cpu > 0) {
-      reservedCpu += c.cpu
-      holdersCpu++
-    }
-    if (c.mem > 0) {
-      reservedMem += c.mem
-      holdersMem++
+  // The policy sees exec-tier local tasks only; a restore is a tar
+  // extract and a pooled task runs on someone else's capacity.
+  const local = (id: string): boolean => !options.restoreTier?.has(id) && poolOf?.(id) === undefined
+  const admits = (id: string): boolean => !local(id) || admitPolicy!(id, running)
+  // With no policy nothing reads `running`, so nothing is tracked: the
+  // count-only dispatch allocates no closure and touches no set per task.
+  const untracked = (): void => {}
+  const track = (id: string): (() => void) => {
+    if (!admitActive || !local(id)) return untracked
+    running.add(id)
+    return () => {
+      running.delete(id)
     }
   }
-  const release = (c: ResourceCost): void => {
-    if (c.cpu > 0 && --holdersCpu === 0) reservedCpu = 0
-    else reservedCpu -= c.cpu
-    if (c.mem > 0 && --holdersMem === 0) reservedMem = 0
-    else reservedMem -= c.mem
-  }
 
   return new Promise<Map<string, TaskOutcome>>((resolve) => {
     const finishOne = (id: string, outcome: TaskOutcome): void => {
@@ -598,8 +519,9 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
     // cached output is reported `cache-hit` even if a dep failed — and
     // they're dep-independent, so they typically restore before a dep
     // could fail anyway.
+    const aborted = (): boolean => options.signal?.aborted === true
     const willSkip = (id: string): boolean => {
-      if (failFastTripped) return true
+      if (failFastTripped || aborted()) return true
       if (restoreTier?.has(id)) return false
       if (continueMode === 'always') return false
       const node = nodes.get(id) as TaskNode
@@ -617,31 +539,43 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
 
     const tick = (): void => {
       if (resolved) return
-      // Exec-tier tasks parked THIS tick on a failed resource fit.
-      // Within one synchronous tick `reserved` only increases (release
-      // happens in the async completion callbacks, which run a fresh
-      // tick), so a task that doesn't fit now cannot fit later in the
-      // same tick — parking it for the tick's remainder is exact, and
-      // each id pops at most once per tick (O(R log R)).
+      // Exec-tier tasks parked THIS tick on a refused admission. Within
+      // one synchronous tick `running` only grows (tasks leave it in the
+      // async completion callbacks, which run a fresh tick), so a task
+      // refused now cannot be admitted later in the same tick — parking
+      // it for the tick's remainder is exact, and each id pops at most
+      // once per tick (O(R log R)).
       const parked: Array<[string, number]> = []
 
       // Highest-priority admissible task: exec tier first (misses own
-      // the pool), then restore tier. With no reservations declared this
+      // the pool), then restore tier. With no admission policy this
       // short-circuits to exactly the legacy takeReady. A would-skip
-      // task returns without a fit check (finishing it is free); restore
-      // tasks cost zero by construction, so they never park.
+      // task returns without asking (finishing it is free); restore
+      // tasks are never asked, so they never park.
       const takeFitting = (): string | undefined => {
-        // No pools declared (the common case): a full local pool admits
-        // nothing, so do not scan — this is the legacy O(1) gate.
-        if (poolOf === undefined && active >= concurrency) return undefined
-        while (execReady.size > 0) {
-          const seq = execReady.peekSeq()
-          const id = execReady.pop() as string
-          if (willSkip(id) || ((!resourcesActive || fits(id)) && hasRoom(id))) return id
-          parked.push([id, seq])
+        // No pools, no policy and a full exec lane: nothing on the
+        // exec queue can be admitted, so it is not scanned — scanning it
+        // would pop and re-park every ready exec task on every tick, and
+        // with a wide frontier that is O(R) per tick, O(R²) per run (the
+        // 6,000-task scale pin went 0.5 s → 28 s when the restore lane's
+        // first cut let the scan run past a full exec lane, 2026-09-10).
+        // This is the legacy O(1) gate, kept per lane.
+        const execAdmissible = poolOf !== undefined || admitActive || active < concurrency
+        if (execAdmissible) {
+          while (execReady.size > 0) {
+            const seq = execReady.peekSeq()
+            const id = execReady.pop() as string
+            if (willSkip(id)) return id
+            if (hasRoom(id)) {
+              if (!admitActive || admits(id)) return id
+              // A free worker, refused by the policy: the hold starts now.
+              if (!heldSince.has(id)) heldSince.set(id, Date.now())
+            }
+            parked.push([id, seq])
+          }
         }
-        // Restore-tier tasks are local work (a cache restore on this disk).
-        return active < concurrency ? restoreReady.pop() : undefined
+        // Restore-tier tasks are local disk work, on their own lane.
+        return activeRestore < restoreConcurrency ? restoreReady.pop() : undefined
       }
 
       for (;;) {
@@ -660,17 +594,26 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
         // because dependents are pushed when `pending` hits 0 regardless
         // of outcome — keeps the propagation logic in one place.
         if (willSkip(id)) {
-          finishOne(id, { node, status: 'skipped', exitCode: 1, durationMs: 0 })
+          finishOne(id, {
+            node,
+            status: aborted() ? 'aborted' : 'skipped',
+            exitCode: 1,
+            durationMs: 0,
+          })
           continue
         }
 
         const leave = admit(id)
-        // Reserve on dispatch; capture the cost so the release in the
-        // completion callbacks is symmetric even if the maps change.
-        const cost = costOf(id)
-        reserve(cost)
+        // Listed as running on dispatch, so the policy's next ask in this
+        // tick sees it; the completion callbacks unlist it.
+        const untrack = track(id)
+        const since = admitActive ? heldSince.get(id) : undefined
+        const heldMs = since === undefined ? 0 : Math.max(1, Date.now() - since)
+        if (since !== undefined) heldSince.delete(id)
+        const withHold = (o: TaskOutcome): TaskOutcome =>
+          heldMs > 0 ? { ...o, admissionHeldMs: heldMs } : o
         // Crash-isolated observer hook — a throwing onStart must not abort
-        // the dispatch loop (it would strand the tick with reservations held).
+        // the dispatch loop (it would strand the tick with the slot held).
         try {
           onStart?.(node)
         } catch (err) {
@@ -681,26 +624,38 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
         // `.then(onFulfilled, onRejected)` — NOT `.then(f).catch(g)`. The
         // rejection arm handles ONLY `execute()` rejecting; a throw from
         // the fulfillment arm (finishOne / onFinish / tick) must NOT also
-        // run the rejection arm, or `active`/`reserved` release twice —
-        // and a double release drives `reserved` negative, permanently
-        // wedging the solo-clamp gate.
+        // run the rejection arm, or `active` releases twice — and a
+        // double release wedges the count gate.
         execute(node, upstream).then(
-          (outcome) => {
+          (raw) => {
+            const outcome = withHold(raw)
             leave()
-            release(cost)
-            finishOne(id, outcome)
+            untrack()
+            // The slot is free now; the dependents wait for what the
+            // outcome still owes (a save landing), if anything.
+            const settled = options.settledOf?.(outcome)
+            if (settled === undefined) {
+              finishOne(id, outcome)
+              tick()
+              return
+            }
             tick()
+            const done = (): void => {
+              finishOne(id, outcome)
+              tick()
+            }
+            void settled.then(done, done)
           },
           (err: unknown) => {
             const message = err instanceof Error ? err.message : String(err)
-            const outcome: TaskOutcome = {
+            const outcome: TaskOutcome = withHold({
               node,
               status: 'failed',
               exitCode: 1,
               durationMs: 0,
-            }
+            })
             leave()
-            release(cost)
+            untrack()
             finishOne(id, outcome)
             // Surface the error live; the outcome itself doesn't
             // carry captured stderr (that's the logger's job). A
@@ -722,7 +677,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
       // is exactly preserved for the next tick's admission pass.
       for (const [id, seq] of parked) execReady.push(id, seq)
 
-      if (outcomes.size === nodes.size && active === 0) {
+      if (outcomes.size === nodes.size && active === 0 && activeRestore === 0) {
         resolved = true
         resolve(outcomes)
       }

package/src/graph/task-graph.ts

@@ -289,7 +289,7 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
           `Task ${id}: dependsOn patterns are not supported in the "pkg#task" form (got "${raw}")`,
         )
       }
-      // CLI `--excludeDependencies=name1,name2` drops edges whose target
+      // CLI `--exclude-dependencies=name1,name2` drops edges whose target
       // task name matches, regardless of bucket (self / deps / cross).
       // Pattern specs re-apply the filter per EXPANDED name below.
       if (skipNames?.has(spec.task)) continue
@@ -331,7 +331,7 @@ export function buildTaskGraph(options: BuildGraphOptions): Map<string, TaskNode
         // With a pattern (`^build.*`), a holder is a package declaring
         // AT LEAST ONE matching task and it receives edges to ALL its
         // matches — holder-ness is about declaration, so a holder still
-        // stops the walk even when every match is --excludeDependencies'd.
+        // stops the walk even when every match is --exclude-dependencies'd.
         //
         // The declaring project seeds `visited`: package graphs may legally
         // contain cycles (the common "b devDepends on a for its tests"