@vzn/vx 0.0.644 → 0.0.646

package/src/exec/executor.ts

@@ -3,9 +3,10 @@
 // request here; an executor decides WHERE/HOW the process runs. Core's own
 // `local-executor.ts` is the in-process spawn, and it sits at the TAIL of
 // every executor list — what a plugin executor declines runs here.
-// Persistent tasks (`exec.persistent`) never reach an executor: they are local by
-// construction (a worker cannot hand the submitter a listening port) and
-// stay on `runPersistent`. An `exec.interactive` task reaches the local one
+// Persistent tasks (`exec.persistent`) never reach an executor as tasks: a
+// worker cannot hand the submitter a listening port. A `servers` executor
+// is handed a task's persistent dependencies instead (`ExecuteRequest.servers`)
+// and starts them where it runs the task; the rest stay on `runPersistent`. An `exec.interactive` task reaches the local one
 // only: no plugin can hand it this terminal.
 //
 // Lives in `exec/` (not `orchestrator/`) so the contract depends only on
@@ -14,7 +15,7 @@
 
 import type { CaptureConfig, RunResult } from './runner.js'
 import type { ResolvedSandboxConfig, SandboxViolation } from './sandbox-runtime.js'
-import { UserError } from '../util/index.js'
+import { UserError, type ui } from '../util/index.js'
 
 /** Sandbox baselines + the user's resolved sandbox block, when the task is sandboxed. */
 export interface ExecuteSandbox {
@@ -34,6 +35,12 @@ export interface ExecuteSandbox {
    */
   readonly reportLinked: readonly string[]
   readonly config: ResolvedSandboxConfig
+  /**
+   * A cached task's: the project directories its key answers for, which
+   * an executor that builds the request again where it runs, on that
+   * machine's paths and platform, hands `sandboxRequestFor`.
+   */
+  readonly keyed?: readonly string[]
 }
 
 /** One declared input file, as the cache key saw it. */
@@ -77,6 +84,18 @@ export interface TaskInputs {
   readonly configDigest: string
 }
 
+/** A persistent dependency a `servers` executor starts where it runs the task, as `runPersistent` does. */
+export interface ServerSpec {
+  readonly taskId: string
+  readonly command: string
+  readonly cwd: string
+  readonly env: NodeJS.ProcessEnv
+  /** `exec.persistent.readyWhen`: ready at the first output line it matches; absent, at spawn. */
+  readonly readyWhen?: string
+  /** Bounds the readiness wait. */
+  readonly timeoutMs?: number
+}
+
 export interface ExecuteRequest {
   /** `${project}#${task}` — for executors that route or log by task. */
   readonly taskId: string
@@ -162,6 +181,12 @@ export interface ExecuteRequest {
    * core's local executor is ever handed such a task (placement.ts).
    */
   readonly terminal?: true
+  /**
+   * The persistent tasks it depends on, through groups, for a `servers`
+   * executor: each started where the command runs, ready before it, and
+   * shared by the tasks that run there. Absent for any other executor.
+   */
+  readonly servers?: readonly ServerSpec[]
 }
 
 export interface ExecuteResult extends RunResult {
@@ -180,9 +205,16 @@ export interface ExecuteResult extends RunResult {
    * Executor-reported placement label — which machine ran the command
    * (a REAPI worker id, a pool member). Absent = this host. Rides
    * `TaskOutcome.where` into telemetry so a task can be attributed to
-   * its worker; never persisted to the analytics store.
+   * its worker, onto its row and frame (`<id>@<where>`), and into the
+   * run history (v44).
    */
   readonly where?: string
+  /**
+   * The cores and memory of the machine `where` names, as it reports
+   * them: the run history keeps them, so a task that used all of one
+   * reads as wanting a bigger one (`TaskResources.saturated*`).
+   */
+  readonly machine?: { readonly cores: number; readonly memoryBytes: number }
 }
 
 /** What an executor sees when a task is PLACED — once per task, before scheduling. */
@@ -193,9 +225,10 @@ export interface TaskPlacement {
   readonly command: string
   /**
    * Must run on this machine: the task is persistent, depends (transitively)
-   * on a persistent or sandboxed task, is sandboxed itself, declares
+   * on a persistent task this executor does not carry or a sandboxed one, is sandboxed itself, declares
    * `exec.remote: false`, or folds a runtime probe into its key (this
-   * machine's `node -v`). A `remote` executor is never offered such a task.
+   * machine's `node -v`) and this executor is not `runtime`. A `remote`
+   * executor is never offered such a task.
    */
   readonly pinnedLocal: boolean
   /** Declares `cache` — the only tasks whose input set is described, and so the only ones that can ship. */
@@ -207,6 +240,29 @@ export interface TaskExecutor {
   readonly name: string
   /** Runs the command somewhere else. Never offered a `pinnedLocal` task. */
   readonly remote?: boolean
+  /**
+   * A `remote` executor that enforces `ExecuteRequest.sandbox` where it
+   * runs the command, as the local one does. It is offered sandboxed tasks
+   * and their dependants; the request's violations come back on the result.
+   */
+  readonly sandbox?: boolean
+  /**
+   * A `remote` executor that starts a task's persistent dependencies where
+   * it runs the task (`ExecuteRequest.servers`). It is offered the
+   * dependants of a persistent task with no `exec.sandbox` and no
+   * `exec.remote: false`; that task then runs here only for a dependant
+   * placed here (owner, 2026-10-22: nothing runs on a distributed run's
+   * orchestrator).
+   */
+  readonly servers?: boolean
+  /**
+   * A `remote` executor that answers a task's `inputs.runtime` and
+   * `workspaceRuntime` probes where it runs the task (`answerRuntime`) and
+   * fails it when one answers other than `TaskInputs` says this machine
+   * did. It is offered tasks whose key folds a probe, which otherwise run
+   * here: their key holds this machine's answer (C-2).
+   */
+  readonly runtime?: boolean
   /**
    * How many tasks this executor runs at once. Its tasks then occupy a pool
    * of this size instead of the local worker slots, so a remote pool is not
@@ -230,9 +286,51 @@ export interface TaskExecutor {
    * for it.
    */
   demand?(remaining: ReadonlySet<string>): void
+  /**
+   * Rows the run's footer shows for this executor, `[label, text]`, the
+   * text plain or a `ui` inline (a tone colors it): what only it knows, such as the machines it ran tasks on. Asked on every
+   * frame of the live footer and once for the final one, so it is cheap;
+   * a throw shows nothing.
+   */
+  footer?(): readonly (readonly [label: string, text: ui.Inline])[]
+  /**
+   * The machines it ran the run's tasks on (`small#1`, `local`), for the
+   * run summary telemetry sinks read; asked once at the end, only when a
+   * sink is on. A throw reports none.
+   */
+  machines?(): readonly MachineReport[]
+  /**
+   * Where a task it was handed runs (`small#2`), for the live row; asked
+   * per frame. `null` while it holds the task for a machine to begin it:
+   * the row says `waiting`, and its clock starts when a machine begins it.
+   */
+  running?(taskId: string): string | null | undefined
   execute(req: ExecuteRequest): Promise<ExecuteResult>
 }
 
+/** One machine an executor ran tasks on, as the run summary reports it. */
+export interface MachineReport {
+  /** As task rows name it (`small#1`). */
+  name: string
+  /** `online`, or `gone` for one that left before the run ended. */
+  status: string
+  cores: number
+  memoryBytes: number
+  done: number
+  failed: number
+  onlineMs: number
+  /** Time with at least one task running. */
+  busyMs: number
+  /** Cores in use at once, at the most and on average over its time online. */
+  cpuPeak: number
+  cpuAvg: number
+  /** Bytes in use at once, at the most and on average. */
+  memoryPeakBytes: number
+  memoryAvgBytes: number
+  /** Core-time its tasks used over the core-time it had, 0 to 1. */
+  utilization: number
+}
+
 /**
  * What a remote executor rejects with when it gives a task back: core runs
  * the same request on the local floor and says `reason` once (a remote that
@@ -307,13 +405,18 @@ export function selectExecutor(
   executors: readonly TaskExecutor[],
   task: TaskPlacement,
   label: (executor: TaskExecutor) => string = (e) => `executor '${e.name}'`,
+  /** `task.pinnedLocal` as `executor` sees it: one that enforces the sandbox or carries servers lifts some pins. */
+  pinnedFor: (executor: TaskExecutor) => boolean = () => task.pinnedLocal,
 ): TaskExecutor {
   for (const executor of executors) {
-    if (task.pinnedLocal && executor.remote === true) continue
+    const pinned = pinnedFor(executor)
+    if (pinned && executor.remote === true) continue
     if (executor.accepts === undefined) return executor
     let taken: unknown
     try {
-      taken = executor.accepts(task)
+      taken = executor.accepts(
+        pinned === task.pinnedLocal ? task : { ...task, pinnedLocal: pinned },
+      )
       // An `async accepts` answers a Promise: truthy, so the executor took
       // every task, and a rejection went unheard (the shape of H-16's admit).
       if (taken instanceof Promise) {

package/src/exec/index.ts

@@ -59,6 +59,8 @@ export {
   type ExecuteResult,
   type ExecuteSandbox,
   type InputFile,
+  type MachineReport,
+  type ServerSpec,
   type TaskExecutor,
   type TaskInputs,
   type TaskPlacement,

package/src/exec/proc-sample.ts

@@ -19,7 +19,10 @@ export interface TreeUsage {
 
 interface Proc {
   ppid: number
+  /** Its own CPU and, on Linux, what its reaped children used (`cutime`, `cstime`). */
   cpuMs: number
+  /** The executable's name; Linux only. */
+  comm?: string
   /** Resident KiB when the table carries it; Linux reads it per tree member. */
   rssKiB?: number
 }
@@ -32,7 +35,11 @@ interface Proc {
 const TICK_MS = 10
 
 /** Each root's tree usage; a root no longer alive has no entry. */
-export async function sampleTrees(roots: readonly number[]): Promise<Map<number, TreeUsage>> {
+export async function sampleTrees(
+  roots: readonly number[],
+  /** Members not counted (their children still are): the sandbox's own tracer. */
+  skip?: ReadonlySet<string>,
+): Promise<Map<number, TreeUsage>> {
   const out = new Map<number, TreeUsage>()
   if (roots.length === 0) return out
   const table = process.platform === 'linux' ? await linuxTable() : await psTable()
@@ -49,7 +56,8 @@ export async function sampleTrees(roots: readonly number[]): Promise<Map<number,
     const stack = [root]
     while (stack.length > 0) {
       const pid = stack.pop()!
-      tree.push(pid)
+      const comm = table.get(pid)!.comm
+      if (skip === undefined || comm === undefined || !skip.has(comm)) tree.push(pid)
       for (const c of children.get(pid) ?? []) stack.push(c)
     }
     let cpuMs = 0
@@ -86,12 +94,16 @@ async function linuxTable(): Promise<Map<number, Proc> | undefined> {
       }
       // `<pid> (<comm>) <state> <ppid> …` — comm may hold spaces and parens,
       // so the fields start after the LAST ')': state is [0], ppid [1],
-      // utime [11], stime [12].
-      const f = stat.slice(stat.lastIndexOf(')') + 2).split(' ')
+      // utime [11], stime [12], cutime [13], cstime [14]. The reaped
+      // children's share keeps a tree's sum from dropping when a member
+      // exits: what a task's short-lived processes used stays counted.
+      const close = stat.lastIndexOf(')')
+      const f = stat.slice(close + 2).split(' ')
       if (f[0] === 'Z' || f[0] === 'X') return
       table.set(Number(entry), {
         ppid: Number(f[1]),
-        cpuMs: (Number(f[11]) + Number(f[12])) * TICK_MS,
+        cpuMs: (Number(f[11]) + Number(f[12]) + Number(f[13]) + Number(f[14])) * TICK_MS,
+        comm: stat.slice(stat.indexOf('(') + 1, close),
       })
     }),
   )
@@ -153,3 +165,53 @@ export function psTimeMs(s: string): number | undefined {
   for (const n of parts) seconds = seconds * 60 + n
   return Math.round((days * 86_400 + seconds) * 1000)
 }
+
+/** How often a watched tree is looked at; one timer serves every tree. */
+const WATCH_EVERY_MS = 250
+
+const watched = new Map<number, TreeUsage & { looks: number }>()
+let watchTimer: ReturnType<typeof setInterval> | undefined
+let looking = false
+const SKIP = new Set(['strace'])
+
+async function look(): Promise<void> {
+  if (looking) return
+  looking = true
+  try {
+    const usage = await sampleTrees([...watched.keys()], SKIP)
+    for (const [pid, u] of usage) {
+      const w = watched.get(pid)
+      if (w === undefined) continue
+      w.cpuMs = Math.max(w.cpuMs, u.cpuMs)
+      w.rssBytes = Math.max(w.rssBytes, u.rssBytes)
+      w.looks++
+    }
+  } catch {
+    // A failed look is a missing point.
+  } finally {
+    looking = false
+  }
+}
+
+/**
+ * Watch a running tree until `stop`, which answers the most CPU and the
+ * peak resident memory any look saw; none when no look landed (a task
+ * shorter than one interval). The sandbox's answer on Linux, where the
+ * wait never sees what bwrap's pid namespace used.
+ */
+export function watchTree(pid: number): { stop(): TreeUsage | undefined } {
+  const w = { cpuMs: 0, rssBytes: 0, looks: 0 }
+  watched.set(pid, w)
+  watchTimer ??= setInterval(() => void look(), WATCH_EVERY_MS)
+  watchTimer.unref?.()
+  return {
+    stop() {
+      if (watched.get(pid) === w) watched.delete(pid)
+      if (watched.size === 0 && watchTimer !== undefined) {
+        clearInterval(watchTimer)
+        watchTimer = undefined
+      }
+      return w.looks === 0 ? undefined : { cpuMs: w.cpuMs, rssBytes: w.rssBytes }
+    },
+  }
+}

package/src/exec/runner.ts

@@ -932,6 +932,10 @@ export function runPersistent(opts: PersistentOptions): PersistentSpawn {
         new Promise((resolve) => setTimeout(resolve, POST_EXIT_DRAIN_MS).unref?.()),
       ])
     }
+    // A clean exit is ready: a setup step (migrate, seed, a `docker
+    // compose up --wait`) has done its work, and what it started, if
+    // anything, is torn down with the run (M 2026-10-22).
+    if (code === 0) markReady()
     if (readyAt === undefined && gaveUpAt === undefined) {
       rejectReady(
         new PersistentReadyError(

package/src/exec/sandbox-runtime.ts

@@ -59,6 +59,7 @@ import {
   type CaptureConfig,
   type RunResult,
 } from './runner.js'
+import { watchTree } from './proc-sample.js'
 import {
   BUN_GLOB_WILDCARDS,
   executablePath,
@@ -2074,6 +2075,7 @@ async function runSandboxedOnce(
 
   args.liveChildren?.add(proc)
   args.onSpawn?.(proc.pid)
+  const watch = process.platform === 'linux' ? watchTree(proc.pid) : undefined
   const timeout = armTimeout(proc, args.timeoutMs)
   const armedAt = Date.now()
   const ac = new AbortController()
@@ -2102,6 +2104,7 @@ async function runSandboxedOnce(
   // aborts at once, otherwise drainOrAbort bounds the post-exit drain.
   await proc.exited
   const ranMs = Date.now() - armedAt
+  const looked = watch?.stop()
   await timeout.settle()
   let cut = false
   if (timeout.timedOut()) ac.abort()
@@ -2276,10 +2279,14 @@ async function runSandboxedOnce(
     // processes used is not folded into bwrap's usage: a 500 ms busy loop
     // read 2 ms under it and 510 without the flag, and the peak read was
     // vx's own high-water mark, inherited at exec (a sandboxed `true` from a
-    // 300 MB vx: 353 MB; B-3). Nothing is reported rather than a number
-    // that is someone else's. macOS's `sandbox-exec` execs the command, so
-    // its usage is the task's.
-    ...(process.platform === 'linux' ? {} : resourceUsageToCpuRss(proc.resourceUsage())),
+    // 300 MB vx: 353 MB; B-3). So the tree is looked at while it runs
+    // (`watchTree`); a task over before the first look reports nothing.
+    // macOS's `sandbox-exec` execs the command, so its usage is the task's.
+    ...(process.platform !== 'linux'
+      ? resourceUsageToCpuRss(proc.resourceUsage())
+      : looked === undefined
+        ? {}
+        : { cpuMs: looked.cpuMs, peakRssBytes: looked.rssBytes }),
     ranMs,
     tracerFailed:
       straceLog !== undefined &&

package/src/graph/scheduler.ts

@@ -98,6 +98,12 @@ export interface TaskOutcome {
   /** v11 analytics: CPU time + peak RSS for this task's child process. */
   cpuMs?: number
   peakRssBytes?: number
+  /**
+   * The most cores one sampling interval burned and the mean resident
+   * memory over the looks; only when a sink samples and a look landed.
+   */
+  peakCores?: number
+  avgRssBytes?: number
   /**
    * A GROUP task's own dependency outcomes — the tasks it stands for.
    * A group has no `exec` and therefore no outputs and no cache entry: its
@@ -116,11 +122,11 @@ export interface TaskOutcome {
    */
   unkeyed?: true
   /**
-   * This task's key folds a task with no `inputs` (or it is one): no run
-   * can derive that key again, so it neither read nor saved an entry, and a
-   * dependant that folds `hash` does neither.
+   * The task with no `inputs` this task's key folds (this task, or one
+   * upstream): no run can derive that key again, so it neither read nor
+   * saved an entry, and a dependant that folds `hash` does neither.
    */
-  keyless?: true
+  keylessBy?: string
   /**
    * The run's cache policy read and wrote nothing (`--no-cache`,
    * `--cache-policy=local:`): no cache answered for this task, so a run of it is
@@ -166,8 +172,11 @@ export interface TaskOutcome {
    */
   inputMoved?: string
   /** Executor-reported placement label (`ExecuteResult.where`) — set only
-   *  when the task ran somewhere other than this host. Telemetry-only. */
+   *  when the task ran somewhere other than this host: telemetry and the task's row. */
   where?: string
+  /** The cores and memory of the machine `where` names (`ExecuteResult.machine`). */
+  whereCores?: number
+  whereMemoryBytes?: number
   /** `'deferred'` when the task's outputs were left in the remote store
    *  (`--cache-download=none`) instead of landing on this machine. Absent for
    *  every ordinary outcome. Telemetry-only. */
@@ -344,6 +353,14 @@ export interface ScheduleOptions {
    * second cache.get. When undefined/empty, behavior is byte-identical.
    */
   restoreTier?: ReadonlySet<string>
+  /**
+   * Servers held back until the caller knows whether their dependants are
+   * hits a remote holds. Each resolves to the dependants it has added to
+   * `restoreTier` since the run began (the caller's set, read live); a
+   * server whose every dependant is then in the tier is idle, as above,
+   * and any other is dispatched as usual.
+   */
+  heldServers?: ReadonlyMap<string, Promise<readonly string[]>>
   /**
    * Pool assignment for tasks placed on an executor with its own capacity
    * (a remote pool). Such a task occupies one of the pool's `capacity` slots
@@ -573,6 +590,10 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
   // without an outcome: it is not part of the run, as a pruned task is not.
   const idle = idleServers(nodes, dependents, restoreTier)
   const dormant = new Set<string>()
+  // Held servers not yet answered, and those of them that became ready.
+  const unanswered = new Set(options.heldServers?.keys() ?? [])
+  for (const id of idle) unanswered.delete(id)
+  const heldReady = new Set<string>()
   let settledIdle = 0
   const execReady = new ReadyHeap(priority)
   const restoreReady = new ReadyHeap(priority)
@@ -595,6 +616,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
     else continue
     readyAt.set(node.id, startedAt)
   }
+  const promoted = new Set<string>()
 
   let active = 0
   // Restore-tier work is disk I/O — a confirmed hit's tar extract is ~8
@@ -613,7 +635,7 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
     serial ? active + activeRestore < 1 : activeRestore < restoreConcurrency
   let resolved = false
   // A refusal's text → the first task that said it. A corrupt cache index
-  // reaches every lookup as one UserError; a repeat names the first.
+  // reaches every lookup as one UserError; on bare stderr a repeat names the first.
   const refusedBy = new Map<string, string>()
 
   // Admission policy over the count limit (a plugin's `admit`). Inactive
@@ -863,6 +885,10 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
           while (execReady.size > 0) {
             const seq = execReady.peekSeq()
             const id = execReady.pop() as string
+            if (unanswered.has(id) && !willSkip(id)) {
+              heldReady.add(id)
+              continue
+            }
             if (willSkip(id) || idle.has(id)) return id
             if (exclusive?.has(id) === true) {
               if (!busy() && (!admitActive || admits(id))) return id
@@ -1045,8 +1071,11 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
               const text = isUserError(err)
                 ? message
                 : `${message} — ${isOutOfFds(err) ? OUT_OF_FDS_HINT : fsRefusalHint(err)}`
-              const first = refusedBy.get(text)
-              if (first === undefined) refusedBy.set(text, id)
+              // In a task's own output block, in full: that block is read
+              // alone, and "as <task> above" named one printed elsewhere or
+              // not at all (M, 2026-10-22). On bare stderr the first is above.
+              const first = options.onError === undefined ? refusedBy.get(text) : undefined
+              if (!refusedBy.has(text)) refusedBy.set(text, id)
               line = `[vx] ${id}: ${first === undefined ? text : `as ${first} above`}\n`
             } else {
               const named = err instanceof Error && err.name !== 'Error' ? `${err.name}: ` : ''
@@ -1089,6 +1118,28 @@ export async function runGraph(options: ScheduleOptions): Promise<Map<string, Ta
       }
     }
 
+    // A held server's answer: the dependants it found go on the restore
+    // lane, and the server is idle when they were all of them.
+    const answer = (id: string, found: readonly string[]): void => {
+      if (resolved || !unanswered.delete(id)) return
+      for (const d of found) {
+        if (promoted.has(d) || outcomes.has(d) || !inRestoreTier(d)) continue
+        promoted.add(d)
+        restoreReady.push(d)
+        readyAt.set(d, Date.now())
+      }
+      if (dependents.get(id)?.every((d) => inRestoreTier(d)) === true) idle.add(id)
+      if (heldReady.delete(id)) execReady.push(id)
+      tick()
+    }
+    for (const [id, found] of options.heldServers ?? []) {
+      if (unanswered.has(id))
+        void found.then(
+          (ids) => answer(id, ids),
+          () => answer(id, []),
+        )
+    }
+
     tick()
   })
 }