@vzn/vx 0.0.652 → 0.0.653

package/src/orchestrator/persistent.ts

@@ -41,13 +41,27 @@ export function selectKeepAlive(
 ): KeepAlive {
   const out: KeepAlive = { nodes: [], children: [] }
   if (!foreground || registry.size === 0) return out
+  const reached = keptServers(nodes, (id) => registry.has(id))
+  for (const [id, child] of registry) {
+    if (!reached.has(id)) continue
+    out.nodes.push(nodes.get(id)!)
+    out.children.push(child)
+  }
+  return out
+}
+
+/** The servers and groups `selectKeepAlive` keeps in the foreground: the requested ones and what they stand on. */
+function keptServers(
+  nodes: ReadonlyMap<string, TaskNode>,
+  isServer: (id: string) => boolean,
+): Set<string> {
   // A requested GROUP is a requested server's stand-in: `vx run app#dev`
   // over `dev: { dependsOn: ['^dev'] }` started every server and exited 0
   // at the end of the graph, stopping them all (C-52).
   const stack: string[] = []
   for (const [id, n] of nodes) {
     if (!n.requested && n.surfaced !== true) continue
-    if (registry.has(id) || isGroupTask(n)) stack.push(id)
+    if (isServer(id) || isGroupTask(n)) stack.push(id)
   }
   const reached = new Set<string>()
   while (stack.length > 0) {
@@ -56,15 +70,120 @@ export function selectKeepAlive(
     reached.add(id)
     for (const dep of nodes.get(id)?.deps ?? []) {
       const d = nodes.get(dep)
-      if (d !== undefined && (registry.has(dep) || isGroupTask(d))) stack.push(dep)
+      if (d !== undefined && (isServer(dep) || isGroupTask(d))) stack.push(dep)
     }
   }
-  for (const [id, child] of registry) {
-    if (!reached.has(id)) continue
-    out.nodes.push(nodes.get(id)!)
-    out.children.push(child)
+  return reached
+}
+
+/**
+ * The servers no task left needs, as tasks finish: a dependency-only
+ * server stops once every task that uses it, directly or through groups,
+ * has finished, not at the end of the graph. A server that uses another
+ * finishes only when it is stopped itself. `finished(id)` answers the
+ * servers to stop now; a server kept in the foreground never is.
+ */
+export function unneededServers(
+  nodes: ReadonlyMap<string, TaskNode>,
+  foreground: boolean,
+): (id: string) => string[] {
+  const isServer = (n: TaskNode): boolean => n.config.exec?.persistent !== undefined
+  const kept = foreground ? keptServers(nodes, (id) => isServer(nodes.get(id)!)) : new Set<string>()
+  const dependents = new Map<string, string[]>()
+  for (const [id, n] of nodes)
+    for (const d of n.deps) {
+      const list = dependents.get(d)
+      if (list === undefined) dependents.set(d, [id])
+      else list.push(id)
+    }
+  // Each server's users: its dependents, a group read through to its own.
+  const users = new Map<string, Set<string>>()
+  const usedBy = new Map<string, string[]>()
+  for (const [id, n] of nodes) {
+    if (!isServer(n) || kept.has(id)) continue
+    const set = new Set<string>()
+    const stack = [...(dependents.get(id) ?? [])]
+    const seen = new Set<string>()
+    while (stack.length > 0) {
+      const d = stack.pop()!
+      if (seen.has(d)) continue
+      seen.add(d)
+      if (isGroupTask(nodes.get(d)!)) stack.push(...(dependents.get(d) ?? []))
+      else set.add(d)
+    }
+    if (set.size === 0) continue
+    users.set(id, set)
+    for (const u of set) {
+      const list = usedBy.get(u)
+      if (list === undefined) usedBy.set(u, [id])
+      else list.push(id)
+    }
+  }
+  const done = (id: string, out: string[]): void => {
+    for (const s of usedBy.get(id) ?? []) {
+      const left = users.get(s)
+      if (left === undefined || !left.delete(id) || left.size > 0) continue
+      users.delete(s)
+      out.push(s)
+      done(s, out)
+    }
+  }
+  return (id) => {
+    if (usedBy.size === 0) return []
+    const n = nodes.get(id)
+    // A server finishes when it is ready, still serving its users.
+    if (n === undefined || isServer(n)) return []
+    const out: string[] = []
+    done(id, out)
+    return out
+  }
+}
+
+/**
+ * The servers that use each server, directly or through groups. A server
+ * stops only once these have: one may still talk to it while it shuts
+ * down (a flush to its database), and both stopped at once took the
+ * database away under that handler.
+ */
+export function serverUsers(
+  nodes: ReadonlyMap<string, TaskNode>,
+): ReadonlyMap<string, readonly string[]> {
+  const isServer = (n: TaskNode | undefined): boolean => n?.config.exec?.persistent !== undefined
+  const users = new Map<string, string[]>()
+  for (const [id, n] of nodes) {
+    if (!isServer(n)) continue
+    const stack = [...n.deps]
+    const seen = new Set<string>()
+    while (stack.length > 0) {
+      const d = stack.pop()!
+      if (seen.has(d)) continue
+      seen.add(d)
+      const dn = nodes.get(d)
+      if (dn !== undefined && isGroupTask(dn)) stack.push(...dn.deps)
+      else if (isServer(dn)) {
+        const list = users.get(d)
+        if (list === undefined) users.set(d, [id])
+        else list.push(id)
+      }
+    }
+  }
+  return users
+}
+
+/** Stop one server as `shutdownPersistent` stops the rest: SIGTERM, then SIGKILL past the grace. */
+export async function stopServer(
+  child: Child,
+  graceMs: number = killGraceMs(PERSISTENT_SHUTDOWN_GRACE_MS),
+): Promise<void> {
+  if (hasEnded(child)) return
+  const letGo = holdGroups([child])
+  try {
+    killTree(child, 'SIGTERM')
+    for (const c of await untilGroupsGone([child], graceMs)) killTree(c, 'SIGKILL')
+    await child.exited
+  } finally {
+    letGo()
   }
-  return out
 }
 
 /**
@@ -81,6 +200,7 @@ export async function shutdownPersistent(
   registry: ReadonlyMap<string, Child>,
   keepAlive: readonly Child[],
   graceMs: number = killGraceMs(PERSISTENT_SHUTDOWN_GRACE_MS),
+  usersOf: ReadonlyMap<string, readonly string[]> = new Map(),
 ): Promise<CrashedPersistent[]> {
   const kept = new Set(keepAlive)
   const crashed: CrashedPersistent[] = []
@@ -88,13 +208,30 @@ export async function shutdownPersistent(
     if (kept.has(child) || !hasEnded(child) || child.exitCode === 0) continue
     crashed.push({ id, code: child.exitCode ?? child.signalCode ?? 'unknown' })
   }
-  const dying = [...registry.values()].filter((c) => !kept.has(c))
+  const dying = [...registry].filter(([, c]) => !kept.has(c))
   if (dying.length === 0) return crashed
-  const letGo = holdGroups(dying)
+  const letGo = holdGroups(dying.map(([, c]) => c))
+  // Each after the servers using it (`serverUsers`); the rest at once.
+  const stops = new Map<string, Promise<void>>()
+  const stop = (id: string, child: Child): Promise<void> => {
+    const known = stops.get(id)
+    if (known !== undefined) return known
+    const users: Promise<void>[] = []
+    for (const u of usersOf.get(id) ?? []) {
+      const c = registry.get(u)
+      if (c !== undefined && !kept.has(c)) users.push(stop(u, c))
+    }
+    const own = async (): Promise<void> => {
+      killTree(child, 'SIGTERM')
+      for (const c of await untilGroupsGone([child], graceMs)) killTree(c, 'SIGKILL')
+      await child.exited
+    }
+    const p = users.length === 0 ? own() : Promise.allSettled(users).then(own)
+    stops.set(id, p)
+    return p
+  }
   try {
-    for (const child of dying) killTree(child, 'SIGTERM')
-    for (const child of await untilGroupsGone(dying, graceMs)) killTree(child, 'SIGKILL')
-    await Promise.allSettled(dying.map((c) => c.exited))
+    await Promise.allSettled(dying.map(([id, c]) => stop(id, c)))
   } finally {
     letGo()
   }
@@ -279,9 +416,9 @@ export async function disposeServers(args: {
       (c) => hasEnded(c) && c.exitCode !== 0 && !keepAlive.children.includes(c),
     )
   if (failedElsewhere && !args.hold && !args.continueAlways) keepAlive = { nodes: [], children: [] }
-  const crashed = (await shutdownPersistent(registry, keepAlive.children)).filter(
-    (c) => endedBeforeStop?.has(registry.get(c.id)!) ?? true,
-  )
+  const crashed = (
+    await shutdownPersistent(registry, keepAlive.children, undefined, serverUsers(args.nodes))
+  ).filter((c) => endedBeforeStop?.has(registry.get(c.id)!) ?? true)
   for (const c of crashed) failServer(outcomes, c.id, c.code)
   keepAlive.nodes.forEach((n, i) => {
     const child = keepAlive.children[i]!

package/src/orchestrator/placement.ts

@@ -2,8 +2,9 @@
 // executors, and the plan-mode view of it. Split from run.ts on 2026-09-10
 // (pure motion). A task is pinned to this machine when it is persistent,
 // depends on a persistent task no `servers` executor carries, is sandboxed, says `exec.remote: false`, or
-// folds a runtime probe into its key; an `exec.interactive` task goes to the
-// local floor directly, as only it can hand over this terminal;
+// folds a runtime probe into its key; a sandboxed task goes to no executor
+// that does not declare `sandbox`, here or remote; an `exec.interactive`
+// task goes to the local floor directly, as only it can hand over this terminal;
 // everything else asks the executors in declaration order, and the local
 // floor takes what nothing claimed. An `exclusive` executor lifts every
 // pin and leaves the floor nothing: a task that would land here refuses.
@@ -204,6 +205,13 @@ export interface Placements {
    * did before the field existed.
    */
   remoteOnlyNoop: Set<string>
+  /**
+   * Tasks that depend on a `remoteOnlyNoop` one, directly or through
+   * others: each saves no entry. `remote` is not in the key, so the no-op's
+   * key is the one a real run of it has, and a dependant's key folding it
+   * would file output built without its outputs where a run over them looks.
+   */
+  belowNoop: Set<string>
   /** `'only'` tasks a remote executor DID take — executed remotely, outputs stay remote. */
   remoteOnly: Set<string>
   /** Each task placed on a `servers` executor → the persistent tasks it starts there. */
@@ -223,8 +231,9 @@ export interface Placements {
 }
 
 /**
- * The tasks that write IN PLACE — everything not placed on a remote
- * executor. `resolveDownloadModes` reads it as "the outputs are already
+ * Whether a task writes IN PLACE — everything not placed on a remote
+ * executor. A predicate, not a set: a set held an entry per task of the
+ * run for its whole length. `resolveDownloadModes` reads it as "the outputs are already
  * here", so a task wrongly left out is one vx thinks it must fetch from a
  * CAS that never held it.
  *
@@ -234,11 +243,27 @@ export interface Placements {
  * remote", never "declared local"; asking the latter drops every task on
  * the floor executor, i.e. every task in a workspace with no executor
  * plugin. It lives here, shared, because `run()` and `--dry` each built
- * this set inline from the same predicate and a second copy is how the
+ * this inline from the same predicate and a second copy is how the
  * plan and the run would disagree about what comes home (441, 442, 445).
  */
-export function locallyPlaced(placements: Placements, ids: Iterable<string>): Set<string> {
-  return new Set([...ids].filter((id) => placements.executors.get(id)?.remote !== true))
+export function locallyPlaced(placements: Placements): (id: string) => boolean {
+  return (id) => placements.executors.get(id)?.remote !== true
+}
+
+/** Why `e`, which runs every task, takes none of this one's: no platform or worker type of it fits. */
+function unrouted(
+  e: TaskExecutor,
+  named: string | undefined,
+  wants: readonly string[] | undefined,
+): string {
+  const types = e.workers
+  if (types !== undefined && named !== undefined) {
+    const type = types.find(([n]) => n === named)
+    if (type === undefined)
+      return `it has no worker type '${named}' that exec.worker names (types: ${types.map(([n]) => n).join(', ')})`
+    return `its worker type '${named}' runs ${type[1].join(', ')}, and exec.platforms asks for ${wants!.join(', ')}`
+  }
+  return `it offers ${e.platforms?.join(', ') ?? 'no platform'}, and exec.platforms asks for ${wants!.join(', ')}`
 }
 
 export function placeTasks(
@@ -267,6 +292,7 @@ export function placeTasks(
   const placements: Placements = {
     executors: new Map(),
     remoteOnlyNoop: new Set(),
+    belowNoop: new Set(),
     remoteOnly: new Set(),
     servers: new Map(),
     carried: new Set(),
@@ -290,12 +316,14 @@ export function placeTasks(
   if (executors.length === 1 && executors[0] === floor) {
     for (const node of nodes.values()) {
       delete node.platform
+      delete node.worker
       if (isGroupTask(node)) continue
       here(node)
       if (node.config.exec?.persistent !== undefined) continue
       placements.executors.set(node.id, floor)
       if (node.config.exec?.remote === 'only') placements.remoteOnlyNoop.add(node.id)
     }
+    markBelowNoop(nodes, placements)
     return placements
   }
   const pinned = pinsOf({})
@@ -318,6 +346,7 @@ export function placeTasks(
         refuse(node.id, 'it starts a server only beside a task that needs it')
   for (const node of nodes.values()) {
     delete node.platform
+    delete node.worker
     if (isGroupTask(node) || node.config.exec?.persistent !== undefined) continue
     if (node.config.exec?.interactive === true && sole === undefined) {
       placements.executors.set(node.id, floor)
@@ -327,8 +356,18 @@ export function placeTasks(
       continue
     }
     const wants: readonly string[] | undefined = node.config.exec?.platforms
+    const sandboxed = node.config.exec?.sandbox !== undefined
+    const named = node.config.exec?.worker
+    const fits = (ps: readonly string[]): boolean =>
+      wants === undefined || ps.some((p) => wants.includes(p))
+    /** Its worker type on `e`, which names types: `exec.worker`'s, else the first that fits. */
+    const typeOn = (e: TaskExecutor): readonly [string, readonly string[]] | undefined =>
+      e.workers?.find(([n, ps]) => (named === undefined || n === named) && fits(ps))
+    // An executor that names no types ignores `exec.worker`, as this machine does.
     const routes = (e: TaskExecutor): boolean =>
-      wants === undefined || e.platforms?.some((p) => wants.includes(p)) === true
+      e.workers !== undefined
+        ? typeOn(e) !== undefined
+        : wants === undefined || e.platforms?.some((p) => wants.includes(p)) === true
     const executor = selectExecutor(
       executors,
       {
@@ -341,20 +380,25 @@ export function placeTasks(
       },
       executorLabel,
       (e) => pinsOf(e).has(node.id) || !routes(e),
+      (e) => sandboxed && e.sandbox !== true && e !== floor,
     )
     if (sole !== undefined && executor === floor) {
       refuse(
         node.id,
-        pinsOf(sole).has(node.id)
-          ? 'its key includes a runtime probe, which it does not check'
-          : routes(sole)
-            ? 'it declined the task'
-            : `it offers ${sole.platforms?.join(', ') ?? 'no platform'}, and exec.platforms asks for ${wants!.join(', ')}`,
+        sandboxed && sole.sandbox !== true
+          ? 'it does not enforce exec.sandbox'
+          : pinsOf(sole).has(node.id)
+            ? 'its key includes a runtime probe, which it does not check'
+            : routes(sole)
+              ? 'it declined the task'
+              : unrouted(sole, named, wants),
       )
     }
     placements.executors.set(node.id, executor)
+    const type = routes(executor) ? typeOn(executor) : undefined
+    if (type !== undefined) node.worker = type[0]
     if (wants !== undefined) {
-      const offered = executor.platforms
+      const offered = type?.[1] ?? executor.platforms
       if (offered !== undefined && routes(executor)) {
         node.platform =
           wants.includes(host) && offered.includes(host)
@@ -386,9 +430,31 @@ export function placeTasks(
     if (sole !== undefined) refuse(node.id, 'it starts a server only beside a task that needs it')
     here(node)
   }
+  markBelowNoop(nodes, placements)
   return placements
 }
 
+/** Fills `belowNoop`: every task that reaches a `remoteOnlyNoop` one. */
+function markBelowNoop(nodes: Map<string, TaskNode>, placements: Placements): void {
+  if (placements.remoteOnlyNoop.size === 0) return
+  const dependants = new Map<string, string[]>()
+  for (const node of nodes.values())
+    for (const dep of node.deps) {
+      const list = dependants.get(dep)
+      if (list === undefined) dependants.set(dep, [node.id])
+      else list.push(node.id)
+    }
+  // An explicit stack: a recursion per edge throws on a deep chain.
+  const stack = [...placements.remoteOnlyNoop]
+  while (stack.length > 0) {
+    for (const id of dependants.get(stack.pop()!) ?? []) {
+      if (placements.belowNoop.has(id)) continue
+      placements.belowNoop.add(id)
+      stack.push(id)
+    }
+  }
+}
+
 /**
  * `executorOf` for `planRun`, or nothing. A remote-only task no remote
  * executor takes is labelled `noop` whatever the executor count: the run
@@ -442,7 +508,7 @@ export async function planExecutorOf(
       : resolveDownloadModes({
           nodes: prepared.nodes,
           policy,
-          localPlaced: locallyPlaced(placements, prepared.nodes.keys()),
+          localPlaced: locallyPlaced(placements),
           remoteOnly: placements.remoteOnly,
         })
   return {

package/src/orchestrator/plan.ts

@@ -12,13 +12,20 @@
 // side-effect-free pure probes.
 
 import type { CacheLayer, CachePolicy, GitFilesCache } from '../cache/index.js'
-import { cachesNothing, FULL_CACHE_POLICY } from '../cache/index.js'
+import { cachesNothing, FULL_CACHE_POLICY, networkConcurrency } from '../cache/index.js'
 import { ranNoCache } from '../report/index.js'
-import { idleServers, isGroupTask, runGraph, type TaskNode } from '../graph/index.js'
+import {
+  idleServers,
+  isGroupTask,
+  isKeyFailed,
+  markKeyFailed,
+  runGraph,
+  type TaskNode,
+} from '../graph/index.js'
 import type { AffectedReason } from './affected-tasks.js'
 import type { HistoryProvider } from '../history/index.js'
 import { hostPlatform } from '../exec/index.js'
-import { computeGroupKey, computeTaskHash } from './task-hash.js'
+import { computeGroupKey, computeTaskHash, keyOf } from './task-hash.js'
 import { dependedOn, keyedOutcome, keylessServer, keyUpstream } from './upstream.js'
 
 export type CacheStatus =
@@ -177,10 +184,12 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
       }
       if (isGroupTask(node)) {
         cacheStatusById.set(node.id, 'group')
-        return keyedOutcome(node, await computeGroupKey(hashArgs))
+        return keyedOutcome(node, await keyOf(computeGroupKey(hashArgs)))
       }
 
-      const hash = keylessServer(node, depended) ? undefined : await computeTaskHash(hashArgs)
+      const hash = keylessServer(node, depended)
+        ? undefined
+        : await keyOf(computeTaskHash(hashArgs))
 
       // Prediction keys off READS: a no-read policy re-executes every task,
       // so nothing is probed. Whether that run is a miss or no cache at all
@@ -205,15 +214,23 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
   })
   if (probed.size > 0) {
     const infos = await args.cache.remoteHasMany?.([...new Set(probed.values())])
-    for (const [id, hash] of probed) {
-      const where = await args.cache.has(hash)
-      cacheStatusById.set(
-        id,
-        where === null ? 'miss' : where === 'remote' ? 'hit-remote' : 'hit-local',
-      )
-      const size = where === 'remote' ? infos?.get(hash)?.size : undefined
-      if (size !== undefined) remoteBytesById.set(id, size)
+    // Asked at once, as a run pulls: in series, a remote that cannot batch
+    // cost its round-trip, or its timeout, once per key.
+    const pending = [...probed]
+    let next = 0
+    const one = async (): Promise<void> => {
+      while (next < pending.length) {
+        const [id, hash] = pending[next++]!
+        const where = await args.cache.has(hash)
+        cacheStatusById.set(
+          id,
+          where === null ? 'miss' : where === 'remote' ? 'hit-remote' : 'hit-local',
+        )
+        const size = where === 'remote' ? infos?.get(hash)?.size : undefined
+        if (size !== undefined) remoteBytesById.set(id, size)
+      }
     }
+    await Promise.all(Array.from({ length: Math.min(pending.length, networkConcurrency()) }, one))
   }
 
   if (args.restorable !== undefined) {
@@ -234,6 +251,11 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
       cacheStatusById.set(id, 'not-started')
   }
 
+  // A task whose key could not be derived fails the run, so the plan is
+  // marked (`isKeyFailed`) and `--dry` exits 1; the scheduler printed the
+  // run's own line for each.
+  const failed = [...outcomes.values()].some(isKeyFailed)
+  const mark = (p: RunPlan): RunPlan => (failed ? markKeyFailed(p) : p)
   const skips = platformSkips(args.nodes, args.otherPlatform)
   const tasks: PlannedTask[] = []
   for (const id of args.nodes.keys()) {
@@ -268,13 +290,13 @@ export async function plan(args: PlanArgs): Promise<RunPlan> {
         const p50 = p50s.get(t.node.id)
         if (p50 !== undefined) t.p50Ms = p50
       }
-      return { tasks, predicted: predictPlan(tasks), ...downgradeField(args) }
+      return mark({ tasks, predicted: predictPlan(tasks), ...downgradeField(args) })
     } catch {
       // Failing open: prediction is a nicety — a broken history read must
       // never break `--dry`.
     }
   }
-  return { tasks, ...downgradeField(args) }
+  return mark({ tasks, ...downgradeField(args) })
 }
 
 /**