@vzn/vx-dte 0.0.651 → 0.0.653

package/src/orchestrator.ts

@@ -25,6 +25,7 @@ import {
   type SandboxConfig,
   type TaskArtifacts,
   type TaskNode,
+  type WorkerAdvice,
 } from '@vzn/vx'
 import {
   bytesOf,
@@ -35,14 +36,24 @@ import {
   type DepOrder,
   type RunOrder,
   type RuntimeOrder,
+  type SentOrder,
+  type WiredDep,
   type ToWorker,
   type Load,
   type ToOrchestrator,
   type WiredResult,
 } from './messages.js'
-import { simulate, type PlanTask, type PlanWorker, type Size, type WarmUp } from './planner.js'
+import {
+  advise,
+  simulate,
+  type Fleets,
+  type PlanTask,
+  type PlanWorker,
+  type Size,
+  type WarmUp,
+} from './planner.js'
 import { Usage, type UsageStats } from './usage.js'
-import { assign, ordered, type Need } from './dispatch.js'
+import { type Need, Queue } from './dispatch.js'
 import { DEFAULT_STRATEGY, type Strategy } from './strategy.js'
 import { RelayError, unsent, type Participant } from './wire.js'
 import { ORCHESTRATOR, Trace } from './trace.js'
@@ -52,7 +63,8 @@ import { names } from './outputs.js'
 
 interface Job {
   readonly req: ExecuteRequest
-  readonly need: Need
+  /** Read as it joined the queue, and again while it waits when history changes (`rerank`). */
+  need: Need
   /** The cores and memory it holds on its worker: its need, or the whole worker for one that filled its worker. */
   held?: Need
   readonly resolve: (r: ExecuteResult) => void
@@ -84,8 +96,15 @@ interface Job {
   reread?: Set<string>
   /** Its upstreams, once the graph named it: walked again at every pump, 8000 queued spent 5 ms a pump on it (2026-10-25). */
   up?: Set<string>
+  /** Its direct dependencies, as placing reads them (`Ask.up`). */
+  deps?: readonly string[] | undefined
+  /** A retry: the worker its last attempt ran on, which it goes elsewhere than while another can take it. */
+  avoid?: string
 }
 
+/** An upstream's files, as an order names them (`DepOrder`). */
+type DepFiles = Pick<DepOrder, 'project' | 'outputs'>
+
 /** The longest a task waits on the worker holding its upstream's files before they are made again (2026-10-23: 10 s on any wait). */
 const PIN_WAIT_MS = 10_000
 
@@ -123,6 +142,14 @@ interface Worker {
   load?: { cpu: number; memory: number; at: number }
   /** Tasks whose outputs it holds: it ran them, or was sent them. */
   readonly has: Set<string>
+  /**
+   * The persistent tasks it runs, each with the base it books there
+   * (`Need.servers`): booked once, by the first task placed there that
+   * needs it, and given back when the run says none needs it.
+   */
+  readonly serving: Map<string, number>
+  /** Each dependency it was sent whole, by the key it was sent under: the next order names it alone. */
+  readonly sentDeps: Map<string, string>
   /** Sends to it go out in order: a run's parts before the run. */
   chain: Promise<void>
   /** It was sent the run's sandbox configs. */
@@ -138,6 +165,8 @@ export interface Declared {
   readonly memory: number
   /** Its size's platform: a worker reporting another is refused. None: planned as this machine's. */
   readonly platform?: string
+  /** Where tasks match it when the run has several worker types: `<type>:<platform>`. */
+  readonly place?: string
 }
 
 export interface OrchestratorOptions {
@@ -147,6 +176,8 @@ export interface OrchestratorOptions {
   readonly tree: string
   readonly nodes: () => ReadonlyMap<string, TaskNode> | undefined
   readonly need: (taskId: string) => Need
+  /** Each server's base, cores, as workers fit them: updated in place. */
+  readonly bases?: Map<string, number>
   /** The run's cache, by key: what workers save to and this machine restores from. */
   readonly artifacts: TaskArtifacts
   /** What workers read the cache under (`ExecutorContext.cacheFacts`). */
@@ -177,6 +208,11 @@ export interface OrchestratorOptions {
   readonly how?: Strategy
   /** The platform a task naming none runs on (`homeOf`). */
   readonly home?: string | undefined
+  /** A joined worker's place, by its size and the platform it reported: absent, the platform. */
+  readonly joinedPlace?: (
+    size: string | undefined,
+    platform: string | undefined,
+  ) => string | undefined
   /** No task began or ended for `stallMs`: what the run waits on, one line a task. */
   readonly stalled?: (waiting: string) => void
   /** A fault the run works around (a message the relay would not take); default stderr. */
@@ -207,6 +243,17 @@ const BATCH_BYTES = 512 * 1024
 const FEW = 2
 /** The unmeasured tasks the summary names; the rest it counts. */
 const NAMED = 40
+/** The footer's label of each `WorkerAdvice`. */
+const ADVICE: Record<WorkerAdvice['of'], readonly [label: string, note: string]> = {
+  declared: ['best', 'the fewest of its sizes'],
+  more: ['more', "past its sizes' max"],
+  any: ['any', 'of any cores, the least core time'],
+}
+/** A platform as shown: a worker type's place (`ubuntu:linux-x64`) without its type, which its size names. */
+const bare = (platform: string): string => platform.slice(platform.indexOf(':') + 1)
+/** `3 × small (4c/16G linux)`. */
+const fleetText = (w: WorkerAdvice['workers'][number]): string =>
+  `${w.count} × ${w.size === undefined ? '' : `${w.size} `}(${w.cores}c/${compact(w.memoryBytes)}${w.platform === undefined ? '' : ` ${w.platform}`})`
 /** Core-time of history the run's tasks finish before their use reshapes bookings: a few tasks' worth. */
 const LEARN_MS = 60_000
 /** The most a worker's bookings may add up to, in its cores, when tasks use less than their history. */
@@ -215,12 +262,12 @@ const OVERBOOK = 1.5
 const RESTARTS = 3
 
 /** `orders` in runs of at most `max` bytes of JSON; one bigger goes alone. */
-function parts<T extends { order: unknown }>(orders: readonly T[], max: number): T[][] {
+function parts<T extends { bytes: number }>(orders: readonly T[], max: number): T[][] {
   const out: T[][] = []
   let part: T[] = []
   let bytes = 0
   for (const o of orders) {
-    const size = bytesOf(o.order)
+    const size = o.bytes
     if (part.length > 0 && bytes + size > max) {
       out.push(part)
       part = []
@@ -259,9 +306,15 @@ export class Orchestrator {
   private readonly workers = new Map<string, Worker>()
   /** The run tree's files, workspace-relative, listed when a task first asks. */
   private inTree?: Promise<Tree | undefined>
-  private readonly queue: Job[] = []
+  /** Waiting tasks, by id, in the order a pump reads them (`Queue`). */
+  private readonly waiting = new Map<string, Job>()
+  /** Handed by core and not yet waiting: held for a machine as the waiting are. */
+  private readonly handed = new Set<string>()
+  private readonly queue: Queue
   /** Where a worker saved each task it ran: its cache key or its run key. */
   private readonly savedAs = new Map<string, string>()
+  /** The worker each task last ran on, which its retry goes elsewhere than. */
+  private readonly ranOn = new Map<string, string>()
   /** Keys the run's requests named for their dependencies. */
   private readonly upstreamKeys = new Map<string, string>()
   /** How many times each task was sent: its run key is one per send. */
@@ -288,6 +341,10 @@ export class Orchestrator {
   private readonly retired = new Set<Worker>()
   /** Workers asked for that never joined, with why. */
   private readonly unstarted = new Map<string, string>()
+  /** A task waited here on a worker: one ended with none joined needed one. */
+  private waited = false
+  /** Why each worker this run refused was refused, in order. */
+  private readonly refused: string[] = []
   /** Started, then not needed before they joined (`release`), or given up as late. */
   private readonly cancelled = new Set<string>()
   /** Of `cancelled`, those given up past their join limit, not let go: taken if they join. */
@@ -354,9 +411,12 @@ export class Orchestrator {
   private relend: ReturnType<typeof setTimeout> | undefined
   /** The plugins any of the run's lends lent a token. */
   private readonly lenders = new Set<string>()
+  /** Each graph's upstreams as orders name them (`dep`). */
+  private readonly deps = new WeakMap<ReadonlyMap<string, TaskNode>, Map<string, DepFiles | null>>()
 
   constructor(private readonly o: OrchestratorOptions) {
     this.sent = o.sent ?? new Map()
+    this.queue = new Queue(o.how ?? DEFAULT_STRATEGY, o.tail, (id) => this.waiting.get(id)!.need)
     // A root that is gone fails the run's own reads first.
     this.realRoot = (() => {
       try {
@@ -371,6 +431,32 @@ export class Orchestrator {
 
   async execute(req: ExecuteRequest): Promise<ExecuteResult> {
     if (this.lost !== undefined) throw new Error(this.lost)
+    this.handed.add(req.taskId)
+    try {
+      return await this.take(req)
+    } finally {
+      this.handed.delete(req.taskId)
+    }
+  }
+
+  private async take(req: ExecuteRequest): Promise<ExecuteResult> {
+    // Walked once here, for every later read: walked again on the way in,
+    // sharing, placing and sending, a run of 8000 tasks spent 17 s of this
+    // machine's CPU on it (2026-11-05).
+    const nodes = this.o.nodes()
+    const up = upstream(nodes, req.taskId)
+    // A worker runs a task in its project's directory of the run's tree: a
+    // project whose config is untracked is not there, and every worker
+    // failed to spawn it, exit 127 (2026-11-07).
+    const dir = path.relative(this.o.root, req.cwd)
+    if (dir !== '' && !dir.startsWith('..')) {
+      this.inTree ??= listTree(this.o.root, this.o.tree)
+      const tree = await this.inTree
+      if (tree !== undefined && !holds(tree, dir))
+        throw new UserError(
+          `${req.taskId} runs in ${dir}, which is not in the commit workers check out: commit its project (its vx.config file is untracked)`,
+        )
+    }
     // A worker runs the run's tree: a file its key includes that the tree
     // lacks (untracked) is not there, and the output it saved under that
     // key was a stale hit for the next run (2026-10-24).
@@ -380,25 +466,16 @@ export class Orchestrator {
       // A dependency's output is restored there before the task runs: a
       // key that reads an upstream's bytes (`inputs.tasks` leaving it out)
       // was refused (2026-10-25).
-      const nodes = this.o.nodes()
-      const restored = [...upstream(nodes, req.taskId)].flatMap((id) => {
-        const n = nodes?.get(id)
-        const outs = n?.config.cache?.outputs
-        return n === undefined || outs === undefined
-          ? []
-          : [
-              names(path.relative(this.o.root, n.projectDir), {
-                files: outs.files,
-                workspaceFiles: outs.workspaceFiles ?? [],
-              }),
-            ]
-      })
+      let restored: ((file: string) => boolean)[] | undefined
+      const made = (file: string): boolean =>
+        (restored ??= [...up].flatMap((id) => {
+          const d = this.dep(nodes, id)
+          return d === undefined ? [] : [names(d.project, d.outputs)]
+        })).some((m) => m(file))
       const stray =
         tree === undefined
           ? undefined
-          : req.inputs.files.find(
-              (f) => !tree.files.has(f.path) && !restored.some((made) => made(f.path)),
-            )
+          : req.inputs.files.find((f) => !tree.files.has(f.path) && !made(f.path))
       if (stray !== undefined) {
         // A committed submodule's files are not the tree's: its gitlink is.
         const link = tree?.links.find((l) => stray.path.startsWith(`${l}/`))
@@ -408,22 +485,42 @@ export class Orchestrator {
             : `${req.taskId} keys ${stray.path}, in submodule ${link}, whose files workers do not check out: leave it out of the task's inputs`,
         )
       }
+      // A worker runs the commit's bytes and modes (`unfilter`): a file on
+      // disk otherwise (CRLF a git filter wrote, an expanded ident, an LFS
+      // file's content) is not what it builds on, and the output saved
+      // under the key of this disk's bytes was a stale hit (2026-11-08).
+      const off =
+        tree === undefined
+          ? undefined
+          : req.inputs.files.find((f) => {
+              const has = tree.files.get(f.path)
+              return has !== undefined && has !== f.digest && !made(f.path)
+            })
+      if (off !== undefined)
+        throw new UserError(
+          `${req.taskId} keys ${off.path}, which is on disk other than the commit workers check out holds it (a git filter's text, eol, ident or LFS bytes, or core.autocrlf): leave it out of the task's inputs, or run the task without dte`,
+        )
     }
     for (const u of req.inputs?.upstream ?? []) this.upstreamKeys.set(u.taskId, u.hash)
-    await this.share(req.taskId)
+    await this.share(up, nodes)
     return new Promise((resolve, reject) => {
       const need = this.o.need(req.taskId)
       const job: Job = {
         req,
+        ...(nodes?.has(req.taskId) === true ? { up } : {}),
         need: { ...need, platform: req.platform ?? this.o.home },
         queuedAt: performance.now(),
         resolve,
         reject,
       }
+      // A retry runs elsewhere: the worker, or what ran beside it, may be the flake.
+      const last = (req.attempt ?? 1) > 1 ? this.ranOn.get(req.taskId) : undefined
+      if (last !== undefined) job.avoid = last
+      this.handed.delete(req.taskId)
       // A signal aborted already fires no `abort`.
       if (req.signal?.aborted === true) return resolve(stopped())
       req.signal?.addEventListener('abort', () => this.abort(job), { once: true })
-      this.queue.push(job)
+      this.enqueue([job])
       this.pumpSoon()
     })
   }
@@ -498,10 +595,21 @@ export class Orchestrator {
     return this.trace.events(this.ended ? this.endedAt : performance.now())
   }
 
+  /** No task left in the run uses server `task`: each worker that started it stops it. */
+  unneeded(task: string): void {
+    for (const a of this.workers.values()) {
+      const base = a.serving.get(task)
+      if (base === undefined) continue
+      a.serving.delete(task)
+      a.usedCores -= base
+      this.post(a, { type: 'unserve', task })
+    }
+  }
+
   private abort(job: Job): void {
-    const at = this.queue.indexOf(job)
-    if (at >= 0) {
-      this.queue.splice(at, 1)
+    if (this.waiting.get(job.req.taskId) === job) {
+      this.waiting.delete(job.req.taskId)
+      this.queue.delete(job.req.taskId)
       job.resolve(stopped())
     } else if (job.worker !== undefined) {
       const a = this.workers.get(job.worker)
@@ -519,27 +627,36 @@ export class Orchestrator {
     })
   }
 
+  /**
+   * History changed (`need`, `tail`): the waiting book and rank by it. On
+   * CI the default branch's timings land a moment after the roots joined
+   * the queue, each booked as unmeasured and ranked as an unknown task:
+   * shards ready later went first, and test.wire waited 129 s (2026-11-07).
+   */
+  rerank(): void {
+    for (const [id, job] of this.waiting)
+      job.need = { ...this.o.need(id), platform: job.need.platform }
+    this.queue.rerank()
+    this.pumpSoon()
+  }
+
+  /** `jobs` wait, in order, behind the waiting tasks or, `front`, before them. */
+  private enqueue(jobs: readonly Job[], front = false): void {
+    for (const j of jobs) this.waiting.set(j.req.taskId, j)
+    if (jobs.length > 0) this.waited = true
+    this.queue.add(
+      jobs.map((j) => j.req.taskId),
+      front,
+    )
+  }
+
   private pump(): void {
     const batches = new Map<Worker, Job[]>()
-    const left: Job[] = []
     // Core asks in its critical-path order, but only as tasks get ready: a
     // long chain ready a moment late queued behind 46 shards ready at once,
     // and its 41 s build began 47 s in, the run's end (main, 2026-10-23).
-    const tail = this.o.tail
-    const how = this.o.how ?? DEFAULT_STRATEGY
-    const waiting = this.queue.splice(0)
     const workers = [...this.workers.values()].filter((a) => !a.faulty)
-    const byId = new Map(waiting.map((j) => [j.req.taskId, j]))
-    const queue =
-      tail === undefined
-        ? waiting
-        : ordered(
-            waiting.map((j) => j.req.taskId),
-            how.order,
-            tail,
-            (id) => byId.get(id)!.need,
-            Math.max(1, ...workers.map((a) => a.cores)),
-          ).map((id) => byId.get(id)!)
+    const byId = this.waiting
     const bins = workers.map((a) => {
       let running = 0
       for (const j of a.busy.values()) if (j.held !== undefined) running++
@@ -551,45 +668,32 @@ export class Orchestrator {
         running,
         has: a.has,
         platform: a.platform,
+        serving: new Set(a.serving.keys()),
         a,
       }
     })
     const nodes = this.o.nodes()
-    const asks = []
-    const placed: Job[] = []
-    // The least any ask needs: once no worker takes it, the pass ends. Each
-    // pump read every queued task, 8000 of them a pump (2026-10-25).
-    const floor = { cores: Infinity, memory: Infinity }
-    for (const job of queue) {
-      const up =
-        job.up ??
-        (nodes?.has(job.req.taskId) === true
-          ? (job.up = upstream(nodes, job.req.taskId))
-          : upstream(nodes, job.req.taskId))
-      const pin = this.pin(job, up)
-      if (pin === 'done') continue
-      if (pin === 'wait') {
-        left.push(job)
-        continue
-      }
-      placed.push(job)
-      const need = this.booking(job.need)
-      floor.cores = Math.min(floor.cores, need.cores)
-      floor.memory = Math.min(floor.memory, need.memory)
-      asks.push({ id: job.req.taskId, need, up, ...pin })
-    }
-    const to = assign(asks, bins, (above, platform) => this.bigger(above, platform), how.place, {
-      floor,
-      kinds: Infinity,
+    const to = this.queue.pass(bins, {
+      book: (need) => this.booking(need),
+      deps: (id) => (byId.get(id)!.deps ??= nodes?.get(id)?.deps),
+      gate: (id) => {
+        const job = byId.get(id)!
+        const up =
+          job.up ?? (nodes?.has(id) === true ? (job.up = upstream(nodes, id)) : upstream(nodes, id))
+        const pin = this.pin(job, up)
+        // It ended: the queue lets it go.
+        if (pin === 'done') byId.delete(id)
+        return pin
+      },
+      waits: (above, platform) => this.bigger(above, platform),
     })
+    const placed = [...to.keys()].map((id) => byId.get(id)!)
+    for (const id of to.keys()) byId.delete(id)
     for (const job of placed) {
-      const a = to.get(job.req.taskId)?.a
-      if (a === undefined) {
-        left.push(job)
-        continue
-      }
+      const a = to.get(job.req.taskId)!.a
       job.worker = a.id
       job.sentAt = performance.now()
+      a.usage.sent(job.sentAt)
       a.busy.set(job.req.taskId, job)
       job.held = {
         cores: Math.min(this.booking(job.need).cores, a.cores),
@@ -606,11 +710,16 @@ export class Orchestrator {
       })
       a.usedCores += job.held.cores
       a.usedMemory += job.held.memory
+      // A server it starts there books its base once, while it runs.
+      for (const sv of job.need.servers ?? [])
+        if (!a.serving.has(sv.id)) {
+          a.serving.set(sv.id, sv.cores)
+          a.usedCores += sv.cores
+        }
       const batch = batches.get(a)
       if (batch === undefined) batches.set(a, [job])
       else batch.push(job)
     }
-    this.queue.push(...left)
     for (const [a, jobs] of batches) this.send(a, jobs)
   }
 
@@ -624,6 +733,23 @@ export class Orchestrator {
   private pin(
     job: Job,
     up: ReadonlySet<string>,
+  ): 'wait' | 'done' | { only?: string; avoid?: string } {
+    const where = this.pinUp(job, up)
+    const avoid = job.avoid
+    if (typeof where !== 'object' || avoid === undefined) return where
+    if (where.only !== undefined || where.avoid !== undefined) return where
+    // With no other worker that can take it, it runs where it failed rather than wait.
+    const platform = job.need.platform
+    for (const w of this.workers.values())
+      if (w.id !== avoid && !w.faulty && (platform === undefined || w.platform === platform))
+        return { avoid }
+    return where
+  }
+
+  /** Where `job` may go as its upstreams' files lie (`pin`, before a retry's own rule). */
+  private pinUp(
+    job: Job,
+    up: ReadonlySet<string>,
   ): 'wait' | 'done' | { only?: string; avoid?: string } {
     if (job.again !== undefined) {
       const { only } = job.again
@@ -677,7 +803,11 @@ export class Orchestrator {
     return this.o.pinWaitMs ?? PIN_WAIT_MS
   }
 
-  /** `ids`, uncached tasks that ran, run again where `where` says, for a dependant. */
+  /**
+   * `ids`, uncached tasks that ran, run again where `where` says, for a
+   * dependant: on its place, which the plan refused when one names another
+   * worker type or platform (`apart`).
+   */
   private remake(
     ids: readonly string[],
     where: { only?: string; avoid?: string; platform?: string | undefined },
@@ -693,24 +823,31 @@ export class Orchestrator {
         this.pumpSoon()
       }
       this.pumpSoon()
-      this.queue.unshift({
-        req: k.req,
-        need: where.platform === undefined ? k.need : { ...k.need, platform: where.platform },
-        queuedAt: performance.now(),
-        pinnedAt: performance.now(),
-        again: {
-          ...(where.only === undefined ? {} : { only: where.only }),
-          ...(where.avoid === undefined ? {} : { avoid: where.avoid }),
-        },
-        resolve: (r) =>
-          done(
-            r.exitCode === 0 ? undefined : `${id} ran again for it and failed (exit ${r.exitCode})`,
-          ),
-        reject: (err) =>
-          done(
-            `${id} ran again for it and failed: ${err instanceof Error ? err.message : String(err)}`,
-          ),
-      })
+      this.enqueue(
+        [
+          {
+            req: k.req,
+            need: where.platform === undefined ? k.need : { ...k.need, platform: where.platform },
+            queuedAt: performance.now(),
+            pinnedAt: performance.now(),
+            again: {
+              ...(where.only === undefined ? {} : { only: where.only }),
+              ...(where.avoid === undefined ? {} : { avoid: where.avoid }),
+            },
+            resolve: (r) =>
+              done(
+                r.exitCode === 0
+                  ? undefined
+                  : `${id} ran again for it and failed (exit ${r.exitCode})`,
+              ),
+            reject: (err) =>
+              done(
+                `${id} ran again for it and failed: ${err instanceof Error ? err.message : String(err)}`,
+              ),
+          },
+        ],
+        true,
+      )
       if (where.only !== undefined || where.avoid !== undefined)
         setTimeout(() => this.pumpSoon(), this.pinWaitMs + 50).unref?.()
     }
@@ -774,7 +911,15 @@ export class Orchestrator {
     if (load === undefined) return
     a.load = { cpu: Number(load.cpu) || 0, memory: Number(load.memory) || 0, at: performance.now() }
     const avg = Number(load.avg)
-    if (Number.isFinite(avg) && avg >= 0) a.usage.machineAvg = avg
+    const self = Number(load.self)
+    if (Number.isFinite(avg) && avg >= 0)
+      a.usage.machine(avg, Number.isFinite(self) && self >= 0 ? self : undefined, a.load.at)
+    // Its servers' bases as it fits them (`server-fit.ts`): what the next
+    // task needing one books, and the next run.
+    if (typeof load.servers === 'object' && load.servers !== null)
+      for (const [id, cores] of Object.entries(load.servers))
+        if (typeof cores === 'number' && Number.isFinite(cores) && cores >= 0 && cores <= a.cores)
+          this.o.bases?.set(id, cores)
     const tasks = Number(load.tasks)
     this.trace.load(a.label, a.load.cpu, a.load.at, Number.isFinite(tasks) ? tasks : undefined)
   }
@@ -790,7 +935,8 @@ export class Orchestrator {
         !joined.has(id) &&
         !this.unstarted.has(id) &&
         !this.cancelled.has(id) &&
-        (platform === undefined || (declared.platform ?? hostPlatform()) === platform) &&
+        (platform === undefined ||
+          (declared.place ?? declared.platform ?? hostPlatform()) === platform) &&
         (declared.cpu > above.cores || declared.memory > above.memory)
       )
         return true
@@ -815,11 +961,13 @@ export class Orchestrator {
    * hit only this cache held (the remote evicted it, or this machine does
    * not write it) failed the dependant there (2026-10-24).
    */
-  private async share(id: string): Promise<void> {
-    const nodes = this.o.nodes()
+  private async share(
+    up: ReadonlySet<string>,
+    nodes: ReadonlyMap<string, TaskNode> | undefined,
+  ): Promise<void> {
     const keys: string[] = []
-    for (const d of upstream(nodes, id)) {
-      if (this.savedAs.has(d) || nodes?.get(d)?.config.cache?.outputs === undefined) continue
+    for (const d of up) {
+      if (this.savedAs.has(d) || this.dep(nodes, d) === undefined) continue
       const key = this.upstreamKeys.get(d) ?? this.o.keyOf(d)
       if (key !== undefined) keys.push(key)
     }
@@ -860,92 +1008,123 @@ export class Orchestrator {
     return got.includes(true)
   }
 
+  /**
+   * Upstream `id` as an order names it, once per graph: its project and
+   * outputs. Undefined: it is not in the graph, or leaves no files.
+   */
+  private dep(nodes: ReadonlyMap<string, TaskNode> | undefined, id: string): DepFiles | undefined {
+    if (nodes === undefined) return undefined
+    let of = this.deps.get(nodes)
+    if (of === undefined) this.deps.set(nodes, (of = new Map()))
+    let d = of.get(id)
+    if (d === undefined) {
+      const n = nodes.get(id)
+      const outs = n?.config.cache === true ? n.config.outputs : undefined
+      d =
+        n === undefined || outs === undefined
+          ? null
+          : {
+              project: path.relative(this.o.root, n.projectDir),
+              outputs: { files: [...outs.files], workspaceFiles: [...(outs.workspaceFiles ?? [])] },
+            }
+      of.set(id, d)
+    }
+    return d ?? undefined
+  }
+
   private send(a: Worker, jobs: readonly Job[]): void {
     const nodes = this.o.nodes()
-    const orders = jobs.map((job) => {
+    const orders = jobs.map((job): { job: Job; order: RunOrder } => {
       const { req } = job
       const deps: DepOrder[] = []
       // Every one, held there or not: the worker restores each key once, and
       // one it was sent but gave back it never ran or restored.
-      for (const id of upstream(nodes, req.taskId)) {
-        const n = nodes?.get(id)
-        const outs = n?.config.cache?.outputs
+      for (const id of job.up ?? upstream(nodes, req.taskId)) {
+        const d = this.dep(nodes, id)
         const key = this.savedAs.get(id) ?? this.upstreamKeys.get(id) ?? this.o.keyOf(id)
-        if (n === undefined || outs === undefined || key === undefined) continue
+        if (d === undefined || key === undefined) continue
         a.has.add(id)
-        deps.push({
-          task: id,
-          key,
-          project: path.relative(this.o.root, n.projectDir),
-          outputs: { files: [...outs.files], workspaceFiles: [...(outs.workspaceFiles ?? [])] },
-        })
+        deps.push({ task: id, key, project: d.project, outputs: d.outputs })
       }
       // A task sent here leaves its outputs here for its dependants.
       a.has.add(req.taskId)
       const node = nodes?.get(req.taskId)
-      const env = defined(req.env, req.envDefine)
-      return {
-        job,
-        order: {
-          task: req.taskId,
-          // Unsaved: under the run's own key, as an uncached task's.
-          ...(req.cacheKey === undefined || req.unsaved === true ? {} : { key: req.cacheKey }),
-          runKey: runKey(this.o.run, req.taskId, this.sends(req.taskId)),
-          // A held key keeps its entry, and restored here it replaced the
-          // forced run's fresh bytes with the ones it bypassed (2026-10-25).
-          ...(req.refresh === true ? { fresh: true as const } : {}),
-          // A HEAD per upload was the workers' only remote probe: the run
-          // already knows the key is in no cache (2026-10-25).
-          ...(req.refresh !== true && this.missed.has(req.taskId) ? { absent: true as const } : {}),
-          deps,
-          nested: node === undefined ? [] : nestedIn(nodes!, node.projectDir, this.o.root),
-          command: withForwardArgs(req.command, req.forwardArgs),
-          cwd: path.relative(this.o.root, req.cwd),
-          project: node === undefined ? '' : path.relative(this.o.root, node.projectDir),
-          env,
-          ...(node?.config.exec?.env?.secret === undefined
-            ? {}
-            : { secret: [...node.config.exec.env.secret] }),
-          ...(req.timeoutMs === undefined ? {} : { timeoutMs: req.timeoutMs }),
-          outputs: {
-            files: [...req.outputs.files],
-            workspaceFiles: [...req.outputs.workspaceFiles],
-          },
-          cores: job.held?.cores ?? 1,
-          memory: job.held?.memory ?? 0,
-          ...(req.sandbox === undefined || node?.config.exec?.sandbox === undefined
-            ? {}
-            : {
-                sandbox: {
-                  config: node.config.exec.sandbox,
-                  ...(req.sandbox.keyed === undefined
-                    ? {}
-                    : { keyed: req.sandbox.keyed.map((d) => path.relative(this.o.root, d)) }),
-                },
-              }),
-          ...runtimeOrders(req),
-          ...(req.servers === undefined || req.servers.length === 0
-            ? {}
-            : {
-                servers: req.servers.map((s) => ({
-                  task: s.taskId,
-                  command: s.command,
-                  cwd: path.relative(this.o.root, s.cwd),
-                  env: defined(s.env, {}),
-                  ...(s.readyWhen === undefined ? {} : { readyWhen: s.readyWhen }),
-                  ...(s.timeoutMs === undefined ? {} : { timeoutMs: s.timeoutMs }),
-                  ...(s.sandbox === undefined
-                    ? {}
-                    : {
-                        sandbox: {
-                          config: s.sandbox.config,
-                          nested: s.sandbox.nested.map((d) => path.relative(this.o.root, d)),
-                        },
-                      }),
-                })),
-              }),
-        } as RunOrder,
-      }
+      const keyed = hostKeyed(req)
+      const env = defined(req.env, req.envDefine, keyed)
+      const unset = keyed.filter((k) => req.env[k] === undefined && !(k in req.envDefine))
+      const order = {
+        task: req.taskId,
+        // Unsaved: under the run's own key, as an uncached task's.
+        ...(req.cacheKey === undefined || req.unsaved === true ? {} : { key: req.cacheKey }),
+        runKey: runKey(this.o.run, req.taskId, this.sends(req.taskId)),
+        // A held key keeps its entry, and restored here it replaced the
+        // forced run's fresh bytes with the ones it bypassed (2026-10-25).
+        ...(req.refresh === true ? { fresh: true as const } : {}),
+        // A HEAD per upload was the workers' only remote probe: the run
+        // already knows the key is in no cache (2026-10-25).
+        ...(req.refresh !== true && this.missed.has(req.taskId) ? { absent: true as const } : {}),
+        deps,
+        nested: node === undefined ? [] : nestedIn(nodes!, node.projectDir, this.o.root),
+        command: withForwardArgs(req.command, req.forwardArgs),
+        cwd: path.relative(this.o.root, req.cwd),
+        project: node === undefined ? '' : path.relative(this.o.root, node.projectDir),
+        env,
+        ...(unset.length === 0 ? {} : { unset }),
+        ...(node?.config.exec?.env?.secret === undefined
+          ? {}
+          : { secret: [...node.config.exec.env.secret] }),
+        ...(req.timeoutMs === undefined ? {} : { timeoutMs: req.timeoutMs }),
+        outputs: {
+          files: [...req.outputs.files],
+          workspaceFiles: [...req.outputs.workspaceFiles],
+        },
+        cores: job.held?.cores ?? 1,
+        memory: job.held?.memory ?? 0,
+        ...(req.sandbox === undefined || node?.config.exec?.sandbox === undefined
+          ? {}
+          : {
+              sandbox: {
+                config: node.config.exec.sandbox,
+                ...(req.sandbox.keyed === undefined
+                  ? {}
+                  : { keyed: req.sandbox.keyed.map((d) => path.relative(this.o.root, d)) }),
+                ...(req.sandbox.dependencyReads === undefined
+                  ? {}
+                  : {
+                      reads: req.sandbox.dependencyReads.map((d) => path.relative(this.o.root, d)),
+                    }),
+                ...(req.sandbox.dependencyReported === undefined
+                  ? {}
+                  : {
+                      reported: req.sandbox.dependencyReported.map((d) =>
+                        path.relative(this.o.root, d),
+                      ),
+                    }),
+              },
+            }),
+        ...runtimeOrders(req),
+        ...(req.servers === undefined || req.servers.length === 0
+          ? {}
+          : {
+              servers: req.servers.map((s) => ({
+                task: s.taskId,
+                command: s.command,
+                cwd: path.relative(this.o.root, s.cwd),
+                env: defined(s.env, {}),
+                ...(s.readyWhen === undefined ? {} : { readyWhen: s.readyWhen }),
+                ...(s.timeoutMs === undefined ? {} : { timeoutMs: s.timeoutMs }),
+                ...(s.sandbox === undefined
+                  ? {}
+                  : {
+                      sandbox: {
+                        config: s.sandbox.config,
+                        nested: s.sandbox.nested.map((d) => path.relative(this.o.root, d)),
+                      },
+                    }),
+              })),
+            }),
+      } as RunOrder
+      return { job, order }
     })
     const sandboxed = orders.some(
       (o) => o.order.sandbox !== undefined || o.order.servers?.some((s) => s.sandbox !== undefined),
@@ -955,13 +1134,32 @@ export class Orchestrator {
     const pending = new Set(jobs)
     a.chain = a.chain
       .then(async () => {
-        for (const [i, part] of parts(orders, BATCH_BYTES).entries()) {
+        // A dependency the worker was sent under the same key goes by name
+        // alone: each order named every upstream whole, and 8000 tasks of a
+        // deep graph sent 219 MB (2026-11-05). Read here, as sends go in
+        // order: one sent before it fails is not counted as sent.
+        const fresh = new Map<string, string>()
+        const wired = orders.map(({ job, order }) => {
+          const intro: [string, string][] = []
+          const deps = order.deps.map((d): WiredDep => {
+            if ((fresh.get(d.task) ?? a.sentDeps.get(d.task)) === d.key) return d.task
+            fresh.set(d.task, d.key)
+            intro.push([d.task, d.key])
+            return d
+          })
+          const sent: SentOrder = { ...order, deps }
+          // Measured once: a giant repo's were measured three times each (2026-11-05).
+          return { job, order: sent, intro, bytes: bytesOf(sent) }
+        })
+        for (const [i, part] of parts(wired, BATCH_BYTES).entries()) {
           // An order too long for a frame (a task of thousands of
           // dependencies was refused, 413) sends its deps ahead.
-          const tasks: RunOrder[] = []
-          for (const { order } of part) {
-            if (bytesOf(order) <= BATCH_BYTES) {
+          const tasks: SentOrder[] = []
+          let bytes = 0
+          for (const { order, bytes: size } of part) {
+            if (size <= BATCH_BYTES) {
               tasks.push(order)
+              bytes += size
               continue
             }
             for (const deps of chunks(order.deps, BATCH_BYTES)) {
@@ -970,7 +1168,9 @@ export class Orchestrator {
               a.usage.messagesIn++
               a.usage.bytesIn += bytesOf(ahead)
             }
-            tasks.push({ ...order, deps: [] })
+            const alone = { ...order, deps: [] }
+            tasks.push(alone)
+            bytes += bytesOf(alone)
           }
           const n = ++a.batches
           for (const { job } of part) job.batch = n
@@ -984,7 +1184,12 @@ export class Orchestrator {
           } satisfies ToWorker
           await this.o.me.send(a.id, batch)
           a.usage.messagesIn++
-          a.usage.bytesIn += bytesOf(batch)
+          // Its orders' bytes, a comma between each, in the batch's own.
+          a.usage.bytesIn += bytesOf({ ...batch, tasks: [] }) + bytes + tasks.length - 1
+          for (const { intro } of part) for (const [t, k] of intro) a.sentDeps.set(t, k)
+          for (const { order } of part)
+            for (const sv of order.servers ?? [])
+              if (!a.serving.has(sv.task)) a.serving.set(sv.task, 0)
           for (const { job } of part) {
             pending.delete(job)
             // Sent, it runs: its worker takes only what it has room for.
@@ -1121,7 +1326,7 @@ export class Orchestrator {
         a.started.delete(task)
         const again = this.retry(a, job, `could not read ${head.unread.join(', ')}`)
         if (again !== undefined) {
-          this.queue.unshift(again)
+          this.enqueue([again], true)
           this.pumpSoon()
         }
         return
@@ -1136,7 +1341,15 @@ export class Orchestrator {
     a.done.push(task)
     a.usage.record(head, job.sentAt)
     const share = shareOf(head.cpuShare)
-    const held = Math.max(1, job.startedAt === undefined ? head.durationMs : endAt - job.startedAt)
+    // At least as long as it ran: begun here once its batch's send resolved,
+    // which can be after its worker began it, it was seen for less, and its
+    // CPU over that read as more cores than it had (#5036: 1.001 each, so
+    // two that ran side by side on 2 cores replayed one after the other).
+    const held = Math.max(
+      1,
+      head.durationMs,
+      job.startedAt === undefined ? 0 : endAt - job.startedAt,
+    )
     this.took.set(task, {
       ms: held,
       cores:
@@ -1162,6 +1375,7 @@ export class Orchestrator {
       this.needMs += Math.min(job.need.cores, a.cores) * ms
     }
     this.savedAs.set(task, key)
+    this.ranOn.set(task, a.id)
     const node = this.o.nodes()?.get(task)
     if (head.exitCode === 0 && node !== undefined && leaves(node))
       this.kept.set(task, { worker: a.id, req: job.req, need: job.need })
@@ -1309,7 +1523,11 @@ export class Orchestrator {
     if (body.type === 'joined') {
       const info = body.info as Partial<WorkerInfo> | null
       const id = body.id!
+      // Only the worker heard why: the run sat silent and its footer said
+      // none joined (2026-11-07).
       const refuse = (reason: string): void => {
+        this.refused.push(reason)
+        this.warn(`dte: refused worker ${typeof info?.id === 'string' ? info.id : id}: ${reason}`)
         void this.o.me
           .send(id, { type: 'refused', reason } satisfies ToWorker)
           .catch(this.unsent('refused', id))
@@ -1381,7 +1599,7 @@ export class Orchestrator {
         cores,
         memory,
         reported: { cores: Number(info.cores), memory: Number(info.memory) },
-        platform,
+        platform: this.o.joinedPlace?.(spawn?.declared.size, platform) ?? platform,
         usedCores: 0,
         usedMemory: 0,
         label,
@@ -1389,6 +1607,8 @@ export class Orchestrator {
         started: new Set(),
         batches: 0,
         has: new Set(),
+        serving: new Map(),
+        sentDeps: new Map(),
         // Its name first, before any batch.
         chain: this.cache()
           .then((cache) =>
@@ -1419,13 +1639,15 @@ export class Orchestrator {
     a.usage.to = performance.now()
     this.trace.left(a.label, 'lost', a.usage.to)
     this.gone.push(a)
+    // Its results' `end`s never come.
+    for (const k of this.parts.keys()) if (k.startsWith(`${a.id}\0`)) this.parts.delete(k)
     // Its tasks run again: first, as they were first to go.
     const back: Job[] = []
     for (const j of [...a.busy.values()]) {
       const again = this.retry(a, j, 'lost')
       if (again !== undefined) back.push(again)
     }
-    this.queue.unshift(...back)
+    this.enqueue(back, true)
     this.o.lost?.(a.label)
     // Workers that join on their own may yet join; ones this run starts
     // are all it gets.
@@ -1484,7 +1706,7 @@ export class Orchestrator {
                 if (again !== undefined) back.push(again)
               }
             if (back.length > 0) {
-              this.queue.unshift(...back)
+              this.enqueue(back, true)
               this.o.faulted?.()
               if (this.spawnedAt.size > 0)
                 this.stranded(`${a.label} no longer runs a task it began and no worker is left`)
@@ -1505,7 +1727,7 @@ export class Orchestrator {
             if (j === undefined) continue
             // It began it whether or not its `start` came first.
             this.begun(a, m.task)
-            if (this.retry(a, j, m.reason) !== undefined) this.queue.unshift(j)
+            if (this.retry(a, j, m.reason) !== undefined) this.enqueue([j], true)
             if (a.faulty) this.o.faulted?.()
             if (a.faulty && this.spawnedAt.size > 0)
               this.stranded(
@@ -1554,7 +1776,7 @@ export class Orchestrator {
   where(taskId: string): string | null | undefined {
     for (const a of this.workers.values())
       if (a.busy.has(taskId)) return a.started.has(taskId) ? a.label : null
-    return this.queue.some((j) => j.req.taskId === taskId) ? null : undefined
+    return this.waiting.has(taskId) || this.handed.has(taskId) ? null : undefined
   }
 
   /**
@@ -1610,7 +1832,7 @@ export class Orchestrator {
   }): void {
     const work = p.work?.map(
       (t) =>
-        `${t.id} ${t.ms === undefined ? '?' : Math.round(t.ms / 100) / 10}s ${Math.round(t.cores * 100) / 100}c${t.platform === undefined ? '' : ` ${t.platform}`}`,
+        `${t.id} ${t.ms === undefined ? '?' : Math.round(t.ms / 100) / 10}s ${Math.round(t.cores * 100) / 100}c${t.platform === undefined ? '' : ` ${bare(t.platform)}`}`,
     )
     const workers = p.workers.map((w) =>
       w.starting === true
@@ -1716,6 +1938,7 @@ export class Orchestrator {
         cores: a.cores,
         memory: a.memory,
         platform: a.platform,
+        ...(a.serving.size === 0 ? {} : { serving: new Map(a.serving) }),
         ...(a.declared === undefined ? {} : { size: a.declared.size }),
         at: 0,
         idle: a.busy.size === 0,
@@ -1727,7 +1950,8 @@ export class Orchestrator {
           id: j.req.taskId,
           cores: Math.min(a.cores, Math.max((j.held?.cores ?? 0) * over, j.need.ask?.cores ?? 0)),
           memory: Math.max(j.held?.memory ?? 0, Math.min(a.memory, j.need.ask?.memory ?? 0)),
-          endsIn: Math.max(0, (ms(j.req.taskId) ?? 0) - (now - j.startedAt!)),
+          // Below 0: past its time, which the plan weighs (`decide`).
+          endsIn: (ms(j.req.taskId) ?? 0) - (now - j.startedAt!),
         })),
       })
     }
@@ -1741,7 +1965,7 @@ export class Orchestrator {
           // A size naming none is planned as this machine's, as `dte()`
           // planned it: on no platform, no task naming one fit it, and the
           // next plan cancelled it.
-          platform: declared.platform ?? hostPlatform(),
+          platform: declared.place ?? declared.platform ?? hostPlatform(),
           size: declared.size,
           at: Math.max(
             0,
@@ -1873,9 +2097,19 @@ export class Orchestrator {
 
   /** Every task sent here and not done fails with `reason`; later ones too. */
   private fail(reason: string): void {
-    for (const a of this.workers.values())
+    const now = performance.now()
+    for (const a of this.workers.values()) {
       for (const j of a.busy.values()) j.reject(new Error(reason))
-    for (const j of this.queue.splice(0)) j.reject(new Error(reason))
+      // Lost, keeping what it ran: dropped, the footer read it as never joined.
+      a.usage.to = now
+      this.trace.left(a.label, 'lost', now)
+      this.gone.push(a)
+    }
+    for (const [id, j] of this.waiting) {
+      this.queue.delete(id)
+      j.reject(new Error(reason))
+    }
+    this.waiting.clear()
     this.workers.clear()
     this.lost = reason
   }
@@ -1899,13 +2133,8 @@ export class Orchestrator {
    * waits on its deps through those no worker ran (groups, hits).
    */
   idealMs(depsOf: (id: string) => readonly string[], sizes: readonly Size[]): number | undefined {
-    if (this.took.size === 0) return undefined
-    const all = [...this.workers.values(), ...this.gone]
-    const warm = new Map<string | undefined, number>()
-    for (const a of all)
-      if (a.warmUpMs !== undefined && !(warm.get(a.platform)! <= a.warmUpMs))
-        warm.set(a.platform, a.warmUpMs)
-    const least = this.warmUpMs() ?? 0
+    const r = this.replay(depsOf)
+    if (r === undefined) return undefined
     const pool: PlanWorker[] = (
       sizes.length > 0
         ? sizes.flatMap((s) =>
@@ -1915,14 +2144,104 @@ export class Orchestrator {
               platform: s.platform,
             })),
           )
-        : all.map((a) => ({ cores: a.cores, memory: a.memory, platform: a.platform }))
+        : [...this.workers.values(), ...this.gone].map((a) => ({
+            cores: a.cores,
+            memory: a.memory,
+            platform: a.platform,
+          }))
     ).map((w, i) => ({
       ...w,
       id: `ideal#${i}`,
-      at: warm.get(w.platform) ?? least,
+      at: r.warm(w.platform),
       running: [],
       idle: false,
     }))
+    const end = simulate(r.tasks, pool)
+    if (!Number.isFinite(end)) return undefined
+    return r.from + end
+  }
+
+  /**
+   * The workers that would have run this run best (`advise`), from the
+   * same replay as `idealMs`: on `sizes`, past their `max`, and of any
+   * cores. With no sizes (workers that join on their own), the shapes
+   * that joined stand for them, as many as joined.
+   */
+  advice(depsOf: (id: string) => readonly string[], sizes: readonly Size[]): WorkerAdvice[] {
+    // The footer and the summary each ask once the run ended: played once.
+    if (this.ended && this.advised !== undefined) return this.advised
+    const out = this.advise(depsOf, sizes)
+    if (this.ended) this.advised = out
+    return out
+  }
+
+  private advised: WorkerAdvice[] | undefined
+
+  private advise(
+    depsOf: (id: string) => readonly string[],
+    sizes: readonly Size[],
+  ): WorkerAdvice[] {
+    const r = this.replay(depsOf)
+    if (r === undefined) return []
+    const shapes: Size[] =
+      sizes.length > 0
+        ? [...sizes]
+        : [
+            ...[...this.workers.values(), ...this.gone]
+              .reduce((by, a) => {
+                const name = `${a.cores}c/${a.platform ?? ''}`
+                const had = by.get(name)
+                by.set(name, {
+                  name,
+                  max: (had?.max ?? 0) + 1,
+                  cpu: a.cores,
+                  memory: a.memory,
+                  platform: a.platform,
+                })
+                return by
+              }, new Map<string, Size>())
+              .values(),
+          ]
+    const a = advise(r.tasks, shapes, r.warm)
+    const wire = (of: WorkerAdvice['of'], f: Fleets | undefined): WorkerAdvice[] =>
+      f === undefined
+        ? []
+        : [
+            {
+              of,
+              endMs: Math.round(r.from + f.endMs),
+              workers: f.fleet.map((w) => ({
+                count: w.count,
+                cores: w.cores,
+                memoryBytes: w.memory,
+                ...(w.platform === undefined ? {} : { platform: bare(w.platform) }),
+                ...(w.size === undefined || sizes.length === 0 ? {} : { size: w.size }),
+              })),
+            },
+          ]
+    return [...wire('declared', a.declared), ...wire('more', a.more), ...wire('any', a.any)]
+  }
+
+  /**
+   * Each task a worker ran, as long as it held its worker and with the
+   * cores it used meanwhile, waiting on its deps through those no worker
+   * ran (groups, hits); each platform's fastest real warm-up; and when
+   * the run first knew its work.
+   */
+  private replay(depsOf: (id: string) => readonly string[]):
+    | {
+        tasks: PlanTask[]
+        warm: (platform: string | undefined) => number
+        from: number
+      }
+    | undefined {
+    if (this.took.size === 0) return undefined
+    const all = [...this.workers.values(), ...this.gone]
+    const warm = new Map<string | undefined, number>()
+    for (const a of all)
+      if (a.warmUpMs !== undefined && !(warm.get(a.platform)! <= a.warmUpMs))
+        warm.set(a.platform, a.warmUpMs)
+    const least = this.warmUpMs() ?? 0
     const seen = new Map<string, string[]>()
     const ran = (id: string): string[] => {
       const known = seen.get(id)
@@ -1932,10 +2251,11 @@ export class Orchestrator {
       seen.set(id, out)
       return out
     }
-    const tasks: PlanTask[] = [...this.took].map(([id, t]) => ({ id, ...t, deps: ran(id) }))
-    const end = simulate(tasks, pool)
-    if (!Number.isFinite(end)) return undefined
-    return (this.probe?.at ?? this.asks[0]?.at ?? 0) + end
+    return {
+      tasks: [...this.took].map(([id, t]) => ({ id, ...t, deps: ran(id) })),
+      warm: (platform) => warm.get(platform) ?? least,
+      from: this.probe?.at ?? this.asks[0]?.at ?? 0,
+    }
   }
 
   /**
@@ -1994,11 +2314,30 @@ export class Orchestrator {
    * each, the cores and memory its size declares and its machine reports;
    * and `ideal`, the run's `idealMs`, asked once it ended.
    */
-  footer(ideal?: () => number | undefined): [string, ui.Inline][] {
+  footer(
+    ideal?: () => number | undefined,
+    advice?: () => readonly WorkerAdvice[],
+  ): [string, ui.Inline][] {
     // Over with none, it says so: core's own `N workers` (this machine's
-    // pool) read as the run's when no task needed one (2026-10-25).
+    // pool) read as the run's when no task needed one (2026-10-25), and
+    // says when one did and none joined (2026-11-06), and why any it
+    // refused was (2026-11-07).
     if (this.spawnedAt.size === 0 && this.workers.size === 0 && this.gone.length === 0)
-      return this.ended ? [['workers', ui.dim('none started: no task needed one')]] : []
+      return this.ended
+        ? [
+            [
+              'workers',
+              this.refused.length > 0
+                ? ui.tone(
+                    'warn',
+                    `none joined: ${this.refused.length} refused (${this.refused[0]})`,
+                  )
+                : this.waited
+                  ? ui.tone('warn', 'none joined: a task waited on one')
+                  : ui.dim('none started: no task needed one'),
+            ],
+          ]
+        : []
     const gone = new Set(this.gone)
     const all = [...this.workers.values(), ...this.gone]
     const joined = new Set(all.map((a) => a.label))
@@ -2068,13 +2407,18 @@ export class Orchestrator {
     // Once over, what each did, under its line.
     const usage = new Map<string, readonly string[]>()
     let cpuMs = 0
+    let tasksMs = 0
+    let workerMs: number | undefined
     let coreMs = 0
     if (this.ended)
       for (const a of all) {
         resources(a.label, a.reported.cores, a.reported.memory, a.declared)
         const st = a.usage.stats(a.cores, this.endedAt)
-        cpuMs += st.cpuMs
-        coreMs += a.cores * st.onlineMs
+        cpuMs += st.workCoreMs
+        tasksMs += st.tasksUtilization * a.cores * st.workMs
+        if (st.workerUtilization !== undefined)
+          workerMs = (workerMs ?? 0) + st.workerUtilization * a.cores * st.workMs
+        coreMs += a.cores * st.workMs
         usage.set(a.label, describe(st, a.usage))
       }
     return [
@@ -2125,6 +2469,17 @@ export class Orchestrator {
                       ),
                     ] as [string, ui.Inline],
                   ])(ideal?.()),
+            // What to declare next time (2026-11-05: "So I can add more
+            // or change their specs").
+            ...(advice?.() ?? []).map(
+              (a) =>
+                [
+                  '',
+                  ui.dim(
+                    `${ADVICE[a.of][0].padEnd(width)}  ${a.workers.map(fleetText).join(' + ')}: ${seconds(a.endMs)}, ${ADVICE[a.of][1]}`,
+                  ),
+                ] as [string, ui.Inline],
+            ),
             ...(this.unmeasured.length === 0
               ? []
               : [
@@ -2141,7 +2496,7 @@ export class Orchestrator {
         ? [
             [
               '',
-              `${'total'.padEnd(width)}  cpu utilization ${percent(cpuMs / coreMs)}${this.needMs < LEARN_MS ? '' : ` · tasks used ${percent(this.usedMs / this.needMs)} of their history`}`,
+              `${'total'.padEnd(width)}  ${utilization(cpuMs / coreMs, tasksMs / coreMs, workerMs === undefined ? undefined : workerMs / coreMs)}${this.needMs < LEARN_MS ? '' : ` · tasks used ${percent(this.usedMs / this.needMs)} of their history`}`,
             ] as [string, ui.Inline],
           ]
         : []),
@@ -2190,15 +2545,28 @@ export function nestedIn(
 ): string[] {
   let of = nested.get(nodes)
   if (of === undefined) {
-    of = { dirs: [...new Set([...nodes.values()].map((n) => n.projectDir))], by: new Map() }
+    of = {
+      dirs: [...new Set([...nodes.values()].map((n) => n.projectDir))].sort(),
+      by: new Map(),
+    }
     nested.set(nodes, of)
   }
   let out = of.by.get(dir)
   if (out === undefined) {
-    out = of.dirs
-      .filter((d) => d.startsWith(dir + path.sep))
-      .map((d) => path.relative(root, d))
-      .sort()
+    // Sorted, the ones under `dir` lie together: each project read every
+    // project, 16 million reads for 4000 (2026-11-05).
+    const under = dir + path.sep
+    let lo = 0
+    let hi = of.dirs.length
+    while (lo < hi) {
+      const mid = (lo + hi) >> 1
+      if (of.dirs[mid]! < under) lo = mid + 1
+      else hi = mid
+    }
+    const found: string[] = []
+    for (let i = lo; i < of.dirs.length && of.dirs[i]!.startsWith(under); i++)
+      found.push(path.relative(root, of.dirs[i]!))
+    out = found.sort()
     of.by.set(dir, out)
   }
   return [...out]
@@ -2208,27 +2576,49 @@ const nested = new WeakMap<
   { dirs: string[]; by: Map<string, readonly string[]> }
 >()
 
-/** A tree's file paths, and its submodules' (gitlinks). */
-type Tree = { files: Set<string>; links: string[] }
+/** A tree's files, each path with its identity as core keys it, and its submodules' (gitlinks); `dirs` once asked. */
+type Tree = { files: Map<string, string>; links: string[]; dirs?: Set<string> }
+
+/** Whether `tree` has a file under `dir`, or `dir` in a submodule (checked out or not, its own say). */
+function holds(tree: Tree, dir: string): boolean {
+  if (tree.dirs === undefined) {
+    const dirs = (tree.dirs = new Set<string>())
+    const above = (f: string): void => {
+      for (let d = path.posix.dirname(f); d !== '.' && !dirs.has(d); d = path.posix.dirname(d))
+        dirs.add(d)
+    }
+    for (const f of tree.files.keys()) above(f)
+    for (const l of tree.links) above(l)
+  }
+  return tree.dirs.has(dir) || tree.links.some((l) => dir === l || dir.startsWith(`${l}/`))
+}
 
 /** `tree`'s files and submodules under `root`, relative to it; undefined when git cannot list it. */
 async function listTree(root: string, tree: string): Promise<Tree | undefined> {
-  const p = Bun.spawn(['git', 'ls-tree', '-r', '-z', tree], {
-    cwd: root,
-    stdin: 'ignore',
-    stdout: 'pipe',
-    stderr: 'ignore',
-  })
+  let p
+  try {
+    p = Bun.spawn(['git', 'ls-tree', '-r', '-z', tree], {
+      cwd: root,
+      stdin: 'ignore',
+      stdout: 'pipe',
+      stderr: 'ignore',
+    })
+  } catch {
+    // quiet: no git, or no root, is git unable to say.
+    return undefined
+  }
   const [out, code] = await Promise.all([new Response(p.stdout).text(), p.exited])
   if (code !== 0) return undefined
-  const files = new Set<string>()
+  const files = new Map<string, string>()
   const links: string[] = []
   for (const e of out.split('\0')) {
     const tab = e.indexOf('\t')
     if (tab === -1) continue
     const name = e.slice(tab + 1)
-    if (e.startsWith('160000 ')) links.push(name)
-    else files.add(name)
+    // `<mode> <type> <oid>`: core's identity is the OID, the mode before it unless a plain file's.
+    const [mode, , oid] = e.slice(0, tab).split(' ')
+    if (mode === '160000') links.push(name)
+    else files.set(name, mode === '100644' ? oid! : `${mode}:${oid}`)
   }
   return { files, links }
 }
@@ -2250,11 +2640,52 @@ function sandboxConfigs(nodes: ReadonlyMap<string, TaskNode> | undefined): Sandb
  * pinning every dependant to its worker held a run's shards on one.
  */
 function leaves(node: TaskNode): boolean {
-  if (node.config.cache !== undefined) return false
+  if (node.config.cache === true) return false
   const sandbox = node.config.exec?.sandbox
   return sandbox === undefined || (sandbox.allow?.write?.length ?? 0) > 0
 }
 
+/**
+ * A task and an upstream of it that `leaves` files, which `bound` says
+ * cannot run where the task does (its `exec.worker` or `exec.platforms`):
+ * the task reads those files where the upstream ran, or runs it again
+ * where it runs itself. Undefined when none.
+ */
+export function apart(
+  nodes: ReadonlyMap<string, TaskNode>,
+  bound: (upstream: TaskNode, dependant: TaskNode) => boolean,
+): { upstream: string; dependant: string } | undefined {
+  const runs = (n: TaskNode): boolean =>
+    n.config.exec?.command !== undefined && n.config.exec.persistent === undefined
+  const named = [...nodes.values()].filter(
+    (n) =>
+      runs(n) &&
+      leaves(n) &&
+      (n.config.exec?.worker !== undefined || n.config.exec?.platforms !== undefined),
+  )
+  if (named.length === 0) return undefined
+  const below = new Map<string, string[]>()
+  for (const n of nodes.values())
+    for (const d of n.deps) {
+      const of = below.get(d)
+      if (of === undefined) below.set(d, [n.id])
+      else of.push(n.id)
+    }
+  for (const u of named) {
+    const seen = new Set<string>()
+    const walk = [...(below.get(u.id) ?? [])]
+    for (let id = walk.pop(); id !== undefined; id = walk.pop()) {
+      if (seen.has(id)) continue
+      seen.add(id)
+      const d = nodes.get(id)
+      if (d === undefined) continue
+      if (runs(d) && bound(u, d)) return { upstream: u.id, dependant: id }
+      walk.push(...(below.get(id) ?? []))
+    }
+  }
+  return undefined
+}
+
 /**
  * Every task `id` depends on, transitively, each after its own
  * dependencies: the order a worker restores them in. Dependant first, an
@@ -2276,11 +2707,17 @@ function upstream(nodes: ReadonlyMap<string, TaskNode> | undefined, id: string):
   return out
 }
 
+/** Core-time busy over core-time had, from first send to last end: its machine, its tasks, its worker. */
+function utilization(machine: number, tasks: number, worker: number | undefined): string {
+  return `cpu utilization machine ${percent(machine)} · tasks ${percent(tasks)}${worker === undefined ? '' : ` · worker ${percent(worker)}`}`
+}
+
 /** A worker's run, spelled out: time, CPU, memory and the relay. */
 function describe(st: UsageStats, u: Usage): string[] {
   return [
     `online ${seconds(st.onlineMs)} · busy ${seconds(st.busyMs)} · idle ${seconds(Math.max(0, st.onlineMs - st.busyMs))}`,
-    `cpu peak ${st.cpuPeak.toFixed(1)} · used avg ${st.cpuAvg.toFixed(1)} cores${st.machineAvg === undefined ? '' : ` · machine avg ${st.machineAvg.toFixed(1)}`} · booked avg ${st.bookedAvg.toFixed(1)} · cpu utilization ${percent(st.utilization)}`,
+    `cpu peak ${st.cpuPeak.toFixed(1)} · used avg ${st.cpuAvg.toFixed(1)} cores${st.machineAvg === undefined ? '' : ` · machine avg ${st.machineAvg.toFixed(1)}`} · booked avg ${st.bookedAvg.toFixed(1)}`,
+    utilization(st.utilization, st.tasksUtilization, st.workerUtilization),
     // No task said what it held: `memory peak 0 B` would be a claim.
     ...(st.memoryPeak > 0
       ? [`memory peak ${bytes(st.memoryPeak)} · avg ${bytes(st.memoryAvg)}`]
@@ -2330,16 +2767,24 @@ function runtimeOrders(req: ExecuteRequest): { runtime?: RuntimeOrder[] } {
 
 /**
  * `env` as it travels: no unset name, and no name whose value is the
- * worker's (`HOST_ENV`) unless `define` sets it. A defined `HOME` ran
- * with the worker's own, saved under a key that folded the defined one
- * (2026-10-24).
+ * worker's (`HOST_ENV`) unless `define` sets it or the task keys it
+ * (`keyed`). A defined `HOME` ran with the worker's own, saved under a key
+ * that folded the defined one (2026-10-24); a keyed `XDG_CONFIG_HOME` did
+ * the same (2026-11-07).
  */
 function defined(
   env: NodeJS.ProcessEnv,
   define: Readonly<Record<string, string>>,
+  keyed: readonly string[] = [],
 ): Record<string, string> {
-  const out: Record<string, string> = {}
+  // No prototype: on `{}`, a name `__proto__` hit the setter and was not sent.
+  const out: Record<string, string> = Object.create(null)
   for (const [k, v] of Object.entries(env))
-    if (v !== undefined && (!HOST_ENV.includes(k) || k in define)) out[k] = v
+    if (v !== undefined && (!HOST_ENV.includes(k) || k in define || keyed.includes(k))) out[k] = v
   return out
 }
+
+/** The `HOST_ENV` names the task's `inputs.env` keys: their value is this machine's, as keyed. */
+function hostKeyed(req: ExecuteRequest): string[] {
+  return (req.inputs?.env ?? []).map((e) => e.name).filter((k) => HOST_ENV.includes(k))
+}