@vzn/vx-dte 0.0.651 → 0.0.652

package/src/orchestrator.ts

@@ -25,6 +25,7 @@ import {
   type SandboxConfig,
   type TaskArtifacts,
   type TaskNode,
+  type WorkerAdvice,
 } from '@vzn/vx'
 import {
   bytesOf,
@@ -35,14 +36,24 @@ import {
   type DepOrder,
   type RunOrder,
   type RuntimeOrder,
+  type SentOrder,
+  type WiredDep,
   type ToWorker,
   type Load,
   type ToOrchestrator,
   type WiredResult,
 } from './messages.js'
-import { simulate, type PlanTask, type PlanWorker, type Size, type WarmUp } from './planner.js'
+import {
+  advise,
+  simulate,
+  type Fleets,
+  type PlanTask,
+  type PlanWorker,
+  type Size,
+  type WarmUp,
+} from './planner.js'
 import { Usage, type UsageStats } from './usage.js'
-import { assign, ordered, type Need } from './dispatch.js'
+import { type Need, Queue } from './dispatch.js'
 import { DEFAULT_STRATEGY, type Strategy } from './strategy.js'
 import { RelayError, unsent, type Participant } from './wire.js'
 import { ORCHESTRATOR, Trace } from './trace.js'
@@ -84,8 +95,15 @@ interface Job {
   reread?: Set<string>
   /** Its upstreams, once the graph named it: walked again at every pump, 8000 queued spent 5 ms a pump on it (2026-10-25). */
   up?: Set<string>
+  /** Its direct dependencies, as placing reads them (`Ask.up`). */
+  deps?: readonly string[] | undefined
+  /** A retry: the worker its last attempt ran on, which it goes elsewhere than while another can take it. */
+  avoid?: string
 }
 
+/** An upstream's files, as an order names them (`DepOrder`). */
+type DepFiles = Pick<DepOrder, 'project' | 'outputs'>
+
 /** The longest a task waits on the worker holding its upstream's files before they are made again (2026-10-23: 10 s on any wait). */
 const PIN_WAIT_MS = 10_000
 
@@ -123,6 +141,8 @@ interface Worker {
   load?: { cpu: number; memory: number; at: number }
   /** Tasks whose outputs it holds: it ran them, or was sent them. */
   readonly has: Set<string>
+  /** Each dependency it was sent whole, by the key it was sent under: the next order names it alone. */
+  readonly sentDeps: Map<string, string>
   /** Sends to it go out in order: a run's parts before the run. */
   chain: Promise<void>
   /** It was sent the run's sandbox configs. */
@@ -207,6 +227,15 @@ const BATCH_BYTES = 512 * 1024
 const FEW = 2
 /** The unmeasured tasks the summary names; the rest it counts. */
 const NAMED = 40
+/** The footer's label of each `WorkerAdvice`. */
+const ADVICE: Record<WorkerAdvice['of'], readonly [label: string, note: string]> = {
+  declared: ['best', 'the fewest of its sizes'],
+  more: ['more', "past its sizes' max"],
+  any: ['any', 'of any cores, the least core time'],
+}
+/** `3 × small (4c/16G linux)`. */
+const fleetText = (w: WorkerAdvice['workers'][number]): string =>
+  `${w.count} × ${w.size === undefined ? '' : `${w.size} `}(${w.cores}c/${compact(w.memoryBytes)}${w.platform === undefined ? '' : ` ${w.platform}`})`
 /** Core-time of history the run's tasks finish before their use reshapes bookings: a few tasks' worth. */
 const LEARN_MS = 60_000
 /** The most a worker's bookings may add up to, in its cores, when tasks use less than their history. */
@@ -215,12 +244,12 @@ const OVERBOOK = 1.5
 const RESTARTS = 3
 
 /** `orders` in runs of at most `max` bytes of JSON; one bigger goes alone. */
-function parts<T extends { order: unknown }>(orders: readonly T[], max: number): T[][] {
+function parts<T extends { bytes: number }>(orders: readonly T[], max: number): T[][] {
   const out: T[][] = []
   let part: T[] = []
   let bytes = 0
   for (const o of orders) {
-    const size = bytesOf(o.order)
+    const size = o.bytes
     if (part.length > 0 && bytes + size > max) {
       out.push(part)
       part = []
@@ -259,9 +288,13 @@ export class Orchestrator {
   private readonly workers = new Map<string, Worker>()
   /** The run tree's files, workspace-relative, listed when a task first asks. */
   private inTree?: Promise<Tree | undefined>
-  private readonly queue: Job[] = []
+  /** Waiting tasks, by id, in the order a pump reads them (`Queue`). */
+  private readonly waiting = new Map<string, Job>()
+  private readonly queue: Queue
   /** Where a worker saved each task it ran: its cache key or its run key. */
   private readonly savedAs = new Map<string, string>()
+  /** The worker each task last ran on, which its retry goes elsewhere than. */
+  private readonly ranOn = new Map<string, string>()
   /** Keys the run's requests named for their dependencies. */
   private readonly upstreamKeys = new Map<string, string>()
   /** How many times each task was sent: its run key is one per send. */
@@ -354,9 +387,12 @@ export class Orchestrator {
   private relend: ReturnType<typeof setTimeout> | undefined
   /** The plugins any of the run's lends lent a token. */
   private readonly lenders = new Set<string>()
+  /** Each graph's upstreams as orders name them (`dep`). */
+  private readonly deps = new WeakMap<ReadonlyMap<string, TaskNode>, Map<string, DepFiles | null>>()
 
   constructor(private readonly o: OrchestratorOptions) {
     this.sent = o.sent ?? new Map()
+    this.queue = new Queue(o.how ?? DEFAULT_STRATEGY, o.tail, (id) => this.waiting.get(id)!.need)
     // A root that is gone fails the run's own reads first.
     this.realRoot = (() => {
       try {
@@ -371,6 +407,11 @@ export class Orchestrator {
 
   async execute(req: ExecuteRequest): Promise<ExecuteResult> {
     if (this.lost !== undefined) throw new Error(this.lost)
+    // Walked once here, for every later read: walked again on the way in,
+    // sharing, placing and sending, a run of 8000 tasks spent 17 s of this
+    // machine's CPU on it (2026-11-05).
+    const nodes = this.o.nodes()
+    const up = upstream(nodes, req.taskId)
     // A worker runs the run's tree: a file its key includes that the tree
     // lacks (untracked) is not there, and the output it saved under that
     // key was a stale hit for the next run (2026-10-24).
@@ -380,25 +421,16 @@ export class Orchestrator {
       // A dependency's output is restored there before the task runs: a
       // key that reads an upstream's bytes (`inputs.tasks` leaving it out)
       // was refused (2026-10-25).
-      const nodes = this.o.nodes()
-      const restored = [...upstream(nodes, req.taskId)].flatMap((id) => {
-        const n = nodes?.get(id)
-        const outs = n?.config.cache?.outputs
-        return n === undefined || outs === undefined
-          ? []
-          : [
-              names(path.relative(this.o.root, n.projectDir), {
-                files: outs.files,
-                workspaceFiles: outs.workspaceFiles ?? [],
-              }),
-            ]
-      })
+      let restored: ((file: string) => boolean)[] | undefined
+      const made = (file: string): boolean =>
+        (restored ??= [...up].flatMap((id) => {
+          const d = this.dep(nodes, id)
+          return d === undefined ? [] : [names(d.project, d.outputs)]
+        })).some((m) => m(file))
       const stray =
         tree === undefined
           ? undefined
-          : req.inputs.files.find(
-              (f) => !tree.files.has(f.path) && !restored.some((made) => made(f.path)),
-            )
+          : req.inputs.files.find((f) => !tree.files.has(f.path) && !made(f.path))
       if (stray !== undefined) {
         // A committed submodule's files are not the tree's: its gitlink is.
         const link = tree?.links.find((l) => stray.path.startsWith(`${l}/`))
@@ -410,20 +442,24 @@ export class Orchestrator {
       }
     }
     for (const u of req.inputs?.upstream ?? []) this.upstreamKeys.set(u.taskId, u.hash)
-    await this.share(req.taskId)
+    await this.share(up, nodes)
     return new Promise((resolve, reject) => {
       const need = this.o.need(req.taskId)
       const job: Job = {
         req,
+        ...(nodes?.has(req.taskId) === true ? { up } : {}),
         need: { ...need, platform: req.platform ?? this.o.home },
         queuedAt: performance.now(),
         resolve,
         reject,
       }
+      // A retry runs elsewhere: the worker, or what ran beside it, may be the flake.
+      const last = (req.attempt ?? 1) > 1 ? this.ranOn.get(req.taskId) : undefined
+      if (last !== undefined) job.avoid = last
       // A signal aborted already fires no `abort`.
       if (req.signal?.aborted === true) return resolve(stopped())
       req.signal?.addEventListener('abort', () => this.abort(job), { once: true })
-      this.queue.push(job)
+      this.enqueue([job])
       this.pumpSoon()
     })
   }
@@ -499,9 +535,9 @@ export class Orchestrator {
   }
 
   private abort(job: Job): void {
-    const at = this.queue.indexOf(job)
-    if (at >= 0) {
-      this.queue.splice(at, 1)
+    if (this.waiting.get(job.req.taskId) === job) {
+      this.waiting.delete(job.req.taskId)
+      this.queue.delete(job.req.taskId)
       job.resolve(stopped())
     } else if (job.worker !== undefined) {
       const a = this.workers.get(job.worker)
@@ -519,27 +555,22 @@ export class Orchestrator {
     })
   }
 
+  /** `jobs` wait, in order, behind the waiting tasks or, `front`, before them. */
+  private enqueue(jobs: readonly Job[], front = false): void {
+    for (const j of jobs) this.waiting.set(j.req.taskId, j)
+    this.queue.add(
+      jobs.map((j) => j.req.taskId),
+      front,
+    )
+  }
+
   private pump(): void {
     const batches = new Map<Worker, Job[]>()
-    const left: Job[] = []
     // Core asks in its critical-path order, but only as tasks get ready: a
     // long chain ready a moment late queued behind 46 shards ready at once,
     // and its 41 s build began 47 s in, the run's end (main, 2026-10-23).
-    const tail = this.o.tail
-    const how = this.o.how ?? DEFAULT_STRATEGY
-    const waiting = this.queue.splice(0)
     const workers = [...this.workers.values()].filter((a) => !a.faulty)
-    const byId = new Map(waiting.map((j) => [j.req.taskId, j]))
-    const queue =
-      tail === undefined
-        ? waiting
-        : ordered(
-            waiting.map((j) => j.req.taskId),
-            how.order,
-            tail,
-            (id) => byId.get(id)!.need,
-            Math.max(1, ...workers.map((a) => a.cores)),
-          ).map((id) => byId.get(id)!)
+    const byId = this.waiting
     const bins = workers.map((a) => {
       let running = 0
       for (const j of a.busy.values()) if (j.held !== undefined) running++
@@ -555,39 +586,24 @@ export class Orchestrator {
       }
     })
     const nodes = this.o.nodes()
-    const asks = []
-    const placed: Job[] = []
-    // The least any ask needs: once no worker takes it, the pass ends. Each
-    // pump read every queued task, 8000 of them a pump (2026-10-25).
-    const floor = { cores: Infinity, memory: Infinity }
-    for (const job of queue) {
-      const up =
-        job.up ??
-        (nodes?.has(job.req.taskId) === true
-          ? (job.up = upstream(nodes, job.req.taskId))
-          : upstream(nodes, job.req.taskId))
-      const pin = this.pin(job, up)
-      if (pin === 'done') continue
-      if (pin === 'wait') {
-        left.push(job)
-        continue
-      }
-      placed.push(job)
-      const need = this.booking(job.need)
-      floor.cores = Math.min(floor.cores, need.cores)
-      floor.memory = Math.min(floor.memory, need.memory)
-      asks.push({ id: job.req.taskId, need, up, ...pin })
-    }
-    const to = assign(asks, bins, (above, platform) => this.bigger(above, platform), how.place, {
-      floor,
-      kinds: Infinity,
+    const to = this.queue.pass(bins, {
+      book: (need) => this.booking(need),
+      deps: (id) => (byId.get(id)!.deps ??= nodes?.get(id)?.deps),
+      gate: (id) => {
+        const job = byId.get(id)!
+        const up =
+          job.up ?? (nodes?.has(id) === true ? (job.up = upstream(nodes, id)) : upstream(nodes, id))
+        const pin = this.pin(job, up)
+        // It ended: the queue lets it go.
+        if (pin === 'done') byId.delete(id)
+        return pin
+      },
+      waits: (above, platform) => this.bigger(above, platform),
     })
+    const placed = [...to.keys()].map((id) => byId.get(id)!)
+    for (const id of to.keys()) byId.delete(id)
     for (const job of placed) {
-      const a = to.get(job.req.taskId)?.a
-      if (a === undefined) {
-        left.push(job)
-        continue
-      }
+      const a = to.get(job.req.taskId)!.a
       job.worker = a.id
       job.sentAt = performance.now()
       a.busy.set(job.req.taskId, job)
@@ -610,7 +626,6 @@ export class Orchestrator {
       if (batch === undefined) batches.set(a, [job])
       else batch.push(job)
     }
-    this.queue.push(...left)
     for (const [a, jobs] of batches) this.send(a, jobs)
   }
 
@@ -624,6 +639,23 @@ export class Orchestrator {
   private pin(
     job: Job,
     up: ReadonlySet<string>,
+  ): 'wait' | 'done' | { only?: string; avoid?: string } {
+    const where = this.pinUp(job, up)
+    const avoid = job.avoid
+    if (typeof where !== 'object' || avoid === undefined) return where
+    if (where.only !== undefined || where.avoid !== undefined) return where
+    // With no other worker that can take it, it runs where it failed rather than wait.
+    const platform = job.need.platform
+    for (const w of this.workers.values())
+      if (w.id !== avoid && !w.faulty && (platform === undefined || w.platform === platform))
+        return { avoid }
+    return where
+  }
+
+  /** Where `job` may go as its upstreams' files lie (`pin`, before a retry's own rule). */
+  private pinUp(
+    job: Job,
+    up: ReadonlySet<string>,
   ): 'wait' | 'done' | { only?: string; avoid?: string } {
     if (job.again !== undefined) {
       const { only } = job.again
@@ -693,24 +725,31 @@ export class Orchestrator {
         this.pumpSoon()
       }
       this.pumpSoon()
-      this.queue.unshift({
-        req: k.req,
-        need: where.platform === undefined ? k.need : { ...k.need, platform: where.platform },
-        queuedAt: performance.now(),
-        pinnedAt: performance.now(),
-        again: {
-          ...(where.only === undefined ? {} : { only: where.only }),
-          ...(where.avoid === undefined ? {} : { avoid: where.avoid }),
-        },
-        resolve: (r) =>
-          done(
-            r.exitCode === 0 ? undefined : `${id} ran again for it and failed (exit ${r.exitCode})`,
-          ),
-        reject: (err) =>
-          done(
-            `${id} ran again for it and failed: ${err instanceof Error ? err.message : String(err)}`,
-          ),
-      })
+      this.enqueue(
+        [
+          {
+            req: k.req,
+            need: where.platform === undefined ? k.need : { ...k.need, platform: where.platform },
+            queuedAt: performance.now(),
+            pinnedAt: performance.now(),
+            again: {
+              ...(where.only === undefined ? {} : { only: where.only }),
+              ...(where.avoid === undefined ? {} : { avoid: where.avoid }),
+            },
+            resolve: (r) =>
+              done(
+                r.exitCode === 0
+                  ? undefined
+                  : `${id} ran again for it and failed (exit ${r.exitCode})`,
+              ),
+            reject: (err) =>
+              done(
+                `${id} ran again for it and failed: ${err instanceof Error ? err.message : String(err)}`,
+              ),
+          },
+        ],
+        true,
+      )
       if (where.only !== undefined || where.avoid !== undefined)
         setTimeout(() => this.pumpSoon(), this.pinWaitMs + 50).unref?.()
     }
@@ -815,11 +854,13 @@ export class Orchestrator {
    * hit only this cache held (the remote evicted it, or this machine does
    * not write it) failed the dependant there (2026-10-24).
    */
-  private async share(id: string): Promise<void> {
-    const nodes = this.o.nodes()
+  private async share(
+    up: ReadonlySet<string>,
+    nodes: ReadonlyMap<string, TaskNode> | undefined,
+  ): Promise<void> {
     const keys: string[] = []
-    for (const d of upstream(nodes, id)) {
-      if (this.savedAs.has(d) || nodes?.get(d)?.config.cache?.outputs === undefined) continue
+    for (const d of up) {
+      if (this.savedAs.has(d) || this.dep(nodes, d) === undefined) continue
       const key = this.upstreamKeys.get(d) ?? this.o.keyOf(d)
       if (key !== undefined) keys.push(key)
     }
@@ -860,92 +901,113 @@ export class Orchestrator {
     return got.includes(true)
   }
 
+  /**
+   * Upstream `id` as an order names it, once per graph: its project and
+   * outputs. Undefined: it is not in the graph, or leaves no files.
+   */
+  private dep(nodes: ReadonlyMap<string, TaskNode> | undefined, id: string): DepFiles | undefined {
+    if (nodes === undefined) return undefined
+    let of = this.deps.get(nodes)
+    if (of === undefined) this.deps.set(nodes, (of = new Map()))
+    let d = of.get(id)
+    if (d === undefined) {
+      const n = nodes.get(id)
+      const outs = n?.config.cache === true ? n.config.outputs : undefined
+      d =
+        n === undefined || outs === undefined
+          ? null
+          : {
+              project: path.relative(this.o.root, n.projectDir),
+              outputs: { files: [...outs.files], workspaceFiles: [...(outs.workspaceFiles ?? [])] },
+            }
+      of.set(id, d)
+    }
+    return d ?? undefined
+  }
+
   private send(a: Worker, jobs: readonly Job[]): void {
     const nodes = this.o.nodes()
-    const orders = jobs.map((job) => {
+    const orders = jobs.map((job): { job: Job; order: RunOrder } => {
       const { req } = job
       const deps: DepOrder[] = []
       // Every one, held there or not: the worker restores each key once, and
       // one it was sent but gave back it never ran or restored.
-      for (const id of upstream(nodes, req.taskId)) {
-        const n = nodes?.get(id)
-        const outs = n?.config.cache?.outputs
+      for (const id of job.up ?? upstream(nodes, req.taskId)) {
+        const d = this.dep(nodes, id)
         const key = this.savedAs.get(id) ?? this.upstreamKeys.get(id) ?? this.o.keyOf(id)
-        if (n === undefined || outs === undefined || key === undefined) continue
+        if (d === undefined || key === undefined) continue
         a.has.add(id)
-        deps.push({
-          task: id,
-          key,
-          project: path.relative(this.o.root, n.projectDir),
-          outputs: { files: [...outs.files], workspaceFiles: [...(outs.workspaceFiles ?? [])] },
-        })
+        deps.push({ task: id, key, project: d.project, outputs: d.outputs })
       }
       // A task sent here leaves its outputs here for its dependants.
       a.has.add(req.taskId)
       const node = nodes?.get(req.taskId)
       const env = defined(req.env, req.envDefine)
-      return {
-        job,
-        order: {
-          task: req.taskId,
-          // Unsaved: under the run's own key, as an uncached task's.
-          ...(req.cacheKey === undefined || req.unsaved === true ? {} : { key: req.cacheKey }),
-          runKey: runKey(this.o.run, req.taskId, this.sends(req.taskId)),
-          // A held key keeps its entry, and restored here it replaced the
-          // forced run's fresh bytes with the ones it bypassed (2026-10-25).
-          ...(req.refresh === true ? { fresh: true as const } : {}),
-          // A HEAD per upload was the workers' only remote probe: the run
-          // already knows the key is in no cache (2026-10-25).
-          ...(req.refresh !== true && this.missed.has(req.taskId) ? { absent: true as const } : {}),
-          deps,
-          nested: node === undefined ? [] : nestedIn(nodes!, node.projectDir, this.o.root),
-          command: withForwardArgs(req.command, req.forwardArgs),
-          cwd: path.relative(this.o.root, req.cwd),
-          project: node === undefined ? '' : path.relative(this.o.root, node.projectDir),
-          env,
-          ...(node?.config.exec?.env?.secret === undefined
-            ? {}
-            : { secret: [...node.config.exec.env.secret] }),
-          ...(req.timeoutMs === undefined ? {} : { timeoutMs: req.timeoutMs }),
-          outputs: {
-            files: [...req.outputs.files],
-            workspaceFiles: [...req.outputs.workspaceFiles],
-          },
-          cores: job.held?.cores ?? 1,
-          memory: job.held?.memory ?? 0,
-          ...(req.sandbox === undefined || node?.config.exec?.sandbox === undefined
-            ? {}
-            : {
-                sandbox: {
-                  config: node.config.exec.sandbox,
-                  ...(req.sandbox.keyed === undefined
-                    ? {}
-                    : { keyed: req.sandbox.keyed.map((d) => path.relative(this.o.root, d)) }),
-                },
-              }),
-          ...runtimeOrders(req),
-          ...(req.servers === undefined || req.servers.length === 0
-            ? {}
-            : {
-                servers: req.servers.map((s) => ({
-                  task: s.taskId,
-                  command: s.command,
-                  cwd: path.relative(this.o.root, s.cwd),
-                  env: defined(s.env, {}),
-                  ...(s.readyWhen === undefined ? {} : { readyWhen: s.readyWhen }),
-                  ...(s.timeoutMs === undefined ? {} : { timeoutMs: s.timeoutMs }),
-                  ...(s.sandbox === undefined
-                    ? {}
-                    : {
-                        sandbox: {
-                          config: s.sandbox.config,
-                          nested: s.sandbox.nested.map((d) => path.relative(this.o.root, d)),
-                        },
-                      }),
-                })),
-              }),
-        } as RunOrder,
-      }
+      const order = {
+        task: req.taskId,
+        // Unsaved: under the run's own key, as an uncached task's.
+        ...(req.cacheKey === undefined || req.unsaved === true ? {} : { key: req.cacheKey }),
+        runKey: runKey(this.o.run, req.taskId, this.sends(req.taskId)),
+        // A held key keeps its entry, and restored here it replaced the
+        // forced run's fresh bytes with the ones it bypassed (2026-10-25).
+        ...(req.refresh === true ? { fresh: true as const } : {}),
+        // A HEAD per upload was the workers' only remote probe: the run
+        // already knows the key is in no cache (2026-10-25).
+        ...(req.refresh !== true && this.missed.has(req.taskId) ? { absent: true as const } : {}),
+        deps,
+        nested: node === undefined ? [] : nestedIn(nodes!, node.projectDir, this.o.root),
+        command: withForwardArgs(req.command, req.forwardArgs),
+        cwd: path.relative(this.o.root, req.cwd),
+        project: node === undefined ? '' : path.relative(this.o.root, node.projectDir),
+        env,
+        ...(node?.config.exec?.env?.secret === undefined
+          ? {}
+          : { secret: [...node.config.exec.env.secret] }),
+        ...(req.timeoutMs === undefined ? {} : { timeoutMs: req.timeoutMs }),
+        outputs: {
+          files: [...req.outputs.files],
+          workspaceFiles: [...req.outputs.workspaceFiles],
+        },
+        cores: job.held?.cores ?? 1,
+        memory: job.held?.memory ?? 0,
+        ...(req.sandbox === undefined || node?.config.exec?.sandbox === undefined
+          ? {}
+          : {
+              sandbox: {
+                config: node.config.exec.sandbox,
+                ...(req.sandbox.keyed === undefined
+                  ? {}
+                  : { keyed: req.sandbox.keyed.map((d) => path.relative(this.o.root, d)) }),
+                ...(req.sandbox.dependencyReads === undefined
+                  ? {}
+                  : {
+                      reads: req.sandbox.dependencyReads.map((d) => path.relative(this.o.root, d)),
+                    }),
+              },
+            }),
+        ...runtimeOrders(req),
+        ...(req.servers === undefined || req.servers.length === 0
+          ? {}
+          : {
+              servers: req.servers.map((s) => ({
+                task: s.taskId,
+                command: s.command,
+                cwd: path.relative(this.o.root, s.cwd),
+                env: defined(s.env, {}),
+                ...(s.readyWhen === undefined ? {} : { readyWhen: s.readyWhen }),
+                ...(s.timeoutMs === undefined ? {} : { timeoutMs: s.timeoutMs }),
+                ...(s.sandbox === undefined
+                  ? {}
+                  : {
+                      sandbox: {
+                        config: s.sandbox.config,
+                        nested: s.sandbox.nested.map((d) => path.relative(this.o.root, d)),
+                      },
+                    }),
+              })),
+            }),
+      } as RunOrder
+      return { job, order }
     })
     const sandboxed = orders.some(
       (o) => o.order.sandbox !== undefined || o.order.servers?.some((s) => s.sandbox !== undefined),
@@ -955,13 +1017,32 @@ export class Orchestrator {
     const pending = new Set(jobs)
     a.chain = a.chain
       .then(async () => {
-        for (const [i, part] of parts(orders, BATCH_BYTES).entries()) {
+        // A dependency the worker was sent under the same key goes by name
+        // alone: each order named every upstream whole, and 8000 tasks of a
+        // deep graph sent 219 MB (2026-11-05). Read here, as sends go in
+        // order: one sent before it fails is not counted as sent.
+        const fresh = new Map<string, string>()
+        const wired = orders.map(({ job, order }) => {
+          const intro: [string, string][] = []
+          const deps = order.deps.map((d): WiredDep => {
+            if ((fresh.get(d.task) ?? a.sentDeps.get(d.task)) === d.key) return d.task
+            fresh.set(d.task, d.key)
+            intro.push([d.task, d.key])
+            return d
+          })
+          const sent: SentOrder = { ...order, deps }
+          // Measured once: a giant repo's were measured three times each (2026-11-05).
+          return { job, order: sent, intro, bytes: bytesOf(sent) }
+        })
+        for (const [i, part] of parts(wired, BATCH_BYTES).entries()) {
           // An order too long for a frame (a task of thousands of
           // dependencies was refused, 413) sends its deps ahead.
-          const tasks: RunOrder[] = []
-          for (const { order } of part) {
-            if (bytesOf(order) <= BATCH_BYTES) {
+          const tasks: SentOrder[] = []
+          let bytes = 0
+          for (const { order, bytes: size } of part) {
+            if (size <= BATCH_BYTES) {
               tasks.push(order)
+              bytes += size
               continue
             }
             for (const deps of chunks(order.deps, BATCH_BYTES)) {
@@ -970,7 +1051,9 @@ export class Orchestrator {
               a.usage.messagesIn++
               a.usage.bytesIn += bytesOf(ahead)
             }
-            tasks.push({ ...order, deps: [] })
+            const alone = { ...order, deps: [] }
+            tasks.push(alone)
+            bytes += bytesOf(alone)
           }
           const n = ++a.batches
           for (const { job } of part) job.batch = n
@@ -984,7 +1067,9 @@ export class Orchestrator {
           } satisfies ToWorker
           await this.o.me.send(a.id, batch)
           a.usage.messagesIn++
-          a.usage.bytesIn += bytesOf(batch)
+          // Its orders' bytes, a comma between each, in the batch's own.
+          a.usage.bytesIn += bytesOf({ ...batch, tasks: [] }) + bytes + tasks.length - 1
+          for (const { intro } of part) for (const [t, k] of intro) a.sentDeps.set(t, k)
           for (const { job } of part) {
             pending.delete(job)
             // Sent, it runs: its worker takes only what it has room for.
@@ -1121,7 +1206,7 @@ export class Orchestrator {
         a.started.delete(task)
         const again = this.retry(a, job, `could not read ${head.unread.join(', ')}`)
         if (again !== undefined) {
-          this.queue.unshift(again)
+          this.enqueue([again], true)
           this.pumpSoon()
         }
         return
@@ -1136,7 +1221,15 @@ export class Orchestrator {
     a.done.push(task)
     a.usage.record(head, job.sentAt)
     const share = shareOf(head.cpuShare)
-    const held = Math.max(1, job.startedAt === undefined ? head.durationMs : endAt - job.startedAt)
+    // At least as long as it ran: begun here once its batch's send resolved,
+    // which can be after its worker began it, it was seen for less, and its
+    // CPU over that read as more cores than it had (#5036: 1.001 each, so
+    // two that ran side by side on 2 cores replayed one after the other).
+    const held = Math.max(
+      1,
+      head.durationMs,
+      job.startedAt === undefined ? 0 : endAt - job.startedAt,
+    )
     this.took.set(task, {
       ms: held,
       cores:
@@ -1162,6 +1255,7 @@ export class Orchestrator {
       this.needMs += Math.min(job.need.cores, a.cores) * ms
     }
     this.savedAs.set(task, key)
+    this.ranOn.set(task, a.id)
     const node = this.o.nodes()?.get(task)
     if (head.exitCode === 0 && node !== undefined && leaves(node))
       this.kept.set(task, { worker: a.id, req: job.req, need: job.need })
@@ -1389,6 +1483,7 @@ export class Orchestrator {
         started: new Set(),
         batches: 0,
         has: new Set(),
+        sentDeps: new Map(),
         // Its name first, before any batch.
         chain: this.cache()
           .then((cache) =>
@@ -1425,7 +1520,7 @@ export class Orchestrator {
       const again = this.retry(a, j, 'lost')
       if (again !== undefined) back.push(again)
     }
-    this.queue.unshift(...back)
+    this.enqueue(back, true)
     this.o.lost?.(a.label)
     // Workers that join on their own may yet join; ones this run starts
     // are all it gets.
@@ -1484,7 +1579,7 @@ export class Orchestrator {
                 if (again !== undefined) back.push(again)
               }
             if (back.length > 0) {
-              this.queue.unshift(...back)
+              this.enqueue(back, true)
               this.o.faulted?.()
               if (this.spawnedAt.size > 0)
                 this.stranded(`${a.label} no longer runs a task it began and no worker is left`)
@@ -1505,7 +1600,7 @@ export class Orchestrator {
             if (j === undefined) continue
             // It began it whether or not its `start` came first.
             this.begun(a, m.task)
-            if (this.retry(a, j, m.reason) !== undefined) this.queue.unshift(j)
+            if (this.retry(a, j, m.reason) !== undefined) this.enqueue([j], true)
             if (a.faulty) this.o.faulted?.()
             if (a.faulty && this.spawnedAt.size > 0)
               this.stranded(
@@ -1554,7 +1649,7 @@ export class Orchestrator {
   where(taskId: string): string | null | undefined {
     for (const a of this.workers.values())
       if (a.busy.has(taskId)) return a.started.has(taskId) ? a.label : null
-    return this.queue.some((j) => j.req.taskId === taskId) ? null : undefined
+    return this.waiting.has(taskId) ? null : undefined
   }
 
   /**
@@ -1727,7 +1822,8 @@ export class Orchestrator {
           id: j.req.taskId,
           cores: Math.min(a.cores, Math.max((j.held?.cores ?? 0) * over, j.need.ask?.cores ?? 0)),
           memory: Math.max(j.held?.memory ?? 0, Math.min(a.memory, j.need.ask?.memory ?? 0)),
-          endsIn: Math.max(0, (ms(j.req.taskId) ?? 0) - (now - j.startedAt!)),
+          // Below 0: past its time, which the plan weighs (`decide`).
+          endsIn: (ms(j.req.taskId) ?? 0) - (now - j.startedAt!),
         })),
       })
     }
@@ -1875,7 +1971,11 @@ export class Orchestrator {
   private fail(reason: string): void {
     for (const a of this.workers.values())
       for (const j of a.busy.values()) j.reject(new Error(reason))
-    for (const j of this.queue.splice(0)) j.reject(new Error(reason))
+    for (const [id, j] of this.waiting) {
+      this.queue.delete(id)
+      j.reject(new Error(reason))
+    }
+    this.waiting.clear()
     this.workers.clear()
     this.lost = reason
   }
@@ -1899,13 +1999,8 @@ export class Orchestrator {
    * waits on its deps through those no worker ran (groups, hits).
    */
   idealMs(depsOf: (id: string) => readonly string[], sizes: readonly Size[]): number | undefined {
-    if (this.took.size === 0) return undefined
-    const all = [...this.workers.values(), ...this.gone]
-    const warm = new Map<string | undefined, number>()
-    for (const a of all)
-      if (a.warmUpMs !== undefined && !(warm.get(a.platform)! <= a.warmUpMs))
-        warm.set(a.platform, a.warmUpMs)
-    const least = this.warmUpMs() ?? 0
+    const r = this.replay(depsOf)
+    if (r === undefined) return undefined
     const pool: PlanWorker[] = (
       sizes.length > 0
         ? sizes.flatMap((s) =>
@@ -1915,14 +2010,104 @@ export class Orchestrator {
               platform: s.platform,
             })),
           )
-        : all.map((a) => ({ cores: a.cores, memory: a.memory, platform: a.platform }))
+        : [...this.workers.values(), ...this.gone].map((a) => ({
+            cores: a.cores,
+            memory: a.memory,
+            platform: a.platform,
+          }))
     ).map((w, i) => ({
       ...w,
       id: `ideal#${i}`,
-      at: warm.get(w.platform) ?? least,
+      at: r.warm(w.platform),
       running: [],
       idle: false,
     }))
+    const end = simulate(r.tasks, pool)
+    if (!Number.isFinite(end)) return undefined
+    return r.from + end
+  }
+
+  /**
+   * The workers that would have run this run best (`advise`), from the
+   * same replay as `idealMs`: on `sizes`, past their `max`, and of any
+   * cores. With no sizes (workers that join on their own), the shapes
+   * that joined stand for them, as many as joined.
+   */
+  advice(depsOf: (id: string) => readonly string[], sizes: readonly Size[]): WorkerAdvice[] {
+    // The footer and the summary each ask once the run ended: played once.
+    if (this.ended && this.advised !== undefined) return this.advised
+    const out = this.advise(depsOf, sizes)
+    if (this.ended) this.advised = out
+    return out
+  }
+
+  private advised: WorkerAdvice[] | undefined
+
+  private advise(
+    depsOf: (id: string) => readonly string[],
+    sizes: readonly Size[],
+  ): WorkerAdvice[] {
+    const r = this.replay(depsOf)
+    if (r === undefined) return []
+    const shapes: Size[] =
+      sizes.length > 0
+        ? [...sizes]
+        : [
+            ...[...this.workers.values(), ...this.gone]
+              .reduce((by, a) => {
+                const name = `${a.cores}c/${a.platform ?? ''}`
+                const had = by.get(name)
+                by.set(name, {
+                  name,
+                  max: (had?.max ?? 0) + 1,
+                  cpu: a.cores,
+                  memory: a.memory,
+                  platform: a.platform,
+                })
+                return by
+              }, new Map<string, Size>())
+              .values(),
+          ]
+    const a = advise(r.tasks, shapes, r.warm)
+    const wire = (of: WorkerAdvice['of'], f: Fleets | undefined): WorkerAdvice[] =>
+      f === undefined
+        ? []
+        : [
+            {
+              of,
+              endMs: Math.round(r.from + f.endMs),
+              workers: f.fleet.map((w) => ({
+                count: w.count,
+                cores: w.cores,
+                memoryBytes: w.memory,
+                ...(w.platform === undefined ? {} : { platform: w.platform }),
+                ...(w.size === undefined || sizes.length === 0 ? {} : { size: w.size }),
+              })),
+            },
+          ]
+    return [...wire('declared', a.declared), ...wire('more', a.more), ...wire('any', a.any)]
+  }
+
+  /**
+   * Each task a worker ran, as long as it held its worker and with the
+   * cores it used meanwhile, waiting on its deps through those no worker
+   * ran (groups, hits); each platform's fastest real warm-up; and when
+   * the run first knew its work.
+   */
+  private replay(depsOf: (id: string) => readonly string[]):
+    | {
+        tasks: PlanTask[]
+        warm: (platform: string | undefined) => number
+        from: number
+      }
+    | undefined {
+    if (this.took.size === 0) return undefined
+    const all = [...this.workers.values(), ...this.gone]
+    const warm = new Map<string | undefined, number>()
+    for (const a of all)
+      if (a.warmUpMs !== undefined && !(warm.get(a.platform)! <= a.warmUpMs))
+        warm.set(a.platform, a.warmUpMs)
+    const least = this.warmUpMs() ?? 0
     const seen = new Map<string, string[]>()
     const ran = (id: string): string[] => {
       const known = seen.get(id)
@@ -1932,10 +2117,11 @@ export class Orchestrator {
       seen.set(id, out)
       return out
     }
-    const tasks: PlanTask[] = [...this.took].map(([id, t]) => ({ id, ...t, deps: ran(id) }))
-    const end = simulate(tasks, pool)
-    if (!Number.isFinite(end)) return undefined
-    return (this.probe?.at ?? this.asks[0]?.at ?? 0) + end
+    return {
+      tasks: [...this.took].map(([id, t]) => ({ id, ...t, deps: ran(id) })),
+      warm: (platform) => warm.get(platform) ?? least,
+      from: this.probe?.at ?? this.asks[0]?.at ?? 0,
+    }
   }
 
   /**
@@ -1994,7 +2180,10 @@ export class Orchestrator {
    * each, the cores and memory its size declares and its machine reports;
    * and `ideal`, the run's `idealMs`, asked once it ended.
    */
-  footer(ideal?: () => number | undefined): [string, ui.Inline][] {
+  footer(
+    ideal?: () => number | undefined,
+    advice?: () => readonly WorkerAdvice[],
+  ): [string, ui.Inline][] {
     // Over with none, it says so: core's own `N workers` (this machine's
     // pool) read as the run's when no task needed one (2026-10-25).
     if (this.spawnedAt.size === 0 && this.workers.size === 0 && this.gone.length === 0)
@@ -2125,6 +2314,17 @@ export class Orchestrator {
                       ),
                     ] as [string, ui.Inline],
                   ])(ideal?.()),
+            // What to declare next time (2026-11-05: "So I can add more
+            // or change their specs").
+            ...(advice?.() ?? []).map(
+              (a) =>
+                [
+                  '',
+                  ui.dim(
+                    `${ADVICE[a.of][0].padEnd(width)}  ${a.workers.map(fleetText).join(' + ')}: ${seconds(a.endMs)}, ${ADVICE[a.of][1]}`,
+                  ),
+                ] as [string, ui.Inline],
+            ),
             ...(this.unmeasured.length === 0
               ? []
               : [
@@ -2190,15 +2390,28 @@ export function nestedIn(
 ): string[] {
   let of = nested.get(nodes)
   if (of === undefined) {
-    of = { dirs: [...new Set([...nodes.values()].map((n) => n.projectDir))], by: new Map() }
+    of = {
+      dirs: [...new Set([...nodes.values()].map((n) => n.projectDir))].sort(),
+      by: new Map(),
+    }
     nested.set(nodes, of)
   }
   let out = of.by.get(dir)
   if (out === undefined) {
-    out = of.dirs
-      .filter((d) => d.startsWith(dir + path.sep))
-      .map((d) => path.relative(root, d))
-      .sort()
+    // Sorted, the ones under `dir` lie together: each project read every
+    // project, 16 million reads for 4000 (2026-11-05).
+    const under = dir + path.sep
+    let lo = 0
+    let hi = of.dirs.length
+    while (lo < hi) {
+      const mid = (lo + hi) >> 1
+      if (of.dirs[mid]! < under) lo = mid + 1
+      else hi = mid
+    }
+    const found: string[] = []
+    for (let i = lo; i < of.dirs.length && of.dirs[i]!.startsWith(under); i++)
+      found.push(path.relative(root, of.dirs[i]!))
+    out = found.sort()
     of.by.set(dir, out)
   }
   return [...out]
@@ -2250,7 +2463,7 @@ function sandboxConfigs(nodes: ReadonlyMap<string, TaskNode> | undefined): Sandb
  * pinning every dependant to its worker held a run's shards on one.
  */
 function leaves(node: TaskNode): boolean {
-  if (node.config.cache !== undefined) return false
+  if (node.config.cache === true) return false
   const sandbox = node.config.exec?.sandbox
   return sandbox === undefined || (sandbox.allow?.write?.length ?? 0) > 0
 }