@vzn/vx-dte 0.0.650 → 0.0.652

package/src/wire.ts

@@ -42,18 +42,24 @@ export function unsent(
  */
 const FIRST_REDIAL_MS = 250
 /**
- * The orchestrator probes a worker quiet this many heartbeats: each probe
- * and its ack are two frames, so a peer heard from is never asked.
+ * A peer quiet this many heartbeats is lost (6 s on the default): the one
+ * number the others derive from.
  */
-const PROBE_BEATS = 1.5
-/** A worker probes its orchestrator quiet this long: after the orchestrator would have. */
-const WORKER_PROBE_BEATS = 1.6
+const LOST_BEATS = 3
 /**
- * A peer quiet this many is lost: a live one, acking a probe at once, is
- * heard within a round trip of it. Checked each tenth, loss lands by two
- * (4 s on the default).
+ * A worker quiet half of `LOST_BEATS` sends its report (3 s), and its
+ * orchestrator's ack answers it: one late frame is not a loss.
  */
-const LOST_BEATS = 1.9
+const QUIET_BEATS = LOST_BEATS / 2
+/**
+ * How long the orchestrator gives its own reaction to a frame (a pump, a
+ * batch) to carry the frame's ack, before the ack goes alone.
+ */
+const REACT_MS = 20
+/** A send unacknowledged as long goes again: its ack comes within it. */
+const RESEND_BEATS = LOST_BEATS / 2
+/** Why a worker ends on its orchestrator's silence. */
+export const ORCHESTRATOR_SILENT = 'relay: the orchestrator went silent'
 /**
  * A relay that forgot this participant (a restart: it stores nothing) is
  * joined again for this many heartbeats; the orchestrator joins as the
@@ -61,8 +67,8 @@ const LOST_BEATS = 1.9
  * rejoin for 2 heartbeats, it's over").
  */
 const REJOIN_BEATS = 2
-/** What a probe delivers: nothing. */
-const PROBED = Symbol('probe')
+/** What a beat delivers: nothing. */
+const BEAT = Symbol('beat')
 /** A frame of several sends stays under this, half the relay's 1 MiB body limit. */
 const FRAME_BYTES = 512 * 1024
 /** How long `leave` waits on its socket's close handshake. */
@@ -121,17 +127,17 @@ const isNumbered = (b: unknown): b is Numbered =>
   'body' in b &&
   Object.keys(b).length === ('acked' in b ? 3 : 2)
 /**
- * A peer quiet a heartbeat is asked: a numbered send with no body, which
- * its receiver acknowledges at once and delivers to no one.
+ * What a side with nothing else to say a heartbeat sends: a numbered send
+ * with no body, carrying its acks, delivered to no one.
  */
-interface Probe {
+interface Beat {
   readonly seq: number
   readonly acked?: number
 }
-const isProbe = (b: unknown): b is Probe =>
+const isBeat = (b: unknown): b is Beat =>
   typeof b === 'object' &&
   b !== null &&
-  typeof (b as Probe).seq === 'number' &&
+  typeof (b as Beat).seq === 'number' &&
   !('body' in b) &&
   Object.keys(b).length === ('acked' in b ? 2 : 1)
 const isAcked = (b: unknown): b is Acked =>
@@ -169,6 +175,12 @@ export interface Participant {
    * said meanwhile, and is not lost for that.
    */
   resumed(f: (peer?: string) => void): void
+  /**
+   * A worker's `f` is told when it said nothing to its orchestrator a
+   * heartbeat: it says something now, its live report. Untold, the wire
+   * sends a beat. The orchestrator is never told: it speaks on no timer.
+   */
+  quiet(f: (peer: string) => void): void
   send(to: string, body: unknown): Promise<void>
   /** Held up to one heartbeat, which also keeps this participant alive. */
   poll(signal?: AbortSignal): Promise<RelayMessage[]>
@@ -255,45 +267,70 @@ export async function join(
   >()
   /** Each sender's sends taken in order, and those that came ahead of a gap. */
   const taken = new Map<string, { upTo: number; ahead: Map<number, unknown> }>()
+  /** Acks owed, by peer: each rides the next send to its peer. */
+  const owing = new Set<string>()
   /**
-   * Acks owed, by peer: each rides the next send to its peer; those with
-   * none in half a heartbeat go together in one frame, before any peer
-   * sends again, in place of a ping.
+   * The orchestrator sends only orders and acks: an ack no order carried
+   * goes on its own after half a heartbeat, every peer's in one frame. A
+   * worker's ride its next send, which it makes within a heartbeat. A peer
+   * the orchestrator said nothing to in a heartbeat (a quiet worker's
+   * report) is answered once the orchestrator reacted: an order sent in
+   * reply carries the ack, and with none it goes alone, `REACT_MS` on.
    */
-  const owing = new Set<string>()
   let acking: ReturnType<typeof setTimeout> | undefined
+  let answering: ReturnType<typeof setTimeout> | undefined
+  const soon = new Set<string>()
+  const ack = (peer: string): void => {
+    const upTo = taken.get(peer)?.upTo
+    owing.delete(peer)
+    said.set(peer, performance.now())
+    // quiet: a lost ack only costs a resend, which is dropped as seen
+    if (upTo !== undefined) put(peer, { acked: upTo } satisfies Acked).catch(() => undefined)
+  }
   const flushAcks = (): void => {
-    clearTimeout(acking)
     acking = undefined
     if (left || ended !== undefined) return
-    for (const peer of owing) {
-      const upTo = taken.get(peer)?.upTo
-      if (upTo === undefined) continue
-      // quiet: a lost ack only costs a resend, which is dropped as seen
-      put(peer, { acked: upTo } satisfies Acked).catch(() => undefined)
-    }
-    owing.clear()
+    for (const peer of [...owing]) ack(peer)
+  }
+  const answer = (): void => {
+    answering = undefined
+    const peers = [...soon]
+    soon.clear()
+    if (left || ended !== undefined) return
+    for (const peer of peers) if (owing.has(peer)) ack(peer)
   }
-  /** `now`: a probe's ack goes at once, so its asker hears it within the loss window. */
-  const ack = (to: string, now = false): void => {
-    owing.add(to)
-    if (now) return flushAcks()
+  const owe = (peer: string): void => {
+    owing.add(peer)
+    if (role !== 'orchestrator') return
+    if (performance.now() - (said.get(peer) ?? 0) >= heartbeatMs) {
+      soon.add(peer)
+      if (answering === undefined) {
+        answering = setTimeout(answer, REACT_MS)
+        answering.unref()
+      }
+      return
+    }
     if (acking !== undefined) return
     acking = setTimeout(flushAcks, heartbeatMs / 2)
     acking.unref()
   }
-  /** The ack owed `to`, if any, for a send to it to carry. */
-  const owed = (to: string): { acked?: number } =>
-    owing.delete(to) ? { acked: taken.get(to)!.upTo } : {}
+  /** When this side last sent each peer anything: a worker quiet a heartbeat says something. */
+  const said = new Map<string, number>()
+  let onQuiet: ((peer: string) => void) | undefined
+  /** The ack owed `to`, if any, for a send to it to carry; and `to` was just said to. */
+  const owed = (to: string): { acked?: number } => {
+    said.set(to, performance.now())
+    return owing.delete(to) ? { acked: taken.get(to)!.upTo } : {}
+  }
   /**
    * When each peer was last heard from: the orchestrator's workers from
    * their `joined`, a worker's orchestrator from its join. One quiet
-   * `PROBE_BEATS` is probed; one quiet `LOST_BEATS` is lost
+   * `LOST_BEATS` is lost
    * (2026-10-23: "if it does not reply for 10s its dead"), on a socket that
    * never closed: the relay loses one whose socket closed and says so.
    */
   const heard = new Map<string, number>()
-  /** `quiet`: an ack, a probe or a resend, which waits on no answer: the relay says only a failure. */
+  /** `quiet`: a beat or a resend, which waits on no answer: the relay says only a failure. */
   const put = (to: string, frame: object, quiet = true): Promise<void> =>
     over((s) => s.send(relayIds.get(to) ?? to, frame, quiet))
   const settle = (from: string, acked: number): void => {
@@ -327,17 +364,23 @@ export async function join(
       const from = heldIds.get(m.from) ?? m.from
       if (heard.has(from)) heard.set(from, performance.now())
       if (isAcked(body)) settle(from, body.acked)
-      else if (isProbe(body)) {
+      else if (isBeat(body)) {
         if (body.acked !== undefined) settle(from, body.acked)
         const t = taken.get(from) ?? { upTo: 0, ahead: new Map() }
         taken.set(from, t)
-        if (body.seq > t.upTo) t.ahead.set(body.seq, PROBED)
+        if (body.seq > t.upTo) t.ahead.set(body.seq, BEAT)
         while (t.ahead.has(t.upTo + 1)) {
           const next = t.ahead.get(++t.upTo)
           t.ahead.delete(t.upTo)
-          if (next !== PROBED) out.push({ from, body: next })
+          if (next !== BEAT) out.push({ from, body: next })
         }
-        ack(from, true)
+        // A beat comes only from a worker quiet `QUIET_BEATS`: half a
+        // heartbeat more would leave it 2 s of `LOST_BEATS` for a stall.
+        if (role === 'orchestrator') {
+          owing.delete(from)
+          // quiet: a lost ack only costs a resend, which is dropped as seen
+          put(from, { acked: t.upTo } satisfies Acked).catch(() => undefined)
+        } else owe(from)
       } else if (isNumbered(body)) {
         if (body.acked !== undefined) settle(from, body.acked)
         let t = taken.get(from)
@@ -346,9 +389,9 @@ export async function join(
         while (t.ahead.has(t.upTo + 1)) {
           const next = t.ahead.get(++t.upTo)
           t.ahead.delete(t.upTo)
-          if (next !== PROBED) out.push({ from, body: next })
+          if (next !== BEAT) out.push({ from, body: next })
         }
-        ack(from)
+        owe(from)
       } else out.push({ from, body })
     }
     return out
@@ -377,46 +420,49 @@ export async function join(
     outbox.delete(peer)
     taken.delete(peer)
     owing.delete(peer)
+    said.delete(peer)
     const now = relayIds.get(peer)
     relayIds.delete(peer)
     if (now !== undefined) heldIds.delete(now)
   }
   /**
-   * Only a quiet peer is probed: the orchestrator a worker quiet
-   * `PROBE_BEATS`, a worker its orchestrator quiet a tenth longer, so in a
-   * quiet pair the orchestrator's probe comes first and the pair trades
-   * one probe and its ack. Either loses a peer quiet `LOST_BEATS`.
+   * A worker says something to its orchestrator whenever it said nothing
+   * `QUIET_BEATS`: its live report (`quiet`), or a beat. The orchestrator
+   * answers with orders and acks only. Either loses a peer quiet `LOST_BEATS`.
    */
   const listen = (): void => {
     // With no socket nobody is heard: the redial or rejoin decides.
     if (socket?.open !== true) return
-    resend(heartbeatMs)
+    resend(RESEND_BEATS * heartbeatMs)
     const now = performance.now()
     // With no peer, the relay is asked instead: a frame it answers.
     if (heard.size === 0 && socket?.open === true && now - socket.heardAt >= heartbeatMs)
       socket.ask()
     for (const [peer, at] of heard) {
-      const quiet = now - at
-      if (quiet >= LOST_BEATS * heartbeatMs) {
+      if (now - at >= LOST_BEATS * heartbeatMs) {
         forget(peer)
         if (peer === 'orchestrator') {
-          end(new RelayError(410, 'relay: the orchestrator went silent'))
+          end(new RelayError(410, ORCHESTRATOR_SILENT))
           return
         }
         inbox.push({ from: 'relay', body: { type: 'lost', id: peer } })
         nudge()
-      } else if (
-        quiet >= (role === 'orchestrator' ? PROBE_BEATS : WORKER_PROBE_BEATS) * heartbeatMs &&
-        // a numbered send's ack may be deferred past the loss line; a probe's is not
-        ![...(outbox.get(peer)?.unacked.values() ?? [])].some((u) => u.body === PROBED)
-      ) {
-        let o = outbox.get(peer)
-        if (o === undefined) outbox.set(peer, (o = { next: 0, unacked: new Map() }))
-        const seq = ++o.next
-        o.unacked.set(seq, { body: PROBED, at: now })
-        // quiet: unacknowledged, it goes again
-        put(peer, { seq, ...owed(peer) } satisfies Probe).catch(() => undefined)
+        continue
+      }
+      // The orchestrator never speaks on a timer: its workers do.
+      if (role === 'orchestrator') continue
+      if (now - (said.get(peer) ?? 0) < QUIET_BEATS * heartbeatMs) continue
+      said.set(peer, now)
+      if (onQuiet !== undefined) {
+        onQuiet(peer)
+        continue
       }
+      let o = outbox.get(peer)
+      if (o === undefined) outbox.set(peer, (o = { next: 0, unacked: new Map() }))
+      const seq = ++o.next
+      o.unacked.set(seq, { body: BEAT, at: now })
+      // quiet: unacknowledged, it goes again
+      put(peer, { seq, ...owed(peer) } satisfies Beat).catch(() => undefined)
     }
   }
   /** Sends again what waited `ms` unacknowledged, to `only` or to all; all of it, at 0. */
@@ -428,7 +474,7 @@ export async function join(
         if (now - u.at < ms) continue
         u.at = now
         // quiet: unacknowledged, it goes again next heartbeat
-        const again = u.body === PROBED ? { seq } : { seq, body: u.body }
+        const again = u.body === BEAT ? { seq } : { seq, body: u.body }
         put(to, { ...again, ...owed(to) }).catch(() => undefined)
       }
     }
@@ -582,8 +628,8 @@ export async function join(
     else again()
   }
   // Every tenth of a heartbeat, so a peer quiet `LOST_BEATS` is lost within
-  // two heartbeats (4 s on the default) of its last word, and what went
-  // unacknowledged a heartbeat goes again before its receiver loses this.
+  // two heartbeats (4 s on the default) of its last word, and a quiet
+  // worker speaks within a tenth of its heartbeat.
   const listening = setInterval(listen, heartbeatMs / 10)
   listening.unref()
   if (role === 'worker') heard.set('orchestrator', performance.now())
@@ -637,6 +683,9 @@ export async function join(
     resumed(f) {
       onResumed = f
     },
+    quiet(f) {
+      onQuiet = f
+    },
     async send(to, body) {
       let o = outbox.get(to)
       if (o === undefined) outbox.set(to, (o = { next: 0, unacked: new Map() }))
@@ -666,7 +715,7 @@ export async function join(
         // One neither the relay nor this side knows waits on nothing, and
         // would be asked every heartbeat for the rest of the run.
         if (err.status === 404 && !heard.has(to)) outbox.delete(to)
-        else o.unacked.set(seq, { body: PROBED, at: 0 })
+        else o.unacked.set(seq, { body: BEAT, at: 0 })
         throw err
       }
     },
@@ -702,7 +751,7 @@ export async function join(
         await said.catch(() => undefined)
       }
       left = true
-      // Ends what still waits on a socket (an ack, a probe): no redial
+      // Ends what still waits on a socket (a send, a beat): no redial
       // would ever bring one, and each waits a heartbeat at a time, for good.
       end(new RelayError(410, 'relay: this participant left'))
       const s = socket
@@ -984,7 +1033,7 @@ async function connect(
       return
     }
     if (m.n === undefined || m.n === null) {
-      // An unnumbered frame's failure: an ack or probe to one the relay no
+      // An unnumbered frame's failure: a beat or resend to one the relay no
       // longer knows (404) is not one, its loss reaching this side apart.
       const failed = [m, ...(m.results ?? [])].filter(
         (a): a is Answer => a?.status !== undefined && a.status !== 404,

package/src/worker.ts

@@ -33,10 +33,10 @@ import {
 import { childrenCpuMs, describeCpu, readPs, topBetween } from './cpu-account.js'
 import {
   bytesOf,
-  HELD_EVERY,
   HOST_ENV,
   PART_LISTS,
   type DepOrder,
+  type WiredDep,
   type Load,
   type WorkerInfo,
   type RunOrder,
@@ -47,7 +47,7 @@ import {
 import { leftUnder, names, outputFiles } from './outputs.js'
 import { snapshot } from './tree.js'
 import { sealWorker, type Proof } from './seal.js'
-import { join, RelayError, unsent as unsentBy } from './wire.js'
+import { join, ORCHESTRATOR_SILENT, RelayError, unsent as unsentBy } from './wire.js'
 
 export async function runWorker(opts: {
   relay: string
@@ -115,7 +115,9 @@ export async function runWorker(opts: {
   const artifacts = opts.artifacts(() => facts)
   const running = new Map<string, AbortController>()
   /** Each order's deps sent ahead of its batch, until the batch comes. */
-  const ahead = new Map<string, DepOrder[]>()
+  const ahead = new Map<string, WiredDep[]>()
+  /** Each dependency sent whole, by its task: an order after names it alone (`WiredDep`). */
+  const known = new Map<string, DepOrder>()
   const work = new Set<Promise<void>>()
   let root = opts.root
   let realRoot = opts.root
@@ -152,7 +154,27 @@ export async function runWorker(opts: {
   const onTerm = onSignal('SIGTERM')
   process.once('SIGINT', onInt)
   process.once('SIGTERM', onTerm)
+  /** Aborted as this worker leaves: what runs here then is stopped, not ended. */
+  const leaving = new AbortController()
   const quitCode = (): number => 128 + constants.signals[quit.signal.reason as 'SIGINT' | 'SIGTERM']
+  /**
+   * The run is over for this worker (410), its job no failure. An end on
+   * the orchestrator's silence stops what runs here, and is said: three
+   * workers' every task stopped in one second, each job green and silent,
+   * and the cause was found only in the run's profile (CI, 2026-10-29).
+   */
+  const ended = (err: RelayError): number => {
+    if (err.message === ORCHESTRATOR_SILENT)
+      ui.print(
+        'err',
+        ui.status(
+          'warn',
+          'vx dte worker',
+          'the orchestrator went silent: the tasks running here were stopped, and the run gives them to other workers',
+        ),
+      )
+    return 0
+  }
 
   /** Saves the task's files (none when `bare`) and output under `key` and says so. */
   const reply = async (
@@ -162,9 +184,11 @@ export async function runWorker(opts: {
     res: WiredResult & { output: CapturedChunk[] },
     bare = false,
   ): Promise<void> => {
-    // Stopped by a signal: its end would fail the task in the run, which
-    // runs it again elsewhere once this worker leaves.
-    if (quit.signal.aborted) return
+    // Stopped by a signal, or by this worker leaving (its orchestrator went
+    // silent): its end would fail the task in the run, which runs it again
+    // elsewhere once this worker leaves. One such end reached the
+    // orchestrator and failed the run (2026-11-05).
+    if (quit.signal.aborted || leaving.signal.aborted) return
     const { output, ...result } = res
     // Forced: the key keeps an entry it holds, so the run's machines restore
     // the fresh bytes under the run key.
@@ -205,26 +229,29 @@ export async function runWorker(opts: {
         true,
       )
     }
-    // No remote took it: the orchestrator reads the remotes alone, so its
-    // dependants come here and its output goes with the `end` (2026-10-25).
+    // No remote took it after its tries: the run's other machines read the
+    // remotes alone, so the task is this worker's fault, run elsewhere.
     const saved = keys.at(-1)!
-    const wired: WiredResult =
-      (await artifacts.uploaded?.(saved)) === false
-        ? { ...result, unshared: true, output: pieces(output) }
-        : result
+    if ((await artifacts.uploaded?.(saved)) === false)
+      throw new Error(`no remote cache took its entry ${saved} (this worker's log says why)`)
     if (result.exitCode === 0)
       for (const k of keys) {
         present.set(k, Promise.resolve())
         holds.set(k, names(m.project, m.outputs))
       }
+    const endedAt = Date.now()
+    const t = times.get(m.task)
     const end: ToOrchestrator = {
       type: 'end',
       task: m.task,
       key: saved,
-      result: wired,
+      result,
+      startedAt: t?.startedAt ?? endedAt,
+      ranAt: t?.ranAt ?? endedAt,
+      endedAt,
       load: frame(),
     }
-    for (const part of parts(end)) await me.send('orchestrator', part)
+    for (const part of parts(end)) await say(part)
   }
 
   // A message the relay would not take is said, each kind once (2026-10-23):
@@ -240,15 +267,47 @@ export async function runWorker(opts: {
       what,
       'the orchestrator',
     )
+  /** Each running task's times: begun, and its command's end once it ran. */
+  const times = new Map<string, { startedAt: number; ranAt?: number }>()
+  /** Tasks whose command ended, not said yet: each rides the next message. */
+  const ran = new Set<string>()
+  /** The last batch read: its report names what it runs as of that batch. */
+  let batch = 0
+  /**
+   * `m` to the orchestrator in one frame with its machine's load, which
+   * every frame carries, and each waiting `ran`: none is ever sent alone.
+   */
+  const say = (event: ToOrchestrator): Promise<void> => {
+    const m = { ...event, load: { ...frame(), at: Date.now() } } as ToOrchestrator
+    // Its `end` says it ran.
+    if (m.type === 'end') ran.delete(m.task)
+    const riding: ToOrchestrator[] = []
+    for (const task of ran) {
+      const t = times.get(task)
+      if (t?.ranAt !== undefined)
+        riding.push({ type: 'ran', task, startedAt: t.startedAt, ranAt: t.ranAt })
+    }
+    ran.clear()
+    return me.send(
+      'orchestrator',
+      riding.length === 0 ? m : { type: 'together', messages: [...riding, m] },
+    )
+  }
+  // A heartbeat with nothing said: its live report, which is how the
+  // orchestrator hears it. Never a second in one: any message carries load.
+  me.quiet(() => {
+    void say({ type: 'report', running: [...running.keys()], batch, load: frame() }).catch(
+      unsent('report'),
+    )
+  })
   /** Runs a task the orchestrator sent: it fits what this worker has free. */
   const start = (m: RunOrder): void => {
-    let ran = false
+    const t: { startedAt: number; ranAt?: number } = { startedAt: Date.now() }
+    times.set(m.task, t)
     const done = (): void => {
-      if (ran) return
-      ran = true
-      void me
-        .send('orchestrator', { type: 'ran', task: m.task, load: frame() })
-        .catch(unsent('ran'))
+      if (t.ranAt !== undefined) return
+      t.ranAt = Date.now()
+      ran.add(m.task)
     }
     const p = runTask(m, done)
       .catch(async (err: unknown) => {
@@ -268,11 +327,13 @@ export async function runWorker(opts: {
         }
         const reason = err instanceof Error ? err.message : String(err)
         ui.print('err', ui.status('error', m.task, `could not run: ${reason}`))
-        await me
-          .send('orchestrator', { type: 'fault', task: m.task, reason })
-          .catch(unsent('fault'))
+        ran.delete(m.task)
+        await say({ type: 'fault', task: m.task, reason }).catch(unsent('fault'))
+      })
+      .finally(() => {
+        work.delete(p)
+        times.delete(m.task)
       })
-      .finally(() => work.delete(p))
     work.add(p)
   }
 
@@ -302,10 +363,6 @@ export async function runWorker(opts: {
   const runTask = async (m: RunOrder, free: () => void): Promise<void> => {
     const stop = new AbortController()
     running.set(m.task, stop)
-    // Not awaited: the task does not wait on the relay to say it began.
-    void me
-      .send('orchestrator', { type: 'start', task: m.task, load: frame() })
-      .catch(unsent('start'))
     try {
       const differs = await runtimeDiffers(m)
       if (differs !== undefined) {
@@ -386,6 +443,8 @@ export async function runWorker(opts: {
                 ? undefined
                 : new Set(m.sandbox.keyed.map((k) => path.join(opts.root, k))),
               m.nested.map((n) => path.join(opts.root, n)),
+              undefined,
+              (m.sandbox.reads ?? []).map((r) => path.join(opts.root, r)),
             )
       const output = new BoundedCapture()
       const req: ExecuteRequest = {
@@ -510,13 +569,6 @@ export async function runWorker(opts: {
     }
   }
 
-  // What it holds, every `HELD_EVERY` heartbeats: the orchestrator's lease on each task.
-  const held = setInterval(() => {
-    if (running.size === 0) return
-    void me
-      .send('orchestrator', { type: 'held', running: [...running.keys()], load: frame() })
-      .catch(unsent('held'))
-  }, HELD_EVERY * me.heartbeatMs)
   try {
     for (;;) {
       let messages
@@ -524,7 +576,7 @@ export async function runWorker(opts: {
         messages = await me.poll(quit.signal)
       } catch (err) {
         if (quit.signal.aborted) return quitCode()
-        if (err instanceof RelayError && err.status === 410) return 0
+        if (err instanceof RelayError && err.status === 410) return ended(err)
         // The relay lost it (silent too long, or restarted): the run gave its
         // tasks to others, so this job has nothing left and is no failure.
         if (err instanceof RelayError && err.status === 404) {
@@ -568,30 +620,47 @@ export async function runWorker(opts: {
           // Every batch of a run names the same orchestrator root.
           root = m.root
           realRoot = m.realRoot
+          batch = m.n
           if (m.sandboxes !== undefined) {
             if (floor === undefined) floor = localFloor(m.sandboxes)
             else await floor.rearm(m.sandboxes)
           }
           for (const t of m.tasks) {
-            // One it runs already, sent again (its lease ended while the
-            // relay was away): it is not run twice.
-            if (running.has(t.task)) continue
-            const deps = ahead.get(t.task)
+            const sent = ahead.get(t.task)
             ahead.delete(t.task)
-            start(deps === undefined ? t : { ...t, deps: [...deps, ...t.deps] })
+            // Learned in the order sent, a task run already too: the run counts them sent.
+            let missing: string | undefined
+            const deps = (sent === undefined ? t.deps : [...sent, ...t.deps]).map((d): DepOrder => {
+              if (typeof d !== 'string') {
+                known.set(d.task, d)
+                return d
+              }
+              const got = known.get(d)
+              if (got === undefined) missing ??= d
+              return got!
+            })
+            // One it runs already is not run twice.
+            if (running.has(t.task)) continue
+            if (missing !== undefined) {
+              const reason = `was sent ${missing} by name, never whole`
+              ui.print('err', ui.status('error', t.task, `could not run: ${reason}`))
+              await say({ type: 'fault', task: t.task, reason }).catch(unsent('fault'))
+              continue
+            }
+            start({ ...t, deps })
           }
         }
       }
     }
   } catch (err) {
     // The run closed under a poll or an answer: nothing is left for this worker.
-    if (err instanceof RelayError && err.status === 410) return 0
+    if (err instanceof RelayError && err.status === 410) return ended(err)
     throw err
   } finally {
     process.off('SIGINT', onInt)
     process.off('SIGTERM', onTerm)
-    clearInterval(held)
     trees.stop()
+    leaving.abort()
     for (const c of running.values()) c.abort()
     await Promise.all(work)
     const onlineMs = performance.now() - started.at
@@ -988,25 +1057,6 @@ const gb = (n: number): string => `${(n / 1e9).toFixed(1)} GB`
 /** The most of a result one send carries: a relay frame is 1 MiB, which sealing grows by a third. */
 const PART_BYTES = 256 * 1024
 
-/** `output` cut so each chunk fits a part, escaped as JSON grows it up to six times. */
-function pieces(output: readonly CapturedChunk[]): CapturedChunk[] {
-  const most = PART_BYTES / 8
-  return output.flatMap((c) => {
-    if (c.text.length <= most) return [c]
-    const cut: CapturedChunk[] = []
-    for (let i = 0; i < c.text.length;) {
-      let end = Math.min(i + most, c.text.length)
-      // Not inside a surrogate pair: each half replayed alone printed as
-      // two replacement characters (2026-10-25).
-      const last = c.text.charCodeAt(end - 1)
-      if (end < c.text.length && last >= 0xd800 && last <= 0xdbff) end--
-      cut.push({ ...c, text: c.text.slice(i, end) })
-      i = end
-    }
-    return cut
-  })
-}
-
 /**
  * `end` as its sends: itself when it fits one frame, else its long lists
  * first, in parts that each fit, and then the rest of it.