@vzn/vx-dte 0.0.649 → 0.0.651

package/src/worker.ts

@@ -33,7 +33,6 @@ import {
 import { childrenCpuMs, describeCpu, readPs, topBetween } from './cpu-account.js'
 import {
   bytesOf,
-  HELD_EVERY,
   HOST_ENV,
   PART_LISTS,
   type DepOrder,
@@ -47,7 +46,7 @@ import {
 import { leftUnder, names, outputFiles } from './outputs.js'
 import { snapshot } from './tree.js'
 import { sealWorker, type Proof } from './seal.js'
-import { join, RelayError, unsent as unsentBy } from './wire.js'
+import { join, ORCHESTRATOR_SILENT, RelayError, unsent as unsentBy } from './wire.js'
 
 export async function runWorker(opts: {
   relay: string
@@ -152,7 +151,27 @@ export async function runWorker(opts: {
   const onTerm = onSignal('SIGTERM')
   process.once('SIGINT', onInt)
   process.once('SIGTERM', onTerm)
+  /** Aborted as this worker leaves: what runs here then is stopped, not ended. */
+  const leaving = new AbortController()
   const quitCode = (): number => 128 + constants.signals[quit.signal.reason as 'SIGINT' | 'SIGTERM']
+  /**
+   * The run is over for this worker (410), its job no failure. An end on
+   * the orchestrator's silence stops what runs here, and is said: three
+   * workers' every task stopped in one second, each job green and silent,
+   * and the cause was found only in the run's profile (CI, 2026-10-29).
+   */
+  const ended = (err: RelayError): number => {
+    if (err.message === ORCHESTRATOR_SILENT)
+      ui.print(
+        'err',
+        ui.status(
+          'warn',
+          'vx dte worker',
+          'the orchestrator went silent: the tasks running here were stopped, and the run gives them to other workers',
+        ),
+      )
+    return 0
+  }
 
   /** Saves the task's files (none when `bare`) and output under `key` and says so. */
   const reply = async (
@@ -162,9 +181,11 @@ export async function runWorker(opts: {
     res: WiredResult & { output: CapturedChunk[] },
     bare = false,
   ): Promise<void> => {
-    // Stopped by a signal: its end would fail the task in the run, which
-    // runs it again elsewhere once this worker leaves.
-    if (quit.signal.aborted) return
+    // Stopped by a signal, or by this worker leaving (its orchestrator went
+    // silent): its end would fail the task in the run, which runs it again
+    // elsewhere once this worker leaves. One such end reached the
+    // orchestrator and failed the run (2026-11-05).
+    if (quit.signal.aborted || leaving.signal.aborted) return
     const { output, ...result } = res
     // Forced: the key keeps an entry it holds, so the run's machines restore
     // the fresh bytes under the run key.
@@ -205,26 +226,29 @@ export async function runWorker(opts: {
         true,
       )
     }
-    // No remote took it: the orchestrator reads the remotes alone, so its
-    // dependants come here and its output goes with the `end` (2026-10-25).
+    // No remote took it after its tries: the run's other machines read the
+    // remotes alone, so the task is this worker's fault, run elsewhere.
     const saved = keys.at(-1)!
-    const wired: WiredResult =
-      (await artifacts.uploaded?.(saved)) === false
-        ? { ...result, unshared: true, output: pieces(output) }
-        : result
+    if ((await artifacts.uploaded?.(saved)) === false)
+      throw new Error(`no remote cache took its entry ${saved} (this worker's log says why)`)
     if (result.exitCode === 0)
       for (const k of keys) {
         present.set(k, Promise.resolve())
         holds.set(k, names(m.project, m.outputs))
       }
+    const endedAt = Date.now()
+    const t = times.get(m.task)
     const end: ToOrchestrator = {
       type: 'end',
       task: m.task,
       key: saved,
-      result: wired,
+      result,
+      startedAt: t?.startedAt ?? endedAt,
+      ranAt: t?.ranAt ?? endedAt,
+      endedAt,
       load: frame(),
     }
-    for (const part of parts(end)) await me.send('orchestrator', part)
+    for (const part of parts(end)) await say(part)
   }
 
   // A message the relay would not take is said, each kind once (2026-10-23):
@@ -240,15 +264,47 @@ export async function runWorker(opts: {
       what,
       'the orchestrator',
     )
+  /** Each running task's times: begun, and its command's end once it ran. */
+  const times = new Map<string, { startedAt: number; ranAt?: number }>()
+  /** Tasks whose command ended, not said yet: each rides the next message. */
+  const ran = new Set<string>()
+  /** The last batch read: its report names what it runs as of that batch. */
+  let batch = 0
+  /**
+   * `m` to the orchestrator in one frame with its machine's load, which
+   * every frame carries, and each waiting `ran`: none is ever sent alone.
+   */
+  const say = (event: ToOrchestrator): Promise<void> => {
+    const m = { ...event, load: { ...frame(), at: Date.now() } } as ToOrchestrator
+    // Its `end` says it ran.
+    if (m.type === 'end') ran.delete(m.task)
+    const riding: ToOrchestrator[] = []
+    for (const task of ran) {
+      const t = times.get(task)
+      if (t?.ranAt !== undefined)
+        riding.push({ type: 'ran', task, startedAt: t.startedAt, ranAt: t.ranAt })
+    }
+    ran.clear()
+    return me.send(
+      'orchestrator',
+      riding.length === 0 ? m : { type: 'together', messages: [...riding, m] },
+    )
+  }
+  // A heartbeat with nothing said: its live report, which is how the
+  // orchestrator hears it. Never a second in one: any message carries load.
+  me.quiet(() => {
+    void say({ type: 'report', running: [...running.keys()], batch, load: frame() }).catch(
+      unsent('report'),
+    )
+  })
   /** Runs a task the orchestrator sent: it fits what this worker has free. */
   const start = (m: RunOrder): void => {
-    let ran = false
+    const t: { startedAt: number; ranAt?: number } = { startedAt: Date.now() }
+    times.set(m.task, t)
     const done = (): void => {
-      if (ran) return
-      ran = true
-      void me
-        .send('orchestrator', { type: 'ran', task: m.task, load: frame() })
-        .catch(unsent('ran'))
+      if (t.ranAt !== undefined) return
+      t.ranAt = Date.now()
+      ran.add(m.task)
     }
     const p = runTask(m, done)
       .catch(async (err: unknown) => {
@@ -268,11 +324,13 @@ export async function runWorker(opts: {
         }
         const reason = err instanceof Error ? err.message : String(err)
         ui.print('err', ui.status('error', m.task, `could not run: ${reason}`))
-        await me
-          .send('orchestrator', { type: 'fault', task: m.task, reason })
-          .catch(unsent('fault'))
+        ran.delete(m.task)
+        await say({ type: 'fault', task: m.task, reason }).catch(unsent('fault'))
+      })
+      .finally(() => {
+        work.delete(p)
+        times.delete(m.task)
       })
-      .finally(() => work.delete(p))
     work.add(p)
   }
 
@@ -302,10 +360,6 @@ export async function runWorker(opts: {
   const runTask = async (m: RunOrder, free: () => void): Promise<void> => {
     const stop = new AbortController()
     running.set(m.task, stop)
-    // Not awaited: the task does not wait on the relay to say it began.
-    void me
-      .send('orchestrator', { type: 'start', task: m.task, load: frame() })
-      .catch(unsent('start'))
     try {
       const differs = await runtimeDiffers(m)
       if (differs !== undefined) {
@@ -510,13 +564,6 @@ export async function runWorker(opts: {
     }
   }
 
-  // What it holds, every `HELD_EVERY` heartbeats: the orchestrator's lease on each task.
-  const held = setInterval(() => {
-    if (running.size === 0) return
-    void me
-      .send('orchestrator', { type: 'held', running: [...running.keys()], load: frame() })
-      .catch(unsent('held'))
-  }, HELD_EVERY * me.heartbeatMs)
   try {
     for (;;) {
       let messages
@@ -524,7 +571,7 @@ export async function runWorker(opts: {
         messages = await me.poll(quit.signal)
       } catch (err) {
         if (quit.signal.aborted) return quitCode()
-        if (err instanceof RelayError && err.status === 410) return 0
+        if (err instanceof RelayError && err.status === 410) return ended(err)
         // The relay lost it (silent too long, or restarted): the run gave its
         // tasks to others, so this job has nothing left and is no failure.
         if (err instanceof RelayError && err.status === 404) {
@@ -568,13 +615,13 @@ export async function runWorker(opts: {
           // Every batch of a run names the same orchestrator root.
           root = m.root
           realRoot = m.realRoot
+          batch = m.n
           if (m.sandboxes !== undefined) {
             if (floor === undefined) floor = localFloor(m.sandboxes)
             else await floor.rearm(m.sandboxes)
           }
           for (const t of m.tasks) {
-            // One it runs already, sent again (its lease ended while the
-            // relay was away): it is not run twice.
+            // One it runs already is not run twice.
             if (running.has(t.task)) continue
             const deps = ahead.get(t.task)
             ahead.delete(t.task)
@@ -585,13 +632,13 @@ export async function runWorker(opts: {
     }
   } catch (err) {
     // The run closed under a poll or an answer: nothing is left for this worker.
-    if (err instanceof RelayError && err.status === 410) return 0
+    if (err instanceof RelayError && err.status === 410) return ended(err)
     throw err
   } finally {
     process.off('SIGINT', onInt)
     process.off('SIGTERM', onTerm)
-    clearInterval(held)
     trees.stop()
+    leaving.abort()
     for (const c of running.values()) c.abort()
     await Promise.all(work)
     const onlineMs = performance.now() - started.at
@@ -988,25 +1035,6 @@ const gb = (n: number): string => `${(n / 1e9).toFixed(1)} GB`
 /** The most of a result one send carries: a relay frame is 1 MiB, which sealing grows by a third. */
 const PART_BYTES = 256 * 1024
 
-/** `output` cut so each chunk fits a part, escaped as JSON grows it up to six times. */
-function pieces(output: readonly CapturedChunk[]): CapturedChunk[] {
-  const most = PART_BYTES / 8
-  return output.flatMap((c) => {
-    if (c.text.length <= most) return [c]
-    const cut: CapturedChunk[] = []
-    for (let i = 0; i < c.text.length;) {
-      let end = Math.min(i + most, c.text.length)
-      // Not inside a surrogate pair: each half replayed alone printed as
-      // two replacement characters (2026-10-25).
-      const last = c.text.charCodeAt(end - 1)
-      if (end < c.text.length && last >= 0xd800 && last <= 0xdbff) end--
-      cut.push({ ...c, text: c.text.slice(i, end) })
-      i = end
-    }
-    return cut
-  })
-}
-
 /**
  * `end` as its sends: itself when it fits one frame, else its long lists
  * first, in parts that each fit, and then the rest of it.