@vzn/vx-dte 0.0.650 → 0.0.651

package/src/orchestrator.ts

@@ -28,7 +28,6 @@ import {
 } from '@vzn/vx'
 import {
   bytesOf,
-  HELD_EVERY,
   HOST_ENV,
   PART_LISTS,
   type PartList,
@@ -65,11 +64,13 @@ interface Job {
   booked?: number
   /** The workers that failed it after beginning it, with why: two fail it. */
   faults?: string[]
-  /** Its worker's lease on it ends then unless the worker says it still holds it. */
-  leaseUntil?: number
+  /** Its worker's `end` came: a report naming it no more is not a loss. */
+  ended?: true
   /** When it was sent to its worker. */
   sentAt?: number
-  /** When its worker began it. */
+  /** The batch that sent it, by its worker's count: a report of that batch or later names it. */
+  batch?: number
+  /** When its worker began it: its send, then the time its worker says. */
   startedAt?: number
   /** When its worker said its command ended: it saves since. */
   ranAt?: number
@@ -112,8 +113,10 @@ interface Worker {
   readonly usage: Usage
   /** Sent to it and not finished: running there or in its queue. */
   readonly busy: Map<string, Job>
-  /** Of `busy`, the ones it began. */
+  /** Of `busy`, the ones it began: each one sent, as a worker runs what it takes at once. */
   readonly started: Set<string>
+  /** Batches sent it: each batch's `n`. */
+  batches: number
   usedCores: number
   usedMemory: number
   /** Its machine's use as its last frame said, and when this machine heard it. */
@@ -164,7 +167,9 @@ export interface OrchestratorOptions {
   readonly joined?: () => void
   /** Worker `id` did not join within twice its size's first join: given up, for its adapter to stop. */
   readonly overdue?: (id: string) => void
-  /** A worker failed a task or its lease: the run plans again, now, so a replacement starts. */
+  /** Worker `id`, given up as late (`overdue`), joined after all and is taken. */
+  readonly kept?: (id: string) => void
+  /** A worker failed a task or dropped it: the run plans again, now, so a replacement starts. */
   readonly faulted?: () => void
   /** How much of the run waits on a task (`tailsMs`): the queue places the most first. */
   readonly tail?: (taskId: string) => number
@@ -361,14 +366,6 @@ export class Orchestrator {
         return o.root
       }
     })()
-    // Leases ran on while either side was away, as a relay restarted: what
-    // a worker holds is still its own, not run again elsewhere
-    // (2026-10-25: "make sure on rejoin we don't repeat work").
-    o.me.resumed((peer) => {
-      for (const a of this.workers.values())
-        if (peer === undefined || a.id === peer)
-          for (const j of a.busy.values()) if (j.leaseUntil !== undefined) this.lease(j)
-    })
     this.loop = this.listen()
   }
 
@@ -975,8 +972,11 @@ export class Orchestrator {
             }
             tasks.push({ ...order, deps: [] })
           }
+          const n = ++a.batches
+          for (const { job } of part) job.batch = n
           const batch = {
             type: 'batch',
+            n,
             root: this.o.root,
             realRoot: this.realRoot,
             tasks,
@@ -986,9 +986,11 @@ export class Orchestrator {
           a.usage.messagesIn++
           a.usage.bytesIn += bytesOf(batch)
           for (const { job } of part) {
-            this.lease(job)
             pending.delete(job)
+            // Sent, it runs: its worker takes only what it has room for.
+            this.begun(a, job.req.taskId)
           }
+          this.pumpSoon()
         }
       })
       .catch((err: unknown) => {
@@ -1006,13 +1008,8 @@ export class Orchestrator {
       })
   }
 
-  /** Three of the worker's `held` from now. */
-  private lease(job: Job): void {
-    job.leaseUntil = performance.now() + 3 * HELD_EVERY * this.o.me.heartbeatMs
-  }
-
   /**
-   * Worker `a` let `job` down: lost, its lease ended, or it could not run
+   * Worker `a` let `job` down: lost, its report named it no more, or it could not run
    * it. One it had begun counts, and two workers fail it; one still in its
    * queue goes back free.
    */
@@ -1032,9 +1029,10 @@ export class Orchestrator {
     a.has.delete(job.req.taskId)
     this.unhold(a, job)
     delete job.worker
-    delete job.leaseUntil
     delete job.sentAt
+    delete job.ended
     delete job.ranAt
+    delete job.batch
     const begun = a.started.delete(job.req.taskId)
     // Stopped, it runs nowhere again; the cancel sent its worker has no one to answer it.
     if (job.req.signal?.aborted === true) {
@@ -1078,25 +1076,6 @@ export class Orchestrator {
     this.o.stalled?.(lines.join('\n'))
   }
 
-  /** Each task whose worker's lease on it ended goes back. */
-  private expire(): void {
-    // With no socket no `held` arrives: the leases wait for it (`resumed`).
-    if (!this.o.me.online) return
-    const now = performance.now()
-    const back: Job[] = []
-    for (const a of this.workers.values())
-      for (const j of [...a.busy.values()])
-        if (j.leaseUntil !== undefined && j.leaseUntil < now) {
-          const again = this.retry(a, j, 'lease expired')
-          if (again !== undefined) back.push(again)
-        }
-    if (back.length === 0) return
-    this.queue.unshift(...back)
-    this.o.faulted?.()
-    if (this.spawnedAt.size > 0) this.stranded(`a task's lease ended and no worker is left`)
-    this.pumpSoon()
-  }
-
   private post(a: Worker, m: ToWorker): void {
     a.chain = a.chain.then(() => this.o.me.send(a.id, m)).catch(this.unsent(m.type, a.label))
   }
@@ -1118,16 +1097,19 @@ export class Orchestrator {
   }
 
   /** Worker `a` ran `task` and saved it under `key`: it is done there, and done here once restored. */
-  private async finish(a: Worker, task: string, key: string, head: WiredResult): Promise<void> {
+  private async finish(
+    a: Worker,
+    task: string,
+    key: string,
+    head: WiredResult,
+    endedAt: number | undefined,
+  ): Promise<void> {
     const job = a.busy.get(task)
     if (job === undefined) return
     // A dependency's remote copy did not read back there (a truncated
     // upload): sent again from this machine, the task runs again. Taken as
     // the task's failure, it failed every run, the remote saying it held the
     // key (2026-10-25).
-    // Ended there, it is in no `held` again: a lease running on through a
-    // slow resend called its worker faulty and gave the task to another.
-    delete job.leaseUntil
     if (head.unread !== undefined) {
       const sent = await this.resend(job, head.unread)
       // Its worker was lost meanwhile: the task went back to the queue.
@@ -1145,7 +1127,7 @@ export class Orchestrator {
         return
       }
     }
-    const endAt = performance.now()
+    const endAt = endedAt ?? performance.now()
     this.traceTask(a, job, head, endAt)
     a.busy.delete(task)
     this.unhold(a, job)
@@ -1181,10 +1163,7 @@ export class Orchestrator {
     }
     this.savedAs.set(task, key)
     const node = this.o.nodes()?.get(task)
-    // No remote took its upload (too big, a 429): its files are there
-    // alone, as an uncached task's are, and its output came with its `end`.
-    const unshared = head.unshared === true
-    if (head.exitCode === 0 && (unshared || (node !== undefined && leaves(node))))
+    if (head.exitCode === 0 && node !== undefined && leaves(node))
       this.kept.set(task, { worker: a.id, req: job.req, need: job.need })
     this.pumpSoon()
     if (job.again !== undefined) return job.resolve(result(head, []))
@@ -1202,17 +1181,14 @@ export class Orchestrator {
     const deferred = job.req.remoteOnly !== true && job.req.download === 'deferred'
     const read = () =>
       job.req.remoteOnly === true || deferred ? this.o.artifacts.output(key) : restore()
-    const log = await (
-      unshared
-        ? Promise.resolve(wiredLog(head.output))
-        : // A failed download reads as a miss: asked once more, since one
-          // stalled download failed a CI run whose upload had landed
-          // (main, 2026-10-28).
-          read().then((got) => got ?? read())
-    ).catch((err: unknown) => {
-      job.reject(err)
-      return undefined
-    })
+    // A failed download reads as a miss: asked once more, since one stalled
+    // download failed a CI run whose upload had landed (main, 2026-10-28).
+    const log = await read()
+      .then((got) => got ?? read())
+      .catch((err: unknown) => {
+        job.reject(err)
+        return undefined
+      })
     this.trace.slice({
       track: ORCHESTRATOR,
       name: task,
@@ -1224,14 +1200,14 @@ export class Orchestrator {
     if (log === null) {
       job.reject(
         new Error(
-          `${task}: ${a.label} ran it, and no cache this machine reads gave its entry, asked twice: a download failed (warned above), ${a.label}'s upload failed (its log says why), or the workers and this machine share no remote cache the workers write`,
+          `${task}: ${a.label} ran it, and no cache this machine reads gave its entry, asked twice: a download failed (warned above), or the workers and this machine share no remote cache the workers write`,
         ),
       )
       return
     }
     // An upstream's file the task removed there is gone here too: a restore
     // lays files, never removes one, and this disk is what core saves from.
-    if (head.removed !== undefined && job.req.remoteOnly !== true && !deferred && !unshared) {
+    if (head.removed !== undefined && job.req.remoteOnly !== true && !deferred) {
       const ours = names(
         path.relative(this.o.root, node?.projectDir ?? job.req.cwd),
         job.req.outputs,
@@ -1267,29 +1243,17 @@ export class Orchestrator {
     for (const c of log) (c.err ? job.req.onStderr : job.req.onStdout)(c.text)
     job.resolve({
       ...result(head, log),
-      ...(unshared
+      ...(deferred
         ? {
             outputs: {
               kind: 'deferred' as const,
-              materialize: () =>
-                Promise.reject(
-                  new UserError(
-                    `${task}: its files are on ${a.label} alone: no remote cache took its upload (its log says why)`,
-                  ),
-                ),
+              materialize: async () => {
+                if ((await restore()) === null)
+                  throw new Error(`${task}'s entry ${key} is in no cache this machine reads`)
+              },
             },
           }
-        : deferred
-          ? {
-              outputs: {
-                kind: 'deferred' as const,
-                materialize: async () => {
-                  if ((await restore()) === null)
-                    throw new Error(`${task}'s entry ${key} is in no cache this machine reads`)
-                },
-              },
-            }
-          : {}),
+        : {}),
       where: a.label,
       machine: {
         cores: a.reported.cores,
@@ -1374,6 +1338,7 @@ export class Orchestrator {
       if (info.id !== undefined && this.late.delete(info.id)) {
         this.cancelled.delete(info.id)
         this.unstarted.delete(info.id)
+        this.o.kept?.(info.id)
       }
       if (info.id !== undefined && this.cancelled.has(info.id)) {
         this.turnedAway.add(info.id)
@@ -1422,6 +1387,7 @@ export class Orchestrator {
         label,
         busy: new Map(),
         started: new Set(),
+        batches: 0,
         has: new Set(),
         // Its name first, before any batch.
         chain: this.cache()
@@ -1487,67 +1453,99 @@ export class Orchestrator {
         }
         const a = this.workers.get(from)
         if (a === undefined) continue
-        const m = body as ToOrchestrator
-        if ('load' in m) this.heard(a, m.load)
-        if (m.type !== 'held') this.movedAt = performance.now()
-        if (m.type === 'start') {
-          this.begun(a, m.task)
-          this.pumpSoon()
-        } else if (m.type === 'held') {
-          for (const t of m.running) {
-            const j = a.busy.get(t)
-            if (j?.leaseUntil !== undefined) this.lease(j)
-            this.begun(a, t)
-          }
-          this.pumpSoon()
-        } else if (m.type === 'ran') {
-          // Its command ended: its cores are the next task's while it saves.
-          const j = a.busy.get(m.task)
-          if (j === undefined) continue
-          this.begun(a, m.task)
-          this.unhold(a, j)
-          // Its worker still says it holds it while it saves.
-          j.ranAt = performance.now()
-          this.pumpSoon()
-        } else if (m.type === 'fault') {
-          const j = a.busy.get(m.task)
-          if (j === undefined) continue
-          // It began it whether or not its `start` came first.
-          this.begun(a, m.task)
-          if (this.retry(a, j, m.reason) !== undefined) this.queue.unshift(j)
-          if (a.faulty) this.o.faulted?.()
-          if (a.faulty && this.spawnedAt.size > 0)
-            this.stranded(`${a.label} could not run ${m.task} (${m.reason}) and no worker is left`)
-          this.pumpSoon()
-        } else if (m.type === 'part') {
-          a.usage.messagesOut++
-          a.usage.bytesOut += bytesOf(m)
-          const k = `${from}\0${m.task}`
-          const held = this.parts.get(k) ?? {}
-          for (const list of PART_LISTS) {
-            const items = m[list] as readonly unknown[] | undefined
-            if (items !== undefined) held[list] = [...(held[list] ?? []), ...items]
-          }
-          this.parts.set(k, held)
-        } else if (m.type === 'end') {
-          a.usage.messagesOut++
-          a.usage.bytesOut += bytesOf(m)
-          const k = `${from}\0${m.task}`
-          const held = this.parts.get(k)
-          this.parts.delete(k)
-          const result: Record<string, unknown> = { ...m.result }
-          for (const list of PART_LISTS) {
-            const items = m.result[list] as readonly unknown[] | undefined
-            if (held?.[list] !== undefined) result[list] = [...held[list], ...(items ?? [])]
+        const said = body as ToOrchestrator
+        const frame = said.type === 'together' ? said.messages : [said]
+        // A worker's times, read against its clock as this frame left it:
+        // no two machines' clocks agree.
+        const now = performance.now()
+        let at: number | undefined
+        for (const m of frame) if ('load' in m && m.load?.at !== undefined) at = Number(m.load.at)
+        const here = (t: unknown): number | undefined =>
+          at === undefined || typeof t !== 'number' || !Number.isFinite(at - t)
+            ? undefined
+            : Math.min(now, now - (at - t))
+        for (const m of frame) {
+          if ('load' in m && m.load !== undefined) this.heard(a, m.load)
+          if (m.type !== 'report') this.movedAt = performance.now()
+          if (m.type === 'report') {
+            // Every task of the batches it read that has not ended is in its
+            // report: one missing it no longer runs, and goes back at once.
+            const running = new Set(m.running)
+            const read = Number(m.batch) || 0
+            const back: Job[] = []
+            for (const [t, j] of [...a.busy])
+              if (
+                a.started.has(t) &&
+                j.ended !== true &&
+                (j.batch ?? 0) <= read &&
+                !running.has(t)
+              ) {
+                const again = this.retry(a, j, 'not in its report')
+                if (again !== undefined) back.push(again)
+              }
+            if (back.length > 0) {
+              this.queue.unshift(...back)
+              this.o.faulted?.()
+              if (this.spawnedAt.size > 0)
+                this.stranded(`${a.label} no longer runs a task it began and no worker is left`)
+            }
+            this.pumpSoon()
+          } else if (m.type === 'ran') {
+            // Its command ended: its cores are the next task's while it saves.
+            const j = a.busy.get(m.task)
+            if (j === undefined) continue
+            this.begun(a, m.task)
+            this.unhold(a, j)
+            // Its worker still says it holds it while it saves.
+            j.startedAt = here(m.startedAt) ?? j.startedAt ?? now
+            j.ranAt = here(m.ranAt) ?? now
+            this.pumpSoon()
+          } else if (m.type === 'fault') {
+            const j = a.busy.get(m.task)
+            if (j === undefined) continue
+            // It began it whether or not its `start` came first.
+            this.begun(a, m.task)
+            if (this.retry(a, j, m.reason) !== undefined) this.queue.unshift(j)
+            if (a.faulty) this.o.faulted?.()
+            if (a.faulty && this.spawnedAt.size > 0)
+              this.stranded(
+                `${a.label} could not run ${m.task} (${m.reason}) and no worker is left`,
+              )
+            this.pumpSoon()
+          } else if (m.type === 'part') {
+            a.usage.messagesOut++
+            a.usage.bytesOut += bytesOf(m)
+            const k = `${from}\0${m.task}`
+            const held = this.parts.get(k) ?? {}
+            for (const list of PART_LISTS) {
+              const items = m[list] as readonly unknown[] | undefined
+              if (items !== undefined) held[list] = [...(held[list] ?? []), ...items]
+            }
+            this.parts.set(k, held)
+          } else if (m.type === 'end') {
+            a.usage.messagesOut++
+            a.usage.bytesOut += bytesOf(m)
+            const k = `${from}\0${m.task}`
+            const held = this.parts.get(k)
+            this.parts.delete(k)
+            const result: Record<string, unknown> = { ...m.result }
+            for (const list of PART_LISTS) {
+              const items = m.result[list] as readonly unknown[] | undefined
+              if (held?.[list] !== undefined) result[list] = [...held[list], ...(items ?? [])]
+            }
+            // Not awaited: one task's restore holds no other message.
+            const j = a.busy.get(m.task)
+            if (j !== undefined) {
+              j.ended = true
+              const began = here(m.startedAt)
+              const ran = here(m.ranAt)
+              if (began !== undefined) j.startedAt = began
+              if (ran !== undefined) j.ranAt = ran
+            }
+            void this.finish(a, m.task, m.key, result as WiredResult, here(m.endedAt))
           }
-          // Not awaited: one task's restore holds no other message.
-          void this.finish(a, m.task, m.key, result as WiredResult)
         }
       }
-      // After the batch: a `held` in it renews its lease first. Expired
-      // before, a stalled loop's waiting `held` came too late and the task
-      // ran again on another worker (2026-10-27).
-      this.expire()
       this.stall()
     }
   }
@@ -2155,7 +2153,7 @@ export class Orchestrator {
 }
 
 function result(r: WiredResult, log: readonly CapturedChunk[]): ExecuteResult {
-  const { cpuShare: _, removed: __, unread: ___, unshared: ____, output: _____, ...rest } = r
+  const { cpuShare: _, removed: __, unread: ___, ...rest } = r
   const violations = (r.violations ?? []).map((v) => ({ ...v, timestamp: new Date(v.timestamp) }))
   const of = (err: boolean) =>
     log
@@ -2165,18 +2163,6 @@ function result(r: WiredResult, log: readonly CapturedChunk[]): ExecuteResult {
   return { ...rest, stdout: of(false), stderr: of(true), violations, outputs: { kind: 'disk' } }
 }
 
-/** A worker's `output`, read at the boundary: its chunks of text. */
-function wiredLog(output: unknown): CapturedChunk[] {
-  return Array.isArray(output)
-    ? output
-        .filter(
-          (c): c is CapturedChunk =>
-            typeof c === 'object' && c !== null && typeof c.text === 'string',
-        )
-        .map((c) => ({ text: c.text, err: c.err === true }))
-    : []
-}
-
 /**
  * A worker's `cpuShare`, read at the boundary: in (0, 1], else 1. A twentieth
  * at least: a share that small is a misread, and dividing by it would book

package/src/plugin.ts

@@ -76,6 +76,8 @@ export interface DteAdapter {
    * on joining, stuck or queued, so stopped whatever its job holds.
    */
   cancel?(ids: readonly string[], late?: boolean): void | Promise<readonly string[]>
+  /** Worker `id`, given up as late, joined after all and the run took it: a stop not yet sent is dropped. */
+  kept?(id: string): void
   /** The CI files that start its workers, which `vx dte init` writes; none for an adapter that starts them on this machine. */
   init?(setup: DteSetup): Promise<DteInit>
   /** Where it says a fault it works around (a status poll or a cancel that failed); called as the run starts. */
@@ -241,7 +243,7 @@ export function dte(options: DteOptions = {}) {
   // With no scheme the run failed in the executor as a bare "Invalid URL".
   if (options.relay !== undefined && !/^https?:\/\//.test(options.relay)) {
     throw new UserError(
-      `dte() option "relay" must be an http(s) URL, got ${JSON.stringify(options.relay)}`,
+      `dte() option "relay" must be an http(s) URL, got ${JSON.stringify(shown(options.relay))}`,
     )
   }
   // On GitHub Actions with none given, one for the relay's URL.
@@ -385,7 +387,7 @@ export function dte(options: DteOptions = {}) {
             throw new UserError(`vx dte worker: needs ${missing.join(', and ')}`)
           }
           if (!/^https?:\/\//.test(relay))
-            throw new UserError(`vx dte worker: VX_DTE_RELAY must be http(s), got ${relay}`)
+            throw new UserError(`vx dte worker: VX_DTE_RELAY must be http(s), got ${shown(relay)}`)
           const secret = givenSecret()
           if (secret === undefined && !onGitHub()) {
             throw new UserError(
@@ -485,13 +487,13 @@ export function dte(options: DteOptions = {}) {
         throw new UserError('dte(): no relay: set relay, or use an adapter that has one')
       const [
         { Orchestrator },
-        { join },
+        { joinOffLoop },
         { decide, knownMisses, slowdown, tailsMs, WARM_UP_MS },
         { snapshot },
         seal,
       ] = await Promise.all([
         import('./orchestrator.js'),
-        import('./wire.js'),
+        import('./wire-thread.js'),
         import('./planner.js'),
         import('./tree.js'),
         import('./seal.js'),
@@ -521,7 +523,9 @@ export function dte(options: DteOptions = {}) {
       let onChange = (_now?: boolean): void => {}
       const o = new Orchestrator({
         me: seal.sealOrchestrator(
-          await join(relay, run, 'orchestrator', null, () => token(relay), headers),
+          // Off this loop: the run's own sync work held its acks, and every
+          // worker then running ended (CI, 2026-10-29).
+          await joinOffLoop(relay, run, null, () => token(relay), headers),
           run,
           proof,
           ctx.warn,
@@ -550,6 +554,7 @@ export function dte(options: DteOptions = {}) {
         joined: () => onChange(),
         // A worker given up on joining: its job, still queued or stuck, is stopped.
         overdue: (id) => void adapter?.cancel?.([id], true),
+        kept: (id) => adapter?.kept?.(id),
         // Before the run counts what is left to start: a worker that let a
         // task's lease end failed main's macOS run with another to start
         // (2026-10-23).
@@ -1016,3 +1021,8 @@ export function homeOf(sizes: readonly Size[], host: string): string | undefined
   const of = new Set(sizes.filter((s) => s.max > 0).map((s) => s.platform))
   return of.has(host) && of.size > 1 ? host : undefined
 }
+
+/** A relay URL as a refusal may print it: a `user:pass@` would reach the CI log. */
+function shown(url: string): string {
+  return url.replace(/^([a-z][a-z0-9+.-]*:\/\/)[^/?#]*@/i, '$1***@')
+}

package/src/seal.ts

@@ -40,17 +40,22 @@ const GITHUB = 'https://token.actions.githubusercontent.com'
 /** GitHub's keys are read again for a key not among them at most this often. */
 const KEYS_REREAD_MS = 60_000
 
+/** One try at GitHub's token endpoint, which answers in well under a second. */
+const TOKEN_TRY_MS = 3_000
+
 /**
- * An OIDC token for `audience` from GitHub's token endpoint. A 5xx or a
- * dropped connection is tried again, three tries in 10 s at most: one 503
- * ended a worker before it joined (PR #4146, 2026-10-23).
+ * An OIDC token for `audience` from GitHub's token endpoint. A 5xx, a
+ * dropped connection or a try that hangs past 3 s is tried again, three
+ * tries in 10 s at most: one 503 ended a worker before it joined (PR #4146,
+ * 2026-10-23), and one held request spent the whole budget (PR #4887,
+ * 2026-10-31).
  */
 export async function oidcToken(url: string, bearer: string, audience: string): Promise<string> {
   const until = performance.now() + 10_000
   for (let attempt = 1; ; attempt++) {
     const res = await fetch(`${url}&audience=${encodeURIComponent(audience)}`, {
       headers: { authorization: `bearer ${bearer}` },
-      signal: AbortSignal.timeout(Math.max(1, until - performance.now())),
+      signal: AbortSignal.timeout(Math.max(1, Math.min(TOKEN_TRY_MS, until - performance.now()))),
     }).catch((err: unknown) => (err instanceof Error ? err.message : String(err)))
     if (typeof res !== 'string' && res.ok) {
       // quiet: a body that is not JSON holds no token, said below
@@ -371,6 +376,7 @@ export function sealOrchestrator(
       return me.online
     },
     resumed: (f) => me.resumed(f),
+    quiet: (f) => me.quiet(f),
     async send(to, body) {
       const s = sessions.get(to)
       // Lost or never admitted: the relay's own answer for a peer it does
@@ -468,6 +474,7 @@ export async function sealWorker(
       return me.online
     },
     resumed: (f) => me.resumed(f),
+    quiet: (f) => me.quiet(f),
     async send(to, body) {
       if (session === undefined) throw new Error('no session with the orchestrator yet')
       await session.send(body, (sealed) => me.send(to, sealed))