@vzn/vx-dte 0.0.651 → 0.0.653

package/src/usage.ts

@@ -17,9 +17,19 @@ export interface UsageStats {
   readonly cpuAvg: number
   readonly memoryPeak: number
   readonly memoryAvg: number
-  /** Core-time its tasks used over the core-time it had. */
+  /**
+   * Its machine's busy core-time over the core-time it had, from its first
+   * task's send to its last one's end: whether the run left cores idle. Its
+   * tasks' own CPU when its machine said none.
+   */
   readonly utilization: number
+  /** Its tasks' core-time over the same, and its worker process's (undefined when it said none). */
+  readonly tasksUtilization: number
+  readonly workerUtilization?: number | undefined
   readonly cpuMs: number
+  /** From its first task's send to its last one's end, and its busy core-time then. */
+  readonly workMs: number
+  readonly workCoreMs: number
   /** Cores the run held for its tasks, on average over its time online: what packing gave it. */
   readonly bookedAvg: number
   /**
@@ -30,6 +40,12 @@ export interface UsageStats {
   readonly machineAvg?: number | undefined
 }
 
+interface Point {
+  readonly at: number
+  readonly busy: number
+  readonly self?: number
+}
+
 export class Usage {
   done = 0
   failed = 0
@@ -48,8 +64,23 @@ export class Usage {
   private bookedAt = 0
   /** Its machine's busy cores on average, as its last frame said. */
   machineAvg: number | undefined
+  /** Its machine's busy core-time since it joined, as its last frame said, and when. */
+  private heard: Point
+  private first: Point | undefined
+  private last: Point | undefined
+  private firstAt: number | undefined
+  private lastAt: number | undefined
 
-  constructor(readonly from: number = performance.now()) {}
+  constructor(readonly from: number = performance.now()) {
+    this.heard = { at: from, busy: 0, self: 0 }
+  }
+
+  /** Its machine said `avg` cores busy on average since it started, its worker process `self`. */
+  machine(avg: number, self: number | undefined, at: number = performance.now()): void {
+    this.machineAvg = avg
+    const ms = Math.max(0, at - this.from)
+    this.heard = { at, busy: avg * ms, ...(self === undefined ? {} : { self: self * ms }) }
+  }
 
   /**
    * A task it finished, as it ends: on it `from` its send (else its
@@ -73,6 +104,8 @@ export class Usage {
       this.aborted++
     else this.failed++
     const end = performance.now()
+    this.lastAt = end
+    this.last = this.heard
     const ms = Math.max(1, from === undefined ? r.durationMs : end - from)
     this.spans.push({
       start: end - ms,
@@ -82,6 +115,13 @@ export class Usage {
     })
   }
 
+  /** The run sent it a task at `at`: its first send opens the window utilization reads. */
+  sent(at: number = performance.now()): void {
+    if (this.firstAt !== undefined) return
+    this.firstAt = at
+    this.first = this.heard
+  }
+
   /** The run now holds `cores` more of it for its tasks (fewer when negative). */
   book(cores: number, at: number = performance.now()): void {
     this.bookedMs += this.booked * (at - this.bookedAt)
@@ -119,6 +159,24 @@ export class Usage {
       cpuMs += s.cores * (s.end - s.start)
       memoryMs += s.memory * (s.end - s.start)
     }
+    const workMs =
+      this.firstAt !== undefined && this.lastAt !== undefined && this.lastAt > this.firstAt
+        ? this.lastAt - this.firstAt
+        : onlineMs
+    const f = this.first
+    const l = this.last
+    // Between the frames nearest its first send and its last end: an avg
+    // since it started says nothing of a window inside that.
+    const framed = f !== undefined && l !== undefined && l.at > f.at
+    const tasksUtilization = Math.min(1, cpuMs / (cores * workMs))
+    const utilization =
+      framed && this.machineAvg !== undefined
+        ? Math.min(1, (l.busy - f.busy) / (cores * (l.at - f.at)))
+        : tasksUtilization
+    const workerUtilization =
+      framed && l.self !== undefined && f.self !== undefined
+        ? Math.min(1, (l.self - f.self) / (cores * (l.at - f.at)))
+        : undefined
     return {
       onlineMs,
       busyMs,
@@ -126,8 +184,12 @@ export class Usage {
       cpuAvg: cpuMs / onlineMs,
       memoryPeak,
       memoryAvg: memoryMs / onlineMs,
-      utilization: cpuMs / (cores * onlineMs),
+      utilization,
+      tasksUtilization,
+      ...(workerUtilization === undefined ? {} : { workerUtilization }),
       cpuMs,
+      workMs,
+      workCoreMs: utilization * cores * workMs,
       bookedAvg: (this.bookedMs + this.booked * Math.max(0, end - this.bookedAt)) / onlineMs,
       machineAvg: this.machineAvg,
     }

package/src/warm-ups.ts

@@ -56,14 +56,24 @@ async function commits(root: string): Promise<string[]> {
 }
 
 /** The newest warm-ups an ancestor's run wrote, the base branch's on a scoped run. */
-export async function readWarmUps(cache: PluginCache, root: string): Promise<Map<string, number>> {
-  const keys = (await commits(root)).map(keyOf)
+export function readWarmUps(cache: PluginCache, root: string): Promise<Map<string, number>> {
+  return newest(cache, root, keyOf, parseWarmUps)
+}
+
+/** The newest blob under an ancestor's `key` that `parse` reads; an empty map for none. */
+async function newest(
+  cache: PluginCache,
+  root: string,
+  key: (sha: string) => string,
+  parse: (bytes: Uint8Array) => Map<string, number> | null,
+): Promise<Map<string, number>> {
+  const keys = (await commits(root)).map(key)
   if (keys.length === 0) return new Map()
   const held = await cache.hasMany(keys, false)
-  for (const key of keys) {
-    if (held !== null && !held.has(key)) continue
-    const bytes = await cache.get(key, false)
-    const got = bytes === null ? null : parseWarmUps(bytes)
+  for (const k of keys) {
+    if (held !== null && !held.has(k)) continue
+    const bytes = await cache.get(k, false)
+    const got = bytes === null ? null : parse(bytes)
     if (got !== null) return got
   }
   return new Map()
@@ -83,3 +93,59 @@ export async function writeWarmUps(
   for (const [name, ms] of [...learned, ...run]) sizes[name] = Math.round(ms)
   await cache.put(keyOf(head), new TextEncoder().encode(JSON.stringify({ version: 1, sizes })))
 }
+
+// Each server's base, the cores it uses with no dependant's work
+// (`server-fit.ts`): booked once on each worker that starts it. Kept as
+// the warm-ups are, under each CI run's commit; untrusted, it only books.
+
+const MAX_SERVERS = 1024
+/** More cores than any machine a run books. */
+const MAX_CORES = 4096
+
+const basesKeyOf = (sha: string): string => `server-bases\0${sha}`
+
+/** `{ version: 1, servers: { id: cores } }`'s bases, each finite in [0, MAX_CORES]; null for anything else. */
+export function parseServerBases(bytes: Uint8Array): Map<string, number> | null {
+  if (bytes.byteLength > MAX_BYTES) return null
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(new TextDecoder().decode(bytes))
+    // quiet: not JSON reads as no blob
+  } catch {
+    return null
+  }
+  if (typeof parsed !== 'object' || parsed === null) return null
+  const { version, servers } = parsed as { version?: unknown; servers?: unknown }
+  if (version !== 1 || typeof servers !== 'object' || servers === null || Array.isArray(servers))
+    return null
+  const entries = Object.entries(servers)
+  if (entries.length > MAX_SERVERS) return null
+  const out = new Map<string, number>()
+  for (const [id, cores] of entries)
+    if (typeof cores === 'number' && Number.isFinite(cores) && cores >= 0 && cores <= MAX_CORES)
+      out.set(id, cores)
+  return out
+}
+
+/** The newest server bases an ancestor's run wrote. */
+export function readServerBases(cache: PluginCache, root: string): Promise<Map<string, number>> {
+  return newest(cache, root, basesKeyOf, parseServerBases)
+}
+
+/** `learned` with this run's own over it, under HEAD. */
+export async function writeServerBases(
+  cache: PluginCache,
+  root: string,
+  learned: ReadonlyMap<string, number>,
+  run: ReadonlyMap<string, number>,
+): Promise<void> {
+  if (run.size === 0) return
+  const [head] = await commits(root)
+  if (head === undefined) return
+  const servers: Record<string, number> = {}
+  for (const [id, cores] of [...learned, ...run]) servers[id] = Math.round(cores * 100) / 100
+  await cache.put(
+    basesKeyOf(head),
+    new TextEncoder().encode(JSON.stringify({ version: 1, servers })),
+  )
+}

package/src/wire.ts

@@ -51,6 +51,11 @@ const LOST_BEATS = 3
  * orchestrator's ack answers it: one late frame is not a loss.
  */
 const QUIET_BEATS = LOST_BEATS / 2
+/**
+ * How long the orchestrator gives its own reaction to a frame (a pump, a
+ * batch) to carry the frame's ack, before the ack goes alone.
+ */
+const REACT_MS = 20
 /** A send unacknowledged as long goes again: its ack comes within it. */
 const RESEND_BEATS = LOST_BEATS / 2
 /** Why a worker ends on its orchestrator's silence. */
@@ -102,48 +107,70 @@ const answerMs = (heartbeatMs: number): number =>
   Math.min(Math.max(heartbeatMs, MIN_WAIT_MS), REPLY_MS)
 
 /**
- * A send as it travels: numbered per receiver, so a resend is dropped as
- * seen, and carrying the sender's ack of what it took from that receiver,
- * so most acks cost no message of their own.
+ * A send as it travels: numbered (`n`) per receiver, so a resend is
+ * dropped as seen, its body `b`, and carrying the sender's ack (`a`) of
+ * what it took from that receiver, so most acks cost no message of their
+ * own. One letter each: they travel in every frame, twice.
  */
 interface Numbered {
-  readonly seq: number
-  readonly body: unknown
-  readonly acked?: number
+  readonly n: number
+  readonly b: unknown
+  readonly a?: number
 }
-/** The receiver took every send to it numbered up to `acked`. */
+/** The receiver took every send to it numbered up to `a`. */
 interface Acked {
-  readonly acked: number
+  readonly a: number
 }
 const isNumbered = (b: unknown): b is Numbered =>
   typeof b === 'object' &&
   b !== null &&
-  typeof (b as Numbered).seq === 'number' &&
-  'body' in b &&
-  Object.keys(b).length === ('acked' in b ? 3 : 2)
+  typeof (b as Numbered).n === 'number' &&
+  'b' in b &&
+  Object.keys(b).length === ('a' in b ? 3 : 2)
 /**
  * What a side with nothing else to say a heartbeat sends: a numbered send
  * with no body, carrying its acks, delivered to no one.
  */
 interface Beat {
-  readonly seq: number
-  readonly acked?: number
+  readonly n: number
+  readonly a?: number
 }
 const isBeat = (b: unknown): b is Beat =>
   typeof b === 'object' &&
   b !== null &&
-  typeof (b as Beat).seq === 'number' &&
-  !('body' in b) &&
-  Object.keys(b).length === ('acked' in b ? 2 : 1)
+  typeof (b as Beat).n === 'number' &&
+  !('b' in b) &&
+  Object.keys(b).length === ('a' in b ? 2 : 1)
 const isAcked = (b: unknown): b is Acked =>
   typeof b === 'object' &&
   b !== null &&
-  typeof (b as Acked).acked === 'number' &&
+  typeof (b as Acked).a === 'number' &&
   Object.keys(b).length === 1
 
+/** The last of `paced`'s tries failed: `cause` is that try's error. */
+class Exhausted extends Error {
+  constructor(
+    override readonly cause: unknown,
+    readonly tries: number,
+    readonly ms: number,
+  ) {
+    super(cause instanceof Error ? cause.message : String(cause))
+  }
+}
+
+/**
+ * The wait before try `tries + 2`: doubling from an eighth of `REPLY_MS`,
+ * jittered down to half, so `RETRIES` waits span 4.4 to 8.75 s of it. A
+ * fault that fails at once (a refused or unresolved host) spent four tries
+ * in about 2 s under a 250 ms start (PR #5082's CI, 2026-11-07).
+ */
+const pause = (tries: number): number =>
+  ((REPLY_MS / 2 ** RETRIES) * 2 ** tries * (1 + Math.random())) / 2
+
 /** Runs `f` again after a fault, at most `RETRIES` times and within `REPLY_MS`. */
 async function paced<T>(f: () => Promise<T>): Promise<T> {
-  const by = performance.now() + REPLY_MS
+  const at = performance.now()
+  const by = at + REPLY_MS
   for (let tries = 0; ; tries++) {
     try {
       return await f()
@@ -151,9 +178,10 @@ async function paced<T>(f: () => Promise<T>): Promise<T> {
       // A request the relay held past its deadline already took the whole wait.
       const fault =
         err instanceof RelayError ? err.status >= 500 : (err as Error).name !== 'TimeoutError'
-      const wait = Math.random() * FIRST_REDIAL_MS * 2 ** tries
-      if (!fault || tries >= RETRIES) throw err
-      if (performance.now() + wait >= by) throw err
+      if (!fault) throw err
+      const wait = pause(tries)
+      if (tries >= RETRIES || performance.now() + wait >= by)
+        throw new Exhausted(err, tries + 1, performance.now() - at)
       await Bun.sleep(wait)
     }
   }
@@ -267,22 +295,45 @@ export async function join(
   /**
    * The orchestrator sends only orders and acks: an ack no order carried
    * goes on its own after half a heartbeat, every peer's in one frame. A
-   * worker's ride its next send, which it makes within a heartbeat.
+   * worker's ride its next send, which it makes within a heartbeat. A peer
+   * the orchestrator said nothing to in a heartbeat (a quiet worker's
+   * report) is answered once the orchestrator reacted: an order sent in
+   * reply carries the ack, and with none it goes alone, `REACT_MS` on.
    */
   let acking: ReturnType<typeof setTimeout> | undefined
+  let answering: ReturnType<typeof setTimeout> | undefined
+  const soon = new Set<string>()
+  const ack = (peer: string): void => {
+    const upTo = taken.get(peer)?.upTo
+    owing.delete(peer)
+    said.set(peer, performance.now())
+    // quiet: a lost ack only costs a resend, which is dropped as seen
+    if (upTo !== undefined) put(peer, { a: upTo } satisfies Acked).catch(() => undefined)
+  }
   const flushAcks = (): void => {
     acking = undefined
     if (left || ended !== undefined) return
-    for (const peer of [...owing]) {
-      const upTo = taken.get(peer)?.upTo
-      owing.delete(peer)
-      // quiet: a lost ack only costs a resend, which is dropped as seen
-      if (upTo !== undefined) put(peer, { acked: upTo } satisfies Acked).catch(() => undefined)
-    }
+    for (const peer of [...owing]) ack(peer)
+  }
+  const answer = (): void => {
+    answering = undefined
+    const peers = [...soon]
+    soon.clear()
+    if (left || ended !== undefined) return
+    for (const peer of peers) if (owing.has(peer)) ack(peer)
   }
   const owe = (peer: string): void => {
     owing.add(peer)
-    if (role !== 'orchestrator' || acking !== undefined) return
+    if (role !== 'orchestrator') return
+    if (performance.now() - (said.get(peer) ?? 0) >= heartbeatMs) {
+      soon.add(peer)
+      if (answering === undefined) {
+        answering = setTimeout(answer, REACT_MS)
+        answering.unref()
+      }
+      return
+    }
+    if (acking !== undefined) return
     acking = setTimeout(flushAcks, heartbeatMs / 2)
     acking.unref()
   }
@@ -290,9 +341,9 @@ export async function join(
   const said = new Map<string, number>()
   let onQuiet: ((peer: string) => void) | undefined
   /** The ack owed `to`, if any, for a send to it to carry; and `to` was just said to. */
-  const owed = (to: string): { acked?: number } => {
+  const owed = (to: string): { a?: number } => {
     said.set(to, performance.now())
-    return owing.delete(to) ? { acked: taken.get(to)!.upTo } : {}
+    return owing.delete(to) ? { a: taken.get(to)!.upTo } : {}
   }
   /**
    * When each peer was last heard from: the orchestrator's workers from
@@ -335,12 +386,12 @@ export async function join(
       }
       const from = heldIds.get(m.from) ?? m.from
       if (heard.has(from)) heard.set(from, performance.now())
-      if (isAcked(body)) settle(from, body.acked)
+      if (isAcked(body)) settle(from, body.a)
       else if (isBeat(body)) {
-        if (body.acked !== undefined) settle(from, body.acked)
+        if (body.a !== undefined) settle(from, body.a)
         const t = taken.get(from) ?? { upTo: 0, ahead: new Map() }
         taken.set(from, t)
-        if (body.seq > t.upTo) t.ahead.set(body.seq, BEAT)
+        if (body.n > t.upTo) t.ahead.set(body.n, BEAT)
         while (t.ahead.has(t.upTo + 1)) {
           const next = t.ahead.get(++t.upTo)
           t.ahead.delete(t.upTo)
@@ -351,13 +402,13 @@ export async function join(
         if (role === 'orchestrator') {
           owing.delete(from)
           // quiet: a lost ack only costs a resend, which is dropped as seen
-          put(from, { acked: t.upTo } satisfies Acked).catch(() => undefined)
+          put(from, { a: t.upTo } satisfies Acked).catch(() => undefined)
         } else owe(from)
       } else if (isNumbered(body)) {
-        if (body.acked !== undefined) settle(from, body.acked)
+        if (body.a !== undefined) settle(from, body.a)
         let t = taken.get(from)
         if (t === undefined) taken.set(from, (t = { upTo: 0, ahead: new Map() }))
-        if (body.seq > t.upTo) t.ahead.set(body.seq, body.body)
+        if (body.n > t.upTo) t.ahead.set(body.n, body.b)
         while (t.ahead.has(t.upTo + 1)) {
           const next = t.ahead.get(++t.upTo)
           t.ahead.delete(t.upTo)
@@ -434,7 +485,7 @@ export async function join(
       const seq = ++o.next
       o.unacked.set(seq, { body: BEAT, at: now })
       // quiet: unacknowledged, it goes again
-      put(peer, { seq, ...owed(peer) } satisfies Beat).catch(() => undefined)
+      put(peer, { n: seq, ...owed(peer) } satisfies Beat).catch(() => undefined)
     }
   }
   /** Sends again what waited `ms` unacknowledged, to `only` or to all; all of it, at 0. */
@@ -446,7 +497,7 @@ export async function join(
         if (now - u.at < ms) continue
         u.at = now
         // quiet: unacknowledged, it goes again next heartbeat
-        const again = u.body === BEAT ? { seq } : { seq, body: u.body }
+        const again = u.body === BEAT ? { n: seq } : { n: seq, b: u.body }
         put(to, { ...again, ...owed(to) }).catch(() => undefined)
       }
     }
@@ -466,16 +517,19 @@ export async function join(
         return { tokenless }
       }
       const s = await connect(base, query, h)
-      // A fault: dialed again within `REPLY_MS`.
       if (s === undefined) throw new Error('no socket')
       return { s }
     })
   } catch (err) {
     if (err instanceof RelayError) throw err
-    throw new RelayError(
-      504,
-      `cannot connect to the relay ${new URL(base).origin}: no answer in ${REPLY_MS / 1000} s`,
-    )
+    const cause = err instanceof Exhausted ? err.cause : err
+    if (cause instanceof RelayError) throw cause
+    // Said as it was: a wait only when one was waited out.
+    const why =
+      err instanceof Exhausted && (cause as Error).name !== 'TimeoutError'
+        ? `${err.message}, ${err.tries} dials in ${(err.ms / 1000).toFixed(1)} s`
+        : `no answer in ${REPLY_MS / 1000} s`
+    throw new RelayError(504, `cannot connect to the relay ${new URL(base).origin}: ${why}`)
   }
   // A token it cannot get is said as itself, not as a relay that never
   // answered: the relay was not asked.
@@ -673,7 +727,7 @@ export async function join(
       }
       try {
         if (s?.open !== true) throw ended!
-        await s.send(relayIds.get(to) ?? to, { seq, body, ...owed(to) } satisfies Numbered)
+        await s.send(relayIds.get(to) ?? to, { n: seq, b: body, ...owed(to) } satisfies Numbered)
       } catch (err) {
         // Closed under it: it goes again on the next socket.
         if (!(err instanceof RelayError)) return
@@ -839,8 +893,14 @@ async function connect(
   // Its handlers are set before it opens: a relay that refuses closes at
   // once, and Bun may report that close, or its frame, with the open.
   let opening: (ok: boolean) => void = () => {}
+  /** Why the dial failed, as the platform said it (`ECONNREFUSED`, `ENOTFOUND`). */
+  let failure = ''
+  let waited = false
   const opened = new Promise<boolean>((resolve) => {
-    const timer = setTimeout(() => resolve(false), wait)
+    const timer = setTimeout(() => {
+      waited = true
+      resolve(false)
+    }, wait)
     opening = (ok) => {
       clearTimeout(timer)
       resolve(ok)
@@ -963,7 +1023,9 @@ async function connect(
     shut()
     opening(false)
   }
-  ws.onerror = () => {
+  ws.onerror = (e) => {
+    const err = (e as ErrorEvent).error as { code?: unknown } | undefined
+    failure ||= typeof err?.code === 'string' ? err.code : ((e as ErrorEvent).message ?? '')
     closed()
     opening(false)
   }
@@ -1051,13 +1113,21 @@ async function connect(
     if (heartbeatMs !== undefined) return undefined
   }
   if (heartbeatMs === undefined) {
-    const timer = setTimeout(() => said(undefined), wait)
+    const timer = setTimeout(() => {
+      waited = true
+      said(undefined)
+    }, wait)
     const j = await joined
     clearTimeout(timer)
     if (j === undefined) {
       await me.close()
       const status = refusal(code)
-      if (status === undefined) return undefined
+      if (status === undefined) {
+        // A fault, dialed again: its words say why, or that it was waited out.
+        const err = new Error(printable(failure || reason || 'socket closed'))
+        if (waited) err.name = 'TimeoutError'
+        throw err
+      }
       // A GitLab job with no `id_tokens` entry was told only `401 no bearer
       // token`, not what gives it one (2026-10-26).
       const hint =