@vzn/vx-dte 0.0.650 → 0.0.652

package/src/plugin.ts

@@ -20,7 +20,7 @@ import type { Need } from './dispatch.js'
 import { booked, DEFAULT_STRATEGY, STRATEGIES, type Strategy } from './strategy.js'
 import type { Orchestrator } from './orchestrator.js'
 import type { Proof } from './seal.js'
-import type { Size } from './planner.js'
+import type { Forecast, Size } from './planner.js'
 
 /** One kind of worker the adapter can start. */
 export interface WorkerSize {
@@ -76,6 +76,8 @@ export interface DteAdapter {
    * on joining, stuck or queued, so stopped whatever its job holds.
    */
   cancel?(ids: readonly string[], late?: boolean): void | Promise<readonly string[]>
+  /** Worker `id`, given up as late, joined after all and the run took it: a stop not yet sent is dropped. */
+  kept?(id: string): void
   /** The CI files that start its workers, which `vx dte init` writes; none for an adapter that starts them on this machine. */
   init?(setup: DteSetup): Promise<DteInit>
   /** Where it says a fault it works around (a status poll or a cancel that failed); called as the run starts. */
@@ -241,7 +243,13 @@ export function dte(options: DteOptions = {}) {
   // With no scheme the run failed in the executor as a bare "Invalid URL".
   if (options.relay !== undefined && !/^https?:\/\//.test(options.relay)) {
     throw new UserError(
-      `dte() option "relay" must be an http(s) URL, got ${JSON.stringify(options.relay)}`,
+      `dte() option "relay" must be an http(s) URL, got ${JSON.stringify(shown(options.relay))}`,
+    )
+  }
+  const unrouted = options.relay === undefined ? undefined : unroutable(options.relay)
+  if (unrouted !== undefined) {
+    throw new UserError(
+      `dte() option "relay" ${unrouted}, got ${JSON.stringify(shown(options.relay!))}`,
     )
   }
   // On GitHub Actions with none given, one for the relay's URL.
@@ -385,7 +393,10 @@ export function dte(options: DteOptions = {}) {
             throw new UserError(`vx dte worker: needs ${missing.join(', and ')}`)
           }
           if (!/^https?:\/\//.test(relay))
-            throw new UserError(`vx dte worker: VX_DTE_RELAY must be http(s), got ${relay}`)
+            throw new UserError(`vx dte worker: VX_DTE_RELAY must be http(s), got ${shown(relay)}`)
+          const unroutedEnv = unroutable(relay)
+          if (unroutedEnv !== undefined)
+            throw new UserError(`vx dte worker: VX_DTE_RELAY ${unroutedEnv}, got ${shown(relay)}`)
           const secret = givenSecret()
           if (secret === undefined && !onGitHub()) {
             throw new UserError(
@@ -485,13 +496,13 @@ export function dte(options: DteOptions = {}) {
         throw new UserError('dte(): no relay: set relay, or use an adapter that has one')
       const [
         { Orchestrator },
-        { join },
+        { joinOffLoop },
         { decide, knownMisses, slowdown, tailsMs, WARM_UP_MS },
         { snapshot },
         seal,
       ] = await Promise.all([
         import('./orchestrator.js'),
-        import('./wire.js'),
+        import('./wire-thread.js'),
         import('./planner.js'),
         import('./tree.js'),
         import('./seal.js'),
@@ -521,7 +532,9 @@ export function dte(options: DteOptions = {}) {
       let onChange = (_now?: boolean): void => {}
       const o = new Orchestrator({
         me: seal.sealOrchestrator(
-          await join(relay, run, 'orchestrator', null, () => token(relay), headers),
+          // Off this loop: the run's own sync work held its acks, and every
+          // worker then running ended (CI, 2026-10-29).
+          await joinOffLoop(relay, run, null, () => token(relay), headers),
           run,
           proof,
           ctx.warn,
@@ -550,6 +563,7 @@ export function dte(options: DteOptions = {}) {
         joined: () => onChange(),
         // A worker given up on joining: its job, still queued or stuck, is stopped.
         overdue: (id) => void adapter?.cancel?.([id], true),
+        kept: (id) => adapter?.kept?.(id),
         // Before the run counts what is left to start: a worker that let a
         // task's lease end failed main's macOS run with another to start
         // (2026-10-23).
@@ -620,6 +634,8 @@ export function dte(options: DteOptions = {}) {
       // The run's deadline (`Plan.boundMs`), and the work its plans knew.
       let deadline: number | undefined
       const known = new Set<string>()
+      // The last plan, which stands while what it saw holds (`decide`).
+      const last: { forecast?: Forecast } = {}
       // A plan that needs a worker later plans again then.
       let again: ReturnType<typeof setTimeout> | undefined
       // The fewest workers that end the run's known work within a margin of
@@ -662,6 +678,8 @@ export function dte(options: DteOptions = {}) {
           how,
           home,
           queued: o.queued(warmUp),
+          last,
+          now: performance.now(),
         })
         deadline = performance.now() + p.boundMs
         for (const t of p.tasks) known.add(t.id)
@@ -782,6 +800,8 @@ export function dte(options: DteOptions = {}) {
       }
       if (needsIn === undefined) scheduled = forecast
       else forecast()
+      // Core asks the footer and the summary after teardown: the last run's graph.
+      const depsOf = (id: string): readonly string[] => (nodes ?? ran)?.get(id)?.deps ?? []
       return {
         name: 'dte',
         remote: true,
@@ -813,9 +833,14 @@ export function dte(options: DteOptions = {}) {
           replan()
           return o.execute(req)
         },
-        footer: () => o.footer(() => o.idealMs((id) => (nodes ?? ran)?.get(id)?.deps ?? [], sizes)),
+        footer: () =>
+          o.footer(
+            () => o.idealMs(depsOf, sizes),
+            () => o.advice(depsOf, sizes),
+          ),
         machines: () => o.machines(),
-        idealMs: () => o.idealMs((id) => (nodes ?? ran)?.get(id)?.deps ?? [], sizes),
+        idealMs: () => o.idealMs(depsOf, sizes),
+        advice: () => o.advice(depsOf, sizes),
         profile: () => o.profile(),
         running: (id) => o.where(id),
         demand(remaining, misses) {
@@ -1016,3 +1041,18 @@ export function homeOf(sizes: readonly Size[], host: string): string | undefined
   const of = new Set(sizes.filter((s) => s.max > 0).map((s) => s.platform))
   return of.has(host) && of.size > 1 ? host : undefined
 }
+
+/**
+ * Why a relay url no route can be appended to is refused: past a `?` or
+ * `#` every socket asked the relay's root, and one naming /v1 asked /v1/v1/….
+ */
+function unroutable(url: string): string | undefined {
+  if (/[?#]/.test(url)) return 'takes no ?query or #fragment: each route is appended to it'
+  if (/\/v1\/*$/.test(url)) return "is the relay's base, without /v1"
+  return undefined
+}
+
+/** A relay URL as a refusal may print it: a `user:pass@` would reach the CI log. */
+function shown(url: string): string {
+  return url.replace(/^([a-z][a-z0-9+.-]*:\/\/)[^/?#]*@/i, '$1***@')
+}

package/src/seal.ts

@@ -40,17 +40,22 @@ const GITHUB = 'https://token.actions.githubusercontent.com'
 /** GitHub's keys are read again for a key not among them at most this often. */
 const KEYS_REREAD_MS = 60_000
 
+/** One try at GitHub's token endpoint, which answers in well under a second. */
+const TOKEN_TRY_MS = 3_000
+
 /**
- * An OIDC token for `audience` from GitHub's token endpoint. A 5xx or a
- * dropped connection is tried again, three tries in 10 s at most: one 503
- * ended a worker before it joined (PR #4146, 2026-10-23).
+ * An OIDC token for `audience` from GitHub's token endpoint. A 5xx, a
+ * dropped connection or a try that hangs past 3 s is tried again, three
+ * tries in 10 s at most: one 503 ended a worker before it joined (PR #4146,
+ * 2026-10-23), and one held request spent the whole budget (PR #4887,
+ * 2026-10-31).
  */
 export async function oidcToken(url: string, bearer: string, audience: string): Promise<string> {
   const until = performance.now() + 10_000
   for (let attempt = 1; ; attempt++) {
     const res = await fetch(`${url}&audience=${encodeURIComponent(audience)}`, {
       headers: { authorization: `bearer ${bearer}` },
-      signal: AbortSignal.timeout(Math.max(1, until - performance.now())),
+      signal: AbortSignal.timeout(Math.max(1, Math.min(TOKEN_TRY_MS, until - performance.now()))),
     }).catch((err: unknown) => (err instanceof Error ? err.message : String(err)))
     if (typeof res !== 'string' && res.ok) {
       // quiet: a body that is not JSON holds no token, said below
@@ -371,6 +376,7 @@ export function sealOrchestrator(
       return me.online
     },
     resumed: (f) => me.resumed(f),
+    quiet: (f) => me.quiet(f),
     async send(to, body) {
       const s = sessions.get(to)
       // Lost or never admitted: the relay's own answer for a peer it does
@@ -468,6 +474,7 @@ export async function sealWorker(
       return me.online
     },
     resumed: (f) => me.resumed(f),
+    quiet: (f) => me.quiet(f),
     async send(to, body) {
       if (session === undefined) throw new Error('no session with the orchestrator yet')
       await session.send(body, (sealed) => me.send(to, sealed))

package/src/wire-host.ts

@@ -0,0 +1,98 @@
+// The thread `joinOffLoop` (wire-thread.ts) starts: the orchestrator's
+// participant, its socket, acks, beats and redials, on a loop the run's
+// work never holds.
+
+import { join, type Participant } from './wire.js'
+
+declare const self: Worker
+
+/**
+ * The run's loop held this many heartbeats (10 s on the default) is taken
+ * as hung: the thread leaves, so the relay closes the run and its workers
+ * end, as they would have for an orchestrator quiet that long (owner,
+ * 2026-10-23: "if it does not reply for 10s its dead").
+ */
+const HELD_BEATS = 5
+
+let me: Participant | undefined
+let n = 0
+const tokens = new Map<number, { resolve(v: string | undefined): void; reject(e: Error): void }>()
+const post = (m: unknown): void => self.postMessage(m)
+const failure = (err: unknown): { status?: number; message: string } => ({
+  ...(typeof (err as { status?: unknown }).status === 'number'
+    ? { status: (err as { status: number }).status }
+    : {}),
+  message: err instanceof Error ? err.message : String(err),
+})
+
+self.onmessage = async (e: MessageEvent<Record<string, unknown>>) => {
+  const m = e.data
+  if (m['type'] === 'token') {
+    const t = tokens.get(m['n'] as number)
+    tokens.delete(m['n'] as number)
+    if (typeof m['error'] === 'string') t?.reject(new Error(m['error']))
+    else t?.resolve(m['value'] as string | undefined)
+    return
+  }
+  const reply = (value?: unknown): void => post({ type: 'done', n: m['n'], value })
+  const fail = (err: unknown): void => post({ type: 'done', n: m['n'], error: failure(err) })
+  try {
+    if (m['type'] === 'join') {
+      const shared = new Float64Array(m['shared'] as SharedArrayBuffer)
+      const token = m['asks']
+        ? () =>
+            new Promise<string | undefined>((resolve, reject) => {
+              const k = ++n
+              tokens.set(k, { resolve, reject })
+              post({ type: 'token', n: k })
+            })
+        : (m['token'] as string | undefined)
+      const p = await join(
+        m['relay'] as string,
+        m['run'] as string,
+        'orchestrator',
+        m['info'],
+        token,
+        m['extra'] as Record<string, string>,
+      )
+      me = p
+      p.resumed((peer) => post({ type: 'resumed', ...(peer === undefined ? {} : { peer }) }))
+      reply({ id: p.id, heartbeatMs: p.heartbeatMs })
+      const watch = setInterval(
+        () => {
+          shared[1] = p.online ? 1 : 0
+          const held = Date.now() - shared[0]!
+          if (held < HELD_BEATS * p.heartbeatMs) return
+          clearInterval(watch)
+          post({
+            type: 'ended',
+            status: 410,
+            message: `dte: this machine's run was held ${Math.round(held / 1000)} s, past ${HELD_BEATS} heartbeats: the run is over`,
+          })
+          // quiet: `ended` above already said the run is over; a failed
+          // leave changes nothing the relay does with it.
+          void p.leave().catch(() => undefined)
+        },
+        Math.max(10, p.heartbeatMs / 10),
+      )
+      for (;;) {
+        let got
+        try {
+          got = await p.poll()
+        } catch (err) {
+          clearInterval(watch)
+          const f = failure(err)
+          post({ type: 'ended', status: f.status ?? 500, message: f.message })
+          return
+        }
+        if (got.length > 0) post({ type: 'messages', messages: got })
+      }
+    }
+    if (me === undefined) throw new Error('dte: not joined')
+    if (m['type'] === 'send') reply(await me.send(m['to'] as string, m['body']))
+    else if (m['type'] === 'remove') reply(await me.remove(m['worker'] as string))
+    else if (m['type'] === 'leave') reply(await me.leave())
+  } catch (err) {
+    fail(err)
+  }
+}

package/src/wire-thread.ts

@@ -0,0 +1,168 @@
+// The orchestrator's relay participant on a thread of its own.
+//
+// Liveness is in the protocol: a quiet worker reports every 1.5
+// heartbeats, its orchestrator acks, and a peer quiet 3 heartbeats is lost. On the orchestrator's loop, its own
+// sync work held its beats: in CI run 40886829588 (2026-10-29) it said
+// nothing for 12 s and every worker then running ended, stopping 13
+// tasks. Here the socket, its acks, beats and redials run on a thread the
+// run's work never holds; the run talks to it by messages.
+
+import { ui } from '@vzn/vx'
+import { RelayError, type Participant, type RelayMessage } from './wire.js'
+
+/** From the thread: what `wire-host.ts` posts. */
+type FromHost =
+  | { type: 'token'; n: number }
+  | { type: 'messages'; messages: RelayMessage[] }
+  | { type: 'resumed'; peer?: string }
+  | { type: 'ended'; status: number; message: string }
+  | { type: 'done'; n: number; value?: unknown; error?: { status?: number; message: string } }
+
+/** How often this loop says it is alive to the thread (`HELD_BEATS` in wire-host.ts reads it). */
+const TICK_MS = 100
+/**
+ * A tick this late says so: the thread kept the run alive, but what held the
+ * loop is not known (run 40886829588's profile had no slice in its 12 s gap),
+ * and the line's time in the log is where to look. Half the 2 s heartbeat.
+ */
+const HELD_SAY_MS = 1000
+
+export async function joinOffLoop(
+  relay: string,
+  run: string,
+  info: unknown,
+  token: string | undefined | (() => Promise<string | undefined>),
+  extra: Readonly<Record<string, string>> = {},
+): Promise<Participant> {
+  // [0]: when this loop last ran (Date.now, the clock both threads share); [1]: online.
+  const shared = new Float64Array(new SharedArrayBuffer(16))
+  const tick = (): void => {
+    const now = Date.now()
+    const held = now - shared[0]! - TICK_MS
+    if (shared[0] && held >= HELD_SAY_MS)
+      ui.say('warn', `[vx] dte: this machine's loop was held ${held} ms`)
+    shared[0] = now
+  }
+  tick()
+  const ticking = setInterval(tick, TICK_MS)
+  ticking.unref()
+  const thread = new Worker(new URL('./wire-host.ts', import.meta.url).href)
+  let n = 0
+  const calls = new Map<number, { resolve(v: unknown): void; reject(e: unknown): void }>()
+  const inbox: RelayMessage[] = []
+  let ended: RelayError | undefined
+  let onResumed: ((peer?: string) => void) | undefined
+  /** The token function's own error, said as itself rather than as the thread's copy. */
+  let tokenError: unknown
+  const waiters = new Set<() => void>()
+  const wake = (): void => {
+    for (const w of [...waiters]) w()
+  }
+  const call = (msg: Record<string, unknown>): Promise<unknown> =>
+    new Promise((resolve, reject) => {
+      const k = ++n
+      calls.set(k, { resolve, reject })
+      thread.postMessage({ ...msg, n: k })
+    })
+  thread.onmessage = (e: MessageEvent<FromHost>) => {
+    const m = e.data
+    if (m.type === 'token') {
+      void (async () => {
+        try {
+          const value = typeof token === 'function' ? await token() : token
+          thread.postMessage({ type: 'token', n: m.n, value })
+        } catch (err) {
+          tokenError = err
+          thread.postMessage({
+            type: 'token',
+            n: m.n,
+            error: err instanceof Error ? err.message : String(err),
+          })
+        }
+      })()
+    } else if (m.type === 'messages') {
+      inbox.push(...m.messages)
+      wake()
+    } else if (m.type === 'resumed') onResumed?.(m.peer)
+    else if (m.type === 'ended') {
+      ended ??= new RelayError(m.status, m.message)
+      wake()
+    } else {
+      const c = calls.get(m.n)
+      calls.delete(m.n)
+      if (m.error === undefined) c?.resolve(m.value)
+      else if (tokenError instanceof Error && tokenError.message === m.error.message)
+        c?.reject(tokenError)
+      else
+        c?.reject(
+          m.error.status === undefined
+            ? new Error(m.error.message)
+            : new RelayError(m.error.status, m.error.message),
+        )
+    }
+  }
+  const stop = (): void => {
+    clearInterval(ticking)
+    thread.terminate()
+  }
+  let joined: { id: string; heartbeatMs: number }
+  try {
+    joined = (await call({
+      type: 'join',
+      relay,
+      run,
+      info,
+      extra,
+      shared: shared.buffer,
+      token: typeof token === 'function' ? undefined : token,
+      asks: typeof token === 'function',
+    })) as { id: string; heartbeatMs: number }
+  } catch (err) {
+    stop()
+    throw err
+  }
+  const idle = (ms: number, signal?: AbortSignal): Promise<void> =>
+    new Promise((resolve) => {
+      const done = (): void => {
+        clearTimeout(timer)
+        signal?.removeEventListener('abort', done)
+        waiters.delete(done)
+        resolve()
+      }
+      const timer = setTimeout(done, ms)
+      signal?.addEventListener('abort', done)
+      waiters.add(done)
+    })
+  return {
+    id: joined.id,
+    heartbeatMs: joined.heartbeatMs,
+    get online() {
+      return shared[1] === 1
+    },
+    resumed(f) {
+      onResumed = f
+    },
+    // The thread's own wire beats for the run: its loop may be held.
+    quiet() {},
+    async send(to, body) {
+      await call({ type: 'send', to, body })
+    },
+    async poll(signal) {
+      if (inbox.length === 0 && ended === undefined && signal?.aborted !== true)
+        await idle(joined.heartbeatMs, signal)
+      if (inbox.length > 0) return inbox.splice(0)
+      if (ended !== undefined) throw ended
+      return []
+    },
+    async remove(worker) {
+      await call({ type: 'remove', worker })
+    },
+    async leave() {
+      try {
+        await call({ type: 'leave' })
+      } finally {
+        stop()
+      }
+    },
+  }
+}