@vzn/vx-dte 0.0.649 → 0.0.651

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -28,7 +28,6 @@ import {
28
28
  } from '@vzn/vx'
29
29
  import {
30
30
  bytesOf,
31
- HELD_EVERY,
32
31
  HOST_ENV,
33
32
  PART_LISTS,
34
33
  type PartList,
@@ -65,11 +64,13 @@ interface Job {
65
64
  booked?: number
66
65
  /** The workers that failed it after beginning it, with why: two fail it. */
67
66
  faults?: string[]
68
- /** Its worker's lease on it ends then unless the worker says it still holds it. */
69
- leaseUntil?: number
67
+ /** Its worker's `end` came: a report naming it no more is not a loss. */
68
+ ended?: true
70
69
  /** When it was sent to its worker. */
71
70
  sentAt?: number
72
- /** When its worker began it. */
71
+ /** The batch that sent it, by its worker's count: a report of that batch or later names it. */
72
+ batch?: number
73
+ /** When its worker began it: its send, then the time its worker says. */
73
74
  startedAt?: number
74
75
  /** When its worker said its command ended: it saves since. */
75
76
  ranAt?: number
@@ -112,8 +113,10 @@ interface Worker {
112
113
  readonly usage: Usage
113
114
  /** Sent to it and not finished: running there or in its queue. */
114
115
  readonly busy: Map<string, Job>
115
- /** Of `busy`, the ones it began. */
116
+ /** Of `busy`, the ones it began: each one sent, as a worker runs what it takes at once. */
116
117
  readonly started: Set<string>
118
+ /** Batches sent it: each batch's `n`. */
119
+ batches: number
117
120
  usedCores: number
118
121
  usedMemory: number
119
122
  /** Its machine's use as its last frame said, and when this machine heard it. */
@@ -164,7 +167,9 @@ export interface OrchestratorOptions {
164
167
  readonly joined?: () => void
165
168
  /** Worker `id` did not join within twice its size's first join: given up, for its adapter to stop. */
166
169
  readonly overdue?: (id: string) => void
167
- /** A worker failed a task or its lease: the run plans again, now, so a replacement starts. */
170
+ /** Worker `id`, given up as late (`overdue`), joined after all and is taken. */
171
+ readonly kept?: (id: string) => void
172
+ /** A worker failed a task or dropped it: the run plans again, now, so a replacement starts. */
168
173
  readonly faulted?: () => void
169
174
  /** How much of the run waits on a task (`tailsMs`): the queue places the most first. */
170
175
  readonly tail?: (taskId: string) => number
@@ -361,14 +366,6 @@ export class Orchestrator {
361
366
  return o.root
362
367
  }
363
368
  })()
364
- // Leases ran on while either side was away, as a relay restarted: what
365
- // a worker holds is still its own, not run again elsewhere
366
- // (2026-10-25: "make sure on rejoin we don't repeat work").
367
- o.me.resumed((peer) => {
368
- for (const a of this.workers.values())
369
- if (peer === undefined || a.id === peer)
370
- for (const j of a.busy.values()) if (j.leaseUntil !== undefined) this.lease(j)
371
- })
372
369
  this.loop = this.listen()
373
370
  }
374
371
 
@@ -975,8 +972,11 @@ export class Orchestrator {
975
972
  }
976
973
  tasks.push({ ...order, deps: [] })
977
974
  }
975
+ const n = ++a.batches
976
+ for (const { job } of part) job.batch = n
978
977
  const batch = {
979
978
  type: 'batch',
979
+ n,
980
980
  root: this.o.root,
981
981
  realRoot: this.realRoot,
982
982
  tasks,
@@ -986,9 +986,11 @@ export class Orchestrator {
986
986
  a.usage.messagesIn++
987
987
  a.usage.bytesIn += bytesOf(batch)
988
988
  for (const { job } of part) {
989
- this.lease(job)
990
989
  pending.delete(job)
990
+ // Sent, it runs: its worker takes only what it has room for.
991
+ this.begun(a, job.req.taskId)
991
992
  }
993
+ this.pumpSoon()
992
994
  }
993
995
  })
994
996
  .catch((err: unknown) => {
@@ -1006,13 +1008,8 @@ export class Orchestrator {
1006
1008
  })
1007
1009
  }
1008
1010
 
1009
- /** Three of the worker's `held` from now. */
1010
- private lease(job: Job): void {
1011
- job.leaseUntil = performance.now() + 3 * HELD_EVERY * this.o.me.heartbeatMs
1012
- }
1013
-
1014
1011
  /**
1015
- * Worker `a` let `job` down: lost, its lease ended, or it could not run
1012
+ * Worker `a` let `job` down: lost, its report named it no more, or it could not run
1016
1013
  * it. One it had begun counts, and two workers fail it; one still in its
1017
1014
  * queue goes back free.
1018
1015
  */
@@ -1032,9 +1029,10 @@ export class Orchestrator {
1032
1029
  a.has.delete(job.req.taskId)
1033
1030
  this.unhold(a, job)
1034
1031
  delete job.worker
1035
- delete job.leaseUntil
1036
1032
  delete job.sentAt
1033
+ delete job.ended
1037
1034
  delete job.ranAt
1035
+ delete job.batch
1038
1036
  const begun = a.started.delete(job.req.taskId)
1039
1037
  // Stopped, it runs nowhere again; the cancel sent its worker has no one to answer it.
1040
1038
  if (job.req.signal?.aborted === true) {
@@ -1078,25 +1076,6 @@ export class Orchestrator {
1078
1076
  this.o.stalled?.(lines.join('\n'))
1079
1077
  }
1080
1078
 
1081
- /** Each task whose worker's lease on it ended goes back. */
1082
- private expire(): void {
1083
- // With no socket no `held` arrives: the leases wait for it (`resumed`).
1084
- if (!this.o.me.online) return
1085
- const now = performance.now()
1086
- const back: Job[] = []
1087
- for (const a of this.workers.values())
1088
- for (const j of [...a.busy.values()])
1089
- if (j.leaseUntil !== undefined && j.leaseUntil < now) {
1090
- const again = this.retry(a, j, 'lease expired')
1091
- if (again !== undefined) back.push(again)
1092
- }
1093
- if (back.length === 0) return
1094
- this.queue.unshift(...back)
1095
- this.o.faulted?.()
1096
- if (this.spawnedAt.size > 0) this.stranded(`a task's lease ended and no worker is left`)
1097
- this.pumpSoon()
1098
- }
1099
-
1100
1079
  private post(a: Worker, m: ToWorker): void {
1101
1080
  a.chain = a.chain.then(() => this.o.me.send(a.id, m)).catch(this.unsent(m.type, a.label))
1102
1081
  }
@@ -1118,16 +1097,19 @@ export class Orchestrator {
1118
1097
  }
1119
1098
 
1120
1099
  /** Worker `a` ran `task` and saved it under `key`: it is done there, and done here once restored. */
1121
- private async finish(a: Worker, task: string, key: string, head: WiredResult): Promise<void> {
1100
+ private async finish(
1101
+ a: Worker,
1102
+ task: string,
1103
+ key: string,
1104
+ head: WiredResult,
1105
+ endedAt: number | undefined,
1106
+ ): Promise<void> {
1122
1107
  const job = a.busy.get(task)
1123
1108
  if (job === undefined) return
1124
1109
  // A dependency's remote copy did not read back there (a truncated
1125
1110
  // upload): sent again from this machine, the task runs again. Taken as
1126
1111
  // the task's failure, it failed every run, the remote saying it held the
1127
1112
  // key (2026-10-25).
1128
- // Ended there, it is in no `held` again: a lease running on through a
1129
- // slow resend called its worker faulty and gave the task to another.
1130
- delete job.leaseUntil
1131
1113
  if (head.unread !== undefined) {
1132
1114
  const sent = await this.resend(job, head.unread)
1133
1115
  // Its worker was lost meanwhile: the task went back to the queue.
@@ -1145,7 +1127,7 @@ export class Orchestrator {
1145
1127
  return
1146
1128
  }
1147
1129
  }
1148
- const endAt = performance.now()
1130
+ const endAt = endedAt ?? performance.now()
1149
1131
  this.traceTask(a, job, head, endAt)
1150
1132
  a.busy.delete(task)
1151
1133
  this.unhold(a, job)
@@ -1181,10 +1163,7 @@ export class Orchestrator {
1181
1163
  }
1182
1164
  this.savedAs.set(task, key)
1183
1165
  const node = this.o.nodes()?.get(task)
1184
- // No remote took its upload (too big, a 429): its files are there
1185
- // alone, as an uncached task's are, and its output came with its `end`.
1186
- const unshared = head.unshared === true
1187
- if (head.exitCode === 0 && (unshared || (node !== undefined && leaves(node))))
1166
+ if (head.exitCode === 0 && node !== undefined && leaves(node))
1188
1167
  this.kept.set(task, { worker: a.id, req: job.req, need: job.need })
1189
1168
  this.pumpSoon()
1190
1169
  if (job.again !== undefined) return job.resolve(result(head, []))
@@ -1202,17 +1181,14 @@ export class Orchestrator {
1202
1181
  const deferred = job.req.remoteOnly !== true && job.req.download === 'deferred'
1203
1182
  const read = () =>
1204
1183
  job.req.remoteOnly === true || deferred ? this.o.artifacts.output(key) : restore()
1205
- const log = await (
1206
- unshared
1207
- ? Promise.resolve(wiredLog(head.output))
1208
- : // A failed download reads as a miss: asked once more, since one
1209
- // stalled download failed a CI run whose upload had landed
1210
- // (main, 2026-10-28).
1211
- read().then((got) => got ?? read())
1212
- ).catch((err: unknown) => {
1213
- job.reject(err)
1214
- return undefined
1215
- })
1184
+ // A failed download reads as a miss: asked once more, since one stalled
1185
+ // download failed a CI run whose upload had landed (main, 2026-10-28).
1186
+ const log = await read()
1187
+ .then((got) => got ?? read())
1188
+ .catch((err: unknown) => {
1189
+ job.reject(err)
1190
+ return undefined
1191
+ })
1216
1192
  this.trace.slice({
1217
1193
  track: ORCHESTRATOR,
1218
1194
  name: task,
@@ -1224,14 +1200,14 @@ export class Orchestrator {
1224
1200
  if (log === null) {
1225
1201
  job.reject(
1226
1202
  new Error(
1227
- `${task}: ${a.label} ran it, and no cache this machine reads gave its entry, asked twice: a download failed (warned above), ${a.label}'s upload failed (its log says why), or the workers and this machine share no remote cache the workers write`,
1203
+ `${task}: ${a.label} ran it, and no cache this machine reads gave its entry, asked twice: a download failed (warned above), or the workers and this machine share no remote cache the workers write`,
1228
1204
  ),
1229
1205
  )
1230
1206
  return
1231
1207
  }
1232
1208
  // An upstream's file the task removed there is gone here too: a restore
1233
1209
  // lays files, never removes one, and this disk is what core saves from.
1234
- if (head.removed !== undefined && job.req.remoteOnly !== true && !deferred && !unshared) {
1210
+ if (head.removed !== undefined && job.req.remoteOnly !== true && !deferred) {
1235
1211
  const ours = names(
1236
1212
  path.relative(this.o.root, node?.projectDir ?? job.req.cwd),
1237
1213
  job.req.outputs,
@@ -1267,29 +1243,17 @@ export class Orchestrator {
1267
1243
  for (const c of log) (c.err ? job.req.onStderr : job.req.onStdout)(c.text)
1268
1244
  job.resolve({
1269
1245
  ...result(head, log),
1270
- ...(unshared
1246
+ ...(deferred
1271
1247
  ? {
1272
1248
  outputs: {
1273
1249
  kind: 'deferred' as const,
1274
- materialize: () =>
1275
- Promise.reject(
1276
- new UserError(
1277
- `${task}: its files are on ${a.label} alone: no remote cache took its upload (its log says why)`,
1278
- ),
1279
- ),
1250
+ materialize: async () => {
1251
+ if ((await restore()) === null)
1252
+ throw new Error(`${task}'s entry ${key} is in no cache this machine reads`)
1253
+ },
1280
1254
  },
1281
1255
  }
1282
- : deferred
1283
- ? {
1284
- outputs: {
1285
- kind: 'deferred' as const,
1286
- materialize: async () => {
1287
- if ((await restore()) === null)
1288
- throw new Error(`${task}'s entry ${key} is in no cache this machine reads`)
1289
- },
1290
- },
1291
- }
1292
- : {}),
1256
+ : {}),
1293
1257
  where: a.label,
1294
1258
  machine: {
1295
1259
  cores: a.reported.cores,
@@ -1374,6 +1338,7 @@ export class Orchestrator {
1374
1338
  if (info.id !== undefined && this.late.delete(info.id)) {
1375
1339
  this.cancelled.delete(info.id)
1376
1340
  this.unstarted.delete(info.id)
1341
+ this.o.kept?.(info.id)
1377
1342
  }
1378
1343
  if (info.id !== undefined && this.cancelled.has(info.id)) {
1379
1344
  this.turnedAway.add(info.id)
@@ -1422,6 +1387,7 @@ export class Orchestrator {
1422
1387
  label,
1423
1388
  busy: new Map(),
1424
1389
  started: new Set(),
1390
+ batches: 0,
1425
1391
  has: new Set(),
1426
1392
  // Its name first, before any batch.
1427
1393
  chain: this.cache()
@@ -1487,67 +1453,99 @@ export class Orchestrator {
1487
1453
  }
1488
1454
  const a = this.workers.get(from)
1489
1455
  if (a === undefined) continue
1490
- const m = body as ToOrchestrator
1491
- if ('load' in m) this.heard(a, m.load)
1492
- if (m.type !== 'held') this.movedAt = performance.now()
1493
- if (m.type === 'start') {
1494
- this.begun(a, m.task)
1495
- this.pumpSoon()
1496
- } else if (m.type === 'held') {
1497
- for (const t of m.running) {
1498
- const j = a.busy.get(t)
1499
- if (j?.leaseUntil !== undefined) this.lease(j)
1500
- this.begun(a, t)
1501
- }
1502
- this.pumpSoon()
1503
- } else if (m.type === 'ran') {
1504
- // Its command ended: its cores are the next task's while it saves.
1505
- const j = a.busy.get(m.task)
1506
- if (j === undefined) continue
1507
- this.begun(a, m.task)
1508
- this.unhold(a, j)
1509
- // Its worker still says it holds it while it saves.
1510
- j.ranAt = performance.now()
1511
- this.pumpSoon()
1512
- } else if (m.type === 'fault') {
1513
- const j = a.busy.get(m.task)
1514
- if (j === undefined) continue
1515
- // It began it whether or not its `start` came first.
1516
- this.begun(a, m.task)
1517
- if (this.retry(a, j, m.reason) !== undefined) this.queue.unshift(j)
1518
- if (a.faulty) this.o.faulted?.()
1519
- if (a.faulty && this.spawnedAt.size > 0)
1520
- this.stranded(`${a.label} could not run ${m.task} (${m.reason}) and no worker is left`)
1521
- this.pumpSoon()
1522
- } else if (m.type === 'part') {
1523
- a.usage.messagesOut++
1524
- a.usage.bytesOut += bytesOf(m)
1525
- const k = `${from}\0${m.task}`
1526
- const held = this.parts.get(k) ?? {}
1527
- for (const list of PART_LISTS) {
1528
- const items = m[list] as readonly unknown[] | undefined
1529
- if (items !== undefined) held[list] = [...(held[list] ?? []), ...items]
1530
- }
1531
- this.parts.set(k, held)
1532
- } else if (m.type === 'end') {
1533
- a.usage.messagesOut++
1534
- a.usage.bytesOut += bytesOf(m)
1535
- const k = `${from}\0${m.task}`
1536
- const held = this.parts.get(k)
1537
- this.parts.delete(k)
1538
- const result: Record<string, unknown> = { ...m.result }
1539
- for (const list of PART_LISTS) {
1540
- const items = m.result[list] as readonly unknown[] | undefined
1541
- if (held?.[list] !== undefined) result[list] = [...held[list], ...(items ?? [])]
1456
+ const said = body as ToOrchestrator
1457
+ const frame = said.type === 'together' ? said.messages : [said]
1458
+ // A worker's times, read against its clock as this frame left it:
1459
+ // no two machines' clocks agree.
1460
+ const now = performance.now()
1461
+ let at: number | undefined
1462
+ for (const m of frame) if ('load' in m && m.load?.at !== undefined) at = Number(m.load.at)
1463
+ const here = (t: unknown): number | undefined =>
1464
+ at === undefined || typeof t !== 'number' || !Number.isFinite(at - t)
1465
+ ? undefined
1466
+ : Math.min(now, now - (at - t))
1467
+ for (const m of frame) {
1468
+ if ('load' in m && m.load !== undefined) this.heard(a, m.load)
1469
+ if (m.type !== 'report') this.movedAt = performance.now()
1470
+ if (m.type === 'report') {
1471
+ // Every task of the batches it read that has not ended is in its
1472
+ // report: one missing it no longer runs, and goes back at once.
1473
+ const running = new Set(m.running)
1474
+ const read = Number(m.batch) || 0
1475
+ const back: Job[] = []
1476
+ for (const [t, j] of [...a.busy])
1477
+ if (
1478
+ a.started.has(t) &&
1479
+ j.ended !== true &&
1480
+ (j.batch ?? 0) <= read &&
1481
+ !running.has(t)
1482
+ ) {
1483
+ const again = this.retry(a, j, 'not in its report')
1484
+ if (again !== undefined) back.push(again)
1485
+ }
1486
+ if (back.length > 0) {
1487
+ this.queue.unshift(...back)
1488
+ this.o.faulted?.()
1489
+ if (this.spawnedAt.size > 0)
1490
+ this.stranded(`${a.label} no longer runs a task it began and no worker is left`)
1491
+ }
1492
+ this.pumpSoon()
1493
+ } else if (m.type === 'ran') {
1494
+ // Its command ended: its cores are the next task's while it saves.
1495
+ const j = a.busy.get(m.task)
1496
+ if (j === undefined) continue
1497
+ this.begun(a, m.task)
1498
+ this.unhold(a, j)
1499
+ // Its worker still says it holds it while it saves.
1500
+ j.startedAt = here(m.startedAt) ?? j.startedAt ?? now
1501
+ j.ranAt = here(m.ranAt) ?? now
1502
+ this.pumpSoon()
1503
+ } else if (m.type === 'fault') {
1504
+ const j = a.busy.get(m.task)
1505
+ if (j === undefined) continue
1506
+ // It began it whether or not its `start` came first.
1507
+ this.begun(a, m.task)
1508
+ if (this.retry(a, j, m.reason) !== undefined) this.queue.unshift(j)
1509
+ if (a.faulty) this.o.faulted?.()
1510
+ if (a.faulty && this.spawnedAt.size > 0)
1511
+ this.stranded(
1512
+ `${a.label} could not run ${m.task} (${m.reason}) and no worker is left`,
1513
+ )
1514
+ this.pumpSoon()
1515
+ } else if (m.type === 'part') {
1516
+ a.usage.messagesOut++
1517
+ a.usage.bytesOut += bytesOf(m)
1518
+ const k = `${from}\0${m.task}`
1519
+ const held = this.parts.get(k) ?? {}
1520
+ for (const list of PART_LISTS) {
1521
+ const items = m[list] as readonly unknown[] | undefined
1522
+ if (items !== undefined) held[list] = [...(held[list] ?? []), ...items]
1523
+ }
1524
+ this.parts.set(k, held)
1525
+ } else if (m.type === 'end') {
1526
+ a.usage.messagesOut++
1527
+ a.usage.bytesOut += bytesOf(m)
1528
+ const k = `${from}\0${m.task}`
1529
+ const held = this.parts.get(k)
1530
+ this.parts.delete(k)
1531
+ const result: Record<string, unknown> = { ...m.result }
1532
+ for (const list of PART_LISTS) {
1533
+ const items = m.result[list] as readonly unknown[] | undefined
1534
+ if (held?.[list] !== undefined) result[list] = [...held[list], ...(items ?? [])]
1535
+ }
1536
+ // Not awaited: one task's restore holds no other message.
1537
+ const j = a.busy.get(m.task)
1538
+ if (j !== undefined) {
1539
+ j.ended = true
1540
+ const began = here(m.startedAt)
1541
+ const ran = here(m.ranAt)
1542
+ if (began !== undefined) j.startedAt = began
1543
+ if (ran !== undefined) j.ranAt = ran
1544
+ }
1545
+ void this.finish(a, m.task, m.key, result as WiredResult, here(m.endedAt))
1542
1546
  }
1543
- // Not awaited: one task's restore holds no other message.
1544
- void this.finish(a, m.task, m.key, result as WiredResult)
1545
1547
  }
1546
1548
  }
1547
- // After the batch: a `held` in it renews its lease first. Expired
1548
- // before, a stalled loop's waiting `held` came too late and the task
1549
- // ran again on another worker (2026-10-27).
1550
- this.expire()
1551
1549
  this.stall()
1552
1550
  }
1553
1551
  }
@@ -2155,7 +2153,7 @@ export class Orchestrator {
2155
2153
  }
2156
2154
 
2157
2155
  function result(r: WiredResult, log: readonly CapturedChunk[]): ExecuteResult {
2158
- const { cpuShare: _, removed: __, unread: ___, unshared: ____, output: _____, ...rest } = r
2156
+ const { cpuShare: _, removed: __, unread: ___, ...rest } = r
2159
2157
  const violations = (r.violations ?? []).map((v) => ({ ...v, timestamp: new Date(v.timestamp) }))
2160
2158
  const of = (err: boolean) =>
2161
2159
  log
@@ -2165,18 +2163,6 @@ function result(r: WiredResult, log: readonly CapturedChunk[]): ExecuteResult {
2165
2163
  return { ...rest, stdout: of(false), stderr: of(true), violations, outputs: { kind: 'disk' } }
2166
2164
  }
2167
2165
 
2168
- /** A worker's `output`, read at the boundary: its chunks of text. */
2169
- function wiredLog(output: unknown): CapturedChunk[] {
2170
- return Array.isArray(output)
2171
- ? output
2172
- .filter(
2173
- (c): c is CapturedChunk =>
2174
- typeof c === 'object' && c !== null && typeof c.text === 'string',
2175
- )
2176
- .map((c) => ({ text: c.text, err: c.err === true }))
2177
- : []
2178
- }
2179
-
2180
2166
  /**
2181
2167
  * A worker's `cpuShare`, read at the boundary: in (0, 1], else 1. A twentieth
2182
2168
  * at least: a share that small is a misread, and dividing by it would book
package/src/plugin.ts CHANGED
@@ -2,7 +2,7 @@ import path from 'node:path'
2
2
  import {
3
3
  definePlugin,
4
4
  hostPlatform,
5
- LocalHistoryProvider,
5
+ localHistory,
6
6
  PLATFORMS,
7
7
  refuseUnknownOptions,
8
8
  taskDefinitions,
@@ -76,6 +76,8 @@ export interface DteAdapter {
76
76
  * on joining, stuck or queued, so stopped whatever its job holds.
77
77
  */
78
78
  cancel?(ids: readonly string[], late?: boolean): void | Promise<readonly string[]>
79
+ /** Worker `id`, given up as late, joined after all and the run took it: a stop not yet sent is dropped. */
80
+ kept?(id: string): void
79
81
  /** The CI files that start its workers, which `vx dte init` writes; none for an adapter that starts them on this machine. */
80
82
  init?(setup: DteSetup): Promise<DteInit>
81
83
  /** Where it says a fault it works around (a status poll or a cancel that failed); called as the run starts. */
@@ -241,7 +243,7 @@ export function dte(options: DteOptions = {}) {
241
243
  // With no scheme the run failed in the executor as a bare "Invalid URL".
242
244
  if (options.relay !== undefined && !/^https?:\/\//.test(options.relay)) {
243
245
  throw new UserError(
244
- `dte() option "relay" must be an http(s) URL, got ${JSON.stringify(options.relay)}`,
246
+ `dte() option "relay" must be an http(s) URL, got ${JSON.stringify(shown(options.relay))}`,
245
247
  )
246
248
  }
247
249
  // On GitHub Actions with none given, one for the relay's URL.
@@ -385,7 +387,7 @@ export function dte(options: DteOptions = {}) {
385
387
  throw new UserError(`vx dte worker: needs ${missing.join(', and ')}`)
386
388
  }
387
389
  if (!/^https?:\/\//.test(relay))
388
- throw new UserError(`vx dte worker: VX_DTE_RELAY must be http(s), got ${relay}`)
390
+ throw new UserError(`vx dte worker: VX_DTE_RELAY must be http(s), got ${shown(relay)}`)
389
391
  const secret = givenSecret()
390
392
  if (secret === undefined && !onGitHub()) {
391
393
  throw new UserError(
@@ -436,12 +438,10 @@ export function dte(options: DteOptions = {}) {
436
438
  const read = async (): Promise<void> => {
437
439
  if (orchestrator === undefined) return
438
440
  try {
439
- peaks = await new LocalHistoryProvider(
440
- ctx.localCache.dbHandle(),
441
- undefined,
442
- ctx.branch,
443
- taskDefinitions(graph.values()),
444
- ).resourcesFor(ids)
441
+ peaks = await localHistory(ctx.localCache, {
442
+ branch: ctx.branch,
443
+ defs: taskDefinitions(graph.values()),
444
+ }).resourcesFor(ids)
445
445
  tails = undefined
446
446
  } catch (err) {
447
447
  ctx.warn(
@@ -487,13 +487,13 @@ export function dte(options: DteOptions = {}) {
487
487
  throw new UserError('dte(): no relay: set relay, or use an adapter that has one')
488
488
  const [
489
489
  { Orchestrator },
490
- { join },
490
+ { joinOffLoop },
491
491
  { decide, knownMisses, slowdown, tailsMs, WARM_UP_MS },
492
492
  { snapshot },
493
493
  seal,
494
494
  ] = await Promise.all([
495
495
  import('./orchestrator.js'),
496
- import('./wire.js'),
496
+ import('./wire-thread.js'),
497
497
  import('./planner.js'),
498
498
  import('./tree.js'),
499
499
  import('./seal.js'),
@@ -523,7 +523,9 @@ export function dte(options: DteOptions = {}) {
523
523
  let onChange = (_now?: boolean): void => {}
524
524
  const o = new Orchestrator({
525
525
  me: seal.sealOrchestrator(
526
- await join(relay, run, 'orchestrator', null, () => token(relay), headers),
526
+ // Off this loop: the run's own sync work held its acks, and every
527
+ // worker then running ended (CI, 2026-10-29).
528
+ await joinOffLoop(relay, run, null, () => token(relay), headers),
527
529
  run,
528
530
  proof,
529
531
  ctx.warn,
@@ -552,6 +554,7 @@ export function dte(options: DteOptions = {}) {
552
554
  joined: () => onChange(),
553
555
  // A worker given up on joining: its job, still queued or stuck, is stopped.
554
556
  overdue: (id) => void adapter?.cancel?.([id], true),
557
+ kept: (id) => adapter?.kept?.(id),
555
558
  // Before the run counts what is left to start: a worker that let a
556
559
  // task's lease end failed main's macOS run with another to start
557
560
  // (2026-10-23).
@@ -1018,3 +1021,8 @@ export function homeOf(sizes: readonly Size[], host: string): string | undefined
1018
1021
  const of = new Set(sizes.filter((s) => s.max > 0).map((s) => s.platform))
1019
1022
  return of.has(host) && of.size > 1 ? host : undefined
1020
1023
  }
1024
+
1025
+ /** A relay URL as a refusal may print it: a `user:pass@` would reach the CI log. */
1026
+ function shown(url: string): string {
1027
+ return url.replace(/^([a-z][a-z0-9+.-]*:\/\/)[^/?#]*@/i, '$1***@')
1028
+ }
package/src/seal.ts CHANGED
@@ -40,17 +40,22 @@ const GITHUB = 'https://token.actions.githubusercontent.com'
40
40
  /** GitHub's keys are read again for a key not among them at most this often. */
41
41
  const KEYS_REREAD_MS = 60_000
42
42
 
43
+ /** One try at GitHub's token endpoint, which answers in well under a second. */
44
+ const TOKEN_TRY_MS = 3_000
45
+
43
46
  /**
44
- * An OIDC token for `audience` from GitHub's token endpoint. A 5xx or a
45
- * dropped connection is tried again, three tries in 10 s at most: one 503
46
- * ended a worker before it joined (PR #4146, 2026-10-23).
47
+ * An OIDC token for `audience` from GitHub's token endpoint. A 5xx, a
48
+ * dropped connection or a try that hangs past 3 s is tried again, three
49
+ * tries in 10 s at most: one 503 ended a worker before it joined (PR #4146,
50
+ * 2026-10-23), and one held request spent the whole budget (PR #4887,
51
+ * 2026-10-31).
47
52
  */
48
53
  export async function oidcToken(url: string, bearer: string, audience: string): Promise<string> {
49
54
  const until = performance.now() + 10_000
50
55
  for (let attempt = 1; ; attempt++) {
51
56
  const res = await fetch(`${url}&audience=${encodeURIComponent(audience)}`, {
52
57
  headers: { authorization: `bearer ${bearer}` },
53
- signal: AbortSignal.timeout(Math.max(1, until - performance.now())),
58
+ signal: AbortSignal.timeout(Math.max(1, Math.min(TOKEN_TRY_MS, until - performance.now()))),
54
59
  }).catch((err: unknown) => (err instanceof Error ? err.message : String(err)))
55
60
  if (typeof res !== 'string' && res.ok) {
56
61
  // quiet: a body that is not JSON holds no token, said below
@@ -371,6 +376,7 @@ export function sealOrchestrator(
371
376
  return me.online
372
377
  },
373
378
  resumed: (f) => me.resumed(f),
379
+ quiet: (f) => me.quiet(f),
374
380
  async send(to, body) {
375
381
  const s = sessions.get(to)
376
382
  // Lost or never admitted: the relay's own answer for a peer it does
@@ -468,6 +474,7 @@ export async function sealWorker(
468
474
  return me.online
469
475
  },
470
476
  resumed: (f) => me.resumed(f),
477
+ quiet: (f) => me.quiet(f),
471
478
  async send(to, body) {
472
479
  if (session === undefined) throw new Error('no session with the orchestrator yet')
473
480
  await session.send(body, (sealed) => me.send(to, sealed))
package/src/tree.ts CHANGED
@@ -2,7 +2,7 @@
2
2
  // them. Workers compare trees, not commits, so a run on uncommitted changes
3
3
  // still matches the worktrees `local()` makes from them.
4
4
 
5
- import { copyFile, mkdtemp, rm } from 'node:fs/promises'
5
+ import { copyFile, mkdtemp, rm, stat, utimes } from 'node:fs/promises'
6
6
  import os from 'node:os'
7
7
  import path from 'node:path'
8
8
 
@@ -27,6 +27,12 @@ export async function snapshot(root: string): Promise<{ commit: string; tree: st
27
27
  const env = { GIT_INDEX_FILE: path.join(dir, 'index') }
28
28
  try {
29
29
  await copyFile(index, env.GIT_INDEX_FILE)
30
+ // The copy keeps the index's mtime (as core's refreshedIndex): git
31
+ // trusts an entry's stat only while its file is older than the index,
32
+ // and a fresh mtime took a same-size edit made in the tick the index
33
+ // was written for clean, its old blob in the tree (2026-10-29).
34
+ const st = await stat(index)
35
+ await utimes(env.GIT_INDEX_FILE, st.atime, st.mtime)
30
36
  } catch {
31
37
  // quiet: no index yet, so HEAD's tree is its tracked set.
32
38
  if ((await git(['read-tree', head], root, env)) === undefined) return none