@vzn/vx-dte 0.0.652 → 0.0.653
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +135 -62
- package/package.json +3 -3
- package/src/dispatch.ts +39 -4
- package/src/github.ts +0 -3
- package/src/index.ts +1 -0
- package/src/init.ts +9 -10
- package/src/local.ts +112 -28
- package/src/messages.ts +14 -3
- package/src/orchestrator.ts +265 -33
- package/src/planner.ts +183 -55
- package/src/plugin.ts +209 -30
- package/src/seal.ts +159 -43
- package/src/server-fit.ts +80 -0
- package/src/tree.ts +215 -7
- package/src/usage.ts +65 -3
- package/src/warm-ups.ts +72 -6
- package/src/wire.ts +84 -42
- package/src/worker.ts +148 -15
package/src/orchestrator.ts
CHANGED
|
@@ -63,7 +63,8 @@ import { names } from './outputs.js'
|
|
|
63
63
|
|
|
64
64
|
interface Job {
|
|
65
65
|
readonly req: ExecuteRequest
|
|
66
|
-
|
|
66
|
+
/** Read as it joined the queue, and again while it waits when history changes (`rerank`). */
|
|
67
|
+
need: Need
|
|
67
68
|
/** The cores and memory it holds on its worker: its need, or the whole worker for one that filled its worker. */
|
|
68
69
|
held?: Need
|
|
69
70
|
readonly resolve: (r: ExecuteResult) => void
|
|
@@ -141,6 +142,12 @@ interface Worker {
|
|
|
141
142
|
load?: { cpu: number; memory: number; at: number }
|
|
142
143
|
/** Tasks whose outputs it holds: it ran them, or was sent them. */
|
|
143
144
|
readonly has: Set<string>
|
|
145
|
+
/**
|
|
146
|
+
* The persistent tasks it runs, each with the base it books there
|
|
147
|
+
* (`Need.servers`): booked once, by the first task placed there that
|
|
148
|
+
* needs it, and given back when the run says none needs it.
|
|
149
|
+
*/
|
|
150
|
+
readonly serving: Map<string, number>
|
|
144
151
|
/** Each dependency it was sent whole, by the key it was sent under: the next order names it alone. */
|
|
145
152
|
readonly sentDeps: Map<string, string>
|
|
146
153
|
/** Sends to it go out in order: a run's parts before the run. */
|
|
@@ -158,6 +165,8 @@ export interface Declared {
|
|
|
158
165
|
readonly memory: number
|
|
159
166
|
/** Its size's platform: a worker reporting another is refused. None: planned as this machine's. */
|
|
160
167
|
readonly platform?: string
|
|
168
|
+
/** Where tasks match it when the run has several worker types: `<type>:<platform>`. */
|
|
169
|
+
readonly place?: string
|
|
161
170
|
}
|
|
162
171
|
|
|
163
172
|
export interface OrchestratorOptions {
|
|
@@ -167,6 +176,8 @@ export interface OrchestratorOptions {
|
|
|
167
176
|
readonly tree: string
|
|
168
177
|
readonly nodes: () => ReadonlyMap<string, TaskNode> | undefined
|
|
169
178
|
readonly need: (taskId: string) => Need
|
|
179
|
+
/** Each server's base, cores, as workers fit them: updated in place. */
|
|
180
|
+
readonly bases?: Map<string, number>
|
|
170
181
|
/** The run's cache, by key: what workers save to and this machine restores from. */
|
|
171
182
|
readonly artifacts: TaskArtifacts
|
|
172
183
|
/** What workers read the cache under (`ExecutorContext.cacheFacts`). */
|
|
@@ -197,6 +208,11 @@ export interface OrchestratorOptions {
|
|
|
197
208
|
readonly how?: Strategy
|
|
198
209
|
/** The platform a task naming none runs on (`homeOf`). */
|
|
199
210
|
readonly home?: string | undefined
|
|
211
|
+
/** A joined worker's place, by its size and the platform it reported: absent, the platform. */
|
|
212
|
+
readonly joinedPlace?: (
|
|
213
|
+
size: string | undefined,
|
|
214
|
+
platform: string | undefined,
|
|
215
|
+
) => string | undefined
|
|
200
216
|
/** No task began or ended for `stallMs`: what the run waits on, one line a task. */
|
|
201
217
|
readonly stalled?: (waiting: string) => void
|
|
202
218
|
/** A fault the run works around (a message the relay would not take); default stderr. */
|
|
@@ -233,6 +249,8 @@ const ADVICE: Record<WorkerAdvice['of'], readonly [label: string, note: string]>
|
|
|
233
249
|
more: ['more', "past its sizes' max"],
|
|
234
250
|
any: ['any', 'of any cores, the least core time'],
|
|
235
251
|
}
|
|
252
|
+
/** A platform as shown: a worker type's place (`ubuntu:linux-x64`) without its type, which its size names. */
|
|
253
|
+
const bare = (platform: string): string => platform.slice(platform.indexOf(':') + 1)
|
|
236
254
|
/** `3 × small (4c/16G linux)`. */
|
|
237
255
|
const fleetText = (w: WorkerAdvice['workers'][number]): string =>
|
|
238
256
|
`${w.count} × ${w.size === undefined ? '' : `${w.size} `}(${w.cores}c/${compact(w.memoryBytes)}${w.platform === undefined ? '' : ` ${w.platform}`})`
|
|
@@ -290,6 +308,8 @@ export class Orchestrator {
|
|
|
290
308
|
private inTree?: Promise<Tree | undefined>
|
|
291
309
|
/** Waiting tasks, by id, in the order a pump reads them (`Queue`). */
|
|
292
310
|
private readonly waiting = new Map<string, Job>()
|
|
311
|
+
/** Handed by core and not yet waiting: held for a machine as the waiting are. */
|
|
312
|
+
private readonly handed = new Set<string>()
|
|
293
313
|
private readonly queue: Queue
|
|
294
314
|
/** Where a worker saved each task it ran: its cache key or its run key. */
|
|
295
315
|
private readonly savedAs = new Map<string, string>()
|
|
@@ -321,6 +341,10 @@ export class Orchestrator {
|
|
|
321
341
|
private readonly retired = new Set<Worker>()
|
|
322
342
|
/** Workers asked for that never joined, with why. */
|
|
323
343
|
private readonly unstarted = new Map<string, string>()
|
|
344
|
+
/** A task waited here on a worker: one ended with none joined needed one. */
|
|
345
|
+
private waited = false
|
|
346
|
+
/** Why each worker this run refused was refused, in order. */
|
|
347
|
+
private readonly refused: string[] = []
|
|
324
348
|
/** Started, then not needed before they joined (`release`), or given up as late. */
|
|
325
349
|
private readonly cancelled = new Set<string>()
|
|
326
350
|
/** Of `cancelled`, those given up past their join limit, not let go: taken if they join. */
|
|
@@ -407,11 +431,32 @@ export class Orchestrator {
|
|
|
407
431
|
|
|
408
432
|
async execute(req: ExecuteRequest): Promise<ExecuteResult> {
|
|
409
433
|
if (this.lost !== undefined) throw new Error(this.lost)
|
|
434
|
+
this.handed.add(req.taskId)
|
|
435
|
+
try {
|
|
436
|
+
return await this.take(req)
|
|
437
|
+
} finally {
|
|
438
|
+
this.handed.delete(req.taskId)
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
private async take(req: ExecuteRequest): Promise<ExecuteResult> {
|
|
410
443
|
// Walked once here, for every later read: walked again on the way in,
|
|
411
444
|
// sharing, placing and sending, a run of 8000 tasks spent 17 s of this
|
|
412
445
|
// machine's CPU on it (2026-11-05).
|
|
413
446
|
const nodes = this.o.nodes()
|
|
414
447
|
const up = upstream(nodes, req.taskId)
|
|
448
|
+
// A worker runs a task in its project's directory of the run's tree: a
|
|
449
|
+
// project whose config is untracked is not there, and every worker
|
|
450
|
+
// failed to spawn it, exit 127 (2026-11-07).
|
|
451
|
+
const dir = path.relative(this.o.root, req.cwd)
|
|
452
|
+
if (dir !== '' && !dir.startsWith('..')) {
|
|
453
|
+
this.inTree ??= listTree(this.o.root, this.o.tree)
|
|
454
|
+
const tree = await this.inTree
|
|
455
|
+
if (tree !== undefined && !holds(tree, dir))
|
|
456
|
+
throw new UserError(
|
|
457
|
+
`${req.taskId} runs in ${dir}, which is not in the commit workers check out: commit its project (its vx.config file is untracked)`,
|
|
458
|
+
)
|
|
459
|
+
}
|
|
415
460
|
// A worker runs the run's tree: a file its key includes that the tree
|
|
416
461
|
// lacks (untracked) is not there, and the output it saved under that
|
|
417
462
|
// key was a stale hit for the next run (2026-10-24).
|
|
@@ -440,6 +485,21 @@ export class Orchestrator {
|
|
|
440
485
|
: `${req.taskId} keys ${stray.path}, in submodule ${link}, whose files workers do not check out: leave it out of the task's inputs`,
|
|
441
486
|
)
|
|
442
487
|
}
|
|
488
|
+
// A worker runs the commit's bytes and modes (`unfilter`): a file on
|
|
489
|
+
// disk otherwise (CRLF a git filter wrote, an expanded ident, an LFS
|
|
490
|
+
// file's content) is not what it builds on, and the output saved
|
|
491
|
+
// under the key of this disk's bytes was a stale hit (2026-11-08).
|
|
492
|
+
const off =
|
|
493
|
+
tree === undefined
|
|
494
|
+
? undefined
|
|
495
|
+
: req.inputs.files.find((f) => {
|
|
496
|
+
const has = tree.files.get(f.path)
|
|
497
|
+
return has !== undefined && has !== f.digest && !made(f.path)
|
|
498
|
+
})
|
|
499
|
+
if (off !== undefined)
|
|
500
|
+
throw new UserError(
|
|
501
|
+
`${req.taskId} keys ${off.path}, which is on disk other than the commit workers check out holds it (a git filter's text, eol, ident or LFS bytes, or core.autocrlf): leave it out of the task's inputs, or run the task without dte`,
|
|
502
|
+
)
|
|
443
503
|
}
|
|
444
504
|
for (const u of req.inputs?.upstream ?? []) this.upstreamKeys.set(u.taskId, u.hash)
|
|
445
505
|
await this.share(up, nodes)
|
|
@@ -456,6 +516,7 @@ export class Orchestrator {
|
|
|
456
516
|
// A retry runs elsewhere: the worker, or what ran beside it, may be the flake.
|
|
457
517
|
const last = (req.attempt ?? 1) > 1 ? this.ranOn.get(req.taskId) : undefined
|
|
458
518
|
if (last !== undefined) job.avoid = last
|
|
519
|
+
this.handed.delete(req.taskId)
|
|
459
520
|
// A signal aborted already fires no `abort`.
|
|
460
521
|
if (req.signal?.aborted === true) return resolve(stopped())
|
|
461
522
|
req.signal?.addEventListener('abort', () => this.abort(job), { once: true })
|
|
@@ -534,6 +595,17 @@ export class Orchestrator {
|
|
|
534
595
|
return this.trace.events(this.ended ? this.endedAt : performance.now())
|
|
535
596
|
}
|
|
536
597
|
|
|
598
|
+
/** No task left in the run uses server `task`: each worker that started it stops it. */
|
|
599
|
+
unneeded(task: string): void {
|
|
600
|
+
for (const a of this.workers.values()) {
|
|
601
|
+
const base = a.serving.get(task)
|
|
602
|
+
if (base === undefined) continue
|
|
603
|
+
a.serving.delete(task)
|
|
604
|
+
a.usedCores -= base
|
|
605
|
+
this.post(a, { type: 'unserve', task })
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
|
|
537
609
|
private abort(job: Job): void {
|
|
538
610
|
if (this.waiting.get(job.req.taskId) === job) {
|
|
539
611
|
this.waiting.delete(job.req.taskId)
|
|
@@ -555,9 +627,23 @@ export class Orchestrator {
|
|
|
555
627
|
})
|
|
556
628
|
}
|
|
557
629
|
|
|
630
|
+
/**
|
|
631
|
+
* History changed (`need`, `tail`): the waiting book and rank by it. On
|
|
632
|
+
* CI the default branch's timings land a moment after the roots joined
|
|
633
|
+
* the queue, each booked as unmeasured and ranked as an unknown task:
|
|
634
|
+
* shards ready later went first, and test.wire waited 129 s (2026-11-07).
|
|
635
|
+
*/
|
|
636
|
+
rerank(): void {
|
|
637
|
+
for (const [id, job] of this.waiting)
|
|
638
|
+
job.need = { ...this.o.need(id), platform: job.need.platform }
|
|
639
|
+
this.queue.rerank()
|
|
640
|
+
this.pumpSoon()
|
|
641
|
+
}
|
|
642
|
+
|
|
558
643
|
/** `jobs` wait, in order, behind the waiting tasks or, `front`, before them. */
|
|
559
644
|
private enqueue(jobs: readonly Job[], front = false): void {
|
|
560
645
|
for (const j of jobs) this.waiting.set(j.req.taskId, j)
|
|
646
|
+
if (jobs.length > 0) this.waited = true
|
|
561
647
|
this.queue.add(
|
|
562
648
|
jobs.map((j) => j.req.taskId),
|
|
563
649
|
front,
|
|
@@ -582,6 +668,7 @@ export class Orchestrator {
|
|
|
582
668
|
running,
|
|
583
669
|
has: a.has,
|
|
584
670
|
platform: a.platform,
|
|
671
|
+
serving: new Set(a.serving.keys()),
|
|
585
672
|
a,
|
|
586
673
|
}
|
|
587
674
|
})
|
|
@@ -606,6 +693,7 @@ export class Orchestrator {
|
|
|
606
693
|
const a = to.get(job.req.taskId)!.a
|
|
607
694
|
job.worker = a.id
|
|
608
695
|
job.sentAt = performance.now()
|
|
696
|
+
a.usage.sent(job.sentAt)
|
|
609
697
|
a.busy.set(job.req.taskId, job)
|
|
610
698
|
job.held = {
|
|
611
699
|
cores: Math.min(this.booking(job.need).cores, a.cores),
|
|
@@ -622,6 +710,12 @@ export class Orchestrator {
|
|
|
622
710
|
})
|
|
623
711
|
a.usedCores += job.held.cores
|
|
624
712
|
a.usedMemory += job.held.memory
|
|
713
|
+
// A server it starts there books its base once, while it runs.
|
|
714
|
+
for (const sv of job.need.servers ?? [])
|
|
715
|
+
if (!a.serving.has(sv.id)) {
|
|
716
|
+
a.serving.set(sv.id, sv.cores)
|
|
717
|
+
a.usedCores += sv.cores
|
|
718
|
+
}
|
|
625
719
|
const batch = batches.get(a)
|
|
626
720
|
if (batch === undefined) batches.set(a, [job])
|
|
627
721
|
else batch.push(job)
|
|
@@ -709,7 +803,11 @@ export class Orchestrator {
|
|
|
709
803
|
return this.o.pinWaitMs ?? PIN_WAIT_MS
|
|
710
804
|
}
|
|
711
805
|
|
|
712
|
-
/**
|
|
806
|
+
/**
|
|
807
|
+
* `ids`, uncached tasks that ran, run again where `where` says, for a
|
|
808
|
+
* dependant: on its place, which the plan refused when one names another
|
|
809
|
+
* worker type or platform (`apart`).
|
|
810
|
+
*/
|
|
713
811
|
private remake(
|
|
714
812
|
ids: readonly string[],
|
|
715
813
|
where: { only?: string; avoid?: string; platform?: string | undefined },
|
|
@@ -813,7 +911,15 @@ export class Orchestrator {
|
|
|
813
911
|
if (load === undefined) return
|
|
814
912
|
a.load = { cpu: Number(load.cpu) || 0, memory: Number(load.memory) || 0, at: performance.now() }
|
|
815
913
|
const avg = Number(load.avg)
|
|
816
|
-
|
|
914
|
+
const self = Number(load.self)
|
|
915
|
+
if (Number.isFinite(avg) && avg >= 0)
|
|
916
|
+
a.usage.machine(avg, Number.isFinite(self) && self >= 0 ? self : undefined, a.load.at)
|
|
917
|
+
// Its servers' bases as it fits them (`server-fit.ts`): what the next
|
|
918
|
+
// task needing one books, and the next run.
|
|
919
|
+
if (typeof load.servers === 'object' && load.servers !== null)
|
|
920
|
+
for (const [id, cores] of Object.entries(load.servers))
|
|
921
|
+
if (typeof cores === 'number' && Number.isFinite(cores) && cores >= 0 && cores <= a.cores)
|
|
922
|
+
this.o.bases?.set(id, cores)
|
|
817
923
|
const tasks = Number(load.tasks)
|
|
818
924
|
this.trace.load(a.label, a.load.cpu, a.load.at, Number.isFinite(tasks) ? tasks : undefined)
|
|
819
925
|
}
|
|
@@ -829,7 +935,8 @@ export class Orchestrator {
|
|
|
829
935
|
!joined.has(id) &&
|
|
830
936
|
!this.unstarted.has(id) &&
|
|
831
937
|
!this.cancelled.has(id) &&
|
|
832
|
-
(platform === undefined ||
|
|
938
|
+
(platform === undefined ||
|
|
939
|
+
(declared.place ?? declared.platform ?? hostPlatform()) === platform) &&
|
|
833
940
|
(declared.cpu > above.cores || declared.memory > above.memory)
|
|
834
941
|
)
|
|
835
942
|
return true
|
|
@@ -942,7 +1049,9 @@ export class Orchestrator {
|
|
|
942
1049
|
// A task sent here leaves its outputs here for its dependants.
|
|
943
1050
|
a.has.add(req.taskId)
|
|
944
1051
|
const node = nodes?.get(req.taskId)
|
|
945
|
-
const
|
|
1052
|
+
const keyed = hostKeyed(req)
|
|
1053
|
+
const env = defined(req.env, req.envDefine, keyed)
|
|
1054
|
+
const unset = keyed.filter((k) => req.env[k] === undefined && !(k in req.envDefine))
|
|
946
1055
|
const order = {
|
|
947
1056
|
task: req.taskId,
|
|
948
1057
|
// Unsaved: under the run's own key, as an uncached task's.
|
|
@@ -960,6 +1069,7 @@ export class Orchestrator {
|
|
|
960
1069
|
cwd: path.relative(this.o.root, req.cwd),
|
|
961
1070
|
project: node === undefined ? '' : path.relative(this.o.root, node.projectDir),
|
|
962
1071
|
env,
|
|
1072
|
+
...(unset.length === 0 ? {} : { unset }),
|
|
963
1073
|
...(node?.config.exec?.env?.secret === undefined
|
|
964
1074
|
? {}
|
|
965
1075
|
: { secret: [...node.config.exec.env.secret] }),
|
|
@@ -983,6 +1093,13 @@ export class Orchestrator {
|
|
|
983
1093
|
: {
|
|
984
1094
|
reads: req.sandbox.dependencyReads.map((d) => path.relative(this.o.root, d)),
|
|
985
1095
|
}),
|
|
1096
|
+
...(req.sandbox.dependencyReported === undefined
|
|
1097
|
+
? {}
|
|
1098
|
+
: {
|
|
1099
|
+
reported: req.sandbox.dependencyReported.map((d) =>
|
|
1100
|
+
path.relative(this.o.root, d),
|
|
1101
|
+
),
|
|
1102
|
+
}),
|
|
986
1103
|
},
|
|
987
1104
|
}),
|
|
988
1105
|
...runtimeOrders(req),
|
|
@@ -1070,6 +1187,9 @@ export class Orchestrator {
|
|
|
1070
1187
|
// Its orders' bytes, a comma between each, in the batch's own.
|
|
1071
1188
|
a.usage.bytesIn += bytesOf({ ...batch, tasks: [] }) + bytes + tasks.length - 1
|
|
1072
1189
|
for (const { intro } of part) for (const [t, k] of intro) a.sentDeps.set(t, k)
|
|
1190
|
+
for (const { order } of part)
|
|
1191
|
+
for (const sv of order.servers ?? [])
|
|
1192
|
+
if (!a.serving.has(sv.task)) a.serving.set(sv.task, 0)
|
|
1073
1193
|
for (const { job } of part) {
|
|
1074
1194
|
pending.delete(job)
|
|
1075
1195
|
// Sent, it runs: its worker takes only what it has room for.
|
|
@@ -1403,7 +1523,11 @@ export class Orchestrator {
|
|
|
1403
1523
|
if (body.type === 'joined') {
|
|
1404
1524
|
const info = body.info as Partial<WorkerInfo> | null
|
|
1405
1525
|
const id = body.id!
|
|
1526
|
+
// Only the worker heard why: the run sat silent and its footer said
|
|
1527
|
+
// none joined (2026-11-07).
|
|
1406
1528
|
const refuse = (reason: string): void => {
|
|
1529
|
+
this.refused.push(reason)
|
|
1530
|
+
this.warn(`dte: refused worker ${typeof info?.id === 'string' ? info.id : id}: ${reason}`)
|
|
1407
1531
|
void this.o.me
|
|
1408
1532
|
.send(id, { type: 'refused', reason } satisfies ToWorker)
|
|
1409
1533
|
.catch(this.unsent('refused', id))
|
|
@@ -1475,7 +1599,7 @@ export class Orchestrator {
|
|
|
1475
1599
|
cores,
|
|
1476
1600
|
memory,
|
|
1477
1601
|
reported: { cores: Number(info.cores), memory: Number(info.memory) },
|
|
1478
|
-
platform,
|
|
1602
|
+
platform: this.o.joinedPlace?.(spawn?.declared.size, platform) ?? platform,
|
|
1479
1603
|
usedCores: 0,
|
|
1480
1604
|
usedMemory: 0,
|
|
1481
1605
|
label,
|
|
@@ -1483,6 +1607,7 @@ export class Orchestrator {
|
|
|
1483
1607
|
started: new Set(),
|
|
1484
1608
|
batches: 0,
|
|
1485
1609
|
has: new Set(),
|
|
1610
|
+
serving: new Map(),
|
|
1486
1611
|
sentDeps: new Map(),
|
|
1487
1612
|
// Its name first, before any batch.
|
|
1488
1613
|
chain: this.cache()
|
|
@@ -1514,6 +1639,8 @@ export class Orchestrator {
|
|
|
1514
1639
|
a.usage.to = performance.now()
|
|
1515
1640
|
this.trace.left(a.label, 'lost', a.usage.to)
|
|
1516
1641
|
this.gone.push(a)
|
|
1642
|
+
// Its results' `end`s never come.
|
|
1643
|
+
for (const k of this.parts.keys()) if (k.startsWith(`${a.id}\0`)) this.parts.delete(k)
|
|
1517
1644
|
// Its tasks run again: first, as they were first to go.
|
|
1518
1645
|
const back: Job[] = []
|
|
1519
1646
|
for (const j of [...a.busy.values()]) {
|
|
@@ -1649,7 +1776,7 @@ export class Orchestrator {
|
|
|
1649
1776
|
where(taskId: string): string | null | undefined {
|
|
1650
1777
|
for (const a of this.workers.values())
|
|
1651
1778
|
if (a.busy.has(taskId)) return a.started.has(taskId) ? a.label : null
|
|
1652
|
-
return this.waiting.has(taskId) ? null : undefined
|
|
1779
|
+
return this.waiting.has(taskId) || this.handed.has(taskId) ? null : undefined
|
|
1653
1780
|
}
|
|
1654
1781
|
|
|
1655
1782
|
/**
|
|
@@ -1705,7 +1832,7 @@ export class Orchestrator {
|
|
|
1705
1832
|
}): void {
|
|
1706
1833
|
const work = p.work?.map(
|
|
1707
1834
|
(t) =>
|
|
1708
|
-
`${t.id} ${t.ms === undefined ? '?' : Math.round(t.ms / 100) / 10}s ${Math.round(t.cores * 100) / 100}c${t.platform === undefined ? '' : ` ${t.platform}`}`,
|
|
1835
|
+
`${t.id} ${t.ms === undefined ? '?' : Math.round(t.ms / 100) / 10}s ${Math.round(t.cores * 100) / 100}c${t.platform === undefined ? '' : ` ${bare(t.platform)}`}`,
|
|
1709
1836
|
)
|
|
1710
1837
|
const workers = p.workers.map((w) =>
|
|
1711
1838
|
w.starting === true
|
|
@@ -1811,6 +1938,7 @@ export class Orchestrator {
|
|
|
1811
1938
|
cores: a.cores,
|
|
1812
1939
|
memory: a.memory,
|
|
1813
1940
|
platform: a.platform,
|
|
1941
|
+
...(a.serving.size === 0 ? {} : { serving: new Map(a.serving) }),
|
|
1814
1942
|
...(a.declared === undefined ? {} : { size: a.declared.size }),
|
|
1815
1943
|
at: 0,
|
|
1816
1944
|
idle: a.busy.size === 0,
|
|
@@ -1837,7 +1965,7 @@ export class Orchestrator {
|
|
|
1837
1965
|
// A size naming none is planned as this machine's, as `dte()`
|
|
1838
1966
|
// planned it: on no platform, no task naming one fit it, and the
|
|
1839
1967
|
// next plan cancelled it.
|
|
1840
|
-
platform: declared.platform ?? hostPlatform(),
|
|
1968
|
+
platform: declared.place ?? declared.platform ?? hostPlatform(),
|
|
1841
1969
|
size: declared.size,
|
|
1842
1970
|
at: Math.max(
|
|
1843
1971
|
0,
|
|
@@ -1969,8 +2097,14 @@ export class Orchestrator {
|
|
|
1969
2097
|
|
|
1970
2098
|
/** Every task sent here and not done fails with `reason`; later ones too. */
|
|
1971
2099
|
private fail(reason: string): void {
|
|
1972
|
-
|
|
2100
|
+
const now = performance.now()
|
|
2101
|
+
for (const a of this.workers.values()) {
|
|
1973
2102
|
for (const j of a.busy.values()) j.reject(new Error(reason))
|
|
2103
|
+
// Lost, keeping what it ran: dropped, the footer read it as never joined.
|
|
2104
|
+
a.usage.to = now
|
|
2105
|
+
this.trace.left(a.label, 'lost', now)
|
|
2106
|
+
this.gone.push(a)
|
|
2107
|
+
}
|
|
1974
2108
|
for (const [id, j] of this.waiting) {
|
|
1975
2109
|
this.queue.delete(id)
|
|
1976
2110
|
j.reject(new Error(reason))
|
|
@@ -2080,7 +2214,7 @@ export class Orchestrator {
|
|
|
2080
2214
|
count: w.count,
|
|
2081
2215
|
cores: w.cores,
|
|
2082
2216
|
memoryBytes: w.memory,
|
|
2083
|
-
...(w.platform === undefined ? {} : { platform: w.platform }),
|
|
2217
|
+
...(w.platform === undefined ? {} : { platform: bare(w.platform) }),
|
|
2084
2218
|
...(w.size === undefined || sizes.length === 0 ? {} : { size: w.size }),
|
|
2085
2219
|
})),
|
|
2086
2220
|
},
|
|
@@ -2185,9 +2319,25 @@ export class Orchestrator {
|
|
|
2185
2319
|
advice?: () => readonly WorkerAdvice[],
|
|
2186
2320
|
): [string, ui.Inline][] {
|
|
2187
2321
|
// Over with none, it says so: core's own `N workers` (this machine's
|
|
2188
|
-
// pool) read as the run's when no task needed one (2026-10-25)
|
|
2322
|
+
// pool) read as the run's when no task needed one (2026-10-25), and
|
|
2323
|
+
// says when one did and none joined (2026-11-06), and why any it
|
|
2324
|
+
// refused was (2026-11-07).
|
|
2189
2325
|
if (this.spawnedAt.size === 0 && this.workers.size === 0 && this.gone.length === 0)
|
|
2190
|
-
return this.ended
|
|
2326
|
+
return this.ended
|
|
2327
|
+
? [
|
|
2328
|
+
[
|
|
2329
|
+
'workers',
|
|
2330
|
+
this.refused.length > 0
|
|
2331
|
+
? ui.tone(
|
|
2332
|
+
'warn',
|
|
2333
|
+
`none joined: ${this.refused.length} refused (${this.refused[0]})`,
|
|
2334
|
+
)
|
|
2335
|
+
: this.waited
|
|
2336
|
+
? ui.tone('warn', 'none joined: a task waited on one')
|
|
2337
|
+
: ui.dim('none started: no task needed one'),
|
|
2338
|
+
],
|
|
2339
|
+
]
|
|
2340
|
+
: []
|
|
2191
2341
|
const gone = new Set(this.gone)
|
|
2192
2342
|
const all = [...this.workers.values(), ...this.gone]
|
|
2193
2343
|
const joined = new Set(all.map((a) => a.label))
|
|
@@ -2257,13 +2407,18 @@ export class Orchestrator {
|
|
|
2257
2407
|
// Once over, what each did, under its line.
|
|
2258
2408
|
const usage = new Map<string, readonly string[]>()
|
|
2259
2409
|
let cpuMs = 0
|
|
2410
|
+
let tasksMs = 0
|
|
2411
|
+
let workerMs: number | undefined
|
|
2260
2412
|
let coreMs = 0
|
|
2261
2413
|
if (this.ended)
|
|
2262
2414
|
for (const a of all) {
|
|
2263
2415
|
resources(a.label, a.reported.cores, a.reported.memory, a.declared)
|
|
2264
2416
|
const st = a.usage.stats(a.cores, this.endedAt)
|
|
2265
|
-
cpuMs += st.
|
|
2266
|
-
|
|
2417
|
+
cpuMs += st.workCoreMs
|
|
2418
|
+
tasksMs += st.tasksUtilization * a.cores * st.workMs
|
|
2419
|
+
if (st.workerUtilization !== undefined)
|
|
2420
|
+
workerMs = (workerMs ?? 0) + st.workerUtilization * a.cores * st.workMs
|
|
2421
|
+
coreMs += a.cores * st.workMs
|
|
2267
2422
|
usage.set(a.label, describe(st, a.usage))
|
|
2268
2423
|
}
|
|
2269
2424
|
return [
|
|
@@ -2341,7 +2496,7 @@ export class Orchestrator {
|
|
|
2341
2496
|
? [
|
|
2342
2497
|
[
|
|
2343
2498
|
'',
|
|
2344
|
-
`${'total'.padEnd(width)}
|
|
2499
|
+
`${'total'.padEnd(width)} ${utilization(cpuMs / coreMs, tasksMs / coreMs, workerMs === undefined ? undefined : workerMs / coreMs)}${this.needMs < LEARN_MS ? '' : ` · tasks used ${percent(this.usedMs / this.needMs)} of their history`}`,
|
|
2345
2500
|
] as [string, ui.Inline],
|
|
2346
2501
|
]
|
|
2347
2502
|
: []),
|
|
@@ -2421,27 +2576,49 @@ const nested = new WeakMap<
|
|
|
2421
2576
|
{ dirs: string[]; by: Map<string, readonly string[]> }
|
|
2422
2577
|
>()
|
|
2423
2578
|
|
|
2424
|
-
/** A tree's
|
|
2425
|
-
type Tree = { files:
|
|
2579
|
+
/** A tree's files, each path with its identity as core keys it, and its submodules' (gitlinks); `dirs` once asked. */
|
|
2580
|
+
type Tree = { files: Map<string, string>; links: string[]; dirs?: Set<string> }
|
|
2581
|
+
|
|
2582
|
+
/** Whether `tree` has a file under `dir`, or `dir` in a submodule (checked out or not, its own say). */
|
|
2583
|
+
function holds(tree: Tree, dir: string): boolean {
|
|
2584
|
+
if (tree.dirs === undefined) {
|
|
2585
|
+
const dirs = (tree.dirs = new Set<string>())
|
|
2586
|
+
const above = (f: string): void => {
|
|
2587
|
+
for (let d = path.posix.dirname(f); d !== '.' && !dirs.has(d); d = path.posix.dirname(d))
|
|
2588
|
+
dirs.add(d)
|
|
2589
|
+
}
|
|
2590
|
+
for (const f of tree.files.keys()) above(f)
|
|
2591
|
+
for (const l of tree.links) above(l)
|
|
2592
|
+
}
|
|
2593
|
+
return tree.dirs.has(dir) || tree.links.some((l) => dir === l || dir.startsWith(`${l}/`))
|
|
2594
|
+
}
|
|
2426
2595
|
|
|
2427
2596
|
/** `tree`'s files and submodules under `root`, relative to it; undefined when git cannot list it. */
|
|
2428
2597
|
async function listTree(root: string, tree: string): Promise<Tree | undefined> {
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2598
|
+
let p
|
|
2599
|
+
try {
|
|
2600
|
+
p = Bun.spawn(['git', 'ls-tree', '-r', '-z', tree], {
|
|
2601
|
+
cwd: root,
|
|
2602
|
+
stdin: 'ignore',
|
|
2603
|
+
stdout: 'pipe',
|
|
2604
|
+
stderr: 'ignore',
|
|
2605
|
+
})
|
|
2606
|
+
} catch {
|
|
2607
|
+
// quiet: no git, or no root, is git unable to say.
|
|
2608
|
+
return undefined
|
|
2609
|
+
}
|
|
2435
2610
|
const [out, code] = await Promise.all([new Response(p.stdout).text(), p.exited])
|
|
2436
2611
|
if (code !== 0) return undefined
|
|
2437
|
-
const files = new
|
|
2612
|
+
const files = new Map<string, string>()
|
|
2438
2613
|
const links: string[] = []
|
|
2439
2614
|
for (const e of out.split('\0')) {
|
|
2440
2615
|
const tab = e.indexOf('\t')
|
|
2441
2616
|
if (tab === -1) continue
|
|
2442
2617
|
const name = e.slice(tab + 1)
|
|
2443
|
-
|
|
2444
|
-
|
|
2618
|
+
// `<mode> <type> <oid>`: core's identity is the OID, the mode before it unless a plain file's.
|
|
2619
|
+
const [mode, , oid] = e.slice(0, tab).split(' ')
|
|
2620
|
+
if (mode === '160000') links.push(name)
|
|
2621
|
+
else files.set(name, mode === '100644' ? oid! : `${mode}:${oid}`)
|
|
2445
2622
|
}
|
|
2446
2623
|
return { files, links }
|
|
2447
2624
|
}
|
|
@@ -2468,6 +2645,47 @@ function leaves(node: TaskNode): boolean {
|
|
|
2468
2645
|
return sandbox === undefined || (sandbox.allow?.write?.length ?? 0) > 0
|
|
2469
2646
|
}
|
|
2470
2647
|
|
|
2648
|
+
/**
|
|
2649
|
+
* A task and an upstream of it that `leaves` files, which `bound` says
|
|
2650
|
+
* cannot run where the task does (its `exec.worker` or `exec.platforms`):
|
|
2651
|
+
* the task reads those files where the upstream ran, or runs it again
|
|
2652
|
+
* where it runs itself. Undefined when none.
|
|
2653
|
+
*/
|
|
2654
|
+
export function apart(
|
|
2655
|
+
nodes: ReadonlyMap<string, TaskNode>,
|
|
2656
|
+
bound: (upstream: TaskNode, dependant: TaskNode) => boolean,
|
|
2657
|
+
): { upstream: string; dependant: string } | undefined {
|
|
2658
|
+
const runs = (n: TaskNode): boolean =>
|
|
2659
|
+
n.config.exec?.command !== undefined && n.config.exec.persistent === undefined
|
|
2660
|
+
const named = [...nodes.values()].filter(
|
|
2661
|
+
(n) =>
|
|
2662
|
+
runs(n) &&
|
|
2663
|
+
leaves(n) &&
|
|
2664
|
+
(n.config.exec?.worker !== undefined || n.config.exec?.platforms !== undefined),
|
|
2665
|
+
)
|
|
2666
|
+
if (named.length === 0) return undefined
|
|
2667
|
+
const below = new Map<string, string[]>()
|
|
2668
|
+
for (const n of nodes.values())
|
|
2669
|
+
for (const d of n.deps) {
|
|
2670
|
+
const of = below.get(d)
|
|
2671
|
+
if (of === undefined) below.set(d, [n.id])
|
|
2672
|
+
else of.push(n.id)
|
|
2673
|
+
}
|
|
2674
|
+
for (const u of named) {
|
|
2675
|
+
const seen = new Set<string>()
|
|
2676
|
+
const walk = [...(below.get(u.id) ?? [])]
|
|
2677
|
+
for (let id = walk.pop(); id !== undefined; id = walk.pop()) {
|
|
2678
|
+
if (seen.has(id)) continue
|
|
2679
|
+
seen.add(id)
|
|
2680
|
+
const d = nodes.get(id)
|
|
2681
|
+
if (d === undefined) continue
|
|
2682
|
+
if (runs(d) && bound(u, d)) return { upstream: u.id, dependant: id }
|
|
2683
|
+
walk.push(...(below.get(id) ?? []))
|
|
2684
|
+
}
|
|
2685
|
+
}
|
|
2686
|
+
return undefined
|
|
2687
|
+
}
|
|
2688
|
+
|
|
2471
2689
|
/**
|
|
2472
2690
|
* Every task `id` depends on, transitively, each after its own
|
|
2473
2691
|
* dependencies: the order a worker restores them in. Dependant first, an
|
|
@@ -2489,11 +2707,17 @@ function upstream(nodes: ReadonlyMap<string, TaskNode> | undefined, id: string):
|
|
|
2489
2707
|
return out
|
|
2490
2708
|
}
|
|
2491
2709
|
|
|
2710
|
+
/** Core-time busy over core-time had, from first send to last end: its machine, its tasks, its worker. */
|
|
2711
|
+
function utilization(machine: number, tasks: number, worker: number | undefined): string {
|
|
2712
|
+
return `cpu utilization machine ${percent(machine)} · tasks ${percent(tasks)}${worker === undefined ? '' : ` · worker ${percent(worker)}`}`
|
|
2713
|
+
}
|
|
2714
|
+
|
|
2492
2715
|
/** A worker's run, spelled out: time, CPU, memory and the relay. */
|
|
2493
2716
|
function describe(st: UsageStats, u: Usage): string[] {
|
|
2494
2717
|
return [
|
|
2495
2718
|
`online ${seconds(st.onlineMs)} · busy ${seconds(st.busyMs)} · idle ${seconds(Math.max(0, st.onlineMs - st.busyMs))}`,
|
|
2496
|
-
`cpu peak ${st.cpuPeak.toFixed(1)} · used avg ${st.cpuAvg.toFixed(1)} cores${st.machineAvg === undefined ? '' : ` · machine avg ${st.machineAvg.toFixed(1)}`} · booked avg ${st.bookedAvg.toFixed(1)}
|
|
2719
|
+
`cpu peak ${st.cpuPeak.toFixed(1)} · used avg ${st.cpuAvg.toFixed(1)} cores${st.machineAvg === undefined ? '' : ` · machine avg ${st.machineAvg.toFixed(1)}`} · booked avg ${st.bookedAvg.toFixed(1)}`,
|
|
2720
|
+
utilization(st.utilization, st.tasksUtilization, st.workerUtilization),
|
|
2497
2721
|
// No task said what it held: `memory peak 0 B` would be a claim.
|
|
2498
2722
|
...(st.memoryPeak > 0
|
|
2499
2723
|
? [`memory peak ${bytes(st.memoryPeak)} · avg ${bytes(st.memoryAvg)}`]
|
|
@@ -2543,16 +2767,24 @@ function runtimeOrders(req: ExecuteRequest): { runtime?: RuntimeOrder[] } {
|
|
|
2543
2767
|
|
|
2544
2768
|
/**
|
|
2545
2769
|
* `env` as it travels: no unset name, and no name whose value is the
|
|
2546
|
-
* worker's (`HOST_ENV`) unless `define` sets it
|
|
2547
|
-
* with the worker's own, saved under a key
|
|
2548
|
-
* (2026-10-24)
|
|
2770
|
+
* worker's (`HOST_ENV`) unless `define` sets it or the task keys it
|
|
2771
|
+
* (`keyed`). A defined `HOME` ran with the worker's own, saved under a key
|
|
2772
|
+
* that folded the defined one (2026-10-24); a keyed `XDG_CONFIG_HOME` did
|
|
2773
|
+
* the same (2026-11-07).
|
|
2549
2774
|
*/
|
|
2550
2775
|
function defined(
|
|
2551
2776
|
env: NodeJS.ProcessEnv,
|
|
2552
2777
|
define: Readonly<Record<string, string>>,
|
|
2778
|
+
keyed: readonly string[] = [],
|
|
2553
2779
|
): Record<string, string> {
|
|
2554
|
-
|
|
2780
|
+
// No prototype: on `{}`, a name `__proto__` hit the setter and was not sent.
|
|
2781
|
+
const out: Record<string, string> = Object.create(null)
|
|
2555
2782
|
for (const [k, v] of Object.entries(env))
|
|
2556
|
-
if (v !== undefined && (!HOST_ENV.includes(k) || k in define)) out[k] = v
|
|
2783
|
+
if (v !== undefined && (!HOST_ENV.includes(k) || k in define || keyed.includes(k))) out[k] = v
|
|
2557
2784
|
return out
|
|
2558
2785
|
}
|
|
2786
|
+
|
|
2787
|
+
/** The `HOST_ENV` names the task's `inputs.env` keys: their value is this machine's, as keyed. */
|
|
2788
|
+
function hostKeyed(req: ExecuteRequest): string[] {
|
|
2789
|
+
return (req.inputs?.env ?? []).map((e) => e.name).filter((k) => HOST_ENV.includes(k))
|
|
2790
|
+
}
|