@vzn/vx-dte 0.0.651 → 0.0.653
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +164 -60
- package/package.json +3 -3
- package/src/dispatch.ts +540 -30
- package/src/forecast.ts +1 -2
- package/src/github.ts +0 -3
- package/src/index.ts +1 -0
- package/src/init.ts +9 -10
- package/src/local.ts +112 -28
- package/src/messages.ts +26 -5
- package/src/orchestrator.ts +678 -233
- package/src/planner.ts +1470 -192
- package/src/plugin.ts +242 -33
- package/src/seal.ts +159 -43
- package/src/server-fit.ts +80 -0
- package/src/tree.ts +215 -7
- package/src/usage.ts +65 -3
- package/src/warm-ups.ts +72 -6
- package/src/wire.ts +119 -49
- package/src/worker.ts +174 -19
package/src/dispatch.ts
CHANGED
|
@@ -16,15 +16,133 @@ export function ordered(
|
|
|
16
16
|
cores: number,
|
|
17
17
|
): string[] {
|
|
18
18
|
if (how === 'fifo') return [...ids]
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
// Each one's key read once: a comparator that read the tails read them
|
|
20
|
+
// q log q times at each event, most of a 10 000-task sim (2026-11-05).
|
|
21
|
+
const tails = ids.map(tail)
|
|
22
|
+
if (how === 'critical-path') return byKey(ids, tails)
|
|
21
23
|
// Troublesome: within a quarter of the longest chain, or half a worker wide.
|
|
22
24
|
let longest = 0
|
|
23
|
-
for (const
|
|
24
|
-
|
|
25
|
-
|
|
25
|
+
for (const t of tails) longest = Math.max(longest, t)
|
|
26
|
+
// Hard ones first: a key above every tail.
|
|
27
|
+
const keys = ids.map((id, i) =>
|
|
28
|
+
tails[i]! >= 0.75 * longest || need(id).cores >= cores / 2
|
|
29
|
+
? tails[i]! + longest + 1
|
|
30
|
+
: tails[i]!,
|
|
31
|
+
)
|
|
32
|
+
return byKey(ids, keys)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* `ids` by `keys`, highest first, ties in the order they came. The queue
|
|
37
|
+
* comes back as it left plus what got ready since: the run of it already
|
|
38
|
+
* in order is merged with the rest, sorted, and not sorted again.
|
|
39
|
+
*/
|
|
40
|
+
function byKey(ids: readonly string[], keys: readonly number[]): string[] {
|
|
41
|
+
let run = 1
|
|
42
|
+
while (run < ids.length && keys[run]! <= keys[run - 1]!) run++
|
|
43
|
+
if (run >= ids.length) return [...ids]
|
|
44
|
+
const rest: number[] = []
|
|
45
|
+
for (let i = run; i < ids.length; i++) rest.push(i)
|
|
46
|
+
rest.sort((x, y) => keys[y]! - keys[x]! || x - y)
|
|
47
|
+
const out: string[] = []
|
|
48
|
+
let i = 0
|
|
49
|
+
let j = 0
|
|
50
|
+
while (i < run && j < rest.length)
|
|
51
|
+
out.push(keys[i]! >= keys[rest[j]!]! ? ids[i++]! : ids[rest[j++]!]!)
|
|
52
|
+
while (i < run) out.push(ids[i++]!)
|
|
53
|
+
while (j < rest.length) out.push(ids[rest[j++]!]!)
|
|
54
|
+
return out
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** How `dispatch` sees the waiting tasks and the run. */
|
|
58
|
+
export interface Dispatching {
|
|
59
|
+
readonly how: Strategy
|
|
60
|
+
/** Each one's longest chain after it (`tailsMs`); none: they go in the order they came. */
|
|
61
|
+
readonly tail?: ((id: string) => number) | undefined
|
|
62
|
+
/** What it asks, as history said. */
|
|
63
|
+
readonly need: (id: string) => Need
|
|
64
|
+
/** What it books now (`Orchestrator.booking`); default `need`'s. */
|
|
65
|
+
readonly book?: (need: Need) => Need
|
|
66
|
+
/** Its direct dependencies (`Ask.up`). */
|
|
67
|
+
readonly deps: (id: string) => readonly string[] | undefined
|
|
68
|
+
/** Whether it may go now, and where (`Orchestrator.pin`): `'wait'` not yet, `'done'` it ended. */
|
|
69
|
+
readonly gate?: (id: string) => 'wait' | 'done' | { only?: string; avoid?: string }
|
|
70
|
+
/** A worker bigger than `above` of `platform` is coming (`assign`'s `waits`). */
|
|
71
|
+
readonly waits?: (above: NonNullable<Need['above']>, platform: string | undefined) => boolean
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Where the `waiting` tasks go now: ordered (`ordered`), gated, and placed
|
|
76
|
+
* (`assign`). The one rule of the live run and of the sims, which keep
|
|
77
|
+
* their waiting tasks in a `Queue` placing as this does: each kept its
|
|
78
|
+
* own, and the sims passed while runs placed otherwise (2026-11-05).
|
|
79
|
+
* `left`: the ones that stay waiting, in order.
|
|
80
|
+
*/
|
|
81
|
+
export function dispatch<B extends Bin>(
|
|
82
|
+
waiting: readonly string[],
|
|
83
|
+
bins: readonly B[],
|
|
84
|
+
o: Dispatching,
|
|
85
|
+
): { to: Map<string, B>; left: string[] } {
|
|
86
|
+
const queue =
|
|
87
|
+
o.tail === undefined
|
|
88
|
+
? waiting
|
|
89
|
+
: ordered(
|
|
90
|
+
waiting,
|
|
91
|
+
o.how.order,
|
|
92
|
+
o.tail,
|
|
93
|
+
o.need,
|
|
94
|
+
bins.reduce((n, b) => Math.max(n, b.cores), 1),
|
|
95
|
+
)
|
|
96
|
+
// The least any ask needs: once no worker takes it, the pass ends. Each
|
|
97
|
+
// pump read every queued task, 8000 of them a pump (2026-10-25).
|
|
98
|
+
const floor = { cores: Infinity, memory: Infinity }
|
|
99
|
+
const needs: Need[] = []
|
|
100
|
+
for (const id of queue) {
|
|
101
|
+
const need = o.book === undefined ? o.need(id) : o.book(o.need(id))
|
|
102
|
+
needs.push(need)
|
|
103
|
+
floor.cores = Math.min(floor.cores, need.cores)
|
|
104
|
+
floor.memory = Math.min(floor.memory, need.memory)
|
|
105
|
+
}
|
|
106
|
+
// Asks are made as the pass reads them: it ends at the first no worker
|
|
107
|
+
// takes, and making every queued one's at each event was most of a
|
|
108
|
+
// 5000-task sim (2026-11-05).
|
|
109
|
+
const read: string[] = []
|
|
110
|
+
// The bins' bounds as assign keeps them: an ask no bin can take or hold
|
|
111
|
+
// for is not made. Each such ask read every bin, and a full run's queue
|
|
112
|
+
// of them at each event was most of a 10 000-task sim (2026-11-05).
|
|
113
|
+
const bounds: Bounds = { done: false, open: true, free: 0, cores: 0, memory: 0 }
|
|
114
|
+
let i = 0
|
|
115
|
+
function* asks(): Generator<Ask> {
|
|
116
|
+
for (; i < queue.length; i++) {
|
|
117
|
+
const id = queue[i]!
|
|
118
|
+
if (bounds.done) return
|
|
119
|
+
if (dead(needs[i]!, bounds)) {
|
|
120
|
+
read.push(id)
|
|
121
|
+
continue
|
|
122
|
+
}
|
|
123
|
+
const gate = o.gate?.(id) ?? {}
|
|
124
|
+
if (gate === 'done') continue
|
|
125
|
+
read.push(id)
|
|
126
|
+
if (gate === 'wait') continue
|
|
127
|
+
yield { id, need: needs[i]!, up: o.deps(id) ?? [], ...gate }
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
const to = assign(asks(), bins, o.waits, o.how.place, { floor, kinds: Infinity, bounds })
|
|
131
|
+
const left = read.filter((id) => !to.has(id))
|
|
132
|
+
// The pass ended at the ask it read last, and took or left it.
|
|
133
|
+
for (let j = i < queue.length && read.at(-1) === queue[i] ? i + 1 : i; j < queue.length; j++)
|
|
134
|
+
left.push(queue[j]!)
|
|
135
|
+
return { to, left }
|
|
26
136
|
}
|
|
27
137
|
|
|
138
|
+
/**
|
|
139
|
+
* The most workers a task is scored on, of those it fits, as Kubernetes
|
|
140
|
+
* scores a share of its nodes: scoring every one cost the orchestrator
|
|
141
|
+
* 15% more CPU at 200 workers, and the sims ended within 0.4% sampling
|
|
142
|
+
* even 2 (2026-11-05).
|
|
143
|
+
*/
|
|
144
|
+
const SAMPLE = 16
|
|
145
|
+
|
|
28
146
|
/** What a task asks of a worker: its history's cores and peak memory. */
|
|
29
147
|
export interface Need {
|
|
30
148
|
readonly cores: number
|
|
@@ -39,6 +157,12 @@ export interface Need {
|
|
|
39
157
|
* free core, but a run of only such tasks still asks enough workers.
|
|
40
158
|
*/
|
|
41
159
|
readonly ask?: { readonly cores: number; readonly memory: number } | undefined
|
|
160
|
+
/**
|
|
161
|
+
* Its persistent dependencies' bases, cores (`server-fit.ts`): a worker
|
|
162
|
+
* not running one yet books its base once beside the task, and keeps it
|
|
163
|
+
* booked while it serves.
|
|
164
|
+
*/
|
|
165
|
+
readonly servers?: readonly { readonly id: string; readonly cores: number }[] | undefined
|
|
42
166
|
}
|
|
43
167
|
|
|
44
168
|
/** A worker as dispatch sees it: its size and what is in use on it now. */
|
|
@@ -54,13 +178,19 @@ export interface Bin {
|
|
|
54
178
|
readonly has?: ReadonlySet<string>
|
|
55
179
|
/** Its machine's platform (`linux-x64`), as it reported or its size declares. */
|
|
56
180
|
readonly platform?: string | undefined
|
|
181
|
+
/** The servers it runs, each base booked: a task needing one books no base here. */
|
|
182
|
+
readonly serving?: Set<string>
|
|
57
183
|
}
|
|
58
184
|
|
|
59
185
|
export interface Ask {
|
|
60
186
|
readonly id: string
|
|
61
187
|
readonly need: Need
|
|
62
|
-
/**
|
|
63
|
-
|
|
188
|
+
/**
|
|
189
|
+
* Its direct dependencies: a worker holding most is preferred. One holds
|
|
190
|
+
* what was sent with each, so the direct ones say enough: counting every
|
|
191
|
+
* upstream for every worker read thousands an ask (2026-11-05).
|
|
192
|
+
*/
|
|
193
|
+
readonly up?: readonly string[]
|
|
64
194
|
/** Names its need: asks of one kind fit and fail alike (`kindOf`). */
|
|
65
195
|
readonly kind?: string
|
|
66
196
|
/** The one bin that may take it: it holds files no cache does. */
|
|
@@ -71,7 +201,14 @@ export interface Ask {
|
|
|
71
201
|
|
|
72
202
|
/** What `Ask.kind` names: every field of a need `assign` reads. */
|
|
73
203
|
export function kindOf(need: Need): string {
|
|
74
|
-
return `${need.cores}\0${need.memory}\0${need.above?.cores}\0${need.above?.memory}\0${need.platform}`
|
|
204
|
+
return `${need.cores}\0${need.memory}\0${need.above?.cores}\0${need.above?.memory}\0${need.platform}\0${need.servers?.map((s) => s.id).join('\0')}`
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** The cores `need` books on `bin`: its own, and the base of each server `bin` does not run yet. */
|
|
208
|
+
function coresOn(bin: Bin, need: Need): number {
|
|
209
|
+
let cores = need.cores
|
|
210
|
+
for (const s of need.servers ?? []) if (bin.serving?.has(s.id) !== true) cores += s.cores
|
|
211
|
+
return cores
|
|
75
212
|
}
|
|
76
213
|
|
|
77
214
|
/**
|
|
@@ -82,9 +219,10 @@ export function kindOf(need: Need): string {
|
|
|
82
219
|
export function fits(bin: Bin, need: Need): boolean {
|
|
83
220
|
if (need.platform !== undefined && bin.platform !== need.platform) return false
|
|
84
221
|
if (bin.running === 0) return true
|
|
222
|
+
const cores = coresOn(bin, need)
|
|
85
223
|
return (
|
|
86
|
-
|
|
87
|
-
bin.used.cores +
|
|
224
|
+
cores < bin.cores &&
|
|
225
|
+
bin.used.cores + cores <= bin.cores + 1e-9 &&
|
|
88
226
|
bin.used.memory + need.memory <= bin.memory
|
|
89
227
|
)
|
|
90
228
|
}
|
|
@@ -130,7 +268,12 @@ export function assign<B extends Bin>(
|
|
|
130
268
|
for (const ask of asks) {
|
|
131
269
|
if (
|
|
132
270
|
all !== undefined &&
|
|
133
|
-
(spent.size >= all.kinds ||
|
|
271
|
+
(spent.size >= all.kinds ||
|
|
272
|
+
// Its bounds, kept as it gives, say it: a read of every bin an ask
|
|
273
|
+
// was most of a plan's play (2026-11-05).
|
|
274
|
+
(bounds !== undefined
|
|
275
|
+
? bounds.done
|
|
276
|
+
: !bins.some((b) => !held.has(b) && fits(b, all.floor))))
|
|
134
277
|
)
|
|
135
278
|
break
|
|
136
279
|
if (ask.kind !== undefined && spent.has(ask.kind)) continue
|
|
@@ -143,20 +286,24 @@ export function assign<B extends Bin>(
|
|
|
143
286
|
: bins.filter((b) => (ask.only === undefined || b.id === ask.only) && b.id !== ask.avoid)
|
|
144
287
|
const of =
|
|
145
288
|
need.platform === undefined ? allowed : allowed.filter((b) => b.platform === need.platform)
|
|
146
|
-
|
|
289
|
+
// With no size to beat, every bin is bigger: no copy of them an ask.
|
|
290
|
+
const anyBigger = need.above === undefined ? of.length > 0 : of.some(bigger)
|
|
147
291
|
if (!anyBigger && need.above !== undefined && waits(need.above, need.platform)) {
|
|
148
292
|
if (ask.kind !== undefined) spent.add(ask.kind)
|
|
149
293
|
continue
|
|
150
294
|
}
|
|
151
|
-
const pool = anyBigger ? of.filter(bigger) : of
|
|
152
|
-
const up = ask.up === undefined ? [] : [...ask.up]
|
|
295
|
+
const pool = anyBigger && need.above !== undefined ? of.filter(bigger) : of
|
|
153
296
|
let best: B | undefined
|
|
154
297
|
let bestScore: [number, number] = [0, 0]
|
|
155
|
-
|
|
298
|
+
let seen = 0
|
|
299
|
+
for (let i = 0; i < pool.length; i++) {
|
|
300
|
+
const b = pool[i]!
|
|
156
301
|
if (held.has(b) || !fits(b, need)) continue
|
|
302
|
+
// Workers fill in order, so the first that fit hold the fullest.
|
|
303
|
+
if (++seen > SAMPLE) break
|
|
157
304
|
let has = 0
|
|
158
|
-
for (const id of up) if (b.has?.has(id) === true) has++
|
|
159
|
-
const score: [number, number] = [has, fit(b, need, place,
|
|
305
|
+
for (const id of ask.up ?? []) if (b.has?.has(id) === true) has++
|
|
306
|
+
const score: [number, number] = [has, fit(b, need, place, i)]
|
|
160
307
|
if (
|
|
161
308
|
best === undefined ||
|
|
162
309
|
score[0] > bestScore[0] ||
|
|
@@ -182,7 +329,8 @@ export function assign<B extends Bin>(
|
|
|
182
329
|
}
|
|
183
330
|
spent.clear()
|
|
184
331
|
out.set(ask.id, best)
|
|
185
|
-
best.used.cores += Math.min(need
|
|
332
|
+
best.used.cores += Math.min(coresOn(best, need), best.cores)
|
|
333
|
+
for (const sv of need.servers ?? []) best.serving?.add(sv.id)
|
|
186
334
|
best.used.memory += Math.min(need.memory, best.memory)
|
|
187
335
|
best.running++
|
|
188
336
|
rebound()
|
|
@@ -207,6 +355,16 @@ export interface Bounds {
|
|
|
207
355
|
cores: number
|
|
208
356
|
/** The least memory free on one. */
|
|
209
357
|
memory: number
|
|
358
|
+
/** The same of each platform's own: a task naming one fits only those. */
|
|
359
|
+
of?: Map<string, Room>
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
/** What `Bounds` says of a set of bins. */
|
|
363
|
+
interface Room {
|
|
364
|
+
open: boolean
|
|
365
|
+
free: number
|
|
366
|
+
cores: number
|
|
367
|
+
memory: number
|
|
210
368
|
}
|
|
211
369
|
|
|
212
370
|
function boundsOf(
|
|
@@ -220,30 +378,41 @@ function boundsOf(
|
|
|
220
378
|
out.free = -Infinity
|
|
221
379
|
out.cores = Infinity
|
|
222
380
|
out.memory = Infinity
|
|
381
|
+
out.of?.clear()
|
|
223
382
|
for (const b of bins) {
|
|
224
383
|
if (held.has(b)) continue
|
|
225
384
|
if (floor !== undefined && fits(b, floor)) out.done = false
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
out.
|
|
229
|
-
|
|
385
|
+
room(out, b)
|
|
386
|
+
if (b.platform === undefined) continue
|
|
387
|
+
out.of ??= new Map()
|
|
388
|
+
let p = out.of.get(b.platform)
|
|
389
|
+
if (p === undefined)
|
|
390
|
+
out.of.set(
|
|
391
|
+
b.platform,
|
|
392
|
+
(p = { open: false, free: -Infinity, cores: Infinity, memory: Infinity }),
|
|
393
|
+
)
|
|
394
|
+
room(p, b)
|
|
230
395
|
}
|
|
231
396
|
}
|
|
232
397
|
|
|
398
|
+
function room(r: Room, b: Bin): void {
|
|
399
|
+
if (b.running === 0) r.open = true
|
|
400
|
+
r.free = Math.max(r.free, b.cores - b.used.cores)
|
|
401
|
+
r.cores = Math.min(r.cores, b.cores)
|
|
402
|
+
r.memory = Math.min(r.memory, b.memory - b.used.memory)
|
|
403
|
+
}
|
|
404
|
+
|
|
233
405
|
/**
|
|
234
406
|
* True when `assign` would fit `need` on no bin and hold none for it, by
|
|
235
407
|
* `fits` and its hold rule: no bin is empty, and on each it is less than
|
|
236
408
|
* the bin, fits its memory, and wants more cores than are free.
|
|
237
409
|
*/
|
|
238
410
|
export function dead(need: Need, b: Bounds): boolean {
|
|
239
|
-
return
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
need.cores < b.cores &&
|
|
245
|
-
need.memory <= b.memory
|
|
246
|
-
)
|
|
411
|
+
if (need.above !== undefined) return false
|
|
412
|
+
// Of its platform's bins: with none free of a hold, no ask of it does a thing.
|
|
413
|
+
const r = need.platform === undefined ? b : b.of?.get(need.platform)
|
|
414
|
+
if (r === undefined) return true
|
|
415
|
+
return !r.open && need.cores > r.free + 1e-9 && need.cores < r.cores && need.memory <= r.memory
|
|
247
416
|
}
|
|
248
417
|
|
|
249
418
|
/** How well `need` suits `b` by `place`: the higher, the better. */
|
|
@@ -266,3 +435,344 @@ function fit(b: Bin, need: Need, place: Strategy['place'], index: number): numbe
|
|
|
266
435
|
}
|
|
267
436
|
}
|
|
268
437
|
}
|
|
438
|
+
|
|
439
|
+
/**
|
|
440
|
+
* The live run's waiting tasks, kept across its passes in the order
|
|
441
|
+
* `ordered` gives (`critical-path`, `fifo`) and grouped by platform and
|
|
442
|
+
* cores' class: a pass places exactly as `dispatch` over the whole list,
|
|
443
|
+
* but passes a group whole while no worker has room for any of it, where
|
|
444
|
+
* each pass ordered and read every waiting task, a fifth of a 10 000-task
|
|
445
|
+
* sim (2026-11-06). `graphene` ranks by the longest waiting tail, which
|
|
446
|
+
* moves as tasks leave: it keeps the list and `dispatch`.
|
|
447
|
+
*/
|
|
448
|
+
export class Queue {
|
|
449
|
+
private list: string[] | undefined
|
|
450
|
+
private readonly groups = new Map<string, Group>()
|
|
451
|
+
private readonly of = new Map<string, Entry>()
|
|
452
|
+
private front = 0
|
|
453
|
+
private back = 0
|
|
454
|
+
|
|
455
|
+
constructor(
|
|
456
|
+
private readonly how: Strategy,
|
|
457
|
+
private readonly tail: ((id: string) => number) | undefined,
|
|
458
|
+
private readonly need: (id: string) => Need,
|
|
459
|
+
) {
|
|
460
|
+
if (how.order === 'graphene' && tail !== undefined) this.list = []
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
get size(): number {
|
|
464
|
+
return this.list?.length ?? this.of.size
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
has(id: string): boolean {
|
|
468
|
+
return this.list?.includes(id) ?? this.of.has(id)
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
/** In the order a pass reads them. */
|
|
472
|
+
ids(): string[] {
|
|
473
|
+
if (this.list !== undefined) return [...this.list]
|
|
474
|
+
return [...this.of.values()].sort((a, b) => (before(a, b) ? -1 : 1)).map((e) => e.id)
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
/** `ids` join, in order, behind every one ranked as high, or `front` before all. */
|
|
478
|
+
add(ids: readonly string[], front = false): void {
|
|
479
|
+
if (this.list !== undefined) {
|
|
480
|
+
if (front) this.list.unshift(...ids)
|
|
481
|
+
else this.list.push(...ids)
|
|
482
|
+
return
|
|
483
|
+
}
|
|
484
|
+
if (front) this.front -= ids.length
|
|
485
|
+
let seq = front ? this.front : this.back
|
|
486
|
+
for (const id of ids) {
|
|
487
|
+
const need = this.need(id)
|
|
488
|
+
const e: Entry = {
|
|
489
|
+
id,
|
|
490
|
+
rank: this.how.order === 'fifo' || this.tail === undefined ? 0 : -this.tail(id),
|
|
491
|
+
seq: seq++,
|
|
492
|
+
need,
|
|
493
|
+
group: undefined!,
|
|
494
|
+
}
|
|
495
|
+
const odd = need.above !== undefined || need.ask !== undefined || need.cores === Infinity
|
|
496
|
+
const key = `${need.platform}\0${odd ? 'odd' : Math.floor(Math.log2(Math.max(need.cores, 1e-6)) * CLASSES)}`
|
|
497
|
+
let g = this.groups.get(key)
|
|
498
|
+
if (g === undefined) this.groups.set(key, (g = new Group(odd, need.platform)))
|
|
499
|
+
e.group = g
|
|
500
|
+
g.add(e)
|
|
501
|
+
this.of.set(id, e)
|
|
502
|
+
}
|
|
503
|
+
if (!front) this.back = seq
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
/**
|
|
507
|
+
* History changed (`tail`, `need`): the waiting rank and group by it,
|
|
508
|
+
* ties in the order they stood. A rank is read once, as a task joins:
|
|
509
|
+
* on CI the roots join before the default branch's timings land, each
|
|
510
|
+
* at an unknown task's 30 s, and every test shard ready later ranked
|
|
511
|
+
* above them, so `test.wire` (73 s) waited 129 s for a worker (main
|
|
512
|
+
* 1c91c9d, 2026-11-07). `graphene` reads ranks at each pass.
|
|
513
|
+
*/
|
|
514
|
+
rerank(): void {
|
|
515
|
+
if (this.list !== undefined) return
|
|
516
|
+
const ids = this.ids()
|
|
517
|
+
this.groups.clear()
|
|
518
|
+
this.of.clear()
|
|
519
|
+
this.front = 0
|
|
520
|
+
this.back = 0
|
|
521
|
+
this.add(ids)
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
delete(id: string): boolean {
|
|
525
|
+
if (this.list !== undefined) {
|
|
526
|
+
const at = this.list.indexOf(id)
|
|
527
|
+
if (at >= 0) this.list.splice(at, 1)
|
|
528
|
+
return at >= 0
|
|
529
|
+
}
|
|
530
|
+
const e = this.of.get(id)
|
|
531
|
+
if (e === undefined) return false
|
|
532
|
+
this.drop(e)
|
|
533
|
+
return true
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
private drop(e: Entry): void {
|
|
537
|
+
this.of.delete(e.id)
|
|
538
|
+
e.gone = true
|
|
539
|
+
if (e.group.drop() === 0)
|
|
540
|
+
for (const [k, g] of this.groups) if (g === e.group) this.groups.delete(k)
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
/**
|
|
544
|
+
* Where the waiting tasks go now, as `dispatch` places them; the ones
|
|
545
|
+
* placed or found done leave. `book`, if given, must keep a need's
|
|
546
|
+
* order by cores among needs alike but for cores.
|
|
547
|
+
*/
|
|
548
|
+
pass<B extends Bin>(
|
|
549
|
+
bins: readonly B[],
|
|
550
|
+
o: Omit<Dispatching, 'how' | 'tail' | 'need'>,
|
|
551
|
+
): Map<string, B> {
|
|
552
|
+
if (this.list !== undefined) {
|
|
553
|
+
const { to, left } = dispatch(this.list, bins, {
|
|
554
|
+
...o,
|
|
555
|
+
how: this.how,
|
|
556
|
+
tail: this.tail,
|
|
557
|
+
need: this.need,
|
|
558
|
+
})
|
|
559
|
+
this.list = left
|
|
560
|
+
return to
|
|
561
|
+
}
|
|
562
|
+
return this.place(bins, o)
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
private place<B extends Bin>(
|
|
566
|
+
bins: readonly B[],
|
|
567
|
+
o: Omit<Dispatching, 'how' | 'tail' | 'need'>,
|
|
568
|
+
): Map<string, B> {
|
|
569
|
+
const book = o.book ?? ((n: Need): Need => n)
|
|
570
|
+
const gs = [...this.groups.values()]
|
|
571
|
+
const floor = { cores: Infinity, memory: Infinity }
|
|
572
|
+
for (const g of gs) {
|
|
573
|
+
g.at = g.head()
|
|
574
|
+
const low = g.low()
|
|
575
|
+
if (low === undefined) continue
|
|
576
|
+
// Booked as its least: what each of its tasks books is at least this.
|
|
577
|
+
g.lo = g.odd ? Infinity : book(low).cores
|
|
578
|
+
floor.cores = Math.min(floor.cores, g.odd ? g.oddFloor(book) : g.lo)
|
|
579
|
+
floor.memory = Math.min(floor.memory, g.memory())
|
|
580
|
+
}
|
|
581
|
+
const bounds: Bounds = { done: false, open: true, free: 0, cores: 0, memory: 0 }
|
|
582
|
+
const done: Entry[] = []
|
|
583
|
+
let cursor: Entry | undefined
|
|
584
|
+
function* asks(): Generator<Ask> {
|
|
585
|
+
for (;;) {
|
|
586
|
+
if (bounds.done) return
|
|
587
|
+
// The group whose next is first, past any no worker has room for now.
|
|
588
|
+
let g: Group | undefined
|
|
589
|
+
let second: Entry | undefined
|
|
590
|
+
for (const x of gs) {
|
|
591
|
+
if (x.at >= x.q.length) continue
|
|
592
|
+
if (!x.odd && x.full(bounds)) continue
|
|
593
|
+
if (cursor !== undefined && !before(cursor, x.q[x.at]!)) x.at = x.after(cursor)
|
|
594
|
+
x.at = x.skip(x.at)
|
|
595
|
+
const e = x.q[x.at]
|
|
596
|
+
if (e === undefined) continue
|
|
597
|
+
if (g === undefined || before(e, g.q[g.at]!)) {
|
|
598
|
+
if (g !== undefined && (second === undefined || before(g.q[g.at]!, second)))
|
|
599
|
+
second = g.q[g.at]!
|
|
600
|
+
g = x
|
|
601
|
+
} else if (second === undefined || before(e, second)) second = e
|
|
602
|
+
}
|
|
603
|
+
if (g === undefined) return
|
|
604
|
+
// Its run up to the next group's first, read one by one as before.
|
|
605
|
+
for (;;) {
|
|
606
|
+
const e = g.q[g.at]
|
|
607
|
+
if (e === undefined || (second !== undefined && !before(e, second))) break
|
|
608
|
+
g.at = g.skip(g.at + 1)
|
|
609
|
+
cursor = e
|
|
610
|
+
if (bounds.done) return
|
|
611
|
+
const need = book(e.need)
|
|
612
|
+
if (dead(need, bounds)) continue
|
|
613
|
+
const gate = o.gate?.(e.id) ?? {}
|
|
614
|
+
if (gate === 'done') {
|
|
615
|
+
done.push(e)
|
|
616
|
+
continue
|
|
617
|
+
}
|
|
618
|
+
if (gate === 'wait') continue
|
|
619
|
+
yield { id: e.id, need, up: o.deps(e.id) ?? [], ...gate }
|
|
620
|
+
break
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
const to = assign(asks(), bins, o.waits, this.how.place, { floor, kinds: Infinity, bounds })
|
|
625
|
+
for (const e of done) if (!e.gone) this.drop(e)
|
|
626
|
+
for (const id of to.keys()) this.drop(this.of.get(id)!)
|
|
627
|
+
return to
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
/** Classes a doubling of cores spans (`Queue`'s groups). */
|
|
632
|
+
const CLASSES = 4
|
|
633
|
+
|
|
634
|
+
interface Entry {
|
|
635
|
+
readonly id: string
|
|
636
|
+
/** The lower first, then the lower `seq`. */
|
|
637
|
+
readonly rank: number
|
|
638
|
+
readonly seq: number
|
|
639
|
+
readonly need: Need
|
|
640
|
+
group: Group
|
|
641
|
+
gone?: boolean
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
function before(a: Entry, b: Entry): boolean {
|
|
645
|
+
return a.rank < b.rank || (a.rank === b.rank && a.seq < b.seq)
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
/** Waiting tasks of one platform and cores' class, in rank order; gone ones linger until half are. */
|
|
649
|
+
class Group {
|
|
650
|
+
q: Entry[] = []
|
|
651
|
+
private gone = 0
|
|
652
|
+
/** Where a pass reads next. */
|
|
653
|
+
at = 0
|
|
654
|
+
/** The least any of it books, this pass. */
|
|
655
|
+
lo = Infinity
|
|
656
|
+
/** The most cores and memory one asked: kept as they joined until it is read again without the gone. */
|
|
657
|
+
private hiCores = -Infinity
|
|
658
|
+
private hiMemory = -Infinity
|
|
659
|
+
/** Its least cores and memory, gone ones lazily left off the top. */
|
|
660
|
+
private readonly byCores: Entry[] = []
|
|
661
|
+
private readonly byMemory: Entry[] = []
|
|
662
|
+
|
|
663
|
+
constructor(
|
|
664
|
+
readonly odd: boolean,
|
|
665
|
+
readonly platform: string | undefined,
|
|
666
|
+
) {}
|
|
667
|
+
|
|
668
|
+
add(e: Entry): void {
|
|
669
|
+
let lo = 0
|
|
670
|
+
let hi = this.q.length
|
|
671
|
+
// Most join last: a search from the end.
|
|
672
|
+
if (hi === 0 || before(this.q[hi - 1]!, e)) lo = hi
|
|
673
|
+
while (lo < hi) {
|
|
674
|
+
const mid = (lo + hi) >> 1
|
|
675
|
+
if (before(this.q[mid]!, e)) lo = mid + 1
|
|
676
|
+
else hi = mid
|
|
677
|
+
}
|
|
678
|
+
this.q.splice(lo, 0, e)
|
|
679
|
+
this.hiCores = Math.max(this.hiCores, e.need.cores)
|
|
680
|
+
this.hiMemory = Math.max(this.hiMemory, e.need.memory)
|
|
681
|
+
push(this.byCores, e, (x) => x.need.cores)
|
|
682
|
+
push(this.byMemory, e, (x) => x.need.memory)
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
/** One of it left: how many are left. */
|
|
686
|
+
drop(): number {
|
|
687
|
+
this.gone++
|
|
688
|
+
const left = this.q.length - this.gone
|
|
689
|
+
if (this.gone > left) {
|
|
690
|
+
this.q = this.q.filter((e) => !e.gone)
|
|
691
|
+
this.gone = 0
|
|
692
|
+
this.hiCores = -Infinity
|
|
693
|
+
this.hiMemory = -Infinity
|
|
694
|
+
for (const e of this.q) {
|
|
695
|
+
this.hiCores = Math.max(this.hiCores, e.need.cores)
|
|
696
|
+
this.hiMemory = Math.max(this.hiMemory, e.need.memory)
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
return left
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
head(): number {
|
|
703
|
+
return this.skip(0)
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
skip(p: number): number {
|
|
707
|
+
while (p < this.q.length && this.q[p]!.gone === true) p++
|
|
708
|
+
return p
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
/** The first place ranked past `cursor`. */
|
|
712
|
+
after(cursor: Entry): number {
|
|
713
|
+
let p = this.at
|
|
714
|
+
let hi = this.q.length
|
|
715
|
+
while (p < hi) {
|
|
716
|
+
const mid = (p + hi) >> 1
|
|
717
|
+
if (!before(cursor, this.q[mid]!)) p = mid + 1
|
|
718
|
+
else hi = mid
|
|
719
|
+
}
|
|
720
|
+
return p
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
low(): Need | undefined {
|
|
724
|
+
return top(this.byCores, (x) => x.need.cores)?.need
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
memory(): number {
|
|
728
|
+
return top(this.byMemory, (x) => x.need.memory)?.need.memory ?? Infinity
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
/** An odd group's least booked: read one by one, as books of each differ. */
|
|
732
|
+
oddFloor(book: (n: Need) => Need): number {
|
|
733
|
+
let least = Infinity
|
|
734
|
+
for (let p = this.at; p < this.q.length; p++)
|
|
735
|
+
if (this.q[p]!.gone !== true) least = Math.min(least, book(this.q[p]!.need).cores)
|
|
736
|
+
return least
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
/** No worker of its platform has room for any of it now, as `dead` says of each. */
|
|
740
|
+
full(b: Bounds): boolean {
|
|
741
|
+
const r = this.platform === undefined ? b : b.of?.get(this.platform)
|
|
742
|
+
if (r === undefined) return true
|
|
743
|
+
return !r.open && this.lo > r.free + 1e-9 && this.hiCores < r.cores && this.hiMemory <= r.memory
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
/** A binary min-heap of entries by `by`. */
|
|
748
|
+
function push(h: Entry[], e: Entry, by: (e: Entry) => number): void {
|
|
749
|
+
h.push(e)
|
|
750
|
+
let i = h.length - 1
|
|
751
|
+
while (i > 0) {
|
|
752
|
+
const p = (i - 1) >> 1
|
|
753
|
+
if (by(h[p]!) <= by(h[i]!)) break
|
|
754
|
+
;[h[p], h[i]] = [h[i]!, h[p]!]
|
|
755
|
+
i = p
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
/** The least of `h` not gone, the gone ones above it popped. */
|
|
760
|
+
function top(h: Entry[], by: (e: Entry) => number): Entry | undefined {
|
|
761
|
+
while (h.length > 0 && h[0]!.gone === true) {
|
|
762
|
+
const last = h.pop()!
|
|
763
|
+
if (h.length === 0) break
|
|
764
|
+
h[0] = last
|
|
765
|
+
let i = 0
|
|
766
|
+
for (;;) {
|
|
767
|
+
const l = 2 * i + 1
|
|
768
|
+
const r = l + 1
|
|
769
|
+
let m = i
|
|
770
|
+
if (l < h.length && by(h[l]!) < by(h[m]!)) m = l
|
|
771
|
+
if (r < h.length && by(h[r]!) < by(h[m]!)) m = r
|
|
772
|
+
if (m === i) break
|
|
773
|
+
;[h[m], h[i]] = [h[i]!, h[m]!]
|
|
774
|
+
i = m
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
return h[0]
|
|
778
|
+
}
|
package/src/forecast.ts
CHANGED
|
@@ -62,7 +62,6 @@ export function likelyMisses(
|
|
|
62
62
|
return c
|
|
63
63
|
}
|
|
64
64
|
for (const [id, n] of nodes)
|
|
65
|
-
if (n.config.exec?.command !== undefined && n.config.cache
|
|
66
|
-
out.add(id)
|
|
65
|
+
if (n.config.exec?.command !== undefined && n.config.cache === true && changes(id)) out.add(id)
|
|
67
66
|
return out
|
|
68
67
|
}
|
package/src/github.ts
CHANGED
|
@@ -14,8 +14,6 @@ import { printable } from './printable.js'
|
|
|
14
14
|
export interface GithubOptions {
|
|
15
15
|
/** The workers' workflow file in `.github/workflows` (default `vx-worker.yml`). */
|
|
16
16
|
readonly workflow?: string
|
|
17
|
-
/** The runner label of each size, by size name, which `vx dte init` writes; absent, its platform's GitHub-hosted runner. */
|
|
18
|
-
readonly runsOn?: Readonly<Record<string, string>>
|
|
19
17
|
/**
|
|
20
18
|
* Variables the run hands its workers, as the workflow's `env` input (a
|
|
21
19
|
* JSON object): a value the run's job decides, such as a server's URL
|
|
@@ -33,7 +31,6 @@ interface Job {
|
|
|
33
31
|
|
|
34
32
|
const KEYS: PluginOptionKinds<GithubOptions> = {
|
|
35
33
|
workflow: 'string',
|
|
36
|
-
runsOn: 'object',
|
|
37
34
|
env: 'object',
|
|
38
35
|
}
|
|
39
36
|
|