@vzn/vx-dte 0.0.650 → 0.0.652
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -31
- package/package.json +3 -3
- package/src/dispatch.ts +501 -26
- package/src/forecast.ts +1 -2
- package/src/github.ts +16 -5
- package/src/messages.ts +52 -31
- package/src/orchestrator.ts +551 -352
- package/src/planner.ts +1322 -172
- package/src/plugin.ts +48 -8
- package/src/seal.ts +11 -4
- package/src/wire-host.ts +98 -0
- package/src/wire-thread.ts +168 -0
- package/src/wire.ts +117 -68
- package/src/worker.ts +112 -62
package/src/dispatch.ts
CHANGED
|
@@ -16,15 +16,133 @@ export function ordered(
|
|
|
16
16
|
cores: number,
|
|
17
17
|
): string[] {
|
|
18
18
|
if (how === 'fifo') return [...ids]
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
// Each one's key read once: a comparator that read the tails read them
|
|
20
|
+
// q log q times at each event, most of a 10 000-task sim (2026-11-05).
|
|
21
|
+
const tails = ids.map(tail)
|
|
22
|
+
if (how === 'critical-path') return byKey(ids, tails)
|
|
21
23
|
// Troublesome: within a quarter of the longest chain, or half a worker wide.
|
|
22
24
|
let longest = 0
|
|
23
|
-
for (const
|
|
24
|
-
|
|
25
|
-
|
|
25
|
+
for (const t of tails) longest = Math.max(longest, t)
|
|
26
|
+
// Hard ones first: a key above every tail.
|
|
27
|
+
const keys = ids.map((id, i) =>
|
|
28
|
+
tails[i]! >= 0.75 * longest || need(id).cores >= cores / 2
|
|
29
|
+
? tails[i]! + longest + 1
|
|
30
|
+
: tails[i]!,
|
|
31
|
+
)
|
|
32
|
+
return byKey(ids, keys)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* `ids` by `keys`, highest first, ties in the order they came. The queue
|
|
37
|
+
* comes back as it left plus what got ready since: the run of it already
|
|
38
|
+
* in order is merged with the rest, sorted, and not sorted again.
|
|
39
|
+
*/
|
|
40
|
+
function byKey(ids: readonly string[], keys: readonly number[]): string[] {
|
|
41
|
+
let run = 1
|
|
42
|
+
while (run < ids.length && keys[run]! <= keys[run - 1]!) run++
|
|
43
|
+
if (run >= ids.length) return [...ids]
|
|
44
|
+
const rest: number[] = []
|
|
45
|
+
for (let i = run; i < ids.length; i++) rest.push(i)
|
|
46
|
+
rest.sort((x, y) => keys[y]! - keys[x]! || x - y)
|
|
47
|
+
const out: string[] = []
|
|
48
|
+
let i = 0
|
|
49
|
+
let j = 0
|
|
50
|
+
while (i < run && j < rest.length)
|
|
51
|
+
out.push(keys[i]! >= keys[rest[j]!]! ? ids[i++]! : ids[rest[j++]!]!)
|
|
52
|
+
while (i < run) out.push(ids[i++]!)
|
|
53
|
+
while (j < rest.length) out.push(ids[rest[j++]!]!)
|
|
54
|
+
return out
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** How `dispatch` sees the waiting tasks and the run. */
|
|
58
|
+
export interface Dispatching {
|
|
59
|
+
readonly how: Strategy
|
|
60
|
+
/** Each one's longest chain after it (`tailsMs`); none: they go in the order they came. */
|
|
61
|
+
readonly tail?: ((id: string) => number) | undefined
|
|
62
|
+
/** What it asks, as history said. */
|
|
63
|
+
readonly need: (id: string) => Need
|
|
64
|
+
/** What it books now (`Orchestrator.booking`); default `need`'s. */
|
|
65
|
+
readonly book?: (need: Need) => Need
|
|
66
|
+
/** Its direct dependencies (`Ask.up`). */
|
|
67
|
+
readonly deps: (id: string) => readonly string[] | undefined
|
|
68
|
+
/** Whether it may go now, and where (`Orchestrator.pin`): `'wait'` not yet, `'done'` it ended. */
|
|
69
|
+
readonly gate?: (id: string) => 'wait' | 'done' | { only?: string; avoid?: string }
|
|
70
|
+
/** A worker bigger than `above` of `platform` is coming (`assign`'s `waits`). */
|
|
71
|
+
readonly waits?: (above: NonNullable<Need['above']>, platform: string | undefined) => boolean
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Where the `waiting` tasks go now: ordered (`ordered`), gated, and placed
|
|
76
|
+
* (`assign`). The one rule of the live run and of the sims, which keep
|
|
77
|
+
* their waiting tasks in a `Queue` placing as this does: each kept its
|
|
78
|
+
* own, and the sims passed while runs placed otherwise (2026-11-05).
|
|
79
|
+
* `left`: the ones that stay waiting, in order.
|
|
80
|
+
*/
|
|
81
|
+
export function dispatch<B extends Bin>(
|
|
82
|
+
waiting: readonly string[],
|
|
83
|
+
bins: readonly B[],
|
|
84
|
+
o: Dispatching,
|
|
85
|
+
): { to: Map<string, B>; left: string[] } {
|
|
86
|
+
const queue =
|
|
87
|
+
o.tail === undefined
|
|
88
|
+
? waiting
|
|
89
|
+
: ordered(
|
|
90
|
+
waiting,
|
|
91
|
+
o.how.order,
|
|
92
|
+
o.tail,
|
|
93
|
+
o.need,
|
|
94
|
+
bins.reduce((n, b) => Math.max(n, b.cores), 1),
|
|
95
|
+
)
|
|
96
|
+
// The least any ask needs: once no worker takes it, the pass ends. Each
|
|
97
|
+
// pump read every queued task, 8000 of them a pump (2026-10-25).
|
|
98
|
+
const floor = { cores: Infinity, memory: Infinity }
|
|
99
|
+
const needs: Need[] = []
|
|
100
|
+
for (const id of queue) {
|
|
101
|
+
const need = o.book === undefined ? o.need(id) : o.book(o.need(id))
|
|
102
|
+
needs.push(need)
|
|
103
|
+
floor.cores = Math.min(floor.cores, need.cores)
|
|
104
|
+
floor.memory = Math.min(floor.memory, need.memory)
|
|
105
|
+
}
|
|
106
|
+
// Asks are made as the pass reads them: it ends at the first no worker
|
|
107
|
+
// takes, and making every queued one's at each event was most of a
|
|
108
|
+
// 5000-task sim (2026-11-05).
|
|
109
|
+
const read: string[] = []
|
|
110
|
+
// The bins' bounds as assign keeps them: an ask no bin can take or hold
|
|
111
|
+
// for is not made. Each such ask read every bin, and a full run's queue
|
|
112
|
+
// of them at each event was most of a 10 000-task sim (2026-11-05).
|
|
113
|
+
const bounds: Bounds = { done: false, open: true, free: 0, cores: 0, memory: 0 }
|
|
114
|
+
let i = 0
|
|
115
|
+
function* asks(): Generator<Ask> {
|
|
116
|
+
for (; i < queue.length; i++) {
|
|
117
|
+
const id = queue[i]!
|
|
118
|
+
if (bounds.done) return
|
|
119
|
+
if (dead(needs[i]!, bounds)) {
|
|
120
|
+
read.push(id)
|
|
121
|
+
continue
|
|
122
|
+
}
|
|
123
|
+
const gate = o.gate?.(id) ?? {}
|
|
124
|
+
if (gate === 'done') continue
|
|
125
|
+
read.push(id)
|
|
126
|
+
if (gate === 'wait') continue
|
|
127
|
+
yield { id, need: needs[i]!, up: o.deps(id) ?? [], ...gate }
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
const to = assign(asks(), bins, o.waits, o.how.place, { floor, kinds: Infinity, bounds })
|
|
131
|
+
const left = read.filter((id) => !to.has(id))
|
|
132
|
+
// The pass ended at the ask it read last, and took or left it.
|
|
133
|
+
for (let j = i < queue.length && read.at(-1) === queue[i] ? i + 1 : i; j < queue.length; j++)
|
|
134
|
+
left.push(queue[j]!)
|
|
135
|
+
return { to, left }
|
|
26
136
|
}
|
|
27
137
|
|
|
138
|
+
/**
|
|
139
|
+
* The most workers a task is scored on, of those it fits, as Kubernetes
|
|
140
|
+
* scores a share of its nodes: scoring every one cost the orchestrator
|
|
141
|
+
* 15% more CPU at 200 workers, and the sims ended within 0.4% sampling
|
|
142
|
+
* even 2 (2026-11-05).
|
|
143
|
+
*/
|
|
144
|
+
const SAMPLE = 16
|
|
145
|
+
|
|
28
146
|
/** What a task asks of a worker: its history's cores and peak memory. */
|
|
29
147
|
export interface Need {
|
|
30
148
|
readonly cores: number
|
|
@@ -59,8 +177,12 @@ export interface Bin {
|
|
|
59
177
|
export interface Ask {
|
|
60
178
|
readonly id: string
|
|
61
179
|
readonly need: Need
|
|
62
|
-
/**
|
|
63
|
-
|
|
180
|
+
/**
|
|
181
|
+
* Its direct dependencies: a worker holding most is preferred. One holds
|
|
182
|
+
* what was sent with each, so the direct ones say enough: counting every
|
|
183
|
+
* upstream for every worker read thousands an ask (2026-11-05).
|
|
184
|
+
*/
|
|
185
|
+
readonly up?: readonly string[]
|
|
64
186
|
/** Names its need: asks of one kind fit and fail alike (`kindOf`). */
|
|
65
187
|
readonly kind?: string
|
|
66
188
|
/** The one bin that may take it: it holds files no cache does. */
|
|
@@ -130,7 +252,12 @@ export function assign<B extends Bin>(
|
|
|
130
252
|
for (const ask of asks) {
|
|
131
253
|
if (
|
|
132
254
|
all !== undefined &&
|
|
133
|
-
(spent.size >= all.kinds ||
|
|
255
|
+
(spent.size >= all.kinds ||
|
|
256
|
+
// Its bounds, kept as it gives, say it: a read of every bin an ask
|
|
257
|
+
// was most of a plan's play (2026-11-05).
|
|
258
|
+
(bounds !== undefined
|
|
259
|
+
? bounds.done
|
|
260
|
+
: !bins.some((b) => !held.has(b) && fits(b, all.floor))))
|
|
134
261
|
)
|
|
135
262
|
break
|
|
136
263
|
if (ask.kind !== undefined && spent.has(ask.kind)) continue
|
|
@@ -143,20 +270,24 @@ export function assign<B extends Bin>(
|
|
|
143
270
|
: bins.filter((b) => (ask.only === undefined || b.id === ask.only) && b.id !== ask.avoid)
|
|
144
271
|
const of =
|
|
145
272
|
need.platform === undefined ? allowed : allowed.filter((b) => b.platform === need.platform)
|
|
146
|
-
|
|
273
|
+
// With no size to beat, every bin is bigger: no copy of them an ask.
|
|
274
|
+
const anyBigger = need.above === undefined ? of.length > 0 : of.some(bigger)
|
|
147
275
|
if (!anyBigger && need.above !== undefined && waits(need.above, need.platform)) {
|
|
148
276
|
if (ask.kind !== undefined) spent.add(ask.kind)
|
|
149
277
|
continue
|
|
150
278
|
}
|
|
151
|
-
const pool = anyBigger ? of.filter(bigger) : of
|
|
152
|
-
const up = ask.up === undefined ? [] : [...ask.up]
|
|
279
|
+
const pool = anyBigger && need.above !== undefined ? of.filter(bigger) : of
|
|
153
280
|
let best: B | undefined
|
|
154
281
|
let bestScore: [number, number] = [0, 0]
|
|
155
|
-
|
|
282
|
+
let seen = 0
|
|
283
|
+
for (let i = 0; i < pool.length; i++) {
|
|
284
|
+
const b = pool[i]!
|
|
156
285
|
if (held.has(b) || !fits(b, need)) continue
|
|
286
|
+
// Workers fill in order, so the first that fit hold the fullest.
|
|
287
|
+
if (++seen > SAMPLE) break
|
|
157
288
|
let has = 0
|
|
158
|
-
for (const id of up) if (b.has?.has(id) === true) has++
|
|
159
|
-
const score: [number, number] = [has, fit(b, need, place,
|
|
289
|
+
for (const id of ask.up ?? []) if (b.has?.has(id) === true) has++
|
|
290
|
+
const score: [number, number] = [has, fit(b, need, place, i)]
|
|
160
291
|
if (
|
|
161
292
|
best === undefined ||
|
|
162
293
|
score[0] > bestScore[0] ||
|
|
@@ -207,6 +338,16 @@ export interface Bounds {
|
|
|
207
338
|
cores: number
|
|
208
339
|
/** The least memory free on one. */
|
|
209
340
|
memory: number
|
|
341
|
+
/** The same of each platform's own: a task naming one fits only those. */
|
|
342
|
+
of?: Map<string, Room>
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
/** What `Bounds` says of a set of bins. */
|
|
346
|
+
interface Room {
|
|
347
|
+
open: boolean
|
|
348
|
+
free: number
|
|
349
|
+
cores: number
|
|
350
|
+
memory: number
|
|
210
351
|
}
|
|
211
352
|
|
|
212
353
|
function boundsOf(
|
|
@@ -220,30 +361,41 @@ function boundsOf(
|
|
|
220
361
|
out.free = -Infinity
|
|
221
362
|
out.cores = Infinity
|
|
222
363
|
out.memory = Infinity
|
|
364
|
+
out.of?.clear()
|
|
223
365
|
for (const b of bins) {
|
|
224
366
|
if (held.has(b)) continue
|
|
225
367
|
if (floor !== undefined && fits(b, floor)) out.done = false
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
out.
|
|
229
|
-
|
|
368
|
+
room(out, b)
|
|
369
|
+
if (b.platform === undefined) continue
|
|
370
|
+
out.of ??= new Map()
|
|
371
|
+
let p = out.of.get(b.platform)
|
|
372
|
+
if (p === undefined)
|
|
373
|
+
out.of.set(
|
|
374
|
+
b.platform,
|
|
375
|
+
(p = { open: false, free: -Infinity, cores: Infinity, memory: Infinity }),
|
|
376
|
+
)
|
|
377
|
+
room(p, b)
|
|
230
378
|
}
|
|
231
379
|
}
|
|
232
380
|
|
|
381
|
+
function room(r: Room, b: Bin): void {
|
|
382
|
+
if (b.running === 0) r.open = true
|
|
383
|
+
r.free = Math.max(r.free, b.cores - b.used.cores)
|
|
384
|
+
r.cores = Math.min(r.cores, b.cores)
|
|
385
|
+
r.memory = Math.min(r.memory, b.memory - b.used.memory)
|
|
386
|
+
}
|
|
387
|
+
|
|
233
388
|
/**
|
|
234
389
|
* True when `assign` would fit `need` on no bin and hold none for it, by
|
|
235
390
|
* `fits` and its hold rule: no bin is empty, and on each it is less than
|
|
236
391
|
* the bin, fits its memory, and wants more cores than are free.
|
|
237
392
|
*/
|
|
238
393
|
export function dead(need: Need, b: Bounds): boolean {
|
|
239
|
-
return
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
need.cores < b.cores &&
|
|
245
|
-
need.memory <= b.memory
|
|
246
|
-
)
|
|
394
|
+
if (need.above !== undefined) return false
|
|
395
|
+
// Of its platform's bins: with none free of a hold, no ask of it does a thing.
|
|
396
|
+
const r = need.platform === undefined ? b : b.of?.get(need.platform)
|
|
397
|
+
if (r === undefined) return true
|
|
398
|
+
return !r.open && need.cores > r.free + 1e-9 && need.cores < r.cores && need.memory <= r.memory
|
|
247
399
|
}
|
|
248
400
|
|
|
249
401
|
/** How well `need` suits `b` by `place`: the higher, the better. */
|
|
@@ -266,3 +418,326 @@ function fit(b: Bin, need: Need, place: Strategy['place'], index: number): numbe
|
|
|
266
418
|
}
|
|
267
419
|
}
|
|
268
420
|
}
|
|
421
|
+
|
|
422
|
+
/**
|
|
423
|
+
* The live run's waiting tasks, kept across its passes in the order
|
|
424
|
+
* `ordered` gives (`critical-path`, `fifo`) and grouped by platform and
|
|
425
|
+
* cores' class: a pass places exactly as `dispatch` over the whole list,
|
|
426
|
+
* but passes a group whole while no worker has room for any of it, where
|
|
427
|
+
* each pass ordered and read every waiting task, a fifth of a 10 000-task
|
|
428
|
+
* sim (2026-11-06). `graphene` ranks by the longest waiting tail, which
|
|
429
|
+
* moves as tasks leave: it keeps the list and `dispatch`.
|
|
430
|
+
*/
|
|
431
|
+
export class Queue {
|
|
432
|
+
private list: string[] | undefined
|
|
433
|
+
private readonly groups = new Map<string, Group>()
|
|
434
|
+
private readonly of = new Map<string, Entry>()
|
|
435
|
+
private front = 0
|
|
436
|
+
private back = 0
|
|
437
|
+
|
|
438
|
+
constructor(
|
|
439
|
+
private readonly how: Strategy,
|
|
440
|
+
private readonly tail: ((id: string) => number) | undefined,
|
|
441
|
+
private readonly need: (id: string) => Need,
|
|
442
|
+
) {
|
|
443
|
+
if (how.order === 'graphene' && tail !== undefined) this.list = []
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
get size(): number {
|
|
447
|
+
return this.list?.length ?? this.of.size
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
has(id: string): boolean {
|
|
451
|
+
return this.list?.includes(id) ?? this.of.has(id)
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/** In the order a pass reads them. */
|
|
455
|
+
ids(): string[] {
|
|
456
|
+
if (this.list !== undefined) return [...this.list]
|
|
457
|
+
return [...this.of.values()].sort((a, b) => (before(a, b) ? -1 : 1)).map((e) => e.id)
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
/** `ids` join, in order, behind every one ranked as high, or `front` before all. */
|
|
461
|
+
add(ids: readonly string[], front = false): void {
|
|
462
|
+
if (this.list !== undefined) {
|
|
463
|
+
if (front) this.list.unshift(...ids)
|
|
464
|
+
else this.list.push(...ids)
|
|
465
|
+
return
|
|
466
|
+
}
|
|
467
|
+
if (front) this.front -= ids.length
|
|
468
|
+
let seq = front ? this.front : this.back
|
|
469
|
+
for (const id of ids) {
|
|
470
|
+
const need = this.need(id)
|
|
471
|
+
const e: Entry = {
|
|
472
|
+
id,
|
|
473
|
+
rank: this.how.order === 'fifo' || this.tail === undefined ? 0 : -this.tail(id),
|
|
474
|
+
seq: seq++,
|
|
475
|
+
need,
|
|
476
|
+
group: undefined!,
|
|
477
|
+
}
|
|
478
|
+
const odd = need.above !== undefined || need.ask !== undefined || need.cores === Infinity
|
|
479
|
+
const key = `${need.platform}\0${odd ? 'odd' : Math.floor(Math.log2(Math.max(need.cores, 1e-6)) * CLASSES)}`
|
|
480
|
+
let g = this.groups.get(key)
|
|
481
|
+
if (g === undefined) this.groups.set(key, (g = new Group(odd, need.platform)))
|
|
482
|
+
e.group = g
|
|
483
|
+
g.add(e)
|
|
484
|
+
this.of.set(id, e)
|
|
485
|
+
}
|
|
486
|
+
if (!front) this.back = seq
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
delete(id: string): boolean {
|
|
490
|
+
if (this.list !== undefined) {
|
|
491
|
+
const at = this.list.indexOf(id)
|
|
492
|
+
if (at >= 0) this.list.splice(at, 1)
|
|
493
|
+
return at >= 0
|
|
494
|
+
}
|
|
495
|
+
const e = this.of.get(id)
|
|
496
|
+
if (e === undefined) return false
|
|
497
|
+
this.drop(e)
|
|
498
|
+
return true
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
private drop(e: Entry): void {
|
|
502
|
+
this.of.delete(e.id)
|
|
503
|
+
e.gone = true
|
|
504
|
+
if (e.group.drop() === 0)
|
|
505
|
+
for (const [k, g] of this.groups) if (g === e.group) this.groups.delete(k)
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
/**
|
|
509
|
+
* Where the waiting tasks go now, as `dispatch` places them; the ones
|
|
510
|
+
* placed or found done leave. `book`, if given, must keep a need's
|
|
511
|
+
* order by cores among needs alike but for cores.
|
|
512
|
+
*/
|
|
513
|
+
pass<B extends Bin>(
|
|
514
|
+
bins: readonly B[],
|
|
515
|
+
o: Omit<Dispatching, 'how' | 'tail' | 'need'>,
|
|
516
|
+
): Map<string, B> {
|
|
517
|
+
if (this.list !== undefined) {
|
|
518
|
+
const { to, left } = dispatch(this.list, bins, {
|
|
519
|
+
...o,
|
|
520
|
+
how: this.how,
|
|
521
|
+
tail: this.tail,
|
|
522
|
+
need: this.need,
|
|
523
|
+
})
|
|
524
|
+
this.list = left
|
|
525
|
+
return to
|
|
526
|
+
}
|
|
527
|
+
return this.place(bins, o)
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
private place<B extends Bin>(
|
|
531
|
+
bins: readonly B[],
|
|
532
|
+
o: Omit<Dispatching, 'how' | 'tail' | 'need'>,
|
|
533
|
+
): Map<string, B> {
|
|
534
|
+
const book = o.book ?? ((n: Need): Need => n)
|
|
535
|
+
const gs = [...this.groups.values()]
|
|
536
|
+
const floor = { cores: Infinity, memory: Infinity }
|
|
537
|
+
for (const g of gs) {
|
|
538
|
+
g.at = g.head()
|
|
539
|
+
const low = g.low()
|
|
540
|
+
if (low === undefined) continue
|
|
541
|
+
// Booked as its least: what each of its tasks books is at least this.
|
|
542
|
+
g.lo = g.odd ? Infinity : book(low).cores
|
|
543
|
+
floor.cores = Math.min(floor.cores, g.odd ? g.oddFloor(book) : g.lo)
|
|
544
|
+
floor.memory = Math.min(floor.memory, g.memory())
|
|
545
|
+
}
|
|
546
|
+
const bounds: Bounds = { done: false, open: true, free: 0, cores: 0, memory: 0 }
|
|
547
|
+
const done: Entry[] = []
|
|
548
|
+
let cursor: Entry | undefined
|
|
549
|
+
function* asks(): Generator<Ask> {
|
|
550
|
+
for (;;) {
|
|
551
|
+
if (bounds.done) return
|
|
552
|
+
// The group whose next is first, past any no worker has room for now.
|
|
553
|
+
let g: Group | undefined
|
|
554
|
+
let second: Entry | undefined
|
|
555
|
+
for (const x of gs) {
|
|
556
|
+
if (x.at >= x.q.length) continue
|
|
557
|
+
if (!x.odd && x.full(bounds)) continue
|
|
558
|
+
if (cursor !== undefined && !before(cursor, x.q[x.at]!)) x.at = x.after(cursor)
|
|
559
|
+
x.at = x.skip(x.at)
|
|
560
|
+
const e = x.q[x.at]
|
|
561
|
+
if (e === undefined) continue
|
|
562
|
+
if (g === undefined || before(e, g.q[g.at]!)) {
|
|
563
|
+
if (g !== undefined && (second === undefined || before(g.q[g.at]!, second)))
|
|
564
|
+
second = g.q[g.at]!
|
|
565
|
+
g = x
|
|
566
|
+
} else if (second === undefined || before(e, second)) second = e
|
|
567
|
+
}
|
|
568
|
+
if (g === undefined) return
|
|
569
|
+
// Its run up to the next group's first, read one by one as before.
|
|
570
|
+
for (;;) {
|
|
571
|
+
const e = g.q[g.at]
|
|
572
|
+
if (e === undefined || (second !== undefined && !before(e, second))) break
|
|
573
|
+
g.at = g.skip(g.at + 1)
|
|
574
|
+
cursor = e
|
|
575
|
+
if (bounds.done) return
|
|
576
|
+
const need = book(e.need)
|
|
577
|
+
if (dead(need, bounds)) continue
|
|
578
|
+
const gate = o.gate?.(e.id) ?? {}
|
|
579
|
+
if (gate === 'done') {
|
|
580
|
+
done.push(e)
|
|
581
|
+
continue
|
|
582
|
+
}
|
|
583
|
+
if (gate === 'wait') continue
|
|
584
|
+
yield { id: e.id, need, up: o.deps(e.id) ?? [], ...gate }
|
|
585
|
+
break
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
const to = assign(asks(), bins, o.waits, this.how.place, { floor, kinds: Infinity, bounds })
|
|
590
|
+
for (const e of done) if (!e.gone) this.drop(e)
|
|
591
|
+
for (const id of to.keys()) this.drop(this.of.get(id)!)
|
|
592
|
+
return to
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
/** Classes a doubling of cores spans (`Queue`'s groups). */
|
|
597
|
+
const CLASSES = 4
|
|
598
|
+
|
|
599
|
+
interface Entry {
|
|
600
|
+
readonly id: string
|
|
601
|
+
/** The lower first, then the lower `seq`. */
|
|
602
|
+
readonly rank: number
|
|
603
|
+
readonly seq: number
|
|
604
|
+
readonly need: Need
|
|
605
|
+
group: Group
|
|
606
|
+
gone?: boolean
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
function before(a: Entry, b: Entry): boolean {
|
|
610
|
+
return a.rank < b.rank || (a.rank === b.rank && a.seq < b.seq)
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
/** Waiting tasks of one platform and cores' class, in rank order; gone ones linger until half are. */
|
|
614
|
+
class Group {
|
|
615
|
+
q: Entry[] = []
|
|
616
|
+
private gone = 0
|
|
617
|
+
/** Where a pass reads next. */
|
|
618
|
+
at = 0
|
|
619
|
+
/** The least any of it books, this pass. */
|
|
620
|
+
lo = Infinity
|
|
621
|
+
/** The most cores and memory one asked: kept as they joined until it is read again without the gone. */
|
|
622
|
+
private hiCores = -Infinity
|
|
623
|
+
private hiMemory = -Infinity
|
|
624
|
+
/** Its least cores and memory, gone ones lazily left off the top. */
|
|
625
|
+
private readonly byCores: Entry[] = []
|
|
626
|
+
private readonly byMemory: Entry[] = []
|
|
627
|
+
|
|
628
|
+
constructor(
|
|
629
|
+
readonly odd: boolean,
|
|
630
|
+
readonly platform: string | undefined,
|
|
631
|
+
) {}
|
|
632
|
+
|
|
633
|
+
add(e: Entry): void {
|
|
634
|
+
let lo = 0
|
|
635
|
+
let hi = this.q.length
|
|
636
|
+
// Most join last: a search from the end.
|
|
637
|
+
if (hi === 0 || before(this.q[hi - 1]!, e)) lo = hi
|
|
638
|
+
while (lo < hi) {
|
|
639
|
+
const mid = (lo + hi) >> 1
|
|
640
|
+
if (before(this.q[mid]!, e)) lo = mid + 1
|
|
641
|
+
else hi = mid
|
|
642
|
+
}
|
|
643
|
+
this.q.splice(lo, 0, e)
|
|
644
|
+
this.hiCores = Math.max(this.hiCores, e.need.cores)
|
|
645
|
+
this.hiMemory = Math.max(this.hiMemory, e.need.memory)
|
|
646
|
+
push(this.byCores, e, (x) => x.need.cores)
|
|
647
|
+
push(this.byMemory, e, (x) => x.need.memory)
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
/** One of it left: how many are left. */
|
|
651
|
+
drop(): number {
|
|
652
|
+
this.gone++
|
|
653
|
+
const left = this.q.length - this.gone
|
|
654
|
+
if (this.gone > left) {
|
|
655
|
+
this.q = this.q.filter((e) => !e.gone)
|
|
656
|
+
this.gone = 0
|
|
657
|
+
this.hiCores = -Infinity
|
|
658
|
+
this.hiMemory = -Infinity
|
|
659
|
+
for (const e of this.q) {
|
|
660
|
+
this.hiCores = Math.max(this.hiCores, e.need.cores)
|
|
661
|
+
this.hiMemory = Math.max(this.hiMemory, e.need.memory)
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
return left
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
head(): number {
|
|
668
|
+
return this.skip(0)
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
skip(p: number): number {
|
|
672
|
+
while (p < this.q.length && this.q[p]!.gone === true) p++
|
|
673
|
+
return p
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
/** The first place ranked past `cursor`. */
|
|
677
|
+
after(cursor: Entry): number {
|
|
678
|
+
let p = this.at
|
|
679
|
+
let hi = this.q.length
|
|
680
|
+
while (p < hi) {
|
|
681
|
+
const mid = (p + hi) >> 1
|
|
682
|
+
if (!before(cursor, this.q[mid]!)) p = mid + 1
|
|
683
|
+
else hi = mid
|
|
684
|
+
}
|
|
685
|
+
return p
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
low(): Need | undefined {
|
|
689
|
+
return top(this.byCores, (x) => x.need.cores)?.need
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
memory(): number {
|
|
693
|
+
return top(this.byMemory, (x) => x.need.memory)?.need.memory ?? Infinity
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/** An odd group's least booked: read one by one, as books of each differ. */
|
|
697
|
+
oddFloor(book: (n: Need) => Need): number {
|
|
698
|
+
let least = Infinity
|
|
699
|
+
for (let p = this.at; p < this.q.length; p++)
|
|
700
|
+
if (this.q[p]!.gone !== true) least = Math.min(least, book(this.q[p]!.need).cores)
|
|
701
|
+
return least
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
/** No worker of its platform has room for any of it now, as `dead` says of each. */
|
|
705
|
+
full(b: Bounds): boolean {
|
|
706
|
+
const r = this.platform === undefined ? b : b.of?.get(this.platform)
|
|
707
|
+
if (r === undefined) return true
|
|
708
|
+
return !r.open && this.lo > r.free + 1e-9 && this.hiCores < r.cores && this.hiMemory <= r.memory
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
/** A binary min-heap of entries by `by`. */
|
|
713
|
+
function push(h: Entry[], e: Entry, by: (e: Entry) => number): void {
|
|
714
|
+
h.push(e)
|
|
715
|
+
let i = h.length - 1
|
|
716
|
+
while (i > 0) {
|
|
717
|
+
const p = (i - 1) >> 1
|
|
718
|
+
if (by(h[p]!) <= by(h[i]!)) break
|
|
719
|
+
;[h[p], h[i]] = [h[i]!, h[p]!]
|
|
720
|
+
i = p
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
/** The least of `h` not gone, the gone ones above it popped. */
|
|
725
|
+
function top(h: Entry[], by: (e: Entry) => number): Entry | undefined {
|
|
726
|
+
while (h.length > 0 && h[0]!.gone === true) {
|
|
727
|
+
const last = h.pop()!
|
|
728
|
+
if (h.length === 0) break
|
|
729
|
+
h[0] = last
|
|
730
|
+
let i = 0
|
|
731
|
+
for (;;) {
|
|
732
|
+
const l = 2 * i + 1
|
|
733
|
+
const r = l + 1
|
|
734
|
+
let m = i
|
|
735
|
+
if (l < h.length && by(h[l]!) < by(h[m]!)) m = l
|
|
736
|
+
if (r < h.length && by(h[r]!) < by(h[m]!)) m = r
|
|
737
|
+
if (m === i) break
|
|
738
|
+
;[h[m], h[i]] = [h[i]!, h[m]!]
|
|
739
|
+
i = m
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
return h[0]
|
|
743
|
+
}
|
package/src/forecast.ts
CHANGED
|
@@ -62,7 +62,6 @@ export function likelyMisses(
|
|
|
62
62
|
return c
|
|
63
63
|
}
|
|
64
64
|
for (const [id, n] of nodes)
|
|
65
|
-
if (n.config.exec?.command !== undefined && n.config.cache
|
|
66
|
-
out.add(id)
|
|
65
|
+
if (n.config.exec?.command !== undefined && n.config.cache === true && changes(id)) out.add(id)
|
|
67
66
|
return out
|
|
68
67
|
}
|
package/src/github.ts
CHANGED
|
@@ -138,9 +138,10 @@ export function github(options: GithubOptions = {}): DteAdapter {
|
|
|
138
138
|
* else is said. One the spent API limit refused is asked again at its
|
|
139
139
|
* reset, when that comes within `CANCEL_WAIT_MS`.
|
|
140
140
|
*/
|
|
141
|
-
const cancelOf = async (run: string): Promise<void> => {
|
|
141
|
+
const cancelOf = async (run: string, still: () => boolean = () => true): Promise<void> => {
|
|
142
142
|
const end = Date.now() + CANCEL_WAIT_MS
|
|
143
143
|
for (;;) {
|
|
144
|
+
if (!still()) return
|
|
144
145
|
const res = await call(`${run}/cancel`, { method: 'POST' }).catch((err: unknown) => {
|
|
145
146
|
failed(`cancelling ${runOf(run)}`, err instanceof Error ? err.message : String(err))
|
|
146
147
|
return undefined
|
|
@@ -227,7 +228,9 @@ export function github(options: GithubOptions = {}): DteAdapter {
|
|
|
227
228
|
return
|
|
228
229
|
}
|
|
229
230
|
for (const id of ids) decide(id, false)
|
|
230
|
-
|
|
231
|
+
// Asked again at an API limit's reset, it stops once the run took
|
|
232
|
+
// one of them after all (`kept`).
|
|
233
|
+
return cancelOf(run, () => ids.every((id) => cancelled.has(id)))
|
|
231
234
|
})
|
|
232
235
|
.finally(() => pending.delete(p))
|
|
233
236
|
pending.add(p)
|
|
@@ -373,10 +376,13 @@ export function github(options: GithubOptions = {}): DteAdapter {
|
|
|
373
376
|
throw new Error(`github(): dispatching ${workflow}: ${res.status} ${text}${fix}`)
|
|
374
377
|
}
|
|
375
378
|
}
|
|
376
|
-
|
|
379
|
+
// quiet: a body that is not JSON names no run, refused below
|
|
380
|
+
const said =
|
|
377
381
|
res.status === 200
|
|
378
|
-
? ((await res.json()) as { workflow_run_id?:
|
|
382
|
+
? ((await res.json().catch(() => null)) as { workflow_run_id?: unknown } | null)
|
|
383
|
+
?.workflow_run_id
|
|
379
384
|
: undefined
|
|
385
|
+
const id = Number.isSafeInteger(said) && (said as number) > 0 ? (said as number) : undefined
|
|
380
386
|
// An API naming no run (a 204) is refused, not trusted by repository
|
|
381
387
|
// alone: any job of it with id-token: write would join as a worker.
|
|
382
388
|
if (id === undefined) {
|
|
@@ -472,6 +478,11 @@ export function github(options: GithubOptions = {}): DteAdapter {
|
|
|
472
478
|
for (const id of gone) decide(id, false)
|
|
473
479
|
return Promise.all(left).then((l) => ids.filter((_, i) => l[i]))
|
|
474
480
|
},
|
|
481
|
+
// One given up as late joined after all: its run is not cancelled.
|
|
482
|
+
kept(id) {
|
|
483
|
+
late.delete(id)
|
|
484
|
+
cancelled.delete(id)
|
|
485
|
+
},
|
|
475
486
|
// A finished run answers 409; one still queued or running is the
|
|
476
487
|
// run's no more: its results are in, or the run failed without them.
|
|
477
488
|
async close() {
|
|
@@ -481,7 +492,7 @@ export function github(options: GithubOptions = {}): DteAdapter {
|
|
|
481
492
|
await Promise.all(pending)
|
|
482
493
|
for (const id of [...deciding.keys()]) decide(id, false)
|
|
483
494
|
// A cancel that fails leaves the run to its timeout, and says so.
|
|
484
|
-
await Promise.all(runs.splice(0).map(cancelOf))
|
|
495
|
+
await Promise.all(runs.splice(0).map((run) => cancelOf(run)))
|
|
485
496
|
},
|
|
486
497
|
}
|
|
487
498
|
}
|