@isoloop/balancer 0.0.0 → 0.1.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +48 -0
- package/dist/index.d.ts +302 -0
- package/dist/index.js +961 -0
- package/package.json +29 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sabin Pandelovitch
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# @isoloop/balancer
|
|
2
|
+
|
|
3
|
+
**Experimental.** Automatic placement for [Isoloop](https://github.com/isoloop/isoloop) isolates. Spawn through the balancer instead of naming a pool: it places each isolate in a pool with room by measured load, moves one out when its pool lags or is full, and keeps one that blocks or crashes in a quarantine pool until it's quiet again ([ADR 0017](https://github.com/isoloop/isoloop/blob/main/docs/adr/0017-opt-in-balancer.md)).
|
|
4
|
+
|
|
5
|
+
```sh
|
|
6
|
+
npm install @isoloop/balancer@next @isoloop/host@next
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
```ts
|
|
10
|
+
import { createKernel } from '@isoloop/host';
|
|
11
|
+
import { createBalancer } from '@isoloop/balancer';
|
|
12
|
+
|
|
13
|
+
const kernel = createKernel({ /* … */ });
|
|
14
|
+
const balancer = createBalancer(kernel, { maxWorkers: 4 });
|
|
15
|
+
|
|
16
|
+
balancer.spawn({ id: 'task-1', src: taskUrl }); // no `pool`: the balancer picks one
|
|
17
|
+
balancer.on('move', (e) => console.log(e.id, e.from, '→', e.to, e.reason));
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
It also reacts when work piles up, if the app says how much is queued (M3i):
|
|
21
|
+
|
|
22
|
+
```ts
|
|
23
|
+
// Once a second or so, per isolate; for a consumer group, the group's backlog on every member.
|
|
24
|
+
balancer.report('task-1', { backlog: queue.length });
|
|
25
|
+
balancer.on('behind', (e) => console.warn(e.id, 'is behind and has nowhere to go'));
|
|
26
|
+
balancer.on('arrived', (e) => console.log(e.id, 'moved in', e.costMs, 'ms'));
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
- **Backlog:** a backlog higher at the end of a 10 s window than at its start, and above 0, moves the isolate somewhere with more CPU: a pool of its own if the cap allows, else a pool with less load than its pool-mates. One such move per fresh 10 s window, so a group's members move one at a time. A report older than 10 s counts as none. With nowhere to go, it emits `behind` (at most every 30 s).
|
|
30
|
+
- **Fewer moves:** a move for load is made only if the moves it starts are predicted to bring the pool back under its line, and each isolate gets at most 3 of them in 5 minutes. Spike, crash and bisect moves protect the others and aren't limited.
|
|
31
|
+
- **Spikes:** a first long task is only reported (`spike`, action `noted`); a second from the same isolate within 60 s sends it to quarantine. Its own fatal error still does at once.
|
|
32
|
+
- **Packing back:** an isolate alone in its pool and steady for 60 s joins another of the balancer's pools if both stay under 40 % of a core, which frees a worker; at most one every 60 s. Turn it off with `pack: { enabled: false }`.
|
|
33
|
+
- **Move cost:** `arrived` follows each `move` with `costMs`, the time from stopping the isolate to its first beat in the new pool.
|
|
34
|
+
|
|
35
|
+
And when its busy time misses work, or the kernel gives up on an isolate (M3j):
|
|
36
|
+
|
|
37
|
+
```ts
|
|
38
|
+
// Work timed by the app, as a running total since the isolate started: for slices resumed after `await`.
|
|
39
|
+
balancer.report('task-1', { backlog: queue.length, busyMs: workMsSoFar });
|
|
40
|
+
balancer.on('failed', (e) => console.warn(e.id, 'was given up on in', e.pool, e.reason));
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
- **Reported busy time:** the balancer takes the total's difference over its 3 s window and uses the larger of that and the beats' `busyMs`, wherever it reads a load.
|
|
44
|
+
- **Packing** goes ahead while a backlog holds steady; only a backlog that rose over the 60 s window blocks it.
|
|
45
|
+
- **The quarantine:** an offender's quiet window is capped at 2 minutes, and offences wear off with each quiet window back in a pool. A bisect starts only with a quarantine pool of its own, never alongside offenders; with none free, the pool's members are `behind`.
|
|
46
|
+
- **Given up:** an isolate past the kernel's restart limit is `failed`. It leaves its pool's load and any bisect, and no move restarts it; after `kernel.restart(id)` it's placed again as new.
|
|
47
|
+
|
|
48
|
+
One balancer per kernel at a time (`dispose()` frees the kernel for another); its pools are named `balancer-…`, which the host reserves. The API, the triggers, the quarantine and what `busyMs` measures are in the [balancer guide](https://github.com/isoloop/isoloop/blob/main/docs/guide/balancer.md).
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The part of the kernel the balancer uses: the public API only (`spawn`,
|
|
3
|
+
* `stop`, `pools`, and the `isolate:beat`, `isolate:crash` and
|
|
4
|
+
* `isolate:failed` events), so `@isoloop/host`'s `Kernel` fits it as is.
|
|
5
|
+
*/
|
|
6
|
+
interface KernelLike<M extends ManifestLike = ManifestLike> {
|
|
7
|
+
spawn(manifest: M): unknown;
|
|
8
|
+
stop(id: string): Promise<void>;
|
|
9
|
+
pools(): readonly PoolLike[];
|
|
10
|
+
on(event: "isolate:crash", fn: (e: CrashLike) => void): () => void;
|
|
11
|
+
/** Every heartbeat as it arrives, once per member of a pool (ADR 0017). */
|
|
12
|
+
on(event: "isolate:beat", fn: (e: BeatLike) => void): () => void;
|
|
13
|
+
/** An isolate past the kernel's restart limit: it stays down until `kernel.restart` (M3j). */
|
|
14
|
+
on(event: "isolate:failed", fn: (e: FailedLike) => void): () => void;
|
|
15
|
+
/**
|
|
16
|
+
* Only for kernels not made by `createKernel` (a simulated one in tests):
|
|
17
|
+
* the claim of the `balancer-*` pools, as the host's `claimPools` gives it.
|
|
18
|
+
* The host's `Kernel` has none; the balancer then calls the host's.
|
|
19
|
+
*/
|
|
20
|
+
claimPools?(owner: "balancer"): {
|
|
21
|
+
spawn(manifest: M): unknown;
|
|
22
|
+
release?(): void;
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
/** What the balancer reads from an isolate manifest: its id, and the pool it sets. */
|
|
26
|
+
type ManifestLike = {
|
|
27
|
+
id: string;
|
|
28
|
+
pool?: string;
|
|
29
|
+
};
|
|
30
|
+
/** The kernel's `isolate:beat` event (ADR 0017). */
|
|
31
|
+
type BeatLike = {
|
|
32
|
+
id: string;
|
|
33
|
+
pool?: string;
|
|
34
|
+
/** The worker's lag at this beat: the same for every member of a pool. */
|
|
35
|
+
lagMs: number;
|
|
36
|
+
/**
|
|
37
|
+
* Time this isolate spent running in the interval the beat covers.
|
|
38
|
+
* Absent from runtimes that don't report it; read as 0.
|
|
39
|
+
*/
|
|
40
|
+
busyMs?: number;
|
|
41
|
+
/** The host's `performance.now()` on receipt. */
|
|
42
|
+
at: number;
|
|
43
|
+
};
|
|
44
|
+
/** The fields of `PoolSnapshot` the balancer reads. */
|
|
45
|
+
type PoolLike = {
|
|
46
|
+
name: string;
|
|
47
|
+
members: readonly string[];
|
|
48
|
+
running: boolean;
|
|
49
|
+
};
|
|
50
|
+
/** The fields of the kernel's `isolate:crash` event the balancer reads. */
|
|
51
|
+
type CrashLike = {
|
|
52
|
+
id: string;
|
|
53
|
+
reason: string;
|
|
54
|
+
pool?: string;
|
|
55
|
+
};
|
|
56
|
+
/** The fields of the kernel's `isolate:failed` event the balancer reads (M3j). */
|
|
57
|
+
type FailedLike = {
|
|
58
|
+
id: string;
|
|
59
|
+
reason: string;
|
|
60
|
+
pool?: string;
|
|
61
|
+
};
|
|
62
|
+
/**
|
|
63
|
+
* Where the balancer keeps each isolate's measured load between page loads,
|
|
64
|
+
* so a reload places from history (M3h, Q17). The app supplies it; the
|
|
65
|
+
* creator backs it with a persistent topic. `get` returning `undefined`
|
|
66
|
+
* means "never measured": the isolate starts in quarantine.
|
|
67
|
+
*/
|
|
68
|
+
interface LoadStore {
|
|
69
|
+
get(id: string): number | undefined;
|
|
70
|
+
set(id: string, loadPct: number): void;
|
|
71
|
+
}
|
|
72
|
+
type BalancerOptions = {
|
|
73
|
+
/**
|
|
74
|
+
* The balancer's own allowance of workers (ADR 0017, amended): its
|
|
75
|
+
* `balancer-N` pools, plus any quarantine worker beyond the first. The
|
|
76
|
+
* quarantine, `balancer-quarantine-1`, is allowed on top, and workers the
|
|
77
|
+
* app runs itself don't count; the kernel's budget stays the page-wide
|
|
78
|
+
* warning. Default `max(2, navigator.hardwareConcurrency - 2)`; at least 1.
|
|
79
|
+
*/
|
|
80
|
+
maxWorkers?: number;
|
|
81
|
+
/**
|
|
82
|
+
* A beat with the pool's `lagMs` above this is a spike, put on the member
|
|
83
|
+
* that was busiest in it. A first spike is only reported (`spike`, action
|
|
84
|
+
* `noted`); a second from the same member within 60 s sends it to
|
|
85
|
+
* quarantine (M3i). Default 100.
|
|
86
|
+
*/
|
|
87
|
+
spike?: {
|
|
88
|
+
lagMs?: number;
|
|
89
|
+
};
|
|
90
|
+
/**
|
|
91
|
+
* The pool's median `lagMs` above `lagMs` over `forMs` moves its heaviest
|
|
92
|
+
* member out. Default 50 ms over 3,000 ms. So does a pool whose members'
|
|
93
|
+
* busy time adds up to 80 % of a core or more over the same window, whatever
|
|
94
|
+
* its lag (a fixed threshold, above `headroom`).
|
|
95
|
+
*/
|
|
96
|
+
sustained?: {
|
|
97
|
+
lagMs?: number;
|
|
98
|
+
forMs?: number;
|
|
99
|
+
};
|
|
100
|
+
/** An isolate joins a pool only while pool load + its load stays under `loadPct`. Default 60. */
|
|
101
|
+
headroom?: {
|
|
102
|
+
loadPct?: number;
|
|
103
|
+
};
|
|
104
|
+
/**
|
|
105
|
+
* Leaving quarantine: an isolate there only to be measured leaves once it
|
|
106
|
+
* has 3 s of beats there, or, if a bisect moved it there, once that bisect
|
|
107
|
+
* ends. An offender leaves after `steadyMs` (default 30,000 ms) with no
|
|
108
|
+
* spike and no crash, whatever its load; each offence doubles the window
|
|
109
|
+
* (30, 60, 120 s), capped at 2 minutes (or `steadyMs`, if longer). Back in
|
|
110
|
+
* a balancer pool, an offender loses one offence for each quiet window it
|
|
111
|
+
* completes there (M3j). After `maxReturns` returns (default 3) the next
|
|
112
|
+
* offence keeps it apart for good. Either goes where the placement rule
|
|
113
|
+
* finds room, or stays, flagged `atCap`, until there is.
|
|
114
|
+
*/
|
|
115
|
+
quarantine?: {
|
|
116
|
+
steadyMs?: number;
|
|
117
|
+
maxReturns?: number;
|
|
118
|
+
};
|
|
119
|
+
loads?: LoadStore;
|
|
120
|
+
/**
|
|
121
|
+
* Packing back (M3i): a member alone in its pool and steady for 60 s (no
|
|
122
|
+
* spike, crash, lagging or full pool, and a backlog no higher than 60 s
|
|
123
|
+
* before, M3j) joins another of the balancer's pools if both together stay
|
|
124
|
+
* under 40 % of a core, freeing a worker. At most one pack every 60 s. On by
|
|
125
|
+
* default.
|
|
126
|
+
*/
|
|
127
|
+
pack?: {
|
|
128
|
+
enabled?: boolean;
|
|
129
|
+
};
|
|
130
|
+
};
|
|
131
|
+
/**
|
|
132
|
+
* What the app tells the balancer about an isolate (`report`): its queued
|
|
133
|
+
* work, its own measured busy time, or both. At least one must be given.
|
|
134
|
+
*/
|
|
135
|
+
type Report = {
|
|
136
|
+
/**
|
|
137
|
+
* Work items waiting for the isolate, in any unit the app likes, as long as
|
|
138
|
+
* it's the same one each time. For a consumer group, report the group's
|
|
139
|
+
* backlog on every member.
|
|
140
|
+
*/
|
|
141
|
+
backlog?: number;
|
|
142
|
+
/**
|
|
143
|
+
* Milliseconds of work the app measured for the isolate since it started:
|
|
144
|
+
* a running total, so a late or missed report still counts (M3j). For work
|
|
145
|
+
* the kernel's `busyMs` misses, such as slices resumed after `await`. The
|
|
146
|
+
* balancer takes its difference over the 3 s window and uses the larger of
|
|
147
|
+
* that and the beats' figure. A total that goes down is taken as a restart
|
|
148
|
+
* of the count.
|
|
149
|
+
*/
|
|
150
|
+
busyMs?: number;
|
|
151
|
+
};
|
|
152
|
+
/**
|
|
153
|
+
* `failed`: the kernel gave up on the isolate (past its restart limit, M3j).
|
|
154
|
+
* The balancer never moves or restarts it; after `kernel.restart(id)` it's
|
|
155
|
+
* placed again as new.
|
|
156
|
+
*/
|
|
157
|
+
type PlacementState = "placed" | "quarantined" | "kept-apart" | "moving" | "failed";
|
|
158
|
+
type Placement = {
|
|
159
|
+
id: string;
|
|
160
|
+
pool: string;
|
|
161
|
+
state: PlacementState;
|
|
162
|
+
/**
|
|
163
|
+
* Measured share of one core, 0–100: the larger of the beats' `busyMs` and
|
|
164
|
+
* the app's reported `busyMs` over the last 3 s; `undefined` until measured.
|
|
165
|
+
*/
|
|
166
|
+
loadPct: number | undefined;
|
|
167
|
+
/** The share of one core from the app's `busyMs` reports alone, while they're fresh (M3j). */
|
|
168
|
+
reportedPct?: number;
|
|
169
|
+
/** No pool had room and the worker cap was reached: placed anyway, or left where it is. */
|
|
170
|
+
atCap?: true;
|
|
171
|
+
/** The latest backlog `report`, while it's fresh (under 10 s old). */
|
|
172
|
+
backlog?: number;
|
|
173
|
+
/**
|
|
174
|
+
* Its backlog is rising, or its pool needs a bisect, and the balancer has
|
|
175
|
+
* nowhere to give it (`behind`).
|
|
176
|
+
*/
|
|
177
|
+
behind?: true;
|
|
178
|
+
/**
|
|
179
|
+
* In one of the balancer's pools, but not placed by it. The host reserves
|
|
180
|
+
* `balancer-*` pools, so this shouldn't happen; it is never moved.
|
|
181
|
+
*/
|
|
182
|
+
foreign?: true;
|
|
183
|
+
};
|
|
184
|
+
/**
|
|
185
|
+
* Why a member moved: `spike` (a second long task within 60 s), `sustained`
|
|
186
|
+
* (the pool's lag stayed high), `full` (the pool's members' busy time reached
|
|
187
|
+
* 80 % of a core, whatever its lag), `crash`, `bisect` (looking for a pool's
|
|
188
|
+
* unnamed culprit), `steady` (back out of quarantine), `backlog` (its
|
|
189
|
+
* reported backlog kept rising, M3i) or `pack` (a quiet member joining
|
|
190
|
+
* another pool to free a worker, M3i).
|
|
191
|
+
*/
|
|
192
|
+
type MoveReason = "spike" | "sustained" | "full" | "crash" | "bisect" | "steady" | "backlog" | "pack";
|
|
193
|
+
/** A move, once the old copy has stopped and the new one is spawned. */
|
|
194
|
+
type MoveEvent = {
|
|
195
|
+
id: string;
|
|
196
|
+
from: string;
|
|
197
|
+
to: string;
|
|
198
|
+
reason: MoveReason;
|
|
199
|
+
};
|
|
200
|
+
/**
|
|
201
|
+
* A moved member's first beat in its new pool (M3i). `costMs` is the time from
|
|
202
|
+
* the balancer stopping it to that beat: what the move cost the isolate. It
|
|
203
|
+
* follows `move`, which fires before the cost is known; a member stopped or
|
|
204
|
+
* moved again before it beats never arrives.
|
|
205
|
+
*/
|
|
206
|
+
type ArrivedEvent = MoveEvent & {
|
|
207
|
+
costMs: number;
|
|
208
|
+
};
|
|
209
|
+
/**
|
|
210
|
+
* A spike put on a member (M3i): `noted` for a first one (nothing moves), or
|
|
211
|
+
* `quarantined` when it sends the member to quarantine, or restarts the quiet
|
|
212
|
+
* window of one already there.
|
|
213
|
+
*/
|
|
214
|
+
type SpikeEvent = {
|
|
215
|
+
id: string;
|
|
216
|
+
pool: string;
|
|
217
|
+
lagMs: number;
|
|
218
|
+
action: "noted" | "quarantined";
|
|
219
|
+
};
|
|
220
|
+
/**
|
|
221
|
+
* A member's reported backlog keeps rising, but it already has its pool to
|
|
222
|
+
* itself or the balancer has no worker or pool to give it (M3i). Also when
|
|
223
|
+
* its pool's trouble needs a bisect and no quarantine pool is free for one
|
|
224
|
+
* (M3j). Emitted when it first happens, then at most every 30 s while it
|
|
225
|
+
* lasts. `backlog` is the latest fresh report, or 0.
|
|
226
|
+
*/
|
|
227
|
+
type BehindEvent = {
|
|
228
|
+
id: string;
|
|
229
|
+
pool: string;
|
|
230
|
+
backlog: number;
|
|
231
|
+
};
|
|
232
|
+
/**
|
|
233
|
+
* The kernel gave up on a member (`isolate:failed`, M3j). It drops out of its
|
|
234
|
+
* pool's load and any bisect, and no move restarts it.
|
|
235
|
+
*/
|
|
236
|
+
type FailedEvent = {
|
|
237
|
+
id: string;
|
|
238
|
+
pool: string;
|
|
239
|
+
reason: string;
|
|
240
|
+
};
|
|
241
|
+
/** An isolate the balancer didn't place turned up in one of its pools (see `Placement.foreign`). */
|
|
242
|
+
type ForeignEvent = {
|
|
243
|
+
id: string;
|
|
244
|
+
pool: string;
|
|
245
|
+
};
|
|
246
|
+
type BalancerEvents = {
|
|
247
|
+
move: MoveEvent;
|
|
248
|
+
arrived: ArrivedEvent;
|
|
249
|
+
spike: SpikeEvent;
|
|
250
|
+
behind: BehindEvent;
|
|
251
|
+
failed: FailedEvent;
|
|
252
|
+
foreign: ForeignEvent;
|
|
253
|
+
};
|
|
254
|
+
/**
|
|
255
|
+
* Pools are named `balancer-1`, `balancer-2`…; the quarantine is
|
|
256
|
+
* `balancer-quarantine-1`, and `-2`, `-3`… when a suspect needs a worker of
|
|
257
|
+
* its own. `createBalancer` throws `balancer_exists` if the kernel has one.
|
|
258
|
+
*/
|
|
259
|
+
interface Balancer<M extends ManifestLike = ManifestLike> {
|
|
260
|
+
/** Like `kernel.spawn`, but the balancer picks `pool`; `manifest.pool` must be absent. */
|
|
261
|
+
spawn(manifest: M): Placement;
|
|
262
|
+
stop(id: string): Promise<void>;
|
|
263
|
+
placements(): Placement[];
|
|
264
|
+
/**
|
|
265
|
+
* `id`'s load over the last 3 s, as a share of one core (0–100), whether
|
|
266
|
+
* the balancer placed it or not: a preview's, for `loads.set` on Save.
|
|
267
|
+
* `undefined` until 3 s of beats have been seen.
|
|
268
|
+
*/
|
|
269
|
+
measure(id: string): number | undefined;
|
|
270
|
+
/** Leaves `id` where it is until the returned function is called (a widget open for editing). */
|
|
271
|
+
hold(id: string): () => void;
|
|
272
|
+
/**
|
|
273
|
+
* The app's view of `id`'s queued work (M3i) and of its own busy time (M3j).
|
|
274
|
+
* A backlog higher at the end of a 10 s window than at its start, and above
|
|
275
|
+
* 0, moves the isolate somewhere with more CPU. A reported `busyMs` total
|
|
276
|
+
* counts toward its load wherever the balancer reads one. A report older
|
|
277
|
+
* than 10 s counts as none, so report at least every few seconds. Throws a
|
|
278
|
+
* `TypeError` for a report with neither field, or a field that isn't a
|
|
279
|
+
* finite number of 0 or more. Ignored for isolates the balancer didn't place
|
|
280
|
+
* and ones the kernel gave up on.
|
|
281
|
+
*/
|
|
282
|
+
report(id: string, report: Report): void;
|
|
283
|
+
on<E extends keyof BalancerEvents>(event: E, fn: (e: BalancerEvents[E]) => void): () => void;
|
|
284
|
+
/**
|
|
285
|
+
* Stops watching and gives back the kernel's claim, so another balancer
|
|
286
|
+
* can be created on it; isolates keep running where they are. `spawn`
|
|
287
|
+
* throws afterwards.
|
|
288
|
+
*/
|
|
289
|
+
dispose(): void;
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Automatic placement for isolates spawned through it (M3h, ADR 0017).
|
|
293
|
+
*
|
|
294
|
+
* @experimental The policy and its options may change before 1.0.
|
|
295
|
+
*
|
|
296
|
+
* It decides on the main thread, from every heartbeat (`isolate:beat`), and moves an
|
|
297
|
+
* isolate by stopping it and spawning it again with another `pool`; the
|
|
298
|
+
* kernel itself never pools on its own (ADR 0001). Isolates spawned with
|
|
299
|
+
* `kernel.spawn` are never touched, and don't count toward `maxWorkers`.
|
|
300
|
+
*/
|
|
301
|
+
export declare function createBalancer<M extends ManifestLike>(kernel: KernelLike<M>, opts?: BalancerOptions): Balancer<M>;
|
|
302
|
+
export type { ArrivedEvent, Balancer, BalancerEvents, BalancerOptions, BeatLike, BehindEvent, CrashLike, FailedEvent, FailedLike, ForeignEvent, KernelLike, LoadStore, ManifestLike, MoveEvent, MoveReason, Placement, PlacementState, PoolLike, Report, SpikeEvent };
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,961 @@
|
|
|
1
|
+
import { claimPools as claimPools$1 } from "@isoloop/host";
|
|
2
|
+
// src/claim.ts
|
|
3
|
+
/**
|
|
4
|
+
* The host's claim, for a kernel from `createKernel`. A `KernelLike` that
|
|
5
|
+
* offers `claimPools` itself (a simulated kernel in tests) is asked
|
|
6
|
+
* directly instead.
|
|
7
|
+
*/
|
|
8
|
+
const claimPools = (kernel, owner) => {
|
|
9
|
+
if (kernel.claimPools) {
|
|
10
|
+
const own = kernel.claimPools(owner);
|
|
11
|
+
return {
|
|
12
|
+
spawn: (manifest) => own.spawn(manifest),
|
|
13
|
+
release: () => own.release?.()
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
const claim = claimPools$1(kernel, owner);
|
|
17
|
+
return {
|
|
18
|
+
spawn: (manifest) => claim.spawn(manifest),
|
|
19
|
+
release: () => claim.release()
|
|
20
|
+
};
|
|
21
|
+
};
|
|
22
|
+
// src/balancer.ts
|
|
23
|
+
/**
|
|
24
|
+
* A pool's beat reaches the balancer as one `isolate:beat` per member, all
|
|
25
|
+
* from the same worker message: events this close together are one beat.
|
|
26
|
+
* Beats are 100 ms apart by default.
|
|
27
|
+
*/
|
|
28
|
+
const BEAT_GAP_MS = 10;
|
|
29
|
+
/** How often it decides. */
|
|
30
|
+
const DECIDE_MS = 500;
|
|
31
|
+
/** How often a changed load is written to the `loads` store. */
|
|
32
|
+
const SAVE_MS = 1e4;
|
|
33
|
+
/** A load is written again only once it moved by this much. */
|
|
34
|
+
const SAVE_DELTA_PCT = 5;
|
|
35
|
+
/** Pool names: `balancer-1`, `balancer-2`…, reserved by the host for the balancer. */
|
|
36
|
+
const POOLS = "balancer";
|
|
37
|
+
/** The quarantine, and `-2`, `-3`… for suspects given a worker of their own. */
|
|
38
|
+
const QUARANTINES = "balancer-quarantine";
|
|
39
|
+
const QUARANTINE = `${QUARANTINES}-1`;
|
|
40
|
+
const now = () => performance.now();
|
|
41
|
+
/** A report's figure: a finite number, 0 or more. */
|
|
42
|
+
const valid = (x) => typeof x === "number" && Number.isFinite(x) && x >= 0;
|
|
43
|
+
/**
|
|
44
|
+
* `busyMs` is approximate: shared framework work is split between members and
|
|
45
|
+
* promise continuations go unattributed. When the members' busy time explains
|
|
46
|
+
* less than this share of the interval, the lag isn't pinned on the busiest
|
|
47
|
+
* member; the balancer bisects instead, as on a kill.
|
|
48
|
+
*/
|
|
49
|
+
const ATTRIBUTED_PCT = 50;
|
|
50
|
+
/**
|
|
51
|
+
* A pool whose members' busy time adds up to this share of a core over the
|
|
52
|
+
* window is full (Q15), whatever its lag: work that yields often keeps the lag
|
|
53
|
+
* low while the members starve each other. Placement stops adding at
|
|
54
|
+
* `headroom` (60 by default); moving starts here, and the gap is the
|
|
55
|
+
* hysteresis.
|
|
56
|
+
*/
|
|
57
|
+
const FULL_PCT = 80;
|
|
58
|
+
/** A second spike from the same member (or unexplained one in the same pool) within this quarantines (M3i). */
|
|
59
|
+
const SPIKE_REPEAT_MS = 6e4;
|
|
60
|
+
/**
|
|
61
|
+
* Backlog (M3i): a reported backlog higher at the end of this window than at
|
|
62
|
+
* its start, and above 0, is rising. After a backlog move, the next one waits
|
|
63
|
+
* for a whole fresh window: that's the cooldown.
|
|
64
|
+
*/
|
|
65
|
+
const BACKLOG_WINDOW_MS = 1e4;
|
|
66
|
+
/** A report older than this is no signal. */
|
|
67
|
+
const STALE_MS = 1e4;
|
|
68
|
+
/** `behind` again at most this often while it lasts. */
|
|
69
|
+
const BEHIND_REPEAT_MS = 3e4;
|
|
70
|
+
/** Moves for load (not spike, crash or bisect) per member per window (M3i). */
|
|
71
|
+
const MOVE_BUDGET = 3;
|
|
72
|
+
const MOVE_BUDGET_MS = 3e5;
|
|
73
|
+
/** Packing back (M3i): quiet this long, into a pool that stays under `PACK_PCT` with it, one every `PACK_EVERY_MS`. */
|
|
74
|
+
const PACK_STEADY_MS = 6e4;
|
|
75
|
+
const PACK_PCT = 40;
|
|
76
|
+
const PACK_EVERY_MS = 6e4;
|
|
77
|
+
/** An offender's quiet window doubles with each offence up to this, or `steadyMs` if longer (M3j). */
|
|
78
|
+
const QUIET_CAP_MS = 12e4;
|
|
79
|
+
/** Worker crash reasons whose culprit the kernel can't name: every member of the pool crashes. */
|
|
80
|
+
const POOL_KILL = /* @__PURE__ */ new Set(["unresponsive", "worker-error"]);
|
|
81
|
+
const defaultMaxWorkers = () => {
|
|
82
|
+
const cores = Reflect.get(globalThis, "navigator")?.hardwareConcurrency ?? 4;
|
|
83
|
+
return Math.max(2, cores - 2);
|
|
84
|
+
};
|
|
85
|
+
const median = (xs) => {
|
|
86
|
+
const s = xs.toSorted((a, b) => a - b);
|
|
87
|
+
const mid = s.length >> 1;
|
|
88
|
+
return s.length % 2 === 1 ? s[mid] ?? 0 : ((s[mid - 1] ?? 0) + (s[mid] ?? 0)) / 2;
|
|
89
|
+
};
|
|
90
|
+
const isQuarantine = (pool) => pool.startsWith(`${QUARANTINES}-`);
|
|
91
|
+
const isOurs = (pool) => pool.startsWith(`${POOLS}-`);
|
|
92
|
+
/** Load as a share of one core over `beats`, if they cover at least 90 % of `windowMs`. */
|
|
93
|
+
const average = (beats, windowMs, t) => {
|
|
94
|
+
const recent = beats.filter((b) => b.at > t - windowMs);
|
|
95
|
+
const dur = recent.reduce((s, b) => s + b.dur, 0);
|
|
96
|
+
if (dur < windowMs * .9) return void 0;
|
|
97
|
+
const busy = recent.reduce((s, b) => s + b.busy, 0);
|
|
98
|
+
return Math.min(100, busy / dur * 100);
|
|
99
|
+
};
|
|
100
|
+
/**
|
|
101
|
+
* Automatic placement for isolates spawned through it (M3h, ADR 0017).
|
|
102
|
+
*
|
|
103
|
+
* @experimental The policy and its options may change before 1.0.
|
|
104
|
+
*
|
|
105
|
+
* It decides on the main thread, from every heartbeat (`isolate:beat`), and moves an
|
|
106
|
+
* isolate by stopping it and spawning it again with another `pool`; the
|
|
107
|
+
* kernel itself never pools on its own (ADR 0001). Isolates spawned with
|
|
108
|
+
* `kernel.spawn` are never touched, and don't count toward `maxWorkers`.
|
|
109
|
+
*/
|
|
110
|
+
function createBalancer(kernel, opts = {}) {
|
|
111
|
+
const claim = claimPools(kernel, "balancer");
|
|
112
|
+
const maxWorkers = Math.max(1, opts.maxWorkers ?? defaultMaxWorkers());
|
|
113
|
+
const spikeLag = opts.spike?.lagMs ?? 100;
|
|
114
|
+
const sustainedLag = opts.sustained?.lagMs ?? 50;
|
|
115
|
+
const windowMs = opts.sustained?.forMs ?? 3e3;
|
|
116
|
+
const headroom = opts.headroom?.loadPct ?? 60;
|
|
117
|
+
const steadyMs = opts.quarantine?.steadyMs ?? 3e4;
|
|
118
|
+
const maxReturns = opts.quarantine?.maxReturns ?? 3;
|
|
119
|
+
const loads = opts.loads;
|
|
120
|
+
const packing = opts.pack?.enabled ?? true;
|
|
121
|
+
const members = /* @__PURE__ */ new Map();
|
|
122
|
+
const pools = /* @__PURE__ */ new Map();
|
|
123
|
+
const listeners = {
|
|
124
|
+
move: /* @__PURE__ */ new Set(),
|
|
125
|
+
arrived: /* @__PURE__ */ new Set(),
|
|
126
|
+
spike: /* @__PURE__ */ new Set(),
|
|
127
|
+
behind: /* @__PURE__ */ new Set(),
|
|
128
|
+
failed: /* @__PURE__ */ new Set(),
|
|
129
|
+
foreign: /* @__PURE__ */ new Set()
|
|
130
|
+
};
|
|
131
|
+
/** Beats of isolates the balancer didn't place, for `measure` (previews included). */
|
|
132
|
+
const seen = /* @__PURE__ */ new Map();
|
|
133
|
+
/** Isolates in our pools we didn't place, by id: their pool. */
|
|
134
|
+
const foreign = /* @__PURE__ */ new Map();
|
|
135
|
+
/** Pools the kernel killed since the last decision (an unnamed culprit). */
|
|
136
|
+
const killed = /* @__PURE__ */ new Set();
|
|
137
|
+
/** An unfinished search for the member that kills its pool (Q20). */
|
|
138
|
+
/**
|
|
139
|
+
* `suspects` narrows with each step; `moved` is everyone it has moved so
|
|
140
|
+
* far, kept in quarantine until it ends so a culprit that only shows with
|
|
141
|
+
* company still meets the members it was moved with.
|
|
142
|
+
*/
|
|
143
|
+
let bisect;
|
|
144
|
+
/** Pools whose latest beat spiked, judged once all its members' events are in. */
|
|
145
|
+
const spiked = /* @__PURE__ */ new Set();
|
|
146
|
+
let judging;
|
|
147
|
+
let lastSave = -Infinity;
|
|
148
|
+
/** The last backlog move: rising is judged only on reports after it. */
|
|
149
|
+
let lastBacklogMoveAt = -Infinity;
|
|
150
|
+
let lastPackAt = -Infinity;
|
|
151
|
+
let disposed = false;
|
|
152
|
+
const pool = (name) => {
|
|
153
|
+
let p = pools.get(name);
|
|
154
|
+
if (!p) {
|
|
155
|
+
p = {
|
|
156
|
+
lags: [],
|
|
157
|
+
beatAt: void 0,
|
|
158
|
+
beat: [],
|
|
159
|
+
since: now(),
|
|
160
|
+
watch: false,
|
|
161
|
+
troubleAt: -Infinity,
|
|
162
|
+
spikeNotedAt: -Infinity
|
|
163
|
+
};
|
|
164
|
+
pools.set(name, p);
|
|
165
|
+
}
|
|
166
|
+
return p;
|
|
167
|
+
};
|
|
168
|
+
/** Running in its pool: not mid-move, and not given up on by the kernel. */
|
|
169
|
+
const live = (m) => m.state !== "moving" && m.state !== "failed";
|
|
170
|
+
const membersOf = (name) => [...members.values()].filter((m) => m.pool === name && live(m));
|
|
171
|
+
const changed = (name) => {
|
|
172
|
+
const p = pool(name);
|
|
173
|
+
p.lags = [];
|
|
174
|
+
p.beatAt = void 0;
|
|
175
|
+
p.beat = [];
|
|
176
|
+
p.since = now();
|
|
177
|
+
};
|
|
178
|
+
/**
|
|
179
|
+
* The app's reported busy time (M3j) as a share of one core: the running
|
|
180
|
+
* total's difference over about a window, from a fresh report back to one
|
|
181
|
+
* taken at least 90 % of a window before it.
|
|
182
|
+
*/
|
|
183
|
+
const reportedOf = (m, t) => {
|
|
184
|
+
const latest = m.busyReports.at(-1);
|
|
185
|
+
if (!latest || t - latest.at > STALE_MS) return void 0;
|
|
186
|
+
const start = m.busyReports.findLast((r) => r.at <= latest.at - windowMs * .9);
|
|
187
|
+
if (!start) return void 0;
|
|
188
|
+
const ms = latest.at - start.at;
|
|
189
|
+
return Math.min(100, (latest.total - start.total) / ms * 100);
|
|
190
|
+
};
|
|
191
|
+
/** The beats' and the report's measures over the last window, whichever is larger. */
|
|
192
|
+
const measured = (m, t) => {
|
|
193
|
+
const beat = average(m.beats, windowMs, t);
|
|
194
|
+
const reported = reportedOf(m, t);
|
|
195
|
+
if (beat === void 0) return reported;
|
|
196
|
+
return reported === void 0 ? beat : Math.max(beat, reported);
|
|
197
|
+
};
|
|
198
|
+
/** Measured load over the last window, or the last one known. */
|
|
199
|
+
const loadOf = (m) => measured(m, now()) ?? m.knownLoad;
|
|
200
|
+
const poolLoad = (name) => membersOf(name).reduce((s, m) => s + (loadOf(m) ?? 0), 0);
|
|
201
|
+
const windowFull = (name) => {
|
|
202
|
+
const p = pool(name);
|
|
203
|
+
const last = p.lags.at(-1);
|
|
204
|
+
return last !== void 0 && last.at - p.since >= windowMs;
|
|
205
|
+
};
|
|
206
|
+
/**
|
|
207
|
+
* Workers counted against `maxWorkers`: the balancer's pools and any
|
|
208
|
+
* quarantine worker beyond the first, which is allowed on top. Workers the
|
|
209
|
+
* app runs itself don't count (ADR 0017, amended).
|
|
210
|
+
*/
|
|
211
|
+
const ownWorkers = () => kernel.pools().filter((p) => p.running && isOurs(p.name) && p.name !== QUARANTINE).length;
|
|
212
|
+
const ownPools = () => [...new Set([...members.values()].filter((m) => m.state !== "failed").map((m) => m.pool))].filter((n) => !isQuarantine(n));
|
|
213
|
+
/** Room for one more worker of the balancer's own. */
|
|
214
|
+
const roomForPool = () => ownWorkers() + 1 <= maxWorkers;
|
|
215
|
+
const freshName = (prefix, first) => {
|
|
216
|
+
const taken = new Set(kernel.pools().map((p) => p.name));
|
|
217
|
+
for (const m of members.values()) taken.add(m.pool);
|
|
218
|
+
for (let n = first;; n++) {
|
|
219
|
+
const name = `${prefix}-${n}`;
|
|
220
|
+
if (!taken.has(name)) return name;
|
|
221
|
+
}
|
|
222
|
+
};
|
|
223
|
+
const recentlySpiked = (name) => membersOf(name).some((m) => now() - m.lastSpikeAt < steadyMs);
|
|
224
|
+
/**
|
|
225
|
+
* Where an isolate with `load` goes (Q7′): the auto pool with the most room
|
|
226
|
+
* that stays under `headroom` with it, else a new pool if the cap allows,
|
|
227
|
+
* else the least-loaded auto pool, flagged at the cap. `undefined` when
|
|
228
|
+
* there's nowhere better than `except`.
|
|
229
|
+
*/
|
|
230
|
+
const target = (load, except) => {
|
|
231
|
+
const best = ownPools().filter((n) => n !== except && !recentlySpiked(n)).map((n) => ({
|
|
232
|
+
n,
|
|
233
|
+
load: poolLoad(n)
|
|
234
|
+
})).filter((c) => c.load + load < headroom).toSorted((a, b) => a.load - b.load)[0];
|
|
235
|
+
if (best) return {
|
|
236
|
+
pool: best.n,
|
|
237
|
+
atCap: false
|
|
238
|
+
};
|
|
239
|
+
if (ownPools().length === 0 || roomForPool()) return {
|
|
240
|
+
pool: freshName(POOLS, 1),
|
|
241
|
+
atCap: false
|
|
242
|
+
};
|
|
243
|
+
if (except !== void 0) return void 0;
|
|
244
|
+
const least = ownPools().map((n) => ({
|
|
245
|
+
n,
|
|
246
|
+
load: poolLoad(n)
|
|
247
|
+
})).toSorted((a, b) => a.load - b.load)[0];
|
|
248
|
+
return least ? {
|
|
249
|
+
pool: least.n,
|
|
250
|
+
atCap: true
|
|
251
|
+
} : {
|
|
252
|
+
pool: freshName(POOLS, 1),
|
|
253
|
+
atCap: true
|
|
254
|
+
};
|
|
255
|
+
};
|
|
256
|
+
const emit = (event, e) => {
|
|
257
|
+
for (const fn of listeners[event]) fn(e);
|
|
258
|
+
};
|
|
259
|
+
/** Moves in the last `MOVE_BUDGET_MS`. */
|
|
260
|
+
const inBudget = (m) => {
|
|
261
|
+
const t = now();
|
|
262
|
+
m.moveTimes = m.moveTimes.filter((at) => t - at < MOVE_BUDGET_MS);
|
|
263
|
+
return m.moveTimes.length < MOVE_BUDGET;
|
|
264
|
+
};
|
|
265
|
+
const save = (m) => {
|
|
266
|
+
const load = loadOf(m);
|
|
267
|
+
if (!loads || load === void 0) return;
|
|
268
|
+
if (m.savedLoad !== void 0 && Math.abs(load - m.savedLoad) < SAVE_DELTA_PCT) return;
|
|
269
|
+
loads.set(m.manifest.id, load);
|
|
270
|
+
m.savedLoad = load;
|
|
271
|
+
};
|
|
272
|
+
const move = (m, to, reason, state) => {
|
|
273
|
+
const from = m.pool;
|
|
274
|
+
const id = m.manifest.id;
|
|
275
|
+
const startedAt = now();
|
|
276
|
+
m.knownLoad = loadOf(m) ?? m.knownLoad;
|
|
277
|
+
m.state = "moving";
|
|
278
|
+
m.beats = [];
|
|
279
|
+
m.lastBeatAt = void 0;
|
|
280
|
+
m.moveTimes.push(startedAt);
|
|
281
|
+
m.backlogs = [];
|
|
282
|
+
m.backlogHistory = [];
|
|
283
|
+
m.busyReports = [];
|
|
284
|
+
m.behind = false;
|
|
285
|
+
delete m.arrival;
|
|
286
|
+
m.pool = to;
|
|
287
|
+
changed(from);
|
|
288
|
+
pool(from).watch = true;
|
|
289
|
+
let stopped;
|
|
290
|
+
try {
|
|
291
|
+
stopped = kernel.stop(id);
|
|
292
|
+
} catch (error) {
|
|
293
|
+
stopped = Promise.reject(error instanceof Error ? error : new Error(String(error)));
|
|
294
|
+
}
|
|
295
|
+
m.stopping = stopped.then(() => {
|
|
296
|
+
if (disposed || members.get(id) !== m) return;
|
|
297
|
+
changed(to);
|
|
298
|
+
claim.spawn({
|
|
299
|
+
...m.manifest,
|
|
300
|
+
pool: to
|
|
301
|
+
});
|
|
302
|
+
m.state = state;
|
|
303
|
+
m.arrivedAt = now();
|
|
304
|
+
m.arrival = {
|
|
305
|
+
from,
|
|
306
|
+
to,
|
|
307
|
+
reason,
|
|
308
|
+
at: startedAt
|
|
309
|
+
};
|
|
310
|
+
delete m.stopping;
|
|
311
|
+
emit("move", {
|
|
312
|
+
id,
|
|
313
|
+
from,
|
|
314
|
+
to,
|
|
315
|
+
reason
|
|
316
|
+
});
|
|
317
|
+
}, () => {
|
|
318
|
+
if (members.get(id) === m) members.delete(id);
|
|
319
|
+
});
|
|
320
|
+
};
|
|
321
|
+
const stateFor = (m) => m.offences > maxReturns ? "kept-apart" : "quarantined";
|
|
322
|
+
/** Sends `m` to quarantine for misbehaving (`offence`) or to bisect. */
|
|
323
|
+
const quarantine = (m, reason, offence) => {
|
|
324
|
+
m.lastSpikeAt = now();
|
|
325
|
+
if (offence) m.offences++;
|
|
326
|
+
const state = stateFor(m);
|
|
327
|
+
m.quarantinedAt = now();
|
|
328
|
+
if (isQuarantine(m.pool)) {
|
|
329
|
+
m.state = state;
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
move(m, QUARANTINE, reason, state);
|
|
333
|
+
};
|
|
334
|
+
const movable = (m) => live(m) && m.holds === 0;
|
|
335
|
+
/**
|
|
336
|
+
* A member the kernel gave up on beats again: the app restarted it
|
|
337
|
+
* (`kernel.restart`). It's placed again as new (M3j): no offences, no
|
|
338
|
+
* history, and its pool's rules from here.
|
|
339
|
+
*/
|
|
340
|
+
const revive = (m) => {
|
|
341
|
+
m.state = isQuarantine(m.pool) ? "quarantined" : "placed";
|
|
342
|
+
m.offences = 0;
|
|
343
|
+
m.quarantinedAt = isQuarantine(m.pool) ? now() : -Infinity;
|
|
344
|
+
m.lastSpikeAt = m.spikeNotedAt = m.decayAt = m.risingAt = m.stuckAt = -Infinity;
|
|
345
|
+
m.arrivedAt = now();
|
|
346
|
+
m.moveTimes = [];
|
|
347
|
+
m.beats = [];
|
|
348
|
+
m.lastBeatAt = void 0;
|
|
349
|
+
m.backlogs = [];
|
|
350
|
+
m.backlogHistory = [];
|
|
351
|
+
m.busyReports = [];
|
|
352
|
+
changed(m.pool);
|
|
353
|
+
};
|
|
354
|
+
const trim = (beats, at) => {
|
|
355
|
+
const cut = at - windowMs;
|
|
356
|
+
while ((beats[0]?.at ?? Infinity) <= cut) beats.shift();
|
|
357
|
+
};
|
|
358
|
+
/** A beat of an isolate the balancer doesn't place, for `measure`. */
|
|
359
|
+
const watchOther = (e) => {
|
|
360
|
+
const w = seen.get(e.id);
|
|
361
|
+
if (!w) {
|
|
362
|
+
seen.set(e.id, {
|
|
363
|
+
lastBeatAt: e.at,
|
|
364
|
+
beats: []
|
|
365
|
+
});
|
|
366
|
+
return;
|
|
367
|
+
}
|
|
368
|
+
w.beats.push({
|
|
369
|
+
at: e.at,
|
|
370
|
+
busy: e.busyMs ?? 0,
|
|
371
|
+
dur: e.at - w.lastBeatAt
|
|
372
|
+
});
|
|
373
|
+
w.lastBeatAt = e.at;
|
|
374
|
+
trim(w.beats, e.at);
|
|
375
|
+
};
|
|
376
|
+
/**
|
|
377
|
+
* Every heartbeat, as the kernel reports it (`isolate:beat`). A pool's
|
|
378
|
+
* beat comes as one event per member; its lag is recorded once, and a
|
|
379
|
+
* spike is judged once all of them are in.
|
|
380
|
+
*/
|
|
381
|
+
const onBeat = (e) => {
|
|
382
|
+
if (disposed) return;
|
|
383
|
+
const m = members.get(e.id);
|
|
384
|
+
if (!m) {
|
|
385
|
+
watchOther(e);
|
|
386
|
+
return;
|
|
387
|
+
}
|
|
388
|
+
if (m.state === "moving" || e.pool !== m.pool) return;
|
|
389
|
+
if (m.state === "failed") revive(m);
|
|
390
|
+
if (m.arrival) {
|
|
391
|
+
const { from, to, reason, at } = m.arrival;
|
|
392
|
+
delete m.arrival;
|
|
393
|
+
emit("arrived", {
|
|
394
|
+
id: e.id,
|
|
395
|
+
from,
|
|
396
|
+
to,
|
|
397
|
+
reason,
|
|
398
|
+
costMs: e.at - at
|
|
399
|
+
});
|
|
400
|
+
}
|
|
401
|
+
if (m.lastBeatAt !== void 0) {
|
|
402
|
+
m.beats.push({
|
|
403
|
+
at: e.at,
|
|
404
|
+
busy: e.busyMs ?? 0,
|
|
405
|
+
dur: e.at - m.lastBeatAt
|
|
406
|
+
});
|
|
407
|
+
trim(m.beats, e.at);
|
|
408
|
+
}
|
|
409
|
+
m.lastBeatAt = e.at;
|
|
410
|
+
const p = pool(m.pool);
|
|
411
|
+
if (p.beatAt === void 0 || e.at - p.beatAt > BEAT_GAP_MS) {
|
|
412
|
+
p.beatAt = e.at;
|
|
413
|
+
p.beat = [];
|
|
414
|
+
p.lags.push({
|
|
415
|
+
at: e.at,
|
|
416
|
+
lagMs: e.lagMs
|
|
417
|
+
});
|
|
418
|
+
const cut = e.at - windowMs;
|
|
419
|
+
while ((p.lags[0]?.at ?? Infinity) <= cut) p.lags.shift();
|
|
420
|
+
if (e.lagMs > spikeLag) {
|
|
421
|
+
spiked.add(m.pool);
|
|
422
|
+
judging ??= setTimeout(judge, 0);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
p.beat.push(m);
|
|
426
|
+
};
|
|
427
|
+
const spike = (e) => emit("spike", e);
|
|
428
|
+
/**
|
|
429
|
+
* A spike (Q6′, M3i): put on the member that was busiest in that beat. A
|
|
430
|
+
* first one is only noted; a second within `SPIKE_REPEAT_MS` sends it to
|
|
431
|
+
* quarantine. One long task is often the data, not the isolate (a file that
|
|
432
|
+
* blocks whichever member gets it), and by the time its beat arrives it's
|
|
433
|
+
* over: moving the member then protects nobody.
|
|
434
|
+
*/
|
|
435
|
+
const judge = () => {
|
|
436
|
+
judging = void 0;
|
|
437
|
+
if (disposed) return;
|
|
438
|
+
const t = now();
|
|
439
|
+
for (const name of spiked) {
|
|
440
|
+
const p = pool(name);
|
|
441
|
+
const lagMs = p.lags.at(-1)?.lagMs ?? 0;
|
|
442
|
+
const beat = p.beat.filter((m) => members.get(m.manifest.id) === m && m.pool === name && m.state !== "moving");
|
|
443
|
+
const dur = Math.max(0, ...beat.map((m) => m.beats.at(-1)?.dur ?? 0));
|
|
444
|
+
const busy = beat.reduce((sum, m) => sum + (m.beats.at(-1)?.busy ?? 0), 0);
|
|
445
|
+
if (dur > 0 && busy / dur * 100 < ATTRIBUTED_PCT) {
|
|
446
|
+
if (bisect && beat.some((m) => bisect?.suspects.has(m.manifest.id)) || t - p.spikeNotedAt < SPIKE_REPEAT_MS) {
|
|
447
|
+
p.spikeNotedAt = -Infinity;
|
|
448
|
+
search(name, "spike");
|
|
449
|
+
} else p.spikeNotedAt = t;
|
|
450
|
+
continue;
|
|
451
|
+
}
|
|
452
|
+
const culprit = beat.filter((m) => m.beats.length > 0).toSorted((a, b) => (b.beats.at(-1)?.busy ?? 0) - (a.beats.at(-1)?.busy ?? 0))[0];
|
|
453
|
+
if (!culprit) continue;
|
|
454
|
+
const id = culprit.manifest.id;
|
|
455
|
+
const repeat = t - culprit.spikeNotedAt < SPIKE_REPEAT_MS;
|
|
456
|
+
if (isQuarantine(name) && culprit.offences > 0) {
|
|
457
|
+
culprit.lastSpikeAt = t;
|
|
458
|
+
spike({
|
|
459
|
+
id,
|
|
460
|
+
pool: name,
|
|
461
|
+
lagMs,
|
|
462
|
+
action: "quarantined"
|
|
463
|
+
});
|
|
464
|
+
continue;
|
|
465
|
+
}
|
|
466
|
+
const harmless = !isQuarantine(name) && membersOf(name).length < 2 && culprit.offences === 0;
|
|
467
|
+
if (!repeat || !movable(culprit) || harmless) {
|
|
468
|
+
culprit.spikeNotedAt = t;
|
|
469
|
+
spike({
|
|
470
|
+
id,
|
|
471
|
+
pool: name,
|
|
472
|
+
lagMs,
|
|
473
|
+
action: "noted"
|
|
474
|
+
});
|
|
475
|
+
continue;
|
|
476
|
+
}
|
|
477
|
+
culprit.spikeNotedAt = -Infinity;
|
|
478
|
+
quarantine(culprit, "spike", true);
|
|
479
|
+
spike({
|
|
480
|
+
id,
|
|
481
|
+
pool: name,
|
|
482
|
+
lagMs,
|
|
483
|
+
action: "quarantined"
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
spiked.clear();
|
|
487
|
+
};
|
|
488
|
+
/** Defensive: the host reserves our pools, so nobody else should be in them. */
|
|
489
|
+
const checkForeign = () => {
|
|
490
|
+
const present = /* @__PURE__ */ new Map();
|
|
491
|
+
for (const p of kernel.pools()) {
|
|
492
|
+
if (!isOurs(p.name)) continue;
|
|
493
|
+
for (const id of p.members) if (!members.has(id)) present.set(id, p.name);
|
|
494
|
+
}
|
|
495
|
+
for (const [id, name] of present) {
|
|
496
|
+
if (foreign.get(id) === name) continue;
|
|
497
|
+
foreign.set(id, name);
|
|
498
|
+
for (const fn of listeners.foreign) fn({
|
|
499
|
+
id,
|
|
500
|
+
pool: name
|
|
501
|
+
});
|
|
502
|
+
}
|
|
503
|
+
for (const id of foreign.keys()) if (!present.has(id)) foreign.delete(id);
|
|
504
|
+
};
|
|
505
|
+
/**
|
|
506
|
+
* A quarantine pool a bisect from `name` can have to itself (M3j): the
|
|
507
|
+
* first quarantine if it holds only members this bisect moved (or none), else a
|
|
508
|
+
* new quarantine pool if the cap allows. Never one that holds offenders.
|
|
509
|
+
*/
|
|
510
|
+
const bisectPool = (name) => {
|
|
511
|
+
const free = membersOf(QUARANTINE).every((o) => o.offences === 0 && bisect?.moved.has(o.manifest.id) === true);
|
|
512
|
+
if (!isQuarantine(name) && free) return QUARANTINE;
|
|
513
|
+
return roomForPool() ? freshName(QUARANTINES, 2) : void 0;
|
|
514
|
+
};
|
|
515
|
+
/**
|
|
516
|
+
* Finds the member behind a pool's trouble when the kernel or `busyMs`
|
|
517
|
+
* can't name it (Q20): the heavier half of the suspects moves, and
|
|
518
|
+
* whichever half acts up again holds the culprit. Only into a quarantine
|
|
519
|
+
* pool of its own (M3j); with none free, the pool stays as it is and its
|
|
520
|
+
* members are `behind`.
|
|
521
|
+
*/
|
|
522
|
+
const search = (name, reason) => {
|
|
523
|
+
const here = membersOf(name).filter((m) => m.holds === 0);
|
|
524
|
+
if (here.length === 0) return;
|
|
525
|
+
const ids = new Set(here.map((m) => m.manifest.id));
|
|
526
|
+
const prior = bisect && [...bisect.suspects].filter((id) => ids.has(id));
|
|
527
|
+
const suspects = prior && prior.length > 0 ? prior : [...ids];
|
|
528
|
+
if (suspects.length === 1) {
|
|
529
|
+
const culprit = members.get(suspects[0] ?? "");
|
|
530
|
+
bisect = void 0;
|
|
531
|
+
if (!culprit) return;
|
|
532
|
+
if (reason !== "crash" && membersOf(name).length === 1 && culprit.offences === 0) return;
|
|
533
|
+
if (membersOf(name).length > 1 && roomForPool()) {
|
|
534
|
+
culprit.offences++;
|
|
535
|
+
culprit.lastSpikeAt = culprit.quarantinedAt = now();
|
|
536
|
+
move(culprit, freshName(QUARANTINES, 2), reason, stateFor(culprit));
|
|
537
|
+
} else quarantine(culprit, reason, true);
|
|
538
|
+
return;
|
|
539
|
+
}
|
|
540
|
+
const sorted = suspects.map((id) => members.get(id)).filter((m) => m !== void 0).toSorted((a, b) => (loadOf(b) ?? 0) - (loadOf(a) ?? 0));
|
|
541
|
+
const to = bisectPool(name);
|
|
542
|
+
if (to === void 0) {
|
|
543
|
+
const t = now();
|
|
544
|
+
for (const m of here) {
|
|
545
|
+
m.stuckAt = t;
|
|
546
|
+
behind(m, t);
|
|
547
|
+
}
|
|
548
|
+
return;
|
|
549
|
+
}
|
|
550
|
+
const half = sorted.slice(0, Math.ceil(sorted.length / 2));
|
|
551
|
+
bisect = {
|
|
552
|
+
suspects: new Set(suspects),
|
|
553
|
+
moved: /* @__PURE__ */ new Set([...bisect?.moved ?? [], ...half.map((m) => m.manifest.id)]),
|
|
554
|
+
at: now()
|
|
555
|
+
};
|
|
556
|
+
for (const m of half) {
|
|
557
|
+
if (!isQuarantine(name)) m.quarantinedAt = now();
|
|
558
|
+
move(m, to, "bisect", stateFor(m));
|
|
559
|
+
}
|
|
560
|
+
};
|
|
561
|
+
/**
|
|
562
|
+
* The moves that would bring pool `name` back under (M3i): the heaviest
|
|
563
|
+
* candidates in turn, each to where the placement rule finds room given
|
|
564
|
+
* the moves before it, until the pool's load is predicted under the line,
|
|
565
|
+
* `FULL_PCT` for a full pool or `headroom` for a lagging one. Returns the
|
|
566
|
+
* first of them, or `undefined` when no such set of moves exists.
|
|
567
|
+
*/
|
|
568
|
+
const relief = (name, candidates, busy, lagging) => {
|
|
569
|
+
const line = lagging ? headroom : FULL_PCT;
|
|
570
|
+
const added = /* @__PURE__ */ new Map();
|
|
571
|
+
let fresh = 0;
|
|
572
|
+
let left = busy;
|
|
573
|
+
let first;
|
|
574
|
+
for (const m of candidates) {
|
|
575
|
+
if (left < line) break;
|
|
576
|
+
const load = loadOf(m) ?? 0;
|
|
577
|
+
const into = [.../* @__PURE__ */ new Set([...ownPools(), ...added.keys()])].filter((n) => n !== name && !recentlySpiked(n)).map((n) => ({
|
|
578
|
+
n,
|
|
579
|
+
load: poolLoad(n) + (added.get(n) ?? 0)
|
|
580
|
+
})).filter((c) => c.load + load < headroom).toSorted((a, b) => a.load - b.load)[0];
|
|
581
|
+
let to;
|
|
582
|
+
if (into) to = into.n;
|
|
583
|
+
else if (ownWorkers() + fresh + 1 <= maxWorkers) {
|
|
584
|
+
to = fresh === 0 ? freshName(POOLS, 1) : `${POOLS}-new-${fresh}`;
|
|
585
|
+
fresh++;
|
|
586
|
+
}
|
|
587
|
+
if (to === void 0) continue;
|
|
588
|
+
added.set(to, (added.get(to) ?? 0) + load);
|
|
589
|
+
left -= load;
|
|
590
|
+
first ??= {
|
|
591
|
+
member: m,
|
|
592
|
+
to
|
|
593
|
+
};
|
|
594
|
+
}
|
|
595
|
+
return left < line ? first : void 0;
|
|
596
|
+
};
|
|
597
|
+
/** The latest report, if fresh. */
|
|
598
|
+
const backlogNow = (m, t) => m.backlog && t - m.backlog.at <= STALE_MS ? m.backlog.value : void 0;
|
|
599
|
+
/**
|
|
600
|
+
* Rising (M3i): the latest fresh report is above 0 and higher than one taken
|
|
601
|
+
* at least a window before it, both since it last moved and since the last
|
|
602
|
+
* backlog move.
|
|
603
|
+
*/
|
|
604
|
+
const rising = (m, t) => {
|
|
605
|
+
const latest = m.backlog;
|
|
606
|
+
if (backlogNow(m, t) === void 0 || !latest || latest.value <= 0) return false;
|
|
607
|
+
const start = m.backlogs.findLast((b) => b.at >= lastBacklogMoveAt && b.at <= latest.at - BACKLOG_WINDOW_MS);
|
|
608
|
+
return start !== void 0 && latest.value > start.value;
|
|
609
|
+
};
|
|
610
|
+
/**
|
|
611
|
+
* Somewhere with more CPU for `m` (M3i): a pool of its own if the cap
|
|
612
|
+
* allows, else the balancer's pool with the least load that stays under
|
|
613
|
+
* `headroom` with it and has less load than its own pool-mates.
|
|
614
|
+
*/
|
|
615
|
+
const roomier = (m) => {
|
|
616
|
+
if (roomForPool()) return freshName(POOLS, 1);
|
|
617
|
+
const load = loadOf(m) ?? 0;
|
|
618
|
+
const mates = poolLoad(m.pool) - load;
|
|
619
|
+
return ownPools().filter((n) => n !== m.pool && !recentlySpiked(n)).map((n) => ({
|
|
620
|
+
n,
|
|
621
|
+
load: poolLoad(n)
|
|
622
|
+
})).filter((c) => c.load + load < headroom && c.load < mates).toSorted((a, b) => a.load - b.load)[0]?.n;
|
|
623
|
+
};
|
|
624
|
+
const behind = (m, t) => {
|
|
625
|
+
m.behind = true;
|
|
626
|
+
if (t - m.behindAt < BEHIND_REPEAT_MS) return;
|
|
627
|
+
m.behindAt = t;
|
|
628
|
+
const e = {
|
|
629
|
+
id: m.manifest.id,
|
|
630
|
+
pool: m.pool,
|
|
631
|
+
backlog: backlogNow(m, t) ?? 0
|
|
632
|
+
};
|
|
633
|
+
emit("behind", e);
|
|
634
|
+
};
|
|
635
|
+
/**
|
|
636
|
+
* Backlog (M3i): of the members whose backlog is rising, the one in the
|
|
637
|
+
* busiest pool (the heaviest there first) moves somewhere with more CPU;
|
|
638
|
+
* one move per fresh window across the balancer, so members of one
|
|
639
|
+
* consumer group move one at a time. One that can't get more CPU is
|
|
640
|
+
* `behind`.
|
|
641
|
+
*/
|
|
642
|
+
const backlogs = (t) => {
|
|
643
|
+
const order = [...members.values()].filter((m) => {
|
|
644
|
+
const r = m.state === "placed" && !isQuarantine(m.pool) && rising(m, t);
|
|
645
|
+
if (r) m.risingAt = t;
|
|
646
|
+
else if (t - m.stuckAt >= windowMs) m.behind = false;
|
|
647
|
+
return r;
|
|
648
|
+
}).toSorted((a, b) => poolLoad(b.pool) - poolLoad(a.pool) || (loadOf(b) ?? 0) - (loadOf(a) ?? 0));
|
|
649
|
+
for (const m of order) {
|
|
650
|
+
if (!movable(m) || !inBudget(m)) continue;
|
|
651
|
+
const to = membersOf(m.pool).length < 2 ? void 0 : roomier(m);
|
|
652
|
+
if (to === void 0) {
|
|
653
|
+
behind(m, t);
|
|
654
|
+
continue;
|
|
655
|
+
}
|
|
656
|
+
m.atCap = false;
|
|
657
|
+
lastBacklogMoveAt = t;
|
|
658
|
+
move(m, to, "backlog", "placed");
|
|
659
|
+
return;
|
|
660
|
+
}
|
|
661
|
+
};
|
|
662
|
+
/**
|
|
663
|
+
* Its backlog rose over the last `PACK_STEADY_MS` (M3j): the latest fresh
|
|
664
|
+
* report is higher than the one at or just before the window's start (or
|
|
665
|
+
* the oldest since it moved, when it hasn't reported that long). A
|
|
666
|
+
* queue that holds steady while work arrives doesn't count, nor does a
|
|
667
|
+
* stale report or none.
|
|
668
|
+
*/
|
|
669
|
+
const roseLately = (m, t) => {
|
|
670
|
+
const latest = backlogNow(m, t);
|
|
671
|
+
if (latest === void 0) return false;
|
|
672
|
+
const start = m.backlogHistory.findLast((b) => b.at <= t - PACK_STEADY_MS) ?? m.backlogHistory[0];
|
|
673
|
+
return start !== void 0 && latest > start.value;
|
|
674
|
+
};
|
|
675
|
+
/**
|
|
676
|
+
* Packing back (M3i): the lightest member alone in its pool and quiet for
|
|
677
|
+
* `PACK_STEADY_MS` joins the fullest other pool that stays under
|
|
678
|
+
* `PACK_PCT` with it, which frees its worker. Never one on hold, whose
|
|
679
|
+
* backlog rose over the window (M3j), in quarantine or kept apart, and
|
|
680
|
+
* never into a pool whose members' backlog rose.
|
|
681
|
+
*/
|
|
682
|
+
const pack = (t) => {
|
|
683
|
+
const quiet = (since) => t - since >= PACK_STEADY_MS;
|
|
684
|
+
const steady = (m) => quiet(Math.max(m.arrivedAt, m.lastSpikeAt, m.spikeNotedAt, pool(m.pool).troubleAt)) && !roseLately(m, t);
|
|
685
|
+
const alone = [...members.values()].filter((m) => m.state === "placed" && isOurs(m.pool) && !isQuarantine(m.pool) && movable(m) && inBudget(m) && membersOf(m.pool).length === 1 && loadOf(m) !== void 0 && steady(m)).toSorted((a, b) => (loadOf(a) ?? 0) - (loadOf(b) ?? 0));
|
|
686
|
+
for (const m of alone) {
|
|
687
|
+
const load = loadOf(m) ?? 0;
|
|
688
|
+
const into = ownPools().filter((n) => n !== m.pool && !recentlySpiked(n) && quiet(pool(n).troubleAt)).filter((n) => [...members.values()].filter((o) => o.pool === n).every((o) => o.state !== "moving" && !roseLately(o, t))).map((n) => ({
|
|
689
|
+
n,
|
|
690
|
+
load: poolLoad(n)
|
|
691
|
+
})).filter((c) => c.load + load < PACK_PCT).toSorted((a, b) => b.load - a.load)[0];
|
|
692
|
+
if (!into) continue;
|
|
693
|
+
lastPackAt = t;
|
|
694
|
+
move(m, into.n, "pack", "placed");
|
|
695
|
+
return;
|
|
696
|
+
}
|
|
697
|
+
};
|
|
698
|
+
/** The quiet window: doubled per offence, capped at 2 minutes (or `steadyMs`, if longer; M3j). */
|
|
699
|
+
const steadyFor = (m) => Math.min(Math.max(steadyMs, QUIET_CAP_MS), steadyMs * 2 ** Math.max(0, m.offences - 1));
|
|
700
|
+
/**
|
|
701
|
+
* Offences decay (M3j): an offender back in a balancer pool loses one for
|
|
702
|
+
* each quiet window it completes there, so one that stays quiet is back to
|
|
703
|
+
* the shortest window. One that offends again soon after leaving still
|
|
704
|
+
* gets a longer one, and `kept-apart` stays apart.
|
|
705
|
+
*/
|
|
706
|
+
const decay = (t) => {
|
|
707
|
+
for (const m of members.values()) {
|
|
708
|
+
if (m.state !== "placed" || m.offences === 0) continue;
|
|
709
|
+
if (t - Math.max(m.decayAt, m.lastSpikeAt, m.spikeNotedAt) < steadyFor(m)) continue;
|
|
710
|
+
m.offences--;
|
|
711
|
+
m.decayAt = t;
|
|
712
|
+
}
|
|
713
|
+
};
|
|
714
|
+
const decide = () => {
|
|
715
|
+
const t = now();
|
|
716
|
+
checkForeign();
|
|
717
|
+
for (const [id, w] of seen) if (t - w.lastBeatAt > windowMs * 2) seen.delete(id);
|
|
718
|
+
for (const name of killed) search(name, "crash");
|
|
719
|
+
killed.clear();
|
|
720
|
+
if (bisect && t - bisect.at > steadyMs * 2) bisect = void 0;
|
|
721
|
+
for (const name of new Set([...members.values()].map((m) => m.pool))) {
|
|
722
|
+
const here = membersOf(name);
|
|
723
|
+
if (!windowFull(name)) continue;
|
|
724
|
+
const p = pool(name);
|
|
725
|
+
const lagging = median(p.lags.map((l) => l.lagMs)) > sustainedLag;
|
|
726
|
+
const busy = here.reduce((sum, m) => sum + (loadOf(m) ?? 0), 0);
|
|
727
|
+
const full = !isQuarantine(name) && busy >= FULL_PCT;
|
|
728
|
+
if (!lagging && !full) {
|
|
729
|
+
p.watch = false;
|
|
730
|
+
continue;
|
|
731
|
+
}
|
|
732
|
+
p.troubleAt = t;
|
|
733
|
+
if (here.length < 2) continue;
|
|
734
|
+
if (lagging && !full && busy < ATTRIBUTED_PCT) {
|
|
735
|
+
search(name, "sustained");
|
|
736
|
+
continue;
|
|
737
|
+
}
|
|
738
|
+
const byLoad = here.toSorted((a, b) => (loadOf(b) ?? 0) - (loadOf(a) ?? 0));
|
|
739
|
+
const first = byLoad[0];
|
|
740
|
+
if (!first || !p.watch && !movable(first)) continue;
|
|
741
|
+
if (isQuarantine(name)) {
|
|
742
|
+
const heaviest = byLoad.find(movable);
|
|
743
|
+
if (heaviest && roomForPool()) move(heaviest, freshName(QUARANTINES, 2), "sustained", heaviest.state);
|
|
744
|
+
continue;
|
|
745
|
+
}
|
|
746
|
+
const reason = lagging ? "sustained" : "full";
|
|
747
|
+
const plan = relief(name, byLoad.filter(movable).filter(inBudget), busy, lagging);
|
|
748
|
+
if (plan) {
|
|
749
|
+
plan.member.atCap = false;
|
|
750
|
+
move(plan.member, plan.to, reason, "placed");
|
|
751
|
+
} else first.atCap = true;
|
|
752
|
+
}
|
|
753
|
+
backlogs(t);
|
|
754
|
+
decay(t);
|
|
755
|
+
for (const m of members.values()) {
|
|
756
|
+
if (m.state !== "quarantined" || !movable(m) || !inBudget(m)) continue;
|
|
757
|
+
const fresh = average(m.beats, windowMs, t);
|
|
758
|
+
if (m.offences === 0) {
|
|
759
|
+
if (fresh === void 0 || bisect?.moved.has(m.manifest.id)) continue;
|
|
760
|
+
} else if (t - Math.max(m.quarantinedAt, m.lastSpikeAt) < steadyFor(m)) continue;
|
|
761
|
+
const load = fresh ?? m.knownLoad;
|
|
762
|
+
if (load === void 0) continue;
|
|
763
|
+
const to = target(load, m.pool);
|
|
764
|
+
if (!to || to.atCap) {
|
|
765
|
+
m.atCap = true;
|
|
766
|
+
continue;
|
|
767
|
+
}
|
|
768
|
+
m.atCap = false;
|
|
769
|
+
m.decayAt = t;
|
|
770
|
+
move(m, to.pool, "steady", "placed");
|
|
771
|
+
}
|
|
772
|
+
if (packing && t - lastPackAt >= PACK_EVERY_MS) pack(t);
|
|
773
|
+
if (t - lastSave >= SAVE_MS) {
|
|
774
|
+
lastSave = t;
|
|
775
|
+
for (const m of members.values()) save(m);
|
|
776
|
+
}
|
|
777
|
+
};
|
|
778
|
+
const onCrash = (e) => {
|
|
779
|
+
const m = members.get(e.id);
|
|
780
|
+
if (!m || !live(m)) return;
|
|
781
|
+
if (POOL_KILL.has(e.reason) && e.pool !== void 0) killed.add(e.pool);
|
|
782
|
+
else if (e.reason === "error") {
|
|
783
|
+
if (m.holds > 0) m.lastSpikeAt = now();
|
|
784
|
+
else quarantine(m, "crash", true);
|
|
785
|
+
}
|
|
786
|
+
};
|
|
787
|
+
/**
|
|
788
|
+
* The kernel gave up on a member (M3j): it leaves its pool's load, the
|
|
789
|
+
* quarantine's count and any bisect, and stays where it is until the app
|
|
790
|
+
* restarts it. A member mid-move was stopped by the balancer itself; its
|
|
791
|
+
* respawn is a fresh start.
|
|
792
|
+
*/
|
|
793
|
+
const onFailed = (e) => {
|
|
794
|
+
const m = members.get(e.id);
|
|
795
|
+
if (!m || !live(m)) return;
|
|
796
|
+
m.state = "failed";
|
|
797
|
+
m.beats = [];
|
|
798
|
+
m.lastBeatAt = void 0;
|
|
799
|
+
m.atCap = false;
|
|
800
|
+
m.behind = false;
|
|
801
|
+
m.busyReports = [];
|
|
802
|
+
delete m.arrival;
|
|
803
|
+
bisect?.suspects.delete(e.id);
|
|
804
|
+
bisect?.moved.delete(e.id);
|
|
805
|
+
changed(m.pool);
|
|
806
|
+
emit("failed", {
|
|
807
|
+
id: e.id,
|
|
808
|
+
pool: m.pool,
|
|
809
|
+
reason: e.reason
|
|
810
|
+
});
|
|
811
|
+
};
|
|
812
|
+
const timer = setInterval(decide, DECIDE_MS);
|
|
813
|
+
const offs = [
|
|
814
|
+
kernel.on("isolate:crash", onCrash),
|
|
815
|
+
kernel.on("isolate:beat", onBeat),
|
|
816
|
+
kernel.on("isolate:failed", onFailed)
|
|
817
|
+
];
|
|
818
|
+
const placement = (m) => {
|
|
819
|
+
const p = {
|
|
820
|
+
id: m.manifest.id,
|
|
821
|
+
pool: m.pool,
|
|
822
|
+
state: m.state,
|
|
823
|
+
loadPct: m.state === "failed" ? void 0 : loadOf(m)
|
|
824
|
+
};
|
|
825
|
+
const reported = m.state === "failed" ? void 0 : reportedOf(m, now());
|
|
826
|
+
if (reported !== void 0) p.reportedPct = reported;
|
|
827
|
+
if (m.atCap) p.atCap = true;
|
|
828
|
+
const backlog = backlogNow(m, now());
|
|
829
|
+
if (backlog !== void 0) p.backlog = backlog;
|
|
830
|
+
if (m.behind) p.behind = true;
|
|
831
|
+
return p;
|
|
832
|
+
};
|
|
833
|
+
return {
|
|
834
|
+
spawn(manifest) {
|
|
835
|
+
if (disposed) throw new TypeError("balancer.spawn: this balancer was disposed");
|
|
836
|
+
if (manifest.pool !== void 0) throw new TypeError(`balancer.spawn: "${manifest.id}" names a pool; the balancer picks it`);
|
|
837
|
+
if (members.has(manifest.id)) throw new TypeError(`balancer.spawn: "${manifest.id}" is already running`);
|
|
838
|
+
const known = loads?.get(manifest.id);
|
|
839
|
+
const dest = (known === void 0 ? {
|
|
840
|
+
pool: QUARANTINE,
|
|
841
|
+
atCap: false
|
|
842
|
+
} : target(known)) ?? {
|
|
843
|
+
pool: QUARANTINE,
|
|
844
|
+
atCap: true
|
|
845
|
+
};
|
|
846
|
+
const m = {
|
|
847
|
+
manifest,
|
|
848
|
+
pool: dest.pool,
|
|
849
|
+
state: isQuarantine(dest.pool) ? "quarantined" : "placed",
|
|
850
|
+
beats: [],
|
|
851
|
+
lastBeatAt: void 0,
|
|
852
|
+
knownLoad: known,
|
|
853
|
+
savedLoad: known,
|
|
854
|
+
offences: 0,
|
|
855
|
+
quarantinedAt: isQuarantine(dest.pool) ? now() : -Infinity,
|
|
856
|
+
decayAt: -Infinity,
|
|
857
|
+
lastSpikeAt: -Infinity,
|
|
858
|
+
spikeNotedAt: -Infinity,
|
|
859
|
+
holds: 0,
|
|
860
|
+
atCap: dest.atCap,
|
|
861
|
+
arrivedAt: now(),
|
|
862
|
+
moveTimes: [],
|
|
863
|
+
backlogs: [],
|
|
864
|
+
backlog: void 0,
|
|
865
|
+
backlogHistory: [],
|
|
866
|
+
busyReports: [],
|
|
867
|
+
risingAt: -Infinity,
|
|
868
|
+
stuckAt: -Infinity,
|
|
869
|
+
behind: false,
|
|
870
|
+
behindAt: -Infinity
|
|
871
|
+
};
|
|
872
|
+
members.set(manifest.id, m);
|
|
873
|
+
changed(dest.pool);
|
|
874
|
+
claim.spawn({
|
|
875
|
+
...manifest,
|
|
876
|
+
pool: dest.pool
|
|
877
|
+
});
|
|
878
|
+
return placement(m);
|
|
879
|
+
},
|
|
880
|
+
async stop(id) {
|
|
881
|
+
const m = members.get(id);
|
|
882
|
+
if (!m) return kernel.stop(id);
|
|
883
|
+
save(m);
|
|
884
|
+
members.delete(id);
|
|
885
|
+
changed(m.pool);
|
|
886
|
+
if (m.state === "moving" && m.stopping) return m.stopping;
|
|
887
|
+
await kernel.stop(id);
|
|
888
|
+
},
|
|
889
|
+
placements() {
|
|
890
|
+
const ours = [...members.values()].map(placement);
|
|
891
|
+
const others = [...foreign].map(([id, name]) => ({
|
|
892
|
+
id,
|
|
893
|
+
pool: name,
|
|
894
|
+
state: "placed",
|
|
895
|
+
loadPct: average(seen.get(id)?.beats ?? [], windowMs, now()),
|
|
896
|
+
foreign: true
|
|
897
|
+
}));
|
|
898
|
+
return [...ours, ...others];
|
|
899
|
+
},
|
|
900
|
+
hold(id) {
|
|
901
|
+
const m = members.get(id);
|
|
902
|
+
if (!m) return () => {};
|
|
903
|
+
m.holds++;
|
|
904
|
+
let released = false;
|
|
905
|
+
return () => {
|
|
906
|
+
if (released) return;
|
|
907
|
+
released = true;
|
|
908
|
+
m.holds--;
|
|
909
|
+
};
|
|
910
|
+
},
|
|
911
|
+
report(id, { backlog, busyMs }) {
|
|
912
|
+
if (backlog === void 0 && busyMs === void 0) throw new TypeError("balancer.report: give a backlog, a busyMs or both");
|
|
913
|
+
if (backlog !== void 0 && !valid(backlog)) throw new TypeError("balancer.report: backlog must be a finite number, 0 or more");
|
|
914
|
+
if (busyMs !== void 0 && !valid(busyMs)) throw new TypeError("balancer.report: busyMs must be a finite number, 0 or more");
|
|
915
|
+
const m = members.get(id);
|
|
916
|
+
if (!m || m.state === "failed") return;
|
|
917
|
+
const t = now();
|
|
918
|
+
if (backlog !== void 0) {
|
|
919
|
+
const sample = {
|
|
920
|
+
at: t,
|
|
921
|
+
value: backlog
|
|
922
|
+
};
|
|
923
|
+
m.backlog = sample;
|
|
924
|
+
m.backlogs.push(sample);
|
|
925
|
+
m.backlogHistory.push(sample);
|
|
926
|
+
while ((m.backlogs[1]?.at ?? Infinity) <= t - BACKLOG_WINDOW_MS) m.backlogs.shift();
|
|
927
|
+
while ((m.backlogHistory[1]?.at ?? Infinity) <= t - PACK_STEADY_MS) m.backlogHistory.shift();
|
|
928
|
+
}
|
|
929
|
+
if (busyMs !== void 0) {
|
|
930
|
+
if ((m.busyReports.at(-1)?.total ?? 0) > busyMs) m.busyReports = [];
|
|
931
|
+
m.busyReports.push({
|
|
932
|
+
at: t,
|
|
933
|
+
total: busyMs
|
|
934
|
+
});
|
|
935
|
+
const cut = t - windowMs - STALE_MS;
|
|
936
|
+
while ((m.busyReports[1]?.at ?? Infinity) <= cut) m.busyReports.shift();
|
|
937
|
+
}
|
|
938
|
+
},
|
|
939
|
+
measure(id) {
|
|
940
|
+
const m = members.get(id);
|
|
941
|
+
if (m) return average(m.beats, windowMs, now());
|
|
942
|
+
const s = seen.get(id);
|
|
943
|
+
return s && average(s.beats, windowMs, now());
|
|
944
|
+
},
|
|
945
|
+
on(event, fn) {
|
|
946
|
+
const set = listeners[event];
|
|
947
|
+
set.add(fn);
|
|
948
|
+
return () => void set.delete(fn);
|
|
949
|
+
},
|
|
950
|
+
dispose() {
|
|
951
|
+
if (disposed) return;
|
|
952
|
+
disposed = true;
|
|
953
|
+
claim.release();
|
|
954
|
+
clearInterval(timer);
|
|
955
|
+
if (judging !== void 0) clearTimeout(judging);
|
|
956
|
+
for (const off of offs) off();
|
|
957
|
+
for (const set of Object.values(listeners)) set.clear();
|
|
958
|
+
}
|
|
959
|
+
};
|
|
960
|
+
}
|
|
961
|
+
export { createBalancer };
|
package/package.json
CHANGED
|
@@ -1,7 +1,32 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@isoloop/balancer",
|
|
3
|
-
"version": "0.0.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.1.0-next.1",
|
|
4
|
+
"description": "Experimental automatic placement for Isoloop isolates: joins a pool or keeps apart from measured load, quarantines what blocks.",
|
|
5
5
|
"license": "MIT",
|
|
6
|
-
"
|
|
7
|
-
|
|
6
|
+
"type": "module",
|
|
7
|
+
"sideEffects": false,
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/isoloop/isoloop.git",
|
|
11
|
+
"directory": "packages/balancer"
|
|
12
|
+
},
|
|
13
|
+
"files": [
|
|
14
|
+
"dist"
|
|
15
|
+
],
|
|
16
|
+
"exports": {
|
|
17
|
+
".": {
|
|
18
|
+
"types": "./dist/index.d.ts",
|
|
19
|
+
"import": "./dist/index.js"
|
|
20
|
+
}
|
|
21
|
+
},
|
|
22
|
+
"dependencies": {
|
|
23
|
+
"@isoloop/host": "0.1.0-next.1"
|
|
24
|
+
},
|
|
25
|
+
"devDependencies": {
|
|
26
|
+
"typescript": "7.0.2"
|
|
27
|
+
},
|
|
28
|
+
"scripts": {
|
|
29
|
+
"build": "tsdown --platform browser --dts",
|
|
30
|
+
"typecheck": "tsc --noEmit"
|
|
31
|
+
}
|
|
32
|
+
}
|