@isoloop/balancer 0.0.0-next.g3322248
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/index.d.ts +174 -0
- package/dist/index.js +544 -0
- package/package.json +32 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sabin Pandelovitch
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The part of the kernel the balancer uses: the public API only (`spawn`,
|
|
3
|
+
* `stop`, `pools`, and the `isolate:beat` and `isolate:crash` events), so
|
|
4
|
+
* `@isoloop/host`'s `Kernel` fits it as is.
|
|
5
|
+
*/
|
|
6
|
+
interface KernelLike<M extends ManifestLike = ManifestLike> {
|
|
7
|
+
spawn(manifest: M): unknown;
|
|
8
|
+
stop(id: string): Promise<void>;
|
|
9
|
+
pools(): readonly PoolLike[];
|
|
10
|
+
on(event: "isolate:crash", fn: (e: CrashLike) => void): () => void;
|
|
11
|
+
/** Every heartbeat as it arrives, once per member of a pool (ADR 0017). */
|
|
12
|
+
on(event: "isolate:beat", fn: (e: BeatLike) => void): () => void;
|
|
13
|
+
/**
|
|
14
|
+
* Only for kernels not made by `createKernel` (a simulated one in tests):
|
|
15
|
+
* the claim of the `balancer-*` pools, as the host's `claimPools` gives it.
|
|
16
|
+
* The host's `Kernel` has none; the balancer then calls the host's.
|
|
17
|
+
*/
|
|
18
|
+
claimPools?(owner: "balancer"): {
|
|
19
|
+
spawn(manifest: M): unknown;
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
/** What the balancer reads from an isolate manifest: its id, and the pool it sets. */
|
|
23
|
+
type ManifestLike = {
|
|
24
|
+
id: string;
|
|
25
|
+
pool?: string;
|
|
26
|
+
};
|
|
27
|
+
/** The kernel's `isolate:beat` event (ADR 0017). */
|
|
28
|
+
type BeatLike = {
|
|
29
|
+
id: string;
|
|
30
|
+
pool?: string;
|
|
31
|
+
/** The worker's lag at this beat: the same for every member of a pool. */
|
|
32
|
+
lagMs: number;
|
|
33
|
+
/**
|
|
34
|
+
* Time this isolate spent running in the interval the beat covers.
|
|
35
|
+
* Absent from runtimes that don't report it; read as 0.
|
|
36
|
+
*/
|
|
37
|
+
busyMs?: number;
|
|
38
|
+
/** The host's `performance.now()` on receipt. */
|
|
39
|
+
at: number;
|
|
40
|
+
};
|
|
41
|
+
/** The fields of `PoolSnapshot` the balancer reads. */
|
|
42
|
+
type PoolLike = {
|
|
43
|
+
name: string;
|
|
44
|
+
members: readonly string[];
|
|
45
|
+
running: boolean;
|
|
46
|
+
};
|
|
47
|
+
/** The fields of the kernel's `isolate:crash` event the balancer reads. */
|
|
48
|
+
type CrashLike = {
|
|
49
|
+
id: string;
|
|
50
|
+
reason: string;
|
|
51
|
+
pool?: string;
|
|
52
|
+
};
|
|
53
|
+
/**
|
|
54
|
+
* Where the balancer keeps each isolate's measured load between page loads,
|
|
55
|
+
* so a reload places from history (M3h, Q17). The app supplies it; the
|
|
56
|
+
* creator backs it with a persistent topic. `get` returning `undefined`
|
|
57
|
+
* means "never measured": the isolate starts in quarantine.
|
|
58
|
+
*/
|
|
59
|
+
interface LoadStore {
|
|
60
|
+
get(id: string): number | undefined;
|
|
61
|
+
set(id: string, loadPct: number): void;
|
|
62
|
+
}
|
|
63
|
+
type BalancerOptions = {
|
|
64
|
+
/**
|
|
65
|
+
* The balancer's own allowance of workers (ADR 0017, amended): its
|
|
66
|
+
* `balancer-N` pools, plus any quarantine worker beyond the first. The
|
|
67
|
+
* quarantine, `balancer-quarantine-1`, is allowed on top, and workers the
|
|
68
|
+
* app runs itself don't count; the kernel's budget stays the page-wide
|
|
69
|
+
* warning. Default `max(2, navigator.hardwareConcurrency - 2)`; at least 1.
|
|
70
|
+
*/
|
|
71
|
+
maxWorkers?: number;
|
|
72
|
+
/** One beat with the pool's `lagMs` above this moves its busiest member to quarantine. Default 100. */
|
|
73
|
+
spike?: {
|
|
74
|
+
lagMs?: number;
|
|
75
|
+
};
|
|
76
|
+
/**
|
|
77
|
+
* The pool's median `lagMs` above `lagMs` over `forMs` moves its heaviest
|
|
78
|
+
* member out. Default 50 ms over 3,000 ms. So does a pool whose members'
|
|
79
|
+
* busy time adds up to 80 % of a core or more over the same window, whatever
|
|
80
|
+
* its lag (a fixed threshold, above `headroom`).
|
|
81
|
+
*/
|
|
82
|
+
sustained?: {
|
|
83
|
+
lagMs?: number;
|
|
84
|
+
forMs?: number;
|
|
85
|
+
};
|
|
86
|
+
/** An isolate joins a pool only while pool load + its load stays under `loadPct`. Default 60. */
|
|
87
|
+
headroom?: {
|
|
88
|
+
loadPct?: number;
|
|
89
|
+
};
|
|
90
|
+
/**
|
|
91
|
+
* Leaving quarantine: an isolate there only to be measured leaves once it
|
|
92
|
+
* has 3 s of beats there. An offender leaves after `steadyMs` (default
|
|
93
|
+
* 30,000 ms) with no spike and no crash, whatever its load; each offence
|
|
94
|
+
* doubles the window (30, 60, 120 s), and after `maxReturns` returns
|
|
95
|
+
* (default 3) the next offence keeps it apart for good. Either goes where
|
|
96
|
+
* the placement rule finds room, or stays, flagged `atCap`, until there is.
|
|
97
|
+
*/
|
|
98
|
+
quarantine?: {
|
|
99
|
+
steadyMs?: number;
|
|
100
|
+
maxReturns?: number;
|
|
101
|
+
};
|
|
102
|
+
loads?: LoadStore;
|
|
103
|
+
};
|
|
104
|
+
type PlacementState = "placed" | "quarantined" | "kept-apart" | "moving";
|
|
105
|
+
type Placement = {
|
|
106
|
+
id: string;
|
|
107
|
+
pool: string;
|
|
108
|
+
state: PlacementState;
|
|
109
|
+
/** Measured share of one core, 0–100; `undefined` until measured. */
|
|
110
|
+
loadPct: number | undefined;
|
|
111
|
+
/** No pool had room and the worker cap was reached: placed anyway, or left where it is. */
|
|
112
|
+
atCap?: true;
|
|
113
|
+
/**
|
|
114
|
+
* In one of the balancer's pools, but not placed by it. The host reserves
|
|
115
|
+
* `balancer-*` pools, so this shouldn't happen; it is never moved.
|
|
116
|
+
*/
|
|
117
|
+
foreign?: true;
|
|
118
|
+
};
|
|
119
|
+
/**
|
|
120
|
+
* Why a member moved: `spike` (one long task), `sustained` (the pool's lag
|
|
121
|
+
* stayed high), `full` (the pool's members' busy time reached 80 % of a core,
|
|
122
|
+
* whatever its lag), `crash`, `bisect` (looking for a pool's unnamed
|
|
123
|
+
* culprit), or `steady` (back out of quarantine).
|
|
124
|
+
*/
|
|
125
|
+
type MoveReason = "spike" | "sustained" | "full" | "crash" | "bisect" | "steady";
|
|
126
|
+
type MoveEvent = {
|
|
127
|
+
id: string;
|
|
128
|
+
from: string;
|
|
129
|
+
to: string;
|
|
130
|
+
reason: MoveReason;
|
|
131
|
+
};
|
|
132
|
+
/** An isolate the balancer didn't place turned up in one of its pools (see `Placement.foreign`). */
|
|
133
|
+
type ForeignEvent = {
|
|
134
|
+
id: string;
|
|
135
|
+
pool: string;
|
|
136
|
+
};
|
|
137
|
+
type BalancerEvents = {
|
|
138
|
+
move: MoveEvent;
|
|
139
|
+
foreign: ForeignEvent;
|
|
140
|
+
};
|
|
141
|
+
/**
|
|
142
|
+
* Pools are named `balancer-1`, `balancer-2`…; the quarantine is
|
|
143
|
+
* `balancer-quarantine-1`, and `-2`, `-3`… when a suspect needs a worker of
|
|
144
|
+
* its own. `createBalancer` throws `balancer_exists` if the kernel has one.
|
|
145
|
+
*/
|
|
146
|
+
interface Balancer<M extends ManifestLike = ManifestLike> {
|
|
147
|
+
/** Like `kernel.spawn`, but the balancer picks `pool`; `manifest.pool` must be absent. */
|
|
148
|
+
spawn(manifest: M): Placement;
|
|
149
|
+
stop(id: string): Promise<void>;
|
|
150
|
+
placements(): Placement[];
|
|
151
|
+
/**
|
|
152
|
+
* `id`'s load over the last 3 s, as a share of one core (0–100), whether
|
|
153
|
+
* the balancer placed it or not: a preview's, for `loads.set` on Save.
|
|
154
|
+
* `undefined` until 3 s of beats have been seen.
|
|
155
|
+
*/
|
|
156
|
+
measure(id: string): number | undefined;
|
|
157
|
+
/** Leaves `id` where it is until the returned function is called (a widget open for editing). */
|
|
158
|
+
hold(id: string): () => void;
|
|
159
|
+
on<E extends keyof BalancerEvents>(event: E, fn: (e: BalancerEvents[E]) => void): () => void;
|
|
160
|
+
/** Stops watching; isolates keep running where they are. */
|
|
161
|
+
dispose(): void;
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Automatic placement for isolates spawned through it (M3h, ADR 0017).
|
|
165
|
+
*
|
|
166
|
+
* @experimental The policy and its options may change before 1.0.
|
|
167
|
+
*
|
|
168
|
+
* It decides on the main thread, from every heartbeat (`isolate:beat`), and moves an
|
|
169
|
+
* isolate by stopping it and spawning it again with another `pool`; the
|
|
170
|
+
* kernel itself never pools on its own (ADR 0001). Isolates spawned with
|
|
171
|
+
* `kernel.spawn` are never touched, and don't count toward `maxWorkers`.
|
|
172
|
+
*/
|
|
173
|
+
export declare function createBalancer<M extends ManifestLike>(kernel: KernelLike<M>, opts?: BalancerOptions): Balancer<M>;
|
|
174
|
+
export type { Balancer, BalancerEvents, BalancerOptions, BeatLike, CrashLike, ForeignEvent, KernelLike, LoadStore, ManifestLike, MoveEvent, MoveReason, Placement, PlacementState, PoolLike };
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,544 @@
|
|
|
1
|
+
import { claimPools as claimPools$1 } from "@isoloop/host";
|
|
2
|
+
// src/claim.ts
|
|
3
|
+
/**
|
|
4
|
+
* The host's claim, for a kernel from `createKernel`. A `KernelLike` that
|
|
5
|
+
* offers `claimPools` itself (a simulated kernel in tests) is asked
|
|
6
|
+
* directly instead.
|
|
7
|
+
*/
|
|
8
|
+
const claimPools = (kernel, owner) => {
|
|
9
|
+
if (kernel.claimPools) return kernel.claimPools(owner);
|
|
10
|
+
const claim = claimPools$1(kernel, owner);
|
|
11
|
+
return { spawn: (manifest) => claim.spawn(manifest) };
|
|
12
|
+
};
|
|
13
|
+
// src/balancer.ts
|
|
14
|
+
/**
|
|
15
|
+
* A pool's beat reaches the balancer as one `isolate:beat` per member, all
|
|
16
|
+
* from the same worker message: events this close together are one beat.
|
|
17
|
+
* Beats are 100 ms apart by default.
|
|
18
|
+
*/
|
|
19
|
+
const BEAT_GAP_MS = 10;
|
|
20
|
+
/** How often it decides. */
|
|
21
|
+
const DECIDE_MS = 500;
|
|
22
|
+
/** How often a changed load is written to the `loads` store. */
|
|
23
|
+
const SAVE_MS = 1e4;
|
|
24
|
+
/** A load is written again only once it moved by this much. */
|
|
25
|
+
const SAVE_DELTA_PCT = 5;
|
|
26
|
+
/** Pool names: `balancer-1`, `balancer-2`…, reserved by the host for the balancer. */
|
|
27
|
+
const POOLS = "balancer";
|
|
28
|
+
/** The quarantine, and `-2`, `-3`… for suspects given a worker of their own. */
|
|
29
|
+
const QUARANTINES = "balancer-quarantine";
|
|
30
|
+
const QUARANTINE = `${QUARANTINES}-1`;
|
|
31
|
+
const now = () => performance.now();
|
|
32
|
+
/**
|
|
33
|
+
* `busyMs` is approximate: shared framework work is split between members and
|
|
34
|
+
* promise continuations go unattributed. When the members' busy time explains
|
|
35
|
+
* less than this share of the interval, the lag isn't pinned on the busiest
|
|
36
|
+
* member; the balancer bisects instead, as on a kill.
|
|
37
|
+
*/
|
|
38
|
+
const ATTRIBUTED_PCT = 50;
|
|
39
|
+
/**
|
|
40
|
+
* A pool whose members' busy time adds up to this share of a core over the
|
|
41
|
+
* window is full (Q15), whatever its lag: work that yields often keeps the lag
|
|
42
|
+
* low while the members starve each other. Placement stops adding at
|
|
43
|
+
* `headroom` (60 by default); moving starts here, and the gap is the
|
|
44
|
+
* hysteresis.
|
|
45
|
+
*/
|
|
46
|
+
const FULL_PCT = 80;
|
|
47
|
+
/** Worker crash reasons whose culprit the kernel can't name: every member of the pool crashes. */
|
|
48
|
+
const POOL_KILL = /* @__PURE__ */ new Set(["unresponsive", "worker-error"]);
|
|
49
|
+
const defaultMaxWorkers = () => {
|
|
50
|
+
const cores = Reflect.get(globalThis, "navigator")?.hardwareConcurrency ?? 4;
|
|
51
|
+
return Math.max(2, cores - 2);
|
|
52
|
+
};
|
|
53
|
+
const median = (xs) => {
|
|
54
|
+
const s = xs.toSorted((a, b) => a - b);
|
|
55
|
+
const mid = s.length >> 1;
|
|
56
|
+
return s.length % 2 === 1 ? s[mid] ?? 0 : ((s[mid - 1] ?? 0) + (s[mid] ?? 0)) / 2;
|
|
57
|
+
};
|
|
58
|
+
const isQuarantine = (pool) => pool.startsWith(`${QUARANTINES}-`);
|
|
59
|
+
const isOurs = (pool) => pool.startsWith(`${POOLS}-`);
|
|
60
|
+
/** Load as a share of one core over `beats`, if they cover at least 90 % of `windowMs`. */
|
|
61
|
+
const average = (beats, windowMs, t) => {
|
|
62
|
+
const recent = beats.filter((b) => b.at > t - windowMs);
|
|
63
|
+
const dur = recent.reduce((s, b) => s + b.dur, 0);
|
|
64
|
+
if (dur < windowMs * .9) return void 0;
|
|
65
|
+
const busy = recent.reduce((s, b) => s + b.busy, 0);
|
|
66
|
+
return Math.min(100, busy / dur * 100);
|
|
67
|
+
};
|
|
68
|
+
/**
|
|
69
|
+
* Automatic placement for isolates spawned through it (M3h, ADR 0017).
|
|
70
|
+
*
|
|
71
|
+
* @experimental The policy and its options may change before 1.0.
|
|
72
|
+
*
|
|
73
|
+
* It decides on the main thread, from every heartbeat (`isolate:beat`), and moves an
|
|
74
|
+
* isolate by stopping it and spawning it again with another `pool`; the
|
|
75
|
+
* kernel itself never pools on its own (ADR 0001). Isolates spawned with
|
|
76
|
+
* `kernel.spawn` are never touched, and don't count toward `maxWorkers`.
|
|
77
|
+
*/
|
|
78
|
+
function createBalancer(kernel, opts = {}) {
|
|
79
|
+
const claim = claimPools(kernel, "balancer");
|
|
80
|
+
const maxWorkers = Math.max(1, opts.maxWorkers ?? defaultMaxWorkers());
|
|
81
|
+
const spikeLag = opts.spike?.lagMs ?? 100;
|
|
82
|
+
const sustainedLag = opts.sustained?.lagMs ?? 50;
|
|
83
|
+
const windowMs = opts.sustained?.forMs ?? 3e3;
|
|
84
|
+
const headroom = opts.headroom?.loadPct ?? 60;
|
|
85
|
+
const steadyMs = opts.quarantine?.steadyMs ?? 3e4;
|
|
86
|
+
const maxReturns = opts.quarantine?.maxReturns ?? 3;
|
|
87
|
+
const loads = opts.loads;
|
|
88
|
+
const members = /* @__PURE__ */ new Map();
|
|
89
|
+
const pools = /* @__PURE__ */ new Map();
|
|
90
|
+
const listeners = {
|
|
91
|
+
move: /* @__PURE__ */ new Set(),
|
|
92
|
+
foreign: /* @__PURE__ */ new Set()
|
|
93
|
+
};
|
|
94
|
+
/** Beats of isolates the balancer didn't place, for `measure` (previews included). */
|
|
95
|
+
const seen = /* @__PURE__ */ new Map();
|
|
96
|
+
/** Isolates in our pools we didn't place, by id: their pool. */
|
|
97
|
+
const foreign = /* @__PURE__ */ new Map();
|
|
98
|
+
/** Pools the kernel killed since the last decision (an unnamed culprit). */
|
|
99
|
+
const killed = /* @__PURE__ */ new Set();
|
|
100
|
+
/** An unfinished search for the member that kills its pool (Q20). */
|
|
101
|
+
/**
|
|
102
|
+
* `suspects` narrows with each step; `moved` is everyone it has moved so
|
|
103
|
+
* far, kept in quarantine until it ends so a culprit that only shows with
|
|
104
|
+
* company still meets the members it was moved with.
|
|
105
|
+
*/
|
|
106
|
+
let bisect;
|
|
107
|
+
/** Pools whose latest beat spiked, judged once all its members' events are in. */
|
|
108
|
+
const spiked = /* @__PURE__ */ new Set();
|
|
109
|
+
let judging;
|
|
110
|
+
let lastSave = -Infinity;
|
|
111
|
+
let disposed = false;
|
|
112
|
+
const pool = (name) => {
|
|
113
|
+
let p = pools.get(name);
|
|
114
|
+
if (!p) {
|
|
115
|
+
p = {
|
|
116
|
+
lags: [],
|
|
117
|
+
beatAt: void 0,
|
|
118
|
+
beat: [],
|
|
119
|
+
since: now(),
|
|
120
|
+
watch: false
|
|
121
|
+
};
|
|
122
|
+
pools.set(name, p);
|
|
123
|
+
}
|
|
124
|
+
return p;
|
|
125
|
+
};
|
|
126
|
+
const membersOf = (name) => [...members.values()].filter((m) => m.pool === name && m.state !== "moving");
|
|
127
|
+
const changed = (name) => {
|
|
128
|
+
const p = pool(name);
|
|
129
|
+
p.lags = [];
|
|
130
|
+
p.beatAt = void 0;
|
|
131
|
+
p.beat = [];
|
|
132
|
+
p.since = now();
|
|
133
|
+
};
|
|
134
|
+
/** Measured load over the last window, or the last one known. */
|
|
135
|
+
const loadOf = (m) => average(m.beats, windowMs, now()) ?? m.knownLoad;
|
|
136
|
+
const poolLoad = (name) => membersOf(name).reduce((s, m) => s + (loadOf(m) ?? 0), 0);
|
|
137
|
+
const windowFull = (name) => {
|
|
138
|
+
const p = pool(name);
|
|
139
|
+
const last = p.lags.at(-1);
|
|
140
|
+
return last !== void 0 && last.at - p.since >= windowMs;
|
|
141
|
+
};
|
|
142
|
+
/**
|
|
143
|
+
* Workers counted against `maxWorkers`: the balancer's pools and any
|
|
144
|
+
* quarantine worker beyond the first, which is allowed on top. Workers the
|
|
145
|
+
* app runs itself don't count (ADR 0017, amended).
|
|
146
|
+
*/
|
|
147
|
+
const ownWorkers = () => kernel.pools().filter((p) => p.running && isOurs(p.name) && p.name !== QUARANTINE).length;
|
|
148
|
+
const ownPools = () => [...new Set([...members.values()].map((m) => m.pool))].filter((n) => !isQuarantine(n));
|
|
149
|
+
/** Room for one more worker of the balancer's own. */
|
|
150
|
+
const roomForPool = () => ownWorkers() + 1 <= maxWorkers;
|
|
151
|
+
const freshName = (prefix, first) => {
|
|
152
|
+
const taken = new Set(kernel.pools().map((p) => p.name));
|
|
153
|
+
for (const m of members.values()) taken.add(m.pool);
|
|
154
|
+
for (let n = first;; n++) {
|
|
155
|
+
const name = `${prefix}-${n}`;
|
|
156
|
+
if (!taken.has(name)) return name;
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
const recentlySpiked = (name) => membersOf(name).some((m) => now() - m.lastSpikeAt < steadyMs);
|
|
160
|
+
/**
|
|
161
|
+
* Where an isolate with `load` goes (Q7′): the auto pool with the most room
|
|
162
|
+
* that stays under `headroom` with it, else a new pool if the cap allows,
|
|
163
|
+
* else the least-loaded auto pool, flagged at the cap. `undefined` when
|
|
164
|
+
* there's nowhere better than `except`.
|
|
165
|
+
*/
|
|
166
|
+
const target = (load, except) => {
|
|
167
|
+
const best = ownPools().filter((n) => n !== except && !recentlySpiked(n)).map((n) => ({
|
|
168
|
+
n,
|
|
169
|
+
load: poolLoad(n)
|
|
170
|
+
})).filter((c) => c.load + load < headroom).toSorted((a, b) => a.load - b.load)[0];
|
|
171
|
+
if (best) return {
|
|
172
|
+
pool: best.n,
|
|
173
|
+
atCap: false
|
|
174
|
+
};
|
|
175
|
+
if (ownPools().length === 0 || roomForPool()) return {
|
|
176
|
+
pool: freshName(POOLS, 1),
|
|
177
|
+
atCap: false
|
|
178
|
+
};
|
|
179
|
+
if (except !== void 0) return void 0;
|
|
180
|
+
const least = ownPools().map((n) => ({
|
|
181
|
+
n,
|
|
182
|
+
load: poolLoad(n)
|
|
183
|
+
})).toSorted((a, b) => a.load - b.load)[0];
|
|
184
|
+
return least ? {
|
|
185
|
+
pool: least.n,
|
|
186
|
+
atCap: true
|
|
187
|
+
} : {
|
|
188
|
+
pool: freshName(POOLS, 1),
|
|
189
|
+
atCap: true
|
|
190
|
+
};
|
|
191
|
+
};
|
|
192
|
+
const emit = (e) => {
|
|
193
|
+
for (const fn of listeners.move) fn(e);
|
|
194
|
+
};
|
|
195
|
+
const save = (m) => {
|
|
196
|
+
const load = loadOf(m);
|
|
197
|
+
if (!loads || load === void 0) return;
|
|
198
|
+
if (m.savedLoad !== void 0 && Math.abs(load - m.savedLoad) < SAVE_DELTA_PCT) return;
|
|
199
|
+
loads.set(m.manifest.id, load);
|
|
200
|
+
m.savedLoad = load;
|
|
201
|
+
};
|
|
202
|
+
const move = (m, to, reason, state) => {
|
|
203
|
+
const from = m.pool;
|
|
204
|
+
const id = m.manifest.id;
|
|
205
|
+
m.knownLoad = loadOf(m) ?? m.knownLoad;
|
|
206
|
+
m.state = "moving";
|
|
207
|
+
m.beats = [];
|
|
208
|
+
m.lastBeatAt = void 0;
|
|
209
|
+
m.pool = to;
|
|
210
|
+
changed(from);
|
|
211
|
+
pool(from).watch = true;
|
|
212
|
+
kernel.stop(id).then(() => {
|
|
213
|
+
if (disposed || members.get(id) !== m) return;
|
|
214
|
+
changed(to);
|
|
215
|
+
claim.spawn({
|
|
216
|
+
...m.manifest,
|
|
217
|
+
pool: to
|
|
218
|
+
});
|
|
219
|
+
m.state = state;
|
|
220
|
+
emit({
|
|
221
|
+
id,
|
|
222
|
+
from,
|
|
223
|
+
to,
|
|
224
|
+
reason
|
|
225
|
+
});
|
|
226
|
+
}, () => {
|
|
227
|
+
members.delete(id);
|
|
228
|
+
});
|
|
229
|
+
};
|
|
230
|
+
const stateFor = (m) => m.offences > maxReturns ? "kept-apart" : "quarantined";
|
|
231
|
+
/** Sends `m` to quarantine for misbehaving (`offence`) or to bisect. */
|
|
232
|
+
const quarantine = (m, reason, offence) => {
|
|
233
|
+
m.lastSpikeAt = now();
|
|
234
|
+
if (offence) m.offences++;
|
|
235
|
+
const state = stateFor(m);
|
|
236
|
+
m.quarantinedAt = now();
|
|
237
|
+
if (isQuarantine(m.pool)) {
|
|
238
|
+
m.state = state;
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
move(m, QUARANTINE, reason, state);
|
|
242
|
+
};
|
|
243
|
+
const movable = (m) => m.state !== "moving" && m.holds === 0;
|
|
244
|
+
const trim = (beats, at) => {
|
|
245
|
+
const cut = at - windowMs;
|
|
246
|
+
while ((beats[0]?.at ?? Infinity) <= cut) beats.shift();
|
|
247
|
+
};
|
|
248
|
+
/** A beat of an isolate the balancer doesn't place, for `measure`. */
|
|
249
|
+
const watchOther = (e) => {
|
|
250
|
+
const w = seen.get(e.id);
|
|
251
|
+
if (!w) {
|
|
252
|
+
seen.set(e.id, {
|
|
253
|
+
lastBeatAt: e.at,
|
|
254
|
+
beats: []
|
|
255
|
+
});
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
w.beats.push({
|
|
259
|
+
at: e.at,
|
|
260
|
+
busy: e.busyMs ?? 0,
|
|
261
|
+
dur: e.at - w.lastBeatAt
|
|
262
|
+
});
|
|
263
|
+
w.lastBeatAt = e.at;
|
|
264
|
+
trim(w.beats, e.at);
|
|
265
|
+
};
|
|
266
|
+
/**
|
|
267
|
+
* Every heartbeat, as the kernel reports it (`isolate:beat`). A pool's
|
|
268
|
+
* beat comes as one event per member; its lag is recorded once, and a
|
|
269
|
+
* spike is judged once all of them are in.
|
|
270
|
+
*/
|
|
271
|
+
const onBeat = (e) => {
|
|
272
|
+
if (disposed) return;
|
|
273
|
+
const m = members.get(e.id);
|
|
274
|
+
if (!m) {
|
|
275
|
+
watchOther(e);
|
|
276
|
+
return;
|
|
277
|
+
}
|
|
278
|
+
if (m.state === "moving" || e.pool !== m.pool) return;
|
|
279
|
+
if (m.lastBeatAt !== void 0) {
|
|
280
|
+
m.beats.push({
|
|
281
|
+
at: e.at,
|
|
282
|
+
busy: e.busyMs ?? 0,
|
|
283
|
+
dur: e.at - m.lastBeatAt
|
|
284
|
+
});
|
|
285
|
+
trim(m.beats, e.at);
|
|
286
|
+
}
|
|
287
|
+
m.lastBeatAt = e.at;
|
|
288
|
+
const p = pool(m.pool);
|
|
289
|
+
if (p.beatAt === void 0 || e.at - p.beatAt > BEAT_GAP_MS) {
|
|
290
|
+
p.beatAt = e.at;
|
|
291
|
+
p.beat = [];
|
|
292
|
+
p.lags.push({
|
|
293
|
+
at: e.at,
|
|
294
|
+
lagMs: e.lagMs
|
|
295
|
+
});
|
|
296
|
+
const cut = e.at - windowMs;
|
|
297
|
+
while ((p.lags[0]?.at ?? Infinity) <= cut) p.lags.shift();
|
|
298
|
+
if (e.lagMs > spikeLag) {
|
|
299
|
+
spiked.add(m.pool);
|
|
300
|
+
judging ??= setTimeout(judge, 0);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
p.beat.push(m);
|
|
304
|
+
};
|
|
305
|
+
/** A spike (Q6′): the member that was busiest in that beat goes to quarantine at once. */
|
|
306
|
+
const judge = () => {
|
|
307
|
+
judging = void 0;
|
|
308
|
+
if (disposed) return;
|
|
309
|
+
for (const name of spiked) {
|
|
310
|
+
const beat = pool(name).beat.filter((m) => members.get(m.manifest.id) === m && m.pool === name && m.state !== "moving");
|
|
311
|
+
const dur = Math.max(0, ...beat.map((m) => m.beats.at(-1)?.dur ?? 0));
|
|
312
|
+
const busy = beat.reduce((sum, m) => sum + (m.beats.at(-1)?.busy ?? 0), 0);
|
|
313
|
+
if (dur > 0 && busy / dur * 100 < ATTRIBUTED_PCT) {
|
|
314
|
+
search(name, "spike");
|
|
315
|
+
continue;
|
|
316
|
+
}
|
|
317
|
+
const culprit = beat.filter((m) => m.beats.length > 0).toSorted((a, b) => (b.beats.at(-1)?.busy ?? 0) - (a.beats.at(-1)?.busy ?? 0))[0];
|
|
318
|
+
if (!culprit) continue;
|
|
319
|
+
culprit.lastSpikeAt = now();
|
|
320
|
+
if (!movable(culprit)) continue;
|
|
321
|
+
if (isQuarantine(name)) quarantine(culprit, "spike", culprit.offences === 0);
|
|
322
|
+
else if (membersOf(name).length > 1 || culprit.offences > 0) quarantine(culprit, "spike", true);
|
|
323
|
+
}
|
|
324
|
+
spiked.clear();
|
|
325
|
+
};
|
|
326
|
+
/** Defensive: the host reserves our pools, so nobody else should be in them. */
|
|
327
|
+
const checkForeign = () => {
|
|
328
|
+
const present = /* @__PURE__ */ new Map();
|
|
329
|
+
for (const p of kernel.pools()) {
|
|
330
|
+
if (!isOurs(p.name)) continue;
|
|
331
|
+
for (const id of p.members) if (!members.has(id)) present.set(id, p.name);
|
|
332
|
+
}
|
|
333
|
+
for (const [id, name] of present) {
|
|
334
|
+
if (foreign.get(id) === name) continue;
|
|
335
|
+
foreign.set(id, name);
|
|
336
|
+
for (const fn of listeners.foreign) fn({
|
|
337
|
+
id,
|
|
338
|
+
pool: name
|
|
339
|
+
});
|
|
340
|
+
}
|
|
341
|
+
for (const id of foreign.keys()) if (!present.has(id)) foreign.delete(id);
|
|
342
|
+
};
|
|
343
|
+
/**
|
|
344
|
+
* Finds the member behind a pool's trouble when the kernel or `busyMs`
|
|
345
|
+
* can't name it (Q20): the heavier half of the suspects moves, and
|
|
346
|
+
* whichever half acts up again holds the culprit.
|
|
347
|
+
*/
|
|
348
|
+
const search = (name, reason) => {
|
|
349
|
+
const here = membersOf(name).filter((m) => m.holds === 0);
|
|
350
|
+
if (here.length === 0) return;
|
|
351
|
+
const ids = new Set(here.map((m) => m.manifest.id));
|
|
352
|
+
const prior = bisect && [...bisect.suspects].filter((id) => ids.has(id));
|
|
353
|
+
const suspects = prior && prior.length > 0 ? prior : [...ids];
|
|
354
|
+
if (suspects.length === 1) {
|
|
355
|
+
const culprit = members.get(suspects[0] ?? "");
|
|
356
|
+
bisect = void 0;
|
|
357
|
+
if (!culprit) return;
|
|
358
|
+
if (reason !== "crash" && membersOf(name).length === 1 && culprit.offences === 0) return;
|
|
359
|
+
if (membersOf(name).length > 1 && roomForPool()) {
|
|
360
|
+
culprit.offences++;
|
|
361
|
+
culprit.lastSpikeAt = culprit.quarantinedAt = now();
|
|
362
|
+
move(culprit, freshName(QUARANTINES, 2), reason, stateFor(culprit));
|
|
363
|
+
} else quarantine(culprit, reason, true);
|
|
364
|
+
return;
|
|
365
|
+
}
|
|
366
|
+
const sorted = suspects.map((id) => members.get(id)).filter((m) => m !== void 0).toSorted((a, b) => (loadOf(b) ?? 0) - (loadOf(a) ?? 0));
|
|
367
|
+
const half = sorted.slice(0, Math.ceil(sorted.length / 2));
|
|
368
|
+
bisect = {
|
|
369
|
+
suspects: new Set(suspects),
|
|
370
|
+
moved: /* @__PURE__ */ new Set([...bisect?.moved ?? [], ...half.map((m) => m.manifest.id)]),
|
|
371
|
+
at: now()
|
|
372
|
+
};
|
|
373
|
+
const to = isQuarantine(name) ? roomForPool() ? freshName(QUARANTINES, 2) : void 0 : QUARANTINE;
|
|
374
|
+
if (to === void 0) return;
|
|
375
|
+
for (const m of half) if (to === QUARANTINE) {
|
|
376
|
+
m.quarantinedAt = now();
|
|
377
|
+
move(m, to, "bisect", stateFor(m));
|
|
378
|
+
} else move(m, to, "bisect", m.state);
|
|
379
|
+
};
|
|
380
|
+
const steadyFor = (m) => steadyMs * 2 ** Math.max(0, m.offences - 1);
|
|
381
|
+
const decide = () => {
|
|
382
|
+
const t = now();
|
|
383
|
+
checkForeign();
|
|
384
|
+
for (const [id, w] of seen) if (t - w.lastBeatAt > windowMs * 2) seen.delete(id);
|
|
385
|
+
for (const name of killed) search(name, "crash");
|
|
386
|
+
killed.clear();
|
|
387
|
+
if (bisect && t - bisect.at > steadyMs * 2) bisect = void 0;
|
|
388
|
+
for (const name of new Set([...members.values()].map((m) => m.pool))) {
|
|
389
|
+
const here = membersOf(name);
|
|
390
|
+
if (!windowFull(name)) continue;
|
|
391
|
+
const p = pool(name);
|
|
392
|
+
const lagging = median(p.lags.map((l) => l.lagMs)) > sustainedLag;
|
|
393
|
+
const busy = here.reduce((sum, m) => sum + (loadOf(m) ?? 0), 0);
|
|
394
|
+
const full = !isQuarantine(name) && busy >= FULL_PCT;
|
|
395
|
+
if (!lagging && !full) {
|
|
396
|
+
p.watch = false;
|
|
397
|
+
continue;
|
|
398
|
+
}
|
|
399
|
+
if (here.length < 2) continue;
|
|
400
|
+
if (lagging && !full && busy < ATTRIBUTED_PCT) {
|
|
401
|
+
search(name, "sustained");
|
|
402
|
+
continue;
|
|
403
|
+
}
|
|
404
|
+
const byLoad = here.toSorted((a, b) => (loadOf(b) ?? 0) - (loadOf(a) ?? 0));
|
|
405
|
+
const heaviest = p.watch ? byLoad.find(movable) : byLoad[0];
|
|
406
|
+
if (!heaviest || !movable(heaviest)) continue;
|
|
407
|
+
if (isQuarantine(name)) {
|
|
408
|
+
if (roomForPool()) move(heaviest, freshName(QUARANTINES, 2), "sustained", heaviest.state);
|
|
409
|
+
continue;
|
|
410
|
+
}
|
|
411
|
+
const to = target(loadOf(heaviest) ?? 0, name);
|
|
412
|
+
if (to) {
|
|
413
|
+
heaviest.atCap = false;
|
|
414
|
+
move(heaviest, to.pool, lagging ? "sustained" : "full", "placed");
|
|
415
|
+
} else heaviest.atCap = true;
|
|
416
|
+
}
|
|
417
|
+
for (const m of members.values()) {
|
|
418
|
+
if (m.state !== "quarantined" || !movable(m)) continue;
|
|
419
|
+
const fresh = average(m.beats, windowMs, t);
|
|
420
|
+
if (m.offences === 0) {
|
|
421
|
+
if (fresh === void 0 || bisect?.moved.has(m.manifest.id)) continue;
|
|
422
|
+
} else if (t - Math.max(m.quarantinedAt, m.lastSpikeAt) < steadyFor(m)) continue;
|
|
423
|
+
const load = fresh ?? m.knownLoad;
|
|
424
|
+
if (load === void 0) continue;
|
|
425
|
+
const to = target(load, m.pool);
|
|
426
|
+
if (!to || to.atCap) {
|
|
427
|
+
m.atCap = true;
|
|
428
|
+
continue;
|
|
429
|
+
}
|
|
430
|
+
m.atCap = false;
|
|
431
|
+
move(m, to.pool, "steady", "placed");
|
|
432
|
+
}
|
|
433
|
+
if (t - lastSave >= SAVE_MS) {
|
|
434
|
+
lastSave = t;
|
|
435
|
+
for (const m of members.values()) save(m);
|
|
436
|
+
}
|
|
437
|
+
};
|
|
438
|
+
const onCrash = (e) => {
|
|
439
|
+
const m = members.get(e.id);
|
|
440
|
+
if (!m || m.state === "moving") return;
|
|
441
|
+
if (POOL_KILL.has(e.reason) && e.pool !== void 0) killed.add(e.pool);
|
|
442
|
+
else if (e.reason === "error") {
|
|
443
|
+
if (m.holds > 0) m.lastSpikeAt = now();
|
|
444
|
+
else quarantine(m, "crash", true);
|
|
445
|
+
}
|
|
446
|
+
};
|
|
447
|
+
const timer = setInterval(decide, DECIDE_MS);
|
|
448
|
+
const offs = [kernel.on("isolate:crash", onCrash), kernel.on("isolate:beat", onBeat)];
|
|
449
|
+
const placement = (m) => {
|
|
450
|
+
const p = {
|
|
451
|
+
id: m.manifest.id,
|
|
452
|
+
pool: m.pool,
|
|
453
|
+
state: m.state,
|
|
454
|
+
loadPct: loadOf(m)
|
|
455
|
+
};
|
|
456
|
+
if (m.atCap) p.atCap = true;
|
|
457
|
+
return p;
|
|
458
|
+
};
|
|
459
|
+
return {
|
|
460
|
+
spawn(manifest) {
|
|
461
|
+
if (manifest.pool !== void 0) throw new TypeError(`balancer.spawn: "${manifest.id}" names a pool; the balancer picks it`);
|
|
462
|
+
if (members.has(manifest.id)) throw new TypeError(`balancer.spawn: "${manifest.id}" is already running`);
|
|
463
|
+
const known = loads?.get(manifest.id);
|
|
464
|
+
const dest = (known === void 0 ? {
|
|
465
|
+
pool: QUARANTINE,
|
|
466
|
+
atCap: false
|
|
467
|
+
} : target(known)) ?? {
|
|
468
|
+
pool: QUARANTINE,
|
|
469
|
+
atCap: true
|
|
470
|
+
};
|
|
471
|
+
const m = {
|
|
472
|
+
manifest,
|
|
473
|
+
pool: dest.pool,
|
|
474
|
+
state: isQuarantine(dest.pool) ? "quarantined" : "placed",
|
|
475
|
+
beats: [],
|
|
476
|
+
lastBeatAt: void 0,
|
|
477
|
+
knownLoad: known,
|
|
478
|
+
savedLoad: known,
|
|
479
|
+
offences: 0,
|
|
480
|
+
quarantinedAt: isQuarantine(dest.pool) ? now() : -Infinity,
|
|
481
|
+
lastSpikeAt: -Infinity,
|
|
482
|
+
holds: 0,
|
|
483
|
+
atCap: dest.atCap
|
|
484
|
+
};
|
|
485
|
+
members.set(manifest.id, m);
|
|
486
|
+
changed(dest.pool);
|
|
487
|
+
claim.spawn({
|
|
488
|
+
...manifest,
|
|
489
|
+
pool: dest.pool
|
|
490
|
+
});
|
|
491
|
+
return placement(m);
|
|
492
|
+
},
|
|
493
|
+
async stop(id) {
|
|
494
|
+
const m = members.get(id);
|
|
495
|
+
if (!m) return kernel.stop(id);
|
|
496
|
+
save(m);
|
|
497
|
+
members.delete(id);
|
|
498
|
+
changed(m.pool);
|
|
499
|
+
await kernel.stop(id);
|
|
500
|
+
},
|
|
501
|
+
placements() {
|
|
502
|
+
const ours = [...members.values()].map(placement);
|
|
503
|
+
const others = [...foreign].map(([id, name]) => ({
|
|
504
|
+
id,
|
|
505
|
+
pool: name,
|
|
506
|
+
state: "placed",
|
|
507
|
+
loadPct: average(seen.get(id)?.beats ?? [], windowMs, now()),
|
|
508
|
+
foreign: true
|
|
509
|
+
}));
|
|
510
|
+
return [...ours, ...others];
|
|
511
|
+
},
|
|
512
|
+
hold(id) {
|
|
513
|
+
const m = members.get(id);
|
|
514
|
+
if (!m) return () => {};
|
|
515
|
+
m.holds++;
|
|
516
|
+
let released = false;
|
|
517
|
+
return () => {
|
|
518
|
+
if (released) return;
|
|
519
|
+
released = true;
|
|
520
|
+
m.holds--;
|
|
521
|
+
};
|
|
522
|
+
},
|
|
523
|
+
measure(id) {
|
|
524
|
+
const m = members.get(id);
|
|
525
|
+
if (m) return average(m.beats, windowMs, now());
|
|
526
|
+
const s = seen.get(id);
|
|
527
|
+
return s && average(s.beats, windowMs, now());
|
|
528
|
+
},
|
|
529
|
+
on(event, fn) {
|
|
530
|
+
const set = listeners[event];
|
|
531
|
+
set.add(fn);
|
|
532
|
+
return () => void set.delete(fn);
|
|
533
|
+
},
|
|
534
|
+
dispose() {
|
|
535
|
+
disposed = true;
|
|
536
|
+
clearInterval(timer);
|
|
537
|
+
if (judging !== void 0) clearTimeout(judging);
|
|
538
|
+
for (const off of offs) off();
|
|
539
|
+
listeners.move.clear();
|
|
540
|
+
listeners.foreign.clear();
|
|
541
|
+
}
|
|
542
|
+
};
|
|
543
|
+
}
|
|
544
|
+
export { createBalancer };
|
package/package.json
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@isoloop/balancer",
|
|
3
|
+
"version": "0.0.0-next.g3322248",
|
|
4
|
+
"description": "Experimental automatic placement for Isoloop isolates: joins a pool or keeps apart from measured load, quarantines what blocks.",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"sideEffects": false,
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/isoloop/isoloop.git",
|
|
11
|
+
"directory": "packages/balancer"
|
|
12
|
+
},
|
|
13
|
+
"files": [
|
|
14
|
+
"dist"
|
|
15
|
+
],
|
|
16
|
+
"exports": {
|
|
17
|
+
".": {
|
|
18
|
+
"types": "./dist/index.d.ts",
|
|
19
|
+
"import": "./dist/index.js"
|
|
20
|
+
}
|
|
21
|
+
},
|
|
22
|
+
"dependencies": {
|
|
23
|
+
"@isoloop/host": "0.0.0-next.g3322248"
|
|
24
|
+
},
|
|
25
|
+
"devDependencies": {
|
|
26
|
+
"typescript": "7.0.2"
|
|
27
|
+
},
|
|
28
|
+
"scripts": {
|
|
29
|
+
"build": "tsdown --platform browser --dts",
|
|
30
|
+
"typecheck": "tsc --noEmit"
|
|
31
|
+
}
|
|
32
|
+
}
|