@isoloop/balancer 0.1.0-next.0 → 0.1.0-next.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -3
- package/dist/index.d.ts +77 -24
- package/dist/index.js +220 -44
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -26,10 +26,23 @@ balancer.on('behind', (e) => console.warn(e.id, 'is behind and has nowhere to go
|
|
|
26
26
|
balancer.on('arrived', (e) => console.log(e.id, 'moved in', e.costMs, 'ms'));
|
|
27
27
|
```
|
|
28
28
|
|
|
29
|
-
- **Backlog:** a backlog
|
|
30
|
-
- **Fewer moves:** a move for load is made only if the moves it starts are predicted to bring the pool back under its line, and each isolate gets at most 3 of them in 5 minutes. Spike, crash and bisect moves protect the others and aren't limited.
|
|
29
|
+
- **Backlog:** a backlog that rose over two 10 s windows in a row, and is above 0, moves the isolate somewhere with more CPU: a pool of its own if the cap allows, else a pool with less load than its pool-mates. One such move per fresh 10 s window, so a group's members move one at a time. A report older than 10 s counts as none. With nowhere to go, it emits `behind` (at most every 30 s).
|
|
30
|
+
- **Fewer moves:** a move for load is made only if the moves it starts are predicted to bring the pool back under its line, and each isolate gets at most 3 of them in 5 minutes. Spike, crash and bisect moves protect the others and aren't limited. A never-measured isolate's way out of the quarantine, where it starts, doesn't count.
|
|
31
31
|
- **Spikes:** a first long task is only reported (`spike`, action `noted`); a second from the same isolate within 60 s sends it to quarantine. Its own fatal error still does at once.
|
|
32
|
-
- **Packing back:**
|
|
32
|
+
- **Packing back:** a pool whose isolates have all been steady for 60 s empties into another of the balancer's pools if both together stay under 40 % of a core, one isolate per pack and at most one pack every 60 s, which frees a worker once the last one is in. Turn it off with `pack: { enabled: false }`.
|
|
33
33
|
- **Move cost:** `arrived` follows each `move` with `costMs`, the time from stopping the isolate to its first beat in the new pool.
|
|
34
34
|
|
|
35
|
+
And when its busy time misses work, or the kernel gives up on an isolate (M3j):
|
|
36
|
+
|
|
37
|
+
```ts
|
|
38
|
+
// Work timed by the app, as a running total since the isolate started: for slices resumed after `await`.
|
|
39
|
+
balancer.report('task-1', { backlog: queue.length, busyMs: workMsSoFar });
|
|
40
|
+
balancer.on('failed', (e) => console.warn(e.id, 'was given up on in', e.pool, e.reason));
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
- **Reported busy time:** the balancer takes the total's difference over its 3 s window and uses the larger of that and the beats' `busyMs`, wherever it reads a load.
|
|
44
|
+
- **Packing** goes ahead while a backlog holds steady; only a backlog that rose over the 60 s window blocks it.
|
|
45
|
+
- **The quarantine:** an offender's quiet window is capped at 2 minutes, and offences wear off with each quiet window back in a pool. A bisect starts only with a quarantine pool of its own, never alongside offenders; with none free, the pool's members are `behind`.
|
|
46
|
+
- **Given up:** an isolate past the kernel's restart limit is `failed`. It leaves its pool's load and any bisect, and no move restarts it; after `kernel.restart(id)` it's placed again as new.
|
|
47
|
+
|
|
35
48
|
One balancer per kernel at a time (`dispose()` frees the kernel for another); its pools are named `balancer-…`, which the host reserves. The API, the triggers, the quarantine and what `busyMs` measures are in the [balancer guide](https://github.com/isoloop/isoloop/blob/main/docs/guide/balancer.md).
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The part of the kernel the balancer uses: the public API only (`spawn`,
|
|
3
|
-
* `stop`, `pools`, and the `isolate:beat
|
|
4
|
-
* `@isoloop/host`'s `Kernel` fits it as is.
|
|
3
|
+
* `stop`, `pools`, and the `isolate:beat`, `isolate:crash` and
|
|
4
|
+
* `isolate:failed` events), so `@isoloop/host`'s `Kernel` fits it as is.
|
|
5
5
|
*/
|
|
6
6
|
interface KernelLike<M extends ManifestLike = ManifestLike> {
|
|
7
7
|
spawn(manifest: M): unknown;
|
|
@@ -10,6 +10,8 @@ interface KernelLike<M extends ManifestLike = ManifestLike> {
|
|
|
10
10
|
on(event: "isolate:crash", fn: (e: CrashLike) => void): () => void;
|
|
11
11
|
/** Every heartbeat as it arrives, once per member of a pool (ADR 0017). */
|
|
12
12
|
on(event: "isolate:beat", fn: (e: BeatLike) => void): () => void;
|
|
13
|
+
/** An isolate past the kernel's restart limit: it stays down until `kernel.restart` (M3j). */
|
|
14
|
+
on(event: "isolate:failed", fn: (e: FailedLike) => void): () => void;
|
|
13
15
|
/**
|
|
14
16
|
* Only for kernels not made by `createKernel` (a simulated one in tests):
|
|
15
17
|
* the claim of the `balancer-*` pools, as the host's `claimPools` gives it.
|
|
@@ -51,6 +53,12 @@ type CrashLike = {
|
|
|
51
53
|
reason: string;
|
|
52
54
|
pool?: string;
|
|
53
55
|
};
|
|
56
|
+
/** The fields of the kernel's `isolate:failed` event the balancer reads (M3j). */
|
|
57
|
+
type FailedLike = {
|
|
58
|
+
id: string;
|
|
59
|
+
reason: string;
|
|
60
|
+
pool?: string;
|
|
61
|
+
};
|
|
54
62
|
/**
|
|
55
63
|
* Where the balancer keeps each isolate's measured load between page loads,
|
|
56
64
|
* so a reload places from history (M3h, Q17). The app supplies it; the
|
|
@@ -95,11 +103,14 @@ type BalancerOptions = {
|
|
|
95
103
|
};
|
|
96
104
|
/**
|
|
97
105
|
* Leaving quarantine: an isolate there only to be measured leaves once it
|
|
98
|
-
* has 3 s of beats there
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
* (
|
|
102
|
-
*
|
|
106
|
+
* has 3 s of beats there, or, if a bisect moved it there, once that bisect
|
|
107
|
+
* ends. An offender leaves after `steadyMs` (default 30,000 ms) with no
|
|
108
|
+
* spike and no crash, whatever its load; each offence doubles the window
|
|
109
|
+
* (30, 60, 120 s), capped at 2 minutes (or `steadyMs`, if longer). Back in
|
|
110
|
+
* a balancer pool, an offender loses one offence for each quiet window it
|
|
111
|
+
* completes there (M3j). After `maxReturns` returns (default 3) the next
|
|
112
|
+
* offence keeps it apart for good. Either goes where the placement rule
|
|
113
|
+
* finds room, or stays, flagged `atCap`, until there is.
|
|
103
114
|
*/
|
|
104
115
|
quarantine?: {
|
|
105
116
|
steadyMs?: number;
|
|
@@ -107,36 +118,63 @@ type BalancerOptions = {
|
|
|
107
118
|
};
|
|
108
119
|
loads?: LoadStore;
|
|
109
120
|
/**
|
|
110
|
-
* Packing back (M3i
|
|
111
|
-
* spike, crash, lagging or full pool,
|
|
112
|
-
*
|
|
113
|
-
*
|
|
121
|
+
* Packing back (M3i; whole pools since M3j): a pool whose members have
|
|
122
|
+
* all been steady for 60 s (no spike, crash, lagging or full pool, and a
|
|
123
|
+
* backlog no higher than 60 s before) empties into another of the
|
|
124
|
+
* balancer's pools if both together stay under 40 % of a core, one member
|
|
125
|
+
* per pack, freeing a worker once the last is in. At most one pack every
|
|
126
|
+
* 60 s. On by default.
|
|
114
127
|
*/
|
|
115
128
|
pack?: {
|
|
116
129
|
enabled?: boolean;
|
|
117
130
|
};
|
|
118
131
|
};
|
|
119
|
-
/**
|
|
132
|
+
/**
|
|
133
|
+
* What the app tells the balancer about an isolate (`report`): its queued
|
|
134
|
+
* work, its own measured busy time, or both. At least one must be given.
|
|
135
|
+
*/
|
|
120
136
|
type Report = {
|
|
121
137
|
/**
|
|
122
138
|
* Work items waiting for the isolate, in any unit the app likes, as long as
|
|
123
139
|
* it's the same one each time. For a consumer group, report the group's
|
|
124
140
|
* backlog on every member.
|
|
125
141
|
*/
|
|
126
|
-
backlog
|
|
142
|
+
backlog?: number;
|
|
143
|
+
/**
|
|
144
|
+
* Milliseconds of work the app measured for the isolate since it started:
|
|
145
|
+
* a running total, so a late or missed report still counts (M3j). For work
|
|
146
|
+
* the kernel's `busyMs` misses, such as slices resumed after `await`. The
|
|
147
|
+
* balancer takes its difference over the 3 s window and uses the larger of
|
|
148
|
+
* that and the beats' figure. A total that goes down is taken as a restart
|
|
149
|
+
* of the count.
|
|
150
|
+
*/
|
|
151
|
+
busyMs?: number;
|
|
127
152
|
};
|
|
128
|
-
|
|
153
|
+
/**
|
|
154
|
+
* `failed`: the kernel gave up on the isolate (past its restart limit, M3j).
|
|
155
|
+
* The balancer never moves or restarts it; after `kernel.restart(id)` it's
|
|
156
|
+
* placed again as new.
|
|
157
|
+
*/
|
|
158
|
+
type PlacementState = "placed" | "quarantined" | "kept-apart" | "moving" | "failed";
|
|
129
159
|
type Placement = {
|
|
130
160
|
id: string;
|
|
131
161
|
pool: string;
|
|
132
162
|
state: PlacementState;
|
|
133
|
-
/**
|
|
163
|
+
/**
|
|
164
|
+
* Measured share of one core, 0–100: the larger of the beats' `busyMs` and
|
|
165
|
+
* the app's reported `busyMs` over the last 3 s; `undefined` until measured.
|
|
166
|
+
*/
|
|
134
167
|
loadPct: number | undefined;
|
|
168
|
+
/** The share of one core from the app's `busyMs` reports alone, while they're fresh (M3j). */
|
|
169
|
+
reportedPct?: number;
|
|
135
170
|
/** No pool had room and the worker cap was reached: placed anyway, or left where it is. */
|
|
136
171
|
atCap?: true;
|
|
137
172
|
/** The latest backlog `report`, while it's fresh (under 10 s old). */
|
|
138
173
|
backlog?: number;
|
|
139
|
-
/**
|
|
174
|
+
/**
|
|
175
|
+
* Its backlog is rising, or its pool needs a bisect, and the balancer has
|
|
176
|
+
* nowhere to give it (`behind`).
|
|
177
|
+
*/
|
|
140
178
|
behind?: true;
|
|
141
179
|
/**
|
|
142
180
|
* In one of the balancer's pools, but not placed by it. The host reserves
|
|
@@ -182,14 +220,25 @@ type SpikeEvent = {
|
|
|
182
220
|
};
|
|
183
221
|
/**
|
|
184
222
|
* A member's reported backlog keeps rising, but it already has its pool to
|
|
185
|
-
* itself or the balancer has no worker or pool to give it (M3i).
|
|
186
|
-
*
|
|
223
|
+
* itself or the balancer has no worker or pool to give it (M3i). Also when
|
|
224
|
+
* its pool's trouble needs a bisect and no quarantine pool is free for one
|
|
225
|
+
* (M3j). Emitted when it first happens, then at most every 30 s while it
|
|
226
|
+
* lasts. `backlog` is the latest fresh report, or 0.
|
|
187
227
|
*/
|
|
188
228
|
type BehindEvent = {
|
|
189
229
|
id: string;
|
|
190
230
|
pool: string;
|
|
191
231
|
backlog: number;
|
|
192
232
|
};
|
|
233
|
+
/**
|
|
234
|
+
* The kernel gave up on a member (`isolate:failed`, M3j). It drops out of its
|
|
235
|
+
* pool's load and any bisect, and no move restarts it.
|
|
236
|
+
*/
|
|
237
|
+
type FailedEvent = {
|
|
238
|
+
id: string;
|
|
239
|
+
pool: string;
|
|
240
|
+
reason: string;
|
|
241
|
+
};
|
|
193
242
|
/** An isolate the balancer didn't place turned up in one of its pools (see `Placement.foreign`). */
|
|
194
243
|
type ForeignEvent = {
|
|
195
244
|
id: string;
|
|
@@ -200,6 +249,7 @@ type BalancerEvents = {
|
|
|
200
249
|
arrived: ArrivedEvent;
|
|
201
250
|
spike: SpikeEvent;
|
|
202
251
|
behind: BehindEvent;
|
|
252
|
+
failed: FailedEvent;
|
|
203
253
|
foreign: ForeignEvent;
|
|
204
254
|
};
|
|
205
255
|
/**
|
|
@@ -221,11 +271,14 @@ interface Balancer<M extends ManifestLike = ManifestLike> {
|
|
|
221
271
|
/** Leaves `id` where it is until the returned function is called (a widget open for editing). */
|
|
222
272
|
hold(id: string): () => void;
|
|
223
273
|
/**
|
|
224
|
-
* The app's view of `id`'s queued work (M3i)
|
|
225
|
-
*
|
|
226
|
-
*
|
|
227
|
-
*
|
|
228
|
-
*
|
|
274
|
+
* The app's view of `id`'s queued work (M3i) and of its own busy time (M3j).
|
|
275
|
+
* A backlog that rose over two 10 s windows in a row (M3j), and is above 0,
|
|
276
|
+
* moves the isolate somewhere with more CPU. A reported `busyMs` total
|
|
277
|
+
* counts toward its load wherever the balancer reads one. A report older
|
|
278
|
+
* than 10 s counts as none, so report at least every few seconds. Throws a
|
|
279
|
+
* `TypeError` for a report with neither field, or a field that isn't a
|
|
280
|
+
* finite number of 0 or more. Ignored for isolates the balancer didn't place
|
|
281
|
+
* and ones the kernel gave up on.
|
|
229
282
|
*/
|
|
230
283
|
report(id: string, report: Report): void;
|
|
231
284
|
on<E extends keyof BalancerEvents>(event: E, fn: (e: BalancerEvents[E]) => void): () => void;
|
|
@@ -247,4 +300,4 @@ interface Balancer<M extends ManifestLike = ManifestLike> {
|
|
|
247
300
|
* `kernel.spawn` are never touched, and don't count toward `maxWorkers`.
|
|
248
301
|
*/
|
|
249
302
|
export declare function createBalancer<M extends ManifestLike>(kernel: KernelLike<M>, opts?: BalancerOptions): Balancer<M>;
|
|
250
|
-
export type { ArrivedEvent, Balancer, BalancerEvents, BalancerOptions, BeatLike, BehindEvent, CrashLike, ForeignEvent, KernelLike, LoadStore, ManifestLike, MoveEvent, MoveReason, Placement, PlacementState, PoolLike, Report, SpikeEvent };
|
|
303
|
+
export type { ArrivedEvent, Balancer, BalancerEvents, BalancerOptions, BeatLike, BehindEvent, CrashLike, FailedEvent, FailedLike, ForeignEvent, KernelLike, LoadStore, ManifestLike, MoveEvent, MoveReason, Placement, PlacementState, PoolLike, Report, SpikeEvent };
|
package/dist/index.js
CHANGED
|
@@ -38,6 +38,8 @@ const POOLS = "balancer";
|
|
|
38
38
|
const QUARANTINES = "balancer-quarantine";
|
|
39
39
|
const QUARANTINE = `${QUARANTINES}-1`;
|
|
40
40
|
const now = () => performance.now();
|
|
41
|
+
/** A report's figure: a finite number, 0 or more. */
|
|
42
|
+
const valid = (x) => typeof x === "number" && Number.isFinite(x) && x >= 0;
|
|
41
43
|
/**
|
|
42
44
|
* `busyMs` is approximate: shared framework work is split between members and
|
|
43
45
|
* promise continuations go unattributed. When the members' busy time explains
|
|
@@ -72,6 +74,8 @@ const MOVE_BUDGET_MS = 3e5;
|
|
|
72
74
|
const PACK_STEADY_MS = 6e4;
|
|
73
75
|
const PACK_PCT = 40;
|
|
74
76
|
const PACK_EVERY_MS = 6e4;
|
|
77
|
+
/** An offender's quiet window doubles with each offence up to this, or `steadyMs` if longer (M3j). */
|
|
78
|
+
const QUIET_CAP_MS = 12e4;
|
|
75
79
|
/** Worker crash reasons whose culprit the kernel can't name: every member of the pool crashes. */
|
|
76
80
|
const POOL_KILL = /* @__PURE__ */ new Set(["unresponsive", "worker-error"]);
|
|
77
81
|
const defaultMaxWorkers = () => {
|
|
@@ -121,6 +125,7 @@ function createBalancer(kernel, opts = {}) {
|
|
|
121
125
|
arrived: /* @__PURE__ */ new Set(),
|
|
122
126
|
spike: /* @__PURE__ */ new Set(),
|
|
123
127
|
behind: /* @__PURE__ */ new Set(),
|
|
128
|
+
failed: /* @__PURE__ */ new Set(),
|
|
124
129
|
foreign: /* @__PURE__ */ new Set()
|
|
125
130
|
};
|
|
126
131
|
/** Beats of isolates the balancer didn't place, for `measure` (previews included). */
|
|
@@ -143,6 +148,8 @@ function createBalancer(kernel, opts = {}) {
|
|
|
143
148
|
/** The last backlog move: rising is judged only on reports after it. */
|
|
144
149
|
let lastBacklogMoveAt = -Infinity;
|
|
145
150
|
let lastPackAt = -Infinity;
|
|
151
|
+
/** A pool being emptied into another by packing, one member at a time (M3j). */
|
|
152
|
+
let merging;
|
|
146
153
|
let disposed = false;
|
|
147
154
|
const pool = (name) => {
|
|
148
155
|
let p = pools.get(name);
|
|
@@ -160,7 +167,11 @@ function createBalancer(kernel, opts = {}) {
|
|
|
160
167
|
}
|
|
161
168
|
return p;
|
|
162
169
|
};
|
|
163
|
-
|
|
170
|
+
/** Running in its pool: not mid-move, and not given up on by the kernel. */
|
|
171
|
+
const live = (m) => m.state !== "moving" && m.state !== "failed";
|
|
172
|
+
/** Its members, moving ones included: the ones a pack has to move to empty it (M3j). */
|
|
173
|
+
const inPool = (name) => [...members.values()].filter((m) => m.pool === name && m.state !== "failed");
|
|
174
|
+
const membersOf = (name) => [...members.values()].filter((m) => m.pool === name && live(m));
|
|
164
175
|
const changed = (name) => {
|
|
165
176
|
const p = pool(name);
|
|
166
177
|
p.lags = [];
|
|
@@ -168,8 +179,28 @@ function createBalancer(kernel, opts = {}) {
|
|
|
168
179
|
p.beat = [];
|
|
169
180
|
p.since = now();
|
|
170
181
|
};
|
|
182
|
+
/**
|
|
183
|
+
* The app's reported busy time (M3j) as a share of one core: the running
|
|
184
|
+
* total's difference over about a window, from a fresh report back to one
|
|
185
|
+
* taken at least 90 % of a window before it.
|
|
186
|
+
*/
|
|
187
|
+
const reportedOf = (m, t) => {
|
|
188
|
+
const latest = m.busyReports.at(-1);
|
|
189
|
+
if (!latest || t - latest.at > STALE_MS) return void 0;
|
|
190
|
+
const start = m.busyReports.findLast((r) => r.at <= latest.at - windowMs * .9);
|
|
191
|
+
if (!start) return void 0;
|
|
192
|
+
const ms = latest.at - start.at;
|
|
193
|
+
return Math.min(100, (latest.total - start.total) / ms * 100);
|
|
194
|
+
};
|
|
195
|
+
/** The beats' and the report's measures over the last window, whichever is larger. */
|
|
196
|
+
const measured = (m, t) => {
|
|
197
|
+
const beat = average(m.beats, windowMs, t);
|
|
198
|
+
const reported = reportedOf(m, t);
|
|
199
|
+
if (beat === void 0) return reported;
|
|
200
|
+
return reported === void 0 ? beat : Math.max(beat, reported);
|
|
201
|
+
};
|
|
171
202
|
/** Measured load over the last window, or the last one known. */
|
|
172
|
-
const loadOf = (m) =>
|
|
203
|
+
const loadOf = (m) => measured(m, now()) ?? m.knownLoad;
|
|
173
204
|
const poolLoad = (name) => membersOf(name).reduce((s, m) => s + (loadOf(m) ?? 0), 0);
|
|
174
205
|
const windowFull = (name) => {
|
|
175
206
|
const p = pool(name);
|
|
@@ -182,7 +213,7 @@ function createBalancer(kernel, opts = {}) {
|
|
|
182
213
|
* app runs itself don't count (ADR 0017, amended).
|
|
183
214
|
*/
|
|
184
215
|
const ownWorkers = () => kernel.pools().filter((p) => p.running && isOurs(p.name) && p.name !== QUARANTINE).length;
|
|
185
|
-
const ownPools = () => [...new Set([...members.values()].map((m) => m.pool))].filter((n) => !isQuarantine(n));
|
|
216
|
+
const ownPools = () => [...new Set([...members.values()].filter((m) => m.state !== "failed").map((m) => m.pool))].filter((n) => !isQuarantine(n));
|
|
186
217
|
/** Room for one more worker of the balancer's own. */
|
|
187
218
|
const roomForPool = () => ownWorkers() + 1 <= maxWorkers;
|
|
188
219
|
const freshName = (prefix, first) => {
|
|
@@ -250,8 +281,11 @@ function createBalancer(kernel, opts = {}) {
|
|
|
250
281
|
m.state = "moving";
|
|
251
282
|
m.beats = [];
|
|
252
283
|
m.lastBeatAt = void 0;
|
|
253
|
-
m.moveTimes.push(startedAt);
|
|
284
|
+
if (!m.unmeasured) m.moveTimes.push(startedAt);
|
|
285
|
+
m.unmeasured = false;
|
|
254
286
|
m.backlogs = [];
|
|
287
|
+
m.backlogHistory = [];
|
|
288
|
+
m.busyReports = [];
|
|
255
289
|
m.behind = false;
|
|
256
290
|
delete m.arrival;
|
|
257
291
|
m.pool = to;
|
|
@@ -293,7 +327,10 @@ function createBalancer(kernel, opts = {}) {
|
|
|
293
327
|
/** Sends `m` to quarantine for misbehaving (`offence`) or to bisect. */
|
|
294
328
|
const quarantine = (m, reason, offence) => {
|
|
295
329
|
m.lastSpikeAt = now();
|
|
296
|
-
if (offence)
|
|
330
|
+
if (offence) {
|
|
331
|
+
m.offences++;
|
|
332
|
+
m.unmeasured = false;
|
|
333
|
+
}
|
|
297
334
|
const state = stateFor(m);
|
|
298
335
|
m.quarantinedAt = now();
|
|
299
336
|
if (isQuarantine(m.pool)) {
|
|
@@ -302,7 +339,27 @@ function createBalancer(kernel, opts = {}) {
|
|
|
302
339
|
}
|
|
303
340
|
move(m, QUARANTINE, reason, state);
|
|
304
341
|
};
|
|
305
|
-
const movable = (m) => m
|
|
342
|
+
const movable = (m) => live(m) && m.holds === 0;
|
|
343
|
+
/**
|
|
344
|
+
* A member the kernel gave up on beats again: the app restarted it
|
|
345
|
+
* (`kernel.restart`). It's placed again as new (M3j): no offences, no
|
|
346
|
+
* history, and its pool's rules from here.
|
|
347
|
+
*/
|
|
348
|
+
const revive = (m) => {
|
|
349
|
+
m.state = isQuarantine(m.pool) ? "quarantined" : "placed";
|
|
350
|
+
m.offences = 0;
|
|
351
|
+
m.quarantinedAt = isQuarantine(m.pool) ? now() : -Infinity;
|
|
352
|
+
m.lastSpikeAt = m.spikeNotedAt = m.decayAt = m.risingAt = m.stuckAt = -Infinity;
|
|
353
|
+
m.arrivedAt = now();
|
|
354
|
+
m.moveTimes = [];
|
|
355
|
+
m.unmeasured = false;
|
|
356
|
+
m.beats = [];
|
|
357
|
+
m.lastBeatAt = void 0;
|
|
358
|
+
m.backlogs = [];
|
|
359
|
+
m.backlogHistory = [];
|
|
360
|
+
m.busyReports = [];
|
|
361
|
+
changed(m.pool);
|
|
362
|
+
};
|
|
306
363
|
const trim = (beats, at) => {
|
|
307
364
|
const cut = at - windowMs;
|
|
308
365
|
while ((beats[0]?.at ?? Infinity) <= cut) beats.shift();
|
|
@@ -338,6 +395,7 @@ function createBalancer(kernel, opts = {}) {
|
|
|
338
395
|
return;
|
|
339
396
|
}
|
|
340
397
|
if (m.state === "moving" || e.pool !== m.pool) return;
|
|
398
|
+
if (m.state === "failed") revive(m);
|
|
341
399
|
if (m.arrival) {
|
|
342
400
|
const { from, to, reason, at } = m.arrival;
|
|
343
401
|
delete m.arrival;
|
|
@@ -454,9 +512,21 @@ function createBalancer(kernel, opts = {}) {
|
|
|
454
512
|
for (const id of foreign.keys()) if (!present.has(id)) foreign.delete(id);
|
|
455
513
|
};
|
|
456
514
|
/**
|
|
515
|
+
* A quarantine pool a bisect from `name` can have to itself (M3j): the
|
|
516
|
+
* first quarantine if it holds only members this bisect moved (or none), else a
|
|
517
|
+
* new quarantine pool if the cap allows. Never one that holds offenders.
|
|
518
|
+
*/
|
|
519
|
+
const bisectPool = (name) => {
|
|
520
|
+
const free = membersOf(QUARANTINE).every((o) => o.offences === 0 && bisect?.moved.has(o.manifest.id) === true);
|
|
521
|
+
if (!isQuarantine(name) && free) return QUARANTINE;
|
|
522
|
+
return roomForPool() ? freshName(QUARANTINES, 2) : void 0;
|
|
523
|
+
};
|
|
524
|
+
/**
|
|
457
525
|
* Finds the member behind a pool's trouble when the kernel or `busyMs`
|
|
458
526
|
* can't name it (Q20): the heavier half of the suspects moves, and
|
|
459
|
-
* whichever half acts up again holds the culprit.
|
|
527
|
+
* whichever half acts up again holds the culprit. Only into a quarantine
|
|
528
|
+
* pool of its own (M3j); with none free, the pool stays as it is and its
|
|
529
|
+
* members are `behind`.
|
|
460
530
|
*/
|
|
461
531
|
const search = (name, reason) => {
|
|
462
532
|
const here = membersOf(name).filter((m) => m.holds === 0);
|
|
@@ -477,18 +547,25 @@ function createBalancer(kernel, opts = {}) {
|
|
|
477
547
|
return;
|
|
478
548
|
}
|
|
479
549
|
const sorted = suspects.map((id) => members.get(id)).filter((m) => m !== void 0).toSorted((a, b) => (loadOf(b) ?? 0) - (loadOf(a) ?? 0));
|
|
550
|
+
const to = bisectPool(name);
|
|
551
|
+
if (to === void 0) {
|
|
552
|
+
const t = now();
|
|
553
|
+
for (const m of here) {
|
|
554
|
+
m.stuckAt = t;
|
|
555
|
+
behind(m, t);
|
|
556
|
+
}
|
|
557
|
+
return;
|
|
558
|
+
}
|
|
480
559
|
const half = sorted.slice(0, Math.ceil(sorted.length / 2));
|
|
481
560
|
bisect = {
|
|
482
561
|
suspects: new Set(suspects),
|
|
483
562
|
moved: /* @__PURE__ */ new Set([...bisect?.moved ?? [], ...half.map((m) => m.manifest.id)]),
|
|
484
563
|
at: now()
|
|
485
564
|
};
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
for (const m of half) if (to === QUARANTINE) {
|
|
489
|
-
m.quarantinedAt = now();
|
|
565
|
+
for (const m of half) {
|
|
566
|
+
if (!isQuarantine(name)) m.quarantinedAt = now();
|
|
490
567
|
move(m, to, "bisect", stateFor(m));
|
|
491
|
-
}
|
|
568
|
+
}
|
|
492
569
|
};
|
|
493
570
|
/**
|
|
494
571
|
* The moves that would bring pool `name` back under (M3i): the heaviest
|
|
@@ -529,15 +606,20 @@ function createBalancer(kernel, opts = {}) {
|
|
|
529
606
|
/** The latest report, if fresh. */
|
|
530
607
|
const backlogNow = (m, t) => m.backlog && t - m.backlog.at <= STALE_MS ? m.backlog.value : void 0;
|
|
531
608
|
/**
|
|
532
|
-
* Rising (M3i): the latest fresh report is above 0
|
|
533
|
-
* at least a window before it,
|
|
534
|
-
*
|
|
609
|
+
* Rising (M3i; two windows since M3j): the latest fresh report is above 0
|
|
610
|
+
* and higher than one taken at least a window before it, which is itself
|
|
611
|
+
* higher than one taken at least a window before that, all since it last
|
|
612
|
+
* moved and since the last backlog move. A queue at a steady rate that
|
|
613
|
+
* jumps once and holds isn't falling behind.
|
|
535
614
|
*/
|
|
536
615
|
const rising = (m, t) => {
|
|
537
616
|
const latest = m.backlog;
|
|
538
617
|
if (backlogNow(m, t) === void 0 || !latest || latest.value <= 0) return false;
|
|
539
|
-
const
|
|
540
|
-
|
|
618
|
+
const before = (at) => m.backlogs.findLast((b) => b.at >= lastBacklogMoveAt && b.at <= at - BACKLOG_WINDOW_MS);
|
|
619
|
+
const mid = before(latest.at);
|
|
620
|
+
if (mid === void 0 || latest.value <= mid.value) return false;
|
|
621
|
+
const early = before(mid.at);
|
|
622
|
+
return early !== void 0 && mid.value > early.value;
|
|
541
623
|
};
|
|
542
624
|
/**
|
|
543
625
|
* Somewhere with more CPU for `m` (M3i): a pool of its own if the cap
|
|
@@ -575,7 +657,7 @@ function createBalancer(kernel, opts = {}) {
|
|
|
575
657
|
const order = [...members.values()].filter((m) => {
|
|
576
658
|
const r = m.state === "placed" && !isQuarantine(m.pool) && rising(m, t);
|
|
577
659
|
if (r) m.risingAt = t;
|
|
578
|
-
else m.behind = false;
|
|
660
|
+
else if (t - m.stuckAt >= windowMs) m.behind = false;
|
|
579
661
|
return r;
|
|
580
662
|
}).toSorted((a, b) => poolLoad(b.pool) - poolLoad(a.pool) || (loadOf(b) ?? 0) - (loadOf(a) ?? 0));
|
|
581
663
|
for (const m of order) {
|
|
@@ -592,29 +674,70 @@ function createBalancer(kernel, opts = {}) {
|
|
|
592
674
|
}
|
|
593
675
|
};
|
|
594
676
|
/**
|
|
595
|
-
*
|
|
596
|
-
*
|
|
597
|
-
*
|
|
598
|
-
*
|
|
599
|
-
*
|
|
677
|
+
* Its backlog rose over the last `PACK_STEADY_MS` (M3j): the latest fresh
|
|
678
|
+
* report is higher than the one at or just before the window's start (or
|
|
679
|
+
* the oldest since it moved, when it hasn't reported that long). A
|
|
680
|
+
* queue that holds steady while work arrives doesn't count, nor does a
|
|
681
|
+
* stale report or none.
|
|
682
|
+
*/
|
|
683
|
+
const roseLately = (m, t) => {
|
|
684
|
+
const latest = backlogNow(m, t);
|
|
685
|
+
if (latest === void 0) return false;
|
|
686
|
+
const start = m.backlogHistory.findLast((b) => b.at <= t - PACK_STEADY_MS) ?? m.backlogHistory[0];
|
|
687
|
+
return start !== void 0 && latest > start.value;
|
|
688
|
+
};
|
|
689
|
+
/**
|
|
690
|
+
* Packing back (M3i; whole pools since M3j): the lightest pool whose
|
|
691
|
+
* members have all been quiet for `PACK_STEADY_MS` empties into the
|
|
692
|
+
* fullest other pool that stays under `PACK_PCT` with all of it, one
|
|
693
|
+
* member per pack, which frees its worker once the last one is in. A merge
|
|
694
|
+
* that has started is finished first. Never a pool with a member on hold,
|
|
695
|
+
* out of budget, whose backlog rose over the window (M3j), or a
|
|
696
|
+
* quarantine; never into a pool that's moving or whose members' backlog
|
|
697
|
+
* rose.
|
|
600
698
|
*/
|
|
601
699
|
const pack = (t) => {
|
|
602
700
|
const quiet = (since) => t - since >= PACK_STEADY_MS;
|
|
603
|
-
const steady = (m) => quiet(Math.max(m.arrivedAt, m.lastSpikeAt, m.spikeNotedAt,
|
|
604
|
-
const
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
701
|
+
const steady = (m) => quiet(Math.max(m.arrivedAt, m.lastSpikeAt, m.spikeNotedAt, pool(m.pool).troubleAt)) && !roseLately(m, t);
|
|
702
|
+
const packable = (m) => m.state === "placed" && movable(m) && inBudget(m) && loadOf(m) !== void 0 && steady(m);
|
|
703
|
+
const sources = ownPools().filter((n) => isOurs(n)).map((n) => ({
|
|
704
|
+
n,
|
|
705
|
+
all: inPool(n),
|
|
706
|
+
load: poolLoad(n)
|
|
707
|
+
})).filter((c) => c.all.length > 0 && c.all.every(packable)).toSorted((a, b) => Number(b.n === merging?.from) - Number(a.n === merging?.from) || a.load - b.load);
|
|
708
|
+
for (const from of sources) {
|
|
709
|
+
const into = ownPools().filter((n) => n !== from.n && isOurs(n) && !recentlySpiked(n) && quiet(pool(n).troubleAt)).filter((n) => inPool(n).every((o) => o.state !== "moving" && !roseLately(o, t))).map((n) => ({
|
|
608
710
|
n,
|
|
609
711
|
load: poolLoad(n)
|
|
610
|
-
})).filter((c) => c.load + load < PACK_PCT).toSorted((a, b) => b.load - a.load)[0];
|
|
712
|
+
})).filter((c) => c.load + from.load < PACK_PCT).toSorted((a, b) => Number(b.n === merging?.into) - Number(a.n === merging?.into) || b.load - a.load)[0];
|
|
611
713
|
if (!into) continue;
|
|
714
|
+
const m = from.all.toSorted((a, b) => (loadOf(a) ?? 0) - (loadOf(b) ?? 0))[0];
|
|
715
|
+
if (!m) continue;
|
|
612
716
|
lastPackAt = t;
|
|
717
|
+
merging = from.all.length > 1 ? {
|
|
718
|
+
from: from.n,
|
|
719
|
+
into: into.n
|
|
720
|
+
} : void 0;
|
|
613
721
|
move(m, into.n, "pack", "placed");
|
|
614
722
|
return;
|
|
615
723
|
}
|
|
616
724
|
};
|
|
617
|
-
|
|
725
|
+
/** The quiet window: doubled per offence, capped at 2 minutes (or `steadyMs`, if longer; M3j). */
|
|
726
|
+
const steadyFor = (m) => Math.min(Math.max(steadyMs, QUIET_CAP_MS), steadyMs * 2 ** Math.max(0, m.offences - 1));
|
|
727
|
+
/**
|
|
728
|
+
* Offences decay (M3j): an offender back in a balancer pool loses one for
|
|
729
|
+
* each quiet window it completes there, so one that stays quiet is back to
|
|
730
|
+
* the shortest window. One that offends again soon after leaving still
|
|
731
|
+
* gets a longer one, and `kept-apart` stays apart.
|
|
732
|
+
*/
|
|
733
|
+
const decay = (t) => {
|
|
734
|
+
for (const m of members.values()) {
|
|
735
|
+
if (m.state !== "placed" || m.offences === 0) continue;
|
|
736
|
+
if (t - Math.max(m.decayAt, m.lastSpikeAt, m.spikeNotedAt) < steadyFor(m)) continue;
|
|
737
|
+
m.offences--;
|
|
738
|
+
m.decayAt = t;
|
|
739
|
+
}
|
|
740
|
+
};
|
|
618
741
|
const decide = () => {
|
|
619
742
|
const t = now();
|
|
620
743
|
checkForeign();
|
|
@@ -655,8 +778,9 @@ function createBalancer(kernel, opts = {}) {
|
|
|
655
778
|
} else first.atCap = true;
|
|
656
779
|
}
|
|
657
780
|
backlogs(t);
|
|
781
|
+
decay(t);
|
|
658
782
|
for (const m of members.values()) {
|
|
659
|
-
if (m.state !== "quarantined" || !movable(m) || !inBudget(m)) continue;
|
|
783
|
+
if (m.state !== "quarantined" || !movable(m) || !(m.unmeasured || inBudget(m))) continue;
|
|
660
784
|
const fresh = average(m.beats, windowMs, t);
|
|
661
785
|
if (m.offences === 0) {
|
|
662
786
|
if (fresh === void 0 || bisect?.moved.has(m.manifest.id)) continue;
|
|
@@ -669,6 +793,7 @@ function createBalancer(kernel, opts = {}) {
|
|
|
669
793
|
continue;
|
|
670
794
|
}
|
|
671
795
|
m.atCap = false;
|
|
796
|
+
m.decayAt = t;
|
|
672
797
|
move(m, to.pool, "steady", "placed");
|
|
673
798
|
}
|
|
674
799
|
if (packing && t - lastPackAt >= PACK_EVERY_MS) pack(t);
|
|
@@ -679,22 +804,53 @@ function createBalancer(kernel, opts = {}) {
|
|
|
679
804
|
};
|
|
680
805
|
const onCrash = (e) => {
|
|
681
806
|
const m = members.get(e.id);
|
|
682
|
-
if (!m || m
|
|
807
|
+
if (!m || !live(m)) return;
|
|
683
808
|
if (POOL_KILL.has(e.reason) && e.pool !== void 0) killed.add(e.pool);
|
|
684
809
|
else if (e.reason === "error") {
|
|
685
810
|
if (m.holds > 0) m.lastSpikeAt = now();
|
|
686
811
|
else quarantine(m, "crash", true);
|
|
687
812
|
}
|
|
688
813
|
};
|
|
814
|
+
/**
|
|
815
|
+
* The kernel gave up on a member (M3j): it leaves its pool's load, the
|
|
816
|
+
* quarantine's count and any bisect, and stays where it is until the app
|
|
817
|
+
* restarts it. A member mid-move was stopped by the balancer itself; its
|
|
818
|
+
* respawn is a fresh start.
|
|
819
|
+
*/
|
|
820
|
+
const onFailed = (e) => {
|
|
821
|
+
const m = members.get(e.id);
|
|
822
|
+
if (!m || !live(m)) return;
|
|
823
|
+
m.state = "failed";
|
|
824
|
+
m.beats = [];
|
|
825
|
+
m.lastBeatAt = void 0;
|
|
826
|
+
m.atCap = false;
|
|
827
|
+
m.behind = false;
|
|
828
|
+
m.busyReports = [];
|
|
829
|
+
delete m.arrival;
|
|
830
|
+
bisect?.suspects.delete(e.id);
|
|
831
|
+
bisect?.moved.delete(e.id);
|
|
832
|
+
changed(m.pool);
|
|
833
|
+
emit("failed", {
|
|
834
|
+
id: e.id,
|
|
835
|
+
pool: m.pool,
|
|
836
|
+
reason: e.reason
|
|
837
|
+
});
|
|
838
|
+
};
|
|
689
839
|
const timer = setInterval(decide, DECIDE_MS);
|
|
690
|
-
const offs = [
|
|
840
|
+
const offs = [
|
|
841
|
+
kernel.on("isolate:crash", onCrash),
|
|
842
|
+
kernel.on("isolate:beat", onBeat),
|
|
843
|
+
kernel.on("isolate:failed", onFailed)
|
|
844
|
+
];
|
|
691
845
|
const placement = (m) => {
|
|
692
846
|
const p = {
|
|
693
847
|
id: m.manifest.id,
|
|
694
848
|
pool: m.pool,
|
|
695
849
|
state: m.state,
|
|
696
|
-
loadPct: loadOf(m)
|
|
850
|
+
loadPct: m.state === "failed" ? void 0 : loadOf(m)
|
|
697
851
|
};
|
|
852
|
+
const reported = m.state === "failed" ? void 0 : reportedOf(m, now());
|
|
853
|
+
if (reported !== void 0) p.reportedPct = reported;
|
|
698
854
|
if (m.atCap) p.atCap = true;
|
|
699
855
|
const backlog = backlogNow(m, now());
|
|
700
856
|
if (backlog !== void 0) p.backlog = backlog;
|
|
@@ -724,15 +880,20 @@ function createBalancer(kernel, opts = {}) {
|
|
|
724
880
|
savedLoad: known,
|
|
725
881
|
offences: 0,
|
|
726
882
|
quarantinedAt: isQuarantine(dest.pool) ? now() : -Infinity,
|
|
883
|
+
decayAt: -Infinity,
|
|
727
884
|
lastSpikeAt: -Infinity,
|
|
728
885
|
spikeNotedAt: -Infinity,
|
|
729
886
|
holds: 0,
|
|
730
887
|
atCap: dest.atCap,
|
|
731
888
|
arrivedAt: now(),
|
|
732
889
|
moveTimes: [],
|
|
890
|
+
unmeasured: known === void 0,
|
|
733
891
|
backlogs: [],
|
|
734
892
|
backlog: void 0,
|
|
893
|
+
backlogHistory: [],
|
|
894
|
+
busyReports: [],
|
|
735
895
|
risingAt: -Infinity,
|
|
896
|
+
stuckAt: -Infinity,
|
|
736
897
|
behind: false,
|
|
737
898
|
behindAt: -Infinity
|
|
738
899
|
};
|
|
@@ -775,18 +936,33 @@ function createBalancer(kernel, opts = {}) {
|
|
|
775
936
|
m.holds--;
|
|
776
937
|
};
|
|
777
938
|
},
|
|
778
|
-
report(id, { backlog }) {
|
|
779
|
-
if (
|
|
939
|
+
report(id, { backlog, busyMs }) {
|
|
940
|
+
if (backlog === void 0 && busyMs === void 0) throw new TypeError("balancer.report: give a backlog, a busyMs or both");
|
|
941
|
+
if (backlog !== void 0 && !valid(backlog)) throw new TypeError("balancer.report: backlog must be a finite number, 0 or more");
|
|
942
|
+
if (busyMs !== void 0 && !valid(busyMs)) throw new TypeError("balancer.report: busyMs must be a finite number, 0 or more");
|
|
780
943
|
const m = members.get(id);
|
|
781
|
-
if (!m) return;
|
|
944
|
+
if (!m || m.state === "failed") return;
|
|
782
945
|
const t = now();
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
946
|
+
if (backlog !== void 0) {
|
|
947
|
+
const sample = {
|
|
948
|
+
at: t,
|
|
949
|
+
value: backlog
|
|
950
|
+
};
|
|
951
|
+
m.backlog = sample;
|
|
952
|
+
m.backlogs.push(sample);
|
|
953
|
+
m.backlogHistory.push(sample);
|
|
954
|
+
while ((m.backlogs[1]?.at ?? Infinity) <= t - 3 * BACKLOG_WINDOW_MS) m.backlogs.shift();
|
|
955
|
+
while ((m.backlogHistory[1]?.at ?? Infinity) <= t - PACK_STEADY_MS) m.backlogHistory.shift();
|
|
956
|
+
}
|
|
957
|
+
if (busyMs !== void 0) {
|
|
958
|
+
if ((m.busyReports.at(-1)?.total ?? 0) > busyMs) m.busyReports = [];
|
|
959
|
+
m.busyReports.push({
|
|
960
|
+
at: t,
|
|
961
|
+
total: busyMs
|
|
962
|
+
});
|
|
963
|
+
const cut = t - windowMs - STALE_MS;
|
|
964
|
+
while ((m.busyReports[1]?.at ?? Infinity) <= cut) m.busyReports.shift();
|
|
965
|
+
}
|
|
790
966
|
},
|
|
791
967
|
measure(id) {
|
|
792
968
|
const m = members.get(id);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@isoloop/balancer",
|
|
3
|
-
"version": "0.1.0-next.
|
|
3
|
+
"version": "0.1.0-next.2",
|
|
4
4
|
"description": "Experimental automatic placement for Isoloop isolates: joins a pool or keeps apart from measured load, quarantines what blocks.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
}
|
|
21
21
|
},
|
|
22
22
|
"dependencies": {
|
|
23
|
-
"@isoloop/host": "0.1.0-next.
|
|
23
|
+
"@isoloop/host": "0.1.0-next.2"
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
26
|
"typescript": "7.0.2"
|