@isoloop/balancer 0.1.0-next.0 → 0.1.0-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -1
- package/dist/index.d.ts +75 -23
- package/dist/index.js +179 -31
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -29,7 +29,20 @@ balancer.on('arrived', (e) => console.log(e.id, 'moved in', e.costMs, 'ms'));
|
|
|
29
29
|
- **Backlog:** a backlog higher at the end of a 10 s window than at its start, and above 0, moves the isolate somewhere with more CPU: a pool of its own if the cap allows, else a pool with less load than its pool-mates. One such move per fresh 10 s window, so a group's members move one at a time. A report older than 10 s counts as none. With nowhere to go, it emits `behind` (at most every 30 s).
|
|
30
30
|
- **Fewer moves:** a move for load is made only if the moves it starts are predicted to bring the pool back under its line, and each isolate gets at most 3 of them in 5 minutes. Spike, crash and bisect moves protect the others and aren't limited.
|
|
31
31
|
- **Spikes:** a first long task is only reported (`spike`, action `noted`); a second from the same isolate within 60 s sends it to quarantine. Its own fatal error still does at once.
|
|
32
|
-
- **Packing back:** an isolate alone in its pool and
|
|
32
|
+
- **Packing back:** an isolate alone in its pool and steady for 60 s joins another of the balancer's pools if both stay under 40 % of a core, which frees a worker; at most one every 60 s. Turn it off with `pack: { enabled: false }`.
|
|
33
33
|
- **Move cost:** `arrived` follows each `move` with `costMs`, the time from stopping the isolate to its first beat in the new pool.
|
|
34
34
|
|
|
35
|
+
And when its busy time misses work, or the kernel gives up on an isolate (M3j):
|
|
36
|
+
|
|
37
|
+
```ts
|
|
38
|
+
// Work timed by the app, as a running total since the isolate started: for slices resumed after `await`.
|
|
39
|
+
balancer.report('task-1', { backlog: queue.length, busyMs: workMsSoFar });
|
|
40
|
+
balancer.on('failed', (e) => console.warn(e.id, 'was given up on in', e.pool, e.reason));
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
- **Reported busy time:** the balancer takes the total's difference over its 3 s window and uses the larger of that and the beats' `busyMs`, wherever it reads a load.
|
|
44
|
+
- **Packing** goes ahead while a backlog holds steady; only a backlog that rose over the 60 s window blocks it.
|
|
45
|
+
- **The quarantine:** an offender's quiet window is capped at 2 minutes, and offences wear off with each quiet window back in a pool. A bisect starts only with a quarantine pool of its own, never alongside offenders; with none free, the pool's members are `behind`.
|
|
46
|
+
- **Given up:** an isolate past the kernel's restart limit is `failed`. It leaves its pool's load and any bisect, and no move restarts it; after `kernel.restart(id)` it's placed again as new.
|
|
47
|
+
|
|
35
48
|
One balancer per kernel at a time (`dispose()` frees the kernel for another); its pools are named `balancer-…`, which the host reserves. The API, the triggers, the quarantine and what `busyMs` measures are in the [balancer guide](https://github.com/isoloop/isoloop/blob/main/docs/guide/balancer.md).
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The part of the kernel the balancer uses: the public API only (`spawn`,
|
|
3
|
-
* `stop`, `pools`, and the `isolate:beat
|
|
4
|
-
* `@isoloop/host`'s `Kernel` fits it as is.
|
|
3
|
+
* `stop`, `pools`, and the `isolate:beat`, `isolate:crash` and
|
|
4
|
+
* `isolate:failed` events), so `@isoloop/host`'s `Kernel` fits it as is.
|
|
5
5
|
*/
|
|
6
6
|
interface KernelLike<M extends ManifestLike = ManifestLike> {
|
|
7
7
|
spawn(manifest: M): unknown;
|
|
@@ -10,6 +10,8 @@ interface KernelLike<M extends ManifestLike = ManifestLike> {
|
|
|
10
10
|
on(event: "isolate:crash", fn: (e: CrashLike) => void): () => void;
|
|
11
11
|
/** Every heartbeat as it arrives, once per member of a pool (ADR 0017). */
|
|
12
12
|
on(event: "isolate:beat", fn: (e: BeatLike) => void): () => void;
|
|
13
|
+
/** An isolate past the kernel's restart limit: it stays down until `kernel.restart` (M3j). */
|
|
14
|
+
on(event: "isolate:failed", fn: (e: FailedLike) => void): () => void;
|
|
13
15
|
/**
|
|
14
16
|
* Only for kernels not made by `createKernel` (a simulated one in tests):
|
|
15
17
|
* the claim of the `balancer-*` pools, as the host's `claimPools` gives it.
|
|
@@ -51,6 +53,12 @@ type CrashLike = {
|
|
|
51
53
|
reason: string;
|
|
52
54
|
pool?: string;
|
|
53
55
|
};
|
|
56
|
+
/** The fields of the kernel's `isolate:failed` event the balancer reads (M3j). */
|
|
57
|
+
type FailedLike = {
|
|
58
|
+
id: string;
|
|
59
|
+
reason: string;
|
|
60
|
+
pool?: string;
|
|
61
|
+
};
|
|
54
62
|
/**
|
|
55
63
|
* Where the balancer keeps each isolate's measured load between page loads,
|
|
56
64
|
* so a reload places from history (M3h, Q17). The app supplies it; the
|
|
@@ -95,11 +103,14 @@ type BalancerOptions = {
|
|
|
95
103
|
};
|
|
96
104
|
/**
|
|
97
105
|
* Leaving quarantine: an isolate there only to be measured leaves once it
|
|
98
|
-
* has 3 s of beats there
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
* (
|
|
102
|
-
*
|
|
106
|
+
* has 3 s of beats there, or, if a bisect moved it there, once that bisect
|
|
107
|
+
* ends. An offender leaves after `steadyMs` (default 30,000 ms) with no
|
|
108
|
+
* spike and no crash, whatever its load; each offence doubles the window
|
|
109
|
+
* (30, 60, 120 s), capped at 2 minutes (or `steadyMs`, if longer). Back in
|
|
110
|
+
* a balancer pool, an offender loses one offence for each quiet window it
|
|
111
|
+
* completes there (M3j). After `maxReturns` returns (default 3) the next
|
|
112
|
+
* offence keeps it apart for good. Either goes where the placement rule
|
|
113
|
+
* finds room, or stays, flagged `atCap`, until there is.
|
|
103
114
|
*/
|
|
104
115
|
quarantine?: {
|
|
105
116
|
steadyMs?: number;
|
|
@@ -108,35 +119,61 @@ type BalancerOptions = {
|
|
|
108
119
|
loads?: LoadStore;
|
|
109
120
|
/**
|
|
110
121
|
* Packing back (M3i): a member alone in its pool and steady for 60 s (no
|
|
111
|
-
* spike, crash, lagging or full pool,
|
|
112
|
-
* joins another of the balancer's pools if both together stay
|
|
113
|
-
* of a core, freeing a worker. At most one pack every 60 s. On by
|
|
122
|
+
* spike, crash, lagging or full pool, and a backlog no higher than 60 s
|
|
123
|
+
* before, M3j) joins another of the balancer's pools if both together stay
|
|
124
|
+
* under 40 % of a core, freeing a worker. At most one pack every 60 s. On by
|
|
125
|
+
* default.
|
|
114
126
|
*/
|
|
115
127
|
pack?: {
|
|
116
128
|
enabled?: boolean;
|
|
117
129
|
};
|
|
118
130
|
};
|
|
119
|
-
/**
|
|
131
|
+
/**
|
|
132
|
+
* What the app tells the balancer about an isolate (`report`): its queued
|
|
133
|
+
* work, its own measured busy time, or both. At least one must be given.
|
|
134
|
+
*/
|
|
120
135
|
type Report = {
|
|
121
136
|
/**
|
|
122
137
|
* Work items waiting for the isolate, in any unit the app likes, as long as
|
|
123
138
|
* it's the same one each time. For a consumer group, report the group's
|
|
124
139
|
* backlog on every member.
|
|
125
140
|
*/
|
|
126
|
-
backlog
|
|
141
|
+
backlog?: number;
|
|
142
|
+
/**
|
|
143
|
+
* Milliseconds of work the app measured for the isolate since it started:
|
|
144
|
+
* a running total, so a late or missed report still counts (M3j). For work
|
|
145
|
+
* the kernel's `busyMs` misses, such as slices resumed after `await`. The
|
|
146
|
+
* balancer takes its difference over the 3 s window and uses the larger of
|
|
147
|
+
* that and the beats' figure. A total that goes down is taken as a restart
|
|
148
|
+
* of the count.
|
|
149
|
+
*/
|
|
150
|
+
busyMs?: number;
|
|
127
151
|
};
|
|
128
|
-
|
|
152
|
+
/**
|
|
153
|
+
* `failed`: the kernel gave up on the isolate (past its restart limit, M3j).
|
|
154
|
+
* The balancer never moves or restarts it; after `kernel.restart(id)` it's
|
|
155
|
+
* placed again as new.
|
|
156
|
+
*/
|
|
157
|
+
type PlacementState = "placed" | "quarantined" | "kept-apart" | "moving" | "failed";
|
|
129
158
|
type Placement = {
|
|
130
159
|
id: string;
|
|
131
160
|
pool: string;
|
|
132
161
|
state: PlacementState;
|
|
133
|
-
/**
|
|
162
|
+
/**
|
|
163
|
+
* Measured share of one core, 0–100: the larger of the beats' `busyMs` and
|
|
164
|
+
* the app's reported `busyMs` over the last 3 s; `undefined` until measured.
|
|
165
|
+
*/
|
|
134
166
|
loadPct: number | undefined;
|
|
167
|
+
/** The share of one core from the app's `busyMs` reports alone, while they're fresh (M3j). */
|
|
168
|
+
reportedPct?: number;
|
|
135
169
|
/** No pool had room and the worker cap was reached: placed anyway, or left where it is. */
|
|
136
170
|
atCap?: true;
|
|
137
171
|
/** The latest backlog `report`, while it's fresh (under 10 s old). */
|
|
138
172
|
backlog?: number;
|
|
139
|
-
/**
|
|
173
|
+
/**
|
|
174
|
+
* Its backlog is rising, or its pool needs a bisect, and the balancer has
|
|
175
|
+
* nowhere to give it (`behind`).
|
|
176
|
+
*/
|
|
140
177
|
behind?: true;
|
|
141
178
|
/**
|
|
142
179
|
* In one of the balancer's pools, but not placed by it. The host reserves
|
|
@@ -182,14 +219,25 @@ type SpikeEvent = {
|
|
|
182
219
|
};
|
|
183
220
|
/**
|
|
184
221
|
* A member's reported backlog keeps rising, but it already has its pool to
|
|
185
|
-
* itself or the balancer has no worker or pool to give it (M3i).
|
|
186
|
-
*
|
|
222
|
+
* itself or the balancer has no worker or pool to give it (M3i). Also when
|
|
223
|
+
* its pool's trouble needs a bisect and no quarantine pool is free for one
|
|
224
|
+
* (M3j). Emitted when it first happens, then at most every 30 s while it
|
|
225
|
+
* lasts. `backlog` is the latest fresh report, or 0.
|
|
187
226
|
*/
|
|
188
227
|
type BehindEvent = {
|
|
189
228
|
id: string;
|
|
190
229
|
pool: string;
|
|
191
230
|
backlog: number;
|
|
192
231
|
};
|
|
232
|
+
/**
|
|
233
|
+
* The kernel gave up on a member (`isolate:failed`, M3j). It drops out of its
|
|
234
|
+
* pool's load and any bisect, and no move restarts it.
|
|
235
|
+
*/
|
|
236
|
+
type FailedEvent = {
|
|
237
|
+
id: string;
|
|
238
|
+
pool: string;
|
|
239
|
+
reason: string;
|
|
240
|
+
};
|
|
193
241
|
/** An isolate the balancer didn't place turned up in one of its pools (see `Placement.foreign`). */
|
|
194
242
|
type ForeignEvent = {
|
|
195
243
|
id: string;
|
|
@@ -200,6 +248,7 @@ type BalancerEvents = {
|
|
|
200
248
|
arrived: ArrivedEvent;
|
|
201
249
|
spike: SpikeEvent;
|
|
202
250
|
behind: BehindEvent;
|
|
251
|
+
failed: FailedEvent;
|
|
203
252
|
foreign: ForeignEvent;
|
|
204
253
|
};
|
|
205
254
|
/**
|
|
@@ -221,11 +270,14 @@ interface Balancer<M extends ManifestLike = ManifestLike> {
|
|
|
221
270
|
/** Leaves `id` where it is until the returned function is called (a widget open for editing). */
|
|
222
271
|
hold(id: string): () => void;
|
|
223
272
|
/**
|
|
224
|
-
* The app's view of `id`'s queued work (M3i)
|
|
225
|
-
* a 10 s window than at its start, and above
|
|
226
|
-
*
|
|
227
|
-
*
|
|
228
|
-
*
|
|
273
|
+
* The app's view of `id`'s queued work (M3i) and of its own busy time (M3j).
|
|
274
|
+
* A backlog higher at the end of a 10 s window than at its start, and above
|
|
275
|
+
* 0, moves the isolate somewhere with more CPU. A reported `busyMs` total
|
|
276
|
+
* counts toward its load wherever the balancer reads one. A report older
|
|
277
|
+
* than 10 s counts as none, so report at least every few seconds. Throws a
|
|
278
|
+
* `TypeError` for a report with neither field, or a field that isn't a
|
|
279
|
+
* finite number of 0 or more. Ignored for isolates the balancer didn't place
|
|
280
|
+
* and ones the kernel gave up on.
|
|
229
281
|
*/
|
|
230
282
|
report(id: string, report: Report): void;
|
|
231
283
|
on<E extends keyof BalancerEvents>(event: E, fn: (e: BalancerEvents[E]) => void): () => void;
|
|
@@ -247,4 +299,4 @@ interface Balancer<M extends ManifestLike = ManifestLike> {
|
|
|
247
299
|
* `kernel.spawn` are never touched, and don't count toward `maxWorkers`.
|
|
248
300
|
*/
|
|
249
301
|
export declare function createBalancer<M extends ManifestLike>(kernel: KernelLike<M>, opts?: BalancerOptions): Balancer<M>;
|
|
250
|
-
export type { ArrivedEvent, Balancer, BalancerEvents, BalancerOptions, BeatLike, BehindEvent, CrashLike, ForeignEvent, KernelLike, LoadStore, ManifestLike, MoveEvent, MoveReason, Placement, PlacementState, PoolLike, Report, SpikeEvent };
|
|
302
|
+
export type { ArrivedEvent, Balancer, BalancerEvents, BalancerOptions, BeatLike, BehindEvent, CrashLike, FailedEvent, FailedLike, ForeignEvent, KernelLike, LoadStore, ManifestLike, MoveEvent, MoveReason, Placement, PlacementState, PoolLike, Report, SpikeEvent };
|
package/dist/index.js
CHANGED
|
@@ -38,6 +38,8 @@ const POOLS = "balancer";
|
|
|
38
38
|
const QUARANTINES = "balancer-quarantine";
|
|
39
39
|
const QUARANTINE = `${QUARANTINES}-1`;
|
|
40
40
|
const now = () => performance.now();
|
|
41
|
+
/** A report's figure: a finite number, 0 or more. */
|
|
42
|
+
const valid = (x) => typeof x === "number" && Number.isFinite(x) && x >= 0;
|
|
41
43
|
/**
|
|
42
44
|
* `busyMs` is approximate: shared framework work is split between members and
|
|
43
45
|
* promise continuations go unattributed. When the members' busy time explains
|
|
@@ -72,6 +74,8 @@ const MOVE_BUDGET_MS = 3e5;
|
|
|
72
74
|
const PACK_STEADY_MS = 6e4;
|
|
73
75
|
const PACK_PCT = 40;
|
|
74
76
|
const PACK_EVERY_MS = 6e4;
|
|
77
|
+
/** An offender's quiet window doubles with each offence up to this, or `steadyMs` if longer (M3j). */
|
|
78
|
+
const QUIET_CAP_MS = 12e4;
|
|
75
79
|
/** Worker crash reasons whose culprit the kernel can't name: every member of the pool crashes. */
|
|
76
80
|
const POOL_KILL = /* @__PURE__ */ new Set(["unresponsive", "worker-error"]);
|
|
77
81
|
const defaultMaxWorkers = () => {
|
|
@@ -121,6 +125,7 @@ function createBalancer(kernel, opts = {}) {
|
|
|
121
125
|
arrived: /* @__PURE__ */ new Set(),
|
|
122
126
|
spike: /* @__PURE__ */ new Set(),
|
|
123
127
|
behind: /* @__PURE__ */ new Set(),
|
|
128
|
+
failed: /* @__PURE__ */ new Set(),
|
|
124
129
|
foreign: /* @__PURE__ */ new Set()
|
|
125
130
|
};
|
|
126
131
|
/** Beats of isolates the balancer didn't place, for `measure` (previews included). */
|
|
@@ -160,7 +165,9 @@ function createBalancer(kernel, opts = {}) {
|
|
|
160
165
|
}
|
|
161
166
|
return p;
|
|
162
167
|
};
|
|
163
|
-
|
|
168
|
+
/** Running in its pool: not mid-move, and not given up on by the kernel. */
|
|
169
|
+
const live = (m) => m.state !== "moving" && m.state !== "failed";
|
|
170
|
+
const membersOf = (name) => [...members.values()].filter((m) => m.pool === name && live(m));
|
|
164
171
|
const changed = (name) => {
|
|
165
172
|
const p = pool(name);
|
|
166
173
|
p.lags = [];
|
|
@@ -168,8 +175,28 @@ function createBalancer(kernel, opts = {}) {
|
|
|
168
175
|
p.beat = [];
|
|
169
176
|
p.since = now();
|
|
170
177
|
};
|
|
178
|
+
/**
|
|
179
|
+
* The app's reported busy time (M3j) as a share of one core: the running
|
|
180
|
+
* total's difference over about a window, from a fresh report back to one
|
|
181
|
+
* taken at least 90 % of a window before it.
|
|
182
|
+
*/
|
|
183
|
+
const reportedOf = (m, t) => {
|
|
184
|
+
const latest = m.busyReports.at(-1);
|
|
185
|
+
if (!latest || t - latest.at > STALE_MS) return void 0;
|
|
186
|
+
const start = m.busyReports.findLast((r) => r.at <= latest.at - windowMs * .9);
|
|
187
|
+
if (!start) return void 0;
|
|
188
|
+
const ms = latest.at - start.at;
|
|
189
|
+
return Math.min(100, (latest.total - start.total) / ms * 100);
|
|
190
|
+
};
|
|
191
|
+
/** The beats' and the report's measures over the last window, whichever is larger. */
|
|
192
|
+
const measured = (m, t) => {
|
|
193
|
+
const beat = average(m.beats, windowMs, t);
|
|
194
|
+
const reported = reportedOf(m, t);
|
|
195
|
+
if (beat === void 0) return reported;
|
|
196
|
+
return reported === void 0 ? beat : Math.max(beat, reported);
|
|
197
|
+
};
|
|
171
198
|
/** Measured load over the last window, or the last one known. */
|
|
172
|
-
const loadOf = (m) =>
|
|
199
|
+
const loadOf = (m) => measured(m, now()) ?? m.knownLoad;
|
|
173
200
|
const poolLoad = (name) => membersOf(name).reduce((s, m) => s + (loadOf(m) ?? 0), 0);
|
|
174
201
|
const windowFull = (name) => {
|
|
175
202
|
const p = pool(name);
|
|
@@ -182,7 +209,7 @@ function createBalancer(kernel, opts = {}) {
|
|
|
182
209
|
* app runs itself don't count (ADR 0017, amended).
|
|
183
210
|
*/
|
|
184
211
|
const ownWorkers = () => kernel.pools().filter((p) => p.running && isOurs(p.name) && p.name !== QUARANTINE).length;
|
|
185
|
-
const ownPools = () => [...new Set([...members.values()].map((m) => m.pool))].filter((n) => !isQuarantine(n));
|
|
212
|
+
const ownPools = () => [...new Set([...members.values()].filter((m) => m.state !== "failed").map((m) => m.pool))].filter((n) => !isQuarantine(n));
|
|
186
213
|
/** Room for one more worker of the balancer's own. */
|
|
187
214
|
const roomForPool = () => ownWorkers() + 1 <= maxWorkers;
|
|
188
215
|
const freshName = (prefix, first) => {
|
|
@@ -252,6 +279,8 @@ function createBalancer(kernel, opts = {}) {
|
|
|
252
279
|
m.lastBeatAt = void 0;
|
|
253
280
|
m.moveTimes.push(startedAt);
|
|
254
281
|
m.backlogs = [];
|
|
282
|
+
m.backlogHistory = [];
|
|
283
|
+
m.busyReports = [];
|
|
255
284
|
m.behind = false;
|
|
256
285
|
delete m.arrival;
|
|
257
286
|
m.pool = to;
|
|
@@ -302,7 +331,26 @@ function createBalancer(kernel, opts = {}) {
|
|
|
302
331
|
}
|
|
303
332
|
move(m, QUARANTINE, reason, state);
|
|
304
333
|
};
|
|
305
|
-
const movable = (m) => m
|
|
334
|
+
const movable = (m) => live(m) && m.holds === 0;
|
|
335
|
+
/**
|
|
336
|
+
* A member the kernel gave up on beats again: the app restarted it
|
|
337
|
+
* (`kernel.restart`). It's placed again as new (M3j): no offences, no
|
|
338
|
+
* history, and its pool's rules from here.
|
|
339
|
+
*/
|
|
340
|
+
const revive = (m) => {
|
|
341
|
+
m.state = isQuarantine(m.pool) ? "quarantined" : "placed";
|
|
342
|
+
m.offences = 0;
|
|
343
|
+
m.quarantinedAt = isQuarantine(m.pool) ? now() : -Infinity;
|
|
344
|
+
m.lastSpikeAt = m.spikeNotedAt = m.decayAt = m.risingAt = m.stuckAt = -Infinity;
|
|
345
|
+
m.arrivedAt = now();
|
|
346
|
+
m.moveTimes = [];
|
|
347
|
+
m.beats = [];
|
|
348
|
+
m.lastBeatAt = void 0;
|
|
349
|
+
m.backlogs = [];
|
|
350
|
+
m.backlogHistory = [];
|
|
351
|
+
m.busyReports = [];
|
|
352
|
+
changed(m.pool);
|
|
353
|
+
};
|
|
306
354
|
const trim = (beats, at) => {
|
|
307
355
|
const cut = at - windowMs;
|
|
308
356
|
while ((beats[0]?.at ?? Infinity) <= cut) beats.shift();
|
|
@@ -338,6 +386,7 @@ function createBalancer(kernel, opts = {}) {
|
|
|
338
386
|
return;
|
|
339
387
|
}
|
|
340
388
|
if (m.state === "moving" || e.pool !== m.pool) return;
|
|
389
|
+
if (m.state === "failed") revive(m);
|
|
341
390
|
if (m.arrival) {
|
|
342
391
|
const { from, to, reason, at } = m.arrival;
|
|
343
392
|
delete m.arrival;
|
|
@@ -454,9 +503,21 @@ function createBalancer(kernel, opts = {}) {
|
|
|
454
503
|
for (const id of foreign.keys()) if (!present.has(id)) foreign.delete(id);
|
|
455
504
|
};
|
|
456
505
|
/**
|
|
506
|
+
* A quarantine pool a bisect from `name` can have to itself (M3j): the
|
|
507
|
+
* first quarantine if it holds only members this bisect moved (or none), else a
|
|
508
|
+
* new quarantine pool if the cap allows. Never one that holds offenders.
|
|
509
|
+
*/
|
|
510
|
+
const bisectPool = (name) => {
|
|
511
|
+
const free = membersOf(QUARANTINE).every((o) => o.offences === 0 && bisect?.moved.has(o.manifest.id) === true);
|
|
512
|
+
if (!isQuarantine(name) && free) return QUARANTINE;
|
|
513
|
+
return roomForPool() ? freshName(QUARANTINES, 2) : void 0;
|
|
514
|
+
};
|
|
515
|
+
/**
|
|
457
516
|
* Finds the member behind a pool's trouble when the kernel or `busyMs`
|
|
458
517
|
* can't name it (Q20): the heavier half of the suspects moves, and
|
|
459
|
-
* whichever half acts up again holds the culprit.
|
|
518
|
+
* whichever half acts up again holds the culprit. Only into a quarantine
|
|
519
|
+
* pool of its own (M3j); with none free, the pool stays as it is and its
|
|
520
|
+
* members are `behind`.
|
|
460
521
|
*/
|
|
461
522
|
const search = (name, reason) => {
|
|
462
523
|
const here = membersOf(name).filter((m) => m.holds === 0);
|
|
@@ -477,18 +538,25 @@ function createBalancer(kernel, opts = {}) {
|
|
|
477
538
|
return;
|
|
478
539
|
}
|
|
479
540
|
const sorted = suspects.map((id) => members.get(id)).filter((m) => m !== void 0).toSorted((a, b) => (loadOf(b) ?? 0) - (loadOf(a) ?? 0));
|
|
541
|
+
const to = bisectPool(name);
|
|
542
|
+
if (to === void 0) {
|
|
543
|
+
const t = now();
|
|
544
|
+
for (const m of here) {
|
|
545
|
+
m.stuckAt = t;
|
|
546
|
+
behind(m, t);
|
|
547
|
+
}
|
|
548
|
+
return;
|
|
549
|
+
}
|
|
480
550
|
const half = sorted.slice(0, Math.ceil(sorted.length / 2));
|
|
481
551
|
bisect = {
|
|
482
552
|
suspects: new Set(suspects),
|
|
483
553
|
moved: /* @__PURE__ */ new Set([...bisect?.moved ?? [], ...half.map((m) => m.manifest.id)]),
|
|
484
554
|
at: now()
|
|
485
555
|
};
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
for (const m of half) if (to === QUARANTINE) {
|
|
489
|
-
m.quarantinedAt = now();
|
|
556
|
+
for (const m of half) {
|
|
557
|
+
if (!isQuarantine(name)) m.quarantinedAt = now();
|
|
490
558
|
move(m, to, "bisect", stateFor(m));
|
|
491
|
-
}
|
|
559
|
+
}
|
|
492
560
|
};
|
|
493
561
|
/**
|
|
494
562
|
* The moves that would bring pool `name` back under (M3i): the heaviest
|
|
@@ -575,7 +643,7 @@ function createBalancer(kernel, opts = {}) {
|
|
|
575
643
|
const order = [...members.values()].filter((m) => {
|
|
576
644
|
const r = m.state === "placed" && !isQuarantine(m.pool) && rising(m, t);
|
|
577
645
|
if (r) m.risingAt = t;
|
|
578
|
-
else m.behind = false;
|
|
646
|
+
else if (t - m.stuckAt >= windowMs) m.behind = false;
|
|
579
647
|
return r;
|
|
580
648
|
}).toSorted((a, b) => poolLoad(b.pool) - poolLoad(a.pool) || (loadOf(b) ?? 0) - (loadOf(a) ?? 0));
|
|
581
649
|
for (const m of order) {
|
|
@@ -592,19 +660,32 @@ function createBalancer(kernel, opts = {}) {
|
|
|
592
660
|
}
|
|
593
661
|
};
|
|
594
662
|
/**
|
|
663
|
+
* Its backlog rose over the last `PACK_STEADY_MS` (M3j): the latest fresh
|
|
664
|
+
* report is higher than the one at or just before the window's start (or
|
|
665
|
+
* the oldest since it moved, when it hasn't reported that long). A
|
|
666
|
+
* queue that holds steady while work arrives doesn't count, nor does a
|
|
667
|
+
* stale report or none.
|
|
668
|
+
*/
|
|
669
|
+
const roseLately = (m, t) => {
|
|
670
|
+
const latest = backlogNow(m, t);
|
|
671
|
+
if (latest === void 0) return false;
|
|
672
|
+
const start = m.backlogHistory.findLast((b) => b.at <= t - PACK_STEADY_MS) ?? m.backlogHistory[0];
|
|
673
|
+
return start !== void 0 && latest > start.value;
|
|
674
|
+
};
|
|
675
|
+
/**
|
|
595
676
|
* Packing back (M3i): the lightest member alone in its pool and quiet for
|
|
596
677
|
* `PACK_STEADY_MS` joins the fullest other pool that stays under
|
|
597
|
-
* `PACK_PCT` with it, which frees its worker. Never one on hold,
|
|
598
|
-
* backlog, in quarantine or kept apart, and
|
|
599
|
-
*
|
|
678
|
+
* `PACK_PCT` with it, which frees its worker. Never one on hold, whose
|
|
679
|
+
* backlog rose over the window (M3j), in quarantine or kept apart, and
|
|
680
|
+
* never into a pool whose members' backlog rose.
|
|
600
681
|
*/
|
|
601
682
|
const pack = (t) => {
|
|
602
683
|
const quiet = (since) => t - since >= PACK_STEADY_MS;
|
|
603
|
-
const steady = (m) => quiet(Math.max(m.arrivedAt, m.lastSpikeAt, m.spikeNotedAt,
|
|
604
|
-
const alone = [...members.values()].filter((m) => m.state === "placed" && isOurs(m.pool) && !isQuarantine(m.pool) && movable(m) && inBudget(m) && membersOf(m.pool).length === 1 &&
|
|
684
|
+
const steady = (m) => quiet(Math.max(m.arrivedAt, m.lastSpikeAt, m.spikeNotedAt, pool(m.pool).troubleAt)) && !roseLately(m, t);
|
|
685
|
+
const alone = [...members.values()].filter((m) => m.state === "placed" && isOurs(m.pool) && !isQuarantine(m.pool) && movable(m) && inBudget(m) && membersOf(m.pool).length === 1 && loadOf(m) !== void 0 && steady(m)).toSorted((a, b) => (loadOf(a) ?? 0) - (loadOf(b) ?? 0));
|
|
605
686
|
for (const m of alone) {
|
|
606
687
|
const load = loadOf(m) ?? 0;
|
|
607
|
-
const into = ownPools().filter((n) => n !== m.pool && !recentlySpiked(n) && quiet(pool(n).troubleAt)).filter((n) =>
|
|
688
|
+
const into = ownPools().filter((n) => n !== m.pool && !recentlySpiked(n) && quiet(pool(n).troubleAt)).filter((n) => [...members.values()].filter((o) => o.pool === n).every((o) => o.state !== "moving" && !roseLately(o, t))).map((n) => ({
|
|
608
689
|
n,
|
|
609
690
|
load: poolLoad(n)
|
|
610
691
|
})).filter((c) => c.load + load < PACK_PCT).toSorted((a, b) => b.load - a.load)[0];
|
|
@@ -614,7 +695,22 @@ function createBalancer(kernel, opts = {}) {
|
|
|
614
695
|
return;
|
|
615
696
|
}
|
|
616
697
|
};
|
|
617
|
-
|
|
698
|
+
/** The quiet window: doubled per offence, capped at 2 minutes (or `steadyMs`, if longer; M3j). */
|
|
699
|
+
const steadyFor = (m) => Math.min(Math.max(steadyMs, QUIET_CAP_MS), steadyMs * 2 ** Math.max(0, m.offences - 1));
|
|
700
|
+
/**
|
|
701
|
+
* Offences decay (M3j): an offender back in a balancer pool loses one for
|
|
702
|
+
* each quiet window it completes there, so one that stays quiet is back to
|
|
703
|
+
* the shortest window. One that offends again soon after leaving still
|
|
704
|
+
* gets a longer one, and `kept-apart` stays apart.
|
|
705
|
+
*/
|
|
706
|
+
const decay = (t) => {
|
|
707
|
+
for (const m of members.values()) {
|
|
708
|
+
if (m.state !== "placed" || m.offences === 0) continue;
|
|
709
|
+
if (t - Math.max(m.decayAt, m.lastSpikeAt, m.spikeNotedAt) < steadyFor(m)) continue;
|
|
710
|
+
m.offences--;
|
|
711
|
+
m.decayAt = t;
|
|
712
|
+
}
|
|
713
|
+
};
|
|
618
714
|
const decide = () => {
|
|
619
715
|
const t = now();
|
|
620
716
|
checkForeign();
|
|
@@ -655,6 +751,7 @@ function createBalancer(kernel, opts = {}) {
|
|
|
655
751
|
} else first.atCap = true;
|
|
656
752
|
}
|
|
657
753
|
backlogs(t);
|
|
754
|
+
decay(t);
|
|
658
755
|
for (const m of members.values()) {
|
|
659
756
|
if (m.state !== "quarantined" || !movable(m) || !inBudget(m)) continue;
|
|
660
757
|
const fresh = average(m.beats, windowMs, t);
|
|
@@ -669,6 +766,7 @@ function createBalancer(kernel, opts = {}) {
|
|
|
669
766
|
continue;
|
|
670
767
|
}
|
|
671
768
|
m.atCap = false;
|
|
769
|
+
m.decayAt = t;
|
|
672
770
|
move(m, to.pool, "steady", "placed");
|
|
673
771
|
}
|
|
674
772
|
if (packing && t - lastPackAt >= PACK_EVERY_MS) pack(t);
|
|
@@ -679,22 +777,53 @@ function createBalancer(kernel, opts = {}) {
|
|
|
679
777
|
};
|
|
680
778
|
const onCrash = (e) => {
|
|
681
779
|
const m = members.get(e.id);
|
|
682
|
-
if (!m || m
|
|
780
|
+
if (!m || !live(m)) return;
|
|
683
781
|
if (POOL_KILL.has(e.reason) && e.pool !== void 0) killed.add(e.pool);
|
|
684
782
|
else if (e.reason === "error") {
|
|
685
783
|
if (m.holds > 0) m.lastSpikeAt = now();
|
|
686
784
|
else quarantine(m, "crash", true);
|
|
687
785
|
}
|
|
688
786
|
};
|
|
787
|
+
/**
|
|
788
|
+
* The kernel gave up on a member (M3j): it leaves its pool's load, the
|
|
789
|
+
* quarantine's count and any bisect, and stays where it is until the app
|
|
790
|
+
* restarts it. A member mid-move was stopped by the balancer itself; its
|
|
791
|
+
* respawn is a fresh start.
|
|
792
|
+
*/
|
|
793
|
+
const onFailed = (e) => {
|
|
794
|
+
const m = members.get(e.id);
|
|
795
|
+
if (!m || !live(m)) return;
|
|
796
|
+
m.state = "failed";
|
|
797
|
+
m.beats = [];
|
|
798
|
+
m.lastBeatAt = void 0;
|
|
799
|
+
m.atCap = false;
|
|
800
|
+
m.behind = false;
|
|
801
|
+
m.busyReports = [];
|
|
802
|
+
delete m.arrival;
|
|
803
|
+
bisect?.suspects.delete(e.id);
|
|
804
|
+
bisect?.moved.delete(e.id);
|
|
805
|
+
changed(m.pool);
|
|
806
|
+
emit("failed", {
|
|
807
|
+
id: e.id,
|
|
808
|
+
pool: m.pool,
|
|
809
|
+
reason: e.reason
|
|
810
|
+
});
|
|
811
|
+
};
|
|
689
812
|
const timer = setInterval(decide, DECIDE_MS);
|
|
690
|
-
const offs = [
|
|
813
|
+
const offs = [
|
|
814
|
+
kernel.on("isolate:crash", onCrash),
|
|
815
|
+
kernel.on("isolate:beat", onBeat),
|
|
816
|
+
kernel.on("isolate:failed", onFailed)
|
|
817
|
+
];
|
|
691
818
|
const placement = (m) => {
|
|
692
819
|
const p = {
|
|
693
820
|
id: m.manifest.id,
|
|
694
821
|
pool: m.pool,
|
|
695
822
|
state: m.state,
|
|
696
|
-
loadPct: loadOf(m)
|
|
823
|
+
loadPct: m.state === "failed" ? void 0 : loadOf(m)
|
|
697
824
|
};
|
|
825
|
+
const reported = m.state === "failed" ? void 0 : reportedOf(m, now());
|
|
826
|
+
if (reported !== void 0) p.reportedPct = reported;
|
|
698
827
|
if (m.atCap) p.atCap = true;
|
|
699
828
|
const backlog = backlogNow(m, now());
|
|
700
829
|
if (backlog !== void 0) p.backlog = backlog;
|
|
@@ -724,6 +853,7 @@ function createBalancer(kernel, opts = {}) {
|
|
|
724
853
|
savedLoad: known,
|
|
725
854
|
offences: 0,
|
|
726
855
|
quarantinedAt: isQuarantine(dest.pool) ? now() : -Infinity,
|
|
856
|
+
decayAt: -Infinity,
|
|
727
857
|
lastSpikeAt: -Infinity,
|
|
728
858
|
spikeNotedAt: -Infinity,
|
|
729
859
|
holds: 0,
|
|
@@ -732,7 +862,10 @@ function createBalancer(kernel, opts = {}) {
|
|
|
732
862
|
moveTimes: [],
|
|
733
863
|
backlogs: [],
|
|
734
864
|
backlog: void 0,
|
|
865
|
+
backlogHistory: [],
|
|
866
|
+
busyReports: [],
|
|
735
867
|
risingAt: -Infinity,
|
|
868
|
+
stuckAt: -Infinity,
|
|
736
869
|
behind: false,
|
|
737
870
|
behindAt: -Infinity
|
|
738
871
|
};
|
|
@@ -775,18 +908,33 @@ function createBalancer(kernel, opts = {}) {
|
|
|
775
908
|
m.holds--;
|
|
776
909
|
};
|
|
777
910
|
},
|
|
778
|
-
report(id, { backlog }) {
|
|
779
|
-
if (
|
|
911
|
+
report(id, { backlog, busyMs }) {
|
|
912
|
+
if (backlog === void 0 && busyMs === void 0) throw new TypeError("balancer.report: give a backlog, a busyMs or both");
|
|
913
|
+
if (backlog !== void 0 && !valid(backlog)) throw new TypeError("balancer.report: backlog must be a finite number, 0 or more");
|
|
914
|
+
if (busyMs !== void 0 && !valid(busyMs)) throw new TypeError("balancer.report: busyMs must be a finite number, 0 or more");
|
|
780
915
|
const m = members.get(id);
|
|
781
|
-
if (!m) return;
|
|
916
|
+
if (!m || m.state === "failed") return;
|
|
782
917
|
const t = now();
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
918
|
+
if (backlog !== void 0) {
|
|
919
|
+
const sample = {
|
|
920
|
+
at: t,
|
|
921
|
+
value: backlog
|
|
922
|
+
};
|
|
923
|
+
m.backlog = sample;
|
|
924
|
+
m.backlogs.push(sample);
|
|
925
|
+
m.backlogHistory.push(sample);
|
|
926
|
+
while ((m.backlogs[1]?.at ?? Infinity) <= t - BACKLOG_WINDOW_MS) m.backlogs.shift();
|
|
927
|
+
while ((m.backlogHistory[1]?.at ?? Infinity) <= t - PACK_STEADY_MS) m.backlogHistory.shift();
|
|
928
|
+
}
|
|
929
|
+
if (busyMs !== void 0) {
|
|
930
|
+
if ((m.busyReports.at(-1)?.total ?? 0) > busyMs) m.busyReports = [];
|
|
931
|
+
m.busyReports.push({
|
|
932
|
+
at: t,
|
|
933
|
+
total: busyMs
|
|
934
|
+
});
|
|
935
|
+
const cut = t - windowMs - STALE_MS;
|
|
936
|
+
while ((m.busyReports[1]?.at ?? Infinity) <= cut) m.busyReports.shift();
|
|
937
|
+
}
|
|
790
938
|
},
|
|
791
939
|
measure(id) {
|
|
792
940
|
const m = members.get(id);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@isoloop/balancer",
|
|
3
|
-
"version": "0.1.0-next.
|
|
3
|
+
"version": "0.1.0-next.1",
|
|
4
4
|
"description": "Experimental automatic placement for Isoloop isolates: joins a pool or keeps apart from measured load, quarantines what blocks.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
}
|
|
21
21
|
},
|
|
22
22
|
"dependencies": {
|
|
23
|
-
"@isoloop/host": "0.1.0-next.
|
|
23
|
+
"@isoloop/host": "0.1.0-next.1"
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
26
|
"typescript": "7.0.2"
|