@isoloop/balancer 0.1.0-next.0 → 0.1.0-next.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -26,10 +26,23 @@ balancer.on('behind', (e) => console.warn(e.id, 'is behind and has nowhere to go
26
26
  balancer.on('arrived', (e) => console.log(e.id, 'moved in', e.costMs, 'ms'));
27
27
  ```
28
28
 
29
- - **Backlog:** a backlog higher at the end of a 10 s window than at its start, and above 0, moves the isolate somewhere with more CPU: a pool of its own if the cap allows, else a pool with less load than its pool-mates. One such move per fresh 10 s window, so a group's members move one at a time. A report older than 10 s counts as none. With nowhere to go, it emits `behind` (at most every 30 s).
30
- - **Fewer moves:** a move for load is made only if the moves it starts are predicted to bring the pool back under its line, and each isolate gets at most 3 of them in 5 minutes. Spike, crash and bisect moves protect the others and aren't limited.
29
+ - **Backlog:** a backlog that rose over two 10 s windows in a row, and is above 0, moves the isolate somewhere with more CPU: a pool of its own if the cap allows, else a pool with less load than its pool-mates. One such move per fresh 10 s window, so a group's members move one at a time. A report older than 10 s counts as none. With nowhere to go, it emits `behind` (at most every 30 s).
30
+ - **Fewer moves:** a move for load is made only if the moves it starts are predicted to bring the pool back under its line, and each isolate gets at most 3 of them in 5 minutes. Spike, crash and bisect moves protect the others and aren't limited. A never-measured isolate's way out of the quarantine, where it starts, doesn't count.
31
31
  - **Spikes:** a first long task is only reported (`spike`, action `noted`); a second from the same isolate within 60 s sends it to quarantine. Its own fatal error still does at once.
32
- - **Packing back:** an isolate alone in its pool and quiet for 60 s joins another of the balancer's pools if both stay under 40 % of a core, which frees a worker; at most one every 60 s. Turn it off with `pack: { enabled: false }`.
32
+ - **Packing back:** a pool whose isolates have all been steady for 60 s empties into another of the balancer's pools if both together stay under 40 % of a core, one isolate per pack and at most one pack every 60 s, which frees a worker once the last one is in. Turn it off with `pack: { enabled: false }`.
33
33
  - **Move cost:** `arrived` follows each `move` with `costMs`, the time from stopping the isolate to its first beat in the new pool.
34
34
 
35
+ And when its busy time misses work, or the kernel gives up on an isolate (M3j):
36
+
37
+ ```ts
38
+ // Work timed by the app, as a running total since the isolate started: for slices resumed after `await`.
39
+ balancer.report('task-1', { backlog: queue.length, busyMs: workMsSoFar });
40
+ balancer.on('failed', (e) => console.warn(e.id, 'was given up on in', e.pool, e.reason));
41
+ ```
42
+
43
+ - **Reported busy time:** the balancer takes the total's difference over its 3 s window and uses the larger of that and the beats' `busyMs`, wherever it reads a load.
44
+ - **Packing** goes ahead while a backlog holds steady; only a backlog that rose over the 60 s window blocks it.
45
+ - **The quarantine:** an offender's quiet window is capped at 2 minutes, and offences wear off with each quiet window back in a pool. A bisect starts only with a quarantine pool of its own, never alongside offenders; with none free, the pool's members are `behind`.
46
+ - **Given up:** an isolate past the kernel's restart limit is `failed`. It leaves its pool's load and any bisect, and no move restarts it; after `kernel.restart(id)` it's placed again as new.
47
+
35
48
  One balancer per kernel at a time (`dispose()` frees the kernel for another); its pools are named `balancer-…`, which the host reserves. The API, the triggers, the quarantine and what `busyMs` measures are in the [balancer guide](https://github.com/isoloop/isoloop/blob/main/docs/guide/balancer.md).
package/dist/index.d.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * The part of the kernel the balancer uses: the public API only (`spawn`,
3
- * `stop`, `pools`, and the `isolate:beat` and `isolate:crash` events), so
4
- * `@isoloop/host`'s `Kernel` fits it as is.
3
+ * `stop`, `pools`, and the `isolate:beat`, `isolate:crash` and
4
+ * `isolate:failed` events), so `@isoloop/host`'s `Kernel` fits it as is.
5
5
  */
6
6
  interface KernelLike<M extends ManifestLike = ManifestLike> {
7
7
  spawn(manifest: M): unknown;
@@ -10,6 +10,8 @@ interface KernelLike<M extends ManifestLike = ManifestLike> {
10
10
  on(event: "isolate:crash", fn: (e: CrashLike) => void): () => void;
11
11
  /** Every heartbeat as it arrives, once per member of a pool (ADR 0017). */
12
12
  on(event: "isolate:beat", fn: (e: BeatLike) => void): () => void;
13
+ /** An isolate past the kernel's restart limit: it stays down until `kernel.restart` (M3j). */
14
+ on(event: "isolate:failed", fn: (e: FailedLike) => void): () => void;
13
15
  /**
14
16
  * Only for kernels not made by `createKernel` (a simulated one in tests):
15
17
  * the claim of the `balancer-*` pools, as the host's `claimPools` gives it.
@@ -51,6 +53,12 @@ type CrashLike = {
51
53
  reason: string;
52
54
  pool?: string;
53
55
  };
56
+ /** The fields of the kernel's `isolate:failed` event the balancer reads (M3j). */
57
+ type FailedLike = {
58
+ id: string;
59
+ reason: string;
60
+ pool?: string;
61
+ };
54
62
  /**
55
63
  * Where the balancer keeps each isolate's measured load between page loads,
56
64
  * so a reload places from history (M3h, Q17). The app supplies it; the
@@ -95,11 +103,14 @@ type BalancerOptions = {
95
103
  };
96
104
  /**
97
105
  * Leaving quarantine: an isolate there only to be measured leaves once it
98
- * has 3 s of beats there. An offender leaves after `steadyMs` (default
99
- * 30,000 ms) with no spike and no crash, whatever its load; each offence
100
- * doubles the window (30, 60, 120 s), and after `maxReturns` returns
101
- * (default 3) the next offence keeps it apart for good. Either goes where
102
- * the placement rule finds room, or stays, flagged `atCap`, until there is.
106
+ * has 3 s of beats there, or, if a bisect moved it there, once that bisect
107
+ * ends. An offender leaves after `steadyMs` (default 30,000 ms) with no
108
+ * spike and no crash, whatever its load; each offence doubles the window
109
+ * (30, 60, 120 s), capped at 2 minutes (or `steadyMs`, if longer). Back in
110
+ * a balancer pool, an offender loses one offence for each quiet window it
111
+ * completes there (M3j). After `maxReturns` returns (default 3) the next
112
+ * offence keeps it apart for good. Either goes where the placement rule
113
+ * finds room, or stays, flagged `atCap`, until there is.
103
114
  */
104
115
  quarantine?: {
105
116
  steadyMs?: number;
@@ -107,36 +118,63 @@ type BalancerOptions = {
107
118
  };
108
119
  loads?: LoadStore;
109
120
  /**
110
- * Packing back (M3i): a member alone in its pool and steady for 60 s (no
111
- * spike, crash, lagging or full pool, rising backlog or backlog above 0)
112
- * joins another of the balancer's pools if both together stay under 40 %
113
- * of a core, freeing a worker. At most one pack every 60 s. On by default.
121
+ * Packing back (M3i; whole pools since M3j): a pool whose members have
122
+ * all been steady for 60 s (no spike, crash, lagging or full pool, and a
123
+ * backlog no higher than 60 s before) empties into another of the
124
+ * balancer's pools if both together stay under 40 % of a core, one member
125
+ * per pack, freeing a worker once the last is in. At most one pack every
126
+ * 60 s. On by default.
114
127
  */
115
128
  pack?: {
116
129
  enabled?: boolean;
117
130
  };
118
131
  };
119
- /** What the app tells the balancer about an isolate's queued work (`report`). */
132
+ /**
133
+ * What the app tells the balancer about an isolate (`report`): its queued
134
+ * work, its own measured busy time, or both. At least one must be given.
135
+ */
120
136
  type Report = {
121
137
  /**
122
138
  * Work items waiting for the isolate, in any unit the app likes, as long as
123
139
  * it's the same one each time. For a consumer group, report the group's
124
140
  * backlog on every member.
125
141
  */
126
- backlog: number;
142
+ backlog?: number;
143
+ /**
144
+ * Milliseconds of work the app measured for the isolate since it started:
145
+ * a running total, so a late or missed report still counts (M3j). For work
146
+ * the kernel's `busyMs` misses, such as slices resumed after `await`. The
147
+ * balancer takes its difference over the 3 s window and uses the larger of
148
+ * that and the beats' figure. A total that goes down is taken as a restart
149
+ * of the count.
150
+ */
151
+ busyMs?: number;
127
152
  };
128
- type PlacementState = "placed" | "quarantined" | "kept-apart" | "moving";
153
+ /**
154
+ * `failed`: the kernel gave up on the isolate (past its restart limit, M3j).
155
+ * The balancer never moves or restarts it; after `kernel.restart(id)` it's
156
+ * placed again as new.
157
+ */
158
+ type PlacementState = "placed" | "quarantined" | "kept-apart" | "moving" | "failed";
129
159
  type Placement = {
130
160
  id: string;
131
161
  pool: string;
132
162
  state: PlacementState;
133
- /** Measured share of one core, 0–100; `undefined` until measured. */
163
+ /**
164
+ * Measured share of one core, 0–100: the larger of the beats' `busyMs` and
165
+ * the app's reported `busyMs` over the last 3 s; `undefined` until measured.
166
+ */
134
167
  loadPct: number | undefined;
168
+ /** The share of one core from the app's `busyMs` reports alone, while they're fresh (M3j). */
169
+ reportedPct?: number;
135
170
  /** No pool had room and the worker cap was reached: placed anyway, or left where it is. */
136
171
  atCap?: true;
137
172
  /** The latest backlog `report`, while it's fresh (under 10 s old). */
138
173
  backlog?: number;
139
- /** Its backlog is rising, and the balancer has nowhere to give it more CPU (`behind`). */
174
+ /**
175
+ * Its backlog is rising, or its pool needs a bisect, and the balancer has
176
+ * nowhere to give it (`behind`).
177
+ */
140
178
  behind?: true;
141
179
  /**
142
180
  * In one of the balancer's pools, but not placed by it. The host reserves
@@ -182,14 +220,25 @@ type SpikeEvent = {
182
220
  };
183
221
  /**
184
222
  * A member's reported backlog keeps rising, but it already has its pool to
185
- * itself or the balancer has no worker or pool to give it (M3i). Emitted when
186
- * it first happens, then at most every 30 s while it lasts.
223
+ * itself or the balancer has no worker or pool to give it (M3i). Also when
224
+ * its pool's trouble needs a bisect and no quarantine pool is free for one
225
+ * (M3j). Emitted when it first happens, then at most every 30 s while it
226
+ * lasts. `backlog` is the latest fresh report, or 0.
187
227
  */
188
228
  type BehindEvent = {
189
229
  id: string;
190
230
  pool: string;
191
231
  backlog: number;
192
232
  };
233
+ /**
234
+ * The kernel gave up on a member (`isolate:failed`, M3j). It drops out of its
235
+ * pool's load and any bisect, and no move restarts it.
236
+ */
237
+ type FailedEvent = {
238
+ id: string;
239
+ pool: string;
240
+ reason: string;
241
+ };
193
242
  /** An isolate the balancer didn't place turned up in one of its pools (see `Placement.foreign`). */
194
243
  type ForeignEvent = {
195
244
  id: string;
@@ -200,6 +249,7 @@ type BalancerEvents = {
200
249
  arrived: ArrivedEvent;
201
250
  spike: SpikeEvent;
202
251
  behind: BehindEvent;
252
+ failed: FailedEvent;
203
253
  foreign: ForeignEvent;
204
254
  };
205
255
  /**
@@ -221,11 +271,14 @@ interface Balancer<M extends ManifestLike = ManifestLike> {
221
271
  /** Leaves `id` where it is until the returned function is called (a widget open for editing). */
222
272
  hold(id: string): () => void;
223
273
  /**
224
- * The app's view of `id`'s queued work (M3i). A backlog higher at the end of
225
- * a 10 s window than at its start, and above 0, moves the isolate somewhere
226
- * with more CPU; a report older than 10 s counts as none, so report at least
227
- * every few seconds while there's work. Ignored for isolates the balancer
228
- * didn't place.
274
+ * The app's view of `id`'s queued work (M3i) and of its own busy time (M3j).
275
+ * A backlog that rose over two 10 s windows in a row (M3j), and is above 0,
276
+ * moves the isolate somewhere with more CPU. A reported `busyMs` total
277
+ * counts toward its load wherever the balancer reads one. A report older
278
+ * than 10 s counts as none, so report at least every few seconds. Throws a
279
+ * `TypeError` for a report with neither field, or a field that isn't a
280
+ * finite number of 0 or more. Ignored for isolates the balancer didn't place
281
+ * and ones the kernel gave up on.
229
282
  */
230
283
  report(id: string, report: Report): void;
231
284
  on<E extends keyof BalancerEvents>(event: E, fn: (e: BalancerEvents[E]) => void): () => void;
@@ -247,4 +300,4 @@ interface Balancer<M extends ManifestLike = ManifestLike> {
247
300
  * `kernel.spawn` are never touched, and don't count toward `maxWorkers`.
248
301
  */
249
302
  export declare function createBalancer<M extends ManifestLike>(kernel: KernelLike<M>, opts?: BalancerOptions): Balancer<M>;
250
- export type { ArrivedEvent, Balancer, BalancerEvents, BalancerOptions, BeatLike, BehindEvent, CrashLike, ForeignEvent, KernelLike, LoadStore, ManifestLike, MoveEvent, MoveReason, Placement, PlacementState, PoolLike, Report, SpikeEvent };
303
+ export type { ArrivedEvent, Balancer, BalancerEvents, BalancerOptions, BeatLike, BehindEvent, CrashLike, FailedEvent, FailedLike, ForeignEvent, KernelLike, LoadStore, ManifestLike, MoveEvent, MoveReason, Placement, PlacementState, PoolLike, Report, SpikeEvent };
package/dist/index.js CHANGED
@@ -38,6 +38,8 @@ const POOLS = "balancer";
38
38
  const QUARANTINES = "balancer-quarantine";
39
39
  const QUARANTINE = `${QUARANTINES}-1`;
40
40
  const now = () => performance.now();
41
+ /** A report's figure: a finite number, 0 or more. */
42
+ const valid = (x) => typeof x === "number" && Number.isFinite(x) && x >= 0;
41
43
  /**
42
44
  * `busyMs` is approximate: shared framework work is split between members and
43
45
  * promise continuations go unattributed. When the members' busy time explains
@@ -72,6 +74,8 @@ const MOVE_BUDGET_MS = 3e5;
72
74
  const PACK_STEADY_MS = 6e4;
73
75
  const PACK_PCT = 40;
74
76
  const PACK_EVERY_MS = 6e4;
77
+ /** An offender's quiet window doubles with each offence up to this, or `steadyMs` if longer (M3j). */
78
+ const QUIET_CAP_MS = 12e4;
75
79
  /** Worker crash reasons whose culprit the kernel can't name: every member of the pool crashes. */
76
80
  const POOL_KILL = /* @__PURE__ */ new Set(["unresponsive", "worker-error"]);
77
81
  const defaultMaxWorkers = () => {
@@ -121,6 +125,7 @@ function createBalancer(kernel, opts = {}) {
121
125
  arrived: /* @__PURE__ */ new Set(),
122
126
  spike: /* @__PURE__ */ new Set(),
123
127
  behind: /* @__PURE__ */ new Set(),
128
+ failed: /* @__PURE__ */ new Set(),
124
129
  foreign: /* @__PURE__ */ new Set()
125
130
  };
126
131
  /** Beats of isolates the balancer didn't place, for `measure` (previews included). */
@@ -143,6 +148,8 @@ function createBalancer(kernel, opts = {}) {
143
148
  /** The last backlog move: rising is judged only on reports after it. */
144
149
  let lastBacklogMoveAt = -Infinity;
145
150
  let lastPackAt = -Infinity;
151
+ /** A pool being emptied into another by packing, one member at a time (M3j). */
152
+ let merging;
146
153
  let disposed = false;
147
154
  const pool = (name) => {
148
155
  let p = pools.get(name);
@@ -160,7 +167,11 @@ function createBalancer(kernel, opts = {}) {
160
167
  }
161
168
  return p;
162
169
  };
163
- const membersOf = (name) => [...members.values()].filter((m) => m.pool === name && m.state !== "moving");
170
+ /** Running in its pool: not mid-move, and not given up on by the kernel. */
171
+ const live = (m) => m.state !== "moving" && m.state !== "failed";
172
+ /** Its members, moving ones included: the ones a pack has to move to empty it (M3j). */
173
+ const inPool = (name) => [...members.values()].filter((m) => m.pool === name && m.state !== "failed");
174
+ const membersOf = (name) => [...members.values()].filter((m) => m.pool === name && live(m));
164
175
  const changed = (name) => {
165
176
  const p = pool(name);
166
177
  p.lags = [];
@@ -168,8 +179,28 @@ function createBalancer(kernel, opts = {}) {
168
179
  p.beat = [];
169
180
  p.since = now();
170
181
  };
182
+ /**
183
+ * The app's reported busy time (M3j) as a share of one core: the running
184
+ * total's difference over about a window, from a fresh report back to one
185
+ * taken at least 90 % of a window before it.
186
+ */
187
+ const reportedOf = (m, t) => {
188
+ const latest = m.busyReports.at(-1);
189
+ if (!latest || t - latest.at > STALE_MS) return void 0;
190
+ const start = m.busyReports.findLast((r) => r.at <= latest.at - windowMs * .9);
191
+ if (!start) return void 0;
192
+ const ms = latest.at - start.at;
193
+ return Math.min(100, (latest.total - start.total) / ms * 100);
194
+ };
195
+ /** The beats' and the report's measures over the last window, whichever is larger. */
196
+ const measured = (m, t) => {
197
+ const beat = average(m.beats, windowMs, t);
198
+ const reported = reportedOf(m, t);
199
+ if (beat === void 0) return reported;
200
+ return reported === void 0 ? beat : Math.max(beat, reported);
201
+ };
171
202
  /** Measured load over the last window, or the last one known. */
172
- const loadOf = (m) => average(m.beats, windowMs, now()) ?? m.knownLoad;
203
+ const loadOf = (m) => measured(m, now()) ?? m.knownLoad;
173
204
  const poolLoad = (name) => membersOf(name).reduce((s, m) => s + (loadOf(m) ?? 0), 0);
174
205
  const windowFull = (name) => {
175
206
  const p = pool(name);
@@ -182,7 +213,7 @@ function createBalancer(kernel, opts = {}) {
182
213
  * app runs itself don't count (ADR 0017, amended).
183
214
  */
184
215
  const ownWorkers = () => kernel.pools().filter((p) => p.running && isOurs(p.name) && p.name !== QUARANTINE).length;
185
- const ownPools = () => [...new Set([...members.values()].map((m) => m.pool))].filter((n) => !isQuarantine(n));
216
+ const ownPools = () => [...new Set([...members.values()].filter((m) => m.state !== "failed").map((m) => m.pool))].filter((n) => !isQuarantine(n));
186
217
  /** Room for one more worker of the balancer's own. */
187
218
  const roomForPool = () => ownWorkers() + 1 <= maxWorkers;
188
219
  const freshName = (prefix, first) => {
@@ -250,8 +281,11 @@ function createBalancer(kernel, opts = {}) {
250
281
  m.state = "moving";
251
282
  m.beats = [];
252
283
  m.lastBeatAt = void 0;
253
- m.moveTimes.push(startedAt);
284
+ if (!m.unmeasured) m.moveTimes.push(startedAt);
285
+ m.unmeasured = false;
254
286
  m.backlogs = [];
287
+ m.backlogHistory = [];
288
+ m.busyReports = [];
255
289
  m.behind = false;
256
290
  delete m.arrival;
257
291
  m.pool = to;
@@ -293,7 +327,10 @@ function createBalancer(kernel, opts = {}) {
293
327
  /** Sends `m` to quarantine for misbehaving (`offence`) or to bisect. */
294
328
  const quarantine = (m, reason, offence) => {
295
329
  m.lastSpikeAt = now();
296
- if (offence) m.offences++;
330
+ if (offence) {
331
+ m.offences++;
332
+ m.unmeasured = false;
333
+ }
297
334
  const state = stateFor(m);
298
335
  m.quarantinedAt = now();
299
336
  if (isQuarantine(m.pool)) {
@@ -302,7 +339,27 @@ function createBalancer(kernel, opts = {}) {
302
339
  }
303
340
  move(m, QUARANTINE, reason, state);
304
341
  };
305
- const movable = (m) => m.state !== "moving" && m.holds === 0;
342
+ const movable = (m) => live(m) && m.holds === 0;
343
+ /**
344
+ * A member the kernel gave up on beats again: the app restarted it
345
+ * (`kernel.restart`). It's placed again as new (M3j): no offences, no
346
+ * history, and its pool's rules from here.
347
+ */
348
+ const revive = (m) => {
349
+ m.state = isQuarantine(m.pool) ? "quarantined" : "placed";
350
+ m.offences = 0;
351
+ m.quarantinedAt = isQuarantine(m.pool) ? now() : -Infinity;
352
+ m.lastSpikeAt = m.spikeNotedAt = m.decayAt = m.risingAt = m.stuckAt = -Infinity;
353
+ m.arrivedAt = now();
354
+ m.moveTimes = [];
355
+ m.unmeasured = false;
356
+ m.beats = [];
357
+ m.lastBeatAt = void 0;
358
+ m.backlogs = [];
359
+ m.backlogHistory = [];
360
+ m.busyReports = [];
361
+ changed(m.pool);
362
+ };
306
363
  const trim = (beats, at) => {
307
364
  const cut = at - windowMs;
308
365
  while ((beats[0]?.at ?? Infinity) <= cut) beats.shift();
@@ -338,6 +395,7 @@ function createBalancer(kernel, opts = {}) {
338
395
  return;
339
396
  }
340
397
  if (m.state === "moving" || e.pool !== m.pool) return;
398
+ if (m.state === "failed") revive(m);
341
399
  if (m.arrival) {
342
400
  const { from, to, reason, at } = m.arrival;
343
401
  delete m.arrival;
@@ -454,9 +512,21 @@ function createBalancer(kernel, opts = {}) {
454
512
  for (const id of foreign.keys()) if (!present.has(id)) foreign.delete(id);
455
513
  };
456
514
  /**
515
+ * A quarantine pool a bisect from `name` can have to itself (M3j): the
516
+ * first quarantine if it holds only members this bisect moved (or none), else a
517
+ * new quarantine pool if the cap allows. Never one that holds offenders.
518
+ */
519
+ const bisectPool = (name) => {
520
+ const free = membersOf(QUARANTINE).every((o) => o.offences === 0 && bisect?.moved.has(o.manifest.id) === true);
521
+ if (!isQuarantine(name) && free) return QUARANTINE;
522
+ return roomForPool() ? freshName(QUARANTINES, 2) : void 0;
523
+ };
524
+ /**
457
525
  * Finds the member behind a pool's trouble when the kernel or `busyMs`
458
526
  * can't name it (Q20): the heavier half of the suspects moves, and
459
- * whichever half acts up again holds the culprit.
527
+ * whichever half acts up again holds the culprit. Only into a quarantine
528
+ * pool of its own (M3j); with none free, the pool stays as it is and its
529
+ * members are `behind`.
460
530
  */
461
531
  const search = (name, reason) => {
462
532
  const here = membersOf(name).filter((m) => m.holds === 0);
@@ -477,18 +547,25 @@ function createBalancer(kernel, opts = {}) {
477
547
  return;
478
548
  }
479
549
  const sorted = suspects.map((id) => members.get(id)).filter((m) => m !== void 0).toSorted((a, b) => (loadOf(b) ?? 0) - (loadOf(a) ?? 0));
550
+ const to = bisectPool(name);
551
+ if (to === void 0) {
552
+ const t = now();
553
+ for (const m of here) {
554
+ m.stuckAt = t;
555
+ behind(m, t);
556
+ }
557
+ return;
558
+ }
480
559
  const half = sorted.slice(0, Math.ceil(sorted.length / 2));
481
560
  bisect = {
482
561
  suspects: new Set(suspects),
483
562
  moved: /* @__PURE__ */ new Set([...bisect?.moved ?? [], ...half.map((m) => m.manifest.id)]),
484
563
  at: now()
485
564
  };
486
- const to = isQuarantine(name) ? roomForPool() ? freshName(QUARANTINES, 2) : void 0 : QUARANTINE;
487
- if (to === void 0) return;
488
- for (const m of half) if (to === QUARANTINE) {
489
- m.quarantinedAt = now();
565
+ for (const m of half) {
566
+ if (!isQuarantine(name)) m.quarantinedAt = now();
490
567
  move(m, to, "bisect", stateFor(m));
491
- } else move(m, to, "bisect", m.state);
568
+ }
492
569
  };
493
570
  /**
494
571
  * The moves that would bring pool `name` back under (M3i): the heaviest
@@ -529,15 +606,20 @@ function createBalancer(kernel, opts = {}) {
529
606
  /** The latest report, if fresh. */
530
607
  const backlogNow = (m, t) => m.backlog && t - m.backlog.at <= STALE_MS ? m.backlog.value : void 0;
531
608
  /**
532
- * Rising (M3i): the latest fresh report is above 0 and higher than one taken
533
- * at least a window before it, both since it last moved and since the last
534
- * backlog move.
609
+ * Rising (M3i; two windows since M3j): the latest fresh report is above 0
610
+ * and higher than one taken at least a window before it, which is itself
611
+ * higher than one taken at least a window before that, all since it last
612
+ * moved and since the last backlog move. A queue at a steady rate that
613
+ * jumps once and holds isn't falling behind.
535
614
  */
536
615
  const rising = (m, t) => {
537
616
  const latest = m.backlog;
538
617
  if (backlogNow(m, t) === void 0 || !latest || latest.value <= 0) return false;
539
- const start = m.backlogs.findLast((b) => b.at >= lastBacklogMoveAt && b.at <= latest.at - BACKLOG_WINDOW_MS);
540
- return start !== void 0 && latest.value > start.value;
618
+ const before = (at) => m.backlogs.findLast((b) => b.at >= lastBacklogMoveAt && b.at <= at - BACKLOG_WINDOW_MS);
619
+ const mid = before(latest.at);
620
+ if (mid === void 0 || latest.value <= mid.value) return false;
621
+ const early = before(mid.at);
622
+ return early !== void 0 && mid.value > early.value;
541
623
  };
542
624
  /**
543
625
  * Somewhere with more CPU for `m` (M3i): a pool of its own if the cap
@@ -575,7 +657,7 @@ function createBalancer(kernel, opts = {}) {
575
657
  const order = [...members.values()].filter((m) => {
576
658
  const r = m.state === "placed" && !isQuarantine(m.pool) && rising(m, t);
577
659
  if (r) m.risingAt = t;
578
- else m.behind = false;
660
+ else if (t - m.stuckAt >= windowMs) m.behind = false;
579
661
  return r;
580
662
  }).toSorted((a, b) => poolLoad(b.pool) - poolLoad(a.pool) || (loadOf(b) ?? 0) - (loadOf(a) ?? 0));
581
663
  for (const m of order) {
@@ -592,29 +674,70 @@ function createBalancer(kernel, opts = {}) {
592
674
  }
593
675
  };
594
676
  /**
595
- * Packing back (M3i): the lightest member alone in its pool and quiet for
596
- * `PACK_STEADY_MS` joins the fullest other pool that stays under
597
- * `PACK_PCT` with it, which frees its worker. Never one on hold, with a
598
- * backlog, in quarantine or kept apart, and never into a pool whose
599
- * members have a backlog.
677
+ * Its backlog rose over the last `PACK_STEADY_MS` (M3j): the latest fresh
678
+ * report is higher than the one at or just before the window's start (or
679
+ * the oldest since it moved, when it hasn't reported that long). A
680
+ * queue that holds steady while work arrives doesn't count, nor does a
681
+ * stale report or none.
682
+ */
683
+ const roseLately = (m, t) => {
684
+ const latest = backlogNow(m, t);
685
+ if (latest === void 0) return false;
686
+ const start = m.backlogHistory.findLast((b) => b.at <= t - PACK_STEADY_MS) ?? m.backlogHistory[0];
687
+ return start !== void 0 && latest > start.value;
688
+ };
689
+ /**
690
+ * Packing back (M3i; whole pools since M3j): the lightest pool whose
691
+ * members have all been quiet for `PACK_STEADY_MS` empties into the
692
+ * fullest other pool that stays under `PACK_PCT` with all of it, one
693
+ * member per pack, which frees its worker once the last one is in. A merge
694
+ * that has started is finished first. Never a pool with a member on hold,
695
+ * out of budget, whose backlog rose over the window (M3j), or a
696
+ * quarantine; never into a pool that's moving or whose members' backlog
697
+ * rose.
600
698
  */
601
699
  const pack = (t) => {
602
700
  const quiet = (since) => t - since >= PACK_STEADY_MS;
603
- const steady = (m) => quiet(Math.max(m.arrivedAt, m.lastSpikeAt, m.spikeNotedAt, m.risingAt, pool(m.pool).troubleAt));
604
- const alone = [...members.values()].filter((m) => m.state === "placed" && isOurs(m.pool) && !isQuarantine(m.pool) && movable(m) && inBudget(m) && membersOf(m.pool).length === 1 && (backlogNow(m, t) ?? 0) === 0 && loadOf(m) !== void 0 && steady(m)).toSorted((a, b) => (loadOf(a) ?? 0) - (loadOf(b) ?? 0));
605
- for (const m of alone) {
606
- const load = loadOf(m) ?? 0;
607
- const into = ownPools().filter((n) => n !== m.pool && !recentlySpiked(n) && quiet(pool(n).troubleAt)).filter((n) => membersOf(n).every((o) => o.state !== "moving" && !(backlogNow(o, t) ?? 0))).map((n) => ({
701
+ const steady = (m) => quiet(Math.max(m.arrivedAt, m.lastSpikeAt, m.spikeNotedAt, pool(m.pool).troubleAt)) && !roseLately(m, t);
702
+ const packable = (m) => m.state === "placed" && movable(m) && inBudget(m) && loadOf(m) !== void 0 && steady(m);
703
+ const sources = ownPools().filter((n) => isOurs(n)).map((n) => ({
704
+ n,
705
+ all: inPool(n),
706
+ load: poolLoad(n)
707
+ })).filter((c) => c.all.length > 0 && c.all.every(packable)).toSorted((a, b) => Number(b.n === merging?.from) - Number(a.n === merging?.from) || a.load - b.load);
708
+ for (const from of sources) {
709
+ const into = ownPools().filter((n) => n !== from.n && isOurs(n) && !recentlySpiked(n) && quiet(pool(n).troubleAt)).filter((n) => inPool(n).every((o) => o.state !== "moving" && !roseLately(o, t))).map((n) => ({
608
710
  n,
609
711
  load: poolLoad(n)
610
- })).filter((c) => c.load + load < PACK_PCT).toSorted((a, b) => b.load - a.load)[0];
712
+ })).filter((c) => c.load + from.load < PACK_PCT).toSorted((a, b) => Number(b.n === merging?.into) - Number(a.n === merging?.into) || b.load - a.load)[0];
611
713
  if (!into) continue;
714
+ const m = from.all.toSorted((a, b) => (loadOf(a) ?? 0) - (loadOf(b) ?? 0))[0];
715
+ if (!m) continue;
612
716
  lastPackAt = t;
717
+ merging = from.all.length > 1 ? {
718
+ from: from.n,
719
+ into: into.n
720
+ } : void 0;
613
721
  move(m, into.n, "pack", "placed");
614
722
  return;
615
723
  }
616
724
  };
617
- const steadyFor = (m) => steadyMs * 2 ** Math.max(0, m.offences - 1);
725
+ /** The quiet window: doubled per offence, capped at 2 minutes (or `steadyMs`, if longer; M3j). */
726
+ const steadyFor = (m) => Math.min(Math.max(steadyMs, QUIET_CAP_MS), steadyMs * 2 ** Math.max(0, m.offences - 1));
727
+ /**
728
+ * Offences decay (M3j): an offender back in a balancer pool loses one for
729
+ * each quiet window it completes there, so one that stays quiet is back to
730
+ * the shortest window. One that offends again soon after leaving still
731
+ * gets a longer one, and `kept-apart` stays apart.
732
+ */
733
+ const decay = (t) => {
734
+ for (const m of members.values()) {
735
+ if (m.state !== "placed" || m.offences === 0) continue;
736
+ if (t - Math.max(m.decayAt, m.lastSpikeAt, m.spikeNotedAt) < steadyFor(m)) continue;
737
+ m.offences--;
738
+ m.decayAt = t;
739
+ }
740
+ };
618
741
  const decide = () => {
619
742
  const t = now();
620
743
  checkForeign();
@@ -655,8 +778,9 @@ function createBalancer(kernel, opts = {}) {
655
778
  } else first.atCap = true;
656
779
  }
657
780
  backlogs(t);
781
+ decay(t);
658
782
  for (const m of members.values()) {
659
- if (m.state !== "quarantined" || !movable(m) || !inBudget(m)) continue;
783
+ if (m.state !== "quarantined" || !movable(m) || !(m.unmeasured || inBudget(m))) continue;
660
784
  const fresh = average(m.beats, windowMs, t);
661
785
  if (m.offences === 0) {
662
786
  if (fresh === void 0 || bisect?.moved.has(m.manifest.id)) continue;
@@ -669,6 +793,7 @@ function createBalancer(kernel, opts = {}) {
669
793
  continue;
670
794
  }
671
795
  m.atCap = false;
796
+ m.decayAt = t;
672
797
  move(m, to.pool, "steady", "placed");
673
798
  }
674
799
  if (packing && t - lastPackAt >= PACK_EVERY_MS) pack(t);
@@ -679,22 +804,53 @@ function createBalancer(kernel, opts = {}) {
679
804
  };
680
805
  const onCrash = (e) => {
681
806
  const m = members.get(e.id);
682
- if (!m || m.state === "moving") return;
807
+ if (!m || !live(m)) return;
683
808
  if (POOL_KILL.has(e.reason) && e.pool !== void 0) killed.add(e.pool);
684
809
  else if (e.reason === "error") {
685
810
  if (m.holds > 0) m.lastSpikeAt = now();
686
811
  else quarantine(m, "crash", true);
687
812
  }
688
813
  };
814
+ /**
815
+ * The kernel gave up on a member (M3j): it leaves its pool's load, the
816
+ * quarantine's count and any bisect, and stays where it is until the app
817
+ * restarts it. A member mid-move was stopped by the balancer itself; its
818
+ * respawn is a fresh start.
819
+ */
820
+ const onFailed = (e) => {
821
+ const m = members.get(e.id);
822
+ if (!m || !live(m)) return;
823
+ m.state = "failed";
824
+ m.beats = [];
825
+ m.lastBeatAt = void 0;
826
+ m.atCap = false;
827
+ m.behind = false;
828
+ m.busyReports = [];
829
+ delete m.arrival;
830
+ bisect?.suspects.delete(e.id);
831
+ bisect?.moved.delete(e.id);
832
+ changed(m.pool);
833
+ emit("failed", {
834
+ id: e.id,
835
+ pool: m.pool,
836
+ reason: e.reason
837
+ });
838
+ };
689
839
  const timer = setInterval(decide, DECIDE_MS);
690
- const offs = [kernel.on("isolate:crash", onCrash), kernel.on("isolate:beat", onBeat)];
840
+ const offs = [
841
+ kernel.on("isolate:crash", onCrash),
842
+ kernel.on("isolate:beat", onBeat),
843
+ kernel.on("isolate:failed", onFailed)
844
+ ];
691
845
  const placement = (m) => {
692
846
  const p = {
693
847
  id: m.manifest.id,
694
848
  pool: m.pool,
695
849
  state: m.state,
696
- loadPct: loadOf(m)
850
+ loadPct: m.state === "failed" ? void 0 : loadOf(m)
697
851
  };
852
+ const reported = m.state === "failed" ? void 0 : reportedOf(m, now());
853
+ if (reported !== void 0) p.reportedPct = reported;
698
854
  if (m.atCap) p.atCap = true;
699
855
  const backlog = backlogNow(m, now());
700
856
  if (backlog !== void 0) p.backlog = backlog;
@@ -724,15 +880,20 @@ function createBalancer(kernel, opts = {}) {
724
880
  savedLoad: known,
725
881
  offences: 0,
726
882
  quarantinedAt: isQuarantine(dest.pool) ? now() : -Infinity,
883
+ decayAt: -Infinity,
727
884
  lastSpikeAt: -Infinity,
728
885
  spikeNotedAt: -Infinity,
729
886
  holds: 0,
730
887
  atCap: dest.atCap,
731
888
  arrivedAt: now(),
732
889
  moveTimes: [],
890
+ unmeasured: known === void 0,
733
891
  backlogs: [],
734
892
  backlog: void 0,
893
+ backlogHistory: [],
894
+ busyReports: [],
735
895
  risingAt: -Infinity,
896
+ stuckAt: -Infinity,
736
897
  behind: false,
737
898
  behindAt: -Infinity
738
899
  };
@@ -775,18 +936,33 @@ function createBalancer(kernel, opts = {}) {
775
936
  m.holds--;
776
937
  };
777
938
  },
778
- report(id, { backlog }) {
779
- if (typeof backlog !== "number" || !Number.isFinite(backlog) || backlog < 0) throw new TypeError("balancer.report: backlog must be a finite number, 0 or more");
939
+ report(id, { backlog, busyMs }) {
940
+ if (backlog === void 0 && busyMs === void 0) throw new TypeError("balancer.report: give a backlog, a busyMs or both");
941
+ if (backlog !== void 0 && !valid(backlog)) throw new TypeError("balancer.report: backlog must be a finite number, 0 or more");
942
+ if (busyMs !== void 0 && !valid(busyMs)) throw new TypeError("balancer.report: busyMs must be a finite number, 0 or more");
780
943
  const m = members.get(id);
781
- if (!m) return;
944
+ if (!m || m.state === "failed") return;
782
945
  const t = now();
783
- const sample = {
784
- at: t,
785
- value: backlog
786
- };
787
- m.backlog = sample;
788
- m.backlogs.push(sample);
789
- while ((m.backlogs[1]?.at ?? Infinity) <= t - BACKLOG_WINDOW_MS) m.backlogs.shift();
946
+ if (backlog !== void 0) {
947
+ const sample = {
948
+ at: t,
949
+ value: backlog
950
+ };
951
+ m.backlog = sample;
952
+ m.backlogs.push(sample);
953
+ m.backlogHistory.push(sample);
954
+ while ((m.backlogs[1]?.at ?? Infinity) <= t - 3 * BACKLOG_WINDOW_MS) m.backlogs.shift();
955
+ while ((m.backlogHistory[1]?.at ?? Infinity) <= t - PACK_STEADY_MS) m.backlogHistory.shift();
956
+ }
957
+ if (busyMs !== void 0) {
958
+ if ((m.busyReports.at(-1)?.total ?? 0) > busyMs) m.busyReports = [];
959
+ m.busyReports.push({
960
+ at: t,
961
+ total: busyMs
962
+ });
963
+ const cut = t - windowMs - STALE_MS;
964
+ while ((m.busyReports[1]?.at ?? Infinity) <= cut) m.busyReports.shift();
965
+ }
790
966
  },
791
967
  measure(id) {
792
968
  const m = members.get(id);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@isoloop/balancer",
3
- "version": "0.1.0-next.0",
3
+ "version": "0.1.0-next.2",
4
4
  "description": "Experimental automatic placement for Isoloop isolates: joins a pool or keeps apart from measured load, quarantines what blocks.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -20,7 +20,7 @@
20
20
  }
21
21
  },
22
22
  "dependencies": {
23
- "@isoloop/host": "0.1.0-next.0"
23
+ "@isoloop/host": "0.1.0-next.2"
24
24
  },
25
25
  "devDependencies": {
26
26
  "typescript": "7.0.2"