@isoloop/balancer 0.1.0-next.0 → 0.1.0-next.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -29,7 +29,20 @@ balancer.on('arrived', (e) => console.log(e.id, 'moved in', e.costMs, 'ms'));
29
29
  - **Backlog:** a backlog higher at the end of a 10 s window than at its start, and above 0, moves the isolate somewhere with more CPU: a pool of its own if the cap allows, else a pool with less load than its pool-mates. One such move per fresh 10 s window, so a group's members move one at a time. A report older than 10 s counts as none. With nowhere to go, it emits `behind` (at most every 30 s).
30
30
  - **Fewer moves:** a move for load is made only if the moves it starts are predicted to bring the pool back under its line, and each isolate gets at most 3 of them in 5 minutes. Spike, crash and bisect moves protect the others and aren't limited.
31
31
  - **Spikes:** a first long task is only reported (`spike`, action `noted`); a second from the same isolate within 60 s sends it to quarantine. Its own fatal error still does at once.
32
- - **Packing back:** an isolate alone in its pool and quiet for 60 s joins another of the balancer's pools if both stay under 40 % of a core, which frees a worker; at most one every 60 s. Turn it off with `pack: { enabled: false }`.
32
+ - **Packing back:** an isolate alone in its pool and steady for 60 s joins another of the balancer's pools if both stay under 40 % of a core, which frees a worker; at most one every 60 s. Turn it off with `pack: { enabled: false }`.
33
33
  - **Move cost:** `arrived` follows each `move` with `costMs`, the time from stopping the isolate to its first beat in the new pool.
34
34
 
35
+ And when its busy time misses work, or the kernel gives up on an isolate (M3j):
36
+
37
+ ```ts
38
+ // Work timed by the app, as a running total since the isolate started: for slices resumed after `await`.
39
+ balancer.report('task-1', { backlog: queue.length, busyMs: workMsSoFar });
40
+ balancer.on('failed', (e) => console.warn(e.id, 'was given up on in', e.pool, e.reason));
41
+ ```
42
+
43
+ - **Reported busy time:** the balancer takes the total's difference over its 3 s window and uses the larger of that and the beats' `busyMs`, wherever it reads a load.
44
+ - **Packing** goes ahead while a backlog holds steady; only a backlog that rose over the 60 s window blocks it.
45
+ - **The quarantine:** an offender's quiet window is capped at 2 minutes, and offences wear off with each quiet window back in a pool. A bisect starts only with a quarantine pool of its own, never alongside offenders; with none free, the pool's members are `behind`.
46
+ - **Given up:** an isolate past the kernel's restart limit is `failed`. It leaves its pool's load and any bisect, and no move restarts it; after `kernel.restart(id)` it's placed again as new.
47
+
35
48
  One balancer per kernel at a time (`dispose()` frees the kernel for another); its pools are named `balancer-…`, which the host reserves. The API, the triggers, the quarantine and what `busyMs` measures are in the [balancer guide](https://github.com/isoloop/isoloop/blob/main/docs/guide/balancer.md).
package/dist/index.d.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * The part of the kernel the balancer uses: the public API only (`spawn`,
3
- * `stop`, `pools`, and the `isolate:beat` and `isolate:crash` events), so
4
- * `@isoloop/host`'s `Kernel` fits it as is.
3
+ * `stop`, `pools`, and the `isolate:beat`, `isolate:crash` and
4
+ * `isolate:failed` events), so `@isoloop/host`'s `Kernel` fits it as is.
5
5
  */
6
6
  interface KernelLike<M extends ManifestLike = ManifestLike> {
7
7
  spawn(manifest: M): unknown;
@@ -10,6 +10,8 @@ interface KernelLike<M extends ManifestLike = ManifestLike> {
10
10
  on(event: "isolate:crash", fn: (e: CrashLike) => void): () => void;
11
11
  /** Every heartbeat as it arrives, once per member of a pool (ADR 0017). */
12
12
  on(event: "isolate:beat", fn: (e: BeatLike) => void): () => void;
13
+ /** An isolate past the kernel's restart limit: it stays down until `kernel.restart` (M3j). */
14
+ on(event: "isolate:failed", fn: (e: FailedLike) => void): () => void;
13
15
  /**
14
16
  * Only for kernels not made by `createKernel` (a simulated one in tests):
15
17
  * the claim of the `balancer-*` pools, as the host's `claimPools` gives it.
@@ -51,6 +53,12 @@ type CrashLike = {
51
53
  reason: string;
52
54
  pool?: string;
53
55
  };
56
+ /** The fields of the kernel's `isolate:failed` event the balancer reads (M3j). */
57
+ type FailedLike = {
58
+ id: string;
59
+ reason: string;
60
+ pool?: string;
61
+ };
54
62
  /**
55
63
  * Where the balancer keeps each isolate's measured load between page loads,
56
64
  * so a reload places from history (M3h, Q17). The app supplies it; the
@@ -95,11 +103,14 @@ type BalancerOptions = {
95
103
  };
96
104
  /**
97
105
  * Leaving quarantine: an isolate there only to be measured leaves once it
98
- * has 3 s of beats there. An offender leaves after `steadyMs` (default
99
- * 30,000 ms) with no spike and no crash, whatever its load; each offence
100
- * doubles the window (30, 60, 120 s), and after `maxReturns` returns
101
- * (default 3) the next offence keeps it apart for good. Either goes where
102
- * the placement rule finds room, or stays, flagged `atCap`, until there is.
106
+ * has 3 s of beats there, or, if a bisect moved it there, once that bisect
107
+ * ends. An offender leaves after `steadyMs` (default 30,000 ms) with no
108
+ * spike and no crash, whatever its load; each offence doubles the window
109
+ * (30, 60, 120 s), capped at 2 minutes (or `steadyMs`, if longer). Back in
110
+ * a balancer pool, an offender loses one offence for each quiet window it
111
+ * completes there (M3j). After `maxReturns` returns (default 3) the next
112
+ * offence keeps it apart for good. Either goes where the placement rule
113
+ * finds room, or stays, flagged `atCap`, until there is.
103
114
  */
104
115
  quarantine?: {
105
116
  steadyMs?: number;
@@ -108,35 +119,61 @@ type BalancerOptions = {
108
119
  loads?: LoadStore;
109
120
  /**
110
121
  * Packing back (M3i): a member alone in its pool and steady for 60 s (no
111
- * spike, crash, lagging or full pool, rising backlog or backlog above 0)
112
- * joins another of the balancer's pools if both together stay under 40 %
113
- * of a core, freeing a worker. At most one pack every 60 s. On by default.
122
+ * spike, crash, lagging or full pool, and a backlog no higher than 60 s
123
+ * before, M3j) joins another of the balancer's pools if both together stay
124
+ * under 40 % of a core, freeing a worker. At most one pack every 60 s. On by
125
+ * default.
114
126
  */
115
127
  pack?: {
116
128
  enabled?: boolean;
117
129
  };
118
130
  };
119
- /** What the app tells the balancer about an isolate's queued work (`report`). */
131
+ /**
132
+ * What the app tells the balancer about an isolate (`report`): its queued
133
+ * work, its own measured busy time, or both. At least one must be given.
134
+ */
120
135
  type Report = {
121
136
  /**
122
137
  * Work items waiting for the isolate, in any unit the app likes, as long as
123
138
  * it's the same one each time. For a consumer group, report the group's
124
139
  * backlog on every member.
125
140
  */
126
- backlog: number;
141
+ backlog?: number;
142
+ /**
143
+ * Milliseconds of work the app measured for the isolate since it started:
144
+ * a running total, so a late or missed report still counts (M3j). For work
145
+ * the kernel's `busyMs` misses, such as slices resumed after `await`. The
146
+ * balancer takes its difference over the 3 s window and uses the larger of
147
+ * that and the beats' figure. A total that goes down is taken as a restart
148
+ * of the count.
149
+ */
150
+ busyMs?: number;
127
151
  };
128
- type PlacementState = "placed" | "quarantined" | "kept-apart" | "moving";
152
+ /**
153
+ * `failed`: the kernel gave up on the isolate (past its restart limit, M3j).
154
+ * The balancer never moves or restarts it; after `kernel.restart(id)` it's
155
+ * placed again as new.
156
+ */
157
+ type PlacementState = "placed" | "quarantined" | "kept-apart" | "moving" | "failed";
129
158
  type Placement = {
130
159
  id: string;
131
160
  pool: string;
132
161
  state: PlacementState;
133
- /** Measured share of one core, 0–100; `undefined` until measured. */
162
+ /**
163
+ * Measured share of one core, 0–100: the larger of the beats' `busyMs` and
164
+ * the app's reported `busyMs` over the last 3 s; `undefined` until measured.
165
+ */
134
166
  loadPct: number | undefined;
167
+ /** The share of one core from the app's `busyMs` reports alone, while they're fresh (M3j). */
168
+ reportedPct?: number;
135
169
  /** No pool had room and the worker cap was reached: placed anyway, or left where it is. */
136
170
  atCap?: true;
137
171
  /** The latest backlog `report`, while it's fresh (under 10 s old). */
138
172
  backlog?: number;
139
- /** Its backlog is rising, and the balancer has nowhere to give it more CPU (`behind`). */
173
+ /**
174
+ * Its backlog is rising, or its pool needs a bisect, and the balancer has
175
+ * nowhere to give it (`behind`).
176
+ */
140
177
  behind?: true;
141
178
  /**
142
179
  * In one of the balancer's pools, but not placed by it. The host reserves
@@ -182,14 +219,25 @@ type SpikeEvent = {
182
219
  };
183
220
  /**
184
221
  * A member's reported backlog keeps rising, but it already has its pool to
185
- * itself or the balancer has no worker or pool to give it (M3i). Emitted when
186
- * it first happens, then at most every 30 s while it lasts.
222
+ * itself or the balancer has no worker or pool to give it (M3i). Also when
223
+ * its pool's trouble needs a bisect and no quarantine pool is free for one
224
+ * (M3j). Emitted when it first happens, then at most every 30 s while it
225
+ * lasts. `backlog` is the latest fresh report, or 0.
187
226
  */
188
227
  type BehindEvent = {
189
228
  id: string;
190
229
  pool: string;
191
230
  backlog: number;
192
231
  };
232
+ /**
233
+ * The kernel gave up on a member (`isolate:failed`, M3j). It drops out of its
234
+ * pool's load and any bisect, and no move restarts it.
235
+ */
236
+ type FailedEvent = {
237
+ id: string;
238
+ pool: string;
239
+ reason: string;
240
+ };
193
241
  /** An isolate the balancer didn't place turned up in one of its pools (see `Placement.foreign`). */
194
242
  type ForeignEvent = {
195
243
  id: string;
@@ -200,6 +248,7 @@ type BalancerEvents = {
200
248
  arrived: ArrivedEvent;
201
249
  spike: SpikeEvent;
202
250
  behind: BehindEvent;
251
+ failed: FailedEvent;
203
252
  foreign: ForeignEvent;
204
253
  };
205
254
  /**
@@ -221,11 +270,14 @@ interface Balancer<M extends ManifestLike = ManifestLike> {
221
270
  /** Leaves `id` where it is until the returned function is called (a widget open for editing). */
222
271
  hold(id: string): () => void;
223
272
  /**
224
- * The app's view of `id`'s queued work (M3i). A backlog higher at the end of
225
- * a 10 s window than at its start, and above 0, moves the isolate somewhere
226
- * with more CPU; a report older than 10 s counts as none, so report at least
227
- * every few seconds while there's work. Ignored for isolates the balancer
228
- * didn't place.
273
+ * The app's view of `id`'s queued work (M3i) and of its own busy time (M3j).
274
+ * A backlog higher at the end of a 10 s window than at its start, and above
275
+ * 0, moves the isolate somewhere with more CPU. A reported `busyMs` total
276
+ * counts toward its load wherever the balancer reads one. A report older
277
+ * than 10 s counts as none, so report at least every few seconds. Throws a
278
+ * `TypeError` for a report with neither field, or a field that isn't a
279
+ * finite number of 0 or more. Ignored for isolates the balancer didn't place
280
+ * and ones the kernel gave up on.
229
281
  */
230
282
  report(id: string, report: Report): void;
231
283
  on<E extends keyof BalancerEvents>(event: E, fn: (e: BalancerEvents[E]) => void): () => void;
@@ -247,4 +299,4 @@ interface Balancer<M extends ManifestLike = ManifestLike> {
247
299
  * `kernel.spawn` are never touched, and don't count toward `maxWorkers`.
248
300
  */
249
301
  export declare function createBalancer<M extends ManifestLike>(kernel: KernelLike<M>, opts?: BalancerOptions): Balancer<M>;
250
- export type { ArrivedEvent, Balancer, BalancerEvents, BalancerOptions, BeatLike, BehindEvent, CrashLike, ForeignEvent, KernelLike, LoadStore, ManifestLike, MoveEvent, MoveReason, Placement, PlacementState, PoolLike, Report, SpikeEvent };
302
+ export type { ArrivedEvent, Balancer, BalancerEvents, BalancerOptions, BeatLike, BehindEvent, CrashLike, FailedEvent, FailedLike, ForeignEvent, KernelLike, LoadStore, ManifestLike, MoveEvent, MoveReason, Placement, PlacementState, PoolLike, Report, SpikeEvent };
package/dist/index.js CHANGED
@@ -38,6 +38,8 @@ const POOLS = "balancer";
38
38
  const QUARANTINES = "balancer-quarantine";
39
39
  const QUARANTINE = `${QUARANTINES}-1`;
40
40
  const now = () => performance.now();
41
+ /** A report's figure: a finite number, 0 or more. */
42
+ const valid = (x) => typeof x === "number" && Number.isFinite(x) && x >= 0;
41
43
  /**
42
44
  * `busyMs` is approximate: shared framework work is split between members and
43
45
  * promise continuations go unattributed. When the members' busy time explains
@@ -72,6 +74,8 @@ const MOVE_BUDGET_MS = 3e5;
72
74
  const PACK_STEADY_MS = 6e4;
73
75
  const PACK_PCT = 40;
74
76
  const PACK_EVERY_MS = 6e4;
77
+ /** An offender's quiet window doubles with each offence up to this, or `steadyMs` if longer (M3j). */
78
+ const QUIET_CAP_MS = 12e4;
75
79
  /** Worker crash reasons whose culprit the kernel can't name: every member of the pool crashes. */
76
80
  const POOL_KILL = /* @__PURE__ */ new Set(["unresponsive", "worker-error"]);
77
81
  const defaultMaxWorkers = () => {
@@ -121,6 +125,7 @@ function createBalancer(kernel, opts = {}) {
121
125
  arrived: /* @__PURE__ */ new Set(),
122
126
  spike: /* @__PURE__ */ new Set(),
123
127
  behind: /* @__PURE__ */ new Set(),
128
+ failed: /* @__PURE__ */ new Set(),
124
129
  foreign: /* @__PURE__ */ new Set()
125
130
  };
126
131
  /** Beats of isolates the balancer didn't place, for `measure` (previews included). */
@@ -160,7 +165,9 @@ function createBalancer(kernel, opts = {}) {
160
165
  }
161
166
  return p;
162
167
  };
163
- const membersOf = (name) => [...members.values()].filter((m) => m.pool === name && m.state !== "moving");
168
+ /** Running in its pool: not mid-move, and not given up on by the kernel. */
169
+ const live = (m) => m.state !== "moving" && m.state !== "failed";
170
+ const membersOf = (name) => [...members.values()].filter((m) => m.pool === name && live(m));
164
171
  const changed = (name) => {
165
172
  const p = pool(name);
166
173
  p.lags = [];
@@ -168,8 +175,28 @@ function createBalancer(kernel, opts = {}) {
168
175
  p.beat = [];
169
176
  p.since = now();
170
177
  };
178
+ /**
179
+ * The app's reported busy time (M3j) as a share of one core: the running
180
+ * total's difference over about a window, from a fresh report back to one
181
+ * taken at least 90 % of a window before it.
182
+ */
183
+ const reportedOf = (m, t) => {
184
+ const latest = m.busyReports.at(-1);
185
+ if (!latest || t - latest.at > STALE_MS) return void 0;
186
+ const start = m.busyReports.findLast((r) => r.at <= latest.at - windowMs * .9);
187
+ if (!start) return void 0;
188
+ const ms = latest.at - start.at;
189
+ return Math.min(100, (latest.total - start.total) / ms * 100);
190
+ };
191
+ /** The beats' and the report's measures over the last window, whichever is larger. */
192
+ const measured = (m, t) => {
193
+ const beat = average(m.beats, windowMs, t);
194
+ const reported = reportedOf(m, t);
195
+ if (beat === void 0) return reported;
196
+ return reported === void 0 ? beat : Math.max(beat, reported);
197
+ };
171
198
  /** Measured load over the last window, or the last one known. */
172
- const loadOf = (m) => average(m.beats, windowMs, now()) ?? m.knownLoad;
199
+ const loadOf = (m) => measured(m, now()) ?? m.knownLoad;
173
200
  const poolLoad = (name) => membersOf(name).reduce((s, m) => s + (loadOf(m) ?? 0), 0);
174
201
  const windowFull = (name) => {
175
202
  const p = pool(name);
@@ -182,7 +209,7 @@ function createBalancer(kernel, opts = {}) {
182
209
  * app runs itself don't count (ADR 0017, amended).
183
210
  */
184
211
  const ownWorkers = () => kernel.pools().filter((p) => p.running && isOurs(p.name) && p.name !== QUARANTINE).length;
185
- const ownPools = () => [...new Set([...members.values()].map((m) => m.pool))].filter((n) => !isQuarantine(n));
212
+ const ownPools = () => [...new Set([...members.values()].filter((m) => m.state !== "failed").map((m) => m.pool))].filter((n) => !isQuarantine(n));
186
213
  /** Room for one more worker of the balancer's own. */
187
214
  const roomForPool = () => ownWorkers() + 1 <= maxWorkers;
188
215
  const freshName = (prefix, first) => {
@@ -252,6 +279,8 @@ function createBalancer(kernel, opts = {}) {
252
279
  m.lastBeatAt = void 0;
253
280
  m.moveTimes.push(startedAt);
254
281
  m.backlogs = [];
282
+ m.backlogHistory = [];
283
+ m.busyReports = [];
255
284
  m.behind = false;
256
285
  delete m.arrival;
257
286
  m.pool = to;
@@ -302,7 +331,26 @@ function createBalancer(kernel, opts = {}) {
302
331
  }
303
332
  move(m, QUARANTINE, reason, state);
304
333
  };
305
- const movable = (m) => m.state !== "moving" && m.holds === 0;
334
+ const movable = (m) => live(m) && m.holds === 0;
335
+ /**
336
+ * A member the kernel gave up on beats again: the app restarted it
337
+ * (`kernel.restart`). It's placed again as new (M3j): no offences, no
338
+ * history, and its pool's rules from here.
339
+ */
340
+ const revive = (m) => {
341
+ m.state = isQuarantine(m.pool) ? "quarantined" : "placed";
342
+ m.offences = 0;
343
+ m.quarantinedAt = isQuarantine(m.pool) ? now() : -Infinity;
344
+ m.lastSpikeAt = m.spikeNotedAt = m.decayAt = m.risingAt = m.stuckAt = -Infinity;
345
+ m.arrivedAt = now();
346
+ m.moveTimes = [];
347
+ m.beats = [];
348
+ m.lastBeatAt = void 0;
349
+ m.backlogs = [];
350
+ m.backlogHistory = [];
351
+ m.busyReports = [];
352
+ changed(m.pool);
353
+ };
306
354
  const trim = (beats, at) => {
307
355
  const cut = at - windowMs;
308
356
  while ((beats[0]?.at ?? Infinity) <= cut) beats.shift();
@@ -338,6 +386,7 @@ function createBalancer(kernel, opts = {}) {
338
386
  return;
339
387
  }
340
388
  if (m.state === "moving" || e.pool !== m.pool) return;
389
+ if (m.state === "failed") revive(m);
341
390
  if (m.arrival) {
342
391
  const { from, to, reason, at } = m.arrival;
343
392
  delete m.arrival;
@@ -454,9 +503,21 @@ function createBalancer(kernel, opts = {}) {
454
503
  for (const id of foreign.keys()) if (!present.has(id)) foreign.delete(id);
455
504
  };
456
505
  /**
506
+ * A quarantine pool a bisect from `name` can have to itself (M3j): the
507
+ * first quarantine if it holds only members this bisect moved (or none), else a
508
+ * new quarantine pool if the cap allows. Never one that holds offenders.
509
+ */
510
+ const bisectPool = (name) => {
511
+ const free = membersOf(QUARANTINE).every((o) => o.offences === 0 && bisect?.moved.has(o.manifest.id) === true);
512
+ if (!isQuarantine(name) && free) return QUARANTINE;
513
+ return roomForPool() ? freshName(QUARANTINES, 2) : void 0;
514
+ };
515
+ /**
457
516
  * Finds the member behind a pool's trouble when the kernel or `busyMs`
458
517
  * can't name it (Q20): the heavier half of the suspects moves, and
459
- * whichever half acts up again holds the culprit.
518
+ * whichever half acts up again holds the culprit. Only into a quarantine
519
+ * pool of its own (M3j); with none free, the pool stays as it is and its
520
+ * members are `behind`.
460
521
  */
461
522
  const search = (name, reason) => {
462
523
  const here = membersOf(name).filter((m) => m.holds === 0);
@@ -477,18 +538,25 @@ function createBalancer(kernel, opts = {}) {
477
538
  return;
478
539
  }
479
540
  const sorted = suspects.map((id) => members.get(id)).filter((m) => m !== void 0).toSorted((a, b) => (loadOf(b) ?? 0) - (loadOf(a) ?? 0));
541
+ const to = bisectPool(name);
542
+ if (to === void 0) {
543
+ const t = now();
544
+ for (const m of here) {
545
+ m.stuckAt = t;
546
+ behind(m, t);
547
+ }
548
+ return;
549
+ }
480
550
  const half = sorted.slice(0, Math.ceil(sorted.length / 2));
481
551
  bisect = {
482
552
  suspects: new Set(suspects),
483
553
  moved: /* @__PURE__ */ new Set([...bisect?.moved ?? [], ...half.map((m) => m.manifest.id)]),
484
554
  at: now()
485
555
  };
486
- const to = isQuarantine(name) ? roomForPool() ? freshName(QUARANTINES, 2) : void 0 : QUARANTINE;
487
- if (to === void 0) return;
488
- for (const m of half) if (to === QUARANTINE) {
489
- m.quarantinedAt = now();
556
+ for (const m of half) {
557
+ if (!isQuarantine(name)) m.quarantinedAt = now();
490
558
  move(m, to, "bisect", stateFor(m));
491
- } else move(m, to, "bisect", m.state);
559
+ }
492
560
  };
493
561
  /**
494
562
  * The moves that would bring pool `name` back under (M3i): the heaviest
@@ -575,7 +643,7 @@ function createBalancer(kernel, opts = {}) {
575
643
  const order = [...members.values()].filter((m) => {
576
644
  const r = m.state === "placed" && !isQuarantine(m.pool) && rising(m, t);
577
645
  if (r) m.risingAt = t;
578
- else m.behind = false;
646
+ else if (t - m.stuckAt >= windowMs) m.behind = false;
579
647
  return r;
580
648
  }).toSorted((a, b) => poolLoad(b.pool) - poolLoad(a.pool) || (loadOf(b) ?? 0) - (loadOf(a) ?? 0));
581
649
  for (const m of order) {
@@ -592,19 +660,32 @@ function createBalancer(kernel, opts = {}) {
592
660
  }
593
661
  };
594
662
  /**
663
+ * Its backlog rose over the last `PACK_STEADY_MS` (M3j): the latest fresh
664
+ * report is higher than the one at or just before the window's start (or
665
+ * the oldest since it moved, when it hasn't reported that long). A
666
+ * queue that holds steady while work arrives doesn't count, nor does a
667
+ * stale report or none.
668
+ */
669
+ const roseLately = (m, t) => {
670
+ const latest = backlogNow(m, t);
671
+ if (latest === void 0) return false;
672
+ const start = m.backlogHistory.findLast((b) => b.at <= t - PACK_STEADY_MS) ?? m.backlogHistory[0];
673
+ return start !== void 0 && latest > start.value;
674
+ };
675
+ /**
595
676
  * Packing back (M3i): the lightest member alone in its pool and quiet for
596
677
  * `PACK_STEADY_MS` joins the fullest other pool that stays under
597
- * `PACK_PCT` with it, which frees its worker. Never one on hold, with a
598
- * backlog, in quarantine or kept apart, and never into a pool whose
599
- * members have a backlog.
678
+ * `PACK_PCT` with it, which frees its worker. Never one on hold, whose
679
+ * backlog rose over the window (M3j), in quarantine or kept apart, and
680
+ * never into a pool whose members' backlog rose.
600
681
  */
601
682
  const pack = (t) => {
602
683
  const quiet = (since) => t - since >= PACK_STEADY_MS;
603
- const steady = (m) => quiet(Math.max(m.arrivedAt, m.lastSpikeAt, m.spikeNotedAt, m.risingAt, pool(m.pool).troubleAt));
604
- const alone = [...members.values()].filter((m) => m.state === "placed" && isOurs(m.pool) && !isQuarantine(m.pool) && movable(m) && inBudget(m) && membersOf(m.pool).length === 1 && (backlogNow(m, t) ?? 0) === 0 && loadOf(m) !== void 0 && steady(m)).toSorted((a, b) => (loadOf(a) ?? 0) - (loadOf(b) ?? 0));
684
+ const steady = (m) => quiet(Math.max(m.arrivedAt, m.lastSpikeAt, m.spikeNotedAt, pool(m.pool).troubleAt)) && !roseLately(m, t);
685
+ const alone = [...members.values()].filter((m) => m.state === "placed" && isOurs(m.pool) && !isQuarantine(m.pool) && movable(m) && inBudget(m) && membersOf(m.pool).length === 1 && loadOf(m) !== void 0 && steady(m)).toSorted((a, b) => (loadOf(a) ?? 0) - (loadOf(b) ?? 0));
605
686
  for (const m of alone) {
606
687
  const load = loadOf(m) ?? 0;
607
- const into = ownPools().filter((n) => n !== m.pool && !recentlySpiked(n) && quiet(pool(n).troubleAt)).filter((n) => membersOf(n).every((o) => o.state !== "moving" && !(backlogNow(o, t) ?? 0))).map((n) => ({
688
+ const into = ownPools().filter((n) => n !== m.pool && !recentlySpiked(n) && quiet(pool(n).troubleAt)).filter((n) => [...members.values()].filter((o) => o.pool === n).every((o) => o.state !== "moving" && !roseLately(o, t))).map((n) => ({
608
689
  n,
609
690
  load: poolLoad(n)
610
691
  })).filter((c) => c.load + load < PACK_PCT).toSorted((a, b) => b.load - a.load)[0];
@@ -614,7 +695,22 @@ function createBalancer(kernel, opts = {}) {
614
695
  return;
615
696
  }
616
697
  };
617
- const steadyFor = (m) => steadyMs * 2 ** Math.max(0, m.offences - 1);
698
+ /** The quiet window: doubled per offence, capped at 2 minutes (or `steadyMs`, if longer; M3j). */
699
+ const steadyFor = (m) => Math.min(Math.max(steadyMs, QUIET_CAP_MS), steadyMs * 2 ** Math.max(0, m.offences - 1));
700
+ /**
701
+ * Offences decay (M3j): an offender back in a balancer pool loses one for
702
+ * each quiet window it completes there, so one that stays quiet is back to
703
+ * the shortest window. One that offends again soon after leaving still
704
+ * gets a longer one, and `kept-apart` stays apart.
705
+ */
706
+ const decay = (t) => {
707
+ for (const m of members.values()) {
708
+ if (m.state !== "placed" || m.offences === 0) continue;
709
+ if (t - Math.max(m.decayAt, m.lastSpikeAt, m.spikeNotedAt) < steadyFor(m)) continue;
710
+ m.offences--;
711
+ m.decayAt = t;
712
+ }
713
+ };
618
714
  const decide = () => {
619
715
  const t = now();
620
716
  checkForeign();
@@ -655,6 +751,7 @@ function createBalancer(kernel, opts = {}) {
655
751
  } else first.atCap = true;
656
752
  }
657
753
  backlogs(t);
754
+ decay(t);
658
755
  for (const m of members.values()) {
659
756
  if (m.state !== "quarantined" || !movable(m) || !inBudget(m)) continue;
660
757
  const fresh = average(m.beats, windowMs, t);
@@ -669,6 +766,7 @@ function createBalancer(kernel, opts = {}) {
669
766
  continue;
670
767
  }
671
768
  m.atCap = false;
769
+ m.decayAt = t;
672
770
  move(m, to.pool, "steady", "placed");
673
771
  }
674
772
  if (packing && t - lastPackAt >= PACK_EVERY_MS) pack(t);
@@ -679,22 +777,53 @@ function createBalancer(kernel, opts = {}) {
679
777
  };
680
778
  const onCrash = (e) => {
681
779
  const m = members.get(e.id);
682
- if (!m || m.state === "moving") return;
780
+ if (!m || !live(m)) return;
683
781
  if (POOL_KILL.has(e.reason) && e.pool !== void 0) killed.add(e.pool);
684
782
  else if (e.reason === "error") {
685
783
  if (m.holds > 0) m.lastSpikeAt = now();
686
784
  else quarantine(m, "crash", true);
687
785
  }
688
786
  };
787
+ /**
788
+ * The kernel gave up on a member (M3j): it leaves its pool's load, the
789
+ * quarantine's count and any bisect, and stays where it is until the app
790
+ * restarts it. A member mid-move was stopped by the balancer itself; its
791
+ * respawn is a fresh start.
792
+ */
793
+ const onFailed = (e) => {
794
+ const m = members.get(e.id);
795
+ if (!m || !live(m)) return;
796
+ m.state = "failed";
797
+ m.beats = [];
798
+ m.lastBeatAt = void 0;
799
+ m.atCap = false;
800
+ m.behind = false;
801
+ m.busyReports = [];
802
+ delete m.arrival;
803
+ bisect?.suspects.delete(e.id);
804
+ bisect?.moved.delete(e.id);
805
+ changed(m.pool);
806
+ emit("failed", {
807
+ id: e.id,
808
+ pool: m.pool,
809
+ reason: e.reason
810
+ });
811
+ };
689
812
  const timer = setInterval(decide, DECIDE_MS);
690
- const offs = [kernel.on("isolate:crash", onCrash), kernel.on("isolate:beat", onBeat)];
813
+ const offs = [
814
+ kernel.on("isolate:crash", onCrash),
815
+ kernel.on("isolate:beat", onBeat),
816
+ kernel.on("isolate:failed", onFailed)
817
+ ];
691
818
  const placement = (m) => {
692
819
  const p = {
693
820
  id: m.manifest.id,
694
821
  pool: m.pool,
695
822
  state: m.state,
696
- loadPct: loadOf(m)
823
+ loadPct: m.state === "failed" ? void 0 : loadOf(m)
697
824
  };
825
+ const reported = m.state === "failed" ? void 0 : reportedOf(m, now());
826
+ if (reported !== void 0) p.reportedPct = reported;
698
827
  if (m.atCap) p.atCap = true;
699
828
  const backlog = backlogNow(m, now());
700
829
  if (backlog !== void 0) p.backlog = backlog;
@@ -724,6 +853,7 @@ function createBalancer(kernel, opts = {}) {
724
853
  savedLoad: known,
725
854
  offences: 0,
726
855
  quarantinedAt: isQuarantine(dest.pool) ? now() : -Infinity,
856
+ decayAt: -Infinity,
727
857
  lastSpikeAt: -Infinity,
728
858
  spikeNotedAt: -Infinity,
729
859
  holds: 0,
@@ -732,7 +862,10 @@ function createBalancer(kernel, opts = {}) {
732
862
  moveTimes: [],
733
863
  backlogs: [],
734
864
  backlog: void 0,
865
+ backlogHistory: [],
866
+ busyReports: [],
735
867
  risingAt: -Infinity,
868
+ stuckAt: -Infinity,
736
869
  behind: false,
737
870
  behindAt: -Infinity
738
871
  };
@@ -775,18 +908,33 @@ function createBalancer(kernel, opts = {}) {
775
908
  m.holds--;
776
909
  };
777
910
  },
778
- report(id, { backlog }) {
779
- if (typeof backlog !== "number" || !Number.isFinite(backlog) || backlog < 0) throw new TypeError("balancer.report: backlog must be a finite number, 0 or more");
911
+ report(id, { backlog, busyMs }) {
912
+ if (backlog === void 0 && busyMs === void 0) throw new TypeError("balancer.report: give a backlog, a busyMs or both");
913
+ if (backlog !== void 0 && !valid(backlog)) throw new TypeError("balancer.report: backlog must be a finite number, 0 or more");
914
+ if (busyMs !== void 0 && !valid(busyMs)) throw new TypeError("balancer.report: busyMs must be a finite number, 0 or more");
780
915
  const m = members.get(id);
781
- if (!m) return;
916
+ if (!m || m.state === "failed") return;
782
917
  const t = now();
783
- const sample = {
784
- at: t,
785
- value: backlog
786
- };
787
- m.backlog = sample;
788
- m.backlogs.push(sample);
789
- while ((m.backlogs[1]?.at ?? Infinity) <= t - BACKLOG_WINDOW_MS) m.backlogs.shift();
918
+ if (backlog !== void 0) {
919
+ const sample = {
920
+ at: t,
921
+ value: backlog
922
+ };
923
+ m.backlog = sample;
924
+ m.backlogs.push(sample);
925
+ m.backlogHistory.push(sample);
926
+ while ((m.backlogs[1]?.at ?? Infinity) <= t - BACKLOG_WINDOW_MS) m.backlogs.shift();
927
+ while ((m.backlogHistory[1]?.at ?? Infinity) <= t - PACK_STEADY_MS) m.backlogHistory.shift();
928
+ }
929
+ if (busyMs !== void 0) {
930
+ if ((m.busyReports.at(-1)?.total ?? 0) > busyMs) m.busyReports = [];
931
+ m.busyReports.push({
932
+ at: t,
933
+ total: busyMs
934
+ });
935
+ const cut = t - windowMs - STALE_MS;
936
+ while ((m.busyReports[1]?.at ?? Infinity) <= cut) m.busyReports.shift();
937
+ }
790
938
  },
791
939
  measure(id) {
792
940
  const m = members.get(id);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@isoloop/balancer",
3
- "version": "0.1.0-next.0",
3
+ "version": "0.1.0-next.1",
4
4
  "description": "Experimental automatic placement for Isoloop isolates: joins a pool or keeps apart from measured load, quarantines what blocks.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -20,7 +20,7 @@
20
20
  }
21
21
  },
22
22
  "dependencies": {
23
- "@isoloop/host": "0.1.0-next.0"
23
+ "@isoloop/host": "0.1.0-next.1"
24
24
  },
25
25
  "devDependencies": {
26
26
  "typescript": "7.0.2"