@isoloop/balancer 0.1.0-next.1 → 0.1.0-next.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/index.d.ts +8 -7
- package/dist/index.js +47 -19
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -26,10 +26,10 @@ balancer.on('behind', (e) => console.warn(e.id, 'is behind and has nowhere to go
|
|
|
26
26
|
balancer.on('arrived', (e) => console.log(e.id, 'moved in', e.costMs, 'ms'));
|
|
27
27
|
```
|
|
28
28
|
|
|
29
|
-
- **Backlog:** a backlog
|
|
30
|
-
- **Fewer moves:** a move for load is made only if the moves it starts are predicted to bring the pool back under its line, and each isolate gets at most 3 of them in 5 minutes. Spike, crash and bisect moves protect the others and aren't limited.
|
|
29
|
+
- **Backlog:** a backlog that rose over two 10 s windows in a row, and is above 0, moves the isolate somewhere with more CPU: a pool of its own if the cap allows, else a pool with less load than its pool-mates. One such move per fresh 10 s window, so a group's members move one at a time. A report older than 10 s counts as none. With nowhere to go, it emits `behind` (at most every 30 s).
|
|
30
|
+
- **Fewer moves:** a move for load is made only if the moves it starts are predicted to bring the pool back under its line, and each isolate gets at most 3 of them in 5 minutes. Spike, crash and bisect moves protect the others and aren't limited. A never-measured isolate's way out of the quarantine, where it starts, doesn't count.
|
|
31
31
|
- **Spikes:** a first long task is only reported (`spike`, action `noted`); a second from the same isolate within 60 s sends it to quarantine. Its own fatal error still does at once.
|
|
32
|
-
- **Packing back:**
|
|
32
|
+
- **Packing back:** a pool whose isolates have all been steady for 60 s empties into another of the balancer's pools if both together stay under 40 % of a core, one isolate per pack and at most one pack every 60 s, which frees a worker once the last one is in. Turn it off with `pack: { enabled: false }`.
|
|
33
33
|
- **Move cost:** `arrived` follows each `move` with `costMs`, the time from stopping the isolate to its first beat in the new pool.
|
|
34
34
|
|
|
35
35
|
And when its busy time misses work, or the kernel gives up on an isolate (M3j):
|
package/dist/index.d.ts
CHANGED
|
@@ -118,11 +118,12 @@ type BalancerOptions = {
|
|
|
118
118
|
};
|
|
119
119
|
loads?: LoadStore;
|
|
120
120
|
/**
|
|
121
|
-
* Packing back (M3i
|
|
122
|
-
* spike, crash, lagging or full pool, and a
|
|
123
|
-
*
|
|
124
|
-
* under 40 % of a core,
|
|
125
|
-
*
|
|
121
|
+
* Packing back (M3i; whole pools since M3j): a pool whose members have
|
|
122
|
+
* all been steady for 60 s (no spike, crash, lagging or full pool, and a
|
|
123
|
+
* backlog no higher than 60 s before) empties into another of the
|
|
124
|
+
* balancer's pools if both together stay under 40 % of a core, one member
|
|
125
|
+
* per pack, freeing a worker once the last is in. At most one pack every
|
|
126
|
+
* 60 s. On by default.
|
|
126
127
|
*/
|
|
127
128
|
pack?: {
|
|
128
129
|
enabled?: boolean;
|
|
@@ -271,8 +272,8 @@ interface Balancer<M extends ManifestLike = ManifestLike> {
|
|
|
271
272
|
hold(id: string): () => void;
|
|
272
273
|
/**
|
|
273
274
|
* The app's view of `id`'s queued work (M3i) and of its own busy time (M3j).
|
|
274
|
-
* A backlog
|
|
275
|
-
*
|
|
275
|
+
* A backlog that rose over two 10 s windows in a row (M3j), and is above 0,
|
|
276
|
+
* moves the isolate somewhere with more CPU. A reported `busyMs` total
|
|
276
277
|
* counts toward its load wherever the balancer reads one. A report older
|
|
277
278
|
* than 10 s counts as none, so report at least every few seconds. Throws a
|
|
278
279
|
* `TypeError` for a report with neither field, or a field that isn't a
|
package/dist/index.js
CHANGED
|
@@ -148,6 +148,8 @@ function createBalancer(kernel, opts = {}) {
|
|
|
148
148
|
/** The last backlog move: rising is judged only on reports after it. */
|
|
149
149
|
let lastBacklogMoveAt = -Infinity;
|
|
150
150
|
let lastPackAt = -Infinity;
|
|
151
|
+
/** A pool being emptied into another by packing, one member at a time (M3j). */
|
|
152
|
+
let merging;
|
|
151
153
|
let disposed = false;
|
|
152
154
|
const pool = (name) => {
|
|
153
155
|
let p = pools.get(name);
|
|
@@ -167,6 +169,8 @@ function createBalancer(kernel, opts = {}) {
|
|
|
167
169
|
};
|
|
168
170
|
/** Running in its pool: not mid-move, and not given up on by the kernel. */
|
|
169
171
|
const live = (m) => m.state !== "moving" && m.state !== "failed";
|
|
172
|
+
/** Its members, moving ones included: the ones a pack has to move to empty it (M3j). */
|
|
173
|
+
const inPool = (name) => [...members.values()].filter((m) => m.pool === name && m.state !== "failed");
|
|
170
174
|
const membersOf = (name) => [...members.values()].filter((m) => m.pool === name && live(m));
|
|
171
175
|
const changed = (name) => {
|
|
172
176
|
const p = pool(name);
|
|
@@ -277,7 +281,8 @@ function createBalancer(kernel, opts = {}) {
|
|
|
277
281
|
m.state = "moving";
|
|
278
282
|
m.beats = [];
|
|
279
283
|
m.lastBeatAt = void 0;
|
|
280
|
-
m.moveTimes.push(startedAt);
|
|
284
|
+
if (!m.unmeasured) m.moveTimes.push(startedAt);
|
|
285
|
+
m.unmeasured = false;
|
|
281
286
|
m.backlogs = [];
|
|
282
287
|
m.backlogHistory = [];
|
|
283
288
|
m.busyReports = [];
|
|
@@ -322,7 +327,10 @@ function createBalancer(kernel, opts = {}) {
|
|
|
322
327
|
/** Sends `m` to quarantine for misbehaving (`offence`) or to bisect. */
|
|
323
328
|
const quarantine = (m, reason, offence) => {
|
|
324
329
|
m.lastSpikeAt = now();
|
|
325
|
-
if (offence)
|
|
330
|
+
if (offence) {
|
|
331
|
+
m.offences++;
|
|
332
|
+
m.unmeasured = false;
|
|
333
|
+
}
|
|
326
334
|
const state = stateFor(m);
|
|
327
335
|
m.quarantinedAt = now();
|
|
328
336
|
if (isQuarantine(m.pool)) {
|
|
@@ -344,6 +352,7 @@ function createBalancer(kernel, opts = {}) {
|
|
|
344
352
|
m.lastSpikeAt = m.spikeNotedAt = m.decayAt = m.risingAt = m.stuckAt = -Infinity;
|
|
345
353
|
m.arrivedAt = now();
|
|
346
354
|
m.moveTimes = [];
|
|
355
|
+
m.unmeasured = false;
|
|
347
356
|
m.beats = [];
|
|
348
357
|
m.lastBeatAt = void 0;
|
|
349
358
|
m.backlogs = [];
|
|
@@ -597,15 +606,20 @@ function createBalancer(kernel, opts = {}) {
|
|
|
597
606
|
/** The latest report, if fresh. */
|
|
598
607
|
const backlogNow = (m, t) => m.backlog && t - m.backlog.at <= STALE_MS ? m.backlog.value : void 0;
|
|
599
608
|
/**
|
|
600
|
-
* Rising (M3i): the latest fresh report is above 0
|
|
601
|
-
* at least a window before it,
|
|
602
|
-
*
|
|
609
|
+
* Rising (M3i; two windows since M3j): the latest fresh report is above 0
|
|
610
|
+
* and higher than one taken at least a window before it, which is itself
|
|
611
|
+
* higher than one taken at least a window before that, all since it last
|
|
612
|
+
* moved and since the last backlog move. A queue at a steady rate that
|
|
613
|
+
* jumps once and holds isn't falling behind.
|
|
603
614
|
*/
|
|
604
615
|
const rising = (m, t) => {
|
|
605
616
|
const latest = m.backlog;
|
|
606
617
|
if (backlogNow(m, t) === void 0 || !latest || latest.value <= 0) return false;
|
|
607
|
-
const
|
|
608
|
-
|
|
618
|
+
const before = (at) => m.backlogs.findLast((b) => b.at >= lastBacklogMoveAt && b.at <= at - BACKLOG_WINDOW_MS);
|
|
619
|
+
const mid = before(latest.at);
|
|
620
|
+
if (mid === void 0 || latest.value <= mid.value) return false;
|
|
621
|
+
const early = before(mid.at);
|
|
622
|
+
return early !== void 0 && mid.value > early.value;
|
|
609
623
|
};
|
|
610
624
|
/**
|
|
611
625
|
* Somewhere with more CPU for `m` (M3i): a pool of its own if the cap
|
|
@@ -673,24 +687,37 @@ function createBalancer(kernel, opts = {}) {
|
|
|
673
687
|
return start !== void 0 && latest > start.value;
|
|
674
688
|
};
|
|
675
689
|
/**
|
|
676
|
-
* Packing back (M3i): the lightest
|
|
677
|
-
*
|
|
678
|
-
*
|
|
679
|
-
*
|
|
680
|
-
*
|
|
690
|
+
* Packing back (M3i; whole pools since M3j): the lightest pool whose
|
|
691
|
+
* members have all been quiet for `PACK_STEADY_MS` empties into the
|
|
692
|
+
* fullest other pool that stays under `PACK_PCT` with all of it, one
|
|
693
|
+
* member per pack, which frees its worker once the last one is in. A merge
|
|
694
|
+
* that has started is finished first. Never a pool with a member on hold,
|
|
695
|
+
* out of budget, whose backlog rose over the window (M3j), or a
|
|
696
|
+
* quarantine; never into a pool that's moving or whose members' backlog
|
|
697
|
+
* rose.
|
|
681
698
|
*/
|
|
682
699
|
const pack = (t) => {
|
|
683
700
|
const quiet = (since) => t - since >= PACK_STEADY_MS;
|
|
684
701
|
const steady = (m) => quiet(Math.max(m.arrivedAt, m.lastSpikeAt, m.spikeNotedAt, pool(m.pool).troubleAt)) && !roseLately(m, t);
|
|
685
|
-
const
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
702
|
+
const packable = (m) => m.state === "placed" && movable(m) && inBudget(m) && loadOf(m) !== void 0 && steady(m);
|
|
703
|
+
const sources = ownPools().filter((n) => isOurs(n)).map((n) => ({
|
|
704
|
+
n,
|
|
705
|
+
all: inPool(n),
|
|
706
|
+
load: poolLoad(n)
|
|
707
|
+
})).filter((c) => c.all.length > 0 && c.all.every(packable)).toSorted((a, b) => Number(b.n === merging?.from) - Number(a.n === merging?.from) || a.load - b.load);
|
|
708
|
+
for (const from of sources) {
|
|
709
|
+
const into = ownPools().filter((n) => n !== from.n && isOurs(n) && !recentlySpiked(n) && quiet(pool(n).troubleAt)).filter((n) => inPool(n).every((o) => o.state !== "moving" && !roseLately(o, t))).map((n) => ({
|
|
689
710
|
n,
|
|
690
711
|
load: poolLoad(n)
|
|
691
|
-
})).filter((c) => c.load + load < PACK_PCT).toSorted((a, b) => b.load - a.load)[0];
|
|
712
|
+
})).filter((c) => c.load + from.load < PACK_PCT).toSorted((a, b) => Number(b.n === merging?.into) - Number(a.n === merging?.into) || b.load - a.load)[0];
|
|
692
713
|
if (!into) continue;
|
|
714
|
+
const m = from.all.toSorted((a, b) => (loadOf(a) ?? 0) - (loadOf(b) ?? 0))[0];
|
|
715
|
+
if (!m) continue;
|
|
693
716
|
lastPackAt = t;
|
|
717
|
+
merging = from.all.length > 1 ? {
|
|
718
|
+
from: from.n,
|
|
719
|
+
into: into.n
|
|
720
|
+
} : void 0;
|
|
694
721
|
move(m, into.n, "pack", "placed");
|
|
695
722
|
return;
|
|
696
723
|
}
|
|
@@ -753,7 +780,7 @@ function createBalancer(kernel, opts = {}) {
|
|
|
753
780
|
backlogs(t);
|
|
754
781
|
decay(t);
|
|
755
782
|
for (const m of members.values()) {
|
|
756
|
-
if (m.state !== "quarantined" || !movable(m) || !inBudget(m)) continue;
|
|
783
|
+
if (m.state !== "quarantined" || !movable(m) || !(m.unmeasured || inBudget(m))) continue;
|
|
757
784
|
const fresh = average(m.beats, windowMs, t);
|
|
758
785
|
if (m.offences === 0) {
|
|
759
786
|
if (fresh === void 0 || bisect?.moved.has(m.manifest.id)) continue;
|
|
@@ -860,6 +887,7 @@ function createBalancer(kernel, opts = {}) {
|
|
|
860
887
|
atCap: dest.atCap,
|
|
861
888
|
arrivedAt: now(),
|
|
862
889
|
moveTimes: [],
|
|
890
|
+
unmeasured: known === void 0,
|
|
863
891
|
backlogs: [],
|
|
864
892
|
backlog: void 0,
|
|
865
893
|
backlogHistory: [],
|
|
@@ -923,7 +951,7 @@ function createBalancer(kernel, opts = {}) {
|
|
|
923
951
|
m.backlog = sample;
|
|
924
952
|
m.backlogs.push(sample);
|
|
925
953
|
m.backlogHistory.push(sample);
|
|
926
|
-
while ((m.backlogs[1]?.at ?? Infinity) <= t - BACKLOG_WINDOW_MS) m.backlogs.shift();
|
|
954
|
+
while ((m.backlogs[1]?.at ?? Infinity) <= t - 3 * BACKLOG_WINDOW_MS) m.backlogs.shift();
|
|
927
955
|
while ((m.backlogHistory[1]?.at ?? Infinity) <= t - PACK_STEADY_MS) m.backlogHistory.shift();
|
|
928
956
|
}
|
|
929
957
|
if (busyMs !== void 0) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@isoloop/balancer",
|
|
3
|
-
"version": "0.1.0-next.
|
|
3
|
+
"version": "0.1.0-next.2",
|
|
4
4
|
"description": "Experimental automatic placement for Isoloop isolates: joins a pool or keeps apart from measured load, quarantines what blocks.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
}
|
|
21
21
|
},
|
|
22
22
|
"dependencies": {
|
|
23
|
-
"@isoloop/host": "0.1.0-next.
|
|
23
|
+
"@isoloop/host": "0.1.0-next.2"
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
26
|
"typescript": "7.0.2"
|