maxpool 1.24.92 → 1.24.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/account-manager.js +155 -4
- package/src/claude-reset-grants.js +8 -0
- package/src/event-log.js +79 -1
- package/src/index.js +325 -149
- package/src/prober.js +51 -9
- package/src/reload-protocol.js +86 -2
- package/src/reset-policy.js +94 -9
- package/src/terminal-mode.js +124 -0
- package/src/tui.js +106 -15
package/src/prober.js
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
import { fetchUsage, fetchProviderUsage } from './oauth.js';
|
|
11
11
|
import { listResetCards, redeemResetCard, fetchZaiModelUsage } from './zai-reset-cards.js';
|
|
12
12
|
import { listSubscriptions, monthlyPriceUSD } from './zai-subs.js';
|
|
13
|
-
import { listResetGrants, claimResetGrant } from './claude-reset-grants.js';
|
|
13
|
+
import { listResetGrants, claimResetGrant, appliesTo, clearsGeneralWeekly } from './claude-reset-grants.js';
|
|
14
14
|
import { decideZai, decideClaudeWithReason, CLAUDE_WALL_WEEKLY } from './reset-policy.js';
|
|
15
15
|
import { zaiHourlyCalls, compareOutsideUsage, isUsableZaiUsage, OUTSIDE_CHECK_MS } from './outside-usage.js';
|
|
16
16
|
// Reset-machinery timing lives in ONE leaf module (re-exported for callers that
|
|
@@ -22,6 +22,10 @@ export { GRANTS_EVERY_MS, GRANTS_RETRY_MS, CLAIM_BACKOFF_STEPS_MS, CLAIM_BACKOFF
|
|
|
22
22
|
// token refresh per window — cheap, and bounded so a truly canceled plan isn't hammered).
|
|
23
23
|
export const SUB_RECHECK_MS = 30 * 60_000;
|
|
24
24
|
|
|
25
|
+
// The sweep-decision skips that mean "this tick's LISTING can't be trusted" — the
|
|
26
|
+
// only skips after a successful probe that hand a held wall to the rescue (r2 #1).
|
|
27
|
+
const STALE_LISTING_SKIPS = new Set(['listing stale', 'listing failed']);
|
|
28
|
+
|
|
25
29
|
export class Prober {
|
|
26
30
|
constructor(accountManager, { intervalMs = 0, probeFn = fetchUsage, providerProbeFn = fetchProviderUsage, grantsFn = listResetGrants, timeoutMs = 10_000, log = console.log, usageGapMs = null, zaiUsageFn = fetchZaiModelUsage, cardsFn = listResetCards, now = Date.now } = {}) {
|
|
27
31
|
this.am = accountManager;
|
|
@@ -360,6 +364,12 @@ export class Prober {
|
|
|
360
364
|
if (r?.ok) {
|
|
361
365
|
this.am.noteClaimSuccess?.(account.name, act.grant.id);
|
|
362
366
|
account.lastClaimAt = Date.now();
|
|
367
|
+
// A spent grant that applies to the general weekly — whatever scope it was
|
|
368
|
+
// claimed at — is the general wall's claim (r3 #3, r4 #3); only one that
|
|
369
|
+
// CLEARS it retires the hold (r5 #1, same rule as the 429 fast path). A
|
|
370
|
+
// Fable-only grant refills nothing the hold is waiting on.
|
|
371
|
+
if (appliesTo(act.grant).includes('general')) account.lastGeneralClaimAt = account.lastClaimAt;
|
|
372
|
+
if (clearsGeneralWeekly(act.grant)) account.lastFullGeneralClaimAt = account.lastClaimAt;
|
|
363
373
|
// Spend it in the STORED listing too: until the next listing (≤5 min) the
|
|
364
374
|
// held count, the lift and the reactive path must not see it as unspent.
|
|
365
375
|
this.am.noteGrantClaimed?.(account, act.grant.id);
|
|
@@ -481,13 +491,11 @@ export class Prober {
|
|
|
481
491
|
usage = await this._withTimeout(this.probeFn(account.credential));
|
|
482
492
|
}
|
|
483
493
|
}
|
|
484
|
-
if (usage == null) { // timed out
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
this.am.recordProbeError?.(account.index, usage.error, usage.status ?? null);
|
|
490
|
-
return { ok: false, status: usage.status ?? null };
|
|
494
|
+
if (usage == null || usage.error) { // timed out / HTTP error (e.g. 429) / fetch failure
|
|
495
|
+
const e = usage == null ? 'probe timed out' : usage.error;
|
|
496
|
+
this.am.recordProbeError?.(account.index, e, usage?.status ?? null);
|
|
497
|
+
await this._rescueHeldWall(account); // red-team r2 #1: a held wall must not strand
|
|
498
|
+
return { ok: false, status: usage?.status ?? null };
|
|
491
499
|
}
|
|
492
500
|
this.am.applyUsageData(account.index, usage, { sentAt: probeSentAt }); // clears the error + stamps freshness
|
|
493
501
|
// C1: decide the claim on the reading JUST stored (this sweep's probe), on
|
|
@@ -496,6 +504,7 @@ export class Prober {
|
|
|
496
504
|
return { ok: true, status: 200 };
|
|
497
505
|
} catch (e) { // best-effort; never let a probe throw
|
|
498
506
|
this.am.recordProbeError?.(account.index, e?.message || String(e), null);
|
|
507
|
+
await this._rescueHeldWall(account); // red-team r2 #1
|
|
499
508
|
return { ok: false, status: null };
|
|
500
509
|
} finally {
|
|
501
510
|
account._subRecheck = false;
|
|
@@ -509,12 +518,27 @@ export class Prober {
|
|
|
509
518
|
if (!this.am.applyResetGrants) return;
|
|
510
519
|
const listing = account.resetGrants;
|
|
511
520
|
if (!listing) return;
|
|
512
|
-
|
|
521
|
+
// R7 (2026-11-13): the GENERAL weekly wall claim (the only one R7 holds — red-team
|
|
522
|
+
// r3) waits for an imminent free refill only while
|
|
523
|
+
// other accounts in THIS pool have room. Counted LIVE per sweep, so the hold
|
|
524
|
+
// ends the moment capacity disappears; unknown capacity claims (never hold).
|
|
525
|
+
// holdersInPool budgets one hold per holdMinPeers roomy peers across the pool
|
|
526
|
+
// (r2 #2); a hold is registered so a later failed probe can still rescue it (r2 #1).
|
|
527
|
+
const peersWithRoom = this.am.countPeersWithRoom ? () => this.am.countPeersWithRoom(account) : null;
|
|
528
|
+
const holdersInPool = this.am.countHoldingWalls ? () => this.am.countHoldingWalls(account) : null;
|
|
529
|
+
const { action, reason, heldScope } = decideClaudeWithReason(account, listing, Date.now(), { peersWithRoom, holdersInPool });
|
|
530
|
+
// heldScope rides on every return once the general wall is held — a dying model
|
|
531
|
+
// grant claimed in the same decision included (r3 #2) — so the hold is always registered.
|
|
532
|
+
if (heldScope === 'general') this.am.noteWallHold?.(account);
|
|
513
533
|
if (action) {
|
|
514
534
|
account._claimSkipKey = null;
|
|
515
535
|
await this._claimClaudeGrant(account, action);
|
|
516
536
|
return;
|
|
517
537
|
}
|
|
538
|
+
// The probe answered but the stored LISTING is stale/failed (the grants call
|
|
539
|
+
// shares the rate-limited endpoint): the fresh decision can't run, so a held
|
|
540
|
+
// wall goes through the rescue exactly as on a failed probe (r2 #1).
|
|
541
|
+
if (STALE_LISTING_SKIPS.has(reason)) await this._rescueHeldWall(account);
|
|
518
542
|
// C5: at a reached limit with a held-but-unspent grant, say WHY once per
|
|
519
543
|
// (account, reason) change — the owner's live check reads exactly these lines.
|
|
520
544
|
// The reason is the POLICY's own rejection (stale listing, cooldown, already
|
|
@@ -537,6 +561,24 @@ export class Prober {
|
|
|
537
561
|
}
|
|
538
562
|
}
|
|
539
563
|
|
|
564
|
+
/** Red-team r2 #1 — the held-wall rescue. A wall this pool HELD (its stamp is
|
|
565
|
+
* still live) must not strand: while the account's own usage endpoint is dark
|
|
566
|
+
* (a 429'd probe, a timeout, a throw — every early return of probeOne), no fresh
|
|
567
|
+
* decision can be made, yet "capacity unknown" means CLAIM. Re-evaluate the hold
|
|
568
|
+
* directly (wallClaimHeld: header/probe refill stamp, stored listing ≤ 6h,
|
|
569
|
+
* peers' LIVE probe freshness) and claim through the reactive route — the same
|
|
570
|
+
* one the 429 fast path uses, whose pick tolerates a stale listing on purpose.
|
|
571
|
+
* A still-roomy pool keeps holding; a refill that moved past 24h (or a dying
|
|
572
|
+
* grant) makes wallClaimHeld itself release. Never claims a wall nobody held.
|
|
573
|
+
* General weekly wall only (red-team r3): no model wall is ever held. */
|
|
574
|
+
async _rescueHeldWall(account) {
|
|
575
|
+
if (!this.am.isHoldingWall?.(account)) return; // never a wall nobody held
|
|
576
|
+
// A listing-side cooldown is a server instruction the reactive route doesn't read.
|
|
577
|
+
const cd = account.resetGrants?.cooldownUntil;
|
|
578
|
+
if (cd != null && cd > Date.now()) return;
|
|
579
|
+
await this.am.tryReactiveClaim(account, { kind: 'weekly' });
|
|
580
|
+
}
|
|
581
|
+
|
|
540
582
|
_withTimeout(promise) {
|
|
541
583
|
return Promise.race([
|
|
542
584
|
promise,
|
package/src/reload-protocol.js
CHANGED
|
@@ -68,21 +68,33 @@ export async function runReloadBaton({
|
|
|
68
68
|
releaseTimeoutMs = 20_000,
|
|
69
69
|
takeoverTimeoutMs = 10_000,
|
|
70
70
|
log = () => {},
|
|
71
|
+
reloadId,
|
|
71
72
|
}) {
|
|
73
|
+
// Every rollback notice names the reload it is about, so the old worker can tell a
|
|
74
|
+
// late notice for an EARLIER reload from one for the reload it has open.
|
|
75
|
+
const rolledBack = reason => {
|
|
76
|
+
try { oldWorker.send({ type: MSG_ROLLED_BACK, reason, reloadId }); } catch { /* old worker gone */ }
|
|
77
|
+
};
|
|
72
78
|
// (a) Confirm the freshly-spawned headless worker booted the new code.
|
|
73
79
|
let ready;
|
|
74
80
|
try {
|
|
75
81
|
newWorker.send({ type: MSG_PROBE_READY });
|
|
76
82
|
ready = await newWorker.waitFor([MSG_READY, MSG_FAILED], readyTimeoutMs);
|
|
77
83
|
} catch (err) {
|
|
84
|
+
// Tell the old worker BEFORE logging: it resumes serving on this notice, and a log
|
|
85
|
+
// line must never stand between the two (2026-11-13: a blocked terminal write did).
|
|
86
|
+
// Only an expired timer is 'timeout' (merely slow: a cold restart may retry it). A
|
|
87
|
+
// new worker that exited or could not be spawned is 'failed' — as is any rejection
|
|
88
|
+
// whose cause is unknown, since a cold restart into a build that cannot boot leaves
|
|
89
|
+
// no working proxy.
|
|
90
|
+
rolledBack(err?.reason === 'timeout' ? 'timeout' : 'failed');
|
|
78
91
|
log(`reload: readiness wait failed (${err.message}); rolling back`);
|
|
79
|
-
try { oldWorker.send({ type: MSG_ROLLED_BACK, reason: 'timeout' }); } catch { /* old worker gone */ }
|
|
80
92
|
return RELOAD_ROLLED_BACK;
|
|
81
93
|
}
|
|
82
94
|
// (b) New worker did not come up cleanly → roll back fully, old stays primary.
|
|
83
95
|
if (!ready || ready.type === MSG_FAILED) {
|
|
96
|
+
rolledBack('failed');
|
|
84
97
|
log(`reload: new worker reported failed (${ready?.reason || 'no ready'}); rolling back`);
|
|
85
|
-
try { oldWorker.send({ type: MSG_ROLLED_BACK, reason: 'failed' }); } catch { /* old worker gone */ }
|
|
86
98
|
return RELOAD_ROLLED_BACK;
|
|
87
99
|
}
|
|
88
100
|
|
|
@@ -128,6 +140,58 @@ export async function runReloadBaton({
|
|
|
128
140
|
return RELOAD_SWAPPED;
|
|
129
141
|
}
|
|
130
142
|
|
|
143
|
+
/**
|
|
144
|
+
* Wrap a child process in a small IPC channel: `.send(msg[,handle])` and an
|
|
145
|
+
* async `.waitFor([types], timeoutMs)` that resolves with the first matching
|
|
146
|
+
* message, or rejects on timeout / premature child exit.
|
|
147
|
+
*/
|
|
148
|
+
export function makeWorkerChannel(child) {
|
|
149
|
+
return {
|
|
150
|
+
child,
|
|
151
|
+
send(msg, handle) {
|
|
152
|
+
try {
|
|
153
|
+
if (handle) child.send(msg, handle);
|
|
154
|
+
else child.send(msg);
|
|
155
|
+
} catch { /* IPC may be torn down mid-swap; baton timeouts cover it */ }
|
|
156
|
+
},
|
|
157
|
+
// A rejection carries `.reason`: 'timeout' (the timer expired), 'exited' (the child
|
|
158
|
+
// exited first — or had already exited) or 'spawn-error'.
|
|
159
|
+
waitFor(types, timeoutMs) {
|
|
160
|
+
const want = Array.isArray(types) ? types : [types];
|
|
161
|
+
const fail = (reason, message) => Object.assign(new Error(message), { reason });
|
|
162
|
+
if (child.exitCode != null || child.signalCode != null) {
|
|
163
|
+
return Promise.reject(fail('exited', `worker already exited (code ${child.exitCode}, signal ${child.signalCode}) before ${want.join('/')}`));
|
|
164
|
+
}
|
|
165
|
+
return new Promise((resolve, reject) => {
|
|
166
|
+
const cleanup = () => {
|
|
167
|
+
clearTimeout(timer);
|
|
168
|
+
child.removeListener('message', onMsg);
|
|
169
|
+
child.removeListener('exit', onExit);
|
|
170
|
+
child.removeListener('error', onError);
|
|
171
|
+
};
|
|
172
|
+
const onMsg = msg => {
|
|
173
|
+
if (msg && want.includes(msg.type)) { cleanup(); resolve(msg); }
|
|
174
|
+
};
|
|
175
|
+
const onExit = (code, signal) => {
|
|
176
|
+
cleanup();
|
|
177
|
+
reject(fail('exited', `worker exited (code ${code}, signal ${signal}) before ${want.join('/')}`));
|
|
178
|
+
};
|
|
179
|
+
// A spawn failure (EMFILE/EAGAIN) surfaces as 'error', not 'exit' — the
|
|
180
|
+
// baton must see it as a failed step (→ ROLLED_BACK/FALLBACK), not hang.
|
|
181
|
+
const onError = err => { cleanup(); reject(fail('spawn-error', `worker spawn error before ${want.join('/')}: ${err.message}`)); };
|
|
182
|
+
const timer = setTimeout(() => {
|
|
183
|
+
cleanup();
|
|
184
|
+
reject(fail('timeout', `timed out waiting for ${want.join('/')}`));
|
|
185
|
+
}, timeoutMs);
|
|
186
|
+
timer.unref?.();
|
|
187
|
+
child.on('message', onMsg);
|
|
188
|
+
child.once('exit', onExit);
|
|
189
|
+
child.once('error', onError);
|
|
190
|
+
});
|
|
191
|
+
},
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
|
|
131
195
|
// Sent to the NEW primary worker once the OLD worker has actually exited. Node calls
|
|
132
196
|
// libuv's uv_tty_reset_mode() on process exit, which restores the termios the dying
|
|
133
197
|
// process saved when IT first enabled raw mode — clobbering the live worker's terminal
|
|
@@ -141,3 +205,23 @@ export function describeOldWorkerExit({ swappedAt, exitedAt, killed = false }) {
|
|
|
141
205
|
const after = ((exitedAt - swappedAt) / 1000).toFixed(1);
|
|
142
206
|
return `reload: old worker exited at ${new Date(exitedAt).toISOString()} (${after}s after swap, ${killed ? 'SIGKILL after the drain cap' : 'clean exit'})`;
|
|
143
207
|
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* What the old worker does with a rollback notice. `noticeId` is the reload id the
|
|
211
|
+
* supervisor echoed (absent from a supervisor that predates ids), `currentId` the id
|
|
212
|
+
* of the reload this worker requested last, `open` whether that reload is still
|
|
213
|
+
* waiting for an outcome.
|
|
214
|
+
* 'settle' — the notice names the open reload: act on it now
|
|
215
|
+
* 'settle-idless' — an id-less notice (a supervisor that predates ids — it survives
|
|
216
|
+
* seamless updates) while a reload is open: resume serving now, but
|
|
217
|
+
* its reason never authorizes a cold restart. That supervisor tags
|
|
218
|
+
* every readiness rejection 'timeout', a new build that died before
|
|
219
|
+
* ready included, and cannot say which reload it means (red-team
|
|
220
|
+
* round 2, 2026-11-13)
|
|
221
|
+
* 'ignore' — a notice about an earlier reload, or nothing is open
|
|
222
|
+
*/
|
|
223
|
+
export function rollbackNoticeAction({ noticeId, currentId, open }) {
|
|
224
|
+
if (!open) return 'ignore';
|
|
225
|
+
if (noticeId == null) return 'settle-idless';
|
|
226
|
+
return noticeId === currentId ? 'settle' : 'ignore';
|
|
227
|
+
}
|
package/src/reset-policy.js
CHANGED
|
@@ -43,7 +43,7 @@
|
|
|
43
43
|
* value cannot survive a wait.
|
|
44
44
|
*/
|
|
45
45
|
import { appliesTo as defaultAppliesTo } from './claude-reset-grants.js';
|
|
46
|
-
import { CLAIM_FRESH_MS } from './reset-timing.js';
|
|
46
|
+
import { CLAIM_FRESH_MS, CLAIM_STALE_LISTING_MAX_MS } from './reset-timing.js';
|
|
47
47
|
|
|
48
48
|
export const POLICY = {
|
|
49
49
|
wallWeekly: 0.90, // weekly at 90% = at the wall (z.ai cards; UNCHANGED — pre-mortem #22)
|
|
@@ -53,8 +53,29 @@ export const POLICY = {
|
|
|
53
53
|
dyingMinUtilSes: 0.50, // 5h refills are cheap capacity; demand more waste first
|
|
54
54
|
freeRefillWaitMs: 24 * 3600 * 1000, // R6: hold a wall redeem this close to the free refill
|
|
55
55
|
freeRefillImmediate: 0.98, // R6: ...unless this full — never block the owner
|
|
56
|
+
// R7 (owner, 2026-11-13): the Claude GENERAL weekly wall claim also waits for the
|
|
57
|
+
// free refill when OTHER accounts can absorb the load. Room per peer: probe-fresh
|
|
58
|
+
// (≤5 min) readings, weekly util with ≥ 35% headroom left, 5h util with ≥ 25%
|
|
59
|
+
// headroom to the 0.90 session bench, routable now.
|
|
60
|
+
holdPeerWeeklyRoom: 0.35,
|
|
61
|
+
holdPeerSessionRoom: 0.25,
|
|
62
|
+
holdMinPeers: 2,
|
|
63
|
+
// A hold is tied to the refill it waited for (red-team r4 #4). The same refill is
|
|
64
|
+
// read from two sources with different precision (rate-limit header: whole
|
|
65
|
+
// seconds; usage probe: fractional), so a reading up to this much later is still
|
|
66
|
+
// that refill; a rolled window moves it by days.
|
|
67
|
+
holdRefillSlackMs: 3600 * 1000,
|
|
68
|
+
// The 429 fast path and the rescue claim from the STORED listing, which they
|
|
69
|
+
// refuse past CLAIM_STALE_LISTING_MAX_MS (6h). A hold therefore lets go this long
|
|
70
|
+
// before that bound (red-team r4 #6), while the reset is still claimable — a dark
|
|
71
|
+
// listing is unknown state, and unknown means claim. The rescue runs every sweep.
|
|
72
|
+
holdListingMarginMs: 30 * 60 * 1000,
|
|
56
73
|
};
|
|
57
74
|
|
|
75
|
+
// The plain-words decision reason for an R7 hold — exported so status/log surfaces
|
|
76
|
+
// and tests share one string.
|
|
77
|
+
export const HOLD_POOL_HAS_ROOM = 'free refill <24h away, other accounts have room';
|
|
78
|
+
|
|
58
79
|
// Claude cedar_ember claims read their OWN wall (pre-mortem #22): 0.99, deliberately
|
|
59
80
|
// NOT POLICY.wallWeekly (0.90) which z.ai shares — moving the shared constant would
|
|
60
81
|
// redeem GLM cards at a different point. Claude benches at 0.999 (weeklyExhausted)
|
|
@@ -69,6 +90,34 @@ export { CLAIM_FRESH_MS };
|
|
|
69
90
|
|
|
70
91
|
function msLeft(t, now) { return t == null ? Infinity : t - now; }
|
|
71
92
|
|
|
93
|
+
/** R7 (owner, 2026-11-13): should a GENERAL weekly WALL claim of `grant` wait for the
|
|
94
|
+
* free refill at `refillAt`? GENERAL ONLY (red-team r3, orchestrator decision): a
|
|
95
|
+
* per-model (Fable) wall and a dying-value grant claim exactly as before R7. The ONE
|
|
96
|
+
* hold rule, shared by the sweep (decideClaudeWithReason) and the
|
|
97
|
+
* reactive 429 path (account-manager tryReactiveClaim) so neither can spend what the
|
|
98
|
+
* other holds. Reads only the refill stamp, the grant's end and a LIVE peer count —
|
|
99
|
+
* never the walled account's own usage reading, which is routinely stale at the wall.
|
|
100
|
+
* Conservative: an unknown / past / ≥24h-away refill, a grant that dies before (or
|
|
101
|
+
* within dyingWindowMs after) the refill — it would sit unspent at ~0% used — no
|
|
102
|
+
* counter, or a count that isn't an integer ≥ the pool-wide budget ⇒ false (claim).
|
|
103
|
+
* BUDGET (red-team r2 #2): roomy peers must cover EVERY wall currently held in this
|
|
104
|
+
* pool — holdMinPeers × holdersInPool(), where holdersInPool counts the walls
|
|
105
|
+
* holding now INCLUDING this one. Without a holders counter, this wall alone (1).
|
|
106
|
+
* LISTING AGE (red-team r4 #6): when the caller passes `listingCheckedAt`, a stored
|
|
107
|
+
* listing within holdListingMarginMs of the 6h claim bound — or of unknown age
|
|
108
|
+
* (null) — releases the hold: past that bound the reset could not be claimed at
|
|
109
|
+
* all while the listing stays dark. Both claim paths pass it. */
|
|
110
|
+
export function wallClaimHeld(refillAt, grant, now, { peersWithRoom, holdersInPool, listingCheckedAt } = {}) {
|
|
111
|
+
if (!(refillAt > now) || refillAt - now >= POLICY.freeRefillWaitMs) return false;
|
|
112
|
+
if (listingCheckedAt !== undefined
|
|
113
|
+
&& !(now - listingCheckedAt < CLAIM_STALE_LISTING_MAX_MS - POLICY.holdListingMarginMs)) return false;
|
|
114
|
+
if (grant?.endsAt != null && grant.endsAt - refillAt < POLICY.dyingWindowMs) return false;
|
|
115
|
+
const holders = holdersInPool ? holdersInPool() : 1;
|
|
116
|
+
if (!Number.isInteger(holders) || holders < 1) return false;
|
|
117
|
+
const count = peersWithRoom?.();
|
|
118
|
+
return Number.isInteger(count) && count >= POLICY.holdMinPeers * holders;
|
|
119
|
+
}
|
|
120
|
+
|
|
72
121
|
/** Decide the z.ai action. Returns {kind:'card', card, resetType} | null. */
|
|
73
122
|
export function decideZai(account, cards, now = Date.now()) {
|
|
74
123
|
if (account?.enabled === false) return null; // R0
|
|
@@ -128,8 +177,11 @@ export function probeSentAt(q) {
|
|
|
128
177
|
|
|
129
178
|
/** Decide the Claude action AND say why when there is none.
|
|
130
179
|
* Returns { action: {kind:'grant', grant, scope, giveBack?, reason} | null,
|
|
131
|
-
* reason: string | null } — `reason` names
|
|
132
|
-
* is null (C5: the skip log prints exactly this), or the
|
|
180
|
+
* reason: string | null, heldScope?: string | null } — `reason` names
|
|
181
|
+
* the rejection when action is null (C5: the skip log prints exactly this), or the
|
|
182
|
+
* claim reason; `heldScope` is 'general' whenever the R7 hold kept the general
|
|
183
|
+
* weekly wall unclaimed on this decision — on EVERY return after that point, a claim
|
|
184
|
+
* of a dying model grant included, so the caller always registers the hold (null: none).
|
|
133
185
|
*
|
|
134
186
|
* C1 (amended, 2026-11-10): the claim is decided per grant by appliesTo (D1),
|
|
135
187
|
* NEVER by the server's nextGrantId order alone (pre-mortem #8: a Fable 429 could
|
|
@@ -139,7 +191,20 @@ export function probeSentAt(q) {
|
|
|
139
191
|
* any claim we already made (no second claim for one wall).
|
|
140
192
|
* D4: a dying-value claim (not at the wall) waits when the free refill of the
|
|
141
193
|
* limit it clears is < 24h away; a wall claim never waits (limit effectively full).
|
|
142
|
-
* Session-only grants are NEVER claimed (owner 2026-10-30).
|
|
194
|
+
* Session-only grants are NEVER claimed (owner 2026-10-30).
|
|
195
|
+
* R7 (owner, 2026-11-13, task-2026-11-13-maxpool-hold-resets-when-pool-has-room):
|
|
196
|
+
* the GENERAL weekly WALL claim is HELD when its free refill is < 24h away
|
|
197
|
+
* (freeRefillWaitMs) AND opts.peersWithRoom() answers at least POLICY.holdMinPeers
|
|
198
|
+
* × opts.holdersInPool() — other accounts in THIS pool can absorb the load. Model
|
|
199
|
+
* (Fable) walls are never held (red-team r3) — but while the general wall is held the
|
|
200
|
+
* model WALL claim is skipped too: the account is benched by its general wall, so a
|
|
201
|
+
* Fable reset would buy nothing until the general refill lands — unless that grant
|
|
202
|
+
* would expire before (or within dyingWindowMs after) the refill: then it is claimed. The hold
|
|
203
|
+
* re-evaluates every decision tick (the caller supplies a live count), and fails
|
|
204
|
+
* CLAIM-side: capacity unknown (no counter, non-number answer) or a grant that
|
|
205
|
+
* would expire before the refill (or within a day after it — it would sit unspent
|
|
206
|
+
* at ~0% used) is claimed as today. Dying-value claims stay exempt: an expiring
|
|
207
|
+
* grant's value cannot survive any wait. */
|
|
143
208
|
export function decideClaudeWithReason(account, listing, now = Date.now(), opts = {}) {
|
|
144
209
|
const skip = reason => ({ action: null, reason });
|
|
145
210
|
if (account?.enabled === false) return skip('account disabled'); // R0
|
|
@@ -178,6 +243,14 @@ export function decideClaudeWithReason(account, listing, now = Date.now(), opts
|
|
|
178
243
|
const weeklyWall = wkUtil != null && wkUtil >= CLAUDE_WALL_WEEKLY;
|
|
179
244
|
// D4: the free refill of a limit < 24h away makes a not-at-the-wall spend redundant.
|
|
180
245
|
const refillSoon = resetAt => resetAt != null && msLeft(resetAt, now) < POLICY.freeRefillWaitMs;
|
|
246
|
+
// R7: hold the GENERAL wall claim for the imminent free refill while other accounts
|
|
247
|
+
// in this pool can absorb the load (wallClaimHeld). A hold NOTES its reason and
|
|
248
|
+
// moves on (red-team #2): the model-scoped loop below must still run, so a dying
|
|
249
|
+
// model grant is spent. `generalHeld` rides on every return after it (red-team r3
|
|
250
|
+
// #2) so the caller registers the hold even when this decision claims a dying model
|
|
251
|
+
// grant — it feeds the pool-wide budget and the held-wall rescue (r2 #1/#2).
|
|
252
|
+
let generalHeld = false;
|
|
253
|
+
const out = (action, why) => ({ action, reason: why, heldScope: generalHeld ? 'general' : null });
|
|
181
254
|
let reason = null;
|
|
182
255
|
const note = r => { if (reason == null) reason = r; };
|
|
183
256
|
|
|
@@ -188,16 +261,21 @@ export function decideClaudeWithReason(account, listing, now = Date.now(), opts
|
|
|
188
261
|
for (const g of general) {
|
|
189
262
|
const giveBack = g.givesBackWeekly === true && !(Array.isArray(g.clears) && g.clears.includes('seven_day'));
|
|
190
263
|
if (weeklyWall) {
|
|
264
|
+
// R7: the free refill is imminent and the pool has room — hold the claim.
|
|
265
|
+
if (wallClaimHeld(q.unified7dReset, g, now,
|
|
266
|
+
{ peersWithRoom: opts.peersWithRoom, holdersInPool: opts.holdersInPool, listingCheckedAt: listing.checkedAt })) {
|
|
267
|
+
generalHeld = true; note(HOLD_POOL_HAS_ROOM); continue;
|
|
268
|
+
}
|
|
191
269
|
// A give-back-only grant (no seven_day in clears) returns only what the model
|
|
192
270
|
// used — the claim snapshots unified7d so the next probe measures the drop.
|
|
193
|
-
return {
|
|
271
|
+
return out({ kind: 'grant', grant: g, scope: 'general', giveBack, reason: 'weekly at wall' }, 'weekly at wall');
|
|
194
272
|
}
|
|
195
273
|
const dying = msLeft(g.endsAt, now) < POLICY.dyingWindowMs
|
|
196
274
|
&& wkUtil != null && wkUtil >= POLICY.dyingMinUtilWeekly; // weekly window only (owner 2026-10-30)
|
|
197
275
|
if (!dying) continue;
|
|
198
276
|
if (g.requiresLimit) { note('requires limit'); continue; }
|
|
199
277
|
if (refillSoon(q.unified7dReset)) { note('free refill <24h away'); continue; }
|
|
200
|
-
return {
|
|
278
|
+
return out({ kind: 'grant', grant: g, scope: 'general', giveBack, reason: 'dying value' }, 'dying value');
|
|
201
279
|
}
|
|
202
280
|
if (weeklyWall && !general.length) note('general not held');
|
|
203
281
|
|
|
@@ -215,14 +293,21 @@ export function decideClaudeWithReason(account, listing, now = Date.now(), opts
|
|
|
215
293
|
const cand = grants.find(g => appliesTo(g).includes(fam));
|
|
216
294
|
if (!cand) continue;
|
|
217
295
|
if (e.utilization >= CLAUDE_WALL_WEEKLY) {
|
|
218
|
-
|
|
296
|
+
// Never held on its own (red-team r3) — but skipped while the GENERAL wall is
|
|
297
|
+
// held: the account is benched by its general wall, so a model reset buys
|
|
298
|
+
// nothing now; the hold re-evaluates every tick and, once the general refill
|
|
299
|
+
// lands (or the hold lets go), this wall claims as before R7. ONLY when the
|
|
300
|
+
// grant outlives the general refill by a day (red-team r4 #1) — one that
|
|
301
|
+
// would expire before then is claimed now, or it is lost.
|
|
302
|
+
if (generalHeld && (cand.endsAt == null || cand.endsAt - q.unified7dReset >= POLICY.dyingWindowMs)) continue;
|
|
303
|
+
return out({ kind: 'grant', grant: cand, scope: fam, reason: `${fam} weekly at wall` }, `${fam} weekly at wall`);
|
|
219
304
|
}
|
|
220
305
|
const dying = msLeft(cand.endsAt, now) < POLICY.dyingWindowMs
|
|
221
306
|
&& e.utilization >= POLICY.dyingMinUtilWeekly;
|
|
222
307
|
if (!dying) continue;
|
|
223
308
|
if (cand.requiresLimit) { note('requires limit'); continue; }
|
|
224
309
|
if (refillSoon(e.resetAt)) { note('free refill <24h away'); continue; }
|
|
225
|
-
return {
|
|
310
|
+
return out({ kind: 'grant', grant: cand, scope: fam, reason: 'dying value' }, 'dying value');
|
|
226
311
|
}
|
|
227
312
|
// A scoped bucket measured at the wall with no grant for it.
|
|
228
313
|
for (const [fam, e] of Object.entries(q.scopedWeekly || {})) {
|
|
@@ -231,5 +316,5 @@ export function decideClaudeWithReason(account, listing, now = Date.now(), opts
|
|
|
231
316
|
}
|
|
232
317
|
|
|
233
318
|
// Session-only grants: NEVER claimed (owner 2026-10-30) — falls through.
|
|
234
|
-
return
|
|
319
|
+
return out(null, reason ?? 'below wall');
|
|
235
320
|
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
// Switch the terminal's raw mode WITHOUT stalling the thread that serves requests.
|
|
2
|
+
//
|
|
3
|
+
// Node switches it with tcsetattr(TCSADRAIN), which waits until the terminal has read
|
|
4
|
+
// everything already queued for it — up to the pty's ~1 KB kernel queue, which no Node
|
|
5
|
+
// API can see. A worker that has just taken over the pool cannot know whether its
|
|
6
|
+
// terminal is reading (red-team 2026-11-13: a dashboard frame under 1 KB, or a terminal
|
|
7
|
+
// that reads in bursts, stalled the new sole acceptor for ~1.7s inside setRawMode).
|
|
8
|
+
// So that switch runs in a helper PROCESS with its own handle on the terminal (our
|
|
9
|
+
// stdin, else /dev/tty): the helper waits for the terminal, the pool keeps serving.
|
|
10
|
+
// termios belongs to the terminal device, so a mode set from the helper is the mode
|
|
11
|
+
// every reader of the terminal sees (verified on a real pty, 2026-11-13).
|
|
12
|
+
//
|
|
13
|
+
// A process, not a worker thread (red-team round 2, 2026-11-13): process.exit JOINS
|
|
14
|
+
// worker threads, a thread inside tcsetattr cannot be interrupted (Worker.terminate()
|
|
15
|
+
// included — libuv retries the call on EINTR), so a worker with a switch pending lived
|
|
16
|
+
// until the terminal read: on the abrupt-restart path it kept the port. A child process
|
|
17
|
+
// is never joined, and SIGKILL ends it mid-wait without applying the pending mode.
|
|
18
|
+
// The helper only ever dies by SIGKILL: a normal exit would run Node's terminal reset
|
|
19
|
+
// and put the terminal back in the mode it found before its first switch.
|
|
20
|
+
import { spawn } from 'node:child_process';
|
|
21
|
+
|
|
22
|
+
const HELPER_SRC = `
|
|
23
|
+
const fs = require('node:fs');
|
|
24
|
+
const tty = require('node:tty');
|
|
25
|
+
const die = () => process.kill(process.pid, 'SIGKILL');
|
|
26
|
+
process.on('disconnect', die); // our parent is gone: never apply a stale mode
|
|
27
|
+
process.on('uncaughtException', die);
|
|
28
|
+
let term = null;
|
|
29
|
+
// fd 3 is the parent's stdin (the terminal the dashboard takes keys from — it need not
|
|
30
|
+
// be the controlling terminal), else the controlling terminal.
|
|
31
|
+
function openTerminal() {
|
|
32
|
+
if (tty.isatty(3)) return new tty.ReadStream(3);
|
|
33
|
+
let fd = null;
|
|
34
|
+
try {
|
|
35
|
+
fd = fs.openSync('/dev/tty', fs.constants.O_RDWR | fs.constants.O_NOCTTY);
|
|
36
|
+
if (tty.isatty(fd)) return new tty.ReadStream(fd);
|
|
37
|
+
} catch (err) {
|
|
38
|
+
if (fd != null) { try { fs.closeSync(fd); } catch {} }
|
|
39
|
+
throw err;
|
|
40
|
+
}
|
|
41
|
+
try { fs.closeSync(fd); } catch {}
|
|
42
|
+
throw Object.assign(new Error('/dev/tty is not a terminal'), { code: 'ENOTTY' });
|
|
43
|
+
}
|
|
44
|
+
process.on('message', ({ id, raw, toggle }) => {
|
|
45
|
+
try {
|
|
46
|
+
// Never read from it (no resume): the parent owns the keyboard.
|
|
47
|
+
if (!term) term = openTerminal();
|
|
48
|
+
// libuv skips a switch to the mode it believes is already set; another process
|
|
49
|
+
// may have changed the terminal since, so 'toggle' goes through the other mode.
|
|
50
|
+
if (toggle) term.setRawMode(!raw);
|
|
51
|
+
term.setRawMode(raw);
|
|
52
|
+
process.send({ id, ok: true });
|
|
53
|
+
} catch (err) {
|
|
54
|
+
process.send({ id, ok: false, error: err.code || err.message });
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
`;
|
|
58
|
+
|
|
59
|
+
const spawnHelper = () => spawn(process.execPath, ['-e', HELPER_SRC], {
|
|
60
|
+
// fd 3 = our stdin; nothing on 0-2, so the helper's own exit never touches them.
|
|
61
|
+
stdio: ['ignore', 'ignore', 'ignore', 0, 'ipc'],
|
|
62
|
+
env: { PATH: process.env.PATH || '' },
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* `setRaw(raw, { toggle })` resolves {ok, error?, ms} once the terminal is in that mode
|
|
67
|
+
* — which may be long after the call while the terminal is not reading; the caller's
|
|
68
|
+
* thread is never blocked. ok:false (no terminal, helper failed) means the caller must
|
|
69
|
+
* fall back to its own switch. The helper starts on demand and is killed once it has
|
|
70
|
+
* nothing pending; close() kills it at once — a pending switch then never lands — and
|
|
71
|
+
* so does this process's exit. `makeHelper` is injectable for tests.
|
|
72
|
+
*/
|
|
73
|
+
export function createTerminalModeSwitcher({ makeHelper = spawnHelper, now = Date.now } = {}) {
|
|
74
|
+
let helper = null;
|
|
75
|
+
let seq = 0;
|
|
76
|
+
const pending = new Map();
|
|
77
|
+
const failAll = error => {
|
|
78
|
+
for (const done of pending.values()) done({ ok: false, error });
|
|
79
|
+
pending.clear();
|
|
80
|
+
};
|
|
81
|
+
const kill = h => { try { h?.kill?.('SIGKILL'); } catch { /* already gone */ } };
|
|
82
|
+
const onProcessExit = () => kill(helper);
|
|
83
|
+
let exitHooked = false;
|
|
84
|
+
const ensure = () => {
|
|
85
|
+
if (helper) return helper;
|
|
86
|
+
const h = makeHelper();
|
|
87
|
+
h.unref?.(); // never keeps this process alive
|
|
88
|
+
h.channel?.unref?.();
|
|
89
|
+
h.on('message', ({ id, ok, error }) => {
|
|
90
|
+
const done = pending.get(id);
|
|
91
|
+
if (!done) return;
|
|
92
|
+
pending.delete(id);
|
|
93
|
+
done({ ok, error });
|
|
94
|
+
// Idle: nothing left for it to switch. No helper process lingers between switches.
|
|
95
|
+
if (!pending.size && helper === h) { helper = null; kill(h); }
|
|
96
|
+
});
|
|
97
|
+
h.on('error', err => { if (helper === h) { helper = null; failAll(err.message); } kill(h); });
|
|
98
|
+
h.on('exit', () => { if (helper === h) { helper = null; failAll('terminal helper exited'); } });
|
|
99
|
+
if (!exitHooked) { process.on('exit', onProcessExit); exitHooked = true; }
|
|
100
|
+
helper = h;
|
|
101
|
+
return h;
|
|
102
|
+
};
|
|
103
|
+
return {
|
|
104
|
+
setRaw(raw, { toggle = false } = {}) {
|
|
105
|
+
const t0 = now();
|
|
106
|
+
return new Promise(resolve => {
|
|
107
|
+
const done = r => resolve({ ...r, ms: now() - t0 });
|
|
108
|
+
let h;
|
|
109
|
+
try { h = ensure(); } catch (err) { done({ ok: false, error: err.message }); return; }
|
|
110
|
+
const id = ++seq;
|
|
111
|
+
pending.set(id, done);
|
|
112
|
+
try { h.send({ id, raw: Boolean(raw), toggle: Boolean(toggle) }); } catch (err) { pending.delete(id); done({ ok: false, error: err.message }); }
|
|
113
|
+
});
|
|
114
|
+
},
|
|
115
|
+
/** Kill the helper NOW (exit paths call this first): a pending switch never lands. */
|
|
116
|
+
close() {
|
|
117
|
+
const h = helper;
|
|
118
|
+
helper = null;
|
|
119
|
+
failAll('closed');
|
|
120
|
+
kill(h);
|
|
121
|
+
if (exitHooked) { process.removeListener('exit', onProcessExit); exitHooked = false; }
|
|
122
|
+
},
|
|
123
|
+
};
|
|
124
|
+
}
|