maxpool 1.24.72 → 1.24.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/account-manager.js +82 -4
- package/src/server.js +18 -0
package/package.json
CHANGED
package/src/account-manager.js
CHANGED
|
@@ -179,6 +179,13 @@ const DEFAULT_SCHEDULER = {
|
|
|
179
179
|
// accounts. So in-flight concurrency is the DOMINANT term, with a steep
|
|
180
180
|
// per-account soft cap; burn-pace is only a soft de-preference (never a
|
|
181
181
|
// bench); quota "use-it-or-lose-it" is intentionally a minor signal here.
|
|
182
|
+
// Cache-affinity discount (owner 2026-11-08, see _cacheAffinity): score points
|
|
183
|
+
// a conversation's last-used healthy account sheds (every mode except sticky).
|
|
184
|
+
// On an undiscounted row one in-flight weight unit costs ~12 points past the
|
|
185
|
+
// per-account target (the bonus is rescaled per row so discounted rows match),
|
|
186
|
+
// and a Claude Code turn weighs 3-30 units: 400 ≈ 33 units, about two large
|
|
187
|
+
// turns of extra depth before the conversation moves. 0 = pure per-request.
|
|
188
|
+
cacheAffinityBonus: 400,
|
|
182
189
|
concurrencyWeight: 2, // multiplies in-flight load (activeWeight+reqWeight) — dominant
|
|
183
190
|
perAccountConcurrencyTarget: 3, // D: soft per-account in-flight target; past it, capPenalty bites
|
|
184
191
|
capPenaltyWeight: 10, // steep penalty per unit of in-flight depth past D (throttle safety floor)
|
|
@@ -335,6 +342,7 @@ const PERSISTED_QUOTA_FIELDS = [
|
|
|
335
342
|
'providerQuotaSource', 'lastProbeOkAt', 'noPlan',
|
|
336
343
|
];
|
|
337
344
|
|
|
345
|
+
const CONVERSATION_AFFINITY_MAX = 5000;
|
|
338
346
|
const PROVIDER_429_MIN_BENCH_SEC = 60;
|
|
339
347
|
const PROVIDER_429_MAX_BENCH_SEC = 15 * 60;
|
|
340
348
|
const PROVIDER_429_STREAK_FORGET_MS = 60 * 60 * 1000;
|
|
@@ -591,6 +599,10 @@ export class AccountManager {
|
|
|
591
599
|
this.routingMode = 'automatic';
|
|
592
600
|
this.preferredAccountName = null;
|
|
593
601
|
this.sessionBindings = new Map();
|
|
602
|
+
// Cache affinity: conversation (terminal + prefix hash) → last account name.
|
|
603
|
+
// Bounded LRU — oldest conversations drop first; a dropped entry only means the
|
|
604
|
+
// next turn is balanced fresh.
|
|
605
|
+
this.conversationLastAccount = new Map();
|
|
594
606
|
this._peakCache = null; // per-UTC-minute peak-window memo (see _peakStateFor)
|
|
595
607
|
// CAPACITY LEDGER (2026-08-22): observed tokens per completed window cycle, so
|
|
596
608
|
// "true capacity" is comparable across providers. Restored from state on boot.
|
|
@@ -870,6 +882,7 @@ export class AccountManager {
|
|
|
870
882
|
// disagree with the peak tier that caused the move (defaults to real time).
|
|
871
883
|
this._bindSession(requestInfo.sessionKey, account, requestInfo.model, requestInfo.now);
|
|
872
884
|
}
|
|
885
|
+
this._noteConversationAccount(requestInfo, account);
|
|
873
886
|
account.inFlight++;
|
|
874
887
|
account.activeWeight += weight;
|
|
875
888
|
account.lastUsedAt = Date.now();
|
|
@@ -2232,9 +2245,9 @@ export class AccountManager {
|
|
|
2232
2245
|
}
|
|
2233
2246
|
}
|
|
2234
2247
|
// Session binding only applies in 'sticky' mode. Every other mode scores each
|
|
2235
|
-
// request
|
|
2236
|
-
//
|
|
2237
|
-
//
|
|
2248
|
+
// request — so 20 sessions that started on one account do NOT keep hammering it
|
|
2249
|
+
// while others idle — but with a bounded cache-affinity discount for the
|
|
2250
|
+
// conversation's last account (_cacheAffinity, owner 2026-11-08). Warmup-pull still works under the non-sticky modes because it keys on
|
|
2238
2251
|
// `_isWarming`, not on a binding.
|
|
2239
2252
|
const isStickyMode = this.scheduler.routingMode === 'sticky';
|
|
2240
2253
|
const bound = isStickyMode
|
|
@@ -3035,6 +3048,8 @@ export class AccountManager {
|
|
|
3035
3048
|
|
|
3036
3049
|
_scoreAccount(account, requestInfo = {}, ctx = null, criticalUnlock = null) {
|
|
3037
3050
|
const now = ctx?.now ?? Date.now();
|
|
3051
|
+
// CACHE AFFINITY — see _cacheAffinity.
|
|
3052
|
+
const affinity = this._cacheAffinity(account, requestInfo);
|
|
3038
3053
|
const reqWeight = Math.max(1, requestInfo.weight || 1);
|
|
3039
3054
|
// POOL LINK scores as the cloud's WORST-case load (owner 2026-10-29: "the team
|
|
3040
3055
|
// pool will respect the accounts within it... included in the balancing act").
|
|
@@ -3146,7 +3161,70 @@ export class AccountManager {
|
|
|
3146
3161
|
// default) learns the real number within a cycle. `probing`/requalify still
|
|
3147
3162
|
// flags a never-seen account for learning — that path is unchanged.
|
|
3148
3163
|
|
|
3149
|
-
return concurrency + capPenalty + paceCost + utilizationCost + scopedPace + spread + ramp + reserveCost + criticalCost + failurePenalty;
|
|
3164
|
+
return Math.max(0, concurrency + capPenalty + paceCost + utilizationCost + scopedPace + spread + ramp + reserveCost + criticalCost + failurePenalty - affinity);
|
|
3165
|
+
}
|
|
3166
|
+
|
|
3167
|
+
/**
|
|
3168
|
+
* CACHE AFFINITY (owner 2026-11-08): Anthropic's prompt cache is PER ACCOUNT, so
|
|
3169
|
+
* a conversation that hops accounts re-writes its whole prefix — 72% of all
|
|
3170
|
+
* Anthropic cache writes over 24h were such re-writes (measured 2026-11-08). In
|
|
3171
|
+
* balance mode the account this session used LAST sheds `cacheAffinityBonus`
|
|
3172
|
+
* score points, so the conversation stays put unless there is a MAJOR reason to
|
|
3173
|
+
* move ("stay unless a major preference or truly no other account" — owner):
|
|
3174
|
+
* - availability gates reject it (limit, exclusion, auth) → never rescued here;
|
|
3175
|
+
* - it is no longer healthy: weekly at/over the reserve tier, the 5h window at
|
|
3176
|
+
* or over the same mark, or a failure streak → affinity off, plain balancing;
|
|
3177
|
+
* - it carries roughly two more large requests than the best sibling (one
|
|
3178
|
+
* weight unit in flight costs ~12 points past the per-account target).
|
|
3179
|
+
* Every account type (owner 2026-11-08: treat GLM the same). Measured the same
|
|
3180
|
+
* day: GLM conversations hopped accounts on 39% of turns yet z.ai still reported
|
|
3181
|
+
* 98.6% cache reads, so z.ai's cache is not per-key — affinity buys GLM nothing
|
|
3182
|
+
* there but costs nothing either, and one rule is simpler to reason about.
|
|
3183
|
+
* Applies in every scored mode (balance, prefer-*); never in sticky (see below).
|
|
3184
|
+
*/
|
|
3185
|
+
_noteConversationAccount(requestInfo, account) {
|
|
3186
|
+
const key = this._affinityKey(requestInfo);
|
|
3187
|
+
if (!key) return;
|
|
3188
|
+
const map = this.conversationLastAccount;
|
|
3189
|
+
map.delete(key); // re-insert = most recent (LRU order)
|
|
3190
|
+
map.set(key, account.name);
|
|
3191
|
+
while (map.size > CONVERSATION_AFFINITY_MAX) map.delete(map.keys().next().value);
|
|
3192
|
+
}
|
|
3193
|
+
|
|
3194
|
+
/** Conversation identity: terminal + prefix hash when the server computed one;
|
|
3195
|
+
* falls back to the terminal session key (callers that never set the hash,
|
|
3196
|
+
* e.g. tests and pre-hash call sites). */
|
|
3197
|
+
_affinityKey(requestInfo = {}) {
|
|
3198
|
+
return requestInfo.affinityKey || requestInfo.sessionKey || null;
|
|
3199
|
+
}
|
|
3200
|
+
|
|
3201
|
+
_cacheAffinity(account, requestInfo = {}) {
|
|
3202
|
+
const base = Number(this.scheduler.cacheAffinityBonus) || 0;
|
|
3203
|
+
// Sticky mode has its own binding; its score loop runs only once the session has
|
|
3204
|
+
// decided to LEAVE a hot account — a discount there would pull it straight back.
|
|
3205
|
+
if (base <= 0 || this.scheduler.routingMode === 'sticky') return 0;
|
|
3206
|
+
const key = this._affinityKey(requestInfo);
|
|
3207
|
+
if (!key || this.conversationLastAccount.get(key) !== account.name) return 0;
|
|
3208
|
+
// No readable quota (Kimi's coding key is console-only) → nothing would ever
|
|
3209
|
+
// release the hold before a 429; give such rows plain balancing (red-team r1).
|
|
3210
|
+
if (account.type === 'provider' && account.quota?.providerQuotaSource === 'console-only') return 0;
|
|
3211
|
+
const weeklyState = this._weeklyRawState(account);
|
|
3212
|
+
// Holds through 'soft' (≥65% weekly — most accounts by mid-week) and 'unknown'
|
|
3213
|
+
// (never probed); reserve/critical/exhausted/capped release the conversation.
|
|
3214
|
+
if (!['normal', 'soft', 'unknown'].includes(weeklyState)) return 0;
|
|
3215
|
+
// 5-hour window: Anthropic reports unified5h; GLM/Kimi report providerSes.
|
|
3216
|
+
const five = account.type === 'provider' ? account.quota?.providerSes : account.quota?.unified5h;
|
|
3217
|
+
if (five != null && five >= this.scheduler.weeklyReserveThreshold) return 0;
|
|
3218
|
+
if ((account.consecutiveFailures || 0) > 0) return 0;
|
|
3219
|
+
// Scale to THIS row's price of one in-flight unit, so every account tolerates the
|
|
3220
|
+
// same extra DEPTH (~33 units) — a fast-refill/soak-discounted GLM row prices a
|
|
3221
|
+
// unit at ~0.4x, and an unscaled bonus let it run ~2.5x deeper (red-team r1).
|
|
3222
|
+
const refillMult = this._fastRefillMultiplier(account);
|
|
3223
|
+
const safetyMult = this.scheduler.soakDiscount < 1
|
|
3224
|
+
? this._fastRefillMultiplier(account, { capFloor: true }) : refillMult;
|
|
3225
|
+
const unit = this.scheduler.concurrencyWeight * refillMult + this.scheduler.capPenaltyWeight * safetyMult;
|
|
3226
|
+
const fullUnit = this.scheduler.concurrencyWeight + this.scheduler.capPenaltyWeight;
|
|
3227
|
+
return base * Math.min(1, unit / fullUnit);
|
|
3150
3228
|
}
|
|
3151
3229
|
|
|
3152
3230
|
/**
|
package/src/server.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
1
2
|
import http from 'node:http';
|
|
2
3
|
import { normalizeClientHeaders, rewriteMetadataUserIdRaw } from './client-identity.js';
|
|
3
4
|
import { appendEventLog } from './event-log.js';
|
|
@@ -496,6 +497,10 @@ export function createProxyServer(accountManager, config, hooks = {}) {
|
|
|
496
497
|
console.error('[Maxpool] x-maxpool-pin-account dropped: the link authenticated with the remote\'s legacy apiKey — mint a named client key (proxy.clients) so pins and per-person attribution work');
|
|
497
498
|
}
|
|
498
499
|
requestInfo.sessionKey = headerValue(req.headers, 'x-maxpool-session');
|
|
500
|
+
// Affinity key = terminal + conversation. Works without the session header
|
|
501
|
+
// too (bare `claude`, teammates' launchers): the conversation hash alone.
|
|
502
|
+
requestInfo.affinityKey = requestInfo.conversationHash
|
|
503
|
+
? `${requestInfo.sessionKey || ''}:${requestInfo.conversationHash}` : null;
|
|
499
504
|
requestInfo.caller = req._caller || null;
|
|
500
505
|
// Read per request (not captured at boot) so a config flip + seamless reload
|
|
501
506
|
// takes effect on the next request with no restart.
|
|
@@ -3473,6 +3478,19 @@ function describeRequest(req, body) {
|
|
|
3473
3478
|
const json = JSON.parse(body.toString());
|
|
3474
3479
|
if (json.model) info.model = json.model;
|
|
3475
3480
|
if (json.stream) info.stream = true;
|
|
3481
|
+
// CONVERSATION identity for cache affinity (red-team 2026-11-08): the
|
|
3482
|
+
// x-maxpool-session header names a TERMINAL, and a main thread plus its parallel
|
|
3483
|
+
// subagents share it — so "last account used by this session" chased whichever
|
|
3484
|
+
// request landed last. A conversation's system prompt + first message never
|
|
3485
|
+
// change within it (Claude Code relies on that for its own cache prefix), so a
|
|
3486
|
+
// short hash of those separates the main thread from each subagent.
|
|
3487
|
+
if (Array.isArray(json.messages) && json.messages.length) {
|
|
3488
|
+
info.conversationHash = createHash('sha256')
|
|
3489
|
+
.update(JSON.stringify(json.system ?? ''))
|
|
3490
|
+
.update('\u0000')
|
|
3491
|
+
.update(JSON.stringify(json.messages[0] ?? ''))
|
|
3492
|
+
.digest('hex').slice(0, 16);
|
|
3493
|
+
}
|
|
3476
3494
|
if (json.max_tokens && json.max_tokens > 16_000) weight += 1;
|
|
3477
3495
|
if (json.thinking || json.effort) weight += 1;
|
|
3478
3496
|
if (requiresAnthropicThinkingIntegrity(json)) {
|