dsh-wtq-pool 1.0.5 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -98
- package/README.en.md +25 -26
- package/README.md +26 -27
- package/lib/client.js +39 -6
- package/lib/index.js +1327 -225
- package/package.json +2 -2
- package/renderer/index.html +14 -10
- package/renderer/panel.css +16 -5
- package/renderer/panel.js +166 -82
package/lib/index.js
CHANGED
|
@@ -25,11 +25,10 @@ import { execFile, execFileSync, spawn } from "node:child_process";
|
|
|
25
25
|
*
|
|
26
26
|
* One constant rather than a literal per call site, because those modules are
|
|
27
27
|
* synced byte for byte between the two builds and a literal is what a sync
|
|
28
|
-
* overwrites.
|
|
29
|
-
*
|
|
30
|
-
* builds do not copy.
|
|
28
|
+
* overwrites. Both builds use the same product name, so the host banner and
|
|
29
|
+
* these lines agree.
|
|
31
30
|
*/
|
|
32
|
-
const LOG_PREFIX = "
|
|
31
|
+
const LOG_PREFIX = "wtq-pool";
|
|
33
32
|
//#endregion
|
|
34
33
|
//#region src/loopback.ts
|
|
35
34
|
const LOOPBACK_HOSTS = /* @__PURE__ */ new Set([
|
|
@@ -55,6 +54,21 @@ function hostIsLoopback(host) {
|
|
|
55
54
|
if (host === void 0 || host.trim() === "") return false;
|
|
56
55
|
return LOOPBACK_HOSTS.has(hostnameOfHost(host));
|
|
57
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* The pathname of a request target.
|
|
59
|
+
*
|
|
60
|
+
* `req.url` keeps the query string and may be an absolute-form proxy target;
|
|
61
|
+
* routing compared it whole, so `?api-version=…` or `GET http://host/v1/…`
|
|
62
|
+
* answered 404 on every route.
|
|
63
|
+
*/
|
|
64
|
+
function requestPathname(raw) {
|
|
65
|
+
if (raw === void 0 || raw === "") return "/";
|
|
66
|
+
try {
|
|
67
|
+
return new URL(raw, "http://127.0.0.1").pathname;
|
|
68
|
+
} catch {
|
|
69
|
+
return "/";
|
|
70
|
+
}
|
|
71
|
+
}
|
|
58
72
|
/** A present Origin must be loopback; non-browser clients send none and pass. */
|
|
59
73
|
function originIsLoopback(origin) {
|
|
60
74
|
if (origin === void 0 || origin.trim() === "") return true;
|
|
@@ -345,12 +359,12 @@ function readRequestBody(req, limitBytes = REQUEST_BODY_LIMIT) {
|
|
|
345
359
|
*/
|
|
346
360
|
async function answerRequest(res, handle, log, answerFailure) {
|
|
347
361
|
res.on?.("error", (error) => {
|
|
348
|
-
log?.warn?.(
|
|
362
|
+
log?.warn?.("wtq-pool: response socket closed mid-write", error);
|
|
349
363
|
});
|
|
350
364
|
try {
|
|
351
365
|
await handle();
|
|
352
366
|
} catch (error) {
|
|
353
|
-
log?.warn?.(
|
|
367
|
+
log?.warn?.("wtq-pool: request failed", error);
|
|
354
368
|
const answer = answerFailure ?? ((target, caught) => {
|
|
355
369
|
if (caught instanceof BodyTooLargeError) writeOpenAIError(target, 413, "request_too_large", String(caught.message));
|
|
356
370
|
else writeOpenAIError(target, 500, "internal", String(caught));
|
|
@@ -359,7 +373,7 @@ async function answerRequest(res, handle, log, answerFailure) {
|
|
|
359
373
|
if (!res.headersSent) answer(res, error);
|
|
360
374
|
else res.end();
|
|
361
375
|
} catch (writeError) {
|
|
362
|
-
log?.warn?.(
|
|
376
|
+
log?.warn?.("wtq-pool: could not answer a failed request", writeError);
|
|
363
377
|
}
|
|
364
378
|
}
|
|
365
379
|
}
|
|
@@ -411,6 +425,27 @@ function relayStreamOf(response) {
|
|
|
411
425
|
return response.body;
|
|
412
426
|
}
|
|
413
427
|
/**
|
|
428
|
+
* Write one relayed chunk, waiting while the client's socket is full.
|
|
429
|
+
*
|
|
430
|
+
* A relay that ignores the false return of `res.write` keeps reading the
|
|
431
|
+
* upstream at full speed and parks the whole answer inside the plugin process,
|
|
432
|
+
* competing with the gateways and the panel for one event loop.
|
|
433
|
+
*/
|
|
434
|
+
async function writeChunk(res, chunk) {
|
|
435
|
+
if (res.destroyed || res.writableEnded) return;
|
|
436
|
+
if (res.write(Buffer.from(chunk))) return;
|
|
437
|
+
if (res.destroyed) return;
|
|
438
|
+
await new Promise((resolve) => {
|
|
439
|
+
const wake = () => {
|
|
440
|
+
res.off("drain", wake);
|
|
441
|
+
res.off("close", wake);
|
|
442
|
+
resolve();
|
|
443
|
+
};
|
|
444
|
+
res.once("drain", wake);
|
|
445
|
+
res.once("close", wake);
|
|
446
|
+
});
|
|
447
|
+
}
|
|
448
|
+
/**
|
|
414
449
|
* Relay one web stream to a Node response, honouring backpressure.
|
|
415
450
|
*
|
|
416
451
|
* Written out rather than `Readable.fromWeb` + `pipe`: that adapter stops
|
|
@@ -557,7 +592,7 @@ function createLoopbackServer(options) {
|
|
|
557
592
|
});
|
|
558
593
|
server.listen(options.port ?? 0, "127.0.0.1");
|
|
559
594
|
server.on("error", (error) => {
|
|
560
|
-
options.logger?.warn?.(
|
|
595
|
+
options.logger?.warn?.("wtq-pool: " + options.label + " server error", error);
|
|
561
596
|
});
|
|
562
597
|
return {
|
|
563
598
|
ready,
|
|
@@ -1194,21 +1229,30 @@ const CN_CHAT_BASE = "https://copilot.tencent.com";
|
|
|
1194
1229
|
const CN_BILLING_BASE = "https://www.codebuddy.cn";
|
|
1195
1230
|
/** Global base for `workbuddy.ai` logins. */
|
|
1196
1231
|
const GLOBAL_BASE$1 = "https://www.workbuddy.ai";
|
|
1197
|
-
/** Chat UA: the official desktop client's three-segment spelling. */
|
|
1198
|
-
const CLIENT_UA = "WorkBuddy/5.5.4 WorkBuddy/5.5.4 CLI/2.137.1";
|
|
1199
1232
|
/**
|
|
1200
|
-
*
|
|
1233
|
+
* One client version for every WorkBuddy call.
|
|
1201
1234
|
*
|
|
1202
|
-
* The
|
|
1203
|
-
*
|
|
1235
|
+
* The version appears twice in a request: `X-IDE-Version` and the version
|
|
1236
|
+
* segment of the UA. They were three literals (5.8.4 in the header, 5.5.4 in the
|
|
1237
|
+
* CN UA, 5.5.2 for the roster), which is a combination no real client produces.
|
|
1238
|
+
* The 5.8.4 desktop spelling still returns the full 28-model roster from the
|
|
1239
|
+
* global config (verified 2026-10-26), so every call reads from here.
|
|
1240
|
+
*/
|
|
1241
|
+
const CLIENT_VERSION = "5.8.4";
|
|
1242
|
+
const CLI_VERSION = "2.161.0";
|
|
1243
|
+
/** CN chat UA: the desktop client's three-segment spelling. */
|
|
1244
|
+
const CLIENT_UA = `WorkBuddy/${CLIENT_VERSION} WorkBuddy/${CLIENT_VERSION} CLI/${CLI_VERSION}`;
|
|
1245
|
+
/**
|
|
1246
|
+
* The international desktop client's own chat UA: the `workbuddy-ai/` segment
|
|
1247
|
+
* only exists on that gateway, and the CN spelling gets the CLI roster there.
|
|
1204
1248
|
*/
|
|
1205
|
-
const GLOBAL_CHAT_UA =
|
|
1249
|
+
const GLOBAL_CHAT_UA = `WorkBuddy/${CLIENT_VERSION} workbuddy-ai/${CLIENT_VERSION} CLI/${CLI_VERSION}`;
|
|
1206
1250
|
/**
|
|
1207
1251
|
* Desktop app UA. The global gateway serves its product config only to this
|
|
1208
1252
|
* client channel: the CLI UA gets a truncated roster (or an HTTP 500), which
|
|
1209
1253
|
* is why the international catalog must be read with the desktop spelling.
|
|
1210
1254
|
*/
|
|
1211
|
-
const DESKTOP_UA =
|
|
1255
|
+
const DESKTOP_UA = `WorkBuddy/${CLIENT_VERSION}`;
|
|
1212
1256
|
/** CN model catalog. */
|
|
1213
1257
|
const MODELS_CATALOG_PATH = "/v2/enterprises/personal/models";
|
|
1214
1258
|
/** Global product config, which carries the international model roster. */
|
|
@@ -1395,6 +1439,11 @@ const HARD_CREDIT_MARKERS = [
|
|
|
1395
1439
|
"额度用尽",
|
|
1396
1440
|
"没有积分"
|
|
1397
1441
|
];
|
|
1442
|
+
/** Whether a text names an exhausted balance rather than a rate limit. */
|
|
1443
|
+
function isHardCreditText(body) {
|
|
1444
|
+
const lower = body.toLowerCase();
|
|
1445
|
+
return HARD_CREDIT_MARKERS.some((marker) => lower.includes(marker.toLowerCase()) || body.includes(marker));
|
|
1446
|
+
}
|
|
1398
1447
|
/**
|
|
1399
1448
|
* Session-invalidation markers that mean "sign in again in the WorkBuddy app".
|
|
1400
1449
|
*
|
|
@@ -1405,6 +1454,23 @@ const HARD_CREDIT_MARKERS = [
|
|
|
1405
1454
|
*/
|
|
1406
1455
|
const SESSION_DEAD_PATTERN = /Offline user session not found|"code"\s*:\s*"?12153\b/u;
|
|
1407
1456
|
/**
|
|
1457
|
+
* The plan gate: the account is fine, the model is not included in it.
|
|
1458
|
+
*
|
|
1459
|
+
* Matched as an envelope value rather than a bare substring, for the same
|
|
1460
|
+
* reason as the session marker: `11158` turns up inside ids, and reclassifying
|
|
1461
|
+
* a request error as a plan gate would cool a model that is actually fine.
|
|
1462
|
+
*/
|
|
1463
|
+
const MODEL_TIER_PATTERN = /"code"\s*:\s*"?11158\b/u;
|
|
1464
|
+
/**
|
|
1465
|
+
* The vendor's account-level restriction: `11140`, "request illegal".
|
|
1466
|
+
*
|
|
1467
|
+
* The body names the account ("your account has been restricted due to unusual
|
|
1468
|
+
* activity") rather than the request, and no retry with the same sign-in can
|
|
1469
|
+
* clear it. Reading it as a plain client error answered 400 and stopped the
|
|
1470
|
+
* rotation, so a second account that could serve the turn was never tried.
|
|
1471
|
+
*/
|
|
1472
|
+
const ACCOUNT_RESTRICTED_PATTERN = /"code"\s*:\s*"?11140\b/u;
|
|
1473
|
+
/**
|
|
1408
1474
|
* Hosts the international product answers on, once each has been stripped of a
|
|
1409
1475
|
* leading label. The WorkBuddy AI desktop app signs in at `workbuddy.ai` (and
|
|
1410
1476
|
* the desktop client itself lists `workbuddy.cc` alongside it); the CodeBuddy
|
|
@@ -1451,7 +1517,8 @@ function commonHeaders(credential) {
|
|
|
1451
1517
|
"Origin": billingBase(credential),
|
|
1452
1518
|
"Referer": `${billingBase(credential)}/`,
|
|
1453
1519
|
"User-Agent": CLIENT_UA,
|
|
1454
|
-
"X-CodeBuddy-Request": "1"
|
|
1520
|
+
"X-CodeBuddy-Request": "1",
|
|
1521
|
+
"Connection": "close"
|
|
1455
1522
|
};
|
|
1456
1523
|
}
|
|
1457
1524
|
/**
|
|
@@ -1489,7 +1556,7 @@ function conversationHeaders() {
|
|
|
1489
1556
|
"X-Agent-Intent": "craft",
|
|
1490
1557
|
"X-Agent-Purpose": "conversation",
|
|
1491
1558
|
"X-Agent-Type": "main",
|
|
1492
|
-
"X-IDE-Version":
|
|
1559
|
+
"X-IDE-Version": CLIENT_VERSION,
|
|
1493
1560
|
"X-Private-Data": "true",
|
|
1494
1561
|
"X-Trace-ID": traceId,
|
|
1495
1562
|
"traceparent": `00-${traceId}-${spanId}-01`,
|
|
@@ -1529,7 +1596,9 @@ function billingHeaders(credential) {
|
|
|
1529
1596
|
"Authorization": `Bearer ${credential.accessToken}`,
|
|
1530
1597
|
"Accept": "application/json",
|
|
1531
1598
|
"Content-Type": "application/json",
|
|
1532
|
-
"X-CodeBuddy-Request": "1"
|
|
1599
|
+
"X-CodeBuddy-Request": "1",
|
|
1600
|
+
"User-Agent": CLIENT_UA,
|
|
1601
|
+
"Connection": "close"
|
|
1533
1602
|
};
|
|
1534
1603
|
if (credential.uid !== "" && credential.uid !== void 0) headers["X-User-Id"] = credential.uid;
|
|
1535
1604
|
if (credential.enterpriseId !== void 0 && credential.enterpriseId !== "") {
|
|
@@ -1579,8 +1648,11 @@ function gatewayRejectionText(status) {
|
|
|
1579
1648
|
* Never rejects: a read that fails keeps whatever arrived, which is usually the
|
|
1580
1649
|
* sentence the client needs. Callers on the error path used to let a body that
|
|
1581
1650
|
* died mid-read replace the upstream's status with a bare abort error.
|
|
1651
|
+
*
|
|
1652
|
+
* `timeoutMs` is a seam for the check: the shipped deadline is ten seconds, and
|
|
1653
|
+
* the case that proves the reader is cancelled should not sit it out.
|
|
1582
1654
|
*/
|
|
1583
|
-
async function readErrorBody(response, limitBytes, signal) {
|
|
1655
|
+
async function readErrorBody(response, limitBytes, signal, timeoutMs = ERROR_BODY_TIMEOUT_MS) {
|
|
1584
1656
|
const body = response.body;
|
|
1585
1657
|
if (body === null) return "";
|
|
1586
1658
|
const reader = body.getReader();
|
|
@@ -1591,7 +1663,7 @@ async function readErrorBody(response, limitBytes, signal) {
|
|
|
1591
1663
|
};
|
|
1592
1664
|
const deadline = setTimeout(() => {
|
|
1593
1665
|
abandon("upstream error body read timed out");
|
|
1594
|
-
},
|
|
1666
|
+
}, timeoutMs);
|
|
1595
1667
|
deadline.unref?.();
|
|
1596
1668
|
const onAbort = () => {
|
|
1597
1669
|
abandon("upstream error body read cancelled by the caller");
|
|
@@ -1619,7 +1691,7 @@ async function readErrorBody(response, limitBytes, signal) {
|
|
|
1619
1691
|
}
|
|
1620
1692
|
async function readEnvelope(response) {
|
|
1621
1693
|
const text = await response.text();
|
|
1622
|
-
if (isGatewayHtmlRejection(response.status, text)) throw new Error(gatewayRejectionText(response.status));
|
|
1694
|
+
if (isGatewayHtmlRejection(response.status, text)) throw Object.assign(new Error(gatewayRejectionText(response.status)), { kind: classifyUpstreamError(response.status, text) });
|
|
1623
1695
|
let parsed;
|
|
1624
1696
|
try {
|
|
1625
1697
|
parsed = JSON.parse(text);
|
|
@@ -1636,8 +1708,23 @@ async function readEnvelope(response) {
|
|
|
1636
1708
|
}
|
|
1637
1709
|
/** Fail an envelope whose business code is non-zero, classified like HTTP errors. */
|
|
1638
1710
|
function envelopeError(status, envelope) {
|
|
1639
|
-
const kind = classifyUpstreamError(status,
|
|
1640
|
-
|
|
1711
|
+
const kind = classifyUpstreamError(status, JSON.stringify({
|
|
1712
|
+
code: envelope.code,
|
|
1713
|
+
msg: envelope.msg
|
|
1714
|
+
}));
|
|
1715
|
+
return Object.assign(/* @__PURE__ */ new Error(`workbuddy upstream ${kind} (http ${status}): ${redactSecrets(envelope.msg).slice(0, 160)}`), { kind });
|
|
1716
|
+
}
|
|
1717
|
+
/**
|
|
1718
|
+
* The classified kind of an upstream failure, for a caller holding the error.
|
|
1719
|
+
*
|
|
1720
|
+
* `chatStream` answers with its kind in the result; every JSON endpoint throws,
|
|
1721
|
+
* and a caller that must tell a refused sign-in from a rate limit would
|
|
1722
|
+
* otherwise match on the message text. Anything without a kind - a local fault,
|
|
1723
|
+
* a bug - reads as undefined rather than being guessed at.
|
|
1724
|
+
*/
|
|
1725
|
+
function upstreamErrorKind(error) {
|
|
1726
|
+
const kind = error?.kind;
|
|
1727
|
+
return typeof kind === "string" ? kind : void 0;
|
|
1641
1728
|
}
|
|
1642
1729
|
/**
|
|
1643
1730
|
* Classify an upstream failure from its HTTP status and body excerpt.
|
|
@@ -1659,11 +1746,12 @@ function envelopeError(status, envelope) {
|
|
|
1659
1746
|
function classifyUpstreamError(status, body) {
|
|
1660
1747
|
if (status === 402) return "hard_credit";
|
|
1661
1748
|
if (status === 429) return "soft_rate";
|
|
1749
|
+
if ((status === 400 || status === 403 || status === 422) && ACCOUNT_RESTRICTED_PATTERN.test(body)) return "account_restricted";
|
|
1750
|
+
if ((status === 400 || status === 403 || status === 422) && MODEL_TIER_PATTERN.test(body)) return "model_tier";
|
|
1662
1751
|
if ((status === 401 || status === 403) && SESSION_DEAD_PATTERN.test(body)) return "session_dead";
|
|
1663
1752
|
if (status === 401) return "session_dead";
|
|
1664
1753
|
if (status === 403) {
|
|
1665
|
-
|
|
1666
|
-
for (const marker of HARD_CREDIT_MARKERS) if (lower.includes(marker.toLowerCase()) || body.includes(marker)) return "hard_credit";
|
|
1754
|
+
if (isHardCreditText(body)) return "hard_credit";
|
|
1667
1755
|
}
|
|
1668
1756
|
if (status === 503 && (body.includes("soft_rate") || body.includes("\"code\":6004") || body.includes("频率限制"))) return "soft_rate";
|
|
1669
1757
|
if (status === 403 && !body.includes("\"code\":") && !body.includes("\"msg\":")) return "waf_block";
|
|
@@ -1922,8 +2010,10 @@ var WorkBuddyUpstreamClient = class {
|
|
|
1922
2010
|
}
|
|
1923
2011
|
if (response.ok) {
|
|
1924
2012
|
if (response.body === null) return {
|
|
1925
|
-
ok:
|
|
1926
|
-
|
|
2013
|
+
ok: false,
|
|
2014
|
+
kind: "server",
|
|
2015
|
+
status: 502,
|
|
2016
|
+
message: "upstream returned 200 without a stream"
|
|
1927
2017
|
};
|
|
1928
2018
|
const relayed = withIdleWatchdog(response.body, timeouts?.idleMs, signal);
|
|
1929
2019
|
return {
|
|
@@ -1988,14 +2078,13 @@ var WorkBuddyUpstreamClient = class {
|
|
|
1988
2078
|
...credential.domain === "" ? {} : { "X-Domain": credential.domain },
|
|
1989
2079
|
"X-Product": "SaaS",
|
|
1990
2080
|
"X-Requested-With": "XMLHttpRequest",
|
|
2081
|
+
"X-CodeBuddy-Request": "1",
|
|
1991
2082
|
"Connection": "close",
|
|
1992
2083
|
"User-Agent": DESKTOP_UA
|
|
1993
2084
|
} : {
|
|
2085
|
+
...commonHeaders(credential),
|
|
1994
2086
|
"Authorization": `Bearer ${credential.accessToken}`,
|
|
1995
|
-
"Accept": "application/json"
|
|
1996
|
-
"Origin": billingBase(credential),
|
|
1997
|
-
"Referer": `${billingBase(credential)}/`,
|
|
1998
|
-
"User-Agent": CLIENT_UA
|
|
2087
|
+
"Accept": "application/json"
|
|
1999
2088
|
};
|
|
2000
2089
|
if (!global && credential.enterpriseId !== void 0 && credential.enterpriseId !== "") headers["X-Enterprise-Id"] = credential.enterpriseId;
|
|
2001
2090
|
const response = await this.fetchImpl(url, {
|
|
@@ -2839,6 +2928,15 @@ async function encryptedFieldOpener() {
|
|
|
2839
2928
|
*/
|
|
2840
2929
|
/** Live auth file name the WorkBuddy desktop app writes. */
|
|
2841
2930
|
const WORKBUDDY_LIVE_FILENAME = "workbuddy-desktop.info";
|
|
2931
|
+
/**
|
|
2932
|
+
* Live auth file name the international client writes.
|
|
2933
|
+
*
|
|
2934
|
+
* One file, no snapshots beside it: the client overwrites this one when the user
|
|
2935
|
+
* switches accounts. Treating it as an abandoned snapshot had the pool take over
|
|
2936
|
+
* the session the app was still signed into and renew it behind the app's back,
|
|
2937
|
+
* which spends the rotation the app needed on its next refresh.
|
|
2938
|
+
*/
|
|
2939
|
+
const WORKBUDDY_GLOBAL_LIVE_FILENAME = "workbuddy-desktop-ai.info";
|
|
2842
2940
|
/** Env override for the auth file or its directory. */
|
|
2843
2941
|
const WORKBUDDY_AUTH_FILE_ENV = "WORKBUDDY_AUTH_FILE";
|
|
2844
2942
|
/**
|
|
@@ -2852,6 +2950,15 @@ const WORKBUDDY_AUTH_FILE_ENV = "WORKBUDDY_AUTH_FILE";
|
|
|
2852
2950
|
const ARCHIVE_FILENAME = "credentials-archive.json";
|
|
2853
2951
|
/** Subdirectory of the plugin's own data root that holds the sign-ins it wrote. */
|
|
2854
2952
|
const OWN_SIGN_IN_DIRNAME = "signins";
|
|
2953
|
+
/**
|
|
2954
|
+
* Sign-ins the user deleted, kept as a tombstone so a scan cannot resurrect them.
|
|
2955
|
+
*
|
|
2956
|
+
* Deleting only the file is not enough: the account's token is still in the
|
|
2957
|
+
* archive, and the desktop client still keeps a superseded snapshot of it, so the
|
|
2958
|
+
* next scan took the snapshot over and wrote the file straight back. The row never
|
|
2959
|
+
* moved, and the delete read as a button that does nothing.
|
|
2960
|
+
*/
|
|
2961
|
+
const DELETED_SIGN_INS_FILENAME = "deleted-signins.json";
|
|
2855
2962
|
/** File-name prefix that marks one of those sign-ins as the plugin's own. */
|
|
2856
2963
|
const OWN_SIGN_IN_PREFIX = "workbuddy-panel-";
|
|
2857
2964
|
/**
|
|
@@ -2994,40 +3101,198 @@ function decryptableString(value, decrypt) {
|
|
|
2994
3101
|
return "";
|
|
2995
3102
|
}
|
|
2996
3103
|
/**
|
|
2997
|
-
*
|
|
2998
|
-
* snapshot it left behind).
|
|
2999
|
-
* app
|
|
3104
|
+
* Whether `path` is the desktop app's live sign-in (as opposed to a backup
|
|
3105
|
+
* snapshot it left behind). A copy of it outranks an abandoned snapshot of the
|
|
3106
|
+
* same token, because the app keeps it current.
|
|
3000
3107
|
*/
|
|
3001
3108
|
function isLiveAuthFile(path) {
|
|
3002
|
-
|
|
3109
|
+
const name = basename(path);
|
|
3110
|
+
return name === WORKBUDDY_LIVE_FILENAME || name === WORKBUDDY_GLOBAL_LIVE_FILENAME;
|
|
3003
3111
|
}
|
|
3004
3112
|
/**
|
|
3005
3113
|
* Which of two credentials for the same account the pool should keep.
|
|
3006
3114
|
*
|
|
3007
3115
|
* Ordering, highest first:
|
|
3008
3116
|
*
|
|
3009
|
-
* 1. the
|
|
3010
|
-
* 2. the
|
|
3117
|
+
* 1. the credential the upstream issued most recently (`lastRefreshAtMs`);
|
|
3118
|
+
* 2. the source the copy came from: the live sign-in file, then a sign-in the
|
|
3119
|
+
* plugin wrote, then a superseded snapshot the client left behind;
|
|
3011
3120
|
* 3. the longer stored expiry, as a fallback for documents that carry no issue
|
|
3012
|
-
* time
|
|
3121
|
+
* time at all.
|
|
3122
|
+
*
|
|
3123
|
+
* Issue time comes first, and source rank only breaks a tie between copies of
|
|
3124
|
+
* the same token. Rank used to come first, on the reasoning that a file the
|
|
3125
|
+
* plugin writes is the one with a future, and that ordering cost the pool a
|
|
3126
|
+
* working token outright: a renewal of an account the desktop was signed into
|
|
3127
|
+
* is written to the archive, the plugin's own copy of that account stays one
|
|
3128
|
+
* rotation behind, and the next scan served the dead copy while the live one sat
|
|
3129
|
+
* next to it. An account was marked "sign in again" with a working token in the
|
|
3130
|
+
* same directory.
|
|
3013
3131
|
*
|
|
3014
3132
|
* The stored expiry alone is NOT a freshness signal: the upstream does not
|
|
3015
3133
|
* rewrite it when it revokes a token, so a long-dead backup can claim to expire
|
|
3016
3134
|
* later than the token that actually works. Selecting on it made every upstream
|
|
3017
3135
|
* call return 401 while a perfectly good credential sat in the same directory.
|
|
3018
3136
|
*/
|
|
3019
|
-
function compareFreshness(a, b) {
|
|
3020
|
-
const aLive = isLiveAuthFile(a.sourcePath) ? 1 : 0;
|
|
3021
|
-
const bLive = isLiveAuthFile(b.sourcePath) ? 1 : 0;
|
|
3022
|
-
if (aLive !== bLive) return bLive - aLive;
|
|
3137
|
+
function compareFreshness(a, b, ownDir) {
|
|
3023
3138
|
const aIssued = a.lastRefreshAtMs ?? 0;
|
|
3024
3139
|
const bIssued = b.lastRefreshAtMs ?? 0;
|
|
3140
|
+
if (aIssued !== bIssued && aIssued > 0 && bIssued > 0) return bIssued - aIssued;
|
|
3141
|
+
const aRank = credentialRank(a, ownDir);
|
|
3142
|
+
const bRank = credentialRank(b, ownDir);
|
|
3143
|
+
if (aIssued > 0 !== bIssued > 0) {
|
|
3144
|
+
if (aIssued > 0 && aRank <= 1) return -1;
|
|
3145
|
+
if (bIssued > 0 && bRank <= 1) return 1;
|
|
3146
|
+
}
|
|
3147
|
+
if (aRank !== bRank) return aRank - bRank;
|
|
3025
3148
|
if (aIssued !== bIssued) return bIssued - aIssued;
|
|
3026
3149
|
return b.expiresAtMs - a.expiresAtMs;
|
|
3027
3150
|
}
|
|
3151
|
+
/**
|
|
3152
|
+
* Which kind of source a credential came from, lowest number first.
|
|
3153
|
+
*
|
|
3154
|
+
* A sign-in the plugin wrote outranks a snapshot the client left behind because
|
|
3155
|
+
* the plugin renews it and nothing renews the snapshot: the same token, but one
|
|
3156
|
+
* of the two copies has a future.
|
|
3157
|
+
*/
|
|
3158
|
+
function credentialRank(credential, ownDir) {
|
|
3159
|
+
if (isLiveAuthFile(credential.sourcePath)) return 0;
|
|
3160
|
+
if (ownSignInPath(credential.sourcePath, ownDir)) return 1;
|
|
3161
|
+
return 2;
|
|
3162
|
+
}
|
|
3028
3163
|
/** True when `candidate` should replace `incumbent` for the same account. */
|
|
3029
|
-
function isFresher(candidate, incumbent) {
|
|
3030
|
-
return compareFreshness(candidate, incumbent) < 0;
|
|
3164
|
+
function isFresher(candidate, incumbent, ownDir) {
|
|
3165
|
+
return compareFreshness(candidate, incumbent, ownDir) < 0;
|
|
3166
|
+
}
|
|
3167
|
+
/**
|
|
3168
|
+
* Queue order for the account list: the live sign-in first, then the copy the
|
|
3169
|
+
* upstream issued last, then the newest expiry.
|
|
3170
|
+
*
|
|
3171
|
+
* This is a different question from `compareFreshness`, which decides which copy
|
|
3172
|
+
* of *one* account serves. That rule falls back to the source only when a copy
|
|
3173
|
+
* carries no issue time, which makes it cycle across three copies (a dated
|
|
3174
|
+
* snapshot, an undated live file, a dated panel sign-in) and leaves a list sort
|
|
3175
|
+
* with the order TimSort happened to produce. A key vector cannot.
|
|
3176
|
+
*/
|
|
3177
|
+
function compareQueueOrder(a, b, ownDir) {
|
|
3178
|
+
const aRank = credentialRank(a, ownDir);
|
|
3179
|
+
const bRank = credentialRank(b, ownDir);
|
|
3180
|
+
if (aRank !== bRank) return aRank - bRank;
|
|
3181
|
+
const aIssued = a.lastRefreshAtMs ?? 0;
|
|
3182
|
+
const bIssued = b.lastRefreshAtMs ?? 0;
|
|
3183
|
+
if (aIssued !== bIssued) return bIssued - aIssued;
|
|
3184
|
+
return b.expiresAtMs - a.expiresAtMs;
|
|
3185
|
+
}
|
|
3186
|
+
function tokenDigest(accessToken) {
|
|
3187
|
+
return createHash("sha256").update(`workbuddy\0deleted\0${accessToken}`).digest("hex").slice(0, 32);
|
|
3188
|
+
}
|
|
3189
|
+
/**
|
|
3190
|
+
* A stamp for one file that changes when its content is replaced.
|
|
3191
|
+
*
|
|
3192
|
+
* Millisecond mtime alone repeats on a coarse filesystem, and two pools over one
|
|
3193
|
+
* data directory then kept a stale tombstone set; the size is in here so a rewrite
|
|
3194
|
+
* that lands in the same reported millisecond still reads as a different file.
|
|
3195
|
+
*/
|
|
3196
|
+
async function signInFileStamp(file) {
|
|
3197
|
+
try {
|
|
3198
|
+
const entry = await stat(file);
|
|
3199
|
+
return `${entry.mtimeMs}:${entry.size}`;
|
|
3200
|
+
} catch {
|
|
3201
|
+
return "";
|
|
3202
|
+
}
|
|
3203
|
+
}
|
|
3204
|
+
/**
|
|
3205
|
+
* Run `work` while holding a lock file, or without it when the lock stays taken.
|
|
3206
|
+
*
|
|
3207
|
+
* The tombstone document is the one file two pools over the same data directory
|
|
3208
|
+
* both write, and a read-union-write is only as good as its serialisation. The
|
|
3209
|
+
* budget is short on purpose: a panel call is answered within seconds, and the
|
|
3210
|
+
* work is a few kilobytes of JSON. A lock older than that is a process that died
|
|
3211
|
+
* holding it, so it is dropped and retried.
|
|
3212
|
+
*/
|
|
3213
|
+
async function withFileLock(lock, work) {
|
|
3214
|
+
let held = false;
|
|
3215
|
+
for (let attempt = 0; attempt < 25 && !held; attempt += 1) try {
|
|
3216
|
+
await writeFile(lock, "", {
|
|
3217
|
+
flag: "wx",
|
|
3218
|
+
mode: 384
|
|
3219
|
+
});
|
|
3220
|
+
held = true;
|
|
3221
|
+
break;
|
|
3222
|
+
} catch (error) {
|
|
3223
|
+
if (error.code !== "EEXIST") break;
|
|
3224
|
+
if (await stat(lock).then((entry) => Date.now() - entry.mtimeMs, () => 0) > 5e3) {
|
|
3225
|
+
await unlink(lock).catch(() => {});
|
|
3226
|
+
continue;
|
|
3227
|
+
}
|
|
3228
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
3229
|
+
}
|
|
3230
|
+
try {
|
|
3231
|
+
await work();
|
|
3232
|
+
} finally {
|
|
3233
|
+
if (held) await unlink(lock).catch(() => {});
|
|
3234
|
+
}
|
|
3235
|
+
}
|
|
3236
|
+
/**
|
|
3237
|
+
* Read the tombstone document.
|
|
3238
|
+
*
|
|
3239
|
+
* Throws when the file exists but cannot be read or parsed, so the caller can keep
|
|
3240
|
+
* what it already knows instead of treating a scanner's EBUSY as "no deleted
|
|
3241
|
+
* sign-ins".
|
|
3242
|
+
*/
|
|
3243
|
+
async function readDeletedSignInsFile(file) {
|
|
3244
|
+
const parsed = JSON.parse(await readFile(file, "utf8"));
|
|
3245
|
+
const tombstones = /* @__PURE__ */ new Map();
|
|
3246
|
+
const entries = parsed?.entries;
|
|
3247
|
+
if (entries === null || typeof entries !== "object" || Array.isArray(entries)) return tombstones;
|
|
3248
|
+
for (const [id, value] of Object.entries(entries)) {
|
|
3249
|
+
const entry = value;
|
|
3250
|
+
if (entry === null || typeof entry !== "object") continue;
|
|
3251
|
+
const digest = entry.tokenDigest;
|
|
3252
|
+
if (typeof digest !== "string" || digest === "") continue;
|
|
3253
|
+
const issued = entry.lastRefreshAtMs;
|
|
3254
|
+
const deletedAt = entry.deletedAtMs;
|
|
3255
|
+
tombstones.set(id, {
|
|
3256
|
+
tokenDigest: digest,
|
|
3257
|
+
lastRefreshAtMs: typeof issued === "number" && Number.isFinite(issued) ? issued : 0,
|
|
3258
|
+
deletedAtMs: typeof deletedAt === "number" && Number.isFinite(deletedAt) ? deletedAt : 0
|
|
3259
|
+
});
|
|
3260
|
+
}
|
|
3261
|
+
return tombstones;
|
|
3262
|
+
}
|
|
3263
|
+
/**
|
|
3264
|
+
* Whether `credential` is a sign-in the user made after deleting the account.
|
|
3265
|
+
*
|
|
3266
|
+
* A re-login is newer than the tombstone, so its issue time is the signal. Files
|
|
3267
|
+
* that carry no issue time cannot be ordered, and for those a different token is
|
|
3268
|
+
* the only evidence of a new sign-in.
|
|
3269
|
+
*
|
|
3270
|
+
* The same token never ends it, whatever stamp the copy carries: a superseded
|
|
3271
|
+
* snapshot the client still keeps is that same sign-in lying in another file, and
|
|
3272
|
+
* a tolerant comparison (same stamp within a few minutes) let one of them bring a
|
|
3273
|
+
* deleted account back the moment the build allowed it. An older-issued token is
|
|
3274
|
+
* not a re-login either, which is why the comparison stays strict: a genuine
|
|
3275
|
+
* re-login is issued after the deleted one, and the alternative — accepting a
|
|
3276
|
+
* slightly older stamp as “close enough” — cannot tell it apart from the next
|
|
3277
|
+
* snapshot down the stack.
|
|
3278
|
+
*/
|
|
3279
|
+
function endsTombstone(tombstone, credential) {
|
|
3280
|
+
if (tokenDigest(credential.accessToken) === tombstone.tokenDigest) return false;
|
|
3281
|
+
const issued = credential.lastRefreshAtMs ?? 0;
|
|
3282
|
+
if (tombstone.lastRefreshAtMs > 0 && issued > 0) return issued > tombstone.lastRefreshAtMs;
|
|
3283
|
+
if (tombstone.lastRefreshAtMs === 0 && issued > 0) return issued >= tombstone.deletedAtMs;
|
|
3284
|
+
return isLiveAuthFile(credential.sourcePath);
|
|
3285
|
+
}
|
|
3286
|
+
/**
|
|
3287
|
+
* Whether `sourcePath` sits inside the directory the plugin writes into.
|
|
3288
|
+
*
|
|
3289
|
+
* A walk that escapes the root is not inside it, and `..` is how a path gets
|
|
3290
|
+
* there. An empty walk means the path IS the directory.
|
|
3291
|
+
*/
|
|
3292
|
+
function ownSignInPath(sourcePath, ownDir) {
|
|
3293
|
+
if (ownDir === "" || sourcePath === "") return false;
|
|
3294
|
+
const walked = relative(resolve(ownDir), resolve(sourcePath));
|
|
3295
|
+
return walked !== "" && !walked.startsWith("..") && !isAbsolute(walked);
|
|
3031
3296
|
}
|
|
3032
3297
|
function workbuddyAccountId(credential) {
|
|
3033
3298
|
const stable = credential.uin ?? credential.uid ?? credential.nickname ?? `path:${credential.sourcePath}`;
|
|
@@ -3120,6 +3385,28 @@ async function readCredential(path) {
|
|
|
3120
3385
|
/** Refresh a credential this long before its expiry, so a request never rides one that dies mid-flight. */
|
|
3121
3386
|
const REFRESH_MARGIN_MS = 3e5;
|
|
3122
3387
|
/**
|
|
3388
|
+
* Oldest a sign-in the plugin itself wrote may get before it is renewed, weeks
|
|
3389
|
+
* of lifetime left or not.
|
|
3390
|
+
*
|
|
3391
|
+
* An expiry-based lead cannot do this job. The upstream revokes a session
|
|
3392
|
+
* without rewriting the token's `expiresAt`, so an account the user signed out
|
|
3393
|
+
* of still claims weeks of life: a lead of any size waits for an expiry that
|
|
3394
|
+
* never arrives, and the dead sign-in is only discovered when a request rides
|
|
3395
|
+
* it. Renewing on age instead means the renewal is visible in the panel, and a
|
|
3396
|
+
* sign-in the upstream no longer honours surfaces within this window rather
|
|
3397
|
+
* than at expiry.
|
|
3398
|
+
*
|
|
3399
|
+
* Only the plugin's own sign-ins qualify. A credential read out of the desktop
|
|
3400
|
+
* app's directory belongs to that app: it refreshes it, and a renewal here would
|
|
3401
|
+
* be discarded by the next scan while still spending an upstream call. Once a
|
|
3402
|
+
* desktop sign-in has been renewed here the copy in force is the plugin's own,
|
|
3403
|
+
* which is what keeps an account whose session ran out under this window.
|
|
3404
|
+
*
|
|
3405
|
+
* Six hours: short enough that a revoked session is noticed the same day, long
|
|
3406
|
+
* enough that the cost stays at one lightweight POST per account per window.
|
|
3407
|
+
*/
|
|
3408
|
+
const REFRESH_AFTER_MS = 216e5;
|
|
3409
|
+
/**
|
|
3123
3410
|
* Longest one penalize may cool an account: same 2h ceiling as the parse
|
|
3124
3411
|
* layer, so a raw caller-supplied timestamp cannot sleep a pool all day.
|
|
3125
3412
|
*
|
|
@@ -3130,8 +3417,28 @@ const REFRESH_MARGIN_MS = 3e5;
|
|
|
3130
3417
|
* value can reach it.
|
|
3131
3418
|
*/
|
|
3132
3419
|
const PENALIZE_CAP_MS = 72e5;
|
|
3133
|
-
/**
|
|
3134
|
-
|
|
3420
|
+
/**
|
|
3421
|
+
* How long a hold the pool decided itself keeps an account out of rotation.
|
|
3422
|
+
*
|
|
3423
|
+
* Neither condition behind a hold clears by itself: the credit balance has to
|
|
3424
|
+
* change, or the user has to sign in again. The configured cooldown window is
|
|
3425
|
+
* sized for a limit that expires, and using it here put a held account back at
|
|
3426
|
+
* the head of the queue every minute, where it failed again and burned the
|
|
3427
|
+
* attempts of the healthy accounts behind it. A hold is a fallback, not a
|
|
3428
|
+
* verdict: a manual cooldown reset, a re-scan and a balance read that finds
|
|
3429
|
+
* credits again all release it.
|
|
3430
|
+
*/
|
|
3431
|
+
const POOL_HOLD_MS = 216e5;
|
|
3432
|
+
/**
|
|
3433
|
+
* What a read on an account whose sign-in the upstream refuses is answered with
|
|
3434
|
+
* while the hold stands.
|
|
3435
|
+
*
|
|
3436
|
+
* The upstream's own sentence names a stale sign-in and sends the reader to the
|
|
3437
|
+
* connection check, which is right for the first refusal and noise on every
|
|
3438
|
+
* poll after it: the hold is the plugin's own statement that this account needs
|
|
3439
|
+
* a new sign-in, and it is the one the panel shows until one arrives.
|
|
3440
|
+
*/
|
|
3441
|
+
const REFUSED_SIGN_IN_MESSAGE = "this account's sign-in was refused by the upstream and its refresh token is spent; sign in again in the WorkBuddy desktop app";
|
|
3135
3442
|
/**
|
|
3136
3443
|
* Live per-model cooldown keys kept per account.
|
|
3137
3444
|
*
|
|
@@ -3167,9 +3474,9 @@ const RATE_LIMIT_STREAK_STEPS = 4;
|
|
|
3167
3474
|
* cycle instead of holding a persistently limited account back.
|
|
3168
3475
|
*/
|
|
3169
3476
|
const RATE_LIMIT_STREAK_WINDOW_MS = RATE_LIMIT_STREAK_CAP * 2;
|
|
3170
|
-
/** Drop expired per-model cooldowns, then cap what is left, soonest first. */
|
|
3477
|
+
/** Drop expired per-model cooldowns and blocks, then cap what is left, soonest first. */
|
|
3171
3478
|
function pruneModelCooldowns(account, now) {
|
|
3172
|
-
for (const [modelId, until] of Object.entries(
|
|
3479
|
+
for (const table of [account.modelCooldowns, account.modelBlocks]) for (const [modelId, until] of Object.entries(table)) if (until <= now) delete table[modelId];
|
|
3173
3480
|
const keys = Object.keys(account.modelCooldowns);
|
|
3174
3481
|
if (keys.length <= MAX_MODEL_COOLDOWN_KEYS) return;
|
|
3175
3482
|
keys.sort((a, b) => (account.modelCooldowns[a] ?? 0) - (account.modelCooldowns[b] ?? 0));
|
|
@@ -3188,6 +3495,28 @@ function ownSignInDir(dataPath) {
|
|
|
3188
3495
|
return dataPath === "" ? "" : join(dataPath, OWN_SIGN_IN_DIRNAME);
|
|
3189
3496
|
}
|
|
3190
3497
|
/**
|
|
3498
|
+
* The sign-in document fields one credential spells, in the vocabulary
|
|
3499
|
+
* `writeOwnSignIn` takes.
|
|
3500
|
+
*
|
|
3501
|
+
* `OwnSignIn` spells "the upstream did not say" as the empty string and a parsed
|
|
3502
|
+
* credential spells it as an absent field; coalescing here keeps that meeting
|
|
3503
|
+
* point in one place rather than in every writer.
|
|
3504
|
+
*/
|
|
3505
|
+
function ownSignInOf(credential) {
|
|
3506
|
+
return {
|
|
3507
|
+
region: regionOf(credential.domain),
|
|
3508
|
+
accessToken: credential.accessToken,
|
|
3509
|
+
refreshToken: credential.refreshToken,
|
|
3510
|
+
expiresAtMs: credential.expiresAtMs,
|
|
3511
|
+
domain: credential.domain,
|
|
3512
|
+
...credential.lastRefreshAtMs === void 0 ? {} : { lastRefreshAtMs: credential.lastRefreshAtMs },
|
|
3513
|
+
uid: credential.uid ?? "",
|
|
3514
|
+
uin: credential.uin ?? "",
|
|
3515
|
+
enterpriseId: credential.enterpriseId ?? "",
|
|
3516
|
+
nickname: credential.nickname ?? ""
|
|
3517
|
+
};
|
|
3518
|
+
}
|
|
3519
|
+
/**
|
|
3191
3520
|
* Write one panel-added sign-in as a `.info` file the scan reads back.
|
|
3192
3521
|
*
|
|
3193
3522
|
* The shape is the desktop app's own, so `parseWorkBuddyAuth` opens it with no
|
|
@@ -3206,7 +3535,7 @@ async function writeOwnSignIn(directory, signIn) {
|
|
|
3206
3535
|
refreshToken: signIn.refreshToken,
|
|
3207
3536
|
expiresAt: signIn.expiresAtMs,
|
|
3208
3537
|
domain: signIn.domain,
|
|
3209
|
-
lastRefreshTime: Date.now()
|
|
3538
|
+
lastRefreshTime: signIn.lastRefreshAtMs ?? Date.now()
|
|
3210
3539
|
},
|
|
3211
3540
|
account: {
|
|
3212
3541
|
...signIn.uid === "" ? {} : { uid: signIn.uid },
|
|
@@ -3218,6 +3547,31 @@ async function writeOwnSignIn(directory, signIn) {
|
|
|
3218
3547
|
return file;
|
|
3219
3548
|
}
|
|
3220
3549
|
/**
|
|
3550
|
+
* Rewrite one of the plugin's own sign-in files with a renewed token.
|
|
3551
|
+
*
|
|
3552
|
+
* Read-modify-write of the document that is already there, so the identity
|
|
3553
|
+
* fields the vendor's own parser looks for survive untouched and only the three
|
|
3554
|
+
* auth values a refresh replaces are written. Atomic for the same reason as the
|
|
3555
|
+
* archive: a truncated `.info` is an account the scan cannot open.
|
|
3556
|
+
*/
|
|
3557
|
+
async function renewOwnSignInFile(file, credential) {
|
|
3558
|
+
const document = JSON.parse(await readFile(file, "utf8"));
|
|
3559
|
+
const auth = document.auth ?? {};
|
|
3560
|
+
auth.accessToken = credential.accessToken;
|
|
3561
|
+
auth.refreshToken = credential.refreshToken;
|
|
3562
|
+
auth.expiresAt = credential.expiresAtMs;
|
|
3563
|
+
auth.lastRefreshTime = credential.lastRefreshAtMs ?? Date.now();
|
|
3564
|
+
document.auth = auth;
|
|
3565
|
+
const temporary = `${file}.${randomUUID()}.tmp`;
|
|
3566
|
+
try {
|
|
3567
|
+
await writeFile(temporary, JSON.stringify(document, null, 2), { mode: 384 });
|
|
3568
|
+
await rename(temporary, file);
|
|
3569
|
+
} catch (error) {
|
|
3570
|
+
await unlink(temporary).catch(() => {});
|
|
3571
|
+
throw error;
|
|
3572
|
+
}
|
|
3573
|
+
}
|
|
3574
|
+
/**
|
|
3221
3575
|
* Soonest future one-off expiry. Missing or past sorts last. None left -> list head.
|
|
3222
3576
|
*
|
|
3223
3577
|
* Strict soonest-first, not a weighted spread: this mode exists so a pack about to
|
|
@@ -3407,6 +3761,17 @@ var WorkBuddyAccountPool = class {
|
|
|
3407
3761
|
archiveQueue = Promise.resolve();
|
|
3408
3762
|
/** The one scan in progress; concurrent callers await it instead of duplicating it. */
|
|
3409
3763
|
scanInflight;
|
|
3764
|
+
/** Torn-down sign-ins, read on first use: a scan cannot resurrect what is here. */
|
|
3765
|
+
/**
|
|
3766
|
+
* Cached tombstones, with the file stamp they were read at.
|
|
3767
|
+
*
|
|
3768
|
+
* A second pool over the same data directory writes this file too. Serving a
|
|
3769
|
+
* cache read before that write and then writing it back verbatim dropped the
|
|
3770
|
+
* other pool's deletions, and a deleted account came back the next time the
|
|
3771
|
+
* client's snapshot was adopted. The stamp makes a write elsewhere visible.
|
|
3772
|
+
*/
|
|
3773
|
+
deletedSignIns;
|
|
3774
|
+
deletedSignInsStamp;
|
|
3410
3775
|
/** The directories the in-flight scan is reading, so a repoint supersedes it. */
|
|
3411
3776
|
scanDirs = [];
|
|
3412
3777
|
/**
|
|
@@ -3429,6 +3794,16 @@ var WorkBuddyAccountPool = class {
|
|
|
3429
3794
|
*/
|
|
3430
3795
|
heldIds = /* @__PURE__ */ new Map();
|
|
3431
3796
|
/**
|
|
3797
|
+
* Accounts the desktop client is signed into right now.
|
|
3798
|
+
*
|
|
3799
|
+
* The pool renews one of these into its own directory when the app's session
|
|
3800
|
+
* runs out, and serves that copy until the app writes a newer one. Renewing it
|
|
3801
|
+
* does not make it the pool's own account: the app still holds the session, and
|
|
3802
|
+
* a delete would hide the account from the pool while the user believed they
|
|
3803
|
+
* were removing something the plugin had added.
|
|
3804
|
+
*/
|
|
3805
|
+
appHeldIds = /* @__PURE__ */ new Set();
|
|
3806
|
+
/**
|
|
3432
3807
|
* The sign-ins each account's own files hold, as of the last scan.
|
|
3433
3808
|
*
|
|
3434
3809
|
* Only the files the client and the panel write, not the archive: the archive
|
|
@@ -3439,10 +3814,11 @@ var WorkBuddyAccountPool = class {
|
|
|
3439
3814
|
/**
|
|
3440
3815
|
* Sign-ins the upstream refused, per account.
|
|
3441
3816
|
*
|
|
3442
|
-
*
|
|
3443
|
-
*
|
|
3444
|
-
*
|
|
3445
|
-
*
|
|
3817
|
+
* A refused token must not come back through the ranking: the copies are
|
|
3818
|
+
* compared by issue time, and a token the upstream rejected can be the newest
|
|
3819
|
+
* one (a re-login that did not work, a sign-in copied from another machine).
|
|
3820
|
+
* Without this the scan would serve it again, and the hold would look released
|
|
3821
|
+
* while the requests kept failing. Kept across scans, and
|
|
3446
3822
|
* dropped once the sign-in is no longer on disk or the cooldowns are reset.
|
|
3447
3823
|
*/
|
|
3448
3824
|
refusedSignIns = /* @__PURE__ */ new Map();
|
|
@@ -3531,10 +3907,16 @@ var WorkBuddyAccountPool = class {
|
|
|
3531
3907
|
* True when this account's sign-in is one the plugin wrote, so it may be
|
|
3532
3908
|
* deleted. The panel asks before it offers the button: deleting a desktop
|
|
3533
3909
|
* app's own sign-in would sign the user out of that app.
|
|
3910
|
+
*
|
|
3911
|
+
* A refreshed desktop sign-in does not qualify. The pool holds a copy of it
|
|
3912
|
+
* after a renewal, but the app is still signed in, so the row stays out of
|
|
3913
|
+
* reach: removing it would suppress an account the user only ever added to the
|
|
3914
|
+
* app, and nothing on that row says so.
|
|
3534
3915
|
*/
|
|
3535
3916
|
ownsSignIn(accountId) {
|
|
3536
3917
|
const account = this.accounts.find((item) => item.id === accountId);
|
|
3537
|
-
|
|
3918
|
+
if (account === void 0 || this.appHeldIds.has(accountId)) return false;
|
|
3919
|
+
return this.isOwnSignInPath(account.credential.sourcePath);
|
|
3538
3920
|
}
|
|
3539
3921
|
/**
|
|
3540
3922
|
* Delete one account's sign-in, and its archive entry with it.
|
|
@@ -3553,22 +3935,233 @@ var WorkBuddyAccountPool = class {
|
|
|
3553
3935
|
async removeOwnSignIn(accountId) {
|
|
3554
3936
|
const account = this.accounts.find((item) => item.id === accountId);
|
|
3555
3937
|
if (account === void 0) return false;
|
|
3556
|
-
if (!this.
|
|
3938
|
+
if (!this.ownsSignIn(accountId)) return false;
|
|
3557
3939
|
const running = this.scanInflight;
|
|
3940
|
+
const known = (await this.loadDeletedSignIns()).has(account.id);
|
|
3941
|
+
if (!await this.rememberDeletedSignIn(account.id, account.credential)) throw new Error("could not record the deleted sign-in");
|
|
3558
3942
|
if (running !== void 0) {
|
|
3559
3943
|
if (!await raceWithTimeout(running.catch(() => {}), DELETE_SCAN_WAIT_MS)) running.catch(() => {}).then(() => this.forgetArchived(account.id));
|
|
3560
3944
|
}
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
})
|
|
3945
|
+
try {
|
|
3946
|
+
await unlink(account.credential.sourcePath);
|
|
3947
|
+
} catch (error) {
|
|
3948
|
+
if (error.code !== "ENOENT") {
|
|
3949
|
+
if (!known) {
|
|
3950
|
+
(await this.loadDeletedSignIns()).delete(account.id);
|
|
3951
|
+
await this.writeDeletedSignIns([account.id]);
|
|
3952
|
+
}
|
|
3953
|
+
throw error;
|
|
3954
|
+
}
|
|
3955
|
+
}
|
|
3564
3956
|
await this.forgetArchived(account.id);
|
|
3957
|
+
this.forgetAccount(account.id);
|
|
3565
3958
|
return true;
|
|
3566
3959
|
}
|
|
3567
|
-
/**
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3960
|
+
/** Drop one account and everything the pool remembers about it. */
|
|
3961
|
+
forgetAccount(accountId) {
|
|
3962
|
+
this.accounts = this.accounts.filter((item) => item.id !== accountId);
|
|
3963
|
+
this.noRefreshWarned.delete(accountId);
|
|
3964
|
+
this.creditBalances.delete(accountId);
|
|
3965
|
+
this.creditExpiries.delete(accountId);
|
|
3966
|
+
this.heldIds.delete(accountId);
|
|
3967
|
+
this.signInTokens.delete(accountId);
|
|
3968
|
+
this.refusedSignIns.delete(accountId);
|
|
3969
|
+
}
|
|
3970
|
+
/**
|
|
3971
|
+
* Record one deleted sign-in, so no later scan puts the account back.
|
|
3972
|
+
*
|
|
3973
|
+
* The file is written before the delete is reported: a tombstone that only
|
|
3974
|
+
* lived in memory would be gone after a restart, and the snapshot it has to
|
|
3975
|
+
* hold off is still sitting in the client's directory.
|
|
3976
|
+
*/
|
|
3977
|
+
async rememberDeletedSignIn(id, credential) {
|
|
3978
|
+
const tombstones = await this.loadDeletedSignIns();
|
|
3979
|
+
const previous = tombstones.get(id);
|
|
3980
|
+
tombstones.set(id, {
|
|
3981
|
+
tokenDigest: tokenDigest(credential.accessToken),
|
|
3982
|
+
lastRefreshAtMs: credential.lastRefreshAtMs ?? 0,
|
|
3983
|
+
deletedAtMs: Date.now()
|
|
3984
|
+
});
|
|
3985
|
+
if (await this.writeDeletedSignIns()) return true;
|
|
3986
|
+
if (previous === void 0) tombstones.delete(id);
|
|
3987
|
+
else tombstones.set(id, previous);
|
|
3988
|
+
return false;
|
|
3989
|
+
}
|
|
3990
|
+
/**
|
|
3991
|
+
* Read the tombstones, re-reading when the file changed underneath us.
|
|
3992
|
+
*
|
|
3993
|
+
* Nothing is pruned by age. A deleted sign-in's copy is still lying in the
|
|
3994
|
+
* client's directory until the client overwrites it, so a tombstone that
|
|
3995
|
+
* expired would bring that copy straight back, refresh token included.
|
|
3996
|
+
*/
|
|
3997
|
+
async loadDeletedSignIns() {
|
|
3998
|
+
const file = this.deletedSignInsFile();
|
|
3999
|
+
const stamp = file === void 0 ? "" : await signInFileStamp(file);
|
|
4000
|
+
if (this.deletedSignIns !== void 0 && this.deletedSignInsStamp === stamp) return this.deletedSignIns;
|
|
4001
|
+
if (file === void 0) {
|
|
4002
|
+
this.deletedSignInsStamp = "";
|
|
4003
|
+
this.deletedSignIns = /* @__PURE__ */ new Map();
|
|
4004
|
+
return this.deletedSignIns;
|
|
4005
|
+
}
|
|
4006
|
+
try {
|
|
4007
|
+
const tombstones = await readDeletedSignInsFile(file);
|
|
4008
|
+
this.deletedSignInsStamp = stamp;
|
|
4009
|
+
this.deletedSignIns = tombstones;
|
|
4010
|
+
return tombstones;
|
|
4011
|
+
} catch (error) {
|
|
4012
|
+
if (error.code === "ENOENT") {
|
|
4013
|
+
this.deletedSignInsStamp = stamp;
|
|
4014
|
+
this.deletedSignIns = /* @__PURE__ */ new Map();
|
|
4015
|
+
return this.deletedSignIns;
|
|
4016
|
+
}
|
|
4017
|
+
this.logger?.warn?.(`${LOG_PREFIX}: could not read the deleted sign-ins: ${String(error)}`);
|
|
4018
|
+
return this.deletedSignIns ?? /* @__PURE__ */ new Map();
|
|
4019
|
+
}
|
|
4020
|
+
}
|
|
4021
|
+
/**
|
|
4022
|
+
* Persist the tombstone set; the caller has already updated the map.
|
|
4023
|
+
*
|
|
4024
|
+
* What is on disk is merged in first, except for the ids this call is removing:
|
|
4025
|
+
* a second pool over the same data directory writes this same file, and writing
|
|
4026
|
+
* our map verbatim dropped a delete it had already recorded. The removals are the
|
|
4027
|
+
* other half of that: a re-login ends a tombstone and a failed delete rolls one
|
|
4028
|
+
* back, and a pure union put both straight back.
|
|
4029
|
+
*
|
|
4030
|
+
* Answers false when the file could not be written, so a delete can refuse to
|
|
4031
|
+
* report success on a state it could not record.
|
|
4032
|
+
*/
|
|
4033
|
+
async writeDeletedSignIns(removed = []) {
|
|
4034
|
+
const tombstones = this.deletedSignIns;
|
|
4035
|
+
const file = this.deletedSignInsFile();
|
|
4036
|
+
if (tombstones === void 0 || file === void 0) return true;
|
|
4037
|
+
let written = false;
|
|
4038
|
+
await withFileLock(`${file}.lock`, async () => {
|
|
4039
|
+
const onDisk = await readDeletedSignInsFile(file).catch(() => /* @__PURE__ */ new Map());
|
|
4040
|
+
for (const [id, entry] of onDisk) {
|
|
4041
|
+
if (removed.includes(id)) continue;
|
|
4042
|
+
const ours = tombstones.get(id);
|
|
4043
|
+
if (ours === void 0 || entry.deletedAtMs > ours.deletedAtMs) tombstones.set(id, entry);
|
|
4044
|
+
}
|
|
4045
|
+
const temporary = `${file}.${randomUUID()}.tmp`;
|
|
4046
|
+
try {
|
|
4047
|
+
await mkdir(dirname(file), { recursive: true });
|
|
4048
|
+
await writeFile(temporary, JSON.stringify({
|
|
4049
|
+
version: 1,
|
|
4050
|
+
entries: Object.fromEntries(tombstones)
|
|
4051
|
+
}, null, 2), { mode: 384 });
|
|
4052
|
+
await rename(temporary, file);
|
|
4053
|
+
this.deletedSignInsStamp = await signInFileStamp(file);
|
|
4054
|
+
written = true;
|
|
4055
|
+
} catch (error) {
|
|
4056
|
+
await unlink(temporary).catch(() => {});
|
|
4057
|
+
this.logger?.warn?.(`${LOG_PREFIX}: could not record the deleted sign-ins: ${String(error)}`);
|
|
4058
|
+
}
|
|
4059
|
+
});
|
|
4060
|
+
return written;
|
|
4061
|
+
}
|
|
4062
|
+
deletedSignInsFile() {
|
|
4063
|
+
const directory = this.dataPath?.();
|
|
4064
|
+
return directory === void 0 || directory === "" ? void 0 : join(directory, DELETED_SIGN_INS_FILENAME);
|
|
4065
|
+
}
|
|
4066
|
+
/** Whether the pool recorded this sign-in as refused by the upstream. */
|
|
4067
|
+
signInRefused(id, token) {
|
|
4068
|
+
if (this.refusedSignIns.get(id)?.includes(token) ?? false) return true;
|
|
4069
|
+
const hold = this.heldIds.get(id);
|
|
4070
|
+
return hold?.reason === "session" && hold.heldSignIns.includes(token);
|
|
4071
|
+
}
|
|
4072
|
+
/**
|
|
4073
|
+
* Every sign-in a refusal could have named for this account.
|
|
4074
|
+
*
|
|
4075
|
+
* The files the last walk read, plus the credential in force when it came from
|
|
4076
|
+
* somewhere else (the archive, or a token this process minted before it was
|
|
4077
|
+
* written to disk). Naming only the files left an account served from the
|
|
4078
|
+
* archive with an empty proof, and a hold with an empty proof is released by
|
|
4079
|
+
* the first scan that follows it.
|
|
4080
|
+
*/
|
|
4081
|
+
refusedSignInsFor(id) {
|
|
4082
|
+
const tokens = (this.signInTokens.get(id) ?? []).map((entry) => entry.accessToken);
|
|
4083
|
+
const inForce = this.accounts.find((account) => account.id === id)?.credential.accessToken;
|
|
4084
|
+
if (inForce !== void 0 && inForce !== "" && !tokens.includes(inForce)) tokens.push(inForce);
|
|
4085
|
+
return tokens;
|
|
4086
|
+
}
|
|
4087
|
+
/** Whether `sourcePath` sits inside the directory the plugin writes into. */
|
|
4088
|
+
isOwnSignInPath(sourcePath) {
|
|
4089
|
+
return ownSignInPath(sourcePath, ownSignInDir(this.dataPath?.() ?? ""));
|
|
4090
|
+
}
|
|
4091
|
+
/**
|
|
4092
|
+
* Take over the sign-ins the desktop client left behind.
|
|
4093
|
+
*
|
|
4094
|
+
* The client keeps one live sign-in and a stack of superseded snapshots beside
|
|
4095
|
+
* it. A snapshot's token is a session the client itself stopped using, so
|
|
4096
|
+
* nothing renews it: it serves until its access token expires and then the
|
|
4097
|
+
* account is gone, holding a refresh token the client's own rotation already
|
|
4098
|
+
* spent. Copying it into the directory the plugin writes makes it the
|
|
4099
|
+
* plugin's account, renewed by the same age rule a panel-added sign-in gets,
|
|
4100
|
+
* so the account outlives the snapshot it came from.
|
|
4101
|
+
*
|
|
4102
|
+
* Only a superseded snapshot is taken over. The live sign-in stays the
|
|
4103
|
+
* client's: it refreshes it, and a second holder of the same session
|
|
4104
|
+
* refreshing on its own schedule is how both end up with a spent refresh
|
|
4105
|
+
* token. An account the client is signed into is skipped for the same reason.
|
|
4106
|
+
*/
|
|
4107
|
+
async adoptOrphanedSignIns(found, tornDown) {
|
|
4108
|
+
const directory = ownSignInDir(this.dataPath?.() ?? "");
|
|
4109
|
+
if (directory === "") return [];
|
|
4110
|
+
const liveIds = /* @__PURE__ */ new Set();
|
|
4111
|
+
const ownedIds = /* @__PURE__ */ new Set();
|
|
4112
|
+
for (const credential of found) {
|
|
4113
|
+
const id = workbuddyAccountId(credential);
|
|
4114
|
+
if (isLiveAuthFile(credential.sourcePath)) liveIds.add(id);
|
|
4115
|
+
else if (ownSignInPath(credential.sourcePath, directory)) ownedIds.add(id);
|
|
4116
|
+
}
|
|
4117
|
+
const adopted = [];
|
|
4118
|
+
for (const credential of found) {
|
|
4119
|
+
if (isLiveAuthFile(credential.sourcePath)) continue;
|
|
4120
|
+
if (ownSignInPath(credential.sourcePath, directory)) continue;
|
|
4121
|
+
const id = workbuddyAccountId(credential);
|
|
4122
|
+
if (liveIds.has(id) || ownedIds.has(id) || tornDown.has(id)) continue;
|
|
4123
|
+
if ((credential.uin ?? credential.uid ?? credential.nickname ?? "") === "") continue;
|
|
4124
|
+
let file;
|
|
4125
|
+
try {
|
|
4126
|
+
file = await writeOwnSignIn(directory, {
|
|
4127
|
+
...ownSignInOf(credential),
|
|
4128
|
+
lastRefreshAtMs: credential.lastRefreshAtMs ?? 0
|
|
4129
|
+
});
|
|
4130
|
+
} catch (error) {
|
|
4131
|
+
this.logger?.warn?.(`${LOG_PREFIX}: could not take over ${accountLabel(credential)}: ${String(error)}`);
|
|
4132
|
+
continue;
|
|
4133
|
+
}
|
|
4134
|
+
ownedIds.add(id);
|
|
4135
|
+
adopted.push({
|
|
4136
|
+
...credential,
|
|
4137
|
+
sourcePath: file
|
|
4138
|
+
});
|
|
4139
|
+
}
|
|
4140
|
+
return adopted;
|
|
4141
|
+
}
|
|
4142
|
+
/**
|
|
4143
|
+
* Write a renewed token back into the sign-in file it was read from.
|
|
4144
|
+
*
|
|
4145
|
+
* Best effort: a failed write costs the renewal its persistence, never the
|
|
4146
|
+
* request already in flight. Anything that did not come from the plugin's own
|
|
4147
|
+
* directory is left alone - the desktop app owns its files.
|
|
4148
|
+
*/
|
|
4149
|
+
async renewOwnSignIn(credential) {
|
|
4150
|
+
if (this.isOwnSignInPath(credential.sourcePath)) {
|
|
4151
|
+
try {
|
|
4152
|
+
await renewOwnSignInFile(credential.sourcePath, credential);
|
|
4153
|
+
} catch (error) {
|
|
4154
|
+
this.logger?.warn?.(`${LOG_PREFIX}: could not write the renewed token back for ${credential.nickname ?? credential.uid}: ${String(error)}`);
|
|
4155
|
+
}
|
|
4156
|
+
return;
|
|
4157
|
+
}
|
|
4158
|
+
const directory = ownSignInDir(this.dataPath?.() ?? "");
|
|
4159
|
+
if (directory === "") return;
|
|
4160
|
+
try {
|
|
4161
|
+
credential.sourcePath = await writeOwnSignIn(directory, ownSignInOf(credential));
|
|
4162
|
+
} catch (error) {
|
|
4163
|
+
this.logger?.warn?.(`${LOG_PREFIX}: could not keep the renewed token for ${credential.nickname ?? credential.uid}: ${String(error)}`);
|
|
4164
|
+
}
|
|
3572
4165
|
}
|
|
3573
4166
|
/**
|
|
3574
4167
|
* Drop one account from the credential archive.
|
|
@@ -3595,10 +4188,11 @@ var WorkBuddyAccountPool = class {
|
|
|
3595
4188
|
/** Drop every account, e.g. when the source is switched off. A later `scan()` rebuilds from disk. */
|
|
3596
4189
|
clear() {
|
|
3597
4190
|
this.accounts = [];
|
|
4191
|
+
this.appHeldIds = /* @__PURE__ */ new Set();
|
|
3598
4192
|
}
|
|
3599
4193
|
/** Rescan the auth directories and merge newly discovered accounts. */
|
|
3600
4194
|
async scan() {
|
|
3601
|
-
const dirs =
|
|
4195
|
+
const dirs = this.scanDirsFor(this.authDirs);
|
|
3602
4196
|
if (this.scanInflight !== void 0 && sameDirs(this.scanDirs, dirs)) return this.scanInflight;
|
|
3603
4197
|
const inflight = this.scanOnce(dirs).finally(() => {
|
|
3604
4198
|
if (this.scanInflight === inflight) {
|
|
@@ -3611,6 +4205,21 @@ var WorkBuddyAccountPool = class {
|
|
|
3611
4205
|
return inflight;
|
|
3612
4206
|
}
|
|
3613
4207
|
/**
|
|
4208
|
+
* The directories a walk reads: the configured ones plus the plugin's own.
|
|
4209
|
+
*
|
|
4210
|
+
* The pool writes sign-ins into its own directory (a renewal it claims, an
|
|
4211
|
+
* account it takes over), so it has to read that directory too, whether or not
|
|
4212
|
+
* the host listed it. Read in both places through this one function: a set that
|
|
4213
|
+
* disagreed with itself read as "the auth directories were repointed mid-walk"
|
|
4214
|
+
* and threw the whole walk away.
|
|
4215
|
+
*/
|
|
4216
|
+
scanDirsFor(authDirs) {
|
|
4217
|
+
const dirs = [...authDirs];
|
|
4218
|
+
const own = ownSignInDir(this.dataPath?.() ?? "");
|
|
4219
|
+
if (own !== "" && !dirs.some((dir) => resolve(dir) === resolve(own))) dirs.push(own);
|
|
4220
|
+
return dirs;
|
|
4221
|
+
}
|
|
4222
|
+
/**
|
|
3614
4223
|
* Keep the credentials this walk saw, so an account whose only file the client
|
|
3615
4224
|
* overwrites on the next sign-in still has a copy to be read back from.
|
|
3616
4225
|
*
|
|
@@ -3619,13 +4228,14 @@ var WorkBuddyAccountPool = class {
|
|
|
3619
4228
|
async archiveCredentials(credentials) {
|
|
3620
4229
|
const file = this.archiveFile();
|
|
3621
4230
|
if (file === void 0 || credentials.length === 0) return;
|
|
4231
|
+
const ownDir = ownSignInDir(this.dataPath?.() ?? "");
|
|
3622
4232
|
const run = this.archiveQueue.then(async () => {
|
|
3623
4233
|
const stored = await readArchivedCredentials(file);
|
|
3624
4234
|
let changed = false;
|
|
3625
4235
|
for (const credential of credentials) {
|
|
3626
4236
|
const id = workbuddyAccountId(credential);
|
|
3627
4237
|
const previous = stored.get(id);
|
|
3628
|
-
if (previous !== void 0 && !isFresher(credential, previous)) continue;
|
|
4238
|
+
if (previous !== void 0 && !isFresher(credential, previous, ownDir)) continue;
|
|
3629
4239
|
stored.set(id, credential);
|
|
3630
4240
|
changed = true;
|
|
3631
4241
|
}
|
|
@@ -3671,7 +4281,31 @@ var WorkBuddyAccountPool = class {
|
|
|
3671
4281
|
else if (read.kind === "sealed") sealedDirs.add(resolve(dir));
|
|
3672
4282
|
}
|
|
3673
4283
|
}
|
|
4284
|
+
const tombstones = await this.loadDeletedSignIns();
|
|
4285
|
+
const tornDown = /* @__PURE__ */ new Set();
|
|
4286
|
+
const releasedIds = [];
|
|
4287
|
+
const held = [];
|
|
4288
|
+
for (const credential of found) {
|
|
4289
|
+
const id = workbuddyAccountId(credential);
|
|
4290
|
+
const tombstone = tombstones.get(id);
|
|
4291
|
+
if (tombstone === void 0) {
|
|
4292
|
+
held.push(credential);
|
|
4293
|
+
continue;
|
|
4294
|
+
}
|
|
4295
|
+
if (!endsTombstone(tombstone, credential)) {
|
|
4296
|
+
tornDown.add(id);
|
|
4297
|
+
continue;
|
|
4298
|
+
}
|
|
4299
|
+
tombstones.delete(id);
|
|
4300
|
+
releasedIds.push(id);
|
|
4301
|
+
held.push(credential);
|
|
4302
|
+
}
|
|
4303
|
+
found.length = 0;
|
|
4304
|
+
found.push(...held);
|
|
4305
|
+
for (const id of tombstones.keys()) tornDown.add(id);
|
|
4306
|
+
if (releasedIds.length > 0) await this.writeDeletedSignIns(releasedIds);
|
|
3674
4307
|
await this.archiveCredentials(found);
|
|
4308
|
+
found.push(...await this.adoptOrphanedSignIns(found, tornDown));
|
|
3675
4309
|
const signInTokens = /* @__PURE__ */ new Map();
|
|
3676
4310
|
for (const credential of found) {
|
|
3677
4311
|
const id = workbuddyAccountId(credential);
|
|
@@ -3680,8 +4314,11 @@ var WorkBuddyAccountPool = class {
|
|
|
3680
4314
|
else held.push(credential);
|
|
3681
4315
|
}
|
|
3682
4316
|
const liveIds = new Set(found.map((credential) => workbuddyAccountId(credential)));
|
|
4317
|
+
const appHeld = new Set(found.filter((credential) => isLiveAuthFile(credential.sourcePath)).map((credential) => workbuddyAccountId(credential)));
|
|
3683
4318
|
for (const credential of await this.archivedCredentials(readableDirs)) {
|
|
3684
|
-
|
|
4319
|
+
const id = workbuddyAccountId(credential);
|
|
4320
|
+
if (tornDown.has(id)) continue;
|
|
4321
|
+
if (found.some((entry) => workbuddyAccountId(entry) === id && entry.accessToken === credential.accessToken)) continue;
|
|
3685
4322
|
found.push(credential);
|
|
3686
4323
|
}
|
|
3687
4324
|
const byId = /* @__PURE__ */ new Map();
|
|
@@ -3695,7 +4332,8 @@ var WorkBuddyAccountPool = class {
|
|
|
3695
4332
|
label: accountLabel(credential),
|
|
3696
4333
|
credential,
|
|
3697
4334
|
cooldownUntilMs: 0,
|
|
3698
|
-
modelCooldowns:
|
|
4335
|
+
modelCooldowns: Object.create(null),
|
|
4336
|
+
modelBlocks: Object.create(null),
|
|
3699
4337
|
rateLimitHits: 0
|
|
3700
4338
|
});
|
|
3701
4339
|
continue;
|
|
@@ -3709,8 +4347,16 @@ var WorkBuddyAccountPool = class {
|
|
|
3709
4347
|
existing.credential = replaced;
|
|
3710
4348
|
this.refusedSignIns.set(id, heldSignIns);
|
|
3711
4349
|
}
|
|
3712
|
-
|
|
3713
|
-
|
|
4350
|
+
if (!this.signInRefused(id, credential.accessToken) && (this.signInRefused(id, existing.credential.accessToken) || isFresher(credential, existing.credential, ownSignInDir(this.dataPath?.() ?? "")))) existing.credential = credential;
|
|
4351
|
+
}
|
|
4352
|
+
for (const [id, account] of byId) {
|
|
4353
|
+
const hold = this.heldIds.get(id);
|
|
4354
|
+
if (hold?.reason !== "session") continue;
|
|
4355
|
+
if (hold.heldSignIns.length === 0) continue;
|
|
4356
|
+
if (this.signInRefused(id, account.credential.accessToken)) continue;
|
|
4357
|
+
this.heldIds.delete(id);
|
|
4358
|
+
if (account.cooldownUntilMs > 0 && account.cooldownUntilMs <= hold.until) account.cooldownUntilMs = 0;
|
|
4359
|
+
this.logger?.info?.(`${LOG_PREFIX}: ${account.label} has a sign-in the upstream did not refuse; releasing the hold`);
|
|
3714
4360
|
}
|
|
3715
4361
|
const foundIds = new Set(found.map((credential) => workbuddyAccountId(credential)));
|
|
3716
4362
|
for (const [id, refused] of this.refusedSignIns) {
|
|
@@ -3728,23 +4374,21 @@ var WorkBuddyAccountPool = class {
|
|
|
3728
4374
|
if (sealedDirs.has(home)) continue;
|
|
3729
4375
|
if (readableDirs.has(home) && !foundIds.has(id)) {
|
|
3730
4376
|
byId.delete(id);
|
|
3731
|
-
this.
|
|
3732
|
-
this.creditBalances.delete(id);
|
|
3733
|
-
this.creditExpiries.delete(id);
|
|
3734
|
-
this.heldIds.delete(id);
|
|
3735
|
-
this.signInTokens.delete(id);
|
|
3736
|
-
this.refusedSignIns.delete(id);
|
|
4377
|
+
this.forgetAccount(id);
|
|
3737
4378
|
}
|
|
3738
4379
|
}
|
|
4380
|
+
this.appHeldIds = appHeld;
|
|
3739
4381
|
const ordered = [...byId.values()];
|
|
3740
|
-
ordered.sort((a, b) =>
|
|
4382
|
+
ordered.sort((a, b) => compareQueueOrder(a.credential, b.credential, ownSignInDir(this.dataPath?.() ?? "")));
|
|
3741
4383
|
for (const account of ordered) {
|
|
3742
4384
|
account.label = accountLabel(account.credential);
|
|
4385
|
+
const home = resolve(dirname(account.credential.sourcePath));
|
|
4386
|
+
if (!readableDirs.has(home) || sealedDirs.has(home)) continue;
|
|
3743
4387
|
if (liveIds.has(account.id)) delete account.archived;
|
|
3744
4388
|
else account.archived = true;
|
|
3745
4389
|
}
|
|
3746
4390
|
disambiguateLabels(ordered);
|
|
3747
|
-
if (!sameDirs(this.authDirs, dirs)) return this.accounts;
|
|
4391
|
+
if (!sameDirs(this.scanDirsFor(this.authDirs), dirs)) return this.accounts;
|
|
3748
4392
|
this.accounts = ordered;
|
|
3749
4393
|
this.signInTokens = signInTokens;
|
|
3750
4394
|
this.lastScanAtMs = Date.now();
|
|
@@ -3757,6 +4401,28 @@ var WorkBuddyAccountPool = class {
|
|
|
3757
4401
|
return kept.filter((account) => regionOf(account.credential.domain) === region);
|
|
3758
4402
|
}
|
|
3759
4403
|
/**
|
|
4404
|
+
* Why this account cannot serve a request, or null when it can.
|
|
4405
|
+
*
|
|
4406
|
+
* One ladder, two readers: `available` filters with it and
|
|
4407
|
+
* `unavailableReason` prints its answer. Keeping them separate meant every
|
|
4408
|
+
* new disqualifier had to be added in two places, and forgetting one made the
|
|
4409
|
+
* panel call an account available that the rotation would never pick.
|
|
4410
|
+
*/
|
|
4411
|
+
ineligible(account, now, modelId, region) {
|
|
4412
|
+
if (this.removedIds.has(account.id)) return "removed";
|
|
4413
|
+
if (this.disabledIds.has(account.id)) return "switched off";
|
|
4414
|
+
if (this.reserveHolds(account.id)) return "under its credit reserve";
|
|
4415
|
+
if (credentialUnusable(account.credential, now)) return "sign-in expired";
|
|
4416
|
+
if (region !== void 0 && regionOf(account.credential.domain) !== region) return "other region";
|
|
4417
|
+
if (account.cooldownUntilMs > now) {
|
|
4418
|
+
const reason = this.heldIds.get(account.id)?.reason;
|
|
4419
|
+
return reason === "session" ? "sign-in refused" : reason === "restricted" ? "restricted by the upstream" : "on hold";
|
|
4420
|
+
}
|
|
4421
|
+
if (modelId !== void 0 && (account.modelBlocks[modelId] ?? 0) > now) return "plan";
|
|
4422
|
+
if (modelId !== void 0 && (account.modelCooldowns[modelId] ?? 0) > now) return "rate-limited";
|
|
4423
|
+
return null;
|
|
4424
|
+
}
|
|
4425
|
+
/**
|
|
3760
4426
|
* Accounts currently eligible to serve a request.
|
|
3761
4427
|
*
|
|
3762
4428
|
* With a `modelId`, an account is eligible when it is not account-wide cooled
|
|
@@ -3769,16 +4435,22 @@ var WorkBuddyAccountPool = class {
|
|
|
3769
4435
|
* for.
|
|
3770
4436
|
*/
|
|
3771
4437
|
available(now, modelId, region) {
|
|
3772
|
-
return this.accounts.filter((account) =>
|
|
3773
|
-
|
|
3774
|
-
|
|
3775
|
-
|
|
3776
|
-
|
|
3777
|
-
|
|
3778
|
-
|
|
3779
|
-
|
|
3780
|
-
|
|
3781
|
-
|
|
4438
|
+
return this.accounts.filter((account) => this.ineligible(account, now, modelId, region) === null);
|
|
4439
|
+
}
|
|
4440
|
+
/**
|
|
4441
|
+
* The pool's own hold on this account, if one is in force right now.
|
|
4442
|
+
*
|
|
4443
|
+
* A caller that is about to spend upstream calls on an account asks first: a
|
|
4444
|
+
* held sign-in cannot answer, so the read is skipped and the hold's reason is
|
|
4445
|
+
* reported instead. Null when no hold applies or the hold has run out.
|
|
4446
|
+
*/
|
|
4447
|
+
activeHold(accountId) {
|
|
4448
|
+
const hold = this.heldIds.get(accountId);
|
|
4449
|
+
if (hold === void 0 || hold.until <= Date.now()) return null;
|
|
4450
|
+
return {
|
|
4451
|
+
reason: hold.reason,
|
|
4452
|
+
untilMs: hold.until
|
|
4453
|
+
};
|
|
3782
4454
|
}
|
|
3783
4455
|
/**
|
|
3784
4456
|
* Why this account is not being served right now, for a caller that has to
|
|
@@ -3791,13 +4463,7 @@ var WorkBuddyAccountPool = class {
|
|
|
3791
4463
|
unavailableReason(accountId, now, modelId) {
|
|
3792
4464
|
const account = this.accounts.find((item) => item.id === accountId);
|
|
3793
4465
|
if (account === void 0) return null;
|
|
3794
|
-
|
|
3795
|
-
if (this.disabledIds.has(account.id)) return "switched off";
|
|
3796
|
-
if (this.reserveHolds(account.id)) return "under its credit reserve";
|
|
3797
|
-
if (credentialUnusable(account.credential, now)) return "sign-in expired";
|
|
3798
|
-
if (account.cooldownUntilMs > now) return this.heldIds.get(account.id)?.reason === "session" ? "sign-in refused" : "on hold";
|
|
3799
|
-
if (modelId !== void 0 && (account.modelCooldowns[modelId] ?? 0) > now) return "rate-limited";
|
|
3800
|
-
return null;
|
|
4466
|
+
return this.ineligible(account, now, modelId);
|
|
3801
4467
|
}
|
|
3802
4468
|
/**
|
|
3803
4469
|
* Pick the account to serve a request.
|
|
@@ -3882,11 +4548,75 @@ var WorkBuddyAccountPool = class {
|
|
|
3882
4548
|
* token is unchanged, so the caller stops retrying the same dead token.
|
|
3883
4549
|
*/
|
|
3884
4550
|
async refreshAccount(accountId) {
|
|
4551
|
+
return await this.refreshForRetry(accountId) === "refreshed";
|
|
4552
|
+
}
|
|
4553
|
+
/**
|
|
4554
|
+
* Refresh after a session-dead answer, keeping a refusal apart from a failure.
|
|
4555
|
+
*
|
|
4556
|
+
* Only the upstream turning the sign-in down justifies a hold. A timeout, a
|
|
4557
|
+
* rate limit or an unreachable host says nothing about the token, and parking
|
|
4558
|
+
* the account for six hours over one bad round trip took a healthy account out
|
|
4559
|
+
* of rotation until a manual reset.
|
|
4560
|
+
*/
|
|
4561
|
+
async refreshForRetry(accountId) {
|
|
3885
4562
|
const account = this.accounts.find((item) => item.id === accountId);
|
|
3886
|
-
if (account === void 0) return
|
|
4563
|
+
if (account === void 0) return "failed";
|
|
3887
4564
|
const previous = account.credential.accessToken;
|
|
3888
|
-
|
|
3889
|
-
|
|
4565
|
+
if (account.credential.refreshToken === "") return "refused";
|
|
4566
|
+
const outcome = await this.ensureFresh(account, true);
|
|
4567
|
+
if (account.credential.accessToken !== previous) return "refreshed";
|
|
4568
|
+
return outcome === "refused" ? "refused" : "failed";
|
|
4569
|
+
}
|
|
4570
|
+
/**
|
|
4571
|
+
* Hold an account whose sign-in the upstream refuses.
|
|
4572
|
+
*
|
|
4573
|
+
* The chat gateway has answered a refused sign-in with this hold since it was
|
|
4574
|
+
* written; the panel's reads and the scheduled passes had no equivalent, so a
|
|
4575
|
+
* revoked account kept spending its two upstream calls on every poll and every
|
|
4576
|
+
* pass. The hold is what stops that, and it is released the way every other
|
|
4577
|
+
* hold is: a new sign-in for the account, a cooldown reset, or its cap.
|
|
4578
|
+
*/
|
|
4579
|
+
holdRefusedSignIn(accountId) {
|
|
4580
|
+
this.penalize(accountId, Date.now() + POOL_HOLD_MS, void 0, "session");
|
|
4581
|
+
}
|
|
4582
|
+
/**
|
|
4583
|
+
* One upstream read that a refused sign-in gets a second chance at.
|
|
4584
|
+
*
|
|
4585
|
+
* The upstream revokes the session a sign-in belongs to without rewriting the
|
|
4586
|
+
* token's own expiry, so an account the user switched away from in the desktop
|
|
4587
|
+
* app keeps a credential that reads as valid for weeks and answers 401 on
|
|
4588
|
+
* every call. A refresh token that still lives recovers the account on the
|
|
4589
|
+
* spot, which is the case the desktop app's own refresh leaves behind; one
|
|
4590
|
+
* that does not leaves the account held instead of retried forever.
|
|
4591
|
+
*
|
|
4592
|
+
* A failure of any other kind is rethrown untouched: a rate limit, a spent
|
|
4593
|
+
* balance and an upstream fault each have their own path, and none of them is
|
|
4594
|
+
* helped by a token this read did not blame.
|
|
4595
|
+
*
|
|
4596
|
+
* While the hold stands the read is not attempted at all. The panel polls
|
|
4597
|
+
* every couple of minutes and the scheduled passes walk the whole pool, and
|
|
4598
|
+
* each of those would otherwise spend the same call on a sign-in that cannot
|
|
4599
|
+
* answer it.
|
|
4600
|
+
*/
|
|
4601
|
+
async readAfterRefresh(accountId, read) {
|
|
4602
|
+
const account = this.accounts.find((item) => item.id === accountId);
|
|
4603
|
+
if (account === void 0) throw new Error(`${LOG_PREFIX}: no account ${accountId} to read for`);
|
|
4604
|
+
if (this.activeHold(accountId)?.reason === "session") throw new Error(REFUSED_SIGN_IN_MESSAGE);
|
|
4605
|
+
try {
|
|
4606
|
+
return await read(account.credential);
|
|
4607
|
+
} catch (error) {
|
|
4608
|
+
if (upstreamErrorKind(error) !== "session_dead") throw error;
|
|
4609
|
+
const outcome = await this.refreshForRetry(accountId);
|
|
4610
|
+
if (outcome === "refreshed") try {
|
|
4611
|
+
return await read(account.credential);
|
|
4612
|
+
} catch (retry) {
|
|
4613
|
+
if (upstreamErrorKind(retry) !== "session_dead") throw retry;
|
|
4614
|
+
this.holdRefusedSignIn(accountId);
|
|
4615
|
+
throw retry;
|
|
4616
|
+
}
|
|
4617
|
+
if (outcome === "refused") this.holdRefusedSignIn(accountId);
|
|
4618
|
+
throw error;
|
|
4619
|
+
}
|
|
3890
4620
|
}
|
|
3891
4621
|
/**
|
|
3892
4622
|
* Refresh the account's access token when it is within the margin (or already
|
|
@@ -3902,12 +4632,11 @@ var WorkBuddyAccountPool = class {
|
|
|
3902
4632
|
async ensureFresh(account, forced = false) {
|
|
3903
4633
|
if (this.client === void 0) return "fresh";
|
|
3904
4634
|
const credential = account.credential;
|
|
3905
|
-
|
|
4635
|
+
const issuedAt = credential.lastRefreshAtMs ?? 0;
|
|
4636
|
+
const stale = this.isOwnSignInPath(credential.sourcePath) && issuedAt > 0 && Date.now() - issuedAt >= REFRESH_AFTER_MS;
|
|
4637
|
+
if (!(forced || credential.expiresAtMs <= 0 || credential.expiresAtMs <= Date.now() + REFRESH_MARGIN_MS || stale)) return "fresh";
|
|
3906
4638
|
const existing = this.refreshInflight.get(account.id);
|
|
3907
|
-
if (existing !== void 0)
|
|
3908
|
-
await existing;
|
|
3909
|
-
return "fresh";
|
|
3910
|
-
}
|
|
4639
|
+
if (existing !== void 0) return await existing;
|
|
3911
4640
|
const run = (async () => {
|
|
3912
4641
|
if (credential.refreshToken === "") {
|
|
3913
4642
|
if (credential.expiresAtMs > Date.now() + 3e4) return "fresh";
|
|
@@ -3928,14 +4657,21 @@ var WorkBuddyAccountPool = class {
|
|
|
3928
4657
|
...outcome.domain === void 0 || outcome.domain === "" ? {} : { domain: outcome.domain },
|
|
3929
4658
|
lastRefreshAtMs: Date.now()
|
|
3930
4659
|
};
|
|
4660
|
+
await this.archiveCredentials([account.credential]);
|
|
4661
|
+
await this.renewOwnSignIn(account.credential);
|
|
3931
4662
|
this.noRefreshWarned.delete(account.id);
|
|
4663
|
+
const sessionHold = this.heldIds.get(account.id);
|
|
4664
|
+
if (sessionHold !== void 0 && sessionHold.reason === "session") {
|
|
4665
|
+
this.heldIds.delete(account.id);
|
|
4666
|
+
if (account.cooldownUntilMs > Date.now() && account.cooldownUntilMs <= sessionHold.until) account.cooldownUntilMs = 0;
|
|
4667
|
+
this.logger?.info?.(`${LOG_PREFIX}: ${account.label} refreshed again; releasing the refused sign-in hold`);
|
|
4668
|
+
}
|
|
3932
4669
|
this.logger?.info?.(`${LOG_PREFIX}: refreshed token for ${account.label}`);
|
|
3933
|
-
await this.archiveCredentials([account.credential]);
|
|
3934
4670
|
return "refreshed";
|
|
3935
4671
|
} catch (error) {
|
|
3936
4672
|
if (credential.expiresAtMs > Date.now() + 3e4) this.logger?.warn?.(`${LOG_PREFIX}: token refresh failed but token still valid for ${account.label}`, error);
|
|
3937
4673
|
else this.logger?.error?.(`${LOG_PREFIX}: token refresh failed and token expired for ${account.label}`, error);
|
|
3938
|
-
return "failed";
|
|
4674
|
+
return upstreamErrorKind(error) === "session_dead" ? "refused" : "failed";
|
|
3939
4675
|
}
|
|
3940
4676
|
})();
|
|
3941
4677
|
this.refreshInflight.set(account.id, run);
|
|
@@ -3953,9 +4689,9 @@ var WorkBuddyAccountPool = class {
|
|
|
3953
4689
|
* limit ("可切换其他模型继续使用"). Without a model id the whole account is
|
|
3954
4690
|
* cooled, which callers should reserve for limits that truly span every model.
|
|
3955
4691
|
*
|
|
3956
|
-
* `hold`
|
|
3957
|
-
* an upstream answer — a spent account, a dead session. It is bounded by
|
|
3958
|
-
*
|
|
4692
|
+
* `hold` names a decision of the pool's own rather than a timestamp read
|
|
4693
|
+
* off an upstream answer — a spent account, a dead session. It is bounded by
|
|
4694
|
+
* POOL_HOLD_MS instead of PENALIZE_CAP_MS, because the two-hour ceiling
|
|
3959
4695
|
* exists to stop a mis-parsed reset from parking an account all day, and
|
|
3960
4696
|
* neither of these conditions clears by itself.
|
|
3961
4697
|
*/
|
|
@@ -3969,8 +4705,8 @@ var WorkBuddyAccountPool = class {
|
|
|
3969
4705
|
account.rateLimitStreak = now - (account.rateLimitStreakAtMs ?? 0) > RATE_LIMIT_STREAK_WINDOW_MS ? 1 : (account.rateLimitStreak ?? 0) + 1;
|
|
3970
4706
|
account.rateLimitStreakAtMs = now;
|
|
3971
4707
|
}
|
|
3972
|
-
const holdReason = hold === false ? void 0 : hold
|
|
3973
|
-
const cap = holdReason !== void 0 ?
|
|
4708
|
+
const holdReason = hold === false ? void 0 : hold;
|
|
4709
|
+
const cap = holdReason !== void 0 ? POOL_HOLD_MS : PENALIZE_CAP_MS;
|
|
3974
4710
|
const streak = Math.min(Math.max((account.rateLimitStreak ?? 1) - 1, 0), RATE_LIMIT_STREAK_STEPS);
|
|
3975
4711
|
const backoff = Math.min(this.cooldownMs * 2 ** streak, Math.max(this.cooldownMs, RATE_LIMIT_STREAK_CAP));
|
|
3976
4712
|
const until = stated ? Math.min(resetAtMs, now + cap) : holdReason !== void 0 ? now + cap : now + backoff;
|
|
@@ -3985,20 +4721,37 @@ var WorkBuddyAccountPool = class {
|
|
|
3985
4721
|
const prior = this.heldIds.get(accountId);
|
|
3986
4722
|
const carried = prior !== void 0 && prior.until > now && prior.reason === "session" ? prior : void 0;
|
|
3987
4723
|
const reason = carried?.reason ?? holdReason;
|
|
3988
|
-
const heldSignIns = carried !== void 0 ? carried.heldSignIns :
|
|
4724
|
+
const heldSignIns = carried !== void 0 ? carried.heldSignIns : this.refusedSignInsFor(accountId);
|
|
3989
4725
|
this.heldIds.set(accountId, {
|
|
3990
4726
|
reason,
|
|
3991
4727
|
until: Math.max(until, carried?.until ?? 0),
|
|
3992
4728
|
heldSignIns
|
|
3993
4729
|
});
|
|
3994
4730
|
}
|
|
3995
|
-
this.logger?.warn(`${LOG_PREFIX}: account ${account.label} rate-limited; cooling until ${new Date(until).toISOString()}`);
|
|
4731
|
+
this.logger?.warn(holdReason !== void 0 ? `${LOG_PREFIX}: account ${account.label} held (${holdReason}); held until ${new Date(until).toISOString()}` : `${LOG_PREFIX}: account ${account.label} rate-limited; cooling until ${new Date(until).toISOString()}`);
|
|
4732
|
+
}
|
|
4733
|
+
/**
|
|
4734
|
+
* Keep one model out of one account's rotation for a stated window.
|
|
4735
|
+
*
|
|
4736
|
+
* `penalize` caps what it writes at `PENALIZE_CAP_MS`, which exists so a
|
|
4737
|
+
* mis-parsed rate-limit reset cannot park work all day. A plan refusal names
|
|
4738
|
+
* no upstream window and clears only when the subscription changes, so it
|
|
4739
|
+
* brings its own deadline; routing it through `penalize` silently shortened
|
|
4740
|
+
* it to the rate-limit ceiling and bought one wasted upstream call per two
|
|
4741
|
+
* hours. The account is left alone: every other model on it still serves.
|
|
4742
|
+
*/
|
|
4743
|
+
blockModel(accountId, modelId, untilMs) {
|
|
4744
|
+
const account = this.accounts.find((item) => item.id === accountId);
|
|
4745
|
+
if (account === void 0 || modelId === "" || untilMs <= Date.now()) return;
|
|
4746
|
+
pruneModelCooldowns(account, Date.now());
|
|
4747
|
+
account.modelBlocks[modelId] = Math.max(account.modelBlocks[modelId] ?? 0, untilMs);
|
|
3996
4748
|
}
|
|
3997
4749
|
/** Clear all cooldowns (account-wide and per-model), e.g. from a reset command. */
|
|
3998
4750
|
resetCooldowns() {
|
|
3999
4751
|
for (const account of this.accounts) {
|
|
4000
4752
|
account.cooldownUntilMs = 0;
|
|
4001
|
-
account.modelCooldowns =
|
|
4753
|
+
account.modelCooldowns = Object.create(null);
|
|
4754
|
+
account.modelBlocks = Object.create(null);
|
|
4002
4755
|
account.rateLimitHits = 0;
|
|
4003
4756
|
account.rateLimitStreak = 0;
|
|
4004
4757
|
account.rateLimitStreakAtMs = 0;
|
|
@@ -4374,6 +5127,15 @@ function decodeTraeEvent(event) {
|
|
|
4374
5127
|
* Loopback guards, the request body bound and the server shell live in
|
|
4375
5128
|
* `./loopback` — one copy, shared with the Trae and Qoder gateways.
|
|
4376
5129
|
*/
|
|
5130
|
+
/** Whether one SSE data frame carries a non-null `finish_reason`. */
|
|
5131
|
+
function carriesFinishReason(data) {
|
|
5132
|
+
try {
|
|
5133
|
+
const parsed = JSON.parse(data);
|
|
5134
|
+
return Array.isArray(parsed.choices) && parsed.choices.some((choice) => typeof choice?.finish_reason === "string" && choice.finish_reason !== "");
|
|
5135
|
+
} catch {
|
|
5136
|
+
return false;
|
|
5137
|
+
}
|
|
5138
|
+
}
|
|
4377
5139
|
/** HTTP status each upstream failure class surfaces as. */
|
|
4378
5140
|
const KIND_STATUS = {
|
|
4379
5141
|
hard_credit: 402,
|
|
@@ -4383,9 +5145,35 @@ const KIND_STATUS = {
|
|
|
4383
5145
|
not_found: 502,
|
|
4384
5146
|
server: 502,
|
|
4385
5147
|
client: 400,
|
|
5148
|
+
model_tier: 400,
|
|
5149
|
+
account_restricted: 403,
|
|
4386
5150
|
transport: 502,
|
|
4387
5151
|
aborted: 499
|
|
4388
5152
|
};
|
|
5153
|
+
/**
|
|
5154
|
+
* How long one account keeps a model it is not entitled to out of rotation.
|
|
5155
|
+
*
|
|
5156
|
+
* A plan does not change inside a working day, and the upstream names no
|
|
5157
|
+
* window of its own. A day is long enough that the gate costs one refusal per
|
|
5158
|
+
* account per day, and short enough that a plan bought later comes back
|
|
5159
|
+
* without a restart.
|
|
5160
|
+
*/
|
|
5161
|
+
const MODEL_TIER_HOLD_MS = 864e5;
|
|
5162
|
+
/**
|
|
5163
|
+
* The business code an account-level 200 envelope carries.
|
|
5164
|
+
*
|
|
5165
|
+
* `classifyUpstreamError` only reads codes on the statuses the upstream uses
|
|
5166
|
+
* for failures, so a refusal wearing a 200 was classed `client` and answered as
|
|
5167
|
+
* a one-off 502. These are the codes that make it the same account property a
|
|
5168
|
+
* non-200 answer would carry.
|
|
5169
|
+
*/
|
|
5170
|
+
function classifyEnvelopeCode(code, text) {
|
|
5171
|
+
if (code === "6004") return "soft_rate";
|
|
5172
|
+
if (code === "11158") return "model_tier";
|
|
5173
|
+
if (code === "11140") return "account_restricted";
|
|
5174
|
+
if (code === "12153") return "session_dead";
|
|
5175
|
+
if (isHardCreditText(text)) return "hard_credit";
|
|
5176
|
+
}
|
|
4389
5177
|
/** Stable key for one conversation: a hash of its first user message. */
|
|
4390
5178
|
function conversationKeyOf(raw) {
|
|
4391
5179
|
try {
|
|
@@ -4403,6 +5191,73 @@ function conversationKeyOf(raw) {
|
|
|
4403
5191
|
}
|
|
4404
5192
|
}
|
|
4405
5193
|
/**
|
|
5194
|
+
* Read enough of a 200 body to tell an SSE stream from a refusal wearing a 200.
|
|
5195
|
+
*
|
|
5196
|
+
* WorkBuddy answers a refusal with a JSON envelope and an upstream hiccup can
|
|
5197
|
+
* end the body before the first frame; both leave the status code claiming
|
|
5198
|
+
* success. The head is what decides: a body that opens with `{` or `[` is the
|
|
5199
|
+
* envelope, a body that already ended empty is no answer at all, and anything
|
|
5200
|
+
* else is relayed as the stream it claims to be.
|
|
5201
|
+
*
|
|
5202
|
+
* The bytes the peek took are returned so the relay can put them back in front.
|
|
5203
|
+
*/
|
|
5204
|
+
async function peekAnswerHead(reader) {
|
|
5205
|
+
const head = [];
|
|
5206
|
+
const decoder = new TextDecoder();
|
|
5207
|
+
let text = "";
|
|
5208
|
+
let bytes = 0;
|
|
5209
|
+
let ended = false;
|
|
5210
|
+
let jsonLooking = false;
|
|
5211
|
+
for (;;) {
|
|
5212
|
+
const next = await reader.read();
|
|
5213
|
+
if (next.done) {
|
|
5214
|
+
ended = true;
|
|
5215
|
+
break;
|
|
5216
|
+
}
|
|
5217
|
+
head.push(next.value);
|
|
5218
|
+
bytes += next.value.byteLength;
|
|
5219
|
+
text += decoder.decode(next.value, { stream: true });
|
|
5220
|
+
const trimmed = text.replace(/^\uFEFF/, "").trimStart();
|
|
5221
|
+
if (trimmed === "") {
|
|
5222
|
+
if (bytes >= 262144 || head.length >= 64) break;
|
|
5223
|
+
continue;
|
|
5224
|
+
}
|
|
5225
|
+
jsonLooking = trimmed.startsWith("{") || trimmed.startsWith("[");
|
|
5226
|
+
if (!jsonLooking) break;
|
|
5227
|
+
if (ended || bytes >= 32768 || head.length >= 64) break;
|
|
5228
|
+
}
|
|
5229
|
+
const trimmed = text.replace(/^\uFEFF/, "").trim();
|
|
5230
|
+
if (trimmed === "" && ended) return {
|
|
5231
|
+
head,
|
|
5232
|
+
ended,
|
|
5233
|
+
refusal: "workbuddy returned an empty answer stream"
|
|
5234
|
+
};
|
|
5235
|
+
if (!jsonLooking) return {
|
|
5236
|
+
head,
|
|
5237
|
+
ended
|
|
5238
|
+
};
|
|
5239
|
+
try {
|
|
5240
|
+
const parsed = JSON.parse(trimmed);
|
|
5241
|
+
const code = parsed["code"] ?? parsed["errCode"] ?? parsed["errorCode"];
|
|
5242
|
+
const msg = parsed["msg"] ?? parsed["message"] ?? parsed["error"];
|
|
5243
|
+
const detail = typeof msg === "string" && msg.trim() !== "" ? msg.trim() : "";
|
|
5244
|
+
const named = code === void 0 || code === null || code === 0 || code === "0" ? "" : `code ${String(code)}`;
|
|
5245
|
+
const sentence = [named, detail].filter((part) => part !== "").join(" · ");
|
|
5246
|
+
return {
|
|
5247
|
+
head,
|
|
5248
|
+
ended,
|
|
5249
|
+
refusal: sentence === "" ? "workbuddy answered 200 with a JSON body where a stream was expected" : `workbuddy answered 200 with ${sentence}`,
|
|
5250
|
+
...named === "" ? {} : { code: String(code) }
|
|
5251
|
+
};
|
|
5252
|
+
} catch {
|
|
5253
|
+
return {
|
|
5254
|
+
head,
|
|
5255
|
+
ended,
|
|
5256
|
+
refusal: `workbuddy answered 200 with a body that is neither a stream nor an envelope: ${safeMessage(trimmed.slice(0, 200))}`
|
|
5257
|
+
};
|
|
5258
|
+
}
|
|
5259
|
+
}
|
|
5260
|
+
/**
|
|
4406
5261
|
* The effort to put back on a request that carries none.
|
|
4407
5262
|
*
|
|
4408
5263
|
* Some providers treat the field as mandatory and reject the whole request
|
|
@@ -4472,23 +5327,13 @@ function isContextTooLong(body) {
|
|
|
4472
5327
|
/**
|
|
4473
5328
|
* How long a spent or dead session is held out of rotation.
|
|
4474
5329
|
*
|
|
4475
|
-
*
|
|
4476
|
-
*
|
|
4477
|
-
*
|
|
4478
|
-
*
|
|
4479
|
-
* healthy accounts behind it. This hold is a fallback, not a verdict: a manual
|
|
4480
|
-
* cooldown reset, a re-scan and a balance read that finds credits again all
|
|
4481
|
-
* release it.
|
|
4482
|
-
*/
|
|
4483
|
-
const SPENT_ACCOUNT_HOLD_MS = 216e5;
|
|
4484
|
-
/**
|
|
4485
|
-
* The hold as the absolute reset `penalize` expects.
|
|
4486
|
-
*
|
|
4487
|
-
* `penalize` takes a timestamp, not a duration: handing it the duration put the
|
|
4488
|
-
* reset in 1970, which reads as already past and falls back to the configured
|
|
5330
|
+
* `penalize` takes an absolute reset rather than a duration, and the duration
|
|
5331
|
+
* itself lives with the pool that owns the hold: see `POOL_HOLD_MS` in
|
|
5332
|
+
* `./accounts`. Handing the duration straight to `penalize` once put the reset
|
|
5333
|
+
* in 1970, which reads as already past and falls back to the configured
|
|
4489
5334
|
* cooldown, so the hold never took effect at all.
|
|
4490
5335
|
*/
|
|
4491
|
-
const
|
|
5336
|
+
const holdUntil = () => Date.now() + POOL_HOLD_MS;
|
|
4492
5337
|
/** Turn an upstream `Retry-After` (seconds) into the absolute reset `penalize` wants. */
|
|
4493
5338
|
function resetFromRetryAfter(seconds) {
|
|
4494
5339
|
if (seconds === void 0) return void 0;
|
|
@@ -4523,6 +5368,16 @@ function unavailableAnswer(pool, live, now, modelId) {
|
|
|
4523
5368
|
detail: `every account is rate-limited for model ${modelId}`,
|
|
4524
5369
|
retryAfterSeconds
|
|
4525
5370
|
};
|
|
5371
|
+
if (only === "plan") return {
|
|
5372
|
+
status: 400,
|
|
5373
|
+
code: "model_tier",
|
|
5374
|
+
detail: `no WorkBuddy account has model ${modelId} in its plan; pick another model, or add an account whose plan includes it`
|
|
5375
|
+
};
|
|
5376
|
+
if (only === "restricted by the upstream") return {
|
|
5377
|
+
status: 403,
|
|
5378
|
+
code: "account_restricted",
|
|
5379
|
+
detail: "every WorkBuddy account was restricted by the upstream; appeal the restriction in the WorkBuddy console, or add another account"
|
|
5380
|
+
};
|
|
4526
5381
|
return {
|
|
4527
5382
|
status: 503,
|
|
4528
5383
|
code: "accounts_unavailable",
|
|
@@ -4544,7 +5399,7 @@ function createWorkBuddyShim(options) {
|
|
|
4544
5399
|
const ready = server.ready;
|
|
4545
5400
|
async function handle(req, res) {
|
|
4546
5401
|
if (!guardsPass(req, res)) return;
|
|
4547
|
-
const url = req.url
|
|
5402
|
+
const url = requestPathname(req.url);
|
|
4548
5403
|
if (req.method === "GET" && (url === "/healthz" || url === "/healthz/")) {
|
|
4549
5404
|
writeJson(res, 200, {
|
|
4550
5405
|
ok: true,
|
|
@@ -4604,18 +5459,16 @@ function createWorkBuddyShim(options) {
|
|
|
4604
5459
|
const tried = [];
|
|
4605
5460
|
let last;
|
|
4606
5461
|
let exhaustedByRateLimit = false;
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
return effort === void 0 ? void 0 : withReasoningEffort(prepared, effort);
|
|
4610
|
-
})();
|
|
5462
|
+
let effortFallback;
|
|
5463
|
+
let effortFallbackBuilt = false;
|
|
4611
5464
|
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
4612
5465
|
if (controller.signal.aborted) return;
|
|
4613
5466
|
const account = await pool.acquire(modelId, region, conversationKey);
|
|
4614
5467
|
if (account === void 0) {
|
|
4615
5468
|
if (exhaustedByRateLimit && last !== void 0) {
|
|
4616
5469
|
const model = ` for model ${modelId}`;
|
|
4617
|
-
const subject = last.kind === "waf_block" ? "every WorkBuddy account was blocked by the upstream gateway (WAF)" : last.kind === "hard_credit" ? `every WorkBuddy account is out of credits${model}` : last.kind === "session_dead" ? `every WorkBuddy sign-in is no longer accepted${model}` : last.kind === "transport" ? `the upstream host could not be reached on any account${model}` : `every account is rate-limited${model}`;
|
|
4618
|
-
const advice = last.kind === "soft_rate" ? "resets at the upstream window" : last.kind === "transport" ? "check the connection to the upstream host" : "sign in again in the desktop app";
|
|
5470
|
+
const subject = last.kind === "waf_block" ? "every WorkBuddy account was blocked by the upstream gateway (WAF)" : last.kind === "hard_credit" ? `every WorkBuddy account is out of credits${model}` : last.kind === "session_dead" ? `every WorkBuddy sign-in is no longer accepted${model}` : last.kind === "model_tier" ? `no WorkBuddy account can use model ${modelId}` : last.kind === "account_restricted" ? "every WorkBuddy account was restricted by the upstream" : last.kind === "transport" ? `the upstream host could not be reached on any account${model}` : `every account is rate-limited${model}`;
|
|
5471
|
+
const advice = last.kind === "soft_rate" ? "resets at the upstream window" : last.kind === "model_tier" ? "pick another model, or add an account whose plan includes it" : last.kind === "account_restricted" ? "appeal the restriction in the WorkBuddy console, or add another account" : last.kind === "transport" ? "check the connection to the upstream host" : "sign in again in the desktop app";
|
|
4619
5472
|
writeOpenAIError(res, KIND_STATUS[last.kind], last.kind, `${subject} (tried ${tried.length}: ${tried.join(", ")}); ${advice} — ${last.message.slice(0, 200)}`);
|
|
4620
5473
|
return;
|
|
4621
5474
|
}
|
|
@@ -4642,27 +5495,33 @@ function createWorkBuddyShim(options) {
|
|
|
4642
5495
|
if (controller.signal.aborted) return;
|
|
4643
5496
|
result = await client.chatStream(account.credential, prepared, controller.signal);
|
|
4644
5497
|
}
|
|
4645
|
-
if (!result.ok && result.kind === "client" &&
|
|
4646
|
-
|
|
4647
|
-
|
|
5498
|
+
if (!result.ok && result.kind === "client" && PARAMETER_REJECTION.test(result.message) && !controller.signal.aborted) {
|
|
5499
|
+
if (!effortFallbackBuilt) {
|
|
5500
|
+
effortFallbackBuilt = true;
|
|
5501
|
+
const effort = declaredEffortOf(catalog, modelId);
|
|
5502
|
+
effortFallback = effort === void 0 ? void 0 : withReasoningEffort(prepared, effort);
|
|
5503
|
+
}
|
|
5504
|
+
if (effortFallback !== void 0) {
|
|
5505
|
+
logger?.info?.(`${LOG_PREFIX}: ${modelId} refused a request that named no effort; retrying with the model's declared default`);
|
|
5506
|
+
result = await client.chatStream(account.credential, effortFallback, controller.signal);
|
|
5507
|
+
}
|
|
4648
5508
|
}
|
|
4649
5509
|
if (result.ok) {
|
|
4650
|
-
|
|
4651
|
-
writeOpenAIError(res, 502, "upstream_error", "workbuddy returned no stream");
|
|
4652
|
-
return;
|
|
4653
|
-
}
|
|
4654
|
-
logger?.info?.(`${LOG_PREFIX}: served by ${account.label}`);
|
|
4655
|
-
res.writeHead(200, {
|
|
4656
|
-
"Content-Type": "text/event-stream",
|
|
4657
|
-
"Cache-Control": "no-cache",
|
|
4658
|
-
"Connection": "keep-alive",
|
|
4659
|
-
"X-Accel-Buffering": "no"
|
|
4660
|
-
});
|
|
4661
|
-
res.flushHeaders();
|
|
5510
|
+
const reader = relayStreamOf(result.response).getReader();
|
|
4662
5511
|
let sawDone = false;
|
|
5512
|
+
let sawFinish = false;
|
|
5513
|
+
let envelopeFailure;
|
|
4663
5514
|
const decoder = new TextDecoder();
|
|
4664
5515
|
const sse = new SseDecoder();
|
|
4665
|
-
const
|
|
5516
|
+
const startStream = () => {
|
|
5517
|
+
res.writeHead(200, {
|
|
5518
|
+
"Content-Type": "text/event-stream",
|
|
5519
|
+
"Cache-Control": "no-cache",
|
|
5520
|
+
"Connection": "keep-alive",
|
|
5521
|
+
"X-Accel-Buffering": "no"
|
|
5522
|
+
});
|
|
5523
|
+
res.flushHeaders();
|
|
5524
|
+
};
|
|
4666
5525
|
/**
|
|
4667
5526
|
* End the relayed stream in a shape the client can finish on.
|
|
4668
5527
|
*
|
|
@@ -4676,7 +5535,7 @@ function createWorkBuddyShim(options) {
|
|
|
4676
5535
|
*/
|
|
4677
5536
|
const failInBand = (error) => {
|
|
4678
5537
|
if (controller.signal.aborted || !res.writable) return;
|
|
4679
|
-
logger?.warn?.(
|
|
5538
|
+
logger?.warn?.("wtq-pool: upstream stream failed mid-flight", safeMessage(error));
|
|
4680
5539
|
try {
|
|
4681
5540
|
res.write(`data: ${JSON.stringify({ error: {
|
|
4682
5541
|
message: safeMessage(error),
|
|
@@ -4693,19 +5552,75 @@ function createWorkBuddyShim(options) {
|
|
|
4693
5552
|
/** End the response with `[DONE]` unless the upstream already sent it. */
|
|
4694
5553
|
function closeWithoutDone() {
|
|
4695
5554
|
if (sawDone || !res.writable) return;
|
|
4696
|
-
logger?.warn?.(
|
|
5555
|
+
logger?.warn?.("wtq-pool: upstream stream ended without [DONE]");
|
|
5556
|
+
if (!sawFinish) res.write(`data: ${JSON.stringify({ choices: [{
|
|
5557
|
+
index: 0,
|
|
5558
|
+
delta: {},
|
|
5559
|
+
finish_reason: "stop"
|
|
5560
|
+
}] })}\n\n`);
|
|
4697
5561
|
res.end("data: [DONE]\n\n");
|
|
4698
5562
|
}
|
|
4699
|
-
|
|
5563
|
+
await (async () => {
|
|
5564
|
+
let head = [];
|
|
5565
|
+
let ended = false;
|
|
5566
|
+
try {
|
|
5567
|
+
const peeked = await peekAnswerHead(reader);
|
|
5568
|
+
head = peeked.head;
|
|
5569
|
+
ended = peeked.ended;
|
|
5570
|
+
if (peeked.refusal !== void 0) {
|
|
5571
|
+
await reader.cancel().catch(() => {});
|
|
5572
|
+
logger?.warn?.(`${LOG_PREFIX}: ${account.label} answered 200 with no stream: ${peeked.refusal}`);
|
|
5573
|
+
const envelopeKind = classifyEnvelopeCode(peeked.code, peeked.refusal);
|
|
5574
|
+
if (envelopeKind === "soft_rate" || envelopeKind === "hard_credit" || envelopeKind === "session_dead" || envelopeKind === "account_restricted" || envelopeKind === "model_tier") {
|
|
5575
|
+
last = {
|
|
5576
|
+
kind: envelopeKind,
|
|
5577
|
+
status: 200,
|
|
5578
|
+
message: peeked.refusal
|
|
5579
|
+
};
|
|
5580
|
+
exhaustedByRateLimit = true;
|
|
5581
|
+
envelopeFailure = envelopeKind;
|
|
5582
|
+
if (envelopeKind === "soft_rate") pool.penalize(account.id, void 0, modelId);
|
|
5583
|
+
else if (envelopeKind === "model_tier") pool.blockModel(account.id, modelId, Date.now() + MODEL_TIER_HOLD_MS);
|
|
5584
|
+
else if (envelopeKind === "session_dead" && await pool.refreshAccount(account.id)) {} else pool.penalize(account.id, holdUntil(), void 0, envelopeKind === "hard_credit" ? "spent" : envelopeKind === "session_dead" ? "session" : "restricted");
|
|
5585
|
+
return;
|
|
5586
|
+
}
|
|
5587
|
+
writeOpenAIError(res, 502, "upstream_error", peeked.refusal);
|
|
5588
|
+
return;
|
|
5589
|
+
}
|
|
5590
|
+
} catch (error) {
|
|
5591
|
+
startStream();
|
|
5592
|
+
failInBand(error);
|
|
5593
|
+
return;
|
|
5594
|
+
}
|
|
5595
|
+
logger?.info?.(`${LOG_PREFIX}: served by ${account.label}`);
|
|
5596
|
+
startStream();
|
|
5597
|
+
let headIndex = 0;
|
|
4700
5598
|
try {
|
|
4701
5599
|
for (;;) {
|
|
4702
|
-
const
|
|
4703
|
-
|
|
5600
|
+
const replayed = headIndex < head.length ? head[headIndex++] : void 0;
|
|
5601
|
+
const { done, value } = replayed !== void 0 ? {
|
|
5602
|
+
done: false,
|
|
5603
|
+
value: replayed
|
|
5604
|
+
} : ended ? {
|
|
5605
|
+
done: true,
|
|
5606
|
+
value: void 0
|
|
5607
|
+
} : await reader.read();
|
|
5608
|
+
if (done || value === void 0) break;
|
|
4704
5609
|
if (!res.writable) return;
|
|
4705
|
-
res
|
|
5610
|
+
await writeChunk(res, value);
|
|
4706
5611
|
try {
|
|
4707
5612
|
const decoded = decoder.decode(value, { stream: true });
|
|
4708
|
-
|
|
5613
|
+
const events = sse.push(decoded);
|
|
5614
|
+
if (!sawFinish && events.some((event) => carriesFinishReason(event.data))) sawFinish = true;
|
|
5615
|
+
if (events.some((event) => event.data === "[DONE]")) {
|
|
5616
|
+
if (!sawFinish && res.writable) {
|
|
5617
|
+
res.write(`data: ${JSON.stringify({ choices: [{
|
|
5618
|
+
index: 0,
|
|
5619
|
+
delta: {},
|
|
5620
|
+
finish_reason: "stop"
|
|
5621
|
+
}] })}\n\n`);
|
|
5622
|
+
sawFinish = true;
|
|
5623
|
+
}
|
|
4709
5624
|
sawDone = true;
|
|
4710
5625
|
if (res.writable) res.end();
|
|
4711
5626
|
await reader.cancel().catch(() => {});
|
|
@@ -4715,7 +5630,9 @@ function createWorkBuddyShim(options) {
|
|
|
4715
5630
|
}
|
|
4716
5631
|
if (!sawDone) try {
|
|
4717
5632
|
const rest = decoder.decode();
|
|
4718
|
-
|
|
5633
|
+
const events = rest === "" ? sse.finish() : [...sse.push(rest), ...sse.finish()];
|
|
5634
|
+
if (events.some((event) => event.data === "[DONE]")) sawDone = true;
|
|
5635
|
+
if (events.some((event) => carriesFinishReason(event.data))) sawFinish = true;
|
|
4719
5636
|
} catch {}
|
|
4720
5637
|
closeWithoutDone();
|
|
4721
5638
|
} catch (error) {
|
|
@@ -4726,6 +5643,8 @@ function createWorkBuddyShim(options) {
|
|
|
4726
5643
|
} catch {}
|
|
4727
5644
|
}
|
|
4728
5645
|
})();
|
|
5646
|
+
if (envelopeFailure === void 0) return;
|
|
5647
|
+
continue;
|
|
4729
5648
|
}
|
|
4730
5649
|
last = {
|
|
4731
5650
|
kind: result.kind,
|
|
@@ -4753,17 +5672,29 @@ function createWorkBuddyShim(options) {
|
|
|
4753
5672
|
* for hours over a transient block.
|
|
4754
5673
|
*
|
|
4755
5674
|
* A `transport` failure is the network path's, not the account's, so the
|
|
4756
|
-
*
|
|
4757
|
-
*
|
|
5675
|
+
* rotation stops: every account here reaches the same host, and trying
|
|
5676
|
+
* each one just spends another header timeout before answering. An
|
|
5677
|
+
* `aborted` turn is the client's own doing and stops the loop too.
|
|
4758
5678
|
*/
|
|
4759
5679
|
if (result.kind === "aborted") break;
|
|
4760
|
-
if (result.kind
|
|
5680
|
+
if (result.kind === "transport") {
|
|
5681
|
+
last = {
|
|
5682
|
+
kind: result.kind,
|
|
5683
|
+
status: result.status,
|
|
5684
|
+
message: result.message
|
|
5685
|
+
};
|
|
5686
|
+
exhaustedByRateLimit = true;
|
|
5687
|
+
break;
|
|
5688
|
+
}
|
|
5689
|
+
if (result.kind !== "soft_rate" && result.kind !== "waf_block" && result.kind !== "hard_credit" && result.kind !== "session_dead" && result.kind !== "model_tier" && result.kind !== "account_restricted") break;
|
|
4761
5690
|
exhaustedByRateLimit = true;
|
|
4762
5691
|
if (result.kind === "soft_rate") {
|
|
4763
5692
|
const bodyReset = parseRateLimitReset(result.message);
|
|
4764
5693
|
const resetAtMs = bodyReset !== void 0 && bodyReset > Date.now() ? bodyReset : resetFromRetryAfter(result.retryAfter);
|
|
4765
5694
|
pool.penalize(account.id, resetAtMs, modelId);
|
|
4766
|
-
} else if (result.kind === "
|
|
5695
|
+
} else if (result.kind === "model_tier") pool.blockModel(account.id, modelId, Date.now() + MODEL_TIER_HOLD_MS);
|
|
5696
|
+
else if (result.kind === "account_restricted") pool.penalize(account.id, holdUntil(), void 0, "restricted");
|
|
5697
|
+
else if (result.kind === "hard_credit" || result.kind === "session_dead") pool.penalize(account.id, holdUntil(), void 0, result.kind === "hard_credit" ? "spent" : "session");
|
|
4767
5698
|
else pool.penalize(account.id);
|
|
4768
5699
|
logger?.warn(`${LOG_PREFIX}: ${account.label} ${result.kind} on ${modelId} (attempt ${attempt + 1}/${maxAttempts}); rotating`);
|
|
4769
5700
|
}
|
|
@@ -4891,6 +5822,7 @@ function nextWindowHour(rule, hour) {
|
|
|
4891
5822
|
const candidate = (hour + step) % 24;
|
|
4892
5823
|
if (rule.hours.includes(candidate)) return candidate;
|
|
4893
5824
|
}
|
|
5825
|
+
return hour;
|
|
4894
5826
|
}
|
|
4895
5827
|
/** The hour the CURRENT window closes, searching forward from `hour`. */
|
|
4896
5828
|
function windowEndHour(rule, hour) {
|
|
@@ -4925,10 +5857,9 @@ function promoStatusFor(model, clock, region) {
|
|
|
4925
5857
|
untilHour
|
|
4926
5858
|
};
|
|
4927
5859
|
}
|
|
4928
|
-
|
|
4929
|
-
if (nextHour !== void 0) return {
|
|
5860
|
+
return {
|
|
4930
5861
|
kind: "night-later",
|
|
4931
|
-
nextHour
|
|
5862
|
+
nextHour: nextWindowHour(rule, clock.hour)
|
|
4932
5863
|
};
|
|
4933
5864
|
}
|
|
4934
5865
|
}
|
|
@@ -6738,6 +7669,13 @@ function bridgeTraeStream(source, model, options = {}) {
|
|
|
6738
7669
|
let emittedFinishReason = false;
|
|
6739
7670
|
let upstreamDone = false;
|
|
6740
7671
|
let usage;
|
|
7672
|
+
let lastKeepaliveAt = 0;
|
|
7673
|
+
const keepalive = (controller) => {
|
|
7674
|
+
const now = Date.now();
|
|
7675
|
+
if (now - lastKeepaliveAt < 1e4) return;
|
|
7676
|
+
lastKeepaliveAt = now;
|
|
7677
|
+
controller.enqueue(encoder.encode(": keepalive\n\n"));
|
|
7678
|
+
};
|
|
6741
7679
|
const chunk = (delta, finishReason = null) => encoder.encode(`data: ${JSON.stringify({
|
|
6742
7680
|
id,
|
|
6743
7681
|
object: "chat.completion.chunk",
|
|
@@ -6765,7 +7703,10 @@ function bridgeTraeStream(source, model, options = {}) {
|
|
|
6765
7703
|
if (decoded.type === "done" || event.event === "done") upstreamDone = true;
|
|
6766
7704
|
const failure = traeFailureOf(decoded);
|
|
6767
7705
|
if (failure !== void 0) throw new Error(describeTraeFailure(failure));
|
|
6768
|
-
if (decoded.type === "unknown")
|
|
7706
|
+
if (decoded.type === "unknown") {
|
|
7707
|
+
keepalive(controller);
|
|
7708
|
+
return;
|
|
7709
|
+
}
|
|
6769
7710
|
if (decoded.type === "delta") {
|
|
6770
7711
|
const delta = {};
|
|
6771
7712
|
if (decoded.text !== "") delta["content"] = decoded.text;
|
|
@@ -6779,7 +7720,10 @@ function bridgeTraeStream(source, model, options = {}) {
|
|
|
6779
7720
|
roleChunk(controller);
|
|
6780
7721
|
controller.enqueue(chunk(delta));
|
|
6781
7722
|
}
|
|
6782
|
-
if (decoded.finishReason === void 0)
|
|
7723
|
+
if (decoded.finishReason === void 0) {
|
|
7724
|
+
keepalive(controller);
|
|
7725
|
+
return;
|
|
7726
|
+
}
|
|
6783
7727
|
}
|
|
6784
7728
|
if (decoded.type === "usage") {
|
|
6785
7729
|
const details = {
|
|
@@ -6795,7 +7739,10 @@ function bridgeTraeStream(source, model, options = {}) {
|
|
|
6795
7739
|
if (emittedFinishReason) controller.enqueue(chunk(null));
|
|
6796
7740
|
return;
|
|
6797
7741
|
}
|
|
6798
|
-
if (decoded.type !== "done" && decoded.type !== "delta")
|
|
7742
|
+
if (decoded.type !== "done" && decoded.type !== "delta") {
|
|
7743
|
+
keepalive(controller);
|
|
7744
|
+
return;
|
|
7745
|
+
}
|
|
6799
7746
|
if (emittedFinishReason) return;
|
|
6800
7747
|
emittedFinishReason = true;
|
|
6801
7748
|
controller.enqueue(chunk({}, openAIFinishReason(decoded.finishReason, sawToolCalls)));
|
|
@@ -6998,7 +7945,8 @@ function capabilityMapOf(models) {
|
|
|
6998
7945
|
*/
|
|
6999
7946
|
function functionGone(status, kind, message) {
|
|
7000
7947
|
if (kind === "not_found") return true;
|
|
7001
|
-
|
|
7948
|
+
if (status !== 400) return false;
|
|
7949
|
+
return /"code"\s*:\s*"?4001\b/.test(message) || /\bcode 4001\b/.test(message);
|
|
7002
7950
|
}
|
|
7003
7951
|
function createTraeGateway(options) {
|
|
7004
7952
|
const logger = options.logger;
|
|
@@ -7148,7 +8096,7 @@ function createTraeGateway(options) {
|
|
|
7148
8096
|
let relayed;
|
|
7149
8097
|
try {
|
|
7150
8098
|
relayed = await openTraeResponse(result.response, model, { onFailure: (error) => {
|
|
7151
|
-
logger?.warn?.(
|
|
8099
|
+
logger?.warn?.("wtq-pool: trae relay stream failed mid-flight", error);
|
|
7152
8100
|
if (!(error instanceof Error && error.name === "AbortError") && !watcher.signal.aborted) options.onAttemptFailure?.(credential, {
|
|
7153
8101
|
status: 0,
|
|
7154
8102
|
kind: "server"
|
|
@@ -7244,13 +8192,13 @@ function createTraeGateway(options) {
|
|
|
7244
8192
|
});
|
|
7245
8193
|
res.flushHeaders();
|
|
7246
8194
|
pumpToResponse(relayStreamOf(relayed), res).catch((error) => {
|
|
7247
|
-
if (!(watcher.signal.aborted || error instanceof Error && error.name === "AbortError")) logger?.warn?.(
|
|
8195
|
+
if (!(watcher.signal.aborted || error instanceof Error && error.name === "AbortError")) logger?.warn?.("wtq-pool: trae relay stream failed mid-flight", error);
|
|
7248
8196
|
if (res.writable) res.end();
|
|
7249
8197
|
});
|
|
7250
8198
|
}
|
|
7251
8199
|
async function handle(req, res) {
|
|
7252
8200
|
if (!guardsPass(req, res)) return;
|
|
7253
|
-
const url = req.url
|
|
8201
|
+
const url = requestPathname(req.url);
|
|
7254
8202
|
if (req.method === "GET" && (url === "/healthz" || url === "/healthz/")) {
|
|
7255
8203
|
writeJson(res, 200, {
|
|
7256
8204
|
ok: true,
|
|
@@ -7270,7 +8218,7 @@ function createTraeGateway(options) {
|
|
|
7270
8218
|
writeOpenAIError(res, 503, "upstream_unavailable", String(error instanceof Error ? error.message : error));
|
|
7271
8219
|
return;
|
|
7272
8220
|
}
|
|
7273
|
-
logger?.warn?.(
|
|
8221
|
+
logger?.warn?.("wtq-pool: trae directory refresh failed; serving the cached roster");
|
|
7274
8222
|
}
|
|
7275
8223
|
const ids = options.listedIds?.();
|
|
7276
8224
|
writeJson(res, 200, {
|
|
@@ -7923,7 +8871,11 @@ async function readQoderCredentials(candidates) {
|
|
|
7923
8871
|
for (const group of byAppDir.values()) {
|
|
7924
8872
|
const result = await readQoderCredential(group);
|
|
7925
8873
|
if (result.credential !== void 0) {
|
|
7926
|
-
|
|
8874
|
+
const fresh = result.credential;
|
|
8875
|
+
const at = credentials.findIndex((existing) => existing.region === fresh.region && existing.userID === fresh.userID);
|
|
8876
|
+
const incumbent = at === -1 ? void 0 : credentials[at];
|
|
8877
|
+
if (incumbent === void 0) credentials.push(fresh);
|
|
8878
|
+
else if (fresh.expiresAtMs > incumbent.expiresAtMs) credentials[at] = fresh;
|
|
7927
8879
|
} else if (result.failure !== void 0 && result.failure.reason !== "no-auth") failures.push(result.failure);
|
|
7928
8880
|
}
|
|
7929
8881
|
return {
|
|
@@ -8039,9 +8991,8 @@ function aesEncryptCBCBase64(plaintext, keyString) {
|
|
|
8039
8991
|
* @param url - the absolute request URL.
|
|
8040
8992
|
* @param credential - `{ userID, token, name, email, machineID }`.
|
|
8041
8993
|
*/
|
|
8042
|
-
function buildCosyAuthHeaders(body, url, credential
|
|
8043
|
-
const
|
|
8044
|
-
const aesKey = options.aesKey ?? uuid().replace(/-/g, "").slice(0, 16);
|
|
8994
|
+
function buildCosyAuthHeaders(body, url, credential) {
|
|
8995
|
+
const aesKey = randomUUID().replace(/-/g, "").slice(0, 16);
|
|
8045
8996
|
const infoB64 = aesEncryptCBCBase64(JSON.stringify({
|
|
8046
8997
|
uid: credential.userID,
|
|
8047
8998
|
security_oauth_token: credential.token,
|
|
@@ -8053,10 +9004,11 @@ function buildCosyAuthHeaders(body, url, credential, options = {}) {
|
|
|
8053
9004
|
key: QODER_RSA_PUBLIC_KEY,
|
|
8054
9005
|
padding: constants.RSA_PKCS1_PADDING
|
|
8055
9006
|
}, Buffer.from(aesKey)).toString("base64");
|
|
8056
|
-
const timestamp = Math.floor(
|
|
9007
|
+
const timestamp = Math.floor(Date.now() / 1e3).toString();
|
|
9008
|
+
const requestId = randomUUID();
|
|
8057
9009
|
const payloadB64 = Buffer.from(JSON.stringify({
|
|
8058
9010
|
version: "v1",
|
|
8059
|
-
requestId
|
|
9011
|
+
requestId,
|
|
8060
9012
|
info: infoB64,
|
|
8061
9013
|
cosyVersion: COSY_VERSION,
|
|
8062
9014
|
ideVersion: ""
|
|
@@ -8084,7 +9036,7 @@ function buildCosyAuthHeaders(body, url, credential, options = {}) {
|
|
|
8084
9036
|
"Cosy-Organization-Id": "",
|
|
8085
9037
|
"Cosy-Organization-Tags": "",
|
|
8086
9038
|
"Login-Version": LOGIN_VERSION,
|
|
8087
|
-
"X-Request-Id":
|
|
9039
|
+
"X-Request-Id": randomUUID()
|
|
8088
9040
|
};
|
|
8089
9041
|
}
|
|
8090
9042
|
//#endregion
|
|
@@ -8445,6 +9397,8 @@ function textOf(content) {
|
|
|
8445
9397
|
function toQoderMessages(messages) {
|
|
8446
9398
|
if (!Array.isArray(messages)) return [];
|
|
8447
9399
|
const out = [];
|
|
9400
|
+
const callsByName = /* @__PURE__ */ new Map();
|
|
9401
|
+
const answered = /* @__PURE__ */ new Set();
|
|
8448
9402
|
for (const raw of messages) {
|
|
8449
9403
|
if (raw === null || typeof raw !== "object") continue;
|
|
8450
9404
|
if (raw["role"] === "system" || raw["role"] === "developer") {
|
|
@@ -8485,19 +9439,40 @@ function toQoderMessages(messages) {
|
|
|
8485
9439
|
role: "assistant",
|
|
8486
9440
|
content: textOf(raw["content"])
|
|
8487
9441
|
};
|
|
8488
|
-
if (toolCalls.length > 0)
|
|
9442
|
+
if (toolCalls.length > 0) {
|
|
9443
|
+
entry["tool_calls"] = toolCalls;
|
|
9444
|
+
for (const call of toolCalls) {
|
|
9445
|
+
const name = String(call["function"]["name"]);
|
|
9446
|
+
callsByName.set(name, [...callsByName.get(name) ?? [], String(call["id"])]);
|
|
9447
|
+
}
|
|
9448
|
+
}
|
|
8489
9449
|
out.push(entry);
|
|
8490
9450
|
continue;
|
|
8491
9451
|
}
|
|
9452
|
+
if (raw["role"] === "function") {
|
|
9453
|
+
const name = raw["name"];
|
|
9454
|
+
const callId = typeof name === "string" ? (callsByName.get(name) ?? []).find((id) => !answered.has(id)) : void 0;
|
|
9455
|
+
if (callId === void 0) throw new Error(`toQoderMessages: function message "${typeof name === "string" ? name : ""}" does not answer any assistant tool_call; the tool loop cannot match it`);
|
|
9456
|
+
answered.add(callId);
|
|
9457
|
+
out.push({
|
|
9458
|
+
role: "tool",
|
|
9459
|
+
tool_call_id: callId,
|
|
9460
|
+
content: textOf(raw["content"])
|
|
9461
|
+
});
|
|
9462
|
+
continue;
|
|
9463
|
+
}
|
|
8492
9464
|
if (raw["role"] === "tool" || raw["role"] === "toolResult") {
|
|
8493
9465
|
const toolCallId = raw["tool_call_id"] ?? raw["toolCallId"];
|
|
8494
9466
|
if (typeof toolCallId !== "string" || toolCallId.length === 0) throw new Error("toQoderMessages: tool result message is missing tool_call_id; it cannot be linked to the calling assistant turn");
|
|
9467
|
+
answered.add(toolCallId);
|
|
8495
9468
|
out.push({
|
|
8496
9469
|
role: "tool",
|
|
8497
9470
|
tool_call_id: toolCallId,
|
|
8498
9471
|
content: textOf(raw["content"])
|
|
8499
9472
|
});
|
|
9473
|
+
continue;
|
|
8500
9474
|
}
|
|
9475
|
+
throw new Error(`toQoderMessages: unsupported message role "${String(raw["role"])}"`);
|
|
8501
9476
|
}
|
|
8502
9477
|
return out;
|
|
8503
9478
|
}
|
|
@@ -8650,11 +9625,30 @@ function qoderContextWindowOf(record) {
|
|
|
8650
9625
|
const sizes = Object.values(config).map(tokenCountOf).filter((value) => value !== void 0);
|
|
8651
9626
|
return sizes.length === 0 ? void 0 : Math.max(...sizes);
|
|
8652
9627
|
}
|
|
9628
|
+
/**
|
|
9629
|
+
* The routes Qoder lists beside its models without naming them.
|
|
9630
|
+
*
|
|
9631
|
+
* The global directory returns four entries whose display names are Qoder's own
|
|
9632
|
+
* codenames — Ultimate, Efficient, Sonus, Cantus — and nothing in the payload
|
|
9633
|
+
* marks them as different from a named model: same `source`, same `format`, same
|
|
9634
|
+
* `is_editable`, and one of them even carries a promotion. Offered in the picker
|
|
9635
|
+
* they read as four opaque names between Auto and Qwen3.8-Max. The keys are what
|
|
9636
|
+
* Qoder's API has returned for them across versions, so they are matched by key
|
|
9637
|
+
* on whichever directory lists them. The `auto` route stays: it is a model choice
|
|
9638
|
+
* a user recognises, and it is the row the directory marks `is_default`.
|
|
9639
|
+
*/
|
|
9640
|
+
const QODER_UNNAMED_ROUTES = /* @__PURE__ */ new Set([
|
|
9641
|
+
"ultimate",
|
|
9642
|
+
"efficient",
|
|
9643
|
+
"smodel",
|
|
9644
|
+
"cmodel"
|
|
9645
|
+
]);
|
|
8653
9646
|
/** One directory row → one callable model, or undefined when unusable. */
|
|
8654
9647
|
function qoderModelOf(entry) {
|
|
8655
9648
|
if (entry === null || typeof entry !== "object") return void 0;
|
|
8656
9649
|
const record = entry;
|
|
8657
9650
|
if (typeof record["key"] !== "string" || record["key"].length === 0) return void 0;
|
|
9651
|
+
if (QODER_UNNAMED_ROUTES.has(record["key"])) return void 0;
|
|
8658
9652
|
if (record["enable"] === false) return void 0;
|
|
8659
9653
|
if (typeof record["display_name"] !== "string" || record["display_name"].length === 0) return void 0;
|
|
8660
9654
|
const config = record["thinking_config"];
|
|
@@ -8718,6 +9712,7 @@ var QoderUpstreamClient = class {
|
|
|
8718
9712
|
const model = qoderModelOf(raw);
|
|
8719
9713
|
if (model !== void 0) models.push(model);
|
|
8720
9714
|
}
|
|
9715
|
+
if (models.length === 0) throw new Error("Qoder model list contained no callable model");
|
|
8721
9716
|
return models;
|
|
8722
9717
|
}
|
|
8723
9718
|
openApiBase() {
|
|
@@ -8828,12 +9823,13 @@ var QoderUpstreamClient = class {
|
|
|
8828
9823
|
let plain;
|
|
8829
9824
|
try {
|
|
8830
9825
|
plain = buildQoderChatBody(openAiBody, model, options);
|
|
8831
|
-
} catch {
|
|
9826
|
+
} catch (error) {
|
|
9827
|
+
const detail = error instanceof Error ? error.message : "";
|
|
8832
9828
|
return {
|
|
8833
9829
|
ok: false,
|
|
8834
9830
|
status: 400,
|
|
8835
9831
|
kind: "client",
|
|
8836
|
-
message: "invalid JSON request"
|
|
9832
|
+
message: detail === "" ? "invalid JSON request" : detail
|
|
8837
9833
|
};
|
|
8838
9834
|
}
|
|
8839
9835
|
let credential;
|
|
@@ -9336,6 +10332,13 @@ function bridgeQoderStream(source, model, options = {}) {
|
|
|
9336
10332
|
let sawToolCalls = false;
|
|
9337
10333
|
let emittedFinishReason = false;
|
|
9338
10334
|
let usage;
|
|
10335
|
+
let lastKeepaliveAt = 0;
|
|
10336
|
+
const keepalive = (controller) => {
|
|
10337
|
+
const now = Date.now();
|
|
10338
|
+
if (now - lastKeepaliveAt < 1e4) return;
|
|
10339
|
+
lastKeepaliveAt = now;
|
|
10340
|
+
controller.enqueue(encoder.encode(": keepalive\n\n"));
|
|
10341
|
+
};
|
|
9339
10342
|
const reader = source.getReader();
|
|
9340
10343
|
let cancelled = false;
|
|
9341
10344
|
let wakeRoom;
|
|
@@ -9374,7 +10377,10 @@ function bridgeQoderStream(source, model, options = {}) {
|
|
|
9374
10377
|
const consume = (payload, controller) => {
|
|
9375
10378
|
if (finished) return;
|
|
9376
10379
|
const decoded = decodeQoderFrame(payload);
|
|
9377
|
-
if (decoded.type === "unknown")
|
|
10380
|
+
if (decoded.type === "unknown") {
|
|
10381
|
+
keepalive(controller);
|
|
10382
|
+
return;
|
|
10383
|
+
}
|
|
9378
10384
|
if (decoded.type === "failure") {
|
|
9379
10385
|
const { error } = classifyQoderFailure({
|
|
9380
10386
|
code: decoded.code,
|
|
@@ -9770,7 +10776,7 @@ function createQoderGateway(options) {
|
|
|
9770
10776
|
let relayed;
|
|
9771
10777
|
try {
|
|
9772
10778
|
relayed = await openQoderResponse(result.response, model, { onFailure: (error) => {
|
|
9773
|
-
logger?.warn?.(
|
|
10779
|
+
logger?.warn?.("wtq-pool: qoder relay stream failed mid-flight", error);
|
|
9774
10780
|
if (!(error instanceof Error && error.name === "AbortError") && !watcher.signal.aborted) options.onAttemptFailure?.(credential, {
|
|
9775
10781
|
status: 0,
|
|
9776
10782
|
kind: "server"
|
|
@@ -9860,13 +10866,13 @@ function createQoderGateway(options) {
|
|
|
9860
10866
|
});
|
|
9861
10867
|
res.flushHeaders();
|
|
9862
10868
|
pumpToResponse(relayStreamOf(relayed), res).catch((error) => {
|
|
9863
|
-
if (!(watcher.signal.aborted || error instanceof Error && error.name === "AbortError")) logger?.warn?.(
|
|
10869
|
+
if (!(watcher.signal.aborted || error instanceof Error && error.name === "AbortError")) logger?.warn?.("wtq-pool: qoder relay stream failed mid-flight", error);
|
|
9864
10870
|
if (res.writable) res.end();
|
|
9865
10871
|
});
|
|
9866
10872
|
}
|
|
9867
10873
|
async function handle(req, res) {
|
|
9868
10874
|
if (!guardsPass(req, res)) return;
|
|
9869
|
-
const url = req.url
|
|
10875
|
+
const url = requestPathname(req.url);
|
|
9870
10876
|
if (req.method === "GET" && (url === "/healthz" || url === "/healthz/")) {
|
|
9871
10877
|
writeJson(res, 200, {
|
|
9872
10878
|
ok: true,
|
|
@@ -9886,7 +10892,7 @@ function createQoderGateway(options) {
|
|
|
9886
10892
|
writeOpenAIError(res, 503, "upstream_unavailable", String(error instanceof Error ? error.message : error));
|
|
9887
10893
|
return;
|
|
9888
10894
|
}
|
|
9889
|
-
logger?.warn?.(
|
|
10895
|
+
logger?.warn?.("wtq-pool: qoder directory refresh failed; serving the cached roster");
|
|
9890
10896
|
}
|
|
9891
10897
|
const ids = options.listedIds?.();
|
|
9892
10898
|
writeJson(res, 200, {
|
|
@@ -9966,7 +10972,8 @@ function loginHeaders(origin) {
|
|
|
9966
10972
|
"Origin": origin,
|
|
9967
10973
|
"Referer": `${origin}/`,
|
|
9968
10974
|
"User-Agent": LOGIN_UA,
|
|
9969
|
-
"X-Requested-With": "XMLHttpRequest"
|
|
10975
|
+
"X-Requested-With": "XMLHttpRequest",
|
|
10976
|
+
"Connection": "close"
|
|
9970
10977
|
};
|
|
9971
10978
|
}
|
|
9972
10979
|
async function callLogin(url, headers, fetchImpl, body) {
|
|
@@ -10222,7 +11229,6 @@ function createSourcePool(host) {
|
|
|
10222
11229
|
* upstream SSE body relayed as it arrives. Nothing leaves this machine except
|
|
10223
11230
|
* to the three vendors' own endpoints.
|
|
10224
11231
|
*/
|
|
10225
|
-
/** Read from the package, so this can never drift behind the version it reports. */
|
|
10226
11232
|
/** The one background service this plugin registers. */
|
|
10227
11233
|
const SERVICE_ID = "provider-host";
|
|
10228
11234
|
/** The two gateways, matching the provider ids this plugin registers. */
|
|
@@ -10506,6 +11512,17 @@ const pendingRuntimeStops = [];
|
|
|
10506
11512
|
* drift while every request to that address was refused.
|
|
10507
11513
|
*/
|
|
10508
11514
|
const gatewayStartFailures = /* @__PURE__ */ new Map();
|
|
11515
|
+
/**
|
|
11516
|
+
* Trae/Qoder regions whose gateway could not start, keyed `source:region`.
|
|
11517
|
+
*
|
|
11518
|
+
* WorkBuddy records its own start failures; without the same record for the two
|
|
11519
|
+
* source gateways a region whose bind failed kept the address the host had
|
|
11520
|
+
* already registered, so every request to it was refused while the doctor and
|
|
11521
|
+
* the panel read the region as healthy. Entries are cleared wherever the
|
|
11522
|
+
* gateway is installed or removed, so a switched-off source never reads as a
|
|
11523
|
+
* failed one.
|
|
11524
|
+
*/
|
|
11525
|
+
const sourceStartFailures = /* @__PURE__ */ new Map();
|
|
10509
11526
|
/** Provider ids the host has confirmed it is serving, this process's lifetime. */
|
|
10510
11527
|
let publishedIds = /* @__PURE__ */ new Set();
|
|
10511
11528
|
/**
|
|
@@ -10677,7 +11694,7 @@ const STREAK_REPORT_READ_GAP_MS = 1200;
|
|
|
10677
11694
|
const travelAdoptTried = /* @__PURE__ */ new Map();
|
|
10678
11695
|
function log(level, message) {
|
|
10679
11696
|
try {
|
|
10680
|
-
console.log("[
|
|
11697
|
+
console.log("[wtq-pool] " + level + ": " + message);
|
|
10681
11698
|
} catch {}
|
|
10682
11699
|
}
|
|
10683
11700
|
/**
|
|
@@ -10854,6 +11871,7 @@ async function onHostSettingsChanged(next) {
|
|
|
10854
11871
|
}
|
|
10855
11872
|
/** Subscribe to the host's settings event, tolerating a host without it. */
|
|
10856
11873
|
function subscribeToHostSettings() {
|
|
11874
|
+
unsubscribeFromHostSettings();
|
|
10857
11875
|
settingsChangedHandler = (next) => {
|
|
10858
11876
|
onHostSettingsChanged(next).then((applied) => {
|
|
10859
11877
|
if (applied) log("info", "settings changed in the host UI; selection re-applied");
|
|
@@ -10870,6 +11888,17 @@ function subscribeToHostSettings() {
|
|
|
10870
11888
|
settingsChangedHandler = null;
|
|
10871
11889
|
return false;
|
|
10872
11890
|
}
|
|
11891
|
+
/** Detach the host settings handler, if one is held. */
|
|
11892
|
+
function unsubscribeFromHostSettings() {
|
|
11893
|
+
const handler = settingsChangedHandler;
|
|
11894
|
+
settingsChangedHandler = null;
|
|
11895
|
+
if (handler === null) return;
|
|
11896
|
+
try {
|
|
11897
|
+
if (pi.events && typeof pi.events.off === "function") pi.events.off("plugin:settingsChanged", handler);
|
|
11898
|
+
} catch (error) {
|
|
11899
|
+
log("warn", "could not unsubscribe from settings changes: " + safeMessage(error));
|
|
11900
|
+
}
|
|
11901
|
+
}
|
|
10873
11902
|
function cooldownMs() {
|
|
10874
11903
|
const value = Number(settings.cooldownSeconds);
|
|
10875
11904
|
if (!Number.isFinite(value) || value <= 0) return 6e4;
|
|
@@ -11706,6 +12735,7 @@ async function reconcileTraeGateways() {
|
|
|
11706
12735
|
for (const [region, entry] of Array.from(traeGateways)) {
|
|
11707
12736
|
if (active.includes(region)) continue;
|
|
11708
12737
|
traeGateways.delete(region);
|
|
12738
|
+
sourceStartFailures.delete("trae:" + region);
|
|
11709
12739
|
try {
|
|
11710
12740
|
await entry.gateway.close();
|
|
11711
12741
|
} catch (error) {
|
|
@@ -11717,8 +12747,12 @@ async function reconcileTraeGateways() {
|
|
|
11717
12747
|
try {
|
|
11718
12748
|
const entry = await createTraeRuntime(region);
|
|
11719
12749
|
if (!stillLoaded(generation) || !activeTraeRegions().includes(region) || traeGateways.has(region)) await entry.gateway.close();
|
|
11720
|
-
else
|
|
12750
|
+
else {
|
|
12751
|
+
traeGateways.set(region, entry);
|
|
12752
|
+
sourceStartFailures.delete("trae:" + region);
|
|
12753
|
+
}
|
|
11721
12754
|
} catch (error) {
|
|
12755
|
+
sourceStartFailures.set("trae:" + region, safeMessage(error));
|
|
11722
12756
|
log("warn", "[trae:" + region + "] gateway could not start: " + safeMessage(error));
|
|
11723
12757
|
}
|
|
11724
12758
|
}
|
|
@@ -11842,6 +12876,7 @@ async function reconcileQoderGateways() {
|
|
|
11842
12876
|
for (const [region, entry] of Array.from(qoderGateways)) {
|
|
11843
12877
|
if (active.includes(region)) continue;
|
|
11844
12878
|
qoderGateways.delete(region);
|
|
12879
|
+
sourceStartFailures.delete("qoder:" + region);
|
|
11845
12880
|
try {
|
|
11846
12881
|
await entry.gateway.close();
|
|
11847
12882
|
} catch (error) {
|
|
@@ -11853,8 +12888,12 @@ async function reconcileQoderGateways() {
|
|
|
11853
12888
|
try {
|
|
11854
12889
|
const entry = await createQoderRuntime(region);
|
|
11855
12890
|
if (!stillLoaded(generation) || !activeQoderRegions().includes(region) || qoderGateways.has(region)) await entry.gateway.close();
|
|
11856
|
-
else
|
|
12891
|
+
else {
|
|
12892
|
+
qoderGateways.set(region, entry);
|
|
12893
|
+
sourceStartFailures.delete("qoder:" + region);
|
|
12894
|
+
}
|
|
11857
12895
|
} catch (error) {
|
|
12896
|
+
sourceStartFailures.set("qoder:" + region, safeMessage(error));
|
|
11858
12897
|
log("warn", "[qoder:" + region + "] gateway could not start: " + safeMessage(error));
|
|
11859
12898
|
}
|
|
11860
12899
|
}
|
|
@@ -12086,7 +13125,8 @@ async function connect(runtime, force) {
|
|
|
12086
13125
|
if (runtime.retired) return false;
|
|
12087
13126
|
runtime.lastError = "";
|
|
12088
13127
|
runtime.lastWarning = "";
|
|
12089
|
-
const
|
|
13128
|
+
const regionAccounts = core.pool.list(runtime.region);
|
|
13129
|
+
const account = regionAccounts.find((item) => core.pool.activeHold(item.id)?.reason !== "session") ?? regionAccounts[0];
|
|
12090
13130
|
if (account === void 0) {
|
|
12091
13131
|
runtime.signedIn = false;
|
|
12092
13132
|
runtime.lastError = "no " + runtime.region + " WorkBuddy sign-in found";
|
|
@@ -12100,7 +13140,7 @@ async function connect(runtime, force) {
|
|
|
12100
13140
|
}
|
|
12101
13141
|
runtime.signedIn = true;
|
|
12102
13142
|
if (force === true || !catalogFresh(runtime)) try {
|
|
12103
|
-
const models = await core.
|
|
13143
|
+
const models = await core.pool.readAfterRefresh(account.id, (credential) => core.client.fetchModels(credential, AbortSignal.timeout(MODELS_TIMEOUT_MS)));
|
|
12104
13144
|
if (runtime.retired) return false;
|
|
12105
13145
|
runtime.catalog.update(catalogFromUpstream(models));
|
|
12106
13146
|
runtime.catalogFetchedAtMs = Date.now();
|
|
@@ -12538,12 +13578,13 @@ function pruneSourceState() {
|
|
|
12538
13578
|
/**
|
|
12539
13579
|
* Directories the WorkBuddy pool reads, in order.
|
|
12540
13580
|
*
|
|
12541
|
-
* The desktop app's own directories come first,
|
|
12542
|
-
*
|
|
13581
|
+
* The desktop app's own directories come first, and that is only the walk
|
|
13582
|
+
* order: which copy of an account serves is decided by issue time, with the
|
|
13583
|
+
* source breaking a tie. The plugin's own directory is appended rather than
|
|
12543
13584
|
* substituted: an account added from the panel is read on exactly the same
|
|
12544
13585
|
* terms as a desktop sign-in, and removing it is a file deletion the user can
|
|
12545
|
-
* see. A repointed `authFile` narrows the walk to that one directory
|
|
12546
|
-
*
|
|
13586
|
+
* see. A repointed `authFile` narrows the walk to that one directory plus the
|
|
13587
|
+
* plugin's own, the way the pool reads whatever it writes.
|
|
12547
13588
|
*/
|
|
12548
13589
|
function poolAuthDirs(authFile) {
|
|
12549
13590
|
if (authFile !== "") return [path.dirname(authFile)];
|
|
@@ -12636,10 +13677,11 @@ async function accountProbe(account, allowNetwork, checkinOnly = false) {
|
|
|
12636
13677
|
const cached = probeCache.get(account.id);
|
|
12637
13678
|
if (allowNetwork !== true) return cached === void 0 ? { pending: true } : cached.value;
|
|
12638
13679
|
if (cached !== void 0 && Date.now() - cached.atMs < PROBE_TTL_MS) return cached.value;
|
|
13680
|
+
if (signInRefused(account)) return {};
|
|
12639
13681
|
const value = {};
|
|
12640
13682
|
if (!checkinOnly) {
|
|
12641
13683
|
try {
|
|
12642
|
-
const credits = await core.
|
|
13684
|
+
const credits = await core.pool.readAfterRefresh(account.id, (credential) => core.client.fetchCredits(credential));
|
|
12643
13685
|
value.credits = {
|
|
12644
13686
|
...credits.total === void 0 ? {} : { total: credits.total },
|
|
12645
13687
|
packages: (credits.packages || []).map((pack) => ({
|
|
@@ -12659,7 +13701,7 @@ async function accountProbe(account, allowNetwork, checkinOnly = false) {
|
|
|
12659
13701
|
if (value.credits !== void 0 && typeof value.credits.total === "number") core.pool.setCreditBalance(account.id, value.credits.total, value.credits.nearestExpiryMs);
|
|
12660
13702
|
}
|
|
12661
13703
|
try {
|
|
12662
|
-
const checkin = await core.
|
|
13704
|
+
const checkin = await core.pool.readAfterRefresh(account.id, (credential) => core.client.fetchCheckinStatus(credential));
|
|
12663
13705
|
value.checkin = {
|
|
12664
13706
|
active: checkin.active,
|
|
12665
13707
|
todayCheckedIn: checkin.todayCheckedIn,
|
|
@@ -12698,6 +13740,28 @@ function catalogWarnings() {
|
|
|
12698
13740
|
}
|
|
12699
13741
|
return Object.keys(warnings).length === 0 ? {} : { warnings };
|
|
12700
13742
|
}
|
|
13743
|
+
/**
|
|
13744
|
+
* Whether this account's sign-in is one the upstream has refused.
|
|
13745
|
+
*
|
|
13746
|
+
* The hold is the pool's own record that a refresh could not save the sign-in,
|
|
13747
|
+
* so every pass that walks the pool asks before it spends a call: a held
|
|
13748
|
+
* account cannot answer, and the panel row already says why.
|
|
13749
|
+
*/
|
|
13750
|
+
function signInRefused(account) {
|
|
13751
|
+
return core.pool.activeHold(account.id)?.reason === "session";
|
|
13752
|
+
}
|
|
13753
|
+
/**
|
|
13754
|
+
* The row fields that describe a hold the pool itself decided, if one is in
|
|
13755
|
+
* force for this account right now.
|
|
13756
|
+
*
|
|
13757
|
+
* Empty spread when the account is simply cooling on an upstream window: that
|
|
13758
|
+
* state already reads as one on the row, and its remedy is to wait.
|
|
13759
|
+
*/
|
|
13760
|
+
function activeHoldOf(accountId, now) {
|
|
13761
|
+
const hold = core.pool.activeHold(accountId);
|
|
13762
|
+
if (hold === null || hold.untilMs <= now) return {};
|
|
13763
|
+
return { hold: hold.reason };
|
|
13764
|
+
}
|
|
12701
13765
|
async function statusDocument(options) {
|
|
12702
13766
|
const allowNetwork = options?.allowNetwork === true;
|
|
12703
13767
|
const probeBudget = typeof options?.budgetMs === "number" ? Math.min(PROBE_BATCH_BUDGET_MS, Math.max(0, options.budgetMs)) : PROBE_BATCH_BUDGET_MS;
|
|
@@ -12736,6 +13800,7 @@ async function statusDocument(options) {
|
|
|
12736
13800
|
...cooling ? { cooldownUntil: new Date(account.cooldownUntilMs).toISOString() } : {},
|
|
12737
13801
|
...modelCooldowns.length === 0 ? {} : { modelCooldowns },
|
|
12738
13802
|
rateLimitHits: account.rateLimitHits,
|
|
13803
|
+
...activeHoldOf(account.id, now),
|
|
12739
13804
|
active: runtimeNextUpId.get(region) === account.id,
|
|
12740
13805
|
preferred: preferredAccountIds[region] === account.id,
|
|
12741
13806
|
disabled: core.pool.isDisabled(account.id),
|
|
@@ -12950,6 +14015,14 @@ function addressDrift() {
|
|
|
12950
14015
|
const published = declaredSourceBaseUrl(entry, QODER_PROVIDER_ID_BY_REGION);
|
|
12951
14016
|
if (published !== void 0 && !samePublishedAddress(published, entry.baseUrl)) return true;
|
|
12952
14017
|
}
|
|
14018
|
+
for (const region of activeTraeRegions()) {
|
|
14019
|
+
if (!sourceStartFailures.has("trae:" + region)) continue;
|
|
14020
|
+
if (declaredSourceBaseUrl({ region }, TRAE_PROVIDER_ID_BY_REGION) !== void 0) return true;
|
|
14021
|
+
}
|
|
14022
|
+
for (const region of activeQoderRegions()) {
|
|
14023
|
+
if (!sourceStartFailures.has("qoder:" + region)) continue;
|
|
14024
|
+
if (declaredSourceBaseUrl({ region }, QODER_PROVIDER_ID_BY_REGION) !== void 0) return true;
|
|
14025
|
+
}
|
|
12953
14026
|
return false;
|
|
12954
14027
|
}
|
|
12955
14028
|
/**
|
|
@@ -13093,6 +14166,19 @@ async function doctorReport() {
|
|
|
13093
14166
|
const detail = fallback ? "port " + ports[entry.region] + " is held by another process, so this session serves " + bound + " - free that port (usually a leftover plugin process), then reload the plugin" : "the published address is " + published + " but this gateway is on " + bound + " - reload the plugin so the host picks the new address up";
|
|
13094
14167
|
push(label + ": published address", false, detail);
|
|
13095
14168
|
}
|
|
14169
|
+
for (const [source, nameByRegion, regions] of [[
|
|
14170
|
+
"trae",
|
|
14171
|
+
TRAE_PROVIDER_NAME_BY_REGION,
|
|
14172
|
+
activeTraeRegions()
|
|
14173
|
+
], [
|
|
14174
|
+
"qoder",
|
|
14175
|
+
QODER_PROVIDER_NAME_BY_REGION,
|
|
14176
|
+
activeQoderRegions()
|
|
14177
|
+
]]) for (const region of regions) {
|
|
14178
|
+
const reason = sourceStartFailures.get(source + ":" + region);
|
|
14179
|
+
if (reason === void 0) continue;
|
|
14180
|
+
push(nameByRegion[region] + ": endpoint", false, reason);
|
|
14181
|
+
}
|
|
13096
14182
|
push("declaration-written", runtimeState.write.ok === true, runtimeState.write.ok !== true ? safeMessage(runtimeState.write.error) : runtimeState.write.changed ? "the model list was just updated; the host discovers it from this gateway's /v1/models" : String(runtimeState.write.skipped || "up to date"));
|
|
13097
14183
|
const ok = findings.every((finding) => finding.ok);
|
|
13098
14184
|
const report = [
|
|
@@ -13258,10 +14344,18 @@ async function onPanelInvoke(channel, payload) {
|
|
|
13258
14344
|
* (the Refresh button) still resolves on a fully refreshed document.
|
|
13259
14345
|
*/
|
|
13260
14346
|
case "xd.state": {
|
|
13261
|
-
if (payload?.wait === true)
|
|
13262
|
-
|
|
13263
|
-
|
|
13264
|
-
|
|
14347
|
+
if (payload?.wait === true) {
|
|
14348
|
+
const refreshed = await refreshAll("panel", true);
|
|
14349
|
+
if (refreshed.stale === true) return {
|
|
14350
|
+
...await statusDocument({ allowNetwork: false }),
|
|
14351
|
+
refreshing: false,
|
|
14352
|
+
stale: true
|
|
14353
|
+
};
|
|
14354
|
+
return {
|
|
14355
|
+
...refreshed,
|
|
14356
|
+
refreshing: false
|
|
14357
|
+
};
|
|
14358
|
+
}
|
|
13265
14359
|
const document = await statusDocument({ allowNetwork: false });
|
|
13266
14360
|
const refreshing = refreshNeeded();
|
|
13267
14361
|
if (refreshing) scheduleRefresh("panel");
|
|
@@ -13285,7 +14379,7 @@ async function onPanelInvoke(channel, payload) {
|
|
|
13285
14379
|
case "xd.fetchModels":
|
|
13286
14380
|
await settleWithin(Promise.all(Array.from(runtimes.values()).map(async (runtime) => {
|
|
13287
14381
|
try {
|
|
13288
|
-
await
|
|
14382
|
+
await refreshRuntime(runtime, "fetch-models", true);
|
|
13289
14383
|
} catch (error) {
|
|
13290
14384
|
runtime.lastError = safeMessage(error);
|
|
13291
14385
|
}
|
|
@@ -13328,7 +14422,7 @@ async function onPanelInvoke(channel, payload) {
|
|
|
13328
14422
|
const token = ++checkinPassToken;
|
|
13329
14423
|
checkinPassRunning = true;
|
|
13330
14424
|
const work = async () => {
|
|
13331
|
-
const before = await core.
|
|
14425
|
+
const before = await core.pool.readAfterRefresh(account.id, (credential) => core.client.fetchCheckinStatus(credential));
|
|
13332
14426
|
if (before.todayCheckedIn) {
|
|
13333
14427
|
probeCache.delete(accountId);
|
|
13334
14428
|
return {
|
|
@@ -13818,7 +14912,15 @@ async function onPanelInvoke(channel, payload) {
|
|
|
13818
14912
|
const generation = loadGeneration;
|
|
13819
14913
|
const counts = await settleWithin((async () => {
|
|
13820
14914
|
try {
|
|
13821
|
-
|
|
14915
|
+
const counts = await claimAllGrowth();
|
|
14916
|
+
if (stillLoaded(generation)) {
|
|
14917
|
+
growthLast = {
|
|
14918
|
+
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
14919
|
+
...counts
|
|
14920
|
+
};
|
|
14921
|
+
scheduleRefresh("growth");
|
|
14922
|
+
}
|
|
14923
|
+
return counts;
|
|
13822
14924
|
} finally {
|
|
13823
14925
|
if (token === growthPassToken) growthPassRunning = false;
|
|
13824
14926
|
}
|
|
@@ -13827,13 +14929,6 @@ async function onPanelInvoke(channel, payload) {
|
|
|
13827
14929
|
state: await statusDocument({ allowNetwork: false }),
|
|
13828
14930
|
running: true
|
|
13829
14931
|
};
|
|
13830
|
-
if (stillLoaded(generation)) {
|
|
13831
|
-
growthLast = {
|
|
13832
|
-
at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
13833
|
-
...counts
|
|
13834
|
-
};
|
|
13835
|
-
scheduleRefresh("growth");
|
|
13836
|
-
}
|
|
13837
14932
|
return {
|
|
13838
14933
|
state: await statusDocument({ allowNetwork: false }),
|
|
13839
14934
|
growth: counts
|
|
@@ -14155,6 +15250,7 @@ async function claimAllGrowth() {
|
|
|
14155
15250
|
for (const account of core.pool.list()) {
|
|
14156
15251
|
if (currentCheckinSignal().aborted) break;
|
|
14157
15252
|
if (core.pool.isDisabled(account.id) || !regionEnabled("workbuddy", regionOf(account.credential.domain))) continue;
|
|
15253
|
+
if (signInRefused(account)) continue;
|
|
14158
15254
|
if (seen > 0) await checkinDelay(GROWTH_ACCOUNT_DELAY_MS);
|
|
14159
15255
|
seen += 1;
|
|
14160
15256
|
counts.detail.accounts += 1;
|
|
@@ -14211,7 +15307,7 @@ function mergeGrowthDetail(total, part) {
|
|
|
14211
15307
|
async function readGrowthState() {
|
|
14212
15308
|
const rows = [];
|
|
14213
15309
|
let failed = 0;
|
|
14214
|
-
const wanted = core.pool.list().filter((account) => !core.pool.isDisabled(account.id) && regionEnabled("workbuddy", regionOf(account.credential.domain)));
|
|
15310
|
+
const wanted = core.pool.list().filter((account) => !core.pool.isDisabled(account.id) && regionEnabled("workbuddy", regionOf(account.credential.domain)) && !signInRefused(account));
|
|
14215
15311
|
const reads = await Promise.all(wanted.map(async (account) => {
|
|
14216
15312
|
try {
|
|
14217
15313
|
const [streak, chances] = await Promise.all([core.client.growthStreakFull(account.credential), core.client.lotteryChances(account.credential)]);
|
|
@@ -14281,12 +15377,16 @@ async function claimAllCheckins() {
|
|
|
14281
15377
|
let seen = 0;
|
|
14282
15378
|
for (const account of core.pool.list()) {
|
|
14283
15379
|
if (core.pool.isDisabled(account.id) || !regionEnabled("workbuddy", regionOf(account.credential.domain))) continue;
|
|
15380
|
+
if (signInRefused(account)) continue;
|
|
14284
15381
|
if (currentCheckinSignal().aborted) break;
|
|
14285
15382
|
if (seen > 0) await checkinDelay(CHECKIN_ACCOUNT_DELAY_MS);
|
|
14286
15383
|
seen += 1;
|
|
14287
15384
|
try {
|
|
14288
|
-
const before = await core.
|
|
14289
|
-
if (!before.active)
|
|
15385
|
+
const before = await core.pool.readAfterRefresh(account.id, (credential) => core.client.fetchCheckinStatus(credential));
|
|
15386
|
+
if (!before.active) {
|
|
15387
|
+
counts.skipped += 1;
|
|
15388
|
+
continue;
|
|
15389
|
+
}
|
|
14290
15390
|
if (before.todayCheckedIn) {
|
|
14291
15391
|
counts.skipped += 1;
|
|
14292
15392
|
continue;
|
|
@@ -14424,6 +15524,7 @@ async function travelAllAccounts() {
|
|
|
14424
15524
|
if (core.pool.isDisabled(account.id) || !regionEnabled("workbuddy", regionOf(account.credential.domain))) continue;
|
|
14425
15525
|
if (regionOf(account.credential.domain) !== "cn") continue;
|
|
14426
15526
|
if (!account.credential.refreshToken) continue;
|
|
15527
|
+
if (signInRefused(account)) continue;
|
|
14427
15528
|
if (currentCheckinSignal().aborted) break;
|
|
14428
15529
|
if (seen > 0) await checkinDelay(TRAVEL_ACCOUNT_DELAY_MS);
|
|
14429
15530
|
seen += 1;
|
|
@@ -14780,8 +15881,10 @@ async function keepAliveTokens() {
|
|
|
14780
15881
|
if (serviceStopped) break;
|
|
14781
15882
|
if (core.pool.isDisabled(account.id)) continue;
|
|
14782
15883
|
if (!regionEnabled("workbuddy", regionOf(account.credential.domain))) continue;
|
|
15884
|
+
if (signInRefused(account)) continue;
|
|
14783
15885
|
const outcome = await core.pool.ensureFresh(account);
|
|
14784
|
-
|
|
15886
|
+
const bucket = outcome === "refused" ? "failed" : outcome;
|
|
15887
|
+
counts[bucket] = (counts[bucket] ?? 0) + 1;
|
|
14785
15888
|
}
|
|
14786
15889
|
return counts;
|
|
14787
15890
|
}
|
|
@@ -14912,6 +16015,8 @@ async function onServiceStop() {
|
|
|
14912
16015
|
}));
|
|
14913
16016
|
traeGateways.clear();
|
|
14914
16017
|
qoderGateways.clear();
|
|
16018
|
+
gatewayStartFailures.clear();
|
|
16019
|
+
sourceStartFailures.clear();
|
|
14915
16020
|
await Promise.all(closing);
|
|
14916
16021
|
}
|
|
14917
16022
|
async function onLoad() {
|
|
@@ -14994,12 +16099,9 @@ async function onUnload() {
|
|
|
14994
16099
|
travelAdoptTried.clear();
|
|
14995
16100
|
traeIdentityBlockers.clear();
|
|
14996
16101
|
gatewayStartFailures.clear();
|
|
14997
|
-
|
|
14998
|
-
|
|
14999
|
-
|
|
15000
|
-
log("warn", "could not unsubscribe from settings changes: " + safeMessage(error));
|
|
15001
|
-
}
|
|
15002
|
-
settingsChangedHandler = null;
|
|
16102
|
+
sourceStartFailures.clear();
|
|
16103
|
+
deviceLogins.clear();
|
|
16104
|
+
unsubscribeFromHostSettings();
|
|
15003
16105
|
}
|
|
15004
16106
|
function composerPointsEnabled() {
|
|
15005
16107
|
return settings.showComposerPoints === true;
|