codex-multi-auth 2.21.0 → 2.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +8 -5
- package/config/schema/config.schema.json +8 -0
- package/dist/lib/accounts.d.ts +5 -0
- package/dist/lib/accounts.js +33 -4
- package/dist/lib/api-route-store.d.ts +7 -0
- package/dist/lib/api-route-store.js +100 -31
- package/dist/lib/auth/account-access.js +2 -1
- package/dist/lib/auth/auth.js +3 -2
- package/dist/lib/auth/device-auth.js +2 -1
- package/dist/lib/auth/http.d.ts +1 -0
- package/dist/lib/auth/http.js +1 -0
- package/dist/lib/codex-manager/account-manager-commands.js +1 -0
- package/dist/lib/codex-manager/api-login-menu.d.ts +1 -1
- package/dist/lib/codex-manager/api-login-menu.js +4 -6
- package/dist/lib/codex-manager/commands/locale.d.ts +12 -0
- package/dist/lib/codex-manager/commands/locale.js +25 -0
- package/dist/lib/codex-manager/commands/status.d.ts +5 -1
- package/dist/lib/codex-manager/commands/status.js +501 -13
- package/dist/lib/codex-manager/help.js +3 -1
- package/dist/lib/codex-manager.js +98 -23
- package/dist/lib/config.js +3 -0
- package/dist/lib/constants.d.ts +1 -0
- package/dist/lib/constants.js +1 -0
- package/dist/lib/locale.d.ts +7 -0
- package/dist/lib/locale.js +35 -0
- package/dist/lib/model-route-policy.d.ts +1 -0
- package/dist/lib/model-route-policy.js +2 -1
- package/dist/lib/quota-probe.js +2 -1
- package/dist/lib/request/proxy-fetch.d.ts +5 -0
- package/dist/lib/request/proxy-fetch.js +37 -0
- package/dist/lib/request/url-rewriting.js +17 -0
- package/dist/lib/runtime/account-model-catalog.d.ts +3 -0
- package/dist/lib/runtime/account-model-catalog.js +7 -2
- package/dist/lib/runtime/api-model-capabilities.d.ts +11 -2
- package/dist/lib/runtime/api-model-capabilities.js +155 -24
- package/dist/lib/runtime/app-bind.d.ts +7 -0
- package/dist/lib/runtime/app-bind.js +92 -1
- package/dist/lib/runtime/automatic-api-checks.d.ts +15 -0
- package/dist/lib/runtime/automatic-api-checks.js +188 -0
- package/dist/lib/runtime/model-discovery-status.d.ts +11 -1
- package/dist/lib/runtime/model-discovery-status.js +64 -49
- package/dist/lib/runtime/persisted-api-capabilities.d.ts +20 -0
- package/dist/lib/runtime/persisted-api-capabilities.js +387 -0
- package/dist/lib/runtime/responses-websocket.d.ts +48 -5
- package/dist/lib/runtime/responses-websocket.js +275 -39
- package/dist/lib/runtime/rotation-account-selection.d.ts +25 -5
- package/dist/lib/runtime/rotation-account-selection.js +94 -4
- package/dist/lib/runtime/rotation-proxy-state.d.ts +26 -0
- package/dist/lib/runtime/rotation-proxy-state.js +376 -4
- package/dist/lib/runtime/rotation-server-types.d.ts +6 -0
- package/dist/lib/runtime/rotation-token-refresh.d.ts +23 -1
- package/dist/lib/runtime/rotation-token-refresh.js +218 -27
- package/dist/lib/runtime-rotation-proxy.js +601 -105
- package/dist/lib/schemas.d.ts +5 -0
- package/dist/lib/schemas.js +1 -0
- package/dist/lib/storage/credential-sidecars.js +34 -2
- package/dist/lib/storage/file-lock.d.ts +2 -0
- package/dist/lib/storage/file-lock.js +44 -30
- package/dist/lib/usage/ledger.js +1 -0
- package/dist/lib/usage/redaction.js +1 -0
- package/dist/lib/usage/types.d.ts +1 -1
- package/dist/lib/web-dashboard-html.d.ts +1 -0
- package/dist/lib/web-dashboard-html.js +66 -0
- package/dist/lib/web-dashboard.d.ts +36 -0
- package/dist/lib/web-dashboard.js +322 -0
- package/package.json +3 -1
- package/scripts/codex-app-router.js +86 -14
- package/scripts/codex-routing.js +1 -0
- package/scripts/codex.js +150 -13
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
import { createAutomaticApiCheck, startAutomaticApiChecks } from "./runtime/automatic-api-checks.js";
|
|
1
2
|
import { startAutomaticAccountChecks } from "./runtime/automatic-account-checks.js";
|
|
2
3
|
import { createAutomaticSubscriptionCheck } from "./runtime/automatic-subscription-checks.js";
|
|
3
4
|
import { ClientCancellationError } from "./request/client-cancellation.js";
|
|
5
|
+
import { fetchWithProxy } from "./request/proxy-fetch.js";
|
|
4
6
|
import { createNativeAccountStorageReader } from "./runtime/native-account-storage.js";
|
|
5
7
|
import { resetSnapshotQuota } from "./runtime/reset-credits.js";
|
|
6
8
|
import { createResetCreditService, loadResetCreditState } from "./runtime/account-reset-credits.js";
|
|
@@ -14,6 +16,7 @@ import { modelScopeId, workspaceModelScopes } from "./runtime/workspace-model-sc
|
|
|
14
16
|
import { RuntimeCapabilityFailures, classifyCapabilityFailure } from "./runtime/runtime-capability-failures.js";
|
|
15
17
|
import { ResponseOutcome } from "./request/response-outcome.js";
|
|
16
18
|
import { ResponsesWebSocketGateway } from "./runtime/responses-websocket.js";
|
|
19
|
+
import { enrichPersistedApiCapabilities } from "./runtime/persisted-api-capabilities.js";
|
|
17
20
|
import { ApiModelCapabilities } from "./runtime/api-model-capabilities.js";
|
|
18
21
|
import { saveModelInventory } from "./runtime/model-discovery-status.js";
|
|
19
22
|
import { loadApiRoutes, updateApiModelDiscovery } from "./api-route-store.js";
|
|
@@ -46,10 +49,10 @@ import { normalizeEmailKey } from "./storage/identity.js";
|
|
|
46
49
|
import { buildPinnedUnavailableErrorBody, buildTokenInvalidationBody, extractErrorCodeFromBody, isTokenInvalidationError, normalizeExhaustionStatus, parseRetryAfterBodyMs, parseRetryAfterHeaderMs, } from "./request/rate-limit-decision.js";
|
|
47
50
|
import { forwardStreamingResponse, HOP_BY_HOP_HEADERS, readErrorBody, responseHeadersForClient, withTimeout, } from "./request/stream-failover-runtime.js";
|
|
48
51
|
import { getAccountRecoveryBoundsForFamily } from "./runtime/account-status.js";
|
|
49
|
-
import { chooseAccount } from "./runtime/rotation-account-selection.js";
|
|
50
|
-
import { createRotationProxyState, recoverStaleRuntimeState, } from "./runtime/rotation-proxy-state.js";
|
|
52
|
+
import { chooseAccount, chooseTranscriptionAccount } from "./runtime/rotation-account-selection.js";
|
|
53
|
+
import { createRotationProxyState, recoverStaleRuntimeState, reloadRuntimeCredentials, } from "./runtime/rotation-proxy-state.js";
|
|
51
54
|
import { readStorageMetaFromDisk } from "./runtime/rotation-storage-meta.js";
|
|
52
|
-
import { applyMonotonicAuthCooldown, DEFAULT_AUTH_FAILURE_COOLDOWN_MS, ensureFreshAccessToken, } from "./runtime/rotation-token-refresh.js";
|
|
55
|
+
import { applyMonotonicAuthCooldown, DEFAULT_AUTH_FAILURE_COOLDOWN_MS, ensureFreshAccessToken, redirectRuntimeRefreshCommits, resolveRuntimeRefreshTarget, } from "./runtime/rotation-token-refresh.js";
|
|
53
56
|
import { SessionAffinityStore } from "./session-affinity.js";
|
|
54
57
|
import { isRecord, sleep } from "./utils.js";
|
|
55
58
|
export { buildPinnedUnavailableErrorBody, buildTokenInvalidationBody, } from "./request/rate-limit-decision.js";
|
|
@@ -294,9 +297,11 @@ const MODEL_CAPACITY_RETRY_ENV = "CODEX_MULTI_AUTH_MODEL_CAPACITY_RETRY_MS";
|
|
|
294
297
|
function getApiModelRuntime(state) {
|
|
295
298
|
if (state.apiModelRuntime)
|
|
296
299
|
return state.apiModelRuntime;
|
|
297
|
-
const capabilities = state.apiModelCapabilities ??= new ApiModelCapabilities();
|
|
298
300
|
const production = state.readApiRoutes === loadApiRoutes;
|
|
299
|
-
|
|
301
|
+
// Public documentation survives refresh outages without retaining paid probe state.
|
|
302
|
+
if (production)
|
|
303
|
+
state.apiModelCapabilities ??= new ApiModelCapabilities(state.upstreamFetchImpl, state.now);
|
|
304
|
+
state.apiModelRuntime = new ApiModelRuntime(state.fetchImpl, state.now, production ? updateApiModelDiscovery : undefined, production ? (models, refresh, route, force) => enrichPersistedApiCapabilities(models, refresh, route, force, { fetchImpl: state.upstreamFetchImpl, now: state.now, metadata: state.apiModelCapabilities, requireStoredRoute: true }) : undefined, () => {
|
|
300
305
|
const runtime = state.apiModelRuntime;
|
|
301
306
|
if (!runtime)
|
|
302
307
|
return;
|
|
@@ -362,6 +367,9 @@ const ALLOWED_IMAGE_PATHS = new Set([
|
|
|
362
367
|
"/v1/images/generations",
|
|
363
368
|
"/v1/images/edits",
|
|
364
369
|
]);
|
|
370
|
+
const ALLOWED_TRANSCRIBE_PATHS = new Set([
|
|
371
|
+
URL_PATHS.TRANSCRIBE,
|
|
372
|
+
]);
|
|
365
373
|
const ALLOWED_THREAD_GOAL_PATHS = new Set([
|
|
366
374
|
"/thread/goal/get",
|
|
367
375
|
"/thread/goal/set",
|
|
@@ -737,6 +745,19 @@ function buildImageRequestContext(req, body, pathname) {
|
|
|
737
745
|
family: "codex",
|
|
738
746
|
};
|
|
739
747
|
}
|
|
748
|
+
function buildTranscribeRequestContext(req, body) {
|
|
749
|
+
return {
|
|
750
|
+
body,
|
|
751
|
+
headers: headersFromIncoming(req),
|
|
752
|
+
method: "POST",
|
|
753
|
+
upstreamPath: URL_PATHS.TRANSCRIBE,
|
|
754
|
+
model: null,
|
|
755
|
+
family: "codex",
|
|
756
|
+
stream: false,
|
|
757
|
+
sessionKey: null,
|
|
758
|
+
stableSessionKey: null,
|
|
759
|
+
};
|
|
760
|
+
}
|
|
740
761
|
function buildModelsRequestContext(req) {
|
|
741
762
|
return {
|
|
742
763
|
body: Buffer.alloc(0),
|
|
@@ -794,7 +815,7 @@ function writeJson(res, status, payload) {
|
|
|
794
815
|
function writeMethodOrPathError(res) {
|
|
795
816
|
writeJson(res, 404, {
|
|
796
817
|
error: {
|
|
797
|
-
message: "Runtime rotation proxy only accepts Responses API, images, model discovery, and Codex thread goal requests.",
|
|
818
|
+
message: "Runtime rotation proxy only accepts Responses API, images, transcription, model discovery, and Codex thread goal requests.",
|
|
798
819
|
code: "runtime_rotation_proxy_not_found",
|
|
799
820
|
},
|
|
800
821
|
});
|
|
@@ -809,7 +830,7 @@ function writeUnauthorized(res) {
|
|
|
809
830
|
}
|
|
810
831
|
function writePoolExhausted(params) {
|
|
811
832
|
const { res, accountManager, family, model, reason } = params;
|
|
812
|
-
const waitMs = accountManager.getMinWaitTimeForFamily(family, model);
|
|
833
|
+
const waitMs = params.retryAfterMs ?? accountManager.getMinWaitTimeForFamily(family, model);
|
|
813
834
|
const accountCount = accountManager.getAccountCount();
|
|
814
835
|
const accountSkipReasons = params.accountSkipReasons ?? {};
|
|
815
836
|
recordRuntimePoolExhaustion({
|
|
@@ -859,7 +880,7 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
859
880
|
const routingMutexMode = getRoutingMutexMode(pluginConfig);
|
|
860
881
|
activeAccountManager.setRoutingMutexMode(routingMutexMode);
|
|
861
882
|
const schedulingStrategy = getSchedulingStrategy(pluginConfig);
|
|
862
|
-
const fetchImpl = options.fetchImpl ??
|
|
883
|
+
const fetchImpl = options.fetchImpl ?? fetchWithProxy;
|
|
863
884
|
const host = options.host ?? DEFAULT_HOST;
|
|
864
885
|
// Defense in depth (runtime-proxy-01): the proxy presents managed OAuth tokens
|
|
865
886
|
// and must never be reachable off-box. It is loopback-only with NO opt-out —
|
|
@@ -959,6 +980,7 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
959
980
|
routingMutexMode,
|
|
960
981
|
schedulingStrategy,
|
|
961
982
|
fetchImpl,
|
|
983
|
+
upstreamFetchImpl: fetchImpl,
|
|
962
984
|
upstreamBaseUrl,
|
|
963
985
|
clientApiKey,
|
|
964
986
|
now,
|
|
@@ -1015,12 +1037,13 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
1015
1037
|
websocketGateway.attach(server, `http://${urlHost}:${resolvedPort}`);
|
|
1016
1038
|
// Embedded managers own their own lifecycle; ordinary CLI/app routers check opted-in accounts periodically.
|
|
1017
1039
|
const automaticChecks = options.accountManager ? undefined : startAutomaticAccountChecks(createAutomaticSubscriptionCheck());
|
|
1040
|
+
const automaticApiChecks = options.accountManager || options.readApiRoutes ? undefined : startAutomaticApiChecks(createAutomaticApiCheck(fetchImpl));
|
|
1018
1041
|
return {
|
|
1019
1042
|
host: bindHost,
|
|
1020
1043
|
port: resolvedPort,
|
|
1021
1044
|
baseUrl: `http://${urlHost}:${resolvedPort}`,
|
|
1022
1045
|
close: async () => {
|
|
1023
|
-
await automaticChecks?.stop();
|
|
1046
|
+
await Promise.all([automaticChecks?.stop(), automaticApiChecks?.stop()]);
|
|
1024
1047
|
websocketGateway.close();
|
|
1025
1048
|
await closeServer(server, sockets);
|
|
1026
1049
|
await flushRuntimeInferenceActivity();
|
|
@@ -1032,8 +1055,13 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
1032
1055
|
getOpenConnectionCount: () => sockets.size,
|
|
1033
1056
|
getStatus: () => ({
|
|
1034
1057
|
...state.status,
|
|
1058
|
+
schedulingStrategy: state.schedulingStrategy,
|
|
1035
1059
|
websocketConnections: websocketGateway.stats.connections,
|
|
1036
1060
|
websocketUpstreamRequests: websocketGateway.stats.upstreamRequests,
|
|
1061
|
+
websocketActiveConnections: websocketGateway.stats.activeConnections,
|
|
1062
|
+
websocketCapacityReclaims: websocketGateway.stats.capacityReclaims,
|
|
1063
|
+
lastWebsocketUpgradeRejection: websocketGateway.stats.lastUpgradeRejection,
|
|
1064
|
+
lastWebsocketDisconnect: websocketGateway.stats.lastDisconnect,
|
|
1037
1065
|
// Redact any email/token material that leaked into a raw upstream or
|
|
1038
1066
|
// refresh error string before exposing it to status/report consumers
|
|
1039
1067
|
// (errors-logging-08). maskString is a no-op for clean diagnostic text.
|
|
@@ -1100,8 +1128,10 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1100
1128
|
const accounts = accountManager.getAccountsSnapshot();
|
|
1101
1129
|
const sameInventory = accounts.length === disk.accounts.length && accounts.every((a, i) => isSameNativeAccount(a, disk.accounts[i]));
|
|
1102
1130
|
if (!sameInventory) {
|
|
1131
|
+
const previousManager = accountManager;
|
|
1103
1132
|
accountManager = new AccountManager(undefined, disk);
|
|
1104
1133
|
accountManager.setRoutingMutexMode(state.routingMutexMode);
|
|
1134
|
+
redirectRuntimeRefreshCommits(previousManager, accountManager);
|
|
1105
1135
|
// Preserve independently learned limits by identity, never by the old index.
|
|
1106
1136
|
// A record id survives an email change; the account identity survives a
|
|
1107
1137
|
// re-login that re-derives an unstored record id. Rows with neither an
|
|
@@ -1172,15 +1202,16 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1172
1202
|
const isResponsesRequest = req.method === "POST" && isResponsesPath(incomingUrl.pathname);
|
|
1173
1203
|
const isModelsRequest = req.method === "GET" && isModelsPath(incomingUrl.pathname);
|
|
1174
1204
|
const isImageRequest = req.method === "POST" && ALLOWED_IMAGE_PATHS.has(incomingUrl.pathname);
|
|
1205
|
+
const isTranscribeRequest = req.method === "POST" && ALLOWED_TRANSCRIBE_PATHS.has(incomingUrl.pathname);
|
|
1175
1206
|
const isThreadGoalRequest = (req.method === "GET" || req.method === "POST") &&
|
|
1176
1207
|
isThreadGoalPath(incomingUrl.pathname);
|
|
1177
1208
|
if (!isResponsesRequest && !isModelsRequest &&
|
|
1178
|
-
!isThreadGoalRequest && !isImageRequest) {
|
|
1209
|
+
!isThreadGoalRequest && !isImageRequest && !isTranscribeRequest) {
|
|
1179
1210
|
writeMethodOrPathError(res);
|
|
1180
1211
|
return;
|
|
1181
1212
|
}
|
|
1182
1213
|
state.status.totalRequests += 1;
|
|
1183
|
-
let requestBody = isResponsesRequest || isImageRequest ||
|
|
1214
|
+
let requestBody = isResponsesRequest || isImageRequest || isTranscribeRequest ||
|
|
1184
1215
|
(isThreadGoalRequest && req.method === "POST")
|
|
1185
1216
|
? await readRequestBody(req, state.maxRequestBodyBytes)
|
|
1186
1217
|
: Buffer.alloc(0);
|
|
@@ -1211,7 +1242,9 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1211
1242
|
? buildThreadGoalRequestContext(req, requestBody, incomingUrl.pathname)
|
|
1212
1243
|
: isImageRequest
|
|
1213
1244
|
? buildImageRequestContext(req, requestBody, incomingUrl.pathname)
|
|
1214
|
-
:
|
|
1245
|
+
: isTranscribeRequest
|
|
1246
|
+
? buildTranscribeRequestContext(req, requestBody)
|
|
1247
|
+
: buildResponsesRequestContext(req, requestBody);
|
|
1215
1248
|
if (isResponsesRequest) {
|
|
1216
1249
|
context.headers.delete("content-encoding");
|
|
1217
1250
|
context.headers.delete("content-length");
|
|
@@ -1263,7 +1296,9 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1263
1296
|
? "thread-goal"
|
|
1264
1297
|
: isImageRequest
|
|
1265
1298
|
? "images"
|
|
1266
|
-
:
|
|
1299
|
+
: isTranscribeRequest
|
|
1300
|
+
? "transcribe"
|
|
1301
|
+
: "responses",
|
|
1267
1302
|
model,
|
|
1268
1303
|
projectKey,
|
|
1269
1304
|
requestId: traceId,
|
|
@@ -1504,6 +1539,74 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1504
1539
|
const readSubscriptionQuota = state.readSubscriptionQuota;
|
|
1505
1540
|
const subscriptionQuotaCache = state.nativeOpenai && isResponsesRequest && readSubscriptionQuota ? await cachedRead(state, "subscription-quota", readSubscriptionQuota).catch(() => null) ?? null : null;
|
|
1506
1541
|
const subscriptionQuotaByAccount = {};
|
|
1542
|
+
const workspaceIds = (account) => new Set([
|
|
1543
|
+
account.accountId?.trim(),
|
|
1544
|
+
...(account.workspaces ?? []).map((workspace) => workspace.id.trim()),
|
|
1545
|
+
].filter((value) => Boolean(value)));
|
|
1546
|
+
const uniqueAccount = (candidates) => candidates.length === 1 ? candidates[0] ?? null : null;
|
|
1547
|
+
const matchReplacementAccount = (previous, previousManager, replacementManager) => {
|
|
1548
|
+
const replacementAccounts = replacementManager.getAccountsSnapshot();
|
|
1549
|
+
// Only a persisted record id is stable across re-login; a derived
|
|
1550
|
+
// in-memory id hashes the refresh token and changes on rotation.
|
|
1551
|
+
if (previous.recordId && previousManager.hasPersistedRecordId(previous)) {
|
|
1552
|
+
return uniqueAccount(replacementAccounts.filter((candidate) => candidate.recordId === previous.recordId));
|
|
1553
|
+
}
|
|
1554
|
+
const sourceIds = workspaceIds(previous);
|
|
1555
|
+
const compatible = replacementAccounts.filter((candidate) => {
|
|
1556
|
+
const candidateIds = workspaceIds(candidate);
|
|
1557
|
+
return !(sourceIds.size > 0 &&
|
|
1558
|
+
candidateIds.size > 0 &&
|
|
1559
|
+
![...sourceIds].some((id) => candidateIds.has(id)));
|
|
1560
|
+
});
|
|
1561
|
+
const sourceEmail = sanitizeEmail(previous.email);
|
|
1562
|
+
return (uniqueAccount(compatible.filter((candidate) => candidate.refreshToken === previous.refreshToken)) ??
|
|
1563
|
+
(previous.accountId && sourceEmail
|
|
1564
|
+
? uniqueAccount(compatible.filter((candidate) => candidate.accountId === previous.accountId && sanitizeEmail(candidate.email) === sourceEmail))
|
|
1565
|
+
: null) ??
|
|
1566
|
+
// accountId alone cannot merge logins: two members of one workspace
|
|
1567
|
+
// share it. Known-but-different emails are different logins, so the
|
|
1568
|
+
// remap must not adopt the sibling row for the removed account.
|
|
1569
|
+
(previous.accountId
|
|
1570
|
+
? uniqueAccount(compatible.filter((candidate) => candidate.accountId === previous.accountId &&
|
|
1571
|
+
(!sourceEmail ||
|
|
1572
|
+
!sanitizeEmail(candidate.email) ||
|
|
1573
|
+
sanitizeEmail(candidate.email) === sourceEmail)))
|
|
1574
|
+
: null) ??
|
|
1575
|
+
(sourceEmail
|
|
1576
|
+
? uniqueAccount(compatible.filter((candidate) => sanitizeEmail(candidate.email) === sourceEmail))
|
|
1577
|
+
: null));
|
|
1578
|
+
};
|
|
1579
|
+
const remapRequestWorkspaceState = (previousManager, replacementManager) => {
|
|
1580
|
+
const previousAccounts = previousManager.getAccountsSnapshot();
|
|
1581
|
+
const oldCandidates = [...workspaceCandidates.entries()];
|
|
1582
|
+
const oldQuota = { ...subscriptionQuotaByAccount };
|
|
1583
|
+
workspaceCandidates.clear();
|
|
1584
|
+
for (const key of Object.keys(subscriptionQuotaByAccount)) {
|
|
1585
|
+
delete subscriptionQuotaByAccount[Number(key)];
|
|
1586
|
+
}
|
|
1587
|
+
for (const [previousIndex, scopes] of oldCandidates) {
|
|
1588
|
+
const previous = previousAccounts.find((account) => account.index === previousIndex);
|
|
1589
|
+
if (!previous)
|
|
1590
|
+
continue;
|
|
1591
|
+
const replacement = matchReplacementAccount(previous, previousManager, replacementManager);
|
|
1592
|
+
if (!replacement)
|
|
1593
|
+
continue;
|
|
1594
|
+
const live = replacementManager.getAccountByIndex(replacement.index);
|
|
1595
|
+
if (!live)
|
|
1596
|
+
continue;
|
|
1597
|
+
const currentScopes = new Map(workspaceModelScopes(live).map((scope) => [scope.accountId, scope]));
|
|
1598
|
+
const remapped = scopes
|
|
1599
|
+
.map((scope) => currentScopes.get(scope.accountId))
|
|
1600
|
+
.filter((scope) => Boolean(scope?.routable));
|
|
1601
|
+
// Keep an explicit empty candidate list: it is a request-local guard that
|
|
1602
|
+
// prevents fallback to a stored workspace binding when every catalog-approved
|
|
1603
|
+
// scope disappeared or became unroutable during the manager swap.
|
|
1604
|
+
workspaceCandidates.set(live.index, remapped);
|
|
1605
|
+
if (oldQuota[previousIndex] !== undefined) {
|
|
1606
|
+
subscriptionQuotaByAccount[live.index] = oldQuota[previousIndex];
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
};
|
|
1507
1610
|
const resetCreditState = state.nativeOpenai && isResponsesRequest ? await cachedRead(state, "reset-credits", loadResetCreditState).catch(() => null) : null;
|
|
1508
1611
|
const quotaForScope = (account, scope) => {
|
|
1509
1612
|
// Scoped observations belong to this credential and workspace; legacy entries only describe the binding.
|
|
@@ -1517,8 +1620,65 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1517
1620
|
// A stored `switch` pin is a hard constraint in native mode too (#702), not a
|
|
1518
1621
|
// preference: it fails with codex_pinned_account_unavailable rather than
|
|
1519
1622
|
// silently serving from another account.
|
|
1520
|
-
|
|
1623
|
+
let pinnedIndex = state.forcedAccountIndex ?? storageMeta.pinnedAccountIndex;
|
|
1521
1624
|
const isPinned = typeof pinnedIndex === "number";
|
|
1625
|
+
let pinnedIdentityAccount = isPinned && typeof pinnedIndex === "number"
|
|
1626
|
+
? accountManager.getAccountByIndex(pinnedIndex) ?? null
|
|
1627
|
+
: null;
|
|
1628
|
+
let pinnedIdentityMissing = false;
|
|
1629
|
+
const matchPinnedReplacementAccount = (previousManager, previous, replacementManager) => {
|
|
1630
|
+
const replacements = replacementManager.getAccountsSnapshot();
|
|
1631
|
+
// A persisted record id is authoritative. Legacy rows receive a derived
|
|
1632
|
+
// in-memory id whose hash includes the refresh token; an external re-login
|
|
1633
|
+
// can therefore change that derived id for the same workspace. In that legacy
|
|
1634
|
+
// case continue to the strict workspace/account fallback below.
|
|
1635
|
+
if (previous.recordId && previousManager.hasPersistedRecordId(previous)) {
|
|
1636
|
+
return uniqueAccount(replacements.filter((candidate) => candidate.recordId === previous.recordId));
|
|
1637
|
+
}
|
|
1638
|
+
const sourceIds = workspaceIds(previous);
|
|
1639
|
+
const compatible = replacements.filter((candidate) => {
|
|
1640
|
+
const candidateIds = workspaceIds(candidate);
|
|
1641
|
+
return !(sourceIds.size > 0 &&
|
|
1642
|
+
candidateIds.size > 0 &&
|
|
1643
|
+
![...sourceIds].some((id) => candidateIds.has(id)));
|
|
1644
|
+
});
|
|
1645
|
+
const sourceEmail = sanitizeEmail(previous.email);
|
|
1646
|
+
// Hard pins deliberately omit the email-only fallback used by request-local
|
|
1647
|
+
// workspace remapping. Same-email sibling logins are not interchangeable.
|
|
1648
|
+
return (uniqueAccount(compatible.filter((candidate) => candidate.refreshToken === previous.refreshToken)) ??
|
|
1649
|
+
(previous.accountId && sourceEmail
|
|
1650
|
+
? uniqueAccount(compatible.filter((candidate) => candidate.accountId === previous.accountId && sanitizeEmail(candidate.email) === sourceEmail))
|
|
1651
|
+
: null) ??
|
|
1652
|
+
(previous.accountId
|
|
1653
|
+
? uniqueAccount(compatible.filter((candidate) => {
|
|
1654
|
+
if (candidate.accountId !== previous.accountId)
|
|
1655
|
+
return false;
|
|
1656
|
+
const candidateEmail = sanitizeEmail(candidate.email);
|
|
1657
|
+
return !sourceEmail || !candidateEmail || candidateEmail === sourceEmail;
|
|
1658
|
+
}))
|
|
1659
|
+
: null));
|
|
1660
|
+
};
|
|
1661
|
+
const remapPinnedAccount = (previousManager, replacementManager) => {
|
|
1662
|
+
if (!isPinned || typeof pinnedIndex !== "number" || pinnedIdentityMissing)
|
|
1663
|
+
return;
|
|
1664
|
+
// If the pin resolved when this request began, bind the request to that
|
|
1665
|
+
// account identity rather than to an index that can be reused after a
|
|
1666
|
+
// concurrent inventory change. A pin that was already out of range keeps
|
|
1667
|
+
// the legacy numeric behavior because there is no identity to preserve.
|
|
1668
|
+
const source = pinnedIdentityAccount ?? previousManager.getAccountByIndex(pinnedIndex) ?? null;
|
|
1669
|
+
if (!source)
|
|
1670
|
+
return;
|
|
1671
|
+
const replacement = matchPinnedReplacementAccount(previousManager, source, replacementManager);
|
|
1672
|
+
const live = replacement ? replacementManager.getAccountByIndex(replacement.index) ?? null : null;
|
|
1673
|
+
if (!live) {
|
|
1674
|
+
pinnedIdentityMissing = true;
|
|
1675
|
+
pinnedIdentityAccount = source;
|
|
1676
|
+
accountSkipReasons.set(pinnedIndex, "missing");
|
|
1677
|
+
return;
|
|
1678
|
+
}
|
|
1679
|
+
pinnedIndex = live.index;
|
|
1680
|
+
pinnedIdentityAccount = live;
|
|
1681
|
+
};
|
|
1522
1682
|
let requestModelCatalog;
|
|
1523
1683
|
/** Rebuild native routing scopes after Luna is rewritten to separately metered Reserve. */
|
|
1524
1684
|
const rebuildReserveWorkspaceCandidates = async () => {
|
|
@@ -1719,13 +1879,23 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1719
1879
|
models = await refresh;
|
|
1720
1880
|
else {
|
|
1721
1881
|
const cached = pickerSnapshot();
|
|
1722
|
-
|
|
1882
|
+
// An API cache can be warm while this client-version's OAuth
|
|
1883
|
+
// catalog is still cold. Give subscription discovery the same
|
|
1884
|
+
// bounded opportunity as an entirely empty picker.
|
|
1885
|
+
const oauthReady = modelCatalog.cachedList(routableKeys).length > 0 ||
|
|
1886
|
+
routableKeys.every(key => modelCatalog.hasUsablePickerSnapshot(key));
|
|
1887
|
+
if (cached.length && oauthReady)
|
|
1723
1888
|
models = cached;
|
|
1724
1889
|
else {
|
|
1725
1890
|
let timer;
|
|
1726
1891
|
try {
|
|
1727
1892
|
models = await Promise.race([refresh, new Promise(resolve => {
|
|
1728
|
-
timer = setTimeout(() =>
|
|
1893
|
+
timer = setTimeout(() => {
|
|
1894
|
+
// Do not let an incomplete API-only success replace the
|
|
1895
|
+
// native client's complete cached catalog for minutes.
|
|
1896
|
+
const missingSubscription = routableKeys.length > 0 && !modelCatalog.cachedList(routableKeys).length;
|
|
1897
|
+
resolve(missingSubscription && routableKeys.some(key => !modelCatalog.hasUsablePickerSnapshot(key)) ? [] : pickerSnapshot());
|
|
1898
|
+
}, 2000);
|
|
1729
1899
|
})]);
|
|
1730
1900
|
}
|
|
1731
1901
|
finally {
|
|
@@ -1734,6 +1904,12 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1734
1904
|
}
|
|
1735
1905
|
}
|
|
1736
1906
|
}
|
|
1907
|
+
// A completed failed fetch is still unknown, not a successful
|
|
1908
|
+
// empty catalog. Never replace native subscription entries with
|
|
1909
|
+
// an API-only success while a routable catalog is unknown.
|
|
1910
|
+
if (!forceCatalogRefresh && !models.some(model => !/^(api|zdr)\//.test(model.slug)) &&
|
|
1911
|
+
routableKeys.some(key => !modelCatalog.hasUsablePickerSnapshot(key)))
|
|
1912
|
+
models = [];
|
|
1737
1913
|
res.setHeader("etag", setCatalogEtag(state, models));
|
|
1738
1914
|
writeJson(res, models.length ? 200 : 503, models.length
|
|
1739
1915
|
? { models }
|
|
@@ -1812,7 +1988,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1812
1988
|
// so move the request to the independently metered Reserve model first.
|
|
1813
1989
|
if (isPinned &&
|
|
1814
1990
|
(context.model === "gpt-6-luna" || context.model === "gpt-5.6-luna")) {
|
|
1815
|
-
const pinnedAccount = accountManager.getAccountByIndex(pinnedIndex);
|
|
1991
|
+
const pinnedAccount = typeof pinnedIndex === "number" ? accountManager.getAccountByIndex(pinnedIndex) : null;
|
|
1816
1992
|
const now = state.now();
|
|
1817
1993
|
const familyResetAt = pinnedAccount?.rateLimitResetTimes[getQuotaKey(context.family)];
|
|
1818
1994
|
const modelResetAt = pinnedAccount?.rateLimitResetTimes[getQuotaKey(context.family, context.model)];
|
|
@@ -1854,9 +2030,10 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1854
2030
|
}
|
|
1855
2031
|
// The token bucket spreads load across a selectable pool. A pin has no
|
|
1856
2032
|
// alternative account, so exhausting that local heuristic can only reject a
|
|
1857
|
-
// request that the pinned account could serve.
|
|
1858
|
-
//
|
|
1859
|
-
|
|
2033
|
+
// request that the pinned account could serve. Transcription is not inference
|
|
2034
|
+
// and must not debit the Codex model pool or claim its inference circuit probe.
|
|
2035
|
+
// Transcription uses credential-only admission below.
|
|
2036
|
+
const bypassPoolTokenBucket = isPinned || isTranscribeRequest;
|
|
1860
2037
|
const refundConsumedPoolToken = (account) => {
|
|
1861
2038
|
if (!bypassPoolTokenBucket) {
|
|
1862
2039
|
accountManager.refundToken(account, context.family, context.model);
|
|
@@ -1948,7 +2125,9 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1948
2125
|
return "retry";
|
|
1949
2126
|
};
|
|
1950
2127
|
let runtimeSelectionIterations = 0;
|
|
1951
|
-
while ((isPinned ||
|
|
2128
|
+
while ((isPinned ||
|
|
2129
|
+
attemptedIndexes.size < accountCount ||
|
|
2130
|
+
(isTranscribeRequest && !state.nativeOpenai && accountCount === 0 && !reloadedAfterNoAccount)) &&
|
|
1952
2131
|
transientAttempts < transientAttemptLimit &&
|
|
1953
2132
|
(!isPinned ||
|
|
1954
2133
|
runtimeSelectionIterations - capacityRetries <
|
|
@@ -1982,6 +2161,10 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1982
2161
|
// calls inside `chooseAccount` are used unchanged and no lock is taken,
|
|
1983
2162
|
// so default behavior and perf are identical to before.
|
|
1984
2163
|
const selectAccount = () => {
|
|
2164
|
+
if (pinnedIdentityMissing && typeof pinnedIndex === "number") {
|
|
2165
|
+
accountSkipReasons.set(pinnedIndex, "missing");
|
|
2166
|
+
return null;
|
|
2167
|
+
}
|
|
1985
2168
|
if (context.model !== LUNA_RESERVE_MODEL) {
|
|
1986
2169
|
for (const [index, scopes] of workspaceCandidates) {
|
|
1987
2170
|
const account = accountManager.getAccountByIndex(index);
|
|
@@ -1993,13 +2176,34 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1993
2176
|
subscriptionQuotaByAccount[index] = subscriptionQuotaPreference(quotaForScope(account, scope), state.now());
|
|
1994
2177
|
}
|
|
1995
2178
|
}
|
|
2179
|
+
const attemptedForSelection = new Set([...attemptedIndexes, ...catalogExcludedIndexes()]);
|
|
2180
|
+
if (isTranscribeRequest) {
|
|
2181
|
+
const candidate = chooseTranscriptionAccount({
|
|
2182
|
+
accountManager,
|
|
2183
|
+
attemptedIndexes: attemptedForSelection,
|
|
2184
|
+
policy: policyDecision,
|
|
2185
|
+
pinnedIndex,
|
|
2186
|
+
now: state.now(),
|
|
2187
|
+
transcriptionRateLimitUntilByRecordId: state.transcriptionRateLimitUntilByRecordId,
|
|
2188
|
+
startIndex: state.transcriptionNextAccountIndex,
|
|
2189
|
+
// Native inference preference must not pin dictation to the current
|
|
2190
|
+
// Responses account; transcription owns an independent round-robin cursor.
|
|
2191
|
+
preferredIndex: null,
|
|
2192
|
+
fallbackPinnedIndex: state.nativeOpenai ? storageMeta.pinnedAccountIndex : null,
|
|
2193
|
+
skipReasons: accountSkipReasons,
|
|
2194
|
+
});
|
|
2195
|
+
if (candidate && pinnedIndex === null) {
|
|
2196
|
+
state.transcriptionNextAccountIndex = candidate.index + 1;
|
|
2197
|
+
}
|
|
2198
|
+
return candidate;
|
|
2199
|
+
}
|
|
1996
2200
|
const result = chooseAccount({
|
|
1997
2201
|
accountManager,
|
|
1998
2202
|
sessionAffinityStore: state.sessionAffinityStore,
|
|
1999
2203
|
sessionKey: context.sessionKey,
|
|
2000
2204
|
family: context.family,
|
|
2001
2205
|
model: context.model,
|
|
2002
|
-
attemptedIndexes:
|
|
2206
|
+
attemptedIndexes: attemptedForSelection,
|
|
2003
2207
|
now: state.now(),
|
|
2004
2208
|
policy: policyDecision,
|
|
2005
2209
|
pinnedIndex,
|
|
@@ -2010,12 +2214,13 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2010
2214
|
stickyBoostByAccount: rotationStickyBoost,
|
|
2011
2215
|
pidOffsetEnabled: state.pidOffsetEnabled,
|
|
2012
2216
|
schedulingStrategy: state.schedulingStrategy,
|
|
2013
|
-
//
|
|
2014
|
-
//
|
|
2015
|
-
//
|
|
2016
|
-
//
|
|
2017
|
-
//
|
|
2018
|
-
allowPinnedCooldown: isPinned && transientAttempts > 0
|
|
2217
|
+
// A shared network cooldown is not evidence that the pinned
|
|
2218
|
+
// credentials cannot serve independent concurrent work. Let each
|
|
2219
|
+
// new request try; its own retries remain bounded and backed off.
|
|
2220
|
+
// Other initial cooldowns still block, as do quota/auth/policy
|
|
2221
|
+
// checks and the circuit admission immediately after selection.
|
|
2222
|
+
allowPinnedCooldown: isPinned && (transientAttempts > 0 ||
|
|
2223
|
+
(typeof pinnedIndex === "number" && accountManager.getAccountByIndex(pinnedIndex)?.cooldownReason === "network-error")),
|
|
2019
2224
|
});
|
|
2020
2225
|
return result;
|
|
2021
2226
|
};
|
|
@@ -2023,6 +2228,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2023
2228
|
? await withRoutingMutex(state.routingMutexMode, async () => {
|
|
2024
2229
|
const candidate = selectAccount();
|
|
2025
2230
|
if (candidate &&
|
|
2231
|
+
!isTranscribeRequest &&
|
|
2026
2232
|
pinnedIndex === null &&
|
|
2027
2233
|
state.schedulingStrategy !== "sequential") {
|
|
2028
2234
|
// Re-commit the cursor under the held mutex. Skipped when a
|
|
@@ -2042,7 +2248,69 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2042
2248
|
for (const index of catalogExcludedIndexes())
|
|
2043
2249
|
accountSkipReasons.set(index, "model-not-supported");
|
|
2044
2250
|
if (!selected) {
|
|
2045
|
-
if (
|
|
2251
|
+
if (isTranscribeRequest &&
|
|
2252
|
+
!state.nativeOpenai &&
|
|
2253
|
+
!reloadedAfterNoAccount) {
|
|
2254
|
+
reloadedAfterNoAccount = true;
|
|
2255
|
+
const reloadedManager = await reloadRuntimeCredentials(state);
|
|
2256
|
+
if (reloadedManager && reloadedManager !== accountManager) {
|
|
2257
|
+
const previousManager = accountManager;
|
|
2258
|
+
remapPinnedAccount(previousManager, reloadedManager);
|
|
2259
|
+
accountManager = reloadedManager;
|
|
2260
|
+
accountCount = accountManager.getAccountCount();
|
|
2261
|
+
transientAttemptLimit = Math.max(1, Math.min(accountCount, state.maxRuntimeAccountAttempts));
|
|
2262
|
+
if (runtimePolicyState) {
|
|
2263
|
+
try {
|
|
2264
|
+
const reloadedPolicyDecision = await evaluateRuntimePolicy({
|
|
2265
|
+
state: runtimePolicyState,
|
|
2266
|
+
accounts: accountManager.getAccountsSnapshot(),
|
|
2267
|
+
model: context.model,
|
|
2268
|
+
now: state.now(),
|
|
2269
|
+
});
|
|
2270
|
+
policyDecision = reloadedPolicyDecision;
|
|
2271
|
+
mutateRuntimeObservabilitySnapshot((snapshot) => {
|
|
2272
|
+
snapshot.policyBlockedIndexes = [...reloadedPolicyDecision.blockedAccountIndexes];
|
|
2273
|
+
snapshot.policyBlockedReasons = Object.fromEntries([...reloadedPolicyDecision.blockedAccountIndexes].map((index) => [String(index), "policy-blocked"]));
|
|
2274
|
+
});
|
|
2275
|
+
if (!reloadedPolicyDecision.allowed) {
|
|
2276
|
+
await usageRecorder.record({
|
|
2277
|
+
outcome: "blocked",
|
|
2278
|
+
statusCode: reloadedPolicyDecision.statusCode,
|
|
2279
|
+
errorCode: reloadedPolicyDecision.errorCode,
|
|
2280
|
+
});
|
|
2281
|
+
writeJson(res, reloadedPolicyDecision.statusCode, {
|
|
2282
|
+
error: {
|
|
2283
|
+
message: "Runtime policy blocked this local request.",
|
|
2284
|
+
code: reloadedPolicyDecision.errorCode ?? "policy_blocked",
|
|
2285
|
+
reasons: reloadedPolicyDecision.reasons,
|
|
2286
|
+
},
|
|
2287
|
+
});
|
|
2288
|
+
return;
|
|
2289
|
+
}
|
|
2290
|
+
}
|
|
2291
|
+
catch (error) {
|
|
2292
|
+
state.status.lastError = error instanceof Error ? error.message : String(error);
|
|
2293
|
+
await usageRecorder.record({
|
|
2294
|
+
outcome: "failure",
|
|
2295
|
+
statusCode: HTTP_STATUS.SERVICE_UNAVAILABLE,
|
|
2296
|
+
errorCode: "runtime_policy_unavailable",
|
|
2297
|
+
});
|
|
2298
|
+
writeJson(res, HTTP_STATUS.SERVICE_UNAVAILABLE, {
|
|
2299
|
+
error: {
|
|
2300
|
+
message: "Runtime policy could not be loaded for this local request.",
|
|
2301
|
+
code: "runtime_policy_unavailable",
|
|
2302
|
+
},
|
|
2303
|
+
});
|
|
2304
|
+
return;
|
|
2305
|
+
}
|
|
2306
|
+
}
|
|
2307
|
+
accountSkipReasons.clear();
|
|
2308
|
+
attemptedIndexes.clear();
|
|
2309
|
+
continue;
|
|
2310
|
+
}
|
|
2311
|
+
}
|
|
2312
|
+
if (!isTranscribeRequest &&
|
|
2313
|
+
!reloadedAfterNoAccount &&
|
|
2046
2314
|
!isPinned &&
|
|
2047
2315
|
accountCount > 0 &&
|
|
2048
2316
|
exhaustionReason === "no-account" &&
|
|
@@ -2080,7 +2348,8 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2080
2348
|
attemptedIndexes.add(selected.index);
|
|
2081
2349
|
const quotaScheduleKey = buildQuotaScheduleKey(selected, context.family, context.model);
|
|
2082
2350
|
const preemptiveDeferral = state.preemptiveQuotaScheduler.getDeferral(quotaScheduleKey, state.now());
|
|
2083
|
-
if (
|
|
2351
|
+
if (!isTranscribeRequest &&
|
|
2352
|
+
context.model !== LUNA_RESERVE_MODEL &&
|
|
2084
2353
|
preemptiveDeferral.defer &&
|
|
2085
2354
|
preemptiveDeferral.waitMs > 0) {
|
|
2086
2355
|
accountSkipReasons.set(selected.index, preemptiveDeferral.reason ?? "quota-near-exhaustion");
|
|
@@ -2092,12 +2361,13 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2092
2361
|
noteRotation();
|
|
2093
2362
|
continue;
|
|
2094
2363
|
}
|
|
2095
|
-
//
|
|
2096
|
-
//
|
|
2097
|
-
//
|
|
2098
|
-
//
|
|
2099
|
-
|
|
2100
|
-
|
|
2364
|
+
// Inference consumes both the pool token and the race-safe circuit probe.
|
|
2365
|
+
// Transcription deliberately bypasses both: its credential-only selector
|
|
2366
|
+
// already enforced disabled/workspace/auth/policy gates, and claiming the
|
|
2367
|
+
// inference probe here could strand a half-open account after dictation.
|
|
2368
|
+
const admission = isTranscribeRequest
|
|
2369
|
+
? { ok: true }
|
|
2370
|
+
: accountManager.consumeTokenWithReason(selected, context.family, context.model, { bypassTokenBucket: bypassPoolTokenBucket });
|
|
2101
2371
|
if (!admission.ok) {
|
|
2102
2372
|
accountSkipReasons.set(selected.index, admission.reason);
|
|
2103
2373
|
exhaustionReason = "rate-limit";
|
|
@@ -2122,8 +2392,75 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2122
2392
|
tokenRefreshSkewMs: state.tokenRefreshSkewMs,
|
|
2123
2393
|
tokenInvalidationCooldownMs: state.tokenInvalidationCooldownMs,
|
|
2124
2394
|
});
|
|
2395
|
+
// A credential reload may have published a replacement manager while this
|
|
2396
|
+
// request was awaiting refresh. Keep the refreshed account and its owner
|
|
2397
|
+
// together for every later mutation/persist; account indexes are local to
|
|
2398
|
+
// one manager and can shift when rows are removed or reordered.
|
|
2399
|
+
if (refreshed.accountManager !== accountManager) {
|
|
2400
|
+
const previousManager = accountManager;
|
|
2401
|
+
remapRequestWorkspaceState(previousManager, refreshed.accountManager);
|
|
2402
|
+
remapPinnedAccount(previousManager, refreshed.accountManager);
|
|
2403
|
+
accountManager = refreshed.accountManager;
|
|
2404
|
+
accountCount = accountManager.getAccountCount();
|
|
2405
|
+
transientAttemptLimit = Math.max(1, Math.min(accountCount, state.maxRuntimeAccountAttempts));
|
|
2406
|
+
if (runtimePolicyState) {
|
|
2407
|
+
try {
|
|
2408
|
+
const refreshedPolicyDecision = await evaluateRuntimePolicy({
|
|
2409
|
+
state: runtimePolicyState,
|
|
2410
|
+
accounts: accountManager.getAccountsSnapshot(),
|
|
2411
|
+
model: context.model,
|
|
2412
|
+
now: state.now(),
|
|
2413
|
+
});
|
|
2414
|
+
policyDecision = refreshedPolicyDecision;
|
|
2415
|
+
mutateRuntimeObservabilitySnapshot((snapshot) => {
|
|
2416
|
+
snapshot.policyBlockedIndexes = [...refreshedPolicyDecision.blockedAccountIndexes];
|
|
2417
|
+
snapshot.policyBlockedReasons = Object.fromEntries([...refreshedPolicyDecision.blockedAccountIndexes].map((index) => [String(index), "policy-blocked"]));
|
|
2418
|
+
});
|
|
2419
|
+
if (!refreshedPolicyDecision.allowed) {
|
|
2420
|
+
await usageRecorder.record({
|
|
2421
|
+
outcome: "blocked",
|
|
2422
|
+
statusCode: refreshedPolicyDecision.statusCode,
|
|
2423
|
+
errorCode: refreshedPolicyDecision.errorCode,
|
|
2424
|
+
});
|
|
2425
|
+
writeJson(res, refreshedPolicyDecision.statusCode, {
|
|
2426
|
+
error: {
|
|
2427
|
+
message: "Runtime policy blocked this local request.",
|
|
2428
|
+
code: refreshedPolicyDecision.errorCode ?? "policy_blocked",
|
|
2429
|
+
reasons: refreshedPolicyDecision.reasons,
|
|
2430
|
+
},
|
|
2431
|
+
});
|
|
2432
|
+
return;
|
|
2433
|
+
}
|
|
2434
|
+
}
|
|
2435
|
+
catch (error) {
|
|
2436
|
+
state.status.lastError = error instanceof Error ? error.message : String(error);
|
|
2437
|
+
await usageRecorder.record({
|
|
2438
|
+
outcome: "failure",
|
|
2439
|
+
statusCode: HTTP_STATUS.SERVICE_UNAVAILABLE,
|
|
2440
|
+
errorCode: "runtime_policy_unavailable",
|
|
2441
|
+
});
|
|
2442
|
+
writeJson(res, HTTP_STATUS.SERVICE_UNAVAILABLE, {
|
|
2443
|
+
error: {
|
|
2444
|
+
message: "Runtime policy could not be loaded for this local request.",
|
|
2445
|
+
code: "runtime_policy_unavailable",
|
|
2446
|
+
},
|
|
2447
|
+
});
|
|
2448
|
+
return;
|
|
2449
|
+
}
|
|
2450
|
+
}
|
|
2451
|
+
accountSkipReasons.clear();
|
|
2452
|
+
attemptedIndexes.clear();
|
|
2453
|
+
}
|
|
2125
2454
|
if (!refreshed.ok) {
|
|
2126
2455
|
refundConsumedPoolToken(selected);
|
|
2456
|
+
if (refreshed.selectionRestart) {
|
|
2457
|
+
// The credential row vanished while refresh was in flight. The manager
|
|
2458
|
+
// swap above already cleared request-local attempt state; restart selection
|
|
2459
|
+
// without charging a transient auth failure to the replacement pool.
|
|
2460
|
+
exhaustionReason = "no-account";
|
|
2461
|
+
capabilityRejectionIsLatest = false;
|
|
2462
|
+
continue;
|
|
2463
|
+
}
|
|
2127
2464
|
// No accountSkipReasons write here: ensureFreshAccessToken always
|
|
2128
2465
|
// applies an auth cooldown before returning !ok, so the next
|
|
2129
2466
|
// selection pass overwrites whatever this set with
|
|
@@ -2157,7 +2494,12 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2157
2494
|
noteRotation();
|
|
2158
2495
|
continue;
|
|
2159
2496
|
}
|
|
2160
|
-
|
|
2497
|
+
let pendingScopes = workspaceCandidates.get(refreshed.account.index);
|
|
2498
|
+
if (isTranscribeRequest) {
|
|
2499
|
+
pendingScopes = workspaceModelScopes(refreshed.account)
|
|
2500
|
+
.filter((scope) => scope.routable)
|
|
2501
|
+
.sort((a, b) => Number(b.selected) - Number(a.selected) || Number(b.bound) - Number(a.bound));
|
|
2502
|
+
}
|
|
2161
2503
|
// Revalidate enabled scopes after token refresh; never write a per-request choice to the account.
|
|
2162
2504
|
const enabledScopes = new Set(workspaceModelScopes(refreshed.account).filter(s => s.routable).map(s => s.id));
|
|
2163
2505
|
while (pendingScopes?.length && !enabledScopes.has(pendingScopes[0]?.id ?? ""))
|
|
@@ -2225,6 +2567,26 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2225
2567
|
}
|
|
2226
2568
|
const accountIdentity = accountIdentityFromAccount(refreshed.account, state.now());
|
|
2227
2569
|
recordLastRuntimeAccount(state.status, accountIdentity, accountId);
|
|
2570
|
+
// A credential/inventory reload can publish while this request is waiting
|
|
2571
|
+
// on upstream. Resolve response-derived health/auth mutations lazily onto
|
|
2572
|
+
// the latest owner so late transport/429/401/5xx/success state cannot land
|
|
2573
|
+
// only on a superseded manager. If the row vanished entirely, retain the
|
|
2574
|
+
// captured object for this request; there is no replacement row to penalize.
|
|
2575
|
+
const responseCredential = {
|
|
2576
|
+
access: refreshed.accessToken,
|
|
2577
|
+
refreshToken: refreshed.account.refreshToken,
|
|
2578
|
+
};
|
|
2579
|
+
const responseStateTarget = () => resolveRuntimeRefreshTarget(accountManager, refreshed.account) ?? {
|
|
2580
|
+
accountManager,
|
|
2581
|
+
account: refreshed.account,
|
|
2582
|
+
};
|
|
2583
|
+
const responseAuthStateTarget = () => {
|
|
2584
|
+
const target = responseStateTarget();
|
|
2585
|
+
return target.account.access === responseCredential.access &&
|
|
2586
|
+
target.account.refreshToken === responseCredential.refreshToken
|
|
2587
|
+
? target
|
|
2588
|
+
: null;
|
|
2589
|
+
};
|
|
2228
2590
|
const outboundHeaders = createOutboundHeaders(context.headers, refreshed.account, refreshed.accessToken, accountId);
|
|
2229
2591
|
let upstream;
|
|
2230
2592
|
// Abort the in-flight upstream fetch when the client disconnects
|
|
@@ -2314,16 +2676,18 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2314
2676
|
return;
|
|
2315
2677
|
}
|
|
2316
2678
|
// A timeout may occur after generation; do not retry within this request.
|
|
2317
|
-
if (isImageRequest) {
|
|
2679
|
+
if (isImageRequest || isTranscribeRequest) {
|
|
2680
|
+
const operation = isImageRequest ? "image" : "transcribe";
|
|
2681
|
+
const errorCode = `${operation}_upstream_transport_error`;
|
|
2318
2682
|
writeJson(res, 502, {
|
|
2319
2683
|
error: {
|
|
2320
|
-
code:
|
|
2321
|
-
message: "Image upstream transport failed; not retried by the proxy
|
|
2684
|
+
code: errorCode,
|
|
2685
|
+
message: `${isImageRequest ? "Image" : "Transcription"} upstream transport failed; not retried by the proxy.`,
|
|
2322
2686
|
},
|
|
2323
2687
|
});
|
|
2324
2688
|
await usageRecorder.record({
|
|
2325
2689
|
outcome: "failure", statusCode: 502,
|
|
2326
|
-
errorCode
|
|
2690
|
+
errorCode, account: refreshed.account,
|
|
2327
2691
|
});
|
|
2328
2692
|
return;
|
|
2329
2693
|
}
|
|
@@ -2339,8 +2703,9 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2339
2703
|
// exhaustion 503 a non-zero `retry_after_ms` to back the client off
|
|
2340
2704
|
// with (`getMinWaitTimeForFamily` returns 0 while any account is
|
|
2341
2705
|
// still selectable).
|
|
2342
|
-
|
|
2343
|
-
accountManager.
|
|
2706
|
+
const transportState = responseStateTarget();
|
|
2707
|
+
transportState.accountManager.markAccountCoolingDown(transportState.account, state.networkErrorCooldownMs, "network-error");
|
|
2708
|
+
transportState.accountManager.saveToDiskDebounced();
|
|
2344
2709
|
accountSkipReasons.set(refreshed.account.index, "network-error");
|
|
2345
2710
|
exhaustionReason = "network-error";
|
|
2346
2711
|
capabilityRejectionIsLatest = false;
|
|
@@ -2371,7 +2736,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2371
2736
|
}
|
|
2372
2737
|
};
|
|
2373
2738
|
const quotaSnapshot = readQuotaSchedulerSnapshot(upstream.headers, upstream.status, state.now());
|
|
2374
|
-
if (quotaSnapshot && context.model !== LUNA_RESERVE_MODEL) {
|
|
2739
|
+
if (quotaSnapshot && !isTranscribeRequest && context.model !== LUNA_RESERVE_MODEL) {
|
|
2375
2740
|
observeQuota(quotaSnapshot, upstream.headers.get("x-codex-plan-type") ?? undefined);
|
|
2376
2741
|
}
|
|
2377
2742
|
if (isResponsesRequest && context.model && [400, 403, 404].includes(upstream.status)) {
|
|
@@ -2416,6 +2781,30 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2416
2781
|
}
|
|
2417
2782
|
if (upstream.status === HTTP_STATUS.TOO_MANY_REQUESTS) {
|
|
2418
2783
|
const bodyText = await readErrorBody(upstream, state.streamStallTimeoutMs);
|
|
2784
|
+
if (isTranscribeRequest) {
|
|
2785
|
+
const now = state.now();
|
|
2786
|
+
const retryAfterMs = parseRetryAfterHeaderMs(upstream.headers, now) ??
|
|
2787
|
+
parseRetryAfterBodyMs(bodyText, now) ??
|
|
2788
|
+
60_000;
|
|
2789
|
+
for (const [recordId, until] of state.transcriptionRateLimitUntilByRecordId) {
|
|
2790
|
+
if (until <= now)
|
|
2791
|
+
state.transcriptionRateLimitUntilByRecordId.delete(recordId);
|
|
2792
|
+
}
|
|
2793
|
+
if (refreshed.account.recordId) {
|
|
2794
|
+
const boundedDelayMs = Math.min(MAX_RATE_LIMIT_DELAY_MS, Math.max(1_000, Math.floor(retryAfterMs)));
|
|
2795
|
+
const existingUntil = state.transcriptionRateLimitUntilByRecordId.get(refreshed.account.recordId) ?? 0;
|
|
2796
|
+
state.transcriptionRateLimitUntilByRecordId.set(refreshed.account.recordId, Math.max(existingUntil, now + boundedDelayMs));
|
|
2797
|
+
}
|
|
2798
|
+
res.writeHead(upstream.status, responseHeadersForClient(upstream.headers));
|
|
2799
|
+
res.end(bodyText);
|
|
2800
|
+
await usageRecorder.record({
|
|
2801
|
+
outcome: "failure",
|
|
2802
|
+
statusCode: upstream.status,
|
|
2803
|
+
errorCode: extractErrorCodeFromBody(bodyText) ?? "transcribe_rate_limited",
|
|
2804
|
+
account: refreshed.account,
|
|
2805
|
+
});
|
|
2806
|
+
return;
|
|
2807
|
+
}
|
|
2419
2808
|
// Keep the upstream HINT separate from the 60s fallback. Passing the
|
|
2420
2809
|
// synthesized default into the capacity wait would make every
|
|
2421
2810
|
// hint-less capacity 429 sleep a full minute and leave the 2s/5s/15s
|
|
@@ -2449,16 +2838,17 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2449
2838
|
}
|
|
2450
2839
|
// Every 429 still drains this account for its retry window, but only an
|
|
2451
2840
|
// explicit subscription-usage signal is persisted as quota exhaustion.
|
|
2452
|
-
|
|
2453
|
-
accountManager.
|
|
2841
|
+
const rateLimitState = responseStateTarget();
|
|
2842
|
+
rateLimitState.accountManager.recordRateLimit(rateLimitState.account, context.family, context.model);
|
|
2843
|
+
rateLimitState.accountManager.markRateLimitedWithReason(rateLimitState.account, retryAfterMs, context.family, subscriptionQuotaExceeded ? "quota" : "unknown", context.model);
|
|
2454
2844
|
if (subscriptionQuotaExceeded &&
|
|
2455
2845
|
(context.model === "gpt-6-luna" || context.model === "gpt-5.6-luna")) {
|
|
2456
2846
|
const confirmedKey = confirmedQuotaExhaustionKey(context.family, context.model);
|
|
2457
2847
|
const confirmedResetAt = state.now() +
|
|
2458
2848
|
Math.min(Math.max(0, Math.floor(retryAfterMs)), MAX_RATE_LIMIT_DELAY_MS);
|
|
2459
|
-
|
|
2849
|
+
rateLimitState.account.rateLimitResetTimes[confirmedKey] = Math.max(rateLimitState.account.rateLimitResetTimes[confirmedKey] ?? 0, confirmedResetAt);
|
|
2460
2850
|
}
|
|
2461
|
-
accountManager.saveToDiskDebounced();
|
|
2851
|
+
rateLimitState.accountManager.saveToDiskDebounced();
|
|
2462
2852
|
if (!modelAtCapacity && subscriptionQuotaExceeded && switchRequestToLunaReserve(context)) {
|
|
2463
2853
|
usageRecorder = createUsageRecorderForModel(context.model);
|
|
2464
2854
|
if (runtimePolicyState) {
|
|
@@ -2506,8 +2896,9 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2506
2896
|
const errorCode = extractErrorCodeFromBody(bodyText);
|
|
2507
2897
|
if (isWorkspaceDisabledError(upstream.status, errorCode, bodyText)) {
|
|
2508
2898
|
if (requestScope && refreshed.account.workspaces?.length) {
|
|
2509
|
-
|
|
2510
|
-
|
|
2899
|
+
const workspaceState = responseStateTarget();
|
|
2900
|
+
if (workspaceState.accountManager.disableWorkspace(workspaceState.account, requestScope.accountId))
|
|
2901
|
+
workspaceState.accountManager.saveToDiskDebounced();
|
|
2511
2902
|
pendingScopes?.shift();
|
|
2512
2903
|
refundConsumedPoolToken(refreshed.account);
|
|
2513
2904
|
if (pendingScopes?.length)
|
|
@@ -2518,13 +2909,13 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2518
2909
|
noteRotation();
|
|
2519
2910
|
continue;
|
|
2520
2911
|
}
|
|
2521
|
-
const
|
|
2522
|
-
|
|
2912
|
+
const disabledState = responseStateTarget();
|
|
2913
|
+
const accountWasEnabled = disabledState.account.enabled !== false;
|
|
2523
2914
|
refundConsumedPoolToken(refreshed.account);
|
|
2524
2915
|
if (accountWasEnabled) {
|
|
2525
|
-
accountManager.recordFailure(
|
|
2526
|
-
accountManager.setAccountEnabled(
|
|
2527
|
-
accountManager.saveToDiskDebounced();
|
|
2916
|
+
disabledState.accountManager.recordFailure(disabledState.account, context.family, context.model);
|
|
2917
|
+
disabledState.accountManager.setAccountEnabled(disabledState.account.index, false);
|
|
2918
|
+
disabledState.accountManager.saveToDiskDebounced();
|
|
2528
2919
|
}
|
|
2529
2920
|
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
2530
2921
|
exhaustionReason = "deactivated";
|
|
@@ -2601,20 +2992,25 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2601
2992
|
if (upstream.status === HTTP_STATUS.UNAUTHORIZED) {
|
|
2602
2993
|
const bodyText = await readErrorBody(upstream, state.streamStallTimeoutMs);
|
|
2603
2994
|
refundConsumedPoolToken(refreshed.account);
|
|
2604
|
-
|
|
2995
|
+
const authState = responseAuthStateTarget();
|
|
2996
|
+
if (authState && !isTranscribeRequest) {
|
|
2997
|
+
authState.accountManager.recordFailure(authState.account, context.family, context.model);
|
|
2998
|
+
}
|
|
2605
2999
|
if (isTokenInvalidationError(bodyText)) {
|
|
2606
3000
|
// The upstream explicitly revoked this OAuth token. Applying a long
|
|
2607
3001
|
// cooldown prevents cascade invalidation: rapidly presenting each
|
|
2608
3002
|
// account's token from the same IP triggers OpenAI's anti-abuse
|
|
2609
3003
|
// detection and invalidates them in sequence. Return the 401 directly
|
|
2610
3004
|
// rather than rotating so the client can prompt for re-login.
|
|
2611
|
-
|
|
2612
|
-
|
|
3005
|
+
if (authState) {
|
|
3006
|
+
authState.accountManager.markAuthInvalidated(authState.account, extractErrorCodeFromBody(bodyText) ?? "token_invalidated");
|
|
3007
|
+
applyMonotonicAuthCooldown(authState.accountManager, authState.account, state.tokenInvalidationCooldownMs);
|
|
3008
|
+
// The invalidation marker must be durable before returning the 401.
|
|
3009
|
+
// A delayed save would allow a proxy restart to reload and route the
|
|
3010
|
+
// revoked account again.
|
|
3011
|
+
await authState.accountManager.saveToDisk();
|
|
3012
|
+
}
|
|
2613
3013
|
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
2614
|
-
// The invalidation marker must be durable before returning the 401.
|
|
2615
|
-
// A delayed save would allow a proxy restart to reload and route the
|
|
2616
|
-
// revoked account again.
|
|
2617
|
-
await accountManager.saveToDisk();
|
|
2618
3014
|
// Emit the same machine-readable shape as the refresh-failure path
|
|
2619
3015
|
// (code: "token_invalidated") instead of forwarding the raw upstream
|
|
2620
3016
|
// body, so the client contract is consistent across both vectors.
|
|
@@ -2630,8 +3026,21 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2630
3026
|
});
|
|
2631
3027
|
return;
|
|
2632
3028
|
}
|
|
2633
|
-
|
|
2634
|
-
|
|
3029
|
+
if (isTranscribeRequest) {
|
|
3030
|
+
res.writeHead(upstream.status, responseHeadersForClient(upstream.headers));
|
|
3031
|
+
res.end(bodyText);
|
|
3032
|
+
await usageRecorder.record({
|
|
3033
|
+
outcome: "failure",
|
|
3034
|
+
statusCode: upstream.status,
|
|
3035
|
+
errorCode: extractErrorCodeFromBody(bodyText) ?? "transcribe_unauthorized",
|
|
3036
|
+
account: refreshed.account,
|
|
3037
|
+
});
|
|
3038
|
+
return;
|
|
3039
|
+
}
|
|
3040
|
+
if (authState) {
|
|
3041
|
+
applyMonotonicAuthCooldown(authState.accountManager, authState.account, DEFAULT_AUTH_FAILURE_COOLDOWN_MS);
|
|
3042
|
+
authState.accountManager.saveToDiskDebounced();
|
|
3043
|
+
}
|
|
2635
3044
|
exhaustionReason = "auth-failure";
|
|
2636
3045
|
capabilityRejectionIsLatest = false;
|
|
2637
3046
|
transientAttempts += 1;
|
|
@@ -2640,6 +3049,19 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2640
3049
|
noteRotation();
|
|
2641
3050
|
continue;
|
|
2642
3051
|
}
|
|
3052
|
+
// Dictation audio may already have been accepted before any error response.
|
|
3053
|
+
// Do not replay it across accounts or let its endpoint-specific status
|
|
3054
|
+
// mutate Codex inference health/quota state. Explicit token invalidation
|
|
3055
|
+
// and workspace-disable handling above remain account-wide signals.
|
|
3056
|
+
if (isTranscribeRequest && upstream.status >= 400) {
|
|
3057
|
+
const forwarded = await forwardStreamingResponse(upstream, res, state.status, () => undefined, state.streamStallTimeoutMs);
|
|
3058
|
+
await usageRecorder.record({
|
|
3059
|
+
outcome: "failure", statusCode: upstream.status,
|
|
3060
|
+
errorCode: forwarded ? "transcribe_upstream_error" : "stream_forward_failed",
|
|
3061
|
+
account: refreshed.account,
|
|
3062
|
+
});
|
|
3063
|
+
return;
|
|
3064
|
+
}
|
|
2643
3065
|
// Forward image validation/moderation/5xx errors without cross-account replay.
|
|
2644
3066
|
// Explicit 429 quota and 401 auth handling above still use the existing pool.
|
|
2645
3067
|
if (isImageRequest && upstream.status >= 400) {
|
|
@@ -2670,9 +3092,10 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2670
3092
|
continue;
|
|
2671
3093
|
}
|
|
2672
3094
|
refundConsumedPoolToken(refreshed.account);
|
|
2673
|
-
|
|
2674
|
-
accountManager.
|
|
2675
|
-
accountManager.
|
|
3095
|
+
const serverState = responseStateTarget();
|
|
3096
|
+
serverState.accountManager.recordFailure(serverState.account, context.family, context.model);
|
|
3097
|
+
serverState.accountManager.markAccountCoolingDown(serverState.account, state.serverErrorCooldownMs, "server-error");
|
|
3098
|
+
serverState.accountManager.saveToDiskDebounced();
|
|
2676
3099
|
exhaustionReason = "server-error";
|
|
2677
3100
|
capabilityRejectionIsLatest = false;
|
|
2678
3101
|
transientAttempts += 1;
|
|
@@ -2707,25 +3130,28 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2707
3130
|
// "rate-limited") lingers on disk until an explicit runtime reset and
|
|
2708
3131
|
// the forecast keeps reporting this working account as unavailable.
|
|
2709
3132
|
// No-op when no reason is recorded, so the hot path stays write-free.
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
state.sessionAffinityStore?.remember(context.sessionKey, refreshed.account.index, state.now());
|
|
2723
|
-
if (refreshed.account.index !== state.lastGlobalAccountIndex) {
|
|
2724
|
-
state.lastGlobalAccountIndex = refreshed.account.index;
|
|
3133
|
+
if (!isTranscribeRequest) {
|
|
3134
|
+
const quotaDeferral = state.preemptiveQuotaScheduler.getDeferral(quotaScheduleKey, state.now());
|
|
3135
|
+
const nearExhaustionWaitMs = context.model === LUNA_RESERVE_MODEL
|
|
3136
|
+
? 0
|
|
3137
|
+
: quotaDeferral.defer
|
|
3138
|
+
? quotaDeferral.waitMs
|
|
3139
|
+
: 0;
|
|
3140
|
+
if (nearExhaustionWaitMs > 0) {
|
|
3141
|
+
const quotaState = responseStateTarget();
|
|
3142
|
+
quotaState.accountManager.markRateLimitedWithReason(quotaState.account, nearExhaustionWaitMs, context.family, "quota", context.model);
|
|
3143
|
+
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
3144
|
+
quotaState.accountManager.saveToDiskDebounced();
|
|
2725
3145
|
}
|
|
2726
|
-
|
|
3146
|
+
else {
|
|
3147
|
+
state.sessionAffinityStore?.remember(context.sessionKey, refreshed.account.index, state.now());
|
|
3148
|
+
if (refreshed.account.index !== state.lastGlobalAccountIndex) {
|
|
3149
|
+
state.lastGlobalAccountIndex = refreshed.account.index;
|
|
3150
|
+
}
|
|
3151
|
+
state.lastGlobalSwitchAt = state.now();
|
|
3152
|
+
}
|
|
3153
|
+
await persistRuntimeActiveAccount(accountManager, refreshed.account, context.family, isPinned && refreshed.account.index === pinnedIndex, state.schedulingStrategy, state.nativeOpenai);
|
|
2727
3154
|
}
|
|
2728
|
-
await persistRuntimeActiveAccount(accountManager, refreshed.account, context.family, isPinned && refreshed.account.index === pinnedIndex, state.schedulingStrategy, state.nativeOpenai);
|
|
2729
3155
|
// Recover the upstream token counts as the body streams past. Without
|
|
2730
3156
|
// them every ledger row lands with all-zero tokens and a zero cost, so
|
|
2731
3157
|
// evaluateBudgetGuard compares `0 >= limit` for maxTokens/maxCostUsd
|
|
@@ -2735,7 +3161,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2735
3161
|
contentType: upstream.headers.get("content-type"),
|
|
2736
3162
|
onEvent: (event) => {
|
|
2737
3163
|
responseOutcome.observe(event);
|
|
2738
|
-
if (context.model === LUNA_RESERVE_MODEL)
|
|
3164
|
+
if (isTranscribeRequest || context.model === LUNA_RESERVE_MODEL)
|
|
2739
3165
|
return;
|
|
2740
3166
|
const snapshot = readSubscriptionQuotaEvent(event, state.now());
|
|
2741
3167
|
if (!snapshot)
|
|
@@ -2749,23 +3175,35 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2749
3175
|
let streamErrored = false;
|
|
2750
3176
|
const forwarded = await forwardStreamingResponse(upstream, res, state.status, () => {
|
|
2751
3177
|
streamErrored = true;
|
|
3178
|
+
if (isTranscribeRequest)
|
|
3179
|
+
return;
|
|
2752
3180
|
// Deliberately NOT the pre-header transport policy above,
|
|
2753
3181
|
// which skips recordFailure (#677). By this point the
|
|
2754
3182
|
// upstream accepted the request and began responding, so a
|
|
2755
3183
|
// broken stream is a signal about THIS account's session
|
|
2756
3184
|
// (upstream tearing it down) rather than about the shared
|
|
2757
3185
|
// network path, and it should still credit the breaker.
|
|
2758
|
-
|
|
2759
|
-
accountManager.
|
|
3186
|
+
const streamState = responseStateTarget();
|
|
3187
|
+
streamState.accountManager.recordFailure(streamState.account, context.family, context.model);
|
|
3188
|
+
streamState.accountManager.markAccountCoolingDown(streamState.account, state.networkErrorCooldownMs, "network-error");
|
|
2760
3189
|
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
2761
|
-
accountManager.saveToDiskDebounced();
|
|
3190
|
+
streamState.accountManager.saveToDiskDebounced();
|
|
2762
3191
|
}, state.streamStallTimeoutMs, usageScanner.push, { ...(budgetAdvisory.level === "soft" ? buildContextBudgetHeaders(budgetAdvisory) : {}), ...(state.catalogEtag ? { "x-models-etag": state.catalogEtag } : {}) }, () => { usageScanner.result(); return responseOutcome.finish(); });
|
|
2763
3192
|
// The client went away mid-stream: no upstream error, and the response was
|
|
2764
3193
|
// neither completed nor failed. That says nothing about the account.
|
|
2765
3194
|
const clientDisconnected = !forwarded && !streamErrored && !res.writableEnded;
|
|
2766
3195
|
if (forwarded && upstream.ok) {
|
|
2767
|
-
|
|
2768
|
-
|
|
3196
|
+
if (isTranscribeRequest && refreshed.account.recordId) {
|
|
3197
|
+
const until = state.transcriptionRateLimitUntilByRecordId.get(refreshed.account.recordId);
|
|
3198
|
+
if (typeof until === "number" && until <= state.now()) {
|
|
3199
|
+
state.transcriptionRateLimitUntilByRecordId.delete(refreshed.account.recordId);
|
|
3200
|
+
}
|
|
3201
|
+
}
|
|
3202
|
+
if (!isTranscribeRequest) {
|
|
3203
|
+
const successState = responseStateTarget();
|
|
3204
|
+
successState.accountManager.recordSuccess(successState.account, context.family, context.model);
|
|
3205
|
+
recordRuntimeAccountRecovery(successState.account.index);
|
|
3206
|
+
}
|
|
2769
3207
|
}
|
|
2770
3208
|
else if (!clientDisconnected) {
|
|
2771
3209
|
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
@@ -2793,7 +3231,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2793
3231
|
// it here inflates the measurement by this turn's thinking budget
|
|
2794
3232
|
// against a deliberately tight threshold. `outputTokens` is already
|
|
2795
3233
|
// reasoning-stripped (lib/usage/usage-extraction.ts).
|
|
2796
|
-
if (usageTokens && context.stableSessionKey && context.model) {
|
|
3234
|
+
if (!isTranscribeRequest && usageTokens && context.stableSessionKey && context.model) {
|
|
2797
3235
|
state.contextBudgetGuard.update(context.stableSessionKey, {
|
|
2798
3236
|
model: context.model,
|
|
2799
3237
|
contextTokens: usageTokens.inputTokens + usageTokens.outputTokens,
|
|
@@ -2841,9 +3279,11 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2841
3279
|
// null reason indicates a forecast/runtime state desync (the pinned
|
|
2842
3280
|
// account was selected but no skip reason was recorded) — see #486.
|
|
2843
3281
|
if (isPinned) {
|
|
2844
|
-
const pinnedAccount =
|
|
2845
|
-
?
|
|
2846
|
-
:
|
|
3282
|
+
const pinnedAccount = pinnedIdentityMissing
|
|
3283
|
+
? null
|
|
3284
|
+
: typeof pinnedIndex === "number"
|
|
3285
|
+
? accountManager.getAccountByIndex(pinnedIndex)
|
|
3286
|
+
: null;
|
|
2847
3287
|
const pinnedSkipReason = typeof pinnedIndex === "number"
|
|
2848
3288
|
? accountSkipReasons.get(pinnedIndex) ?? null
|
|
2849
3289
|
: null;
|
|
@@ -2877,23 +3317,44 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2877
3317
|
// them and report a rate-limited pin as bounded by something else.
|
|
2878
3318
|
const evaluatedAtMs = state.now();
|
|
2879
3319
|
// Both bounds come from a single pass over the account's records.
|
|
2880
|
-
const pinnedRecoveryBounds = pinnedAccount
|
|
2881
|
-
?
|
|
2882
|
-
:
|
|
3320
|
+
const pinnedRecoveryBounds = !isTranscribeRequest && pinnedAccount !== null
|
|
3321
|
+
? getAccountRecoveryBoundsForFamily(pinnedAccount, evaluatedAtMs, context.family, context.model)
|
|
3322
|
+
: null;
|
|
2883
3323
|
const pinnedStateRecoveryAtMs = pinnedRecoveryBounds?.recoveryAtMs ?? null;
|
|
2884
3324
|
// An open circuit outlives the short failure cooldowns that tripped
|
|
2885
3325
|
// it; its deadline lives in the breaker, not the account record.
|
|
2886
|
-
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
3326
|
+
// Transcription intentionally ignores the inference circuit entirely.
|
|
3327
|
+
const pinnedCircuitRecoveryAtMs = !isTranscribeRequest && pinnedAccount !== null
|
|
3328
|
+
? accountManager.getCircuitRecoveryTime(pinnedAccount, evaluatedAtMs)
|
|
3329
|
+
: null;
|
|
3330
|
+
const pinnedTranscriptionRateLimitAtMs = isTranscribeRequest && pinnedAccount?.recordId
|
|
3331
|
+
? state.transcriptionRateLimitUntilByRecordId.get(pinnedAccount.recordId) ?? null
|
|
3332
|
+
: null;
|
|
3333
|
+
const livePinnedTranscriptionRateLimitAtMs = typeof pinnedTranscriptionRateLimitAtMs === "number" &&
|
|
3334
|
+
pinnedTranscriptionRateLimitAtMs > evaluatedAtMs
|
|
3335
|
+
? pinnedTranscriptionRateLimitAtMs
|
|
3336
|
+
: null;
|
|
3337
|
+
const pinnedAuthCooldownAtMs = isTranscribeRequest &&
|
|
3338
|
+
pinnedAccount?.cooldownReason === "auth-failure" &&
|
|
3339
|
+
typeof pinnedAccount.coolingDownUntil === "number" &&
|
|
3340
|
+
pinnedAccount.coolingDownUntil > evaluatedAtMs
|
|
3341
|
+
? pinnedAccount.coolingDownUntil
|
|
3342
|
+
: null;
|
|
3343
|
+
const pinnedResetAtMs = pinnedBlockedPermanently
|
|
2891
3344
|
? null
|
|
2892
|
-
:
|
|
3345
|
+
: isTranscribeRequest
|
|
3346
|
+
? livePinnedTranscriptionRateLimitAtMs === null && pinnedAuthCooldownAtMs === null
|
|
3347
|
+
? null
|
|
3348
|
+
: Math.max(livePinnedTranscriptionRateLimitAtMs ?? 0, pinnedAuthCooldownAtMs ?? 0)
|
|
3349
|
+
: pinnedStateRecoveryAtMs === null && pinnedCircuitRecoveryAtMs === null
|
|
3350
|
+
? null
|
|
3351
|
+
: Math.max(pinnedStateRecoveryAtMs ?? 0, pinnedCircuitRecoveryAtMs ?? 0);
|
|
2893
3352
|
// The message only words the recovery deadline as a rate-limit reset
|
|
2894
|
-
// when the rate-limit records are in fact what supplies it
|
|
3353
|
+
// when the rate-limit records are in fact what supplies it - a
|
|
2895
3354
|
// breaker or cooldown can end later and then bounds it instead.
|
|
2896
|
-
const pinnedRateLimitResetAtMs =
|
|
3355
|
+
const pinnedRateLimitResetAtMs = isTranscribeRequest
|
|
3356
|
+
? livePinnedTranscriptionRateLimitAtMs
|
|
3357
|
+
: pinnedRecoveryBounds?.rateLimitAtMs ?? null;
|
|
2897
3358
|
const recoveryBound = pinnedResetAtMs !== null &&
|
|
2898
3359
|
pinnedRateLimitResetAtMs !== null &&
|
|
2899
3360
|
pinnedRateLimitResetAtMs >= pinnedResetAtMs
|
|
@@ -2935,12 +3396,47 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2935
3396
|
writeJson(res, HTTP_STATUS.OK, { goal: null });
|
|
2936
3397
|
}
|
|
2937
3398
|
else {
|
|
3399
|
+
const transcriptionRetryAfterMs = isTranscribeRequest
|
|
3400
|
+
? (() => {
|
|
3401
|
+
const now = state.now();
|
|
3402
|
+
let minWaitMs = null;
|
|
3403
|
+
// Read live account state, not only selection skip reasons. The final
|
|
3404
|
+
// refresh attempt can install an auth cooldown after the last selection
|
|
3405
|
+
// pass, so that blocker may never have been written to accountSkipReasons.
|
|
3406
|
+
for (const account of accountManager.getAccountsSnapshot()) {
|
|
3407
|
+
if (policyDecision?.blockedAccountIndexes.has(account.index))
|
|
3408
|
+
continue;
|
|
3409
|
+
if (account.enabled === false)
|
|
3410
|
+
continue;
|
|
3411
|
+
if (!accountManager.hasEnabledWorkspaces(account))
|
|
3412
|
+
continue;
|
|
3413
|
+
if (accountManager.isAccountAuthInvalidated(account))
|
|
3414
|
+
continue;
|
|
3415
|
+
const transcriptionUntil = account.recordId
|
|
3416
|
+
? state.transcriptionRateLimitUntilByRecordId.get(account.recordId)
|
|
3417
|
+
: undefined;
|
|
3418
|
+
const authUntil = account.cooldownReason === "auth-failure" &&
|
|
3419
|
+
typeof account.coolingDownUntil === "number"
|
|
3420
|
+
? account.coolingDownUntil
|
|
3421
|
+
: undefined;
|
|
3422
|
+
const readyAt = Math.max(typeof transcriptionUntil === "number" && transcriptionUntil > now
|
|
3423
|
+
? transcriptionUntil
|
|
3424
|
+
: 0, typeof authUntil === "number" && authUntil > now ? authUntil : 0);
|
|
3425
|
+
if (readyAt <= now)
|
|
3426
|
+
return 0;
|
|
3427
|
+
const waitMs = readyAt - now;
|
|
3428
|
+
minWaitMs = minWaitMs === null ? waitMs : Math.min(minWaitMs, waitMs);
|
|
3429
|
+
}
|
|
3430
|
+
return minWaitMs ?? 0;
|
|
3431
|
+
})()
|
|
3432
|
+
: undefined;
|
|
2938
3433
|
writePoolExhausted({
|
|
2939
3434
|
res,
|
|
2940
3435
|
accountManager,
|
|
2941
3436
|
family: context.family,
|
|
2942
3437
|
model: context.model,
|
|
2943
3438
|
reason: exhaustionReason,
|
|
3439
|
+
retryAfterMs: transcriptionRetryAfterMs,
|
|
2944
3440
|
accountSkipReasons: Object.fromEntries([...accountSkipReasons.entries()].map(([index, reason]) => [
|
|
2945
3441
|
String(index),
|
|
2946
3442
|
reason,
|