codex-multi-auth 2.20.0 → 2.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +5 -4
- package/dist/lib/accounts.d.ts +5 -0
- package/dist/lib/accounts.js +33 -4
- package/dist/lib/api-route-store.d.ts +6 -0
- package/dist/lib/api-route-store.js +98 -30
- package/dist/lib/codex-manager/api-login-menu.d.ts +1 -1
- package/dist/lib/codex-manager/api-login-menu.js +4 -6
- package/dist/lib/codex-manager/commands/status.d.ts +1 -1
- package/dist/lib/codex-manager/commands/status.js +26 -9
- package/dist/lib/codex-manager.js +7 -1
- package/dist/lib/constants.d.ts +1 -0
- package/dist/lib/constants.js +1 -0
- package/dist/lib/model-route-policy.d.ts +1 -0
- package/dist/lib/model-route-policy.js +2 -1
- package/dist/lib/runtime/account-model-catalog.d.ts +1 -0
- package/dist/lib/runtime/api-model-capabilities.d.ts +11 -2
- package/dist/lib/runtime/api-model-capabilities.js +155 -24
- package/dist/lib/runtime/app-bind.d.ts +4 -0
- package/dist/lib/runtime/app-bind.js +32 -1
- package/dist/lib/runtime/automatic-api-checks.d.ts +15 -0
- package/dist/lib/runtime/automatic-api-checks.js +188 -0
- package/dist/lib/runtime/model-discovery-status.d.ts +11 -1
- package/dist/lib/runtime/model-discovery-status.js +64 -49
- package/dist/lib/runtime/persisted-api-capabilities.d.ts +20 -0
- package/dist/lib/runtime/persisted-api-capabilities.js +387 -0
- package/dist/lib/runtime/responses-websocket.d.ts +18 -5
- package/dist/lib/runtime/responses-websocket.js +33 -11
- package/dist/lib/runtime/rotation-account-selection.d.ts +20 -1
- package/dist/lib/runtime/rotation-account-selection.js +94 -4
- package/dist/lib/runtime/rotation-proxy-state.d.ts +26 -0
- package/dist/lib/runtime/rotation-proxy-state.js +376 -4
- package/dist/lib/runtime/rotation-server-types.d.ts +3 -0
- package/dist/lib/runtime/rotation-token-refresh.d.ts +23 -1
- package/dist/lib/runtime/rotation-token-refresh.js +218 -27
- package/dist/lib/runtime-rotation-proxy.js +571 -96
- package/dist/lib/storage/credential-sidecars.js +34 -2
- package/dist/lib/storage/file-lock.d.ts +2 -0
- package/dist/lib/storage/file-lock.js +44 -30
- package/dist/lib/usage/ledger.js +1 -0
- package/dist/lib/usage/redaction.js +1 -0
- package/dist/lib/usage/types.d.ts +1 -1
- package/package.json +1 -1
- package/scripts/codex-app-router.js +42 -14
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createAutomaticApiCheck, startAutomaticApiChecks } from "./runtime/automatic-api-checks.js";
|
|
1
2
|
import { startAutomaticAccountChecks } from "./runtime/automatic-account-checks.js";
|
|
2
3
|
import { createAutomaticSubscriptionCheck } from "./runtime/automatic-subscription-checks.js";
|
|
3
4
|
import { ClientCancellationError } from "./request/client-cancellation.js";
|
|
@@ -14,6 +15,7 @@ import { modelScopeId, workspaceModelScopes } from "./runtime/workspace-model-sc
|
|
|
14
15
|
import { RuntimeCapabilityFailures, classifyCapabilityFailure } from "./runtime/runtime-capability-failures.js";
|
|
15
16
|
import { ResponseOutcome } from "./request/response-outcome.js";
|
|
16
17
|
import { ResponsesWebSocketGateway } from "./runtime/responses-websocket.js";
|
|
18
|
+
import { enrichPersistedApiCapabilities } from "./runtime/persisted-api-capabilities.js";
|
|
17
19
|
import { ApiModelCapabilities } from "./runtime/api-model-capabilities.js";
|
|
18
20
|
import { saveModelInventory } from "./runtime/model-discovery-status.js";
|
|
19
21
|
import { loadApiRoutes, updateApiModelDiscovery } from "./api-route-store.js";
|
|
@@ -46,10 +48,10 @@ import { normalizeEmailKey } from "./storage/identity.js";
|
|
|
46
48
|
import { buildPinnedUnavailableErrorBody, buildTokenInvalidationBody, extractErrorCodeFromBody, isTokenInvalidationError, normalizeExhaustionStatus, parseRetryAfterBodyMs, parseRetryAfterHeaderMs, } from "./request/rate-limit-decision.js";
|
|
47
49
|
import { forwardStreamingResponse, HOP_BY_HOP_HEADERS, readErrorBody, responseHeadersForClient, withTimeout, } from "./request/stream-failover-runtime.js";
|
|
48
50
|
import { getAccountRecoveryBoundsForFamily } from "./runtime/account-status.js";
|
|
49
|
-
import { chooseAccount } from "./runtime/rotation-account-selection.js";
|
|
50
|
-
import { createRotationProxyState, recoverStaleRuntimeState, } from "./runtime/rotation-proxy-state.js";
|
|
51
|
+
import { chooseAccount, chooseTranscriptionAccount } from "./runtime/rotation-account-selection.js";
|
|
52
|
+
import { createRotationProxyState, recoverStaleRuntimeState, reloadRuntimeCredentials, } from "./runtime/rotation-proxy-state.js";
|
|
51
53
|
import { readStorageMetaFromDisk } from "./runtime/rotation-storage-meta.js";
|
|
52
|
-
import { applyMonotonicAuthCooldown, DEFAULT_AUTH_FAILURE_COOLDOWN_MS, ensureFreshAccessToken, } from "./runtime/rotation-token-refresh.js";
|
|
54
|
+
import { applyMonotonicAuthCooldown, DEFAULT_AUTH_FAILURE_COOLDOWN_MS, ensureFreshAccessToken, redirectRuntimeRefreshCommits, resolveRuntimeRefreshTarget, } from "./runtime/rotation-token-refresh.js";
|
|
53
55
|
import { SessionAffinityStore } from "./session-affinity.js";
|
|
54
56
|
import { isRecord, sleep } from "./utils.js";
|
|
55
57
|
export { buildPinnedUnavailableErrorBody, buildTokenInvalidationBody, } from "./request/rate-limit-decision.js";
|
|
@@ -294,9 +296,11 @@ const MODEL_CAPACITY_RETRY_ENV = "CODEX_MULTI_AUTH_MODEL_CAPACITY_RETRY_MS";
|
|
|
294
296
|
function getApiModelRuntime(state) {
|
|
295
297
|
if (state.apiModelRuntime)
|
|
296
298
|
return state.apiModelRuntime;
|
|
297
|
-
const capabilities = state.apiModelCapabilities ??= new ApiModelCapabilities();
|
|
298
299
|
const production = state.readApiRoutes === loadApiRoutes;
|
|
299
|
-
|
|
300
|
+
// Public documentation survives refresh outages without retaining paid probe state.
|
|
301
|
+
if (production)
|
|
302
|
+
state.apiModelCapabilities ??= new ApiModelCapabilities(state.upstreamFetchImpl, state.now);
|
|
303
|
+
state.apiModelRuntime = new ApiModelRuntime(state.fetchImpl, state.now, production ? updateApiModelDiscovery : undefined, production ? (models, refresh, route, force) => enrichPersistedApiCapabilities(models, refresh, route, force, { fetchImpl: state.upstreamFetchImpl, now: state.now, metadata: state.apiModelCapabilities, requireStoredRoute: true }) : undefined, () => {
|
|
300
304
|
const runtime = state.apiModelRuntime;
|
|
301
305
|
if (!runtime)
|
|
302
306
|
return;
|
|
@@ -362,6 +366,9 @@ const ALLOWED_IMAGE_PATHS = new Set([
|
|
|
362
366
|
"/v1/images/generations",
|
|
363
367
|
"/v1/images/edits",
|
|
364
368
|
]);
|
|
369
|
+
const ALLOWED_TRANSCRIBE_PATHS = new Set([
|
|
370
|
+
URL_PATHS.TRANSCRIBE,
|
|
371
|
+
]);
|
|
365
372
|
const ALLOWED_THREAD_GOAL_PATHS = new Set([
|
|
366
373
|
"/thread/goal/get",
|
|
367
374
|
"/thread/goal/set",
|
|
@@ -737,6 +744,19 @@ function buildImageRequestContext(req, body, pathname) {
|
|
|
737
744
|
family: "codex",
|
|
738
745
|
};
|
|
739
746
|
}
|
|
747
|
+
function buildTranscribeRequestContext(req, body) {
|
|
748
|
+
return {
|
|
749
|
+
body,
|
|
750
|
+
headers: headersFromIncoming(req),
|
|
751
|
+
method: "POST",
|
|
752
|
+
upstreamPath: URL_PATHS.TRANSCRIBE,
|
|
753
|
+
model: null,
|
|
754
|
+
family: "codex",
|
|
755
|
+
stream: false,
|
|
756
|
+
sessionKey: null,
|
|
757
|
+
stableSessionKey: null,
|
|
758
|
+
};
|
|
759
|
+
}
|
|
740
760
|
function buildModelsRequestContext(req) {
|
|
741
761
|
return {
|
|
742
762
|
body: Buffer.alloc(0),
|
|
@@ -794,7 +814,7 @@ function writeJson(res, status, payload) {
|
|
|
794
814
|
function writeMethodOrPathError(res) {
|
|
795
815
|
writeJson(res, 404, {
|
|
796
816
|
error: {
|
|
797
|
-
message: "Runtime rotation proxy only accepts Responses API, images, model discovery, and Codex thread goal requests.",
|
|
817
|
+
message: "Runtime rotation proxy only accepts Responses API, images, transcription, model discovery, and Codex thread goal requests.",
|
|
798
818
|
code: "runtime_rotation_proxy_not_found",
|
|
799
819
|
},
|
|
800
820
|
});
|
|
@@ -809,7 +829,7 @@ function writeUnauthorized(res) {
|
|
|
809
829
|
}
|
|
810
830
|
function writePoolExhausted(params) {
|
|
811
831
|
const { res, accountManager, family, model, reason } = params;
|
|
812
|
-
const waitMs = accountManager.getMinWaitTimeForFamily(family, model);
|
|
832
|
+
const waitMs = params.retryAfterMs ?? accountManager.getMinWaitTimeForFamily(family, model);
|
|
813
833
|
const accountCount = accountManager.getAccountCount();
|
|
814
834
|
const accountSkipReasons = params.accountSkipReasons ?? {};
|
|
815
835
|
recordRuntimePoolExhaustion({
|
|
@@ -959,6 +979,7 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
959
979
|
routingMutexMode,
|
|
960
980
|
schedulingStrategy,
|
|
961
981
|
fetchImpl,
|
|
982
|
+
upstreamFetchImpl: fetchImpl,
|
|
962
983
|
upstreamBaseUrl,
|
|
963
984
|
clientApiKey,
|
|
964
985
|
now,
|
|
@@ -1015,12 +1036,13 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
1015
1036
|
websocketGateway.attach(server, `http://${urlHost}:${resolvedPort}`);
|
|
1016
1037
|
// Embedded managers own their own lifecycle; ordinary CLI/app routers check opted-in accounts periodically.
|
|
1017
1038
|
const automaticChecks = options.accountManager ? undefined : startAutomaticAccountChecks(createAutomaticSubscriptionCheck());
|
|
1039
|
+
const automaticApiChecks = options.accountManager || options.readApiRoutes ? undefined : startAutomaticApiChecks(createAutomaticApiCheck(fetchImpl));
|
|
1018
1040
|
return {
|
|
1019
1041
|
host: bindHost,
|
|
1020
1042
|
port: resolvedPort,
|
|
1021
1043
|
baseUrl: `http://${urlHost}:${resolvedPort}`,
|
|
1022
1044
|
close: async () => {
|
|
1023
|
-
await automaticChecks?.stop();
|
|
1045
|
+
await Promise.all([automaticChecks?.stop(), automaticApiChecks?.stop()]);
|
|
1024
1046
|
websocketGateway.close();
|
|
1025
1047
|
await closeServer(server, sockets);
|
|
1026
1048
|
await flushRuntimeInferenceActivity();
|
|
@@ -1032,8 +1054,10 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
1032
1054
|
getOpenConnectionCount: () => sockets.size,
|
|
1033
1055
|
getStatus: () => ({
|
|
1034
1056
|
...state.status,
|
|
1057
|
+
schedulingStrategy: state.schedulingStrategy,
|
|
1035
1058
|
websocketConnections: websocketGateway.stats.connections,
|
|
1036
1059
|
websocketUpstreamRequests: websocketGateway.stats.upstreamRequests,
|
|
1060
|
+
lastWebsocketDisconnect: websocketGateway.stats.lastDisconnect,
|
|
1037
1061
|
// Redact any email/token material that leaked into a raw upstream or
|
|
1038
1062
|
// refresh error string before exposing it to status/report consumers
|
|
1039
1063
|
// (errors-logging-08). maskString is a no-op for clean diagnostic text.
|
|
@@ -1100,8 +1124,10 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1100
1124
|
const accounts = accountManager.getAccountsSnapshot();
|
|
1101
1125
|
const sameInventory = accounts.length === disk.accounts.length && accounts.every((a, i) => isSameNativeAccount(a, disk.accounts[i]));
|
|
1102
1126
|
if (!sameInventory) {
|
|
1127
|
+
const previousManager = accountManager;
|
|
1103
1128
|
accountManager = new AccountManager(undefined, disk);
|
|
1104
1129
|
accountManager.setRoutingMutexMode(state.routingMutexMode);
|
|
1130
|
+
redirectRuntimeRefreshCommits(previousManager, accountManager);
|
|
1105
1131
|
// Preserve independently learned limits by identity, never by the old index.
|
|
1106
1132
|
// A record id survives an email change; the account identity survives a
|
|
1107
1133
|
// re-login that re-derives an unstored record id. Rows with neither an
|
|
@@ -1172,15 +1198,16 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1172
1198
|
const isResponsesRequest = req.method === "POST" && isResponsesPath(incomingUrl.pathname);
|
|
1173
1199
|
const isModelsRequest = req.method === "GET" && isModelsPath(incomingUrl.pathname);
|
|
1174
1200
|
const isImageRequest = req.method === "POST" && ALLOWED_IMAGE_PATHS.has(incomingUrl.pathname);
|
|
1201
|
+
const isTranscribeRequest = req.method === "POST" && ALLOWED_TRANSCRIBE_PATHS.has(incomingUrl.pathname);
|
|
1175
1202
|
const isThreadGoalRequest = (req.method === "GET" || req.method === "POST") &&
|
|
1176
1203
|
isThreadGoalPath(incomingUrl.pathname);
|
|
1177
1204
|
if (!isResponsesRequest && !isModelsRequest &&
|
|
1178
|
-
!isThreadGoalRequest && !isImageRequest) {
|
|
1205
|
+
!isThreadGoalRequest && !isImageRequest && !isTranscribeRequest) {
|
|
1179
1206
|
writeMethodOrPathError(res);
|
|
1180
1207
|
return;
|
|
1181
1208
|
}
|
|
1182
1209
|
state.status.totalRequests += 1;
|
|
1183
|
-
let requestBody = isResponsesRequest || isImageRequest ||
|
|
1210
|
+
let requestBody = isResponsesRequest || isImageRequest || isTranscribeRequest ||
|
|
1184
1211
|
(isThreadGoalRequest && req.method === "POST")
|
|
1185
1212
|
? await readRequestBody(req, state.maxRequestBodyBytes)
|
|
1186
1213
|
: Buffer.alloc(0);
|
|
@@ -1211,7 +1238,9 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1211
1238
|
? buildThreadGoalRequestContext(req, requestBody, incomingUrl.pathname)
|
|
1212
1239
|
: isImageRequest
|
|
1213
1240
|
? buildImageRequestContext(req, requestBody, incomingUrl.pathname)
|
|
1214
|
-
:
|
|
1241
|
+
: isTranscribeRequest
|
|
1242
|
+
? buildTranscribeRequestContext(req, requestBody)
|
|
1243
|
+
: buildResponsesRequestContext(req, requestBody);
|
|
1215
1244
|
if (isResponsesRequest) {
|
|
1216
1245
|
context.headers.delete("content-encoding");
|
|
1217
1246
|
context.headers.delete("content-length");
|
|
@@ -1263,7 +1292,9 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1263
1292
|
? "thread-goal"
|
|
1264
1293
|
: isImageRequest
|
|
1265
1294
|
? "images"
|
|
1266
|
-
:
|
|
1295
|
+
: isTranscribeRequest
|
|
1296
|
+
? "transcribe"
|
|
1297
|
+
: "responses",
|
|
1267
1298
|
model,
|
|
1268
1299
|
projectKey,
|
|
1269
1300
|
requestId: traceId,
|
|
@@ -1504,6 +1535,74 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1504
1535
|
const readSubscriptionQuota = state.readSubscriptionQuota;
|
|
1505
1536
|
const subscriptionQuotaCache = state.nativeOpenai && isResponsesRequest && readSubscriptionQuota ? await cachedRead(state, "subscription-quota", readSubscriptionQuota).catch(() => null) ?? null : null;
|
|
1506
1537
|
const subscriptionQuotaByAccount = {};
|
|
1538
|
+
const workspaceIds = (account) => new Set([
|
|
1539
|
+
account.accountId?.trim(),
|
|
1540
|
+
...(account.workspaces ?? []).map((workspace) => workspace.id.trim()),
|
|
1541
|
+
].filter((value) => Boolean(value)));
|
|
1542
|
+
const uniqueAccount = (candidates) => candidates.length === 1 ? candidates[0] ?? null : null;
|
|
1543
|
+
const matchReplacementAccount = (previous, previousManager, replacementManager) => {
|
|
1544
|
+
const replacementAccounts = replacementManager.getAccountsSnapshot();
|
|
1545
|
+
// Only a persisted record id is stable across re-login; a derived
|
|
1546
|
+
// in-memory id hashes the refresh token and changes on rotation.
|
|
1547
|
+
if (previous.recordId && previousManager.hasPersistedRecordId(previous)) {
|
|
1548
|
+
return uniqueAccount(replacementAccounts.filter((candidate) => candidate.recordId === previous.recordId));
|
|
1549
|
+
}
|
|
1550
|
+
const sourceIds = workspaceIds(previous);
|
|
1551
|
+
const compatible = replacementAccounts.filter((candidate) => {
|
|
1552
|
+
const candidateIds = workspaceIds(candidate);
|
|
1553
|
+
return !(sourceIds.size > 0 &&
|
|
1554
|
+
candidateIds.size > 0 &&
|
|
1555
|
+
![...sourceIds].some((id) => candidateIds.has(id)));
|
|
1556
|
+
});
|
|
1557
|
+
const sourceEmail = sanitizeEmail(previous.email);
|
|
1558
|
+
return (uniqueAccount(compatible.filter((candidate) => candidate.refreshToken === previous.refreshToken)) ??
|
|
1559
|
+
(previous.accountId && sourceEmail
|
|
1560
|
+
? uniqueAccount(compatible.filter((candidate) => candidate.accountId === previous.accountId && sanitizeEmail(candidate.email) === sourceEmail))
|
|
1561
|
+
: null) ??
|
|
1562
|
+
// accountId alone cannot merge logins: two members of one workspace
|
|
1563
|
+
// share it. Known-but-different emails are different logins, so the
|
|
1564
|
+
// remap must not adopt the sibling row for the removed account.
|
|
1565
|
+
(previous.accountId
|
|
1566
|
+
? uniqueAccount(compatible.filter((candidate) => candidate.accountId === previous.accountId &&
|
|
1567
|
+
(!sourceEmail ||
|
|
1568
|
+
!sanitizeEmail(candidate.email) ||
|
|
1569
|
+
sanitizeEmail(candidate.email) === sourceEmail)))
|
|
1570
|
+
: null) ??
|
|
1571
|
+
(sourceEmail
|
|
1572
|
+
? uniqueAccount(compatible.filter((candidate) => sanitizeEmail(candidate.email) === sourceEmail))
|
|
1573
|
+
: null));
|
|
1574
|
+
};
|
|
1575
|
+
const remapRequestWorkspaceState = (previousManager, replacementManager) => {
|
|
1576
|
+
const previousAccounts = previousManager.getAccountsSnapshot();
|
|
1577
|
+
const oldCandidates = [...workspaceCandidates.entries()];
|
|
1578
|
+
const oldQuota = { ...subscriptionQuotaByAccount };
|
|
1579
|
+
workspaceCandidates.clear();
|
|
1580
|
+
for (const key of Object.keys(subscriptionQuotaByAccount)) {
|
|
1581
|
+
delete subscriptionQuotaByAccount[Number(key)];
|
|
1582
|
+
}
|
|
1583
|
+
for (const [previousIndex, scopes] of oldCandidates) {
|
|
1584
|
+
const previous = previousAccounts.find((account) => account.index === previousIndex);
|
|
1585
|
+
if (!previous)
|
|
1586
|
+
continue;
|
|
1587
|
+
const replacement = matchReplacementAccount(previous, previousManager, replacementManager);
|
|
1588
|
+
if (!replacement)
|
|
1589
|
+
continue;
|
|
1590
|
+
const live = replacementManager.getAccountByIndex(replacement.index);
|
|
1591
|
+
if (!live)
|
|
1592
|
+
continue;
|
|
1593
|
+
const currentScopes = new Map(workspaceModelScopes(live).map((scope) => [scope.accountId, scope]));
|
|
1594
|
+
const remapped = scopes
|
|
1595
|
+
.map((scope) => currentScopes.get(scope.accountId))
|
|
1596
|
+
.filter((scope) => Boolean(scope?.routable));
|
|
1597
|
+
// Keep an explicit empty candidate list: it is a request-local guard that
|
|
1598
|
+
// prevents fallback to a stored workspace binding when every catalog-approved
|
|
1599
|
+
// scope disappeared or became unroutable during the manager swap.
|
|
1600
|
+
workspaceCandidates.set(live.index, remapped);
|
|
1601
|
+
if (oldQuota[previousIndex] !== undefined) {
|
|
1602
|
+
subscriptionQuotaByAccount[live.index] = oldQuota[previousIndex];
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
};
|
|
1507
1606
|
const resetCreditState = state.nativeOpenai && isResponsesRequest ? await cachedRead(state, "reset-credits", loadResetCreditState).catch(() => null) : null;
|
|
1508
1607
|
const quotaForScope = (account, scope) => {
|
|
1509
1608
|
// Scoped observations belong to this credential and workspace; legacy entries only describe the binding.
|
|
@@ -1517,8 +1616,65 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1517
1616
|
// A stored `switch` pin is a hard constraint in native mode too (#702), not a
|
|
1518
1617
|
// preference: it fails with codex_pinned_account_unavailable rather than
|
|
1519
1618
|
// silently serving from another account.
|
|
1520
|
-
|
|
1619
|
+
let pinnedIndex = state.forcedAccountIndex ?? storageMeta.pinnedAccountIndex;
|
|
1521
1620
|
const isPinned = typeof pinnedIndex === "number";
|
|
1621
|
+
let pinnedIdentityAccount = isPinned && typeof pinnedIndex === "number"
|
|
1622
|
+
? accountManager.getAccountByIndex(pinnedIndex) ?? null
|
|
1623
|
+
: null;
|
|
1624
|
+
let pinnedIdentityMissing = false;
|
|
1625
|
+
const matchPinnedReplacementAccount = (previousManager, previous, replacementManager) => {
|
|
1626
|
+
const replacements = replacementManager.getAccountsSnapshot();
|
|
1627
|
+
// A persisted record id is authoritative. Legacy rows receive a derived
|
|
1628
|
+
// in-memory id whose hash includes the refresh token; an external re-login
|
|
1629
|
+
// can therefore change that derived id for the same workspace. In that legacy
|
|
1630
|
+
// case continue to the strict workspace/account fallback below.
|
|
1631
|
+
if (previous.recordId && previousManager.hasPersistedRecordId(previous)) {
|
|
1632
|
+
return uniqueAccount(replacements.filter((candidate) => candidate.recordId === previous.recordId));
|
|
1633
|
+
}
|
|
1634
|
+
const sourceIds = workspaceIds(previous);
|
|
1635
|
+
const compatible = replacements.filter((candidate) => {
|
|
1636
|
+
const candidateIds = workspaceIds(candidate);
|
|
1637
|
+
return !(sourceIds.size > 0 &&
|
|
1638
|
+
candidateIds.size > 0 &&
|
|
1639
|
+
![...sourceIds].some((id) => candidateIds.has(id)));
|
|
1640
|
+
});
|
|
1641
|
+
const sourceEmail = sanitizeEmail(previous.email);
|
|
1642
|
+
// Hard pins deliberately omit the email-only fallback used by request-local
|
|
1643
|
+
// workspace remapping. Same-email sibling logins are not interchangeable.
|
|
1644
|
+
return (uniqueAccount(compatible.filter((candidate) => candidate.refreshToken === previous.refreshToken)) ??
|
|
1645
|
+
(previous.accountId && sourceEmail
|
|
1646
|
+
? uniqueAccount(compatible.filter((candidate) => candidate.accountId === previous.accountId && sanitizeEmail(candidate.email) === sourceEmail))
|
|
1647
|
+
: null) ??
|
|
1648
|
+
(previous.accountId
|
|
1649
|
+
? uniqueAccount(compatible.filter((candidate) => {
|
|
1650
|
+
if (candidate.accountId !== previous.accountId)
|
|
1651
|
+
return false;
|
|
1652
|
+
const candidateEmail = sanitizeEmail(candidate.email);
|
|
1653
|
+
return !sourceEmail || !candidateEmail || candidateEmail === sourceEmail;
|
|
1654
|
+
}))
|
|
1655
|
+
: null));
|
|
1656
|
+
};
|
|
1657
|
+
const remapPinnedAccount = (previousManager, replacementManager) => {
|
|
1658
|
+
if (!isPinned || typeof pinnedIndex !== "number" || pinnedIdentityMissing)
|
|
1659
|
+
return;
|
|
1660
|
+
// If the pin resolved when this request began, bind the request to that
|
|
1661
|
+
// account identity rather than to an index that can be reused after a
|
|
1662
|
+
// concurrent inventory change. A pin that was already out of range keeps
|
|
1663
|
+
// the legacy numeric behavior because there is no identity to preserve.
|
|
1664
|
+
const source = pinnedIdentityAccount ?? previousManager.getAccountByIndex(pinnedIndex) ?? null;
|
|
1665
|
+
if (!source)
|
|
1666
|
+
return;
|
|
1667
|
+
const replacement = matchPinnedReplacementAccount(previousManager, source, replacementManager);
|
|
1668
|
+
const live = replacement ? replacementManager.getAccountByIndex(replacement.index) ?? null : null;
|
|
1669
|
+
if (!live) {
|
|
1670
|
+
pinnedIdentityMissing = true;
|
|
1671
|
+
pinnedIdentityAccount = source;
|
|
1672
|
+
accountSkipReasons.set(pinnedIndex, "missing");
|
|
1673
|
+
return;
|
|
1674
|
+
}
|
|
1675
|
+
pinnedIndex = live.index;
|
|
1676
|
+
pinnedIdentityAccount = live;
|
|
1677
|
+
};
|
|
1522
1678
|
let requestModelCatalog;
|
|
1523
1679
|
/** Rebuild native routing scopes after Luna is rewritten to separately metered Reserve. */
|
|
1524
1680
|
const rebuildReserveWorkspaceCandidates = async () => {
|
|
@@ -1812,7 +1968,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1812
1968
|
// so move the request to the independently metered Reserve model first.
|
|
1813
1969
|
if (isPinned &&
|
|
1814
1970
|
(context.model === "gpt-6-luna" || context.model === "gpt-5.6-luna")) {
|
|
1815
|
-
const pinnedAccount = accountManager.getAccountByIndex(pinnedIndex);
|
|
1971
|
+
const pinnedAccount = typeof pinnedIndex === "number" ? accountManager.getAccountByIndex(pinnedIndex) : null;
|
|
1816
1972
|
const now = state.now();
|
|
1817
1973
|
const familyResetAt = pinnedAccount?.rateLimitResetTimes[getQuotaKey(context.family)];
|
|
1818
1974
|
const modelResetAt = pinnedAccount?.rateLimitResetTimes[getQuotaKey(context.family, context.model)];
|
|
@@ -1854,9 +2010,10 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1854
2010
|
}
|
|
1855
2011
|
// The token bucket spreads load across a selectable pool. A pin has no
|
|
1856
2012
|
// alternative account, so exhausting that local heuristic can only reject a
|
|
1857
|
-
// request that the pinned account could serve.
|
|
1858
|
-
//
|
|
1859
|
-
|
|
2013
|
+
// request that the pinned account could serve. Transcription is not inference
|
|
2014
|
+
// and must not debit the Codex model pool or claim its inference circuit probe.
|
|
2015
|
+
// Transcription uses credential-only admission below.
|
|
2016
|
+
const bypassPoolTokenBucket = isPinned || isTranscribeRequest;
|
|
1860
2017
|
const refundConsumedPoolToken = (account) => {
|
|
1861
2018
|
if (!bypassPoolTokenBucket) {
|
|
1862
2019
|
accountManager.refundToken(account, context.family, context.model);
|
|
@@ -1948,7 +2105,9 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1948
2105
|
return "retry";
|
|
1949
2106
|
};
|
|
1950
2107
|
let runtimeSelectionIterations = 0;
|
|
1951
|
-
while ((isPinned ||
|
|
2108
|
+
while ((isPinned ||
|
|
2109
|
+
attemptedIndexes.size < accountCount ||
|
|
2110
|
+
(isTranscribeRequest && !state.nativeOpenai && accountCount === 0 && !reloadedAfterNoAccount)) &&
|
|
1952
2111
|
transientAttempts < transientAttemptLimit &&
|
|
1953
2112
|
(!isPinned ||
|
|
1954
2113
|
runtimeSelectionIterations - capacityRetries <
|
|
@@ -1982,6 +2141,10 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1982
2141
|
// calls inside `chooseAccount` are used unchanged and no lock is taken,
|
|
1983
2142
|
// so default behavior and perf are identical to before.
|
|
1984
2143
|
const selectAccount = () => {
|
|
2144
|
+
if (pinnedIdentityMissing && typeof pinnedIndex === "number") {
|
|
2145
|
+
accountSkipReasons.set(pinnedIndex, "missing");
|
|
2146
|
+
return null;
|
|
2147
|
+
}
|
|
1985
2148
|
if (context.model !== LUNA_RESERVE_MODEL) {
|
|
1986
2149
|
for (const [index, scopes] of workspaceCandidates) {
|
|
1987
2150
|
const account = accountManager.getAccountByIndex(index);
|
|
@@ -1993,13 +2156,34 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
1993
2156
|
subscriptionQuotaByAccount[index] = subscriptionQuotaPreference(quotaForScope(account, scope), state.now());
|
|
1994
2157
|
}
|
|
1995
2158
|
}
|
|
2159
|
+
const attemptedForSelection = new Set([...attemptedIndexes, ...catalogExcludedIndexes()]);
|
|
2160
|
+
if (isTranscribeRequest) {
|
|
2161
|
+
const candidate = chooseTranscriptionAccount({
|
|
2162
|
+
accountManager,
|
|
2163
|
+
attemptedIndexes: attemptedForSelection,
|
|
2164
|
+
policy: policyDecision,
|
|
2165
|
+
pinnedIndex,
|
|
2166
|
+
now: state.now(),
|
|
2167
|
+
transcriptionRateLimitUntilByRecordId: state.transcriptionRateLimitUntilByRecordId,
|
|
2168
|
+
startIndex: state.transcriptionNextAccountIndex,
|
|
2169
|
+
// Native inference preference must not pin dictation to the current
|
|
2170
|
+
// Responses account; transcription owns an independent round-robin cursor.
|
|
2171
|
+
preferredIndex: null,
|
|
2172
|
+
fallbackPinnedIndex: state.nativeOpenai ? storageMeta.pinnedAccountIndex : null,
|
|
2173
|
+
skipReasons: accountSkipReasons,
|
|
2174
|
+
});
|
|
2175
|
+
if (candidate && pinnedIndex === null) {
|
|
2176
|
+
state.transcriptionNextAccountIndex = candidate.index + 1;
|
|
2177
|
+
}
|
|
2178
|
+
return candidate;
|
|
2179
|
+
}
|
|
1996
2180
|
const result = chooseAccount({
|
|
1997
2181
|
accountManager,
|
|
1998
2182
|
sessionAffinityStore: state.sessionAffinityStore,
|
|
1999
2183
|
sessionKey: context.sessionKey,
|
|
2000
2184
|
family: context.family,
|
|
2001
2185
|
model: context.model,
|
|
2002
|
-
attemptedIndexes:
|
|
2186
|
+
attemptedIndexes: attemptedForSelection,
|
|
2003
2187
|
now: state.now(),
|
|
2004
2188
|
policy: policyDecision,
|
|
2005
2189
|
pinnedIndex,
|
|
@@ -2023,6 +2207,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2023
2207
|
? await withRoutingMutex(state.routingMutexMode, async () => {
|
|
2024
2208
|
const candidate = selectAccount();
|
|
2025
2209
|
if (candidate &&
|
|
2210
|
+
!isTranscribeRequest &&
|
|
2026
2211
|
pinnedIndex === null &&
|
|
2027
2212
|
state.schedulingStrategy !== "sequential") {
|
|
2028
2213
|
// Re-commit the cursor under the held mutex. Skipped when a
|
|
@@ -2042,7 +2227,69 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2042
2227
|
for (const index of catalogExcludedIndexes())
|
|
2043
2228
|
accountSkipReasons.set(index, "model-not-supported");
|
|
2044
2229
|
if (!selected) {
|
|
2045
|
-
if (
|
|
2230
|
+
if (isTranscribeRequest &&
|
|
2231
|
+
!state.nativeOpenai &&
|
|
2232
|
+
!reloadedAfterNoAccount) {
|
|
2233
|
+
reloadedAfterNoAccount = true;
|
|
2234
|
+
const reloadedManager = await reloadRuntimeCredentials(state);
|
|
2235
|
+
if (reloadedManager && reloadedManager !== accountManager) {
|
|
2236
|
+
const previousManager = accountManager;
|
|
2237
|
+
remapPinnedAccount(previousManager, reloadedManager);
|
|
2238
|
+
accountManager = reloadedManager;
|
|
2239
|
+
accountCount = accountManager.getAccountCount();
|
|
2240
|
+
transientAttemptLimit = Math.max(1, Math.min(accountCount, state.maxRuntimeAccountAttempts));
|
|
2241
|
+
if (runtimePolicyState) {
|
|
2242
|
+
try {
|
|
2243
|
+
const reloadedPolicyDecision = await evaluateRuntimePolicy({
|
|
2244
|
+
state: runtimePolicyState,
|
|
2245
|
+
accounts: accountManager.getAccountsSnapshot(),
|
|
2246
|
+
model: context.model,
|
|
2247
|
+
now: state.now(),
|
|
2248
|
+
});
|
|
2249
|
+
policyDecision = reloadedPolicyDecision;
|
|
2250
|
+
mutateRuntimeObservabilitySnapshot((snapshot) => {
|
|
2251
|
+
snapshot.policyBlockedIndexes = [...reloadedPolicyDecision.blockedAccountIndexes];
|
|
2252
|
+
snapshot.policyBlockedReasons = Object.fromEntries([...reloadedPolicyDecision.blockedAccountIndexes].map((index) => [String(index), "policy-blocked"]));
|
|
2253
|
+
});
|
|
2254
|
+
if (!reloadedPolicyDecision.allowed) {
|
|
2255
|
+
await usageRecorder.record({
|
|
2256
|
+
outcome: "blocked",
|
|
2257
|
+
statusCode: reloadedPolicyDecision.statusCode,
|
|
2258
|
+
errorCode: reloadedPolicyDecision.errorCode,
|
|
2259
|
+
});
|
|
2260
|
+
writeJson(res, reloadedPolicyDecision.statusCode, {
|
|
2261
|
+
error: {
|
|
2262
|
+
message: "Runtime policy blocked this local request.",
|
|
2263
|
+
code: reloadedPolicyDecision.errorCode ?? "policy_blocked",
|
|
2264
|
+
reasons: reloadedPolicyDecision.reasons,
|
|
2265
|
+
},
|
|
2266
|
+
});
|
|
2267
|
+
return;
|
|
2268
|
+
}
|
|
2269
|
+
}
|
|
2270
|
+
catch (error) {
|
|
2271
|
+
state.status.lastError = error instanceof Error ? error.message : String(error);
|
|
2272
|
+
await usageRecorder.record({
|
|
2273
|
+
outcome: "failure",
|
|
2274
|
+
statusCode: HTTP_STATUS.SERVICE_UNAVAILABLE,
|
|
2275
|
+
errorCode: "runtime_policy_unavailable",
|
|
2276
|
+
});
|
|
2277
|
+
writeJson(res, HTTP_STATUS.SERVICE_UNAVAILABLE, {
|
|
2278
|
+
error: {
|
|
2279
|
+
message: "Runtime policy could not be loaded for this local request.",
|
|
2280
|
+
code: "runtime_policy_unavailable",
|
|
2281
|
+
},
|
|
2282
|
+
});
|
|
2283
|
+
return;
|
|
2284
|
+
}
|
|
2285
|
+
}
|
|
2286
|
+
accountSkipReasons.clear();
|
|
2287
|
+
attemptedIndexes.clear();
|
|
2288
|
+
continue;
|
|
2289
|
+
}
|
|
2290
|
+
}
|
|
2291
|
+
if (!isTranscribeRequest &&
|
|
2292
|
+
!reloadedAfterNoAccount &&
|
|
2046
2293
|
!isPinned &&
|
|
2047
2294
|
accountCount > 0 &&
|
|
2048
2295
|
exhaustionReason === "no-account" &&
|
|
@@ -2080,7 +2327,8 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2080
2327
|
attemptedIndexes.add(selected.index);
|
|
2081
2328
|
const quotaScheduleKey = buildQuotaScheduleKey(selected, context.family, context.model);
|
|
2082
2329
|
const preemptiveDeferral = state.preemptiveQuotaScheduler.getDeferral(quotaScheduleKey, state.now());
|
|
2083
|
-
if (
|
|
2330
|
+
if (!isTranscribeRequest &&
|
|
2331
|
+
context.model !== LUNA_RESERVE_MODEL &&
|
|
2084
2332
|
preemptiveDeferral.defer &&
|
|
2085
2333
|
preemptiveDeferral.waitMs > 0) {
|
|
2086
2334
|
accountSkipReasons.set(selected.index, preemptiveDeferral.reason ?? "quota-near-exhaustion");
|
|
@@ -2092,12 +2340,13 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2092
2340
|
noteRotation();
|
|
2093
2341
|
continue;
|
|
2094
2342
|
}
|
|
2095
|
-
//
|
|
2096
|
-
//
|
|
2097
|
-
//
|
|
2098
|
-
//
|
|
2099
|
-
|
|
2100
|
-
|
|
2343
|
+
// Inference consumes both the pool token and the race-safe circuit probe.
|
|
2344
|
+
// Transcription deliberately bypasses both: its credential-only selector
|
|
2345
|
+
// already enforced disabled/workspace/auth/policy gates, and claiming the
|
|
2346
|
+
// inference probe here could strand a half-open account after dictation.
|
|
2347
|
+
const admission = isTranscribeRequest
|
|
2348
|
+
? { ok: true }
|
|
2349
|
+
: accountManager.consumeTokenWithReason(selected, context.family, context.model, { bypassTokenBucket: bypassPoolTokenBucket });
|
|
2101
2350
|
if (!admission.ok) {
|
|
2102
2351
|
accountSkipReasons.set(selected.index, admission.reason);
|
|
2103
2352
|
exhaustionReason = "rate-limit";
|
|
@@ -2122,8 +2371,75 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2122
2371
|
tokenRefreshSkewMs: state.tokenRefreshSkewMs,
|
|
2123
2372
|
tokenInvalidationCooldownMs: state.tokenInvalidationCooldownMs,
|
|
2124
2373
|
});
|
|
2374
|
+
// A credential reload may have published a replacement manager while this
|
|
2375
|
+
// request was awaiting refresh. Keep the refreshed account and its owner
|
|
2376
|
+
// together for every later mutation/persist; account indexes are local to
|
|
2377
|
+
// one manager and can shift when rows are removed or reordered.
|
|
2378
|
+
if (refreshed.accountManager !== accountManager) {
|
|
2379
|
+
const previousManager = accountManager;
|
|
2380
|
+
remapRequestWorkspaceState(previousManager, refreshed.accountManager);
|
|
2381
|
+
remapPinnedAccount(previousManager, refreshed.accountManager);
|
|
2382
|
+
accountManager = refreshed.accountManager;
|
|
2383
|
+
accountCount = accountManager.getAccountCount();
|
|
2384
|
+
transientAttemptLimit = Math.max(1, Math.min(accountCount, state.maxRuntimeAccountAttempts));
|
|
2385
|
+
if (runtimePolicyState) {
|
|
2386
|
+
try {
|
|
2387
|
+
const refreshedPolicyDecision = await evaluateRuntimePolicy({
|
|
2388
|
+
state: runtimePolicyState,
|
|
2389
|
+
accounts: accountManager.getAccountsSnapshot(),
|
|
2390
|
+
model: context.model,
|
|
2391
|
+
now: state.now(),
|
|
2392
|
+
});
|
|
2393
|
+
policyDecision = refreshedPolicyDecision;
|
|
2394
|
+
mutateRuntimeObservabilitySnapshot((snapshot) => {
|
|
2395
|
+
snapshot.policyBlockedIndexes = [...refreshedPolicyDecision.blockedAccountIndexes];
|
|
2396
|
+
snapshot.policyBlockedReasons = Object.fromEntries([...refreshedPolicyDecision.blockedAccountIndexes].map((index) => [String(index), "policy-blocked"]));
|
|
2397
|
+
});
|
|
2398
|
+
if (!refreshedPolicyDecision.allowed) {
|
|
2399
|
+
await usageRecorder.record({
|
|
2400
|
+
outcome: "blocked",
|
|
2401
|
+
statusCode: refreshedPolicyDecision.statusCode,
|
|
2402
|
+
errorCode: refreshedPolicyDecision.errorCode,
|
|
2403
|
+
});
|
|
2404
|
+
writeJson(res, refreshedPolicyDecision.statusCode, {
|
|
2405
|
+
error: {
|
|
2406
|
+
message: "Runtime policy blocked this local request.",
|
|
2407
|
+
code: refreshedPolicyDecision.errorCode ?? "policy_blocked",
|
|
2408
|
+
reasons: refreshedPolicyDecision.reasons,
|
|
2409
|
+
},
|
|
2410
|
+
});
|
|
2411
|
+
return;
|
|
2412
|
+
}
|
|
2413
|
+
}
|
|
2414
|
+
catch (error) {
|
|
2415
|
+
state.status.lastError = error instanceof Error ? error.message : String(error);
|
|
2416
|
+
await usageRecorder.record({
|
|
2417
|
+
outcome: "failure",
|
|
2418
|
+
statusCode: HTTP_STATUS.SERVICE_UNAVAILABLE,
|
|
2419
|
+
errorCode: "runtime_policy_unavailable",
|
|
2420
|
+
});
|
|
2421
|
+
writeJson(res, HTTP_STATUS.SERVICE_UNAVAILABLE, {
|
|
2422
|
+
error: {
|
|
2423
|
+
message: "Runtime policy could not be loaded for this local request.",
|
|
2424
|
+
code: "runtime_policy_unavailable",
|
|
2425
|
+
},
|
|
2426
|
+
});
|
|
2427
|
+
return;
|
|
2428
|
+
}
|
|
2429
|
+
}
|
|
2430
|
+
accountSkipReasons.clear();
|
|
2431
|
+
attemptedIndexes.clear();
|
|
2432
|
+
}
|
|
2125
2433
|
if (!refreshed.ok) {
|
|
2126
2434
|
refundConsumedPoolToken(selected);
|
|
2435
|
+
if (refreshed.selectionRestart) {
|
|
2436
|
+
// The credential row vanished while refresh was in flight. The manager
|
|
2437
|
+
// swap above already cleared request-local attempt state; restart selection
|
|
2438
|
+
// without charging a transient auth failure to the replacement pool.
|
|
2439
|
+
exhaustionReason = "no-account";
|
|
2440
|
+
capabilityRejectionIsLatest = false;
|
|
2441
|
+
continue;
|
|
2442
|
+
}
|
|
2127
2443
|
// No accountSkipReasons write here: ensureFreshAccessToken always
|
|
2128
2444
|
// applies an auth cooldown before returning !ok, so the next
|
|
2129
2445
|
// selection pass overwrites whatever this set with
|
|
@@ -2157,7 +2473,12 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2157
2473
|
noteRotation();
|
|
2158
2474
|
continue;
|
|
2159
2475
|
}
|
|
2160
|
-
|
|
2476
|
+
let pendingScopes = workspaceCandidates.get(refreshed.account.index);
|
|
2477
|
+
if (isTranscribeRequest) {
|
|
2478
|
+
pendingScopes = workspaceModelScopes(refreshed.account)
|
|
2479
|
+
.filter((scope) => scope.routable)
|
|
2480
|
+
.sort((a, b) => Number(b.selected) - Number(a.selected) || Number(b.bound) - Number(a.bound));
|
|
2481
|
+
}
|
|
2161
2482
|
// Revalidate enabled scopes after token refresh; never write a per-request choice to the account.
|
|
2162
2483
|
const enabledScopes = new Set(workspaceModelScopes(refreshed.account).filter(s => s.routable).map(s => s.id));
|
|
2163
2484
|
while (pendingScopes?.length && !enabledScopes.has(pendingScopes[0]?.id ?? ""))
|
|
@@ -2225,6 +2546,26 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2225
2546
|
}
|
|
2226
2547
|
const accountIdentity = accountIdentityFromAccount(refreshed.account, state.now());
|
|
2227
2548
|
recordLastRuntimeAccount(state.status, accountIdentity, accountId);
|
|
2549
|
+
// A credential/inventory reload can publish while this request is waiting
|
|
2550
|
+
// on upstream. Resolve response-derived health/auth mutations lazily onto
|
|
2551
|
+
// the latest owner so late transport/429/401/5xx/success state cannot land
|
|
2552
|
+
// only on a superseded manager. If the row vanished entirely, retain the
|
|
2553
|
+
// captured object for this request; there is no replacement row to penalize.
|
|
2554
|
+
const responseCredential = {
|
|
2555
|
+
access: refreshed.accessToken,
|
|
2556
|
+
refreshToken: refreshed.account.refreshToken,
|
|
2557
|
+
};
|
|
2558
|
+
const responseStateTarget = () => resolveRuntimeRefreshTarget(accountManager, refreshed.account) ?? {
|
|
2559
|
+
accountManager,
|
|
2560
|
+
account: refreshed.account,
|
|
2561
|
+
};
|
|
2562
|
+
const responseAuthStateTarget = () => {
|
|
2563
|
+
const target = responseStateTarget();
|
|
2564
|
+
return target.account.access === responseCredential.access &&
|
|
2565
|
+
target.account.refreshToken === responseCredential.refreshToken
|
|
2566
|
+
? target
|
|
2567
|
+
: null;
|
|
2568
|
+
};
|
|
2228
2569
|
const outboundHeaders = createOutboundHeaders(context.headers, refreshed.account, refreshed.accessToken, accountId);
|
|
2229
2570
|
let upstream;
|
|
2230
2571
|
// Abort the in-flight upstream fetch when the client disconnects
|
|
@@ -2314,16 +2655,18 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2314
2655
|
return;
|
|
2315
2656
|
}
|
|
2316
2657
|
// A timeout may occur after generation; do not retry within this request.
|
|
2317
|
-
if (isImageRequest) {
|
|
2658
|
+
if (isImageRequest || isTranscribeRequest) {
|
|
2659
|
+
const operation = isImageRequest ? "image" : "transcribe";
|
|
2660
|
+
const errorCode = `${operation}_upstream_transport_error`;
|
|
2318
2661
|
writeJson(res, 502, {
|
|
2319
2662
|
error: {
|
|
2320
|
-
code:
|
|
2321
|
-
message: "Image upstream transport failed; not retried by the proxy
|
|
2663
|
+
code: errorCode,
|
|
2664
|
+
message: `${isImageRequest ? "Image" : "Transcription"} upstream transport failed; not retried by the proxy.`,
|
|
2322
2665
|
},
|
|
2323
2666
|
});
|
|
2324
2667
|
await usageRecorder.record({
|
|
2325
2668
|
outcome: "failure", statusCode: 502,
|
|
2326
|
-
errorCode
|
|
2669
|
+
errorCode, account: refreshed.account,
|
|
2327
2670
|
});
|
|
2328
2671
|
return;
|
|
2329
2672
|
}
|
|
@@ -2339,8 +2682,9 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2339
2682
|
// exhaustion 503 a non-zero `retry_after_ms` to back the client off
|
|
2340
2683
|
// with (`getMinWaitTimeForFamily` returns 0 while any account is
|
|
2341
2684
|
// still selectable).
|
|
2342
|
-
|
|
2343
|
-
accountManager.
|
|
2685
|
+
const transportState = responseStateTarget();
|
|
2686
|
+
transportState.accountManager.markAccountCoolingDown(transportState.account, state.networkErrorCooldownMs, "network-error");
|
|
2687
|
+
transportState.accountManager.saveToDiskDebounced();
|
|
2344
2688
|
accountSkipReasons.set(refreshed.account.index, "network-error");
|
|
2345
2689
|
exhaustionReason = "network-error";
|
|
2346
2690
|
capabilityRejectionIsLatest = false;
|
|
@@ -2371,7 +2715,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2371
2715
|
}
|
|
2372
2716
|
};
|
|
2373
2717
|
const quotaSnapshot = readQuotaSchedulerSnapshot(upstream.headers, upstream.status, state.now());
|
|
2374
|
-
if (quotaSnapshot && context.model !== LUNA_RESERVE_MODEL) {
|
|
2718
|
+
if (quotaSnapshot && !isTranscribeRequest && context.model !== LUNA_RESERVE_MODEL) {
|
|
2375
2719
|
observeQuota(quotaSnapshot, upstream.headers.get("x-codex-plan-type") ?? undefined);
|
|
2376
2720
|
}
|
|
2377
2721
|
if (isResponsesRequest && context.model && [400, 403, 404].includes(upstream.status)) {
|
|
@@ -2416,6 +2760,30 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2416
2760
|
}
|
|
2417
2761
|
if (upstream.status === HTTP_STATUS.TOO_MANY_REQUESTS) {
|
|
2418
2762
|
const bodyText = await readErrorBody(upstream, state.streamStallTimeoutMs);
|
|
2763
|
+
if (isTranscribeRequest) {
|
|
2764
|
+
const now = state.now();
|
|
2765
|
+
const retryAfterMs = parseRetryAfterHeaderMs(upstream.headers, now) ??
|
|
2766
|
+
parseRetryAfterBodyMs(bodyText, now) ??
|
|
2767
|
+
60_000;
|
|
2768
|
+
for (const [recordId, until] of state.transcriptionRateLimitUntilByRecordId) {
|
|
2769
|
+
if (until <= now)
|
|
2770
|
+
state.transcriptionRateLimitUntilByRecordId.delete(recordId);
|
|
2771
|
+
}
|
|
2772
|
+
if (refreshed.account.recordId) {
|
|
2773
|
+
const boundedDelayMs = Math.min(MAX_RATE_LIMIT_DELAY_MS, Math.max(1_000, Math.floor(retryAfterMs)));
|
|
2774
|
+
const existingUntil = state.transcriptionRateLimitUntilByRecordId.get(refreshed.account.recordId) ?? 0;
|
|
2775
|
+
state.transcriptionRateLimitUntilByRecordId.set(refreshed.account.recordId, Math.max(existingUntil, now + boundedDelayMs));
|
|
2776
|
+
}
|
|
2777
|
+
res.writeHead(upstream.status, responseHeadersForClient(upstream.headers));
|
|
2778
|
+
res.end(bodyText);
|
|
2779
|
+
await usageRecorder.record({
|
|
2780
|
+
outcome: "failure",
|
|
2781
|
+
statusCode: upstream.status,
|
|
2782
|
+
errorCode: extractErrorCodeFromBody(bodyText) ?? "transcribe_rate_limited",
|
|
2783
|
+
account: refreshed.account,
|
|
2784
|
+
});
|
|
2785
|
+
return;
|
|
2786
|
+
}
|
|
2419
2787
|
// Keep the upstream HINT separate from the 60s fallback. Passing the
|
|
2420
2788
|
// synthesized default into the capacity wait would make every
|
|
2421
2789
|
// hint-less capacity 429 sleep a full minute and leave the 2s/5s/15s
|
|
@@ -2449,16 +2817,17 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2449
2817
|
}
|
|
2450
2818
|
// Every 429 still drains this account for its retry window, but only an
|
|
2451
2819
|
// explicit subscription-usage signal is persisted as quota exhaustion.
|
|
2452
|
-
|
|
2453
|
-
accountManager.
|
|
2820
|
+
const rateLimitState = responseStateTarget();
|
|
2821
|
+
rateLimitState.accountManager.recordRateLimit(rateLimitState.account, context.family, context.model);
|
|
2822
|
+
rateLimitState.accountManager.markRateLimitedWithReason(rateLimitState.account, retryAfterMs, context.family, subscriptionQuotaExceeded ? "quota" : "unknown", context.model);
|
|
2454
2823
|
if (subscriptionQuotaExceeded &&
|
|
2455
2824
|
(context.model === "gpt-6-luna" || context.model === "gpt-5.6-luna")) {
|
|
2456
2825
|
const confirmedKey = confirmedQuotaExhaustionKey(context.family, context.model);
|
|
2457
2826
|
const confirmedResetAt = state.now() +
|
|
2458
2827
|
Math.min(Math.max(0, Math.floor(retryAfterMs)), MAX_RATE_LIMIT_DELAY_MS);
|
|
2459
|
-
|
|
2828
|
+
rateLimitState.account.rateLimitResetTimes[confirmedKey] = Math.max(rateLimitState.account.rateLimitResetTimes[confirmedKey] ?? 0, confirmedResetAt);
|
|
2460
2829
|
}
|
|
2461
|
-
accountManager.saveToDiskDebounced();
|
|
2830
|
+
rateLimitState.accountManager.saveToDiskDebounced();
|
|
2462
2831
|
if (!modelAtCapacity && subscriptionQuotaExceeded && switchRequestToLunaReserve(context)) {
|
|
2463
2832
|
usageRecorder = createUsageRecorderForModel(context.model);
|
|
2464
2833
|
if (runtimePolicyState) {
|
|
@@ -2506,8 +2875,9 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2506
2875
|
const errorCode = extractErrorCodeFromBody(bodyText);
|
|
2507
2876
|
if (isWorkspaceDisabledError(upstream.status, errorCode, bodyText)) {
|
|
2508
2877
|
if (requestScope && refreshed.account.workspaces?.length) {
|
|
2509
|
-
|
|
2510
|
-
|
|
2878
|
+
const workspaceState = responseStateTarget();
|
|
2879
|
+
if (workspaceState.accountManager.disableWorkspace(workspaceState.account, requestScope.accountId))
|
|
2880
|
+
workspaceState.accountManager.saveToDiskDebounced();
|
|
2511
2881
|
pendingScopes?.shift();
|
|
2512
2882
|
refundConsumedPoolToken(refreshed.account);
|
|
2513
2883
|
if (pendingScopes?.length)
|
|
@@ -2518,13 +2888,13 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2518
2888
|
noteRotation();
|
|
2519
2889
|
continue;
|
|
2520
2890
|
}
|
|
2521
|
-
const
|
|
2522
|
-
|
|
2891
|
+
const disabledState = responseStateTarget();
|
|
2892
|
+
const accountWasEnabled = disabledState.account.enabled !== false;
|
|
2523
2893
|
refundConsumedPoolToken(refreshed.account);
|
|
2524
2894
|
if (accountWasEnabled) {
|
|
2525
|
-
accountManager.recordFailure(
|
|
2526
|
-
accountManager.setAccountEnabled(
|
|
2527
|
-
accountManager.saveToDiskDebounced();
|
|
2895
|
+
disabledState.accountManager.recordFailure(disabledState.account, context.family, context.model);
|
|
2896
|
+
disabledState.accountManager.setAccountEnabled(disabledState.account.index, false);
|
|
2897
|
+
disabledState.accountManager.saveToDiskDebounced();
|
|
2528
2898
|
}
|
|
2529
2899
|
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
2530
2900
|
exhaustionReason = "deactivated";
|
|
@@ -2601,20 +2971,25 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2601
2971
|
if (upstream.status === HTTP_STATUS.UNAUTHORIZED) {
|
|
2602
2972
|
const bodyText = await readErrorBody(upstream, state.streamStallTimeoutMs);
|
|
2603
2973
|
refundConsumedPoolToken(refreshed.account);
|
|
2604
|
-
|
|
2974
|
+
const authState = responseAuthStateTarget();
|
|
2975
|
+
if (authState && !isTranscribeRequest) {
|
|
2976
|
+
authState.accountManager.recordFailure(authState.account, context.family, context.model);
|
|
2977
|
+
}
|
|
2605
2978
|
if (isTokenInvalidationError(bodyText)) {
|
|
2606
2979
|
// The upstream explicitly revoked this OAuth token. Applying a long
|
|
2607
2980
|
// cooldown prevents cascade invalidation: rapidly presenting each
|
|
2608
2981
|
// account's token from the same IP triggers OpenAI's anti-abuse
|
|
2609
2982
|
// detection and invalidates them in sequence. Return the 401 directly
|
|
2610
2983
|
// rather than rotating so the client can prompt for re-login.
|
|
2611
|
-
|
|
2612
|
-
|
|
2984
|
+
if (authState) {
|
|
2985
|
+
authState.accountManager.markAuthInvalidated(authState.account, extractErrorCodeFromBody(bodyText) ?? "token_invalidated");
|
|
2986
|
+
applyMonotonicAuthCooldown(authState.accountManager, authState.account, state.tokenInvalidationCooldownMs);
|
|
2987
|
+
// The invalidation marker must be durable before returning the 401.
|
|
2988
|
+
// A delayed save would allow a proxy restart to reload and route the
|
|
2989
|
+
// revoked account again.
|
|
2990
|
+
await authState.accountManager.saveToDisk();
|
|
2991
|
+
}
|
|
2613
2992
|
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
2614
|
-
// The invalidation marker must be durable before returning the 401.
|
|
2615
|
-
// A delayed save would allow a proxy restart to reload and route the
|
|
2616
|
-
// revoked account again.
|
|
2617
|
-
await accountManager.saveToDisk();
|
|
2618
2993
|
// Emit the same machine-readable shape as the refresh-failure path
|
|
2619
2994
|
// (code: "token_invalidated") instead of forwarding the raw upstream
|
|
2620
2995
|
// body, so the client contract is consistent across both vectors.
|
|
@@ -2630,8 +3005,21 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2630
3005
|
});
|
|
2631
3006
|
return;
|
|
2632
3007
|
}
|
|
2633
|
-
|
|
2634
|
-
|
|
3008
|
+
if (isTranscribeRequest) {
|
|
3009
|
+
res.writeHead(upstream.status, responseHeadersForClient(upstream.headers));
|
|
3010
|
+
res.end(bodyText);
|
|
3011
|
+
await usageRecorder.record({
|
|
3012
|
+
outcome: "failure",
|
|
3013
|
+
statusCode: upstream.status,
|
|
3014
|
+
errorCode: extractErrorCodeFromBody(bodyText) ?? "transcribe_unauthorized",
|
|
3015
|
+
account: refreshed.account,
|
|
3016
|
+
});
|
|
3017
|
+
return;
|
|
3018
|
+
}
|
|
3019
|
+
if (authState) {
|
|
3020
|
+
applyMonotonicAuthCooldown(authState.accountManager, authState.account, DEFAULT_AUTH_FAILURE_COOLDOWN_MS);
|
|
3021
|
+
authState.accountManager.saveToDiskDebounced();
|
|
3022
|
+
}
|
|
2635
3023
|
exhaustionReason = "auth-failure";
|
|
2636
3024
|
capabilityRejectionIsLatest = false;
|
|
2637
3025
|
transientAttempts += 1;
|
|
@@ -2640,6 +3028,19 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2640
3028
|
noteRotation();
|
|
2641
3029
|
continue;
|
|
2642
3030
|
}
|
|
3031
|
+
// Dictation audio may already have been accepted before any error response.
|
|
3032
|
+
// Do not replay it across accounts or let its endpoint-specific status
|
|
3033
|
+
// mutate Codex inference health/quota state. Explicit token invalidation
|
|
3034
|
+
// and workspace-disable handling above remain account-wide signals.
|
|
3035
|
+
if (isTranscribeRequest && upstream.status >= 400) {
|
|
3036
|
+
const forwarded = await forwardStreamingResponse(upstream, res, state.status, () => undefined, state.streamStallTimeoutMs);
|
|
3037
|
+
await usageRecorder.record({
|
|
3038
|
+
outcome: "failure", statusCode: upstream.status,
|
|
3039
|
+
errorCode: forwarded ? "transcribe_upstream_error" : "stream_forward_failed",
|
|
3040
|
+
account: refreshed.account,
|
|
3041
|
+
});
|
|
3042
|
+
return;
|
|
3043
|
+
}
|
|
2643
3044
|
// Forward image validation/moderation/5xx errors without cross-account replay.
|
|
2644
3045
|
// Explicit 429 quota and 401 auth handling above still use the existing pool.
|
|
2645
3046
|
if (isImageRequest && upstream.status >= 400) {
|
|
@@ -2670,9 +3071,10 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2670
3071
|
continue;
|
|
2671
3072
|
}
|
|
2672
3073
|
refundConsumedPoolToken(refreshed.account);
|
|
2673
|
-
|
|
2674
|
-
accountManager.
|
|
2675
|
-
accountManager.
|
|
3074
|
+
const serverState = responseStateTarget();
|
|
3075
|
+
serverState.accountManager.recordFailure(serverState.account, context.family, context.model);
|
|
3076
|
+
serverState.accountManager.markAccountCoolingDown(serverState.account, state.serverErrorCooldownMs, "server-error");
|
|
3077
|
+
serverState.accountManager.saveToDiskDebounced();
|
|
2676
3078
|
exhaustionReason = "server-error";
|
|
2677
3079
|
capabilityRejectionIsLatest = false;
|
|
2678
3080
|
transientAttempts += 1;
|
|
@@ -2707,25 +3109,28 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2707
3109
|
// "rate-limited") lingers on disk until an explicit runtime reset and
|
|
2708
3110
|
// the forecast keeps reporting this working account as unavailable.
|
|
2709
3111
|
// No-op when no reason is recorded, so the hot path stays write-free.
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
state.sessionAffinityStore?.remember(context.sessionKey, refreshed.account.index, state.now());
|
|
2723
|
-
if (refreshed.account.index !== state.lastGlobalAccountIndex) {
|
|
2724
|
-
state.lastGlobalAccountIndex = refreshed.account.index;
|
|
3112
|
+
if (!isTranscribeRequest) {
|
|
3113
|
+
const quotaDeferral = state.preemptiveQuotaScheduler.getDeferral(quotaScheduleKey, state.now());
|
|
3114
|
+
const nearExhaustionWaitMs = context.model === LUNA_RESERVE_MODEL
|
|
3115
|
+
? 0
|
|
3116
|
+
: quotaDeferral.defer
|
|
3117
|
+
? quotaDeferral.waitMs
|
|
3118
|
+
: 0;
|
|
3119
|
+
if (nearExhaustionWaitMs > 0) {
|
|
3120
|
+
const quotaState = responseStateTarget();
|
|
3121
|
+
quotaState.accountManager.markRateLimitedWithReason(quotaState.account, nearExhaustionWaitMs, context.family, "quota", context.model);
|
|
3122
|
+
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
3123
|
+
quotaState.accountManager.saveToDiskDebounced();
|
|
2725
3124
|
}
|
|
2726
|
-
|
|
3125
|
+
else {
|
|
3126
|
+
state.sessionAffinityStore?.remember(context.sessionKey, refreshed.account.index, state.now());
|
|
3127
|
+
if (refreshed.account.index !== state.lastGlobalAccountIndex) {
|
|
3128
|
+
state.lastGlobalAccountIndex = refreshed.account.index;
|
|
3129
|
+
}
|
|
3130
|
+
state.lastGlobalSwitchAt = state.now();
|
|
3131
|
+
}
|
|
3132
|
+
await persistRuntimeActiveAccount(accountManager, refreshed.account, context.family, isPinned && refreshed.account.index === pinnedIndex, state.schedulingStrategy, state.nativeOpenai);
|
|
2727
3133
|
}
|
|
2728
|
-
await persistRuntimeActiveAccount(accountManager, refreshed.account, context.family, isPinned && refreshed.account.index === pinnedIndex, state.schedulingStrategy, state.nativeOpenai);
|
|
2729
3134
|
// Recover the upstream token counts as the body streams past. Without
|
|
2730
3135
|
// them every ledger row lands with all-zero tokens and a zero cost, so
|
|
2731
3136
|
// evaluateBudgetGuard compares `0 >= limit` for maxTokens/maxCostUsd
|
|
@@ -2735,7 +3140,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2735
3140
|
contentType: upstream.headers.get("content-type"),
|
|
2736
3141
|
onEvent: (event) => {
|
|
2737
3142
|
responseOutcome.observe(event);
|
|
2738
|
-
if (context.model === LUNA_RESERVE_MODEL)
|
|
3143
|
+
if (isTranscribeRequest || context.model === LUNA_RESERVE_MODEL)
|
|
2739
3144
|
return;
|
|
2740
3145
|
const snapshot = readSubscriptionQuotaEvent(event, state.now());
|
|
2741
3146
|
if (!snapshot)
|
|
@@ -2749,23 +3154,35 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2749
3154
|
let streamErrored = false;
|
|
2750
3155
|
const forwarded = await forwardStreamingResponse(upstream, res, state.status, () => {
|
|
2751
3156
|
streamErrored = true;
|
|
3157
|
+
if (isTranscribeRequest)
|
|
3158
|
+
return;
|
|
2752
3159
|
// Deliberately NOT the pre-header transport policy above,
|
|
2753
3160
|
// which skips recordFailure (#677). By this point the
|
|
2754
3161
|
// upstream accepted the request and began responding, so a
|
|
2755
3162
|
// broken stream is a signal about THIS account's session
|
|
2756
3163
|
// (upstream tearing it down) rather than about the shared
|
|
2757
3164
|
// network path, and it should still credit the breaker.
|
|
2758
|
-
|
|
2759
|
-
accountManager.
|
|
3165
|
+
const streamState = responseStateTarget();
|
|
3166
|
+
streamState.accountManager.recordFailure(streamState.account, context.family, context.model);
|
|
3167
|
+
streamState.accountManager.markAccountCoolingDown(streamState.account, state.networkErrorCooldownMs, "network-error");
|
|
2760
3168
|
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
2761
|
-
accountManager.saveToDiskDebounced();
|
|
3169
|
+
streamState.accountManager.saveToDiskDebounced();
|
|
2762
3170
|
}, state.streamStallTimeoutMs, usageScanner.push, { ...(budgetAdvisory.level === "soft" ? buildContextBudgetHeaders(budgetAdvisory) : {}), ...(state.catalogEtag ? { "x-models-etag": state.catalogEtag } : {}) }, () => { usageScanner.result(); return responseOutcome.finish(); });
|
|
2763
3171
|
// The client went away mid-stream: no upstream error, and the response was
|
|
2764
3172
|
// neither completed nor failed. That says nothing about the account.
|
|
2765
3173
|
const clientDisconnected = !forwarded && !streamErrored && !res.writableEnded;
|
|
2766
3174
|
if (forwarded && upstream.ok) {
|
|
2767
|
-
|
|
2768
|
-
|
|
3175
|
+
if (isTranscribeRequest && refreshed.account.recordId) {
|
|
3176
|
+
const until = state.transcriptionRateLimitUntilByRecordId.get(refreshed.account.recordId);
|
|
3177
|
+
if (typeof until === "number" && until <= state.now()) {
|
|
3178
|
+
state.transcriptionRateLimitUntilByRecordId.delete(refreshed.account.recordId);
|
|
3179
|
+
}
|
|
3180
|
+
}
|
|
3181
|
+
if (!isTranscribeRequest) {
|
|
3182
|
+
const successState = responseStateTarget();
|
|
3183
|
+
successState.accountManager.recordSuccess(successState.account, context.family, context.model);
|
|
3184
|
+
recordRuntimeAccountRecovery(successState.account.index);
|
|
3185
|
+
}
|
|
2769
3186
|
}
|
|
2770
3187
|
else if (!clientDisconnected) {
|
|
2771
3188
|
state.sessionAffinityStore?.forgetSession(context.sessionKey);
|
|
@@ -2793,7 +3210,7 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2793
3210
|
// it here inflates the measurement by this turn's thinking budget
|
|
2794
3211
|
// against a deliberately tight threshold. `outputTokens` is already
|
|
2795
3212
|
// reasoning-stripped (lib/usage/usage-extraction.ts).
|
|
2796
|
-
if (usageTokens && context.stableSessionKey && context.model) {
|
|
3213
|
+
if (!isTranscribeRequest && usageTokens && context.stableSessionKey && context.model) {
|
|
2797
3214
|
state.contextBudgetGuard.update(context.stableSessionKey, {
|
|
2798
3215
|
model: context.model,
|
|
2799
3216
|
contextTokens: usageTokens.inputTokens + usageTokens.outputTokens,
|
|
@@ -2841,9 +3258,11 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2841
3258
|
// null reason indicates a forecast/runtime state desync (the pinned
|
|
2842
3259
|
// account was selected but no skip reason was recorded) — see #486.
|
|
2843
3260
|
if (isPinned) {
|
|
2844
|
-
const pinnedAccount =
|
|
2845
|
-
?
|
|
2846
|
-
:
|
|
3261
|
+
const pinnedAccount = pinnedIdentityMissing
|
|
3262
|
+
? null
|
|
3263
|
+
: typeof pinnedIndex === "number"
|
|
3264
|
+
? accountManager.getAccountByIndex(pinnedIndex)
|
|
3265
|
+
: null;
|
|
2847
3266
|
const pinnedSkipReason = typeof pinnedIndex === "number"
|
|
2848
3267
|
? accountSkipReasons.get(pinnedIndex) ?? null
|
|
2849
3268
|
: null;
|
|
@@ -2877,23 +3296,44 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2877
3296
|
// them and report a rate-limited pin as bounded by something else.
|
|
2878
3297
|
const evaluatedAtMs = state.now();
|
|
2879
3298
|
// Both bounds come from a single pass over the account's records.
|
|
2880
|
-
const pinnedRecoveryBounds = pinnedAccount
|
|
2881
|
-
?
|
|
2882
|
-
:
|
|
3299
|
+
const pinnedRecoveryBounds = !isTranscribeRequest && pinnedAccount !== null
|
|
3300
|
+
? getAccountRecoveryBoundsForFamily(pinnedAccount, evaluatedAtMs, context.family, context.model)
|
|
3301
|
+
: null;
|
|
2883
3302
|
const pinnedStateRecoveryAtMs = pinnedRecoveryBounds?.recoveryAtMs ?? null;
|
|
2884
3303
|
// An open circuit outlives the short failure cooldowns that tripped
|
|
2885
3304
|
// it; its deadline lives in the breaker, not the account record.
|
|
2886
|
-
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
3305
|
+
// Transcription intentionally ignores the inference circuit entirely.
|
|
3306
|
+
const pinnedCircuitRecoveryAtMs = !isTranscribeRequest && pinnedAccount !== null
|
|
3307
|
+
? accountManager.getCircuitRecoveryTime(pinnedAccount, evaluatedAtMs)
|
|
3308
|
+
: null;
|
|
3309
|
+
const pinnedTranscriptionRateLimitAtMs = isTranscribeRequest && pinnedAccount?.recordId
|
|
3310
|
+
? state.transcriptionRateLimitUntilByRecordId.get(pinnedAccount.recordId) ?? null
|
|
3311
|
+
: null;
|
|
3312
|
+
const livePinnedTranscriptionRateLimitAtMs = typeof pinnedTranscriptionRateLimitAtMs === "number" &&
|
|
3313
|
+
pinnedTranscriptionRateLimitAtMs > evaluatedAtMs
|
|
3314
|
+
? pinnedTranscriptionRateLimitAtMs
|
|
3315
|
+
: null;
|
|
3316
|
+
const pinnedAuthCooldownAtMs = isTranscribeRequest &&
|
|
3317
|
+
pinnedAccount?.cooldownReason === "auth-failure" &&
|
|
3318
|
+
typeof pinnedAccount.coolingDownUntil === "number" &&
|
|
3319
|
+
pinnedAccount.coolingDownUntil > evaluatedAtMs
|
|
3320
|
+
? pinnedAccount.coolingDownUntil
|
|
3321
|
+
: null;
|
|
3322
|
+
const pinnedResetAtMs = pinnedBlockedPermanently
|
|
2891
3323
|
? null
|
|
2892
|
-
:
|
|
3324
|
+
: isTranscribeRequest
|
|
3325
|
+
? livePinnedTranscriptionRateLimitAtMs === null && pinnedAuthCooldownAtMs === null
|
|
3326
|
+
? null
|
|
3327
|
+
: Math.max(livePinnedTranscriptionRateLimitAtMs ?? 0, pinnedAuthCooldownAtMs ?? 0)
|
|
3328
|
+
: pinnedStateRecoveryAtMs === null && pinnedCircuitRecoveryAtMs === null
|
|
3329
|
+
? null
|
|
3330
|
+
: Math.max(pinnedStateRecoveryAtMs ?? 0, pinnedCircuitRecoveryAtMs ?? 0);
|
|
2893
3331
|
// The message only words the recovery deadline as a rate-limit reset
|
|
2894
|
-
// when the rate-limit records are in fact what supplies it
|
|
3332
|
+
// when the rate-limit records are in fact what supplies it - a
|
|
2895
3333
|
// breaker or cooldown can end later and then bounds it instead.
|
|
2896
|
-
const pinnedRateLimitResetAtMs =
|
|
3334
|
+
const pinnedRateLimitResetAtMs = isTranscribeRequest
|
|
3335
|
+
? livePinnedTranscriptionRateLimitAtMs
|
|
3336
|
+
: pinnedRecoveryBounds?.rateLimitAtMs ?? null;
|
|
2897
3337
|
const recoveryBound = pinnedResetAtMs !== null &&
|
|
2898
3338
|
pinnedRateLimitResetAtMs !== null &&
|
|
2899
3339
|
pinnedRateLimitResetAtMs >= pinnedResetAtMs
|
|
@@ -2935,12 +3375,47 @@ async function handleRequestInner(state, req, res, traceId) {
|
|
|
2935
3375
|
writeJson(res, HTTP_STATUS.OK, { goal: null });
|
|
2936
3376
|
}
|
|
2937
3377
|
else {
|
|
3378
|
+
const transcriptionRetryAfterMs = isTranscribeRequest
|
|
3379
|
+
? (() => {
|
|
3380
|
+
const now = state.now();
|
|
3381
|
+
let minWaitMs = null;
|
|
3382
|
+
// Read live account state, not only selection skip reasons. The final
|
|
3383
|
+
// refresh attempt can install an auth cooldown after the last selection
|
|
3384
|
+
// pass, so that blocker may never have been written to accountSkipReasons.
|
|
3385
|
+
for (const account of accountManager.getAccountsSnapshot()) {
|
|
3386
|
+
if (policyDecision?.blockedAccountIndexes.has(account.index))
|
|
3387
|
+
continue;
|
|
3388
|
+
if (account.enabled === false)
|
|
3389
|
+
continue;
|
|
3390
|
+
if (!accountManager.hasEnabledWorkspaces(account))
|
|
3391
|
+
continue;
|
|
3392
|
+
if (accountManager.isAccountAuthInvalidated(account))
|
|
3393
|
+
continue;
|
|
3394
|
+
const transcriptionUntil = account.recordId
|
|
3395
|
+
? state.transcriptionRateLimitUntilByRecordId.get(account.recordId)
|
|
3396
|
+
: undefined;
|
|
3397
|
+
const authUntil = account.cooldownReason === "auth-failure" &&
|
|
3398
|
+
typeof account.coolingDownUntil === "number"
|
|
3399
|
+
? account.coolingDownUntil
|
|
3400
|
+
: undefined;
|
|
3401
|
+
const readyAt = Math.max(typeof transcriptionUntil === "number" && transcriptionUntil > now
|
|
3402
|
+
? transcriptionUntil
|
|
3403
|
+
: 0, typeof authUntil === "number" && authUntil > now ? authUntil : 0);
|
|
3404
|
+
if (readyAt <= now)
|
|
3405
|
+
return 0;
|
|
3406
|
+
const waitMs = readyAt - now;
|
|
3407
|
+
minWaitMs = minWaitMs === null ? waitMs : Math.min(minWaitMs, waitMs);
|
|
3408
|
+
}
|
|
3409
|
+
return minWaitMs ?? 0;
|
|
3410
|
+
})()
|
|
3411
|
+
: undefined;
|
|
2938
3412
|
writePoolExhausted({
|
|
2939
3413
|
res,
|
|
2940
3414
|
accountManager,
|
|
2941
3415
|
family: context.family,
|
|
2942
3416
|
model: context.model,
|
|
2943
3417
|
reason: exhaustionReason,
|
|
3418
|
+
retryAfterMs: transcriptionRetryAfterMs,
|
|
2944
3419
|
accountSkipReasons: Object.fromEntries([...accountSkipReasons.entries()].map(([index, reason]) => [
|
|
2945
3420
|
String(index),
|
|
2946
3421
|
reason,
|