codex-multi-auth 2.1.9 → 2.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/assets/codex-multi-auth-icon.svg +39 -0
- package/dist/lib/accounts.d.ts +2 -0
- package/dist/lib/accounts.d.ts.map +1 -1
- package/dist/lib/accounts.js +26 -2
- package/dist/lib/accounts.js.map +1 -1
- package/dist/lib/codex-manager/commands/forecast.d.ts +5 -1
- package/dist/lib/codex-manager/commands/forecast.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/forecast.js +14 -1
- package/dist/lib/codex-manager/commands/forecast.js.map +1 -1
- package/dist/lib/codex-manager/commands/report.d.ts +2 -0
- package/dist/lib/codex-manager/commands/report.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/report.js +17 -1
- package/dist/lib/codex-manager/commands/report.js.map +1 -1
- package/dist/lib/codex-manager/commands/rotation.d.ts +1 -1
- package/dist/lib/codex-manager/commands/rotation.d.ts.map +1 -1
- package/dist/lib/codex-manager/commands/rotation.js +71 -1
- package/dist/lib/codex-manager/commands/rotation.js.map +1 -1
- package/dist/lib/codex-manager/repair-commands.d.ts.map +1 -1
- package/dist/lib/codex-manager/repair-commands.js +35 -0
- package/dist/lib/codex-manager/repair-commands.js.map +1 -1
- package/dist/lib/codex-manager.d.ts.map +1 -1
- package/dist/lib/codex-manager.js +2 -0
- package/dist/lib/codex-manager.js.map +1 -1
- package/dist/lib/forecast.d.ts +9 -0
- package/dist/lib/forecast.d.ts.map +1 -1
- package/dist/lib/forecast.js +36 -0
- package/dist/lib/forecast.js.map +1 -1
- package/dist/lib/runtime/runtime-observability.d.ts +17 -0
- package/dist/lib/runtime/runtime-observability.d.ts.map +1 -1
- package/dist/lib/runtime/runtime-observability.js +48 -0
- package/dist/lib/runtime/runtime-observability.js.map +1 -1
- package/dist/lib/runtime-rotation-proxy.d.ts +1 -0
- package/dist/lib/runtime-rotation-proxy.d.ts.map +1 -1
- package/dist/lib/runtime-rotation-proxy.js +119 -20
- package/dist/lib/runtime-rotation-proxy.js.map +1 -1
- package/package.json +12 -4
- package/scripts/audit-dev-allowlist.js +0 -220
- package/scripts/bench-format/codex-host.mjs +0 -275
- package/scripts/bench-format/hashline-v2.mjs +0 -642
- package/scripts/bench-format/models.mjs +0 -110
- package/scripts/bench-format/render.mjs +0 -496
- package/scripts/bench-format/stats.mjs +0 -54
- package/scripts/bench-format/tasks.mjs +0 -151
- package/scripts/benchmark-edit-formats.mjs +0 -1164
- package/scripts/benchmark-render-dashboard.mjs +0 -49
- package/scripts/benchmark-runtime-path.mjs +0 -166
- package/scripts/check-pack-budget-lib.js +0 -157
- package/scripts/check-pack-budget.mjs +0 -10
- package/scripts/copy-oauth-success.js +0 -76
- package/scripts/install-codex-auth.js +0 -258
- package/scripts/repo-hygiene.js +0 -320
- package/scripts/test-all-models.sh +0 -7
- package/scripts/test-model-matrix.js +0 -687
- package/scripts/update-vendor-provenance.mjs +0 -96
- package/scripts/validate-model-map.sh +0 -7
- package/scripts/verify-vendor-provenance.mjs +0 -53
|
@@ -42,10 +42,27 @@ export interface RuntimeObservabilitySnapshot {
|
|
|
42
42
|
lastAccountEmail?: string | null;
|
|
43
43
|
lastAccountId?: string | null;
|
|
44
44
|
lastAccountUpdatedAt?: number | null;
|
|
45
|
+
lastPoolExhaustionReason?: string | null;
|
|
46
|
+
lastPoolExhaustionRetryAfterMs?: number | null;
|
|
47
|
+
lastPoolExhaustionSkipReasons?: Record<string, string>;
|
|
48
|
+
lastRuntimeResetAt?: number | null;
|
|
49
|
+
lastRuntimeResetReason?: string | null;
|
|
50
|
+
lastRuntimeReloadAt?: number | null;
|
|
51
|
+
lastRuntimeReloadReason?: string | null;
|
|
52
|
+
accountSkipReasons?: Record<string, string>;
|
|
53
|
+
policyBlockedIndexes?: number[];
|
|
54
|
+
policyBlockedReasons?: Record<string, string>;
|
|
45
55
|
runtimeMetrics: RuntimeMetricsSnapshot;
|
|
46
56
|
}
|
|
47
57
|
export declare function getRuntimeObservabilitySnapshot(): RuntimeObservabilitySnapshot;
|
|
48
58
|
export declare function mutateRuntimeObservabilitySnapshot(mutator: (snapshot: RuntimeObservabilitySnapshot) => void): void;
|
|
59
|
+
export declare function recordRuntimePoolExhaustion(params: {
|
|
60
|
+
reason: string;
|
|
61
|
+
retryAfterMs: number;
|
|
62
|
+
accountSkipReasons: Record<string, string>;
|
|
63
|
+
}): void;
|
|
64
|
+
export declare function recordRuntimeReload(reason: string): void;
|
|
65
|
+
export declare function recordRuntimeReset(reason: string): void;
|
|
49
66
|
export declare function loadPersistedRuntimeObservabilitySnapshot(): Promise<RuntimeObservabilitySnapshot | null>;
|
|
50
67
|
export declare function resetRuntimeObservabilitySnapshotForTests(): void;
|
|
51
68
|
//# sourceMappingURL=runtime-observability.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runtime-observability.d.ts","sourceRoot":"","sources":["../../../lib/runtime/runtime-observability.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,sBAAsB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;IAChC,4BAA4B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,+BAA+B,EAAE,MAAM,CAAC;IACxC,+BAA+B,EAAE,MAAM,CAAC;IACxC,uBAAuB,EAAE,MAAM,CAAC;IAChC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,kCAAkC,EAAE,MAAM,CAAC;IAC3C,gCAAgC,EAAE,MAAM,CAAC;IACzC,wBAAwB,EAAE,MAAM,CAAC;IACjC,oCAAoC,EAAE,MAAM,CAAC;IAC7C,mBAAmB,EAAE,MAAM,CAAC;IAC5B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,MAAM,WAAW,4BAA4B;IAC5C,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;IAChC,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,wBAAwB,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,cAAc,EAAE,sBAAsB,CAAC;CACvC;
|
|
1
|
+
{"version":3,"file":"runtime-observability.d.ts","sourceRoot":"","sources":["../../../lib/runtime/runtime-observability.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,sBAAsB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;IAChC,4BAA4B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5C,+BAA+B,EAAE,MAAM,CAAC;IACxC,+BAA+B,EAAE,MAAM,CAAC;IACxC,uBAAuB,EAAE,MAAM,CAAC;IAChC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,kCAAkC,EAAE,MAAM,CAAC;IAC3C,gCAAgC,EAAE,MAAM,CAAC;IACzC,wBAAwB,EAAE,MAAM,CAAC;IACjC,oCAAoC,EAAE,MAAM,CAAC;IAC7C,mBAAmB,EAAE,MAAM,CAAC;IAC5B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED,MAAM,WAAW,4BAA4B;IAC5C,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;IAChC,2BAA2B,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3C,wBAAwB,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,wBAAwB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzC,8BAA8B,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/C,6BAA6B,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvD,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,sBAAsB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,uBAAuB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5C,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC,oBAAoB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9C,cAAc,EAAE,sBAAsB,CAAC;CACvC;AAwKD,wBAAgB,+BAA+B,IAAI,4BAA4B,CAE9E;AAED,wBAAgB,kCAAkC,CACjD,OAAO,EAAE,CAAC,QAAQ,EAAE,4BAA4B,KAAK,IAAI,GACvD,IAAI,CAYN;AAED,wBAAgB,2BAA2B,CAAC,MAAM,EAAE;IACnD,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC3C,GAAG,IAAI,CAOP;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAKxD;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAWvD;AAED,wBAAsB,yCAAyC,IAAI,OAAO,CAAC,4BAA4B,GAAG,IAAI,CAAC,CAY9G;AAED,wBAAgB,yCAAyC,IAAI,IAAI,CAGhE"}
|
|
@@ -25,6 +25,16 @@ function createDefaultSnapshot() {
|
|
|
25
25
|
lastAccountEmail: null,
|
|
26
26
|
lastAccountId: null,
|
|
27
27
|
lastAccountUpdatedAt: null,
|
|
28
|
+
lastPoolExhaustionReason: null,
|
|
29
|
+
lastPoolExhaustionRetryAfterMs: null,
|
|
30
|
+
lastPoolExhaustionSkipReasons: {},
|
|
31
|
+
lastRuntimeResetAt: null,
|
|
32
|
+
lastRuntimeResetReason: null,
|
|
33
|
+
lastRuntimeReloadAt: null,
|
|
34
|
+
lastRuntimeReloadReason: null,
|
|
35
|
+
accountSkipReasons: {},
|
|
36
|
+
policyBlockedIndexes: [],
|
|
37
|
+
policyBlockedReasons: {},
|
|
28
38
|
runtimeMetrics: {
|
|
29
39
|
startedAt: 0,
|
|
30
40
|
totalRequests: 0,
|
|
@@ -74,6 +84,18 @@ function normalizePersistedSnapshot(parsed) {
|
|
|
74
84
|
...base.runtimeMetrics,
|
|
75
85
|
...(parsed.runtimeMetrics ?? {}),
|
|
76
86
|
},
|
|
87
|
+
lastPoolExhaustionSkipReasons: {
|
|
88
|
+
...(parsed.lastPoolExhaustionSkipReasons ?? {}),
|
|
89
|
+
},
|
|
90
|
+
accountSkipReasons: {
|
|
91
|
+
...(parsed.accountSkipReasons ?? {}),
|
|
92
|
+
},
|
|
93
|
+
policyBlockedIndexes: Array.isArray(parsed.policyBlockedIndexes)
|
|
94
|
+
? parsed.policyBlockedIndexes.filter((value) => Number.isInteger(value))
|
|
95
|
+
: [],
|
|
96
|
+
policyBlockedReasons: {
|
|
97
|
+
...(parsed.policyBlockedReasons ?? {}),
|
|
98
|
+
},
|
|
77
99
|
};
|
|
78
100
|
}
|
|
79
101
|
function loadPersistedRuntimeObservabilitySnapshotSync() {
|
|
@@ -151,6 +173,32 @@ export function mutateRuntimeObservabilitySnapshot(mutator) {
|
|
|
151
173
|
.then(() => writeSnapshot(nextSnapshot))
|
|
152
174
|
.catch(() => undefined);
|
|
153
175
|
}
|
|
176
|
+
export function recordRuntimePoolExhaustion(params) {
|
|
177
|
+
mutateRuntimeObservabilitySnapshot((snapshot) => {
|
|
178
|
+
snapshot.lastPoolExhaustionReason = params.reason;
|
|
179
|
+
snapshot.lastPoolExhaustionRetryAfterMs = params.retryAfterMs;
|
|
180
|
+
snapshot.lastPoolExhaustionSkipReasons = { ...params.accountSkipReasons };
|
|
181
|
+
snapshot.accountSkipReasons = { ...params.accountSkipReasons };
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
export function recordRuntimeReload(reason) {
|
|
185
|
+
mutateRuntimeObservabilitySnapshot((snapshot) => {
|
|
186
|
+
snapshot.lastRuntimeReloadAt = Date.now();
|
|
187
|
+
snapshot.lastRuntimeReloadReason = reason;
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
export function recordRuntimeReset(reason) {
|
|
191
|
+
mutateRuntimeObservabilitySnapshot((snapshot) => {
|
|
192
|
+
snapshot.lastRuntimeResetAt = Date.now();
|
|
193
|
+
snapshot.lastRuntimeResetReason = reason;
|
|
194
|
+
snapshot.lastPoolExhaustionReason = null;
|
|
195
|
+
snapshot.lastPoolExhaustionRetryAfterMs = null;
|
|
196
|
+
snapshot.lastPoolExhaustionSkipReasons = {};
|
|
197
|
+
snapshot.accountSkipReasons = {};
|
|
198
|
+
snapshot.policyBlockedIndexes = [];
|
|
199
|
+
snapshot.policyBlockedReasons = {};
|
|
200
|
+
});
|
|
201
|
+
}
|
|
154
202
|
export async function loadPersistedRuntimeObservabilitySnapshot() {
|
|
155
203
|
const path = getSnapshotPath();
|
|
156
204
|
if (!existsSync(path)) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runtime-observability.js","sourceRoot":"","sources":["../../../lib/runtime/runtime-observability.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"runtime-observability.js","sourceRoot":"","sources":["../../../lib/runtime/runtime-observability.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACnE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AA4D3D,MAAM,kBAAkB,GAAG,4BAA4B,CAAC;AACxD,MAAM,wBAAwB,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,KAAK,MAAM,CAAC;AAC/D,MAAM,sCAAsC,GAAG,CAAC,CAAC;AACjD,MAAM,yBAAyB,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;AAE9D,IAAI,aAAa,GAAwC,IAAI,CAAC;AAC9D,IAAI,YAAY,GAAyB,IAAI,CAAC;AAE9C,SAAS,eAAe;IACvB,OAAO,IAAI,CAAC,oBAAoB,EAAE,EAAE,kBAAkB,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,qBAAqB;IAC7B,OAAO;QACN,OAAO,EAAE,sCAAsC;QAC/C,SAAS,EAAE,CAAC;QACZ,gBAAgB,EAAE,IAAI;QACtB,iBAAiB,EAAE,CAAC;QACpB,mBAAmB,EAAE,CAAC;QACtB,uBAAuB,EAAE,CAAC;QAC1B,2BAA2B,EAAE,IAAI;QACjC,wBAAwB,EAAE,IAAI;QAC9B,gBAAgB,EAAE,IAAI;QACtB,gBAAgB,EAAE,IAAI;QACtB,gBAAgB,EAAE,IAAI;QACtB,aAAa,EAAE,IAAI;QACnB,oBAAoB,EAAE,IAAI;QAC1B,wBAAwB,EAAE,IAAI;QAC9B,8BAA8B,EAAE,IAAI;QACpC,6BAA6B,EAAE,EAAE;QACjC,kBAAkB,EAAE,IAAI;QACxB,sBAAsB,EAAE,IAAI;QAC5B,mBAAmB,EAAE,IAAI;QACzB,uBAAuB,EAAE,IAAI;QAC7B,kBAAkB,EAAE,EAAE;QACtB,oBAAoB,EAAE,EAAE;QACxB,oBAAoB,EAAE,EAAE;QACxB,cAAc,EAAE;YACf,SAAS,EAAE,CAAC;YACZ,aAAa,EAAE,CAAC;YAChB,kBAAkB,EAAE,CAAC;YACrB,cAAc,EAAE,CAAC;YACjB,iBAAiB,EAAE,CAAC;YACpB,mBAAmB,EAAE,CAAC;YACtB,uBAAuB,EAAE,CAAC;YAC1B,4BAA4B,EAAE,IAAI;YAClC,+BAA+B,EAAE,CAAC;YAClC,+BAA+B,EAAE,CAAC;YAClC,uBAAuB,EAAE,CAAC;YAC1B,oBAAoB,EAAE,CAAC;YACvB,oBAAoB,EAAE,CAAC;YACvB,YAAY,EAAE,CAAC;YACf,aAAa,EAAE,CAAC;YAChB,UAAU,EAAE,CAAC;YACb,mBAAmB,EAAE,CAAC;YACtB,oBAAoB,EAAE,CAAC;YACvB,gBAAgB,EAAE,CAAC;YACnB,kBAAkB,EAAE,CAAC;YACrB,sBAAsB,EAAE,CAAC;YACzB,kCAAkC,EAAE,CAAC;YACrC,gCAAgC,EAAE,CAAC;YACnC,wBAAwB,EAAE,CAAC;YAC3B,oCAAoC,EAAE,CAAC;YACvC,mBAAmB,EAAE,CAAC;YACtB,aAAa,EAAE,IAAI;YACnB,SAAS,EAAE,IAAI;SACf;KACD,CAAC;AACH,CAAC;AAED,SAAS,0BAA0B,CAClC,MAAoD;IAEpD,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IACC,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ;QAClC,MAAM,CAAC,OAAO,KAAK,sCAAsC,EACxD,CAAC;QACF,OAAO,IAAI,CAAC;IACb,CAAC;IACD,MAAM,IAAI,GAAG,qBAAqB,EAAE,CAAC;IACrC,OAAO;QACN,GAAG,IAAI;QACP,GAAG,MAAM;QACT,OAAO,EAAE,sCAAsC;QAC/C,cAAc,EAAE;YACf,GAAG,IAAI,CAAC,cAAc;YACtB,GAAG,CAAC,MAAM,CAAC,cAAc,IAAI,EAAE,CAAC;SAChC;QACD,6BAA6B,EAAE;YAC9B,GAAG,CAAC,MAAM,CAAC,6BAA6B,IAAI,EAAE,CAAC;SAC/C;QACD,kBAAkB,EAAE;YACnB,GAAG,CAAC,MAAM,CAAC,kBAAkB,IAAI,EAAE,CAAC;SACpC;QACD,oBAAoB,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,oBAAoB,CAAC;YAC/D,CAAC,CAAC,MAAM,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAC9D,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CACvB;YACF,CAAC,CAAC,EAAE;QACL,oBAAoB,EAAE;YACrB,GAAG,CAAC,MAAM,CAAC,oBAAoB,IAAI,EAAE,CAAC;SACtC;KACD,CAAC;AACH,CAAC;AAED,SAAS,6CAA6C;IACrD,MAAM,IAAI,GAAG,eAAe,EAAE,CAAC;IAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,CAAC;QACJ,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAiD,CAAC;QAC/E,OAAO,0BAA0B,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAC;IACb,CAAC;AACF,CAAC;AAED,SAAS,mBAAmB;IAC3B,IAAI,CAAC,aAAa,EAAE,CAAC;QACpB,aAAa;YACZ,CAAC,wBAAwB;gBACxB,CAAC,CAAC,6CAA6C,EAAE;gBACjD,CAAC,CAAC,IAAI,CAAC,IAAI,qBAAqB,EAAE,CAAC;IACtC,CAAC;IACD,OAAO,aAAa,CAAC;AACtB,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,QAAsC;IAClE,MAAM,GAAG,GAAG,oBAAoB,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,eAAe,EAAE,CAAC;IAC/B,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,IAAI,SAAS,GAAY,IAAI,CAAC;IAC9B,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,QAAQ,GAAG,GAAG,IAAI,IAAI,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,OAAO,MAAM,CAAC;QACvE,IAAI,KAAK,GAAG,KAAK,CAAC;QAClB,IAAI,CAAC;YACJ,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YACzE,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAChC,KAAK,GAAG,IAAI,CAAC;YACb,OAAO;QACR,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,SAAS,GAAG,KAAK,CAAC;YAClB,MAAM,IAAI,GAAI,KAA2C,EAAE,IAAI,IAAI,EAAE,CAAC;YACtE,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;gBAC1D,MAAM,KAAK,CAAC;YACb,CAAC;QACF,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,IAAI,CAAC;oBACJ,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC3B,CAAC;gBAAC,MAAM,CAAC;oBACR,8CAA8C;gBAC/C,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IACD,IAAI,SAAS,EAAE,CAAC;QACf,MAAM,SAAS,CAAC;IACjB,CAAC;AACF,CAAC;AAED,MAAM,UAAU,+BAA+B;IAC9C,OAAO,eAAe,CAAC,mBAAmB,EAAE,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,kCAAkC,CACjD,OAAyD;IAEzD,MAAM,QAAQ,GAAG,mBAAmB,EAAE,CAAC;IACvC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClB,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChC,IAAI,CAAC,wBAAwB,EAAE,CAAC;QAC/B,OAAO;IACR,CAAC;IACD,MAAM,YAAY,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IAC/C,YAAY,GAAG,CAAC,YAAY,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;SAChD,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC;SACtB,IAAI,CAAC,GAAG,EAAE,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC;SACvC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,MAI3C;IACA,kCAAkC,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC/C,QAAQ,CAAC,wBAAwB,GAAG,MAAM,CAAC,MAAM,CAAC;QAClD,QAAQ,CAAC,8BAA8B,GAAG,MAAM,CAAC,YAAY,CAAC;QAC9D,QAAQ,CAAC,6BAA6B,GAAG,EAAE,GAAG,MAAM,CAAC,kBAAkB,EAAE,CAAC;QAC1E,QAAQ,CAAC,kBAAkB,GAAG,EAAE,GAAG,MAAM,CAAC,kBAAkB,EAAE,CAAC;IAChE,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,MAAc;IACjD,kCAAkC,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC/C,QAAQ,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC1C,QAAQ,CAAC,uBAAuB,GAAG,MAAM,CAAC;IAC3C,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,MAAc;IAChD,kCAAkC,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC/C,QAAQ,CAAC,kBAAkB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzC,QAAQ,CAAC,sBAAsB,GAAG,MAAM,CAAC;QACzC,QAAQ,CAAC,wBAAwB,GAAG,IAAI,CAAC;QACzC,QAAQ,CAAC,8BAA8B,GAAG,IAAI,CAAC;QAC/C,QAAQ,CAAC,6BAA6B,GAAG,EAAE,CAAC;QAC5C,QAAQ,CAAC,kBAAkB,GAAG,EAAE,CAAC;QACjC,QAAQ,CAAC,oBAAoB,GAAG,EAAE,CAAC;QACnC,QAAQ,CAAC,oBAAoB,GAAG,EAAE,CAAC;IACpC,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yCAAyC;IAC9D,MAAM,IAAI,GAAG,eAAe,EAAE,CAAC;IAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IACb,CAAC;IACD,IAAI,CAAC;QACJ,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAiD,CAAC;QAC/E,OAAO,0BAA0B,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAC;IACb,CAAC;AACF,CAAC;AAED,MAAM,UAAU,yCAAyC;IACxD,aAAa,GAAG,qBAAqB,EAAE,CAAC;IACxC,YAAY,GAAG,IAAI,CAAC;AACrB,CAAC"}
|
|
@@ -79,6 +79,7 @@ export declare function chooseAccount(params: {
|
|
|
79
79
|
now: number;
|
|
80
80
|
policy: RuntimePolicyDecision | null;
|
|
81
81
|
pinnedIndex: number | null;
|
|
82
|
+
skipReasons?: Map<number, string>;
|
|
82
83
|
}): ManagedAccount | null;
|
|
83
84
|
export declare function startRuntimeRotationProxy(options: RuntimeRotationProxyOptions): Promise<RuntimeRotationProxyServer>;
|
|
84
85
|
//# sourceMappingURL=runtime-rotation-proxy.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runtime-rotation-proxy.d.ts","sourceRoot":"","sources":["../../lib/runtime-rotation-proxy.ts"],"names":[],"mappings":"AAIA,OAAO,EACN,cAAc,EAEd,KAAK,cAAc,EACnB,MAAM,eAAe,CAAC;AAqBvB,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"runtime-rotation-proxy.d.ts","sourceRoot":"","sources":["../../lib/runtime-rotation-proxy.ts"],"names":[],"mappings":"AAIA,OAAO,EACN,cAAc,EAEd,KAAK,cAAc,EACnB,MAAM,eAAe,CAAC;AAqBvB,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAQtE,OAAO,EAIN,KAAK,qBAAqB,EAC1B,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAI7D,MAAM,WAAW,0BAA0B;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,SAAS,EAAE,MAAM,0BAA0B,CAAC;CAC5C;AAED,MAAM,WAAW,0BAA0B;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;CACpC;AAED,MAAM,WAAW,2BAA2B;IAC3C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,8BAA8B,CAAC,EAAE,MAAM,CAAC;IACxC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,oBAAoB,CAAC,EAAE,MAAM,CAAC;CAC9B;AA6MD,MAAM,WAAW,WAAW;IAC3B,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,kBAAkB,EAAE,MAAM,CAAC;CAC3B;AAkBD;;;;;;;;;;GAUG;AACH,wBAAgB,uBAAuB,CACtC,WAAW,GAAE,MAAyB,GACpC,WAAW,CAgDb;AAED;;;GAGG;AACH,wBAAgB,8BAA8B,CAC7C,WAAW,GAAE,MAAyB,GACpC,MAAM,GAAG,IAAI,CAEf;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,IAAI,IAAI,CAE9C;AAED;;;;;;GAMG;AACH,wBAAgB,+BAA+B,CAC9C,oBAAoB,EAAE,oBAAoB,GAAG,IAAI,EACjD,sBAAsB,EAAE,MAAM,EAC9B,WAAW,GAAE,MAAyB,GACpC,MAAM,CAOR;AAqcD,wBAAgB,aAAa,CAAC,MAAM,EAAE;IACrC,cAAc,EAAE,cAAc,CAAC;IAC/B,oBAAoB,EAAE,oBAAoB,GAAG,IAAI,CAAC;IAClD,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,MAAM,EAAE,WAAW,CAAC;IACpB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gBAAgB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACtC,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,qBAAqB,GAAG,IAAI,CAAC;IACrC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC,GAAG,cAAc,GAAG,IAAI,CA+GxB;AAwID,wBAAsB,yBAAyB,CAC9C,OAAO,EAAE,2BAA2B,GAClC,OAAO,CAAC,0BAA0B,CAAC,CA+tBrC"}
|
|
@@ -7,7 +7,7 @@ import { getFetchTimeoutMs, getNetworkErrorCooldownMs, getRetryAllAccountsMaxRet
|
|
|
7
7
|
import { CODEX_BASE_URL, HTTP_STATUS, OPENAI_HEADERS, OPENAI_HEADER_VALUES, URL_PATHS, } from "./constants.js";
|
|
8
8
|
import { getModelFamily } from "./prompts/codex.js";
|
|
9
9
|
import { queuedRefresh } from "./refresh-queue.js";
|
|
10
|
-
import { mutateRuntimeObservabilitySnapshot } from "./runtime/runtime-observability.js";
|
|
10
|
+
import { mutateRuntimeObservabilitySnapshot, recordRuntimePoolExhaustion, recordRuntimeReload, recordRuntimeReset, } from "./runtime/runtime-observability.js";
|
|
11
11
|
import { createRuntimeUsageRecorder, evaluateRuntimePolicy, loadRuntimePolicyState, } from "./policy/runtime-policy.js";
|
|
12
12
|
import { isWorkspaceDisabledError } from "./request/fetch-helpers.js";
|
|
13
13
|
import { SessionAffinityStore } from "./session-affinity.js";
|
|
@@ -616,22 +616,31 @@ function getQuotaNearExhaustionWaitMs(headers, remainingThreshold, now) {
|
|
|
616
616
|
return candidates.length > 0 ? Math.max(...candidates) : 0;
|
|
617
617
|
}
|
|
618
618
|
export function chooseAccount(params) {
|
|
619
|
-
const { accountManager, sessionAffinityStore, sessionKey, family, model, attemptedIndexes, now, policy, pinnedIndex, } = params;
|
|
619
|
+
const { accountManager, sessionAffinityStore, sessionKey, family, model, attemptedIndexes, now, policy, pinnedIndex, skipReasons, } = params;
|
|
620
620
|
// Manual pin (from `codex-multi-auth switch <n>`) overrides every other
|
|
621
621
|
// selection signal. We do NOT call markSwitched here — the proxy must not
|
|
622
622
|
// clobber the pin set by the CLI. See issue #474.
|
|
623
623
|
if (typeof pinnedIndex === "number") {
|
|
624
|
-
if (attemptedIndexes.has(pinnedIndex))
|
|
624
|
+
if (attemptedIndexes.has(pinnedIndex)) {
|
|
625
|
+
skipReasons?.set(pinnedIndex, "already-attempted");
|
|
625
626
|
return null;
|
|
627
|
+
}
|
|
626
628
|
if (pinnedIndex < 0 || pinnedIndex >= accountManager.getAccountCount()) {
|
|
629
|
+
skipReasons?.set(pinnedIndex, "missing");
|
|
627
630
|
return null;
|
|
628
631
|
}
|
|
629
|
-
if (policy?.blockedAccountIndexes.has(pinnedIndex))
|
|
632
|
+
if (policy?.blockedAccountIndexes.has(pinnedIndex)) {
|
|
633
|
+
skipReasons?.set(pinnedIndex, "policy-blocked");
|
|
630
634
|
return null;
|
|
635
|
+
}
|
|
631
636
|
const pinned = accountManager.getAccountByIndex(pinnedIndex);
|
|
632
|
-
if (!pinned || pinned.enabled === false)
|
|
637
|
+
if (!pinned || pinned.enabled === false) {
|
|
638
|
+
skipReasons?.set(pinnedIndex, "disabled");
|
|
633
639
|
return null;
|
|
634
|
-
|
|
640
|
+
}
|
|
641
|
+
const reason = accountManager.getAccountRuntimeSkipReason(pinnedIndex, family, model);
|
|
642
|
+
if (reason) {
|
|
643
|
+
skipReasons?.set(pinnedIndex, reason);
|
|
635
644
|
return null;
|
|
636
645
|
}
|
|
637
646
|
return pinned;
|
|
@@ -641,10 +650,15 @@ export function chooseAccount(params) {
|
|
|
641
650
|
!attemptedIndexes.has(preferredIndex) &&
|
|
642
651
|
!policy?.blockedAccountIndexes.has(preferredIndex)) {
|
|
643
652
|
const preferred = accountManager.getAccountByIndex(preferredIndex);
|
|
644
|
-
if (preferred
|
|
645
|
-
accountManager.
|
|
646
|
-
|
|
647
|
-
|
|
653
|
+
if (preferred) {
|
|
654
|
+
const reason = accountManager.getAccountRuntimeSkipReason(preferred.index, family, model);
|
|
655
|
+
if (reason) {
|
|
656
|
+
skipReasons?.set(preferred.index, reason);
|
|
657
|
+
}
|
|
658
|
+
else {
|
|
659
|
+
accountManager.markSwitched(preferred, "rotation", family);
|
|
660
|
+
return preferred;
|
|
661
|
+
}
|
|
648
662
|
}
|
|
649
663
|
}
|
|
650
664
|
const selected = accountManager.getCurrentOrNextForFamilyHybrid(family, model, {
|
|
@@ -653,20 +667,29 @@ export function chooseAccount(params) {
|
|
|
653
667
|
if (selected &&
|
|
654
668
|
!attemptedIndexes.has(selected.index) &&
|
|
655
669
|
!policy?.blockedAccountIndexes.has(selected.index)) {
|
|
656
|
-
|
|
670
|
+
const reason = accountManager.getAccountRuntimeSkipReason(selected.index, family, model);
|
|
671
|
+
if (!reason)
|
|
672
|
+
return selected;
|
|
673
|
+
skipReasons?.set(selected.index, reason);
|
|
657
674
|
}
|
|
658
675
|
for (const account of accountManager.getAccountsSnapshot()) {
|
|
659
|
-
if (attemptedIndexes.has(account.index))
|
|
676
|
+
if (attemptedIndexes.has(account.index)) {
|
|
677
|
+
skipReasons?.set(account.index, "already-attempted");
|
|
660
678
|
continue;
|
|
661
|
-
|
|
679
|
+
}
|
|
680
|
+
if (policy?.blockedAccountIndexes.has(account.index)) {
|
|
681
|
+
skipReasons?.set(account.index, "policy-blocked");
|
|
662
682
|
continue;
|
|
663
|
-
|
|
683
|
+
}
|
|
684
|
+
const reason = accountManager.getAccountRuntimeSkipReason(account.index, family, model);
|
|
685
|
+
if (!reason) {
|
|
664
686
|
const live = accountManager.getAccountByIndex(account.index);
|
|
665
687
|
if (!live)
|
|
666
688
|
continue;
|
|
667
689
|
accountManager.markSwitched(live, "rotation", family);
|
|
668
690
|
return live;
|
|
669
691
|
}
|
|
692
|
+
skipReasons?.set(account.index, reason);
|
|
670
693
|
}
|
|
671
694
|
return null;
|
|
672
695
|
}
|
|
@@ -696,13 +719,24 @@ function normalizeExhaustionStatus(reason) {
|
|
|
696
719
|
function writePoolExhausted(params) {
|
|
697
720
|
const { res, accountManager, family, model, reason } = params;
|
|
698
721
|
const waitMs = accountManager.getMinWaitTimeForFamily(family, model);
|
|
722
|
+
const accountCount = accountManager.getAccountCount();
|
|
723
|
+
const accountSkipReasons = params.accountSkipReasons ?? {};
|
|
724
|
+
recordRuntimePoolExhaustion({
|
|
725
|
+
reason,
|
|
726
|
+
retryAfterMs: waitMs,
|
|
727
|
+
accountSkipReasons,
|
|
728
|
+
});
|
|
729
|
+
const hint = reason === "no-account" && accountCount > 0
|
|
730
|
+
? "Accounts exist but all failed runtime availability checks. Run `codex-multi-auth report --json` to inspect runtime skip reasons, or `codex-multi-auth rotation reset-runtime` to reload the runtime proxy."
|
|
731
|
+
: "Run `codex-multi-auth rotation status` to inspect account state.";
|
|
699
732
|
writeJson(res, normalizeExhaustionStatus(reason), {
|
|
700
733
|
error: {
|
|
701
734
|
message: "All managed Codex accounts are temporarily unavailable for this runtime request.",
|
|
702
735
|
code: "codex_runtime_rotation_pool_exhausted",
|
|
703
736
|
reason,
|
|
704
737
|
retry_after_ms: waitMs,
|
|
705
|
-
|
|
738
|
+
account_skip_reasons: accountSkipReasons,
|
|
739
|
+
hint,
|
|
706
740
|
},
|
|
707
741
|
});
|
|
708
742
|
}
|
|
@@ -762,7 +796,8 @@ async function forwardStreamingResponse(upstream, res, status, onStreamError, st
|
|
|
762
796
|
}
|
|
763
797
|
export async function startRuntimeRotationProxy(options) {
|
|
764
798
|
const pluginConfig = loadPluginConfig();
|
|
765
|
-
|
|
799
|
+
let activeAccountManager = options.accountManager ?? (await AccountManager.loadFromDisk());
|
|
800
|
+
const knownAccountManagers = new Set([activeAccountManager]);
|
|
766
801
|
const fetchImpl = options.fetchImpl ?? fetch;
|
|
767
802
|
const host = options.host ?? DEFAULT_HOST;
|
|
768
803
|
const port = options.port ?? 0;
|
|
@@ -810,8 +845,38 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
810
845
|
lastAccountUpdatedAt: null,
|
|
811
846
|
};
|
|
812
847
|
const threadGoalFallbacks = new Map();
|
|
848
|
+
let staleRuntimeReloadPromise = null;
|
|
849
|
+
let lastStaleRuntimeReloadAt = 0;
|
|
850
|
+
const staleRuntimeReloadDedupeMs = 1_000;
|
|
851
|
+
const recoverStaleRuntimeState = async () => {
|
|
852
|
+
if (Date.now() - lastStaleRuntimeReloadAt <= staleRuntimeReloadDedupeMs) {
|
|
853
|
+
return activeAccountManager;
|
|
854
|
+
}
|
|
855
|
+
if (staleRuntimeReloadPromise) {
|
|
856
|
+
return staleRuntimeReloadPromise;
|
|
857
|
+
}
|
|
858
|
+
staleRuntimeReloadPromise = (async () => {
|
|
859
|
+
AccountManager.resetVolatileRuntimeState();
|
|
860
|
+
recordRuntimeReset("pool-exhausted-no-account");
|
|
861
|
+
const reloaded = await AccountManager.loadFromDisk();
|
|
862
|
+
activeAccountManager = reloaded;
|
|
863
|
+
knownAccountManagers.add(reloaded);
|
|
864
|
+
lastStaleRuntimeReloadAt = Date.now();
|
|
865
|
+
recordRuntimeReload("pool-exhausted-no-account");
|
|
866
|
+
return reloaded;
|
|
867
|
+
})()
|
|
868
|
+
.catch((error) => {
|
|
869
|
+
status.lastError = error instanceof Error ? error.message : String(error);
|
|
870
|
+
return null;
|
|
871
|
+
})
|
|
872
|
+
.finally(() => {
|
|
873
|
+
staleRuntimeReloadPromise = null;
|
|
874
|
+
});
|
|
875
|
+
return staleRuntimeReloadPromise;
|
|
876
|
+
};
|
|
813
877
|
const handleRequest = async (req, res) => {
|
|
814
878
|
let usageRecorder = null;
|
|
879
|
+
let accountManager = activeAccountManager;
|
|
815
880
|
try {
|
|
816
881
|
const incomingUrl = new URL(req.url ?? "/", "http://127.0.0.1");
|
|
817
882
|
const isResponsesRequest = req.method === "POST" && isResponsesPath(incomingUrl.pathname);
|
|
@@ -849,6 +914,12 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
849
914
|
model: context.model,
|
|
850
915
|
now: requestStartedAt,
|
|
851
916
|
});
|
|
917
|
+
mutateRuntimeObservabilitySnapshot((snapshot) => {
|
|
918
|
+
snapshot.policyBlockedIndexes = [
|
|
919
|
+
...(policyDecision?.blockedAccountIndexes ?? new Set()),
|
|
920
|
+
];
|
|
921
|
+
snapshot.policyBlockedReasons = Object.fromEntries([...(policyDecision?.blockedAccountIndexes ?? new Set())].map((index) => [String(index), "policy-blocked"]));
|
|
922
|
+
});
|
|
852
923
|
}
|
|
853
924
|
catch (error) {
|
|
854
925
|
policyError = error instanceof Error ? error.message : String(error);
|
|
@@ -898,10 +969,12 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
898
969
|
const upstreamUrl = buildUpstreamUrl(req, upstreamBaseUrl, context.upstreamPath);
|
|
899
970
|
const attemptedIndexes = new Set();
|
|
900
971
|
let exhaustionReason = "no-account";
|
|
901
|
-
|
|
902
|
-
|
|
972
|
+
let accountCount = accountManager.getAccountCount();
|
|
973
|
+
let transientAttemptLimit = Math.max(1, Math.min(accountCount, maxRuntimeAccountAttempts));
|
|
903
974
|
let transientAttempts = 0;
|
|
904
975
|
let transientExhaustionReason = null;
|
|
976
|
+
const accountSkipReasons = new Map();
|
|
977
|
+
let reloadedAfterNoAccount = false;
|
|
905
978
|
// Read the manual pin and affinity generation from disk (mtime-cached)
|
|
906
979
|
// on each request so a `codex-multi-auth switch|unpin|best` invocation
|
|
907
980
|
// in another process is honored without forcing a full AccountManager
|
|
@@ -929,11 +1002,33 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
929
1002
|
now: now(),
|
|
930
1003
|
policy: policyDecision,
|
|
931
1004
|
pinnedIndex,
|
|
1005
|
+
skipReasons: accountSkipReasons,
|
|
932
1006
|
});
|
|
933
|
-
if (!selected)
|
|
1007
|
+
if (!selected) {
|
|
1008
|
+
if (!reloadedAfterNoAccount &&
|
|
1009
|
+
!isPinned &&
|
|
1010
|
+
accountCount > 0 &&
|
|
1011
|
+
exhaustionReason === "no-account" &&
|
|
1012
|
+
(policyDecision?.blockedAccountIndexes.size ?? 0) === 0 &&
|
|
1013
|
+
![...accountSkipReasons.values()].some((reason) => reason === "rate-limited" ||
|
|
1014
|
+
reason.startsWith("cooling-down") ||
|
|
1015
|
+
reason === "policy-blocked")) {
|
|
1016
|
+
reloadedAfterNoAccount = true;
|
|
1017
|
+
const reloadedManager = await recoverStaleRuntimeState();
|
|
1018
|
+
if (reloadedManager) {
|
|
1019
|
+
accountManager = reloadedManager;
|
|
1020
|
+
accountCount = accountManager.getAccountCount();
|
|
1021
|
+
transientAttemptLimit = Math.max(1, Math.min(accountCount, maxRuntimeAccountAttempts));
|
|
1022
|
+
accountSkipReasons.clear();
|
|
1023
|
+
attemptedIndexes.clear();
|
|
1024
|
+
continue;
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
934
1027
|
break;
|
|
1028
|
+
}
|
|
935
1029
|
attemptedIndexes.add(selected.index);
|
|
936
1030
|
if (!accountManager.consumeToken(selected, context.family, context.model)) {
|
|
1031
|
+
accountSkipReasons.set(selected.index, "token-exhausted");
|
|
937
1032
|
exhaustionReason = "rate-limit";
|
|
938
1033
|
continue;
|
|
939
1034
|
}
|
|
@@ -1210,6 +1305,10 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
1210
1305
|
family: context.family,
|
|
1211
1306
|
model: context.model,
|
|
1212
1307
|
reason: exhaustionReason,
|
|
1308
|
+
accountSkipReasons: Object.fromEntries([...accountSkipReasons.entries()].map(([index, reason]) => [
|
|
1309
|
+
String(index),
|
|
1310
|
+
reason,
|
|
1311
|
+
])),
|
|
1213
1312
|
});
|
|
1214
1313
|
}
|
|
1215
1314
|
}
|
|
@@ -1282,7 +1381,7 @@ export async function startRuntimeRotationProxy(options) {
|
|
|
1282
1381
|
baseUrl: `http://${host}:${resolvedPort}`,
|
|
1283
1382
|
close: async () => {
|
|
1284
1383
|
await closeServer(server, sockets);
|
|
1285
|
-
await
|
|
1384
|
+
await activeAccountManager.flushPendingSave();
|
|
1286
1385
|
},
|
|
1287
1386
|
getStatus: () => ({ ...status }),
|
|
1288
1387
|
};
|