codex-multi-auth 2.20.0 → 2.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +5 -4
- package/dist/lib/accounts.d.ts +5 -0
- package/dist/lib/accounts.js +33 -4
- package/dist/lib/api-route-store.d.ts +6 -0
- package/dist/lib/api-route-store.js +98 -30
- package/dist/lib/codex-manager/api-login-menu.d.ts +1 -1
- package/dist/lib/codex-manager/api-login-menu.js +4 -6
- package/dist/lib/codex-manager/commands/status.d.ts +1 -1
- package/dist/lib/codex-manager/commands/status.js +26 -9
- package/dist/lib/codex-manager.js +7 -1
- package/dist/lib/constants.d.ts +1 -0
- package/dist/lib/constants.js +1 -0
- package/dist/lib/model-route-policy.d.ts +1 -0
- package/dist/lib/model-route-policy.js +2 -1
- package/dist/lib/runtime/account-model-catalog.d.ts +1 -0
- package/dist/lib/runtime/api-model-capabilities.d.ts +11 -2
- package/dist/lib/runtime/api-model-capabilities.js +155 -24
- package/dist/lib/runtime/app-bind.d.ts +4 -0
- package/dist/lib/runtime/app-bind.js +32 -1
- package/dist/lib/runtime/automatic-api-checks.d.ts +15 -0
- package/dist/lib/runtime/automatic-api-checks.js +188 -0
- package/dist/lib/runtime/model-discovery-status.d.ts +11 -1
- package/dist/lib/runtime/model-discovery-status.js +64 -49
- package/dist/lib/runtime/persisted-api-capabilities.d.ts +20 -0
- package/dist/lib/runtime/persisted-api-capabilities.js +387 -0
- package/dist/lib/runtime/responses-websocket.d.ts +18 -5
- package/dist/lib/runtime/responses-websocket.js +33 -11
- package/dist/lib/runtime/rotation-account-selection.d.ts +20 -1
- package/dist/lib/runtime/rotation-account-selection.js +94 -4
- package/dist/lib/runtime/rotation-proxy-state.d.ts +26 -0
- package/dist/lib/runtime/rotation-proxy-state.js +376 -4
- package/dist/lib/runtime/rotation-server-types.d.ts +3 -0
- package/dist/lib/runtime/rotation-token-refresh.d.ts +23 -1
- package/dist/lib/runtime/rotation-token-refresh.js +218 -27
- package/dist/lib/runtime-rotation-proxy.js +571 -96
- package/dist/lib/storage/credential-sidecars.js +34 -2
- package/dist/lib/storage/file-lock.d.ts +2 -0
- package/dist/lib/storage/file-lock.js +44 -30
- package/dist/lib/usage/ledger.js +1 -0
- package/dist/lib/usage/redaction.js +1 -0
- package/dist/lib/usage/types.d.ts +1 -1
- package/package.json +1 -1
- package/scripts/codex-app-router.js +42 -14
|
@@ -14,11 +14,14 @@ const efforts = new Set([
|
|
|
14
14
|
]);
|
|
15
15
|
/** Read the explicit model-specific statement, never surrounding examples or another model's settings. */
|
|
16
16
|
export function parseApiReasoningDocumentation(id, text) {
|
|
17
|
+
return parseReasoningStatement(id, text) ?? [];
|
|
18
|
+
}
|
|
19
|
+
function parseReasoningStatement(id, text) {
|
|
17
20
|
if (text.match(/^Model ID:\s*`([^`]+)`/m)?.[1] !== id)
|
|
18
|
-
return
|
|
21
|
+
return undefined;
|
|
19
22
|
const statement = text.match(/`?reasoning\.effort`?\s+supports\s*:?\s*([^\n.]+)\./i)?.[1];
|
|
20
|
-
if (
|
|
21
|
-
return
|
|
23
|
+
if (statement === undefined)
|
|
24
|
+
return undefined;
|
|
22
25
|
const values = statement
|
|
23
26
|
.replace(/`|\(default\)/g, " ")
|
|
24
27
|
.split(/,|\band\b/)
|
|
@@ -29,9 +32,25 @@ export function parseApiReasoningDocumentation(id, text) {
|
|
|
29
32
|
values.some((v) => !/^[a-z][a-z0-9_-]{0,31}$/.test(v) ||
|
|
30
33
|
v === "ultra" ||
|
|
31
34
|
v === "persistent"))
|
|
32
|
-
return
|
|
35
|
+
return undefined;
|
|
33
36
|
return [...new Set(values)];
|
|
34
37
|
}
|
|
38
|
+
/** Candidates describe checks, never request defaults or granted entitlements. */
|
|
39
|
+
function programCandidates(route, id) {
|
|
40
|
+
const configured = (route.modelAccessPrograms?.[id] ?? route.accessPrograms)?.cyber;
|
|
41
|
+
const candidates = [...(configured ?? []), ...(/(?:^|-)cyber(?:-|$)|daybreak-red/.test(id)
|
|
42
|
+
? ["daybreak_red"] : /^gpt-[56](?:[.-]|$)/.test(id) ? ["standard", "daybreak_blue"] : [])];
|
|
43
|
+
return [...new Set(candidates)].filter(value => ["standard", "daybreak_blue", "daybreak_red"].includes(value));
|
|
44
|
+
}
|
|
45
|
+
function programsFresh(result, programs, now) {
|
|
46
|
+
return result.programRetry !== true && result.programCheckedAt !== undefined &&
|
|
47
|
+
Number.isFinite(result.programCheckedAt) && result.programCheckedAt >= 0 && result.programCheckedAt <= 8.64e15 &&
|
|
48
|
+
result.programCheckedAt <= now && result.programCheckedAt + 86400_000 > now &&
|
|
49
|
+
programs.every(program => ["verified", "unsupported"].includes(result.status[`program:cyber:${program}`] ?? ""));
|
|
50
|
+
}
|
|
51
|
+
function probeKey(route, id) {
|
|
52
|
+
return createHash("sha256").update(route.id).update("\0").update(route.apiKey).update("\0").update(id).digest("hex");
|
|
53
|
+
}
|
|
35
54
|
/** Only the exact model's input declaration can enable native image attachments. */
|
|
36
55
|
function parseInputModalities(id, text) {
|
|
37
56
|
if (text.match(/^Model ID:\s*`([^`]+)`/m)?.[1] !== id)
|
|
@@ -81,9 +100,14 @@ function classifyProbe403(data, setting) {
|
|
|
81
100
|
export class ApiModelCapabilities {
|
|
82
101
|
fetchImpl;
|
|
83
102
|
now;
|
|
103
|
+
probeSlot;
|
|
104
|
+
documentationSource;
|
|
105
|
+
programRefreshBefore;
|
|
84
106
|
activeProbes = 0;
|
|
85
107
|
probeWaiters = [];
|
|
86
108
|
async withProbeSlot(operation) {
|
|
109
|
+
if (this.probeSlot)
|
|
110
|
+
return this.probeSlot(operation);
|
|
87
111
|
if (this.activeProbes >= 4)
|
|
88
112
|
await new Promise((resolve) => this.probeWaiters.push(resolve));
|
|
89
113
|
else
|
|
@@ -102,9 +126,16 @@ export class ApiModelCapabilities {
|
|
|
102
126
|
inFlight = new Map();
|
|
103
127
|
probes = new Map();
|
|
104
128
|
cache = new Map();
|
|
105
|
-
constructor(fetchImpl = fetch, now = Date.now
|
|
129
|
+
constructor(fetchImpl = fetch, now = Date.now, probeSlot,
|
|
130
|
+
// Keep public documentation across short-lived credential/model leases.
|
|
131
|
+
documentationSource,
|
|
132
|
+
// Scheduled checks need program evidence to last until their next run.
|
|
133
|
+
programRefreshBefore) {
|
|
106
134
|
this.fetchImpl = fetchImpl;
|
|
107
135
|
this.now = now;
|
|
136
|
+
this.probeSlot = probeSlot;
|
|
137
|
+
this.documentationSource = documentationSource;
|
|
138
|
+
this.programRefreshBefore = programRefreshBefore;
|
|
108
139
|
}
|
|
109
140
|
/** Probe results keyed by an opaque hash, for a fresh CLI process to honour the 15-minute cache. */
|
|
110
141
|
exportProbes() {
|
|
@@ -121,6 +152,8 @@ export class ApiModelCapabilities {
|
|
|
121
152
|
return (await this.documentation(id, refresh)).levels;
|
|
122
153
|
}
|
|
123
154
|
async documentation(id, refresh = false) {
|
|
155
|
+
if (this.documentationSource)
|
|
156
|
+
return this.documentationSource.documentation(id, refresh);
|
|
124
157
|
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(id))
|
|
125
158
|
return { levels: [] };
|
|
126
159
|
const cached = this.cache.get(id);
|
|
@@ -153,41 +186,60 @@ export class ApiModelCapabilities {
|
|
|
153
186
|
await reader.cancel();
|
|
154
187
|
}
|
|
155
188
|
const text = Buffer.concat(chunks).toString("utf8");
|
|
189
|
+
const reasoning = parseReasoningStatement(id, text);
|
|
156
190
|
metadata = {
|
|
157
|
-
levels:
|
|
191
|
+
levels: reasoning ?? [],
|
|
192
|
+
reasoningAvailable: reasoning !== undefined,
|
|
158
193
|
inputModalities: parseInputModalities(id, text),
|
|
159
194
|
};
|
|
160
195
|
}
|
|
161
196
|
catch {
|
|
162
197
|
// An outage is not a capability revocation. Keep last successful evidence.
|
|
163
|
-
metadata = cached ?? metadata;
|
|
198
|
+
metadata = { ...(cached ?? metadata), reasoningAvailable: false };
|
|
164
199
|
}
|
|
165
200
|
if (this.cache.size >= 1000)
|
|
166
201
|
this.cache.delete(this.cache.keys().next().value ?? "");
|
|
167
202
|
this.cache.set(id, { ...metadata, at: this.now() });
|
|
168
203
|
return metadata;
|
|
169
204
|
}
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
.
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
.update("\0")
|
|
176
|
-
.update(id)
|
|
177
|
-
.digest("hex");
|
|
178
|
-
const old = this.probes.get(key);
|
|
205
|
+
programsCoverCheck(result, programs) {
|
|
206
|
+
return programsFresh(result, programs, this.now()) &&
|
|
207
|
+
(this.programRefreshBefore === undefined || (result.programCheckedAt ?? -Infinity) + 86400_000 >= this.programRefreshBefore);
|
|
208
|
+
}
|
|
209
|
+
freshProbe(route, id, old) {
|
|
179
210
|
// A probe cut short by 401/403/429 is retried after a minute, not trusted for 15.
|
|
180
211
|
const freshFor = old?.status.retry === "transient" ? 60_000 : 900_000;
|
|
181
|
-
|
|
212
|
+
const programs = programCandidates(route, id);
|
|
213
|
+
const oldPrograms = old ? Object.keys(old.status).filter(name => name.startsWith("program:cyber:")) : [];
|
|
214
|
+
const samePrograms = oldPrograms.length === programs.length && programs.every(program => oldPrograms.includes(`program:cyber:${program}`));
|
|
215
|
+
const programCacheFresh = old && samePrograms && (programs.length
|
|
216
|
+
? this.programsCoverCheck(old, programs) || ((old.programRetry === true || old.status.retry === "transient") && old.at + 60_000 > this.now())
|
|
217
|
+
: old.programCheckedAt === undefined);
|
|
218
|
+
if (old && old.at <= this.now() && old.at + freshFor > this.now() && programCacheFresh)
|
|
182
219
|
return old;
|
|
220
|
+
return undefined;
|
|
221
|
+
}
|
|
222
|
+
reconcileReasoning(probe, documented, available) {
|
|
223
|
+
if (!available)
|
|
224
|
+
return probe;
|
|
225
|
+
return { ...probe, levels: probe.levels.filter(effort => documented.includes(effort) || probe.status[`effort:${effort}`] === "verified") };
|
|
226
|
+
}
|
|
227
|
+
async probe(route, id, documented, force = false, reasoningAvailable = false) {
|
|
228
|
+
const key = probeKey(route, id);
|
|
229
|
+
const fresh = this.freshProbe(route, id, this.probes.get(key));
|
|
230
|
+
if (!force && fresh) {
|
|
231
|
+
const reconciled = this.reconcileReasoning(fresh, documented, reasoningAvailable);
|
|
232
|
+
this.probes.set(key, reconciled);
|
|
233
|
+
return reconciled;
|
|
234
|
+
}
|
|
183
235
|
const running = this.inFlight.get(key);
|
|
184
236
|
if (running)
|
|
185
237
|
return running;
|
|
186
|
-
const pending = this.runProbe(route, id, documented, key).finally(() => this.inFlight.delete(key));
|
|
238
|
+
const pending = this.runProbe(route, id, documented, key, force, reasoningAvailable).finally(() => this.inFlight.delete(key));
|
|
187
239
|
this.inFlight.set(key, pending);
|
|
188
240
|
return pending;
|
|
189
241
|
}
|
|
190
|
-
async runProbe(route, id, documented, key) {
|
|
242
|
+
async runProbe(route, id, documented, key, force, reasoningAvailable) {
|
|
191
243
|
const result = {
|
|
192
244
|
at: this.now(),
|
|
193
245
|
levels: [...documented],
|
|
@@ -200,6 +252,7 @@ export class ApiModelCapabilities {
|
|
|
200
252
|
// inconclusive and keeps what an earlier probe verified. Only
|
|
201
253
|
// "unsupported" or "downgraded" removes a setting otherwise.
|
|
202
254
|
let lostAccess = false;
|
|
255
|
+
let programAuthFailure = false;
|
|
203
256
|
// A model-level denial hides the model on this credential: no efforts, no tiers.
|
|
204
257
|
let modelDenied = false;
|
|
205
258
|
const previous = this.probes.get(key);
|
|
@@ -222,6 +275,7 @@ export class ApiModelCapabilities {
|
|
|
222
275
|
if (kind === "setting")
|
|
223
276
|
return "unsupported";
|
|
224
277
|
credentialUnavailable = true;
|
|
278
|
+
programAuthFailure = Boolean(setting.program);
|
|
225
279
|
return "unverified";
|
|
226
280
|
};
|
|
227
281
|
return this.withProbeSlot(async () => {
|
|
@@ -266,6 +320,7 @@ export class ApiModelCapabilities {
|
|
|
266
320
|
? { reasoning: { effort: setting.effort } }
|
|
267
321
|
: {}),
|
|
268
322
|
...(setting.tier ? { service_tier: setting.tier } : {}),
|
|
323
|
+
...(setting.program ? { access_programs: { cyber: setting.program } } : {}),
|
|
269
324
|
}),
|
|
270
325
|
});
|
|
271
326
|
if ([401, 429].includes(response.status))
|
|
@@ -299,6 +354,21 @@ export class ApiModelCapabilities {
|
|
|
299
354
|
catch {
|
|
300
355
|
return response.status === 403 ? denied403(null) : "unverified";
|
|
301
356
|
}
|
|
357
|
+
if (setting.program) {
|
|
358
|
+
const error = isRecord(data) && isRecord(data.error) ? data.error : null;
|
|
359
|
+
if ([400, 403].includes(response.status) && error &&
|
|
360
|
+
(["invalid_access_program", "unsupported_access_program", "access_program_not_enabled"].includes(String(error.code)) ||
|
|
361
|
+
["access_programs", "access_programs.cyber"].includes(String(error.param))))
|
|
362
|
+
return "unsupported";
|
|
363
|
+
if (response.status === 403) {
|
|
364
|
+
programAuthFailure = true;
|
|
365
|
+
return denied403(data);
|
|
366
|
+
}
|
|
367
|
+
if (!response.ok)
|
|
368
|
+
return "unverified";
|
|
369
|
+
return isRecord(data) && ["completed", "incomplete"].includes(String(data.status)) && isRecord(data.access_programs) && data.access_programs.cyber === setting.program
|
|
370
|
+
? "verified" : "unverified";
|
|
371
|
+
}
|
|
302
372
|
if (response.status === 403)
|
|
303
373
|
return denied403(data);
|
|
304
374
|
if (!response.ok) {
|
|
@@ -357,7 +427,10 @@ export class ApiModelCapabilities {
|
|
|
357
427
|
for (const { effort, status } of checked) {
|
|
358
428
|
const effective = keep(`effort:${effort}`, status);
|
|
359
429
|
result.status[`effort:${effort}`] = effective;
|
|
360
|
-
|
|
430
|
+
// Persisted levels can originate in public documentation, without a
|
|
431
|
+
// verified effort probe. Keep that scoped evidence through an
|
|
432
|
+
// inconclusive retry only while current documentation is unavailable.
|
|
433
|
+
if (effective === "verified" || (status === "unverified" && !reasoningAvailable && !lostAccess && !modelDenied && previous?.levels.includes(effort)))
|
|
361
434
|
result.levels.push(effort);
|
|
362
435
|
}
|
|
363
436
|
}
|
|
@@ -373,12 +446,54 @@ export class ApiModelCapabilities {
|
|
|
373
446
|
if (effective === "verified")
|
|
374
447
|
result.tiers.push(tier);
|
|
375
448
|
}
|
|
449
|
+
const programs = programCandidates(route, id);
|
|
450
|
+
const reusable = !force && previous && this.programsCoverCheck(previous, programs);
|
|
451
|
+
if (programs.length && reusable && !lostAccess && !modelDenied && !programAuthFailure) {
|
|
452
|
+
for (const program of programs)
|
|
453
|
+
result.status[`program:cyber:${program}`] = previous.status[`program:cyber:${program}`] ?? "unverified";
|
|
454
|
+
result.programCheckedAt = previous.programCheckedAt;
|
|
455
|
+
}
|
|
456
|
+
else if (programs.length) {
|
|
457
|
+
let inconclusive = false;
|
|
458
|
+
for (const program of programs) {
|
|
459
|
+
const name = `program:cyber:${program}`;
|
|
460
|
+
const status = await attempt({ program, effort });
|
|
461
|
+
if (status === "unverified")
|
|
462
|
+
inconclusive = true;
|
|
463
|
+
result.status[name] = !force && !programAuthFailure ? keep(name, status) : status;
|
|
464
|
+
}
|
|
465
|
+
// Never renew old evidence after throttling, transport failure or a missing echo.
|
|
466
|
+
result.programCheckedAt = inconclusive ? previous?.programCheckedAt : this.now();
|
|
467
|
+
if (inconclusive) {
|
|
468
|
+
result.status.retry = "transient";
|
|
469
|
+
result.programRetry = true;
|
|
470
|
+
}
|
|
471
|
+
if (programAuthFailure) {
|
|
472
|
+
for (const program of programs)
|
|
473
|
+
result.status[`program:cyber:${program}`] = "unverified";
|
|
474
|
+
}
|
|
475
|
+
}
|
|
376
476
|
if (modelDenied) {
|
|
377
477
|
result.levels = [];
|
|
378
478
|
result.tiers = [];
|
|
379
479
|
for (const name of Object.keys(result.status))
|
|
380
480
|
result.status[name] = "unsupported";
|
|
381
481
|
result.status.responses = "unsupported";
|
|
482
|
+
if (!lostAccess) {
|
|
483
|
+
// A structured model denial conclusively denies every candidate,
|
|
484
|
+
// including checks skipped after the denial. Do not retry it as a
|
|
485
|
+
// transport failure merely because those POSTs were unnecessary.
|
|
486
|
+
delete result.status.retry;
|
|
487
|
+
delete result.programRetry;
|
|
488
|
+
if (programs.length)
|
|
489
|
+
result.programCheckedAt = this.now();
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
if (lostAccess) {
|
|
493
|
+
result.levels = [];
|
|
494
|
+
result.tiers = [];
|
|
495
|
+
for (const name of Object.keys(result.status))
|
|
496
|
+
result.status[name] = "unverified";
|
|
382
497
|
}
|
|
383
498
|
if (credentialUnavailable && !modelDenied)
|
|
384
499
|
result.status.retry = "transient";
|
|
@@ -388,15 +503,21 @@ export class ApiModelCapabilities {
|
|
|
388
503
|
this.probes.set(key, result);
|
|
389
504
|
return result;
|
|
390
505
|
}
|
|
391
|
-
async enrich(models, refresh = false, route, forceProbes = false
|
|
506
|
+
async enrich(models, refresh = false, route, forceProbes = false,
|
|
507
|
+
// Supplying evidence limits this call to cache reads, even when forced.
|
|
508
|
+
cachedProbesOnly) {
|
|
509
|
+
const cached = cachedProbesOnly === undefined ? undefined : new Map(cachedProbesOnly);
|
|
392
510
|
return mapWithConcurrency(models, 3, async (model) => {
|
|
393
511
|
const documented = await this.documentation(model.slug, refresh);
|
|
394
512
|
let levels = documented.levels;
|
|
395
513
|
let extra = {};
|
|
396
514
|
if (route?.enabled && route.probeCapabilities) {
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
515
|
+
let probed = cached ? this.freshProbe(route, model.slug, cached.get(probeKey(route, model.slug))) : await this.probe(route, model.slug, levels, forceProbes, documented.reasoningAvailable === true);
|
|
516
|
+
if (probed && cached)
|
|
517
|
+
probed = this.reconcileReasoning(probed, levels, documented.reasoningAvailable === true);
|
|
518
|
+
if (probed)
|
|
519
|
+
levels = probed.levels;
|
|
520
|
+
extra = probed ? {
|
|
400
521
|
service_tiers: probed.tiers.map((id) => ({
|
|
401
522
|
id: id === "fast" ? "priority" : id,
|
|
402
523
|
name: id === "fast" ? "Fast" : "Ultrafast",
|
|
@@ -406,6 +527,16 @@ export class ApiModelCapabilities {
|
|
|
406
527
|
additional_speed_tiers: probed.tiers,
|
|
407
528
|
capability_probe_status: probed.status,
|
|
408
529
|
capability_checked_at: probed.at,
|
|
530
|
+
program_checked_at: probed.programCheckedAt,
|
|
531
|
+
} : {
|
|
532
|
+
// A previously enriched model must not smuggle expired or
|
|
533
|
+
// differently scoped evidence through a cache-only miss.
|
|
534
|
+
service_tiers: [],
|
|
535
|
+
default_service_tier: "default",
|
|
536
|
+
additional_speed_tiers: [],
|
|
537
|
+
capability_probe_status: undefined,
|
|
538
|
+
capability_checked_at: undefined,
|
|
539
|
+
program_checked_at: undefined,
|
|
409
540
|
};
|
|
410
541
|
}
|
|
411
542
|
// Ultra is a native orchestration mode. Codex resolves it to this
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { WebSocketDisconnectDiagnostic } from "./responses-websocket.js";
|
|
1
2
|
export interface AppBindPaths {
|
|
2
3
|
codexHome: string;
|
|
3
4
|
configPath: string;
|
|
@@ -37,6 +38,8 @@ export interface AppBindState {
|
|
|
37
38
|
updatedAt: number;
|
|
38
39
|
}
|
|
39
40
|
export interface AppBindRouterStatus {
|
|
41
|
+
lastWebsocketDisconnect?: WebSocketDisconnectDiagnostic | null;
|
|
42
|
+
schedulingStrategy?: "hybrid" | "sequential";
|
|
40
43
|
state: string | null;
|
|
41
44
|
pid: number | null;
|
|
42
45
|
startedAt?: number | null;
|
|
@@ -142,4 +145,5 @@ export declare function stopRuntimeRotationAppHelperProcess(helper: RuntimeRotat
|
|
|
142
145
|
export declare function getAppBindStatus(options?: AppBindOptions): Promise<AppBindStatus>;
|
|
143
146
|
export declare function bindCodexAppRuntimeRotation(options?: AppBindOptions): Promise<AppBindResult>;
|
|
144
147
|
export declare function unbindCodexAppRuntimeRotation(options?: AppBindOptions): Promise<AppBindResult>;
|
|
148
|
+
export declare function formatWebsocketDisconnect(value: unknown): string | null;
|
|
145
149
|
export declare function formatAppBindStatus(status: AppBindStatus): string;
|
|
@@ -228,11 +228,34 @@ async function readAppBindBackup(path) {
|
|
|
228
228
|
createdAt,
|
|
229
229
|
};
|
|
230
230
|
}
|
|
231
|
+
function readWebsocketDisconnect(value) {
|
|
232
|
+
if (!isRecord(value) || typeof value.at !== "number" || !Number.isFinite(value.at) || value.at < 0)
|
|
233
|
+
return null;
|
|
234
|
+
const { cause, stage, closeCode, routeKind, connectionAgeMs, eventCount } = value;
|
|
235
|
+
if (routeKind !== undefined && routeKind !== "oauth" && routeKind !== "api" && routeKind !== "zdr")
|
|
236
|
+
return null;
|
|
237
|
+
for (const count of [connectionAgeMs, eventCount])
|
|
238
|
+
if (count !== undefined && (typeof count !== "number" || !Number.isSafeInteger(count) || count < 0))
|
|
239
|
+
return null;
|
|
240
|
+
if (cause !== "close" && cause !== "transport_error" && cause !== "invalid_event" && cause !== "buffer_limit" && cause !== "send_error")
|
|
241
|
+
return null;
|
|
242
|
+
if (stage !== "awaiting_first_event" && stage !== "streaming")
|
|
243
|
+
return null;
|
|
244
|
+
if (closeCode !== undefined && (typeof closeCode !== "number" || !Number.isInteger(closeCode) || closeCode < 1000 || closeCode > 4999))
|
|
245
|
+
return null;
|
|
246
|
+
return { at: value.at, cause, stage, ...(typeof closeCode === "number" ? { closeCode } : {}),
|
|
247
|
+
...(routeKind === "oauth" || routeKind === "api" || routeKind === "zdr" ? { routeKind } : {}),
|
|
248
|
+
...(typeof connectionAgeMs === "number" ? { connectionAgeMs } : {}),
|
|
249
|
+
...(typeof eventCount === "number" ? { eventCount } : {}) };
|
|
250
|
+
}
|
|
231
251
|
async function readRouterStatus(path) {
|
|
232
252
|
const record = await readJsonFile(path);
|
|
233
253
|
if (!record)
|
|
234
254
|
return null;
|
|
235
255
|
return {
|
|
256
|
+
lastWebsocketDisconnect: readWebsocketDisconnect(record.lastWebsocketDisconnect),
|
|
257
|
+
schedulingStrategy: record.schedulingStrategy === "hybrid" || record.schedulingStrategy === "sequential"
|
|
258
|
+
? record.schedulingStrategy : undefined,
|
|
236
259
|
state: readString(record, "state"),
|
|
237
260
|
pid: readNumber(record, "pid"),
|
|
238
261
|
startedAt: readNumber(record, "startedAt"),
|
|
@@ -1380,6 +1403,12 @@ async function unbindCodexAppRuntimeRotationLocked(options, paths) {
|
|
|
1380
1403
|
message,
|
|
1381
1404
|
};
|
|
1382
1405
|
}
|
|
1406
|
+
export function formatWebsocketDisconnect(value) {
|
|
1407
|
+
const diagnostic = readWebsocketDisconnect(value);
|
|
1408
|
+
if (!diagnostic || diagnostic.at > 8.64e15)
|
|
1409
|
+
return null;
|
|
1410
|
+
return `Last WebSocket disconnect: ${new Date(diagnostic.at).toISOString()}; cause=${diagnostic.cause}; stage=${diagnostic.stage}${diagnostic.closeCode !== undefined ? `; closeCode=${diagnostic.closeCode}` : ""}${diagnostic.routeKind !== undefined ? `; route=${diagnostic.routeKind}` : ""}${diagnostic.connectionAgeMs !== undefined ? `; connectionAgeMs=${diagnostic.connectionAgeMs}` : ""}${diagnostic.eventCount !== undefined ? `; events=${diagnostic.eventCount}` : ""}`;
|
|
1411
|
+
}
|
|
1383
1412
|
export function formatAppBindStatus(status) {
|
|
1384
1413
|
if (status.unmanagedBind && !status.state) {
|
|
1385
1414
|
return [
|
|
@@ -1404,10 +1433,12 @@ export function formatAppBindStatus(status) {
|
|
|
1404
1433
|
else if (status.router?.lastAccountIndex !== null && status.router?.lastAccountIndex !== undefined) {
|
|
1405
1434
|
parts.push(`lastAccount=Account ${status.router.lastAccountIndex + 1}`);
|
|
1406
1435
|
}
|
|
1436
|
+
const diagnostic = formatWebsocketDisconnect(status.router?.lastWebsocketDisconnect);
|
|
1407
1437
|
if (status.state.nativeOpenai)
|
|
1408
|
-
return `Codex app bind: ${parts.join(", ")}, native OpenAI, live account catalogs
|
|
1438
|
+
return [`Codex app bind: ${parts.join(", ")}, native OpenAI, live account catalogs`, ...(diagnostic ? [diagnostic] : [])].join("\n");
|
|
1409
1439
|
return [
|
|
1410
1440
|
`Codex app bind: ${parts.join(", ")}`,
|
|
1441
|
+
...(diagnostic ? [diagnostic] : []),
|
|
1411
1442
|
[
|
|
1412
1443
|
"Note: Codex Desktop may hide history while the app bind selects the",
|
|
1413
1444
|
"codex-multi-auth-runtime-proxy provider; use `codex-multi-auth rotation",
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { type ApiRouteCredential } from "../api-route-store.js";
|
|
2
|
+
export declare const AUTOMATIC_API_CHECK_INTERVAL_MS: number;
|
|
3
|
+
export declare function runAutomaticApiChecks(options: {
|
|
4
|
+
path: string;
|
|
5
|
+
loadRoutes: () => Promise<ApiRouteCredential[]>;
|
|
6
|
+
prepareRoute?: (route: ApiRouteCredential, signal: AbortSignal) => Promise<ApiRouteCredential | undefined>;
|
|
7
|
+
check: (route: ApiRouteCredential, signal: AbortSignal, programRefreshBefore: number) => Promise<void>;
|
|
8
|
+
now?: () => number;
|
|
9
|
+
signal?: AbortSignal;
|
|
10
|
+
}): Promise<number | undefined>;
|
|
11
|
+
export declare function createAutomaticApiCheck(upstreamFetch?: typeof fetch): (signal: AbortSignal) => Promise<number | undefined>;
|
|
12
|
+
/** Daily checks follow persisted claim deadlines, never overlap, and drain on shutdown. */
|
|
13
|
+
export declare function startAutomaticApiChecks(run: (signal: AbortSignal) => Promise<number | void>): {
|
|
14
|
+
stop(): Promise<void>;
|
|
15
|
+
};
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { promises as fs } from "node:fs";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
import { loadApiRoutes, updateApiModelDiscovery, ensureApiRouteCapabilityEpoch } from "../api-route-store.js";
|
|
6
|
+
import { getCodexMultiAuthDir } from "../runtime-paths.js";
|
|
7
|
+
import { withFileTransactionLock } from "../storage/file-lock.js";
|
|
8
|
+
import { tempPathFor } from "../temp-path.js";
|
|
9
|
+
import { withRetry } from "../fs-retry.js";
|
|
10
|
+
import { combineSignals } from "../utils.js";
|
|
11
|
+
import { logWarn } from "../logger.js";
|
|
12
|
+
import { discoverModelInventory, saveModelInventory } from "./model-discovery-status.js";
|
|
13
|
+
export const AUTOMATIC_API_CHECK_INTERVAL_MS = 24 * 60 * 60_000;
|
|
14
|
+
const PREPARATION_RETRY_MS = 60_000;
|
|
15
|
+
const schema = z.record(z.string().regex(/^[a-f0-9]{64}$/), z.number().finite().nonnegative());
|
|
16
|
+
const retry = { maxAttempts: 6, backoffMs: 25 };
|
|
17
|
+
function key(route) { return createHash("sha256").update(JSON.stringify([route.id, route.kind, route.apiKey, route.visibleModels])).digest("hex"); }
|
|
18
|
+
const eligible = (route) => route.enabled && route.probeCapabilities === true;
|
|
19
|
+
// Visibility can expand during discovery; daily spending remains credential-scoped.
|
|
20
|
+
function claimKey(route) { return createHash("sha256").update(JSON.stringify([route.id, route.kind, route.apiKey])).digest("hex"); }
|
|
21
|
+
// Compare visibility as a set without changing stored order or legacy claim hashes.
|
|
22
|
+
function semanticRoute(route) { return { ...route, visibleModels: [...new Set(route.visibleModels)].sort() }; }
|
|
23
|
+
function sameRoute(a, b) { return JSON.stringify(Object.entries(semanticRoute(a)).sort(([x], [y]) => x.localeCompare(y))) === JSON.stringify(Object.entries(semanticRoute(b)).sort(([x], [y]) => x.localeCompare(y))); }
|
|
24
|
+
export async function runAutomaticApiChecks(options) {
|
|
25
|
+
const signal = options.signal ?? new AbortController().signal;
|
|
26
|
+
// Claims name the scheduled pass, so I/O and later routes cannot move its horizon.
|
|
27
|
+
const startedAt = options.now?.() ?? Date.now();
|
|
28
|
+
let nextRunAt;
|
|
29
|
+
const includeDeadline = (at) => { nextRunAt = Math.min(nextRunAt ?? Infinity, at + AUTOMATIC_API_CHECK_INTERVAL_MS); };
|
|
30
|
+
if (signal.aborted)
|
|
31
|
+
return;
|
|
32
|
+
const routes = (await options.loadRoutes()).filter(eligible);
|
|
33
|
+
if (!routes.length)
|
|
34
|
+
return;
|
|
35
|
+
for (const route of routes) {
|
|
36
|
+
if (signal.aborted)
|
|
37
|
+
return;
|
|
38
|
+
const observed = (await options.loadRoutes()).find(r => eligible(r) && key(semanticRoute(r)) === key(semanticRoute(route)));
|
|
39
|
+
if (!observed)
|
|
40
|
+
continue;
|
|
41
|
+
// Complete store-owned migrations before capturing the exact claim snapshot.
|
|
42
|
+
let current;
|
|
43
|
+
try {
|
|
44
|
+
current = options.prepareRoute ? await options.prepareRoute(observed, signal) : observed;
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
if (signal.aborted)
|
|
48
|
+
return;
|
|
49
|
+
nextRunAt = Math.min(nextRunAt ?? Infinity, (options.now?.() ?? Date.now()) + PREPARATION_RETRY_MS);
|
|
50
|
+
logWarn("Automatic API route preparation failed; retrying shortly.");
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
if (!current || !eligible(current) || signal.aborted)
|
|
54
|
+
continue;
|
|
55
|
+
// Commit the claim before network I/O. Concurrent routers and restarts share it.
|
|
56
|
+
let programRefreshBefore = startedAt + AUTOMATIC_API_CHECK_INTERVAL_MS;
|
|
57
|
+
const claimed = await withFileTransactionLock(options.path, async () => {
|
|
58
|
+
// A credential wipe can finish while this checker is queued for the lock.
|
|
59
|
+
if (signal.aborted || !(await options.loadRoutes()).some(r => eligible(r) && sameRoute(r, current)))
|
|
60
|
+
return false;
|
|
61
|
+
let attempts = {};
|
|
62
|
+
let raw, repairHistory = false;
|
|
63
|
+
try {
|
|
64
|
+
raw = await withRetry(() => fs.readFile(options.path, "utf8"), retry);
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
if (error.code !== "ENOENT")
|
|
68
|
+
throw error;
|
|
69
|
+
}
|
|
70
|
+
if (raw !== undefined)
|
|
71
|
+
try {
|
|
72
|
+
if (Buffer.byteLength(raw) > 1024 * 1024)
|
|
73
|
+
throw Error("API-check history too large");
|
|
74
|
+
attempts = schema.parse(JSON.parse(raw));
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
repairHistory = true;
|
|
78
|
+
}
|
|
79
|
+
const now = options.now?.() ?? Date.now(), id = claimKey(current);
|
|
80
|
+
// Unknown prior attempts must not trigger another billable pass on repair.
|
|
81
|
+
// Seed every eligible route so a restart can resume normally the next day.
|
|
82
|
+
if (repairHistory)
|
|
83
|
+
attempts = Object.fromEntries((await options.loadRoutes()).filter(eligible).map(route => [claimKey(route), now]));
|
|
84
|
+
const legacyClaims = [attempts[key(current)], attempts[key(observed)], attempts[key(route)]].filter((at) => at !== undefined && Number.isFinite(at) && at <= now + 300_000);
|
|
85
|
+
const last = attempts[id] ?? (legacyClaims.length ? Math.max(...legacyClaims) : undefined);
|
|
86
|
+
if (signal.aborted)
|
|
87
|
+
return false;
|
|
88
|
+
if (!repairHistory && last !== undefined && last <= now + 300_000 && now - last < AUTOMATIC_API_CHECK_INTERVAL_MS) {
|
|
89
|
+
includeDeadline(last);
|
|
90
|
+
return false;
|
|
91
|
+
}
|
|
92
|
+
// Keep other processes' claims; prune only the oldest history at a fixed bound.
|
|
93
|
+
for (const [old] of Object.entries(attempts).sort((a, b) => b[1] - a[1]).slice(999))
|
|
94
|
+
delete attempts[old];
|
|
95
|
+
const claimedAt = repairHistory ? now : Math.min(startedAt, now);
|
|
96
|
+
attempts[id] = claimedAt;
|
|
97
|
+
programRefreshBefore = claimedAt + AUTOMATIC_API_CHECK_INTERVAL_MS;
|
|
98
|
+
includeDeadline(claimedAt);
|
|
99
|
+
const temp = tempPathFor(options.path);
|
|
100
|
+
await fs.mkdir(dirname(options.path), { recursive: true, mode: 0o700 });
|
|
101
|
+
try {
|
|
102
|
+
await fs.writeFile(temp, JSON.stringify(attempts) + "\n", { mode: 0o600, flag: "wx" });
|
|
103
|
+
await withRetry(() => fs.rename(temp, options.path), retry);
|
|
104
|
+
}
|
|
105
|
+
finally {
|
|
106
|
+
await withRetry(() => fs.rm(temp, { force: true }), retry);
|
|
107
|
+
}
|
|
108
|
+
if (repairHistory)
|
|
109
|
+
logWarn("Automatic API check history repaired; billable checks deferred for one daily interval.");
|
|
110
|
+
return !repairHistory;
|
|
111
|
+
}, { signal });
|
|
112
|
+
if (!claimed || signal.aborted)
|
|
113
|
+
continue;
|
|
114
|
+
try {
|
|
115
|
+
await options.check(current, signal, programRefreshBefore);
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
if (!signal.aborted)
|
|
119
|
+
logWarn("Automatic API capability check failed; retry deferred until the next daily interval.");
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
return nextRunAt;
|
|
123
|
+
}
|
|
124
|
+
export function createAutomaticApiCheck(upstreamFetch = fetch) {
|
|
125
|
+
return async (signal) => {
|
|
126
|
+
return runAutomaticApiChecks({
|
|
127
|
+
path: join(getCodexMultiAuthDir(), "automatic-api-checks.json"), loadRoutes: loadApiRoutes, signal, prepareRoute: ensureApiRouteCapabilityEpoch,
|
|
128
|
+
check: async (route, stop, programRefreshBefore) => {
|
|
129
|
+
let acceptedRoute = route, discoveryAccepted = true;
|
|
130
|
+
const updateDiscovery = async (original, models) => {
|
|
131
|
+
discoveryAccepted = false;
|
|
132
|
+
const known = new Set(original.knownModels ?? models), visible = new Set(original.visibleModels);
|
|
133
|
+
for (const model of models) {
|
|
134
|
+
if (!known.has(model) && /^gpt-/i.test(model))
|
|
135
|
+
visible.add(model);
|
|
136
|
+
known.add(model);
|
|
137
|
+
}
|
|
138
|
+
const expected = { ...original, knownModels: [...known], visibleModels: [...visible] };
|
|
139
|
+
const updated = await updateApiModelDiscovery(original, models);
|
|
140
|
+
// Accept this discovery's expansion only, never a concurrent policy edit.
|
|
141
|
+
if (!sameRoute(updated, expected))
|
|
142
|
+
throw Error("API check route changed");
|
|
143
|
+
acceptedRoute = updated;
|
|
144
|
+
discoveryAccepted = true;
|
|
145
|
+
return updated;
|
|
146
|
+
};
|
|
147
|
+
// Link shutdown to each existing bounded request without weakening its timeout.
|
|
148
|
+
const fetcher = async (input, init) => {
|
|
149
|
+
const signal = combineSignals(stop, init?.signal);
|
|
150
|
+
signal.throwIfAborted();
|
|
151
|
+
if (init?.method?.toUpperCase() === "POST" && (!discoveryAccepted || !(await loadApiRoutes()).some(r => eligible(r) && sameRoute(r, acceptedRoute))))
|
|
152
|
+
throw Error("API check route changed");
|
|
153
|
+
signal.throwIfAborted();
|
|
154
|
+
return upstreamFetch(input, { ...init, signal });
|
|
155
|
+
};
|
|
156
|
+
const inventory = await discoverModelInventory(null, [route], { fetchImpl: fetcher, signal: stop, persistProbes: true, programRefreshBefore, updateApiDiscovery: updateDiscovery });
|
|
157
|
+
if (stop.aborted || !discoveryAccepted)
|
|
158
|
+
return;
|
|
159
|
+
const current = (await loadApiRoutes()).find(r => eligible(r) && sameRoute(r, acceptedRoute));
|
|
160
|
+
if (current)
|
|
161
|
+
await saveModelInventory(inventory, { mergeEntries: true, signal: stop });
|
|
162
|
+
},
|
|
163
|
+
});
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
/** Daily checks follow persisted claim deadlines, never overlap, and drain on shutdown. */
|
|
167
|
+
export function startAutomaticApiChecks(run) {
|
|
168
|
+
const controller = new AbortController();
|
|
169
|
+
let pending;
|
|
170
|
+
let timer;
|
|
171
|
+
const schedule = (delay) => { if (!controller.signal.aborted) {
|
|
172
|
+
timer = setTimeout(tick, delay);
|
|
173
|
+
timer.unref();
|
|
174
|
+
} };
|
|
175
|
+
const tick = () => {
|
|
176
|
+
timer = undefined;
|
|
177
|
+
if (controller.signal.aborted)
|
|
178
|
+
return;
|
|
179
|
+
let nextRunAt = Date.now() + AUTOMATIC_API_CHECK_INTERVAL_MS;
|
|
180
|
+
pending = Promise.resolve().then(() => run(controller.signal)).then(due => { if (typeof due === "number" && Number.isFinite(due))
|
|
181
|
+
nextRunAt = due; })
|
|
182
|
+
.catch(() => { if (!controller.signal.aborted)
|
|
183
|
+
logWarn("Automatic API capability checks unavailable; retry deferred."); })
|
|
184
|
+
.finally(() => { pending = undefined; schedule(Math.min(AUTOMATIC_API_CHECK_INTERVAL_MS, Math.max(0, nextRunAt - Date.now()))); });
|
|
185
|
+
};
|
|
186
|
+
schedule(5000);
|
|
187
|
+
return { async stop() { clearTimeout(timer); controller.abort(); await pending; } };
|
|
188
|
+
}
|
|
@@ -22,6 +22,7 @@ declare const schema: z.ZodObject<{
|
|
|
22
22
|
entitlements: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
23
23
|
model: z.ZodString;
|
|
24
24
|
checkedAt: z.ZodOptional<z.ZodNumber>;
|
|
25
|
+
programCheckedAt: z.ZodOptional<z.ZodNumber>;
|
|
25
26
|
probes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodEnum<{
|
|
26
27
|
downgraded: "downgraded";
|
|
27
28
|
unsupported: "unsupported";
|
|
@@ -50,6 +51,7 @@ declare const schema: z.ZodObject<{
|
|
|
50
51
|
entitlements: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
51
52
|
model: z.ZodString;
|
|
52
53
|
checkedAt: z.ZodOptional<z.ZodNumber>;
|
|
54
|
+
programCheckedAt: z.ZodOptional<z.ZodNumber>;
|
|
53
55
|
probes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodEnum<{
|
|
54
56
|
downgraded: "downgraded";
|
|
55
57
|
unsupported: "unsupported";
|
|
@@ -68,6 +70,7 @@ declare const schema: z.ZodObject<{
|
|
|
68
70
|
export type ModelInventory = z.infer<typeof schema>;
|
|
69
71
|
export declare function discoverModelInventory(storage: AccountStorageV3 | null, routes: ApiRouteCredential[], options?: {
|
|
70
72
|
fetchImpl?: typeof fetch;
|
|
73
|
+
signal?: AbortSignal;
|
|
71
74
|
now?: () => number;
|
|
72
75
|
clientVersion?: string;
|
|
73
76
|
updateApiDiscovery?: typeof updateApiModelDiscovery;
|
|
@@ -75,9 +78,16 @@ export declare function discoverModelInventory(storage: AccountStorageV3 | null,
|
|
|
75
78
|
forceProbes?: boolean;
|
|
76
79
|
/** Share probe results with later CLI processes through the multi-auth directory. */
|
|
77
80
|
persistProbes?: boolean;
|
|
81
|
+
/** Refresh program evidence that expires before the next scheduled check. */
|
|
82
|
+
programRefreshBefore?: number;
|
|
78
83
|
}): Promise<ModelInventory>;
|
|
79
84
|
export declare function loadModelInventory(): Promise<ModelInventory | null>;
|
|
80
|
-
|
|
85
|
+
/** Drain queued writes before resetting process-local test state. */
|
|
86
|
+
export declare function resetModelInventoryWritesForTests(): Promise<void>;
|
|
87
|
+
export declare function saveModelInventory(value: ModelInventory, options?: {
|
|
88
|
+
mergeEntries?: boolean;
|
|
89
|
+
signal?: AbortSignal;
|
|
90
|
+
}): Promise<void>;
|
|
81
91
|
export declare function formatModelInventory(inventory: ModelInventory | null, now?: number): string[];
|
|
82
92
|
export declare function refreshAndPrintModelInventory(log: (message: string) => void, deps?: {
|
|
83
93
|
getBinding?: () => Promise<{
|