@askalf/dario 6.0.13 → 6.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/codex-backend.d.ts +7 -1
- package/dist/codex-backend.js +16 -1
- package/dist/live-fingerprint.d.ts +1 -1
- package/dist/live-fingerprint.js +1 -1
- package/dist/provider-cooldown.d.ts +78 -0
- package/dist/provider-cooldown.js +120 -0
- package/dist/proxy.js +119 -10
- package/package.json +1 -1
package/dist/codex-backend.d.ts
CHANGED
|
@@ -60,6 +60,12 @@ export interface CodexForwardOutcome {
|
|
|
60
60
|
model: string;
|
|
61
61
|
alias: string;
|
|
62
62
|
}
|
|
63
|
+
/** Why a forward DECLINED, for the caller's cool-down bookkeeping. `status` is
|
|
64
|
+
* the upstream status, or 0 when the request never got one (transport). */
|
|
65
|
+
export interface CodexDecline {
|
|
66
|
+
status: number;
|
|
67
|
+
retryAfterMs: number | null;
|
|
68
|
+
}
|
|
63
69
|
/** The cached slug list for an alias WITHOUT fetching. For the admin surface:
|
|
64
70
|
* a status read must never cost an upstream call or a token refresh. */
|
|
65
71
|
export declare function peekCodexModelSlugs(alias: string): readonly string[] | null;
|
|
@@ -216,4 +222,4 @@ export declare function buildCodexHeaders(creds: CodexAccountCredentials): Recor
|
|
|
216
222
|
* testable without network (test/codex-backend.mjs), matching the pattern
|
|
217
223
|
* test/codex-oauth.mjs already uses.
|
|
218
224
|
*/
|
|
219
|
-
export declare function forwardToCodex(req: IncomingMessage, res: ServerResponse, body: Buffer, creds: CodexAccountCredentials, corsOrigin: string, securityHeaders: Record<string, string>, upstreamTimeoutMs: number, verbose: boolean, shape?: CodexRequestShape, fetchImpl?: typeof fetch, deferOnUnavailable?: boolean, onDone?: (outcome: CodexForwardOutcome) => void): Promise<boolean>;
|
|
225
|
+
export declare function forwardToCodex(req: IncomingMessage, res: ServerResponse, body: Buffer, creds: CodexAccountCredentials, corsOrigin: string, securityHeaders: Record<string, string>, upstreamTimeoutMs: number, verbose: boolean, shape?: CodexRequestShape, fetchImpl?: typeof fetch, deferOnUnavailable?: boolean, onDone?: (outcome: CodexForwardOutcome) => void, onDecline?: (info: CodexDecline) => void): Promise<boolean>;
|
package/dist/codex-backend.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { anthropicToResponsesRequest, createResponsesSSEParser, formatResponsesAnthropicSSE, createAnthropicMessageAssembler, responsesStreamToAnthropicSSE, } from './anthropic-responses-translate.js';
|
|
2
2
|
import { resolveClaudeTarget } from './claude-model.js';
|
|
3
3
|
import { BAKED_BASE_MODELS } from './model-catalog.js';
|
|
4
|
+
import { parseRetryAfterMs } from './provider-cooldown.js';
|
|
4
5
|
export const CODEX_BACKEND_BASE_URL = process.env.DARIO_CODEX_BASE_URL || 'https://chatgpt.com/backend-api/codex';
|
|
5
6
|
/** Originator string the codex CLI identifies itself with. */
|
|
6
7
|
const CODEX_ORIGINATOR = 'codex_cli_rs';
|
|
@@ -518,7 +519,7 @@ export function buildCodexHeaders(creds) {
|
|
|
518
519
|
* testable without network (test/codex-backend.mjs), matching the pattern
|
|
519
520
|
* test/codex-oauth.mjs already uses.
|
|
520
521
|
*/
|
|
521
|
-
export async function forwardToCodex(req, res, body, creds, corsOrigin, securityHeaders, upstreamTimeoutMs, verbose, shape = 'openai', fetchImpl = fetch, deferOnUnavailable = false, onDone) {
|
|
522
|
+
export async function forwardToCodex(req, res, body, creds, corsOrigin, securityHeaders, upstreamTimeoutMs, verbose, shape = 'openai', fetchImpl = fetch, deferOnUnavailable = false, onDone, onDecline) {
|
|
522
523
|
void req;
|
|
523
524
|
const isAnthropic = shape === 'anthropic';
|
|
524
525
|
// Reported exactly once, on every exit that answered the client. Without
|
|
@@ -580,6 +581,14 @@ export async function forwardToCodex(req, res, body, creds, corsOrigin, security
|
|
|
580
581
|
const unavailable = upstream.status === 429 || upstream.status >= 500;
|
|
581
582
|
if (deferOnUnavailable && unavailable) {
|
|
582
583
|
console.log(`[dario] codex account ${creds.alias} unavailable (${upstream.status}) — deferring to the next provider`);
|
|
584
|
+
// A decline is the only exit that tells the caller nothing was served,
|
|
585
|
+
// and until now it carried no WHY: a 429 and a 503 were the same false.
|
|
586
|
+
// The chain needs the status (to cool a rate limit but not an outage)
|
|
587
|
+
// and the upstream's own `retry-after` (to cool it for the right long).
|
|
588
|
+
try {
|
|
589
|
+
onDecline?.({ status: upstream.status, retryAfterMs: parseRetryAfterMs(upstream.headers.get('retry-after')) });
|
|
590
|
+
}
|
|
591
|
+
catch { /* a reporting failure must never break a declined request */ }
|
|
583
592
|
return false;
|
|
584
593
|
}
|
|
585
594
|
res.writeHead(upstream.status, { 'Content-Type': 'application/json', ...securityHeaders });
|
|
@@ -737,6 +746,12 @@ export async function forwardToCodex(req, res, body, creds, corsOrigin, security
|
|
|
737
746
|
// the request to anyone else.
|
|
738
747
|
if (deferOnUnavailable && !res.headersSent) {
|
|
739
748
|
console.log(`[dario] codex account ${creds.alias} unreachable (${detail}) — deferring to the next provider`);
|
|
749
|
+
// status 0: no HTTP status ever arrived. Reported so the caller can tell
|
|
750
|
+
// an outage from a rate limit — an unreachable backend is not quota.
|
|
751
|
+
try {
|
|
752
|
+
onDecline?.({ status: 0, retryAfterMs: null });
|
|
753
|
+
}
|
|
754
|
+
catch { /* as above */ }
|
|
740
755
|
return false;
|
|
741
756
|
}
|
|
742
757
|
if (!res.headersSent) {
|
|
@@ -496,7 +496,7 @@ export declare function detectDrift(t: TemplateData, installedOverride?: string
|
|
|
496
496
|
*/
|
|
497
497
|
export declare const SUPPORTED_CC_RANGE: {
|
|
498
498
|
readonly min: "1.0.0";
|
|
499
|
-
readonly maxTested: "2.1.
|
|
499
|
+
readonly maxTested: "2.1.259";
|
|
500
500
|
};
|
|
501
501
|
/**
|
|
502
502
|
* Compare two dotted-numeric version strings. Returns negative if `a<b`,
|
package/dist/live-fingerprint.js
CHANGED
|
@@ -1194,7 +1194,7 @@ export function detectDrift(t, installedOverride) {
|
|
|
1194
1194
|
*/
|
|
1195
1195
|
export const SUPPORTED_CC_RANGE = {
|
|
1196
1196
|
min: '1.0.0',
|
|
1197
|
-
maxTested: '2.1.
|
|
1197
|
+
maxTested: '2.1.259',
|
|
1198
1198
|
};
|
|
1199
1199
|
/**
|
|
1200
1200
|
* Compare two dotted-numeric version strings. Returns negative if `a<b`,
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-provider rate-limit cool-down for the failover chain (DEV-f66b131c).
|
|
3
|
+
*
|
|
4
|
+
* The chain used to have no memory. Every 429 was a fresh discovery, so with
|
|
5
|
+
* BOTH ends capped — a spent Claude 5h window and a 429-ing ChatGPT
|
|
6
|
+
* subscription — one request walked codex → claude → codex before dropping,
|
|
7
|
+
* and the next request did it again. 185 such lines in 38 minutes on
|
|
8
|
+
* 2026-09-02. That is the wrong shape twice over: at the moment quota is the
|
|
9
|
+
* scarce resource the chain spends TWICE as much of it discovering there is
|
|
10
|
+
* none, and whatever is cycling when a window rolls over consumes the fresh
|
|
11
|
+
* window ahead of legitimately queued work.
|
|
12
|
+
*
|
|
13
|
+
* Two rules, both enforced here so the request path can stay a straight line:
|
|
14
|
+
*
|
|
15
|
+
* WITHIN a request — an entry that already declined is never revisited
|
|
16
|
+
* ({@link canAttempt} against the attempted set).
|
|
17
|
+
* ACROSS requests — a 429 cools the entry for a bounded interval, honouring
|
|
18
|
+
* `retry-after` when the upstream sends one.
|
|
19
|
+
*
|
|
20
|
+
* When every entry is cooled the caller fails fast with one terminal verdict
|
|
21
|
+
* instead of a retry storm — see {@link allProvidersCooled}.
|
|
22
|
+
*
|
|
23
|
+
* Provider-granular, not entry-granular: a chain names MODELS
|
|
24
|
+
* (`gpt-5.6-sol,claude-sonnet-5`) but a 429 is a property of the ACCOUNT
|
|
25
|
+
* behind them, so cooling `codex` also cools every other codex slug in the
|
|
26
|
+
* chain, which is the correct blast radius.
|
|
27
|
+
*
|
|
28
|
+
* The clock is injectable so the whole thing is testable without sleeping.
|
|
29
|
+
*/
|
|
30
|
+
/** Cool-down applied when the upstream gives no `retry-after`. */
|
|
31
|
+
export declare const DEFAULT_COOLDOWN_MS = 60000;
|
|
32
|
+
/**
|
|
33
|
+
* Ceiling on any cool-down. An upstream is free to say "come back in 4 hours";
|
|
34
|
+
* honouring that literally would keep a provider parked long after a window
|
|
35
|
+
* reset we cannot observe. One re-probe per 15 minutes is cheap.
|
|
36
|
+
*/
|
|
37
|
+
export declare const MAX_COOLDOWN_MS: number;
|
|
38
|
+
/** Machine-readable terminal verdict. Distinct from `pool exhausted` (which
|
|
39
|
+
* implies a peer exists) and from the billing/credential classes. */
|
|
40
|
+
export declare const ALL_PROVIDERS_RATE_LIMITED = "all-providers-rate-limited";
|
|
41
|
+
/**
|
|
42
|
+
* `retry-after` as milliseconds. The header is either delta-seconds or an
|
|
43
|
+
* HTTP-date (RFC 9110 §10.2.3); both are accepted, anything else is null so
|
|
44
|
+
* the caller falls back to its default rather than trusting a parse it did not
|
|
45
|
+
* get. A date already in the past yields 0 — "retry now" — not a negative.
|
|
46
|
+
*/
|
|
47
|
+
export declare function parseRetryAfterMs(value: string | null | undefined, now?: number): number | null;
|
|
48
|
+
/** A provider that declined, and for how long it should stay declined. */
|
|
49
|
+
export declare class ProviderCooldowns {
|
|
50
|
+
private readonly until;
|
|
51
|
+
private readonly now;
|
|
52
|
+
private readonly defaultMs;
|
|
53
|
+
constructor(now?: () => number, defaultMs?: number);
|
|
54
|
+
/**
|
|
55
|
+
* Record a 429 (or an equivalent "not right now") for `provider`. Returns the
|
|
56
|
+
* cool-down actually applied, so the caller can put it in its one log line.
|
|
57
|
+
* A `retryAfterMs` of 0 is honoured as "retry now" and cools nothing.
|
|
58
|
+
*/
|
|
59
|
+
note(provider: string, retryAfterMs?: number | null): number;
|
|
60
|
+
/** Clear on a success — a provider that just served is not rate-limited. */
|
|
61
|
+
clear(provider: string): void;
|
|
62
|
+
remainingMs(provider: string): number;
|
|
63
|
+
isCooled(provider: string): boolean;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* May this request try `provider`? False when it already declined during THIS
|
|
67
|
+
* request (the codex → claude → codex revisit) or when it is cooling from an
|
|
68
|
+
* earlier one.
|
|
69
|
+
*/
|
|
70
|
+
export declare function canAttempt(provider: string, attempted: ReadonlySet<string>, cooldowns: ProviderCooldowns): boolean;
|
|
71
|
+
/**
|
|
72
|
+
* Every named provider is cooling — the fail-fast condition. An EMPTY list is
|
|
73
|
+
* NOT "all cooled": nothing configured is a configuration answer, not a rate
|
|
74
|
+
* limit, and must keep its existing message.
|
|
75
|
+
*/
|
|
76
|
+
export declare function allProvidersCooled(providers: readonly string[], cooldowns: ProviderCooldowns): boolean;
|
|
77
|
+
/** Longest remaining cool-down across `providers` — what to put in `retry-after`. */
|
|
78
|
+
export declare function cooldownRetryAfterMs(providers: readonly string[], cooldowns: ProviderCooldowns): number;
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-provider rate-limit cool-down for the failover chain (DEV-f66b131c).
|
|
3
|
+
*
|
|
4
|
+
* The chain used to have no memory. Every 429 was a fresh discovery, so with
|
|
5
|
+
* BOTH ends capped — a spent Claude 5h window and a 429-ing ChatGPT
|
|
6
|
+
* subscription — one request walked codex → claude → codex before dropping,
|
|
7
|
+
* and the next request did it again. 185 such lines in 38 minutes on
|
|
8
|
+
* 2026-09-02. That is the wrong shape twice over: at the moment quota is the
|
|
9
|
+
* scarce resource the chain spends TWICE as much of it discovering there is
|
|
10
|
+
* none, and whatever is cycling when a window rolls over consumes the fresh
|
|
11
|
+
* window ahead of legitimately queued work.
|
|
12
|
+
*
|
|
13
|
+
* Two rules, both enforced here so the request path can stay a straight line:
|
|
14
|
+
*
|
|
15
|
+
* WITHIN a request — an entry that already declined is never revisited
|
|
16
|
+
* ({@link canAttempt} against the attempted set).
|
|
17
|
+
* ACROSS requests — a 429 cools the entry for a bounded interval, honouring
|
|
18
|
+
* `retry-after` when the upstream sends one.
|
|
19
|
+
*
|
|
20
|
+
* When every entry is cooled the caller fails fast with one terminal verdict
|
|
21
|
+
* instead of a retry storm — see {@link allProvidersCooled}.
|
|
22
|
+
*
|
|
23
|
+
* Provider-granular, not entry-granular: a chain names MODELS
|
|
24
|
+
* (`gpt-5.6-sol,claude-sonnet-5`) but a 429 is a property of the ACCOUNT
|
|
25
|
+
* behind them, so cooling `codex` also cools every other codex slug in the
|
|
26
|
+
* chain, which is the correct blast radius.
|
|
27
|
+
*
|
|
28
|
+
* The clock is injectable so the whole thing is testable without sleeping.
|
|
29
|
+
*/
|
|
30
|
+
/** Cool-down applied when the upstream gives no `retry-after`. */
|
|
31
|
+
export const DEFAULT_COOLDOWN_MS = 60_000;
|
|
32
|
+
/**
|
|
33
|
+
* Ceiling on any cool-down. An upstream is free to say "come back in 4 hours";
|
|
34
|
+
* honouring that literally would keep a provider parked long after a window
|
|
35
|
+
* reset we cannot observe. One re-probe per 15 minutes is cheap.
|
|
36
|
+
*/
|
|
37
|
+
export const MAX_COOLDOWN_MS = 15 * 60_000;
|
|
38
|
+
/** Machine-readable terminal verdict. Distinct from `pool exhausted` (which
|
|
39
|
+
* implies a peer exists) and from the billing/credential classes. */
|
|
40
|
+
export const ALL_PROVIDERS_RATE_LIMITED = 'all-providers-rate-limited';
|
|
41
|
+
/**
|
|
42
|
+
* `retry-after` as milliseconds. The header is either delta-seconds or an
|
|
43
|
+
* HTTP-date (RFC 9110 §10.2.3); both are accepted, anything else is null so
|
|
44
|
+
* the caller falls back to its default rather than trusting a parse it did not
|
|
45
|
+
* get. A date already in the past yields 0 — "retry now" — not a negative.
|
|
46
|
+
*/
|
|
47
|
+
export function parseRetryAfterMs(value, now = Date.now()) {
|
|
48
|
+
if (value == null)
|
|
49
|
+
return null;
|
|
50
|
+
const trimmed = value.trim();
|
|
51
|
+
if (trimmed.length === 0)
|
|
52
|
+
return null;
|
|
53
|
+
if (/^\d+$/.test(trimmed))
|
|
54
|
+
return Math.min(Number(trimmed) * 1000, MAX_COOLDOWN_MS);
|
|
55
|
+
const at = Date.parse(trimmed);
|
|
56
|
+
if (Number.isNaN(at))
|
|
57
|
+
return null;
|
|
58
|
+
return Math.min(Math.max(at - now, 0), MAX_COOLDOWN_MS);
|
|
59
|
+
}
|
|
60
|
+
/** A provider that declined, and for how long it should stay declined. */
|
|
61
|
+
export class ProviderCooldowns {
|
|
62
|
+
until = new Map();
|
|
63
|
+
now;
|
|
64
|
+
defaultMs;
|
|
65
|
+
constructor(now = () => Date.now(), defaultMs = DEFAULT_COOLDOWN_MS) {
|
|
66
|
+
this.now = now;
|
|
67
|
+
this.defaultMs = defaultMs;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Record a 429 (or an equivalent "not right now") for `provider`. Returns the
|
|
71
|
+
* cool-down actually applied, so the caller can put it in its one log line.
|
|
72
|
+
* A `retryAfterMs` of 0 is honoured as "retry now" and cools nothing.
|
|
73
|
+
*/
|
|
74
|
+
note(provider, retryAfterMs) {
|
|
75
|
+
const ms = retryAfterMs == null ? this.defaultMs : Math.min(Math.max(retryAfterMs, 0), MAX_COOLDOWN_MS);
|
|
76
|
+
this.until.set(provider, this.now() + ms);
|
|
77
|
+
return ms;
|
|
78
|
+
}
|
|
79
|
+
/** Clear on a success — a provider that just served is not rate-limited. */
|
|
80
|
+
clear(provider) {
|
|
81
|
+
this.until.delete(provider);
|
|
82
|
+
}
|
|
83
|
+
remainingMs(provider) {
|
|
84
|
+
const at = this.until.get(provider);
|
|
85
|
+
if (at == null)
|
|
86
|
+
return 0;
|
|
87
|
+
const left = at - this.now();
|
|
88
|
+
if (left <= 0) {
|
|
89
|
+
// Expired entries are dropped on read; nothing else sweeps this map and
|
|
90
|
+
// a long-lived proxy would otherwise accumulate one entry per provider
|
|
91
|
+
// forever (bounded, but pointlessly).
|
|
92
|
+
this.until.delete(provider);
|
|
93
|
+
return 0;
|
|
94
|
+
}
|
|
95
|
+
return left;
|
|
96
|
+
}
|
|
97
|
+
isCooled(provider) {
|
|
98
|
+
return this.remainingMs(provider) > 0;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* May this request try `provider`? False when it already declined during THIS
|
|
103
|
+
* request (the codex → claude → codex revisit) or when it is cooling from an
|
|
104
|
+
* earlier one.
|
|
105
|
+
*/
|
|
106
|
+
export function canAttempt(provider, attempted, cooldowns) {
|
|
107
|
+
return !attempted.has(provider) && !cooldowns.isCooled(provider);
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Every named provider is cooling — the fail-fast condition. An EMPTY list is
|
|
111
|
+
* NOT "all cooled": nothing configured is a configuration answer, not a rate
|
|
112
|
+
* limit, and must keep its existing message.
|
|
113
|
+
*/
|
|
114
|
+
export function allProvidersCooled(providers, cooldowns) {
|
|
115
|
+
return providers.length > 0 && providers.every(p => cooldowns.isCooled(p));
|
|
116
|
+
}
|
|
117
|
+
/** Longest remaining cool-down across `providers` — what to put in `retry-after`. */
|
|
118
|
+
export function cooldownRetryAfterMs(providers, cooldowns) {
|
|
119
|
+
return providers.reduce((max, p) => Math.max(max, cooldowns.remainingMs(p)), 0);
|
|
120
|
+
}
|
package/dist/proxy.js
CHANGED
|
@@ -28,6 +28,7 @@ import { RequestQueue, QueueFullError, QueueTimeoutError, DEFAULT_MAX_CONCURRENT
|
|
|
28
28
|
import { redactSecrets } from './redact.js';
|
|
29
29
|
import { BAKED_BASE_MODELS, withLongContextVariants, buildOpenAIModelsList, getModelCatalog, getCachedBases, resolveAliasAgainst, prewarmModelCatalog, retryModelCatalogNow, isSuspendedModel } from './model-catalog.js';
|
|
30
30
|
import { classifyUpstreamRejection, diagnosticSnippet } from './upstream-rejection.js';
|
|
31
|
+
import { ProviderCooldowns, canAttempt, allProvidersCooled, cooldownRetryAfterMs, parseRetryAfterMs, ALL_PROVIDERS_RATE_LIMITED, } from './provider-cooldown.js';
|
|
31
32
|
const ANTHROPIC_API = 'https://api.anthropic.com';
|
|
32
33
|
const DEFAULT_PORT = 3456;
|
|
33
34
|
const MAX_BODY_BYTES = 10 * 1024 * 1024; // 10 MB — generous for large prompts, prevents abuse
|
|
@@ -1328,6 +1329,10 @@ export async function startProxy(opts = {}) {
|
|
|
1328
1329
|
// Per-alias request counts for GET /codex — the pool has requestCount per
|
|
1329
1330
|
// account; the codex accounts had nothing until now.
|
|
1330
1331
|
const codexRequestCounts = new Map();
|
|
1332
|
+
// Rate-limit memory for the failover chain (DEV-f66b131c). Process-wide and
|
|
1333
|
+
// deliberately NOT per-request: the point is that request #269 does not
|
|
1334
|
+
// re-discover the limit #267 already found. Cleared on any success.
|
|
1335
|
+
const providerCooldowns = new ProviderCooldowns();
|
|
1331
1336
|
// Overage-guard (v4.1, dario#288). Resolved from opts with built-in
|
|
1332
1337
|
// defaults (enabled=true, behavior='halt', cooldown=30min, notifyOs=true)
|
|
1333
1338
|
// so an opts-less proxy still gets protection. The notifier is wired
|
|
@@ -1828,9 +1833,14 @@ export async function startProxy(opts = {}) {
|
|
|
1828
1833
|
* a silently swapped model family is precisely the surprise this project
|
|
1829
1834
|
* exists to avoid.
|
|
1830
1835
|
*/
|
|
1831
|
-
const tryCodexPoolFallback = async (req, res, body, fallbackModels, shape, why) => {
|
|
1836
|
+
const tryCodexPoolFallback = async (req, res, body, fallbackModels, shape, why, attempted) => {
|
|
1832
1837
|
if (fallbackModels.length === 0)
|
|
1833
1838
|
return false;
|
|
1839
|
+
// Never a second codex attempt in the same request, and never one while it
|
|
1840
|
+
// is cooling from an earlier 429. This is the revisit the 20:13:40 log
|
|
1841
|
+
// sequence shows: codex -> claude -> codex, all inside request #267.
|
|
1842
|
+
if (!canAttempt('codex', attempted, providerCooldowns))
|
|
1843
|
+
return false;
|
|
1834
1844
|
if (!(await hasAnyCodexAccount().catch(() => false)))
|
|
1835
1845
|
return false;
|
|
1836
1846
|
const stored = await selectCodexAccount().catch(() => null);
|
|
@@ -1852,6 +1862,7 @@ export async function startProxy(opts = {}) {
|
|
|
1852
1862
|
return false;
|
|
1853
1863
|
console.log(`[dario] #${requestCount} ${why} → codex account ${creds.alias} as ${fallbackModel}`);
|
|
1854
1864
|
requestCount++;
|
|
1865
|
+
attempted.add('codex');
|
|
1855
1866
|
// If an api-key backend could ALSO serve this request, let the subscription
|
|
1856
1867
|
// decline a 429/5xx rather than answer with it, and report not-served so the
|
|
1857
1868
|
// caller falls through to that backend. This helper's contract has always
|
|
@@ -1862,7 +1873,15 @@ export async function startProxy(opts = {}) {
|
|
|
1862
1873
|
// With NO next option, do not defer: the real upstream error is more useful
|
|
1863
1874
|
// to the client than replacing it with a generic 503.
|
|
1864
1875
|
const hasNextOption = openaiBackend !== null && shape === 'openai';
|
|
1865
|
-
|
|
1876
|
+
const served = await forwardToCodex(req, res, fallbackBody, creds, corsOrigin, { ...SECURITY_HEADERS, 'x-dario-pool-fallback': fallbackModel }, upstreamTimeoutMs, verbose, shape, fetch, hasNextOption, undefined,
|
|
1877
|
+
// Only a rate limit cools the provider. A 5xx or a transport failure is
|
|
1878
|
+
// an outage, not quota — cooling it would park a provider that may be
|
|
1879
|
+
// back on the next request, which is the opposite of the fix.
|
|
1880
|
+
(d) => { if (d.status === 429)
|
|
1881
|
+
providerCooldowns.note('codex', d.retryAfterMs); });
|
|
1882
|
+
if (served)
|
|
1883
|
+
providerCooldowns.clear('codex');
|
|
1884
|
+
return served;
|
|
1866
1885
|
};
|
|
1867
1886
|
/**
|
|
1868
1887
|
* Mid-flight 429, no peer account left to retry — the pool is genuinely
|
|
@@ -1885,8 +1904,8 @@ export async function startProxy(opts = {}) {
|
|
|
1885
1904
|
* bug structurally impossible to reintroduce: a future third call site gets
|
|
1886
1905
|
* this by construction, not by remembering to copy six lines correctly.
|
|
1887
1906
|
*/
|
|
1888
|
-
const attemptPoolFallbackOn429 = async (req, res, body, isOpenAI) => {
|
|
1889
|
-
if (await tryCodexPoolFallback(req, res, body, poolFallbackModels, isOpenAI ? 'openai' : 'anthropic', 'pool exhausted mid-flight (429, no peer)')) {
|
|
1907
|
+
const attemptPoolFallbackOn429 = async (req, res, body, isOpenAI, attempted) => {
|
|
1908
|
+
if (await tryCodexPoolFallback(req, res, body, poolFallbackModels, isOpenAI ? 'openai' : 'anthropic', 'pool exhausted mid-flight (429, no peer)', attempted)) {
|
|
1890
1909
|
return true;
|
|
1891
1910
|
}
|
|
1892
1911
|
if (isOpenAI && poolFallbackModel && openaiBackend) {
|
|
@@ -2434,6 +2453,10 @@ export async function startProxy(opts = {}) {
|
|
|
2434
2453
|
// form gives the declared type without seeding a narrowing.
|
|
2435
2454
|
let poolAccount = null;
|
|
2436
2455
|
let accessToken = '';
|
|
2456
|
+
// Providers this ONE request has already asked. Paired with the
|
|
2457
|
+
// process-wide cool-downs: the set stops a revisit inside the request,
|
|
2458
|
+
// the cool-downs stop the next request repeating the discovery.
|
|
2459
|
+
const attemptedProviders = new Set();
|
|
2437
2460
|
/**
|
|
2438
2461
|
* Take a Claude pool account for this request. Returns false when it has
|
|
2439
2462
|
* already answered the client (empty or fully drained pool with no viable
|
|
@@ -2464,6 +2487,35 @@ export async function startProxy(opts = {}) {
|
|
|
2464
2487
|
message: 'all accounts are rate-limited or in auth cool-down; retry shortly',
|
|
2465
2488
|
}));
|
|
2466
2489
|
};
|
|
2490
|
+
/**
|
|
2491
|
+
* Terminal verdict for DEV-f66b131c: every provider that could serve this
|
|
2492
|
+
* request is inside a 429 cool-down, so there is nothing left to try and
|
|
2493
|
+
* no reason to spend an upstream attempt proving it. ONE log line and one
|
|
2494
|
+
* response — the defect being fixed is a three-line-per-request storm.
|
|
2495
|
+
*
|
|
2496
|
+
* 429 rather than the 503 `writePoolUnavailable` gives, and distinct from
|
|
2497
|
+
* `pool exhausted` (which implies a peer exists): this IS a rate limit, it
|
|
2498
|
+
* self-clears, and `retry-after` says when. The
|
|
2499
|
+
* `x-dario-upstream-rejection` marker is the same channel the billing and
|
|
2500
|
+
* credential classes use, so the platform-side sustained-fallback tracker
|
|
2501
|
+
* reads one field for all of them.
|
|
2502
|
+
*/
|
|
2503
|
+
const writeAllProvidersRateLimited = (providers) => {
|
|
2504
|
+
const retryAfterSec = Math.max(1, Math.ceil(cooldownRetryAfterMs(providers, providerCooldowns) / 1000));
|
|
2505
|
+
console.log(`[dario] #${requestCount} ${ALL_PROVIDERS_RATE_LIMITED} (${providers.join('+')}) — no attempt made, retry in ${retryAfterSec}s`);
|
|
2506
|
+
res.writeHead(429, {
|
|
2507
|
+
...JSON_HEADERS,
|
|
2508
|
+
'retry-after': String(retryAfterSec),
|
|
2509
|
+
'x-dario-upstream-rejection': ALL_PROVIDERS_RATE_LIMITED,
|
|
2510
|
+
});
|
|
2511
|
+
res.end(JSON.stringify({
|
|
2512
|
+
error: {
|
|
2513
|
+
type: 'rate_limit_error',
|
|
2514
|
+
message: `All configured providers (${providers.join(', ')}) are rate-limited; retry in ${retryAfterSec}s.`,
|
|
2515
|
+
},
|
|
2516
|
+
reason: ALL_PROVIDERS_RATE_LIMITED,
|
|
2517
|
+
}));
|
|
2518
|
+
};
|
|
2467
2519
|
const selectPoolAccount = () => {
|
|
2468
2520
|
if (upstreamApiKey) {
|
|
2469
2521
|
// Per-token API-key mode: no OAuth, no pool selection. `poolAccount`
|
|
@@ -2752,8 +2804,19 @@ export async function startProxy(opts = {}) {
|
|
|
2752
2804
|
const claudeTarget = pickClaudeTarget(poolFallbackModels, codexModels, getCachedBases(), (m) => resolveClaudeAlias(applyModelAlias(m, modelAliases) ?? m));
|
|
2753
2805
|
const claudeTargetModel = claudeTarget?.model ?? null;
|
|
2754
2806
|
const canDefer = claudeTarget !== null && pool.size > 0 && !upstreamApiKey;
|
|
2807
|
+
// DEV-f66b131c: an account still inside its 429 cool-down is not
|
|
2808
|
+
// asked again. Skipping straight to the Claude half of the chain is
|
|
2809
|
+
// the whole saving — the doomed attempt was spending quota on BOTH
|
|
2810
|
+
// accounts to rediscover a limit the previous request already found.
|
|
2811
|
+
const codexAvailable = canAttempt('codex', attemptedProviders, providerCooldowns);
|
|
2812
|
+
if (!codexAvailable && !canDefer) {
|
|
2813
|
+
// Nothing else in the chain can pick this up, so an attempt could
|
|
2814
|
+
// only rediscover the limit. One verdict, no upstream request.
|
|
2815
|
+
writeAllProvidersRateLimited(['codex']);
|
|
2816
|
+
return;
|
|
2817
|
+
}
|
|
2755
2818
|
const codexReq = requestCount;
|
|
2756
|
-
const served = await forwardToCodex(req, res, body, codexCreds, corsOrigin, SECURITY_HEADERS, upstreamTimeoutMs, verbose, isOpenAI ? 'openai' : 'anthropic', fetch, canDefer,
|
|
2819
|
+
const served = codexAvailable && await forwardToCodex(req, res, body, codexCreds, corsOrigin, SECURITY_HEADERS, upstreamTimeoutMs, verbose, isOpenAI ? 'openai' : 'anthropic', fetch, canDefer,
|
|
2757
2820
|
// Before this hook a codex request left no trace: nothing in
|
|
2758
2821
|
// /analytics, nothing in the request log, no per-account count.
|
|
2759
2822
|
// The dock (and anyone reading /analytics) saw a proxy that
|
|
@@ -2780,9 +2843,26 @@ export async function startProxy(opts = {}) {
|
|
|
2780
2843
|
status: o.status, latency_ms: o.latencyMs, in_tokens: o.inputTokens, out_tokens: o.outputTokens,
|
|
2781
2844
|
claim: CODEX_CLAIM, bucket: 'subscription', account: o.alias, stream: o.stream,
|
|
2782
2845
|
});
|
|
2783
|
-
}
|
|
2784
|
-
|
|
2846
|
+
},
|
|
2847
|
+
// Cool codex on a rate limit only — a 5xx or an unreachable backend
|
|
2848
|
+
// is an outage, and parking a provider for that would keep it out
|
|
2849
|
+
// of the chain while it was already coming back.
|
|
2850
|
+
(d) => { if (d.status === 429)
|
|
2851
|
+
providerCooldowns.note('codex', d.retryAfterMs); });
|
|
2852
|
+
if (served) {
|
|
2853
|
+
// A provider that just served is not rate-limited.
|
|
2854
|
+
providerCooldowns.clear('codex');
|
|
2855
|
+
return;
|
|
2856
|
+
}
|
|
2857
|
+
if (codexAvailable)
|
|
2858
|
+
attemptedProviders.add('codex');
|
|
2859
|
+
// Both ends of the chain limited — the case this ticket is about.
|
|
2860
|
+
// Fail fast with the machine-readable verdict instead of handing the
|
|
2861
|
+
// request on to a Claude pool that is itself cooling.
|
|
2862
|
+
if (providerCooldowns.isCooled('codex') && providerCooldowns.isCooled('claude')) {
|
|
2863
|
+
writeAllProvidersRateLimited(['codex', 'claude']);
|
|
2785
2864
|
return;
|
|
2865
|
+
}
|
|
2786
2866
|
const swapped = buildPoolFallbackBody(body, claudeTargetModel);
|
|
2787
2867
|
if (!swapped) {
|
|
2788
2868
|
res.writeHead(503, { 'Content-Type': 'application/json', ...SECURITY_HEADERS });
|
|
@@ -2836,7 +2916,12 @@ export async function startProxy(opts = {}) {
|
|
|
2836
2916
|
// response carries `x-dario-pool-fallback` — a substituted model must
|
|
2837
2917
|
// never be silent. GPT-bound requests never reach here (the routing
|
|
2838
2918
|
// block above already forwarded them; they don't need the pool).
|
|
2839
|
-
|
|
2919
|
+
// Selection returns null only for an empty pool or all-account auth-cooldown.
|
|
2920
|
+
// Genuine rate limits return an account and are cooled from the upstream 429.
|
|
2921
|
+
if (!upstreamApiKey && !poolAccount) {
|
|
2922
|
+
attemptedProviders.add('claude');
|
|
2923
|
+
}
|
|
2924
|
+
if (!upstreamApiKey && !poolAccount && await tryCodexPoolFallback(req, res, body, poolFallbackModels, isOpenAI ? 'openai' : 'anthropic', 'pool exhausted', attemptedProviders)) {
|
|
2840
2925
|
return;
|
|
2841
2926
|
}
|
|
2842
2927
|
// `isOpenAI` is REQUIRED here and was not, before v6.0.0 — the selector's
|
|
@@ -2864,6 +2949,12 @@ export async function startProxy(opts = {}) {
|
|
|
2864
2949
|
// would fall through to the Claude path with no account and an empty
|
|
2865
2950
|
// bearer token, turning a clean 503 into a confusing upstream 401.
|
|
2866
2951
|
if (!upstreamApiKey && !poolAccount) {
|
|
2952
|
+
// A chain where every entry is cooling is a rate limit, not a
|
|
2953
|
+
// misconfiguration — say so in the machine-readable way, once.
|
|
2954
|
+
if (allProvidersCooled(poolFallbackModels.length > 0 ? ['codex', 'claude'] : ['claude'], providerCooldowns)) {
|
|
2955
|
+
writeAllProvidersRateLimited(poolFallbackModels.length > 0 ? ['codex', 'claude'] : ['claude']);
|
|
2956
|
+
return;
|
|
2957
|
+
}
|
|
2867
2958
|
console.log(`[dario] #${requestCount} pool exhausted and no fallback provider could serve ${poolFallbackModels.join(', ') || '(none configured)'}`);
|
|
2868
2959
|
writePoolUnavailable();
|
|
2869
2960
|
return;
|
|
@@ -3658,7 +3749,16 @@ export async function startProxy(opts = {}) {
|
|
|
3658
3749
|
// attemptPoolFallbackOn429 below, so the file goes back to having
|
|
3659
3750
|
// exactly ONE copy of this logic instead of two near-identical ones
|
|
3660
3751
|
// — which is the exact shape of bug this incident already was.
|
|
3661
|
-
|
|
3752
|
+
// The Claude half is out of quota for this request AND for the next
|
|
3753
|
+
// one — cool it here, honouring the upstream `retry-after` when it
|
|
3754
|
+
// sent one, so the following request does not re-walk the chain.
|
|
3755
|
+
attemptedProviders.add('claude');
|
|
3756
|
+
providerCooldowns.note('claude', parseRetryAfterMs(upstream.headers.get('retry-after')));
|
|
3757
|
+
if (await attemptPoolFallbackOn429(req, res, body, isOpenAI, attemptedProviders)) {
|
|
3758
|
+
return;
|
|
3759
|
+
}
|
|
3760
|
+
if (allProvidersCooled(['codex', 'claude'], providerCooldowns)) {
|
|
3761
|
+
writeAllProvidersRateLimited(['codex', 'claude']);
|
|
3662
3762
|
return;
|
|
3663
3763
|
}
|
|
3664
3764
|
const enriched = enrich429(peekedBody, upstream.headers);
|
|
@@ -3776,7 +3876,14 @@ export async function startProxy(opts = {}) {
|
|
|
3776
3876
|
// translation (OpenAI-shape clients) went into finalBody, never back
|
|
3777
3877
|
// into body. See attemptPoolFallbackOn429's doc comment for why this
|
|
3778
3878
|
// is one shared function rather than an inline copy.
|
|
3779
|
-
|
|
3879
|
+
// Same bookkeeping as the other mid-flight site — see there.
|
|
3880
|
+
attemptedProviders.add('claude');
|
|
3881
|
+
providerCooldowns.note('claude', parseRetryAfterMs(upstream.headers.get('retry-after')));
|
|
3882
|
+
if (await attemptPoolFallbackOn429(req, res, body, isOpenAI, attemptedProviders)) {
|
|
3883
|
+
return;
|
|
3884
|
+
}
|
|
3885
|
+
if (allProvidersCooled(['codex', 'claude'], providerCooldowns)) {
|
|
3886
|
+
writeAllProvidersRateLimited(['codex', 'claude']);
|
|
3780
3887
|
return;
|
|
3781
3888
|
}
|
|
3782
3889
|
const errBody = await upstream.text().catch(() => '');
|
|
@@ -3813,6 +3920,8 @@ export async function startProxy(opts = {}) {
|
|
|
3813
3920
|
// so its consecutive-failure counter resets. dario#234.
|
|
3814
3921
|
if (poolAccount && upstream.status >= 200 && upstream.status < 300) {
|
|
3815
3922
|
pool.clearAuthFailure(poolAccount.alias);
|
|
3923
|
+
// Served — whatever cooled this provider has passed.
|
|
3924
|
+
providerCooldowns.clear('claude');
|
|
3816
3925
|
}
|
|
3817
3926
|
break;
|
|
3818
3927
|
} // end dispatchLoop: while (true)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@askalf/dario",
|
|
3
|
-
"version": "6.0.
|
|
3
|
+
"version": "6.0.15",
|
|
4
4
|
"description": "Use your Claude Pro/Max subscription in any tool — Cursor, Cline, Aider, the Agent SDK, your scripts — at subscription pricing, not per-token API bills. One local Anthropic + OpenAI-compatible endpoint.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|