@askalf/dario 6.0.12 → 6.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/codex-backend.d.ts +7 -1
- package/dist/codex-backend.js +16 -1
- package/dist/doctor-core.d.ts +239 -0
- package/dist/doctor-core.js +1234 -0
- package/dist/doctor-serving.d.ts +7 -0
- package/dist/doctor-serving.js +17 -0
- package/dist/doctor.d.ts +17 -234
- package/dist/doctor.js +32 -1226
- package/dist/provider-cooldown.d.ts +78 -0
- package/dist/provider-cooldown.js +120 -0
- package/dist/proxy.js +119 -10
- package/dist/serving-probe.d.ts +7 -92
- package/dist/serving-probe.js +34 -112
- package/dist/upstream-rejection.d.ts +6 -2
- package/dist/upstream-rejection.js +26 -0
- package/package.json +1 -1
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-provider rate-limit cool-down for the failover chain (DEV-f66b131c).
|
|
3
|
+
*
|
|
4
|
+
* The chain used to have no memory. Every 429 was a fresh discovery, so with
|
|
5
|
+
* BOTH ends capped — a spent Claude 5h window and a 429-ing ChatGPT
|
|
6
|
+
* subscription — one request walked codex → claude → codex before dropping,
|
|
7
|
+
* and the next request did it again. 185 such lines in 38 minutes on
|
|
8
|
+
* 2026-09-02. That is the wrong shape twice over: at the moment quota is the
|
|
9
|
+
* scarce resource the chain spends TWICE as much of it discovering there is
|
|
10
|
+
* none, and whatever is cycling when a window rolls over consumes the fresh
|
|
11
|
+
* window ahead of legitimately queued work.
|
|
12
|
+
*
|
|
13
|
+
* Two rules, both enforced here so the request path can stay a straight line:
|
|
14
|
+
*
|
|
15
|
+
* WITHIN a request — an entry that already declined is never revisited
|
|
16
|
+
* ({@link canAttempt} against the attempted set).
|
|
17
|
+
* ACROSS requests — a 429 cools the entry for a bounded interval, honouring
|
|
18
|
+
* `retry-after` when the upstream sends one.
|
|
19
|
+
*
|
|
20
|
+
* When every entry is cooled the caller fails fast with one terminal verdict
|
|
21
|
+
* instead of a retry storm — see {@link allProvidersCooled}.
|
|
22
|
+
*
|
|
23
|
+
* Provider-granular, not entry-granular: a chain names MODELS
|
|
24
|
+
* (`gpt-5.6-sol,claude-sonnet-5`) but a 429 is a property of the ACCOUNT
|
|
25
|
+
* behind them, so cooling `codex` also cools every other codex slug in the
|
|
26
|
+
* chain, which is the correct blast radius.
|
|
27
|
+
*
|
|
28
|
+
* The clock is injectable so the whole thing is testable without sleeping.
|
|
29
|
+
*/
|
|
30
|
+
/** Cool-down applied when the upstream gives no `retry-after`. */
|
|
31
|
+
export declare const DEFAULT_COOLDOWN_MS = 60000;
|
|
32
|
+
/**
|
|
33
|
+
* Ceiling on any cool-down. An upstream is free to say "come back in 4 hours";
|
|
34
|
+
* honouring that literally would keep a provider parked long after a window
|
|
35
|
+
* reset we cannot observe. One re-probe per 15 minutes is cheap.
|
|
36
|
+
*/
|
|
37
|
+
export declare const MAX_COOLDOWN_MS: number;
|
|
38
|
+
/** Machine-readable terminal verdict. Distinct from `pool exhausted` (which
|
|
39
|
+
* implies a peer exists) and from the billing/credential classes. */
|
|
40
|
+
export declare const ALL_PROVIDERS_RATE_LIMITED = "all-providers-rate-limited";
|
|
41
|
+
/**
|
|
42
|
+
* `retry-after` as milliseconds. The header is either delta-seconds or an
|
|
43
|
+
* HTTP-date (RFC 9110 §10.2.3); both are accepted, anything else is null so
|
|
44
|
+
* the caller falls back to its default rather than trusting a parse it did not
|
|
45
|
+
* get. A date already in the past yields 0 — "retry now" — not a negative.
|
|
46
|
+
*/
|
|
47
|
+
export declare function parseRetryAfterMs(value: string | null | undefined, now?: number): number | null;
|
|
48
|
+
/** A provider that declined, and for how long it should stay declined. */
|
|
49
|
+
export declare class ProviderCooldowns {
|
|
50
|
+
private readonly until;
|
|
51
|
+
private readonly now;
|
|
52
|
+
private readonly defaultMs;
|
|
53
|
+
constructor(now?: () => number, defaultMs?: number);
|
|
54
|
+
/**
|
|
55
|
+
* Record a 429 (or an equivalent "not right now") for `provider`. Returns the
|
|
56
|
+
* cool-down actually applied, so the caller can put it in its one log line.
|
|
57
|
+
* A `retryAfterMs` of 0 is honoured as "retry now" and cools nothing.
|
|
58
|
+
*/
|
|
59
|
+
note(provider: string, retryAfterMs?: number | null): number;
|
|
60
|
+
/** Clear on a success — a provider that just served is not rate-limited. */
|
|
61
|
+
clear(provider: string): void;
|
|
62
|
+
remainingMs(provider: string): number;
|
|
63
|
+
isCooled(provider: string): boolean;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* May this request try `provider`? False when it already declined during THIS
|
|
67
|
+
* request (the codex → claude → codex revisit) or when it is cooling from an
|
|
68
|
+
* earlier one.
|
|
69
|
+
*/
|
|
70
|
+
export declare function canAttempt(provider: string, attempted: ReadonlySet<string>, cooldowns: ProviderCooldowns): boolean;
|
|
71
|
+
/**
|
|
72
|
+
* Every named provider is cooling — the fail-fast condition. An EMPTY list is
|
|
73
|
+
* NOT "all cooled": nothing configured is a configuration answer, not a rate
|
|
74
|
+
* limit, and must keep its existing message.
|
|
75
|
+
*/
|
|
76
|
+
export declare function allProvidersCooled(providers: readonly string[], cooldowns: ProviderCooldowns): boolean;
|
|
77
|
+
/** Longest remaining cool-down across `providers` — what to put in `retry-after`. */
|
|
78
|
+
export declare function cooldownRetryAfterMs(providers: readonly string[], cooldowns: ProviderCooldowns): number;
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-provider rate-limit cool-down for the failover chain (DEV-f66b131c).
|
|
3
|
+
*
|
|
4
|
+
* The chain used to have no memory. Every 429 was a fresh discovery, so with
|
|
5
|
+
* BOTH ends capped — a spent Claude 5h window and a 429-ing ChatGPT
|
|
6
|
+
* subscription — one request walked codex → claude → codex before dropping,
|
|
7
|
+
* and the next request did it again. 185 such lines in 38 minutes on
|
|
8
|
+
* 2026-09-02. That is the wrong shape twice over: at the moment quota is the
|
|
9
|
+
* scarce resource the chain spends TWICE as much of it discovering there is
|
|
10
|
+
* none, and whatever is cycling when a window rolls over consumes the fresh
|
|
11
|
+
* window ahead of legitimately queued work.
|
|
12
|
+
*
|
|
13
|
+
* Two rules, both enforced here so the request path can stay a straight line:
|
|
14
|
+
*
|
|
15
|
+
* WITHIN a request — an entry that already declined is never revisited
|
|
16
|
+
* ({@link canAttempt} against the attempted set).
|
|
17
|
+
* ACROSS requests — a 429 cools the entry for a bounded interval, honouring
|
|
18
|
+
* `retry-after` when the upstream sends one.
|
|
19
|
+
*
|
|
20
|
+
* When every entry is cooled the caller fails fast with one terminal verdict
|
|
21
|
+
* instead of a retry storm — see {@link allProvidersCooled}.
|
|
22
|
+
*
|
|
23
|
+
* Provider-granular, not entry-granular: a chain names MODELS
|
|
24
|
+
* (`gpt-5.6-sol,claude-sonnet-5`) but a 429 is a property of the ACCOUNT
|
|
25
|
+
* behind them, so cooling `codex` also cools every other codex slug in the
|
|
26
|
+
* chain, which is the correct blast radius.
|
|
27
|
+
*
|
|
28
|
+
* The clock is injectable so the whole thing is testable without sleeping.
|
|
29
|
+
*/
|
|
30
|
+
/** Cool-down applied when the upstream gives no `retry-after`. */
|
|
31
|
+
export const DEFAULT_COOLDOWN_MS = 60_000;
|
|
32
|
+
/**
|
|
33
|
+
* Ceiling on any cool-down. An upstream is free to say "come back in 4 hours";
|
|
34
|
+
* honouring that literally would keep a provider parked long after a window
|
|
35
|
+
* reset we cannot observe. One re-probe per 15 minutes is cheap.
|
|
36
|
+
*/
|
|
37
|
+
export const MAX_COOLDOWN_MS = 15 * 60_000;
|
|
38
|
+
/** Machine-readable terminal verdict. Distinct from `pool exhausted` (which
|
|
39
|
+
* implies a peer exists) and from the billing/credential classes. */
|
|
40
|
+
export const ALL_PROVIDERS_RATE_LIMITED = 'all-providers-rate-limited';
|
|
41
|
+
/**
|
|
42
|
+
* `retry-after` as milliseconds. The header is either delta-seconds or an
|
|
43
|
+
* HTTP-date (RFC 9110 §10.2.3); both are accepted, anything else is null so
|
|
44
|
+
* the caller falls back to its default rather than trusting a parse it did not
|
|
45
|
+
* get. A date already in the past yields 0 — "retry now" — not a negative.
|
|
46
|
+
*/
|
|
47
|
+
export function parseRetryAfterMs(value, now = Date.now()) {
|
|
48
|
+
if (value == null)
|
|
49
|
+
return null;
|
|
50
|
+
const trimmed = value.trim();
|
|
51
|
+
if (trimmed.length === 0)
|
|
52
|
+
return null;
|
|
53
|
+
if (/^\d+$/.test(trimmed))
|
|
54
|
+
return Math.min(Number(trimmed) * 1000, MAX_COOLDOWN_MS);
|
|
55
|
+
const at = Date.parse(trimmed);
|
|
56
|
+
if (Number.isNaN(at))
|
|
57
|
+
return null;
|
|
58
|
+
return Math.min(Math.max(at - now, 0), MAX_COOLDOWN_MS);
|
|
59
|
+
}
|
|
60
|
+
/** A provider that declined, and for how long it should stay declined. */
|
|
61
|
+
export class ProviderCooldowns {
|
|
62
|
+
until = new Map();
|
|
63
|
+
now;
|
|
64
|
+
defaultMs;
|
|
65
|
+
constructor(now = () => Date.now(), defaultMs = DEFAULT_COOLDOWN_MS) {
|
|
66
|
+
this.now = now;
|
|
67
|
+
this.defaultMs = defaultMs;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Record a 429 (or an equivalent "not right now") for `provider`. Returns the
|
|
71
|
+
* cool-down actually applied, so the caller can put it in its one log line.
|
|
72
|
+
* A `retryAfterMs` of 0 is honoured as "retry now" and cools nothing.
|
|
73
|
+
*/
|
|
74
|
+
note(provider, retryAfterMs) {
|
|
75
|
+
const ms = retryAfterMs == null ? this.defaultMs : Math.min(Math.max(retryAfterMs, 0), MAX_COOLDOWN_MS);
|
|
76
|
+
this.until.set(provider, this.now() + ms);
|
|
77
|
+
return ms;
|
|
78
|
+
}
|
|
79
|
+
/** Clear on a success — a provider that just served is not rate-limited. */
|
|
80
|
+
clear(provider) {
|
|
81
|
+
this.until.delete(provider);
|
|
82
|
+
}
|
|
83
|
+
remainingMs(provider) {
|
|
84
|
+
const at = this.until.get(provider);
|
|
85
|
+
if (at == null)
|
|
86
|
+
return 0;
|
|
87
|
+
const left = at - this.now();
|
|
88
|
+
if (left <= 0) {
|
|
89
|
+
// Expired entries are dropped on read; nothing else sweeps this map and
|
|
90
|
+
// a long-lived proxy would otherwise accumulate one entry per provider
|
|
91
|
+
// forever (bounded, but pointlessly).
|
|
92
|
+
this.until.delete(provider);
|
|
93
|
+
return 0;
|
|
94
|
+
}
|
|
95
|
+
return left;
|
|
96
|
+
}
|
|
97
|
+
isCooled(provider) {
|
|
98
|
+
return this.remainingMs(provider) > 0;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* May this request try `provider`? False when it already declined during THIS
|
|
103
|
+
* request (the codex → claude → codex revisit) or when it is cooling from an
|
|
104
|
+
* earlier one.
|
|
105
|
+
*/
|
|
106
|
+
export function canAttempt(provider, attempted, cooldowns) {
|
|
107
|
+
return !attempted.has(provider) && !cooldowns.isCooled(provider);
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Every named provider is cooling — the fail-fast condition. An EMPTY list is
|
|
111
|
+
* NOT "all cooled": nothing configured is a configuration answer, not a rate
|
|
112
|
+
* limit, and must keep its existing message.
|
|
113
|
+
*/
|
|
114
|
+
export function allProvidersCooled(providers, cooldowns) {
|
|
115
|
+
return providers.length > 0 && providers.every(p => cooldowns.isCooled(p));
|
|
116
|
+
}
|
|
117
|
+
/** Longest remaining cool-down across `providers` — what to put in `retry-after`. */
|
|
118
|
+
export function cooldownRetryAfterMs(providers, cooldowns) {
|
|
119
|
+
return providers.reduce((max, p) => Math.max(max, cooldowns.remainingMs(p)), 0);
|
|
120
|
+
}
|
package/dist/proxy.js
CHANGED
|
@@ -28,6 +28,7 @@ import { RequestQueue, QueueFullError, QueueTimeoutError, DEFAULT_MAX_CONCURRENT
|
|
|
28
28
|
import { redactSecrets } from './redact.js';
|
|
29
29
|
import { BAKED_BASE_MODELS, withLongContextVariants, buildOpenAIModelsList, getModelCatalog, getCachedBases, resolveAliasAgainst, prewarmModelCatalog, retryModelCatalogNow, isSuspendedModel } from './model-catalog.js';
|
|
30
30
|
import { classifyUpstreamRejection, diagnosticSnippet } from './upstream-rejection.js';
|
|
31
|
+
import { ProviderCooldowns, canAttempt, allProvidersCooled, cooldownRetryAfterMs, parseRetryAfterMs, ALL_PROVIDERS_RATE_LIMITED, } from './provider-cooldown.js';
|
|
31
32
|
const ANTHROPIC_API = 'https://api.anthropic.com';
|
|
32
33
|
const DEFAULT_PORT = 3456;
|
|
33
34
|
const MAX_BODY_BYTES = 10 * 1024 * 1024; // 10 MB — generous for large prompts, prevents abuse
|
|
@@ -1328,6 +1329,10 @@ export async function startProxy(opts = {}) {
|
|
|
1328
1329
|
// Per-alias request counts for GET /codex — the pool has requestCount per
|
|
1329
1330
|
// account; the codex accounts had nothing until now.
|
|
1330
1331
|
const codexRequestCounts = new Map();
|
|
1332
|
+
// Rate-limit memory for the failover chain (DEV-f66b131c). Process-wide and
|
|
1333
|
+
// deliberately NOT per-request: the point is that request #269 does not
|
|
1334
|
+
// re-discover the limit #267 already found. Cleared on any success.
|
|
1335
|
+
const providerCooldowns = new ProviderCooldowns();
|
|
1331
1336
|
// Overage-guard (v4.1, dario#288). Resolved from opts with built-in
|
|
1332
1337
|
// defaults (enabled=true, behavior='halt', cooldown=30min, notifyOs=true)
|
|
1333
1338
|
// so an opts-less proxy still gets protection. The notifier is wired
|
|
@@ -1828,9 +1833,14 @@ export async function startProxy(opts = {}) {
|
|
|
1828
1833
|
* a silently swapped model family is precisely the surprise this project
|
|
1829
1834
|
* exists to avoid.
|
|
1830
1835
|
*/
|
|
1831
|
-
const tryCodexPoolFallback = async (req, res, body, fallbackModels, shape, why) => {
|
|
1836
|
+
const tryCodexPoolFallback = async (req, res, body, fallbackModels, shape, why, attempted) => {
|
|
1832
1837
|
if (fallbackModels.length === 0)
|
|
1833
1838
|
return false;
|
|
1839
|
+
// Never a second codex attempt in the same request, and never one while it
|
|
1840
|
+
// is cooling from an earlier 429. This is the revisit the 20:13:40 log
|
|
1841
|
+
// sequence shows: codex -> claude -> codex, all inside request #267.
|
|
1842
|
+
if (!canAttempt('codex', attempted, providerCooldowns))
|
|
1843
|
+
return false;
|
|
1834
1844
|
if (!(await hasAnyCodexAccount().catch(() => false)))
|
|
1835
1845
|
return false;
|
|
1836
1846
|
const stored = await selectCodexAccount().catch(() => null);
|
|
@@ -1852,6 +1862,7 @@ export async function startProxy(opts = {}) {
|
|
|
1852
1862
|
return false;
|
|
1853
1863
|
console.log(`[dario] #${requestCount} ${why} → codex account ${creds.alias} as ${fallbackModel}`);
|
|
1854
1864
|
requestCount++;
|
|
1865
|
+
attempted.add('codex');
|
|
1855
1866
|
// If an api-key backend could ALSO serve this request, let the subscription
|
|
1856
1867
|
// decline a 429/5xx rather than answer with it, and report not-served so the
|
|
1857
1868
|
// caller falls through to that backend. This helper's contract has always
|
|
@@ -1862,7 +1873,15 @@ export async function startProxy(opts = {}) {
|
|
|
1862
1873
|
// With NO next option, do not defer: the real upstream error is more useful
|
|
1863
1874
|
// to the client than replacing it with a generic 503.
|
|
1864
1875
|
const hasNextOption = openaiBackend !== null && shape === 'openai';
|
|
1865
|
-
|
|
1876
|
+
const served = await forwardToCodex(req, res, fallbackBody, creds, corsOrigin, { ...SECURITY_HEADERS, 'x-dario-pool-fallback': fallbackModel }, upstreamTimeoutMs, verbose, shape, fetch, hasNextOption, undefined,
|
|
1877
|
+
// Only a rate limit cools the provider. A 5xx or a transport failure is
|
|
1878
|
+
// an outage, not quota — cooling it would park a provider that may be
|
|
1879
|
+
// back on the next request, which is the opposite of the fix.
|
|
1880
|
+
(d) => { if (d.status === 429)
|
|
1881
|
+
providerCooldowns.note('codex', d.retryAfterMs); });
|
|
1882
|
+
if (served)
|
|
1883
|
+
providerCooldowns.clear('codex');
|
|
1884
|
+
return served;
|
|
1866
1885
|
};
|
|
1867
1886
|
/**
|
|
1868
1887
|
* Mid-flight 429, no peer account left to retry — the pool is genuinely
|
|
@@ -1885,8 +1904,8 @@ export async function startProxy(opts = {}) {
|
|
|
1885
1904
|
* bug structurally impossible to reintroduce: a future third call site gets
|
|
1886
1905
|
* this by construction, not by remembering to copy six lines correctly.
|
|
1887
1906
|
*/
|
|
1888
|
-
const attemptPoolFallbackOn429 = async (req, res, body, isOpenAI) => {
|
|
1889
|
-
if (await tryCodexPoolFallback(req, res, body, poolFallbackModels, isOpenAI ? 'openai' : 'anthropic', 'pool exhausted mid-flight (429, no peer)')) {
|
|
1907
|
+
const attemptPoolFallbackOn429 = async (req, res, body, isOpenAI, attempted) => {
|
|
1908
|
+
if (await tryCodexPoolFallback(req, res, body, poolFallbackModels, isOpenAI ? 'openai' : 'anthropic', 'pool exhausted mid-flight (429, no peer)', attempted)) {
|
|
1890
1909
|
return true;
|
|
1891
1910
|
}
|
|
1892
1911
|
if (isOpenAI && poolFallbackModel && openaiBackend) {
|
|
@@ -2434,6 +2453,10 @@ export async function startProxy(opts = {}) {
|
|
|
2434
2453
|
// form gives the declared type without seeding a narrowing.
|
|
2435
2454
|
let poolAccount = null;
|
|
2436
2455
|
let accessToken = '';
|
|
2456
|
+
// Providers this ONE request has already asked. Paired with the
|
|
2457
|
+
// process-wide cool-downs: the set stops a revisit inside the request,
|
|
2458
|
+
// the cool-downs stop the next request repeating the discovery.
|
|
2459
|
+
const attemptedProviders = new Set();
|
|
2437
2460
|
/**
|
|
2438
2461
|
* Take a Claude pool account for this request. Returns false when it has
|
|
2439
2462
|
* already answered the client (empty or fully drained pool with no viable
|
|
@@ -2464,6 +2487,35 @@ export async function startProxy(opts = {}) {
|
|
|
2464
2487
|
message: 'all accounts are rate-limited or in auth cool-down; retry shortly',
|
|
2465
2488
|
}));
|
|
2466
2489
|
};
|
|
2490
|
+
/**
|
|
2491
|
+
* Terminal verdict for DEV-f66b131c: every provider that could serve this
|
|
2492
|
+
* request is inside a 429 cool-down, so there is nothing left to try and
|
|
2493
|
+
* no reason to spend an upstream attempt proving it. ONE log line and one
|
|
2494
|
+
* response — the defect being fixed is a three-line-per-request storm.
|
|
2495
|
+
*
|
|
2496
|
+
* 429 rather than the 503 `writePoolUnavailable` gives, and distinct from
|
|
2497
|
+
* `pool exhausted` (which implies a peer exists): this IS a rate limit, it
|
|
2498
|
+
* self-clears, and `retry-after` says when. The
|
|
2499
|
+
* `x-dario-upstream-rejection` marker is the same channel the billing and
|
|
2500
|
+
* credential classes use, so the platform-side sustained-fallback tracker
|
|
2501
|
+
* reads one field for all of them.
|
|
2502
|
+
*/
|
|
2503
|
+
const writeAllProvidersRateLimited = (providers) => {
|
|
2504
|
+
const retryAfterSec = Math.max(1, Math.ceil(cooldownRetryAfterMs(providers, providerCooldowns) / 1000));
|
|
2505
|
+
console.log(`[dario] #${requestCount} ${ALL_PROVIDERS_RATE_LIMITED} (${providers.join('+')}) — no attempt made, retry in ${retryAfterSec}s`);
|
|
2506
|
+
res.writeHead(429, {
|
|
2507
|
+
...JSON_HEADERS,
|
|
2508
|
+
'retry-after': String(retryAfterSec),
|
|
2509
|
+
'x-dario-upstream-rejection': ALL_PROVIDERS_RATE_LIMITED,
|
|
2510
|
+
});
|
|
2511
|
+
res.end(JSON.stringify({
|
|
2512
|
+
error: {
|
|
2513
|
+
type: 'rate_limit_error',
|
|
2514
|
+
message: `All configured providers (${providers.join(', ')}) are rate-limited; retry in ${retryAfterSec}s.`,
|
|
2515
|
+
},
|
|
2516
|
+
reason: ALL_PROVIDERS_RATE_LIMITED,
|
|
2517
|
+
}));
|
|
2518
|
+
};
|
|
2467
2519
|
const selectPoolAccount = () => {
|
|
2468
2520
|
if (upstreamApiKey) {
|
|
2469
2521
|
// Per-token API-key mode: no OAuth, no pool selection. `poolAccount`
|
|
@@ -2752,8 +2804,19 @@ export async function startProxy(opts = {}) {
|
|
|
2752
2804
|
const claudeTarget = pickClaudeTarget(poolFallbackModels, codexModels, getCachedBases(), (m) => resolveClaudeAlias(applyModelAlias(m, modelAliases) ?? m));
|
|
2753
2805
|
const claudeTargetModel = claudeTarget?.model ?? null;
|
|
2754
2806
|
const canDefer = claudeTarget !== null && pool.size > 0 && !upstreamApiKey;
|
|
2807
|
+
// DEV-f66b131c: an account still inside its 429 cool-down is not
|
|
2808
|
+
// asked again. Skipping straight to the Claude half of the chain is
|
|
2809
|
+
// the whole saving — the doomed attempt was spending quota on BOTH
|
|
2810
|
+
// accounts to rediscover a limit the previous request already found.
|
|
2811
|
+
const codexAvailable = canAttempt('codex', attemptedProviders, providerCooldowns);
|
|
2812
|
+
if (!codexAvailable && !canDefer) {
|
|
2813
|
+
// Nothing else in the chain can pick this up, so an attempt could
|
|
2814
|
+
// only rediscover the limit. One verdict, no upstream request.
|
|
2815
|
+
writeAllProvidersRateLimited(['codex']);
|
|
2816
|
+
return;
|
|
2817
|
+
}
|
|
2755
2818
|
const codexReq = requestCount;
|
|
2756
|
-
const served = await forwardToCodex(req, res, body, codexCreds, corsOrigin, SECURITY_HEADERS, upstreamTimeoutMs, verbose, isOpenAI ? 'openai' : 'anthropic', fetch, canDefer,
|
|
2819
|
+
const served = codexAvailable && await forwardToCodex(req, res, body, codexCreds, corsOrigin, SECURITY_HEADERS, upstreamTimeoutMs, verbose, isOpenAI ? 'openai' : 'anthropic', fetch, canDefer,
|
|
2757
2820
|
// Before this hook a codex request left no trace: nothing in
|
|
2758
2821
|
// /analytics, nothing in the request log, no per-account count.
|
|
2759
2822
|
// The dock (and anyone reading /analytics) saw a proxy that
|
|
@@ -2780,9 +2843,26 @@ export async function startProxy(opts = {}) {
|
|
|
2780
2843
|
status: o.status, latency_ms: o.latencyMs, in_tokens: o.inputTokens, out_tokens: o.outputTokens,
|
|
2781
2844
|
claim: CODEX_CLAIM, bucket: 'subscription', account: o.alias, stream: o.stream,
|
|
2782
2845
|
});
|
|
2783
|
-
}
|
|
2784
|
-
|
|
2846
|
+
},
|
|
2847
|
+
// Cool codex on a rate limit only — a 5xx or an unreachable backend
|
|
2848
|
+
// is an outage, and parking a provider for that would keep it out
|
|
2849
|
+
// of the chain while it was already coming back.
|
|
2850
|
+
(d) => { if (d.status === 429)
|
|
2851
|
+
providerCooldowns.note('codex', d.retryAfterMs); });
|
|
2852
|
+
if (served) {
|
|
2853
|
+
// A provider that just served is not rate-limited.
|
|
2854
|
+
providerCooldowns.clear('codex');
|
|
2855
|
+
return;
|
|
2856
|
+
}
|
|
2857
|
+
if (codexAvailable)
|
|
2858
|
+
attemptedProviders.add('codex');
|
|
2859
|
+
// Both ends of the chain limited — the case this ticket is about.
|
|
2860
|
+
// Fail fast with the machine-readable verdict instead of handing the
|
|
2861
|
+
// request on to a Claude pool that is itself cooling.
|
|
2862
|
+
if (providerCooldowns.isCooled('codex') && providerCooldowns.isCooled('claude')) {
|
|
2863
|
+
writeAllProvidersRateLimited(['codex', 'claude']);
|
|
2785
2864
|
return;
|
|
2865
|
+
}
|
|
2786
2866
|
const swapped = buildPoolFallbackBody(body, claudeTargetModel);
|
|
2787
2867
|
if (!swapped) {
|
|
2788
2868
|
res.writeHead(503, { 'Content-Type': 'application/json', ...SECURITY_HEADERS });
|
|
@@ -2836,7 +2916,12 @@ export async function startProxy(opts = {}) {
|
|
|
2836
2916
|
// response carries `x-dario-pool-fallback` — a substituted model must
|
|
2837
2917
|
// never be silent. GPT-bound requests never reach here (the routing
|
|
2838
2918
|
// block above already forwarded them; they don't need the pool).
|
|
2839
|
-
|
|
2919
|
+
// Selection returns null only for an empty pool or all-account auth-cooldown.
|
|
2920
|
+
// Genuine rate limits return an account and are cooled from the upstream 429.
|
|
2921
|
+
if (!upstreamApiKey && !poolAccount) {
|
|
2922
|
+
attemptedProviders.add('claude');
|
|
2923
|
+
}
|
|
2924
|
+
if (!upstreamApiKey && !poolAccount && await tryCodexPoolFallback(req, res, body, poolFallbackModels, isOpenAI ? 'openai' : 'anthropic', 'pool exhausted', attemptedProviders)) {
|
|
2840
2925
|
return;
|
|
2841
2926
|
}
|
|
2842
2927
|
// `isOpenAI` is REQUIRED here and was not, before v6.0.0 — the selector's
|
|
@@ -2864,6 +2949,12 @@ export async function startProxy(opts = {}) {
|
|
|
2864
2949
|
// would fall through to the Claude path with no account and an empty
|
|
2865
2950
|
// bearer token, turning a clean 503 into a confusing upstream 401.
|
|
2866
2951
|
if (!upstreamApiKey && !poolAccount) {
|
|
2952
|
+
// A chain where every entry is cooling is a rate limit, not a
|
|
2953
|
+
// misconfiguration — say so in the machine-readable way, once.
|
|
2954
|
+
if (allProvidersCooled(poolFallbackModels.length > 0 ? ['codex', 'claude'] : ['claude'], providerCooldowns)) {
|
|
2955
|
+
writeAllProvidersRateLimited(poolFallbackModels.length > 0 ? ['codex', 'claude'] : ['claude']);
|
|
2956
|
+
return;
|
|
2957
|
+
}
|
|
2867
2958
|
console.log(`[dario] #${requestCount} pool exhausted and no fallback provider could serve ${poolFallbackModels.join(', ') || '(none configured)'}`);
|
|
2868
2959
|
writePoolUnavailable();
|
|
2869
2960
|
return;
|
|
@@ -3658,7 +3749,16 @@ export async function startProxy(opts = {}) {
|
|
|
3658
3749
|
// attemptPoolFallbackOn429 below, so the file goes back to having
|
|
3659
3750
|
// exactly ONE copy of this logic instead of two near-identical ones
|
|
3660
3751
|
// — which is the exact shape of bug this incident already was.
|
|
3661
|
-
|
|
3752
|
+
// The Claude half is out of quota for this request AND for the next
|
|
3753
|
+
// one — cool it here, honouring the upstream `retry-after` when it
|
|
3754
|
+
// sent one, so the following request does not re-walk the chain.
|
|
3755
|
+
attemptedProviders.add('claude');
|
|
3756
|
+
providerCooldowns.note('claude', parseRetryAfterMs(upstream.headers.get('retry-after')));
|
|
3757
|
+
if (await attemptPoolFallbackOn429(req, res, body, isOpenAI, attemptedProviders)) {
|
|
3758
|
+
return;
|
|
3759
|
+
}
|
|
3760
|
+
if (allProvidersCooled(['codex', 'claude'], providerCooldowns)) {
|
|
3761
|
+
writeAllProvidersRateLimited(['codex', 'claude']);
|
|
3662
3762
|
return;
|
|
3663
3763
|
}
|
|
3664
3764
|
const enriched = enrich429(peekedBody, upstream.headers);
|
|
@@ -3776,7 +3876,14 @@ export async function startProxy(opts = {}) {
|
|
|
3776
3876
|
// translation (OpenAI-shape clients) went into finalBody, never back
|
|
3777
3877
|
// into body. See attemptPoolFallbackOn429's doc comment for why this
|
|
3778
3878
|
// is one shared function rather than an inline copy.
|
|
3779
|
-
|
|
3879
|
+
// Same bookkeeping as the other mid-flight site — see there.
|
|
3880
|
+
attemptedProviders.add('claude');
|
|
3881
|
+
providerCooldowns.note('claude', parseRetryAfterMs(upstream.headers.get('retry-after')));
|
|
3882
|
+
if (await attemptPoolFallbackOn429(req, res, body, isOpenAI, attemptedProviders)) {
|
|
3883
|
+
return;
|
|
3884
|
+
}
|
|
3885
|
+
if (allProvidersCooled(['codex', 'claude'], providerCooldowns)) {
|
|
3886
|
+
writeAllProvidersRateLimited(['codex', 'claude']);
|
|
3780
3887
|
return;
|
|
3781
3888
|
}
|
|
3782
3889
|
const errBody = await upstream.text().catch(() => '');
|
|
@@ -3813,6 +3920,8 @@ export async function startProxy(opts = {}) {
|
|
|
3813
3920
|
// so its consecutive-failure counter resets. dario#234.
|
|
3814
3921
|
if (poolAccount && upstream.status >= 200 && upstream.status < 300) {
|
|
3815
3922
|
pool.clearAuthFailure(poolAccount.alias);
|
|
3923
|
+
// Served — whatever cooled this provider has passed.
|
|
3924
|
+
providerCooldowns.clear('claude');
|
|
3816
3925
|
}
|
|
3817
3926
|
break;
|
|
3818
3927
|
} // end dispatchLoop: while (true)
|
package/dist/serving-probe.d.ts
CHANGED
|
@@ -1,120 +1,35 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* serving-probe.ts — the "can dario actually serve a request?" check.
|
|
3
|
-
*
|
|
4
|
-
* WHY THIS EXISTS (dario#905, dario#921)
|
|
5
|
-
*
|
|
6
|
-
* dario's existing health surfaces are both STRUCTURAL — they inspect state,
|
|
7
|
-
* they never prove anything end-to-end:
|
|
8
|
-
*
|
|
9
|
-
* /livez the HTTP server is accepting connections. Always 200.
|
|
10
|
-
* /health credentials/pool look sane: token not expired, refresh not broken,
|
|
11
|
-
* accounts not all in auth-cooldown, queue snapshot attached.
|
|
12
|
-
*
|
|
13
|
-
* Neither can catch the failure class that actually took a user down for ~14h
|
|
14
|
-
* across four episodes (#905): dario looking perfectly healthy from the outside
|
|
15
|
-
* while every real request failed. The reporter's remediation was an external
|
|
16
|
-
* watchdog that sends a real inference call and restarts the service when it
|
|
17
|
-
* doesn't come back — i.e. the user had to write the liveness check dario
|
|
18
|
-
* should own. This module is that check, brought in-house.
|
|
19
|
-
*
|
|
20
|
-
* It also covers a class /health provably cannot see: an account whose token is
|
|
21
|
-
* unexpired and refreshable (so `status: healthy`) but whose accountUuid has
|
|
22
|
-
* drifted, so upstream 401s every request. Structural inspection says fine;
|
|
23
|
-
* only a round-trip says otherwise.
|
|
24
|
-
*
|
|
25
|
-
* WHAT IT PROVES, AND WHAT IT DELIBERATELY DOES NOT
|
|
26
|
-
*
|
|
27
|
-
* The probe is a minimal `POST /v1/messages` (max_tokens 1) sent DIRECTLY to
|
|
28
|
-
* api.anthropic.com with the same auth the request path uses — pool bearer, or
|
|
29
|
-
* `x-api-key` in upstream-API-key mode. That proves: a token can be acquired,
|
|
30
|
-
* the network path is up, and upstream accepts our credential.
|
|
31
|
-
*
|
|
32
|
-
* It deliberately does NOT go through dario's own proxy path:
|
|
33
|
-
* - No recursion, no self-deadlock, no port/auth assumptions.
|
|
34
|
-
* - It must not take a concurrency slot. A probe that queues behind real
|
|
35
|
-
* traffic would report "unhealthy" during a legitimate burst and hand a
|
|
36
|
-
* watchdog a reason to restart a busy-but-fine dario. Slot exhaustion is
|
|
37
|
-
* covered instead by `queue.saturatedSince` (request-queue.ts), which is
|
|
38
|
-
* free, needs no tokens, and cannot false-positive on a short burst.
|
|
39
|
-
*
|
|
40
|
-
* So: the probe covers the credential/network axis, the queue snapshot covers
|
|
41
|
-
* the concurrency axis. Neither claims to cover the transform path.
|
|
42
|
-
*
|
|
43
|
-
* COST AND EXPOSURE
|
|
44
|
-
*
|
|
45
|
-
* A probe is a real billed request. Three guards, all load-bearing:
|
|
46
|
-
* 1. OPT-IN. Only runs when a caller explicitly asks (`/health?probe=1`).
|
|
47
|
-
* A plain /health never spends a token — existing docker healthchecks and
|
|
48
|
-
* uptime monitors keep their current cost profile, which is zero.
|
|
49
|
-
* 2. TRUSTED CALLERS ONLY. proxy.ts honours `?probe=1` only for callers that
|
|
50
|
-
* already pass shouldDiscloseHealthInternals. A world-readable /health
|
|
51
|
-
* behind a Cloudflare tunnel bypass must not be a button the internet can
|
|
52
|
-
* press to spend the operator's tokens.
|
|
53
|
-
* 3. CACHED + SINGLE-FLIGHTED. Results are reused for `ttlMs` (default 60s)
|
|
54
|
-
* and concurrent callers share one in-flight request, so a monitor polling
|
|
55
|
-
* every second still costs at most one probe per minute.
|
|
56
|
-
*
|
|
57
|
-
* VERDICT SEMANTICS — why 429 is NOT a failure
|
|
58
|
-
*
|
|
59
|
-
* `ok` answers "would a restart or an alert help?", not "did the call return
|
|
60
|
-
* 200". Rate-limited (429) and upstream-overloaded (529) are healthy states:
|
|
61
|
-
* dario is working, the answer is legitimately "not right now", and a watchdog
|
|
62
|
-
* that restarts on them just thrashes. That is the same lesson /livez already
|
|
63
|
-
* encodes in proxy.ts — a shared-refresh-family outage once had dario restart
|
|
64
|
-
* -looping for 4h+ because the healthcheck keyed on a condition a restart
|
|
65
|
-
* cannot fix. Auth rejection, 5xx, network failure and timeout DO set ok=false.
|
|
66
|
-
*/
|
|
67
|
-
/** Cheapest family, and the one doctor's own probe already uses. */
|
|
68
1
|
export declare const DEFAULT_PROBE_MODEL = "claude-haiku-4-5";
|
|
69
2
|
export declare const DEFAULT_PROBE_TTL_MS = 60000;
|
|
70
3
|
export declare const DEFAULT_PROBE_TIMEOUT_MS = 15000;
|
|
71
|
-
|
|
72
|
-
* Why the round-trip ended the way it did. Carried on both verdicts so an
|
|
73
|
-
* operator reading a green probe can still tell "served" from "rate-limited".
|
|
74
|
-
*/
|
|
75
|
-
export type ProbeReason = 'served' | 'rate-limited' | 'upstream-overloaded' | 'no-token' | 'auth-rejected' | 'upstream-error' | 'timeout' | 'network-error';
|
|
4
|
+
export type ProbeReason = 'served' | 'billing-required' | 'rate-limited' | 'upstream-overloaded' | 'no-token' | 'auth-rejected' | 'upstream-error' | 'timeout' | 'network-error';
|
|
76
5
|
export interface ProbeResult {
|
|
77
|
-
/** False only for conditions where dario is the problem — see module header. */
|
|
78
6
|
ok: boolean;
|
|
79
7
|
reason: ProbeReason;
|
|
80
|
-
/** Epoch ms the round-trip completed. */
|
|
81
8
|
checkedAt: number;
|
|
82
|
-
/** Wall time of the round-trip, including token acquisition. */
|
|
83
9
|
latencyMs: number;
|
|
84
10
|
model: string;
|
|
85
|
-
/** Upstream HTTP status, when upstream answered at all. */
|
|
86
11
|
status?: number;
|
|
87
|
-
/** Short failure detail. Never carries a token or a raw body. */
|
|
88
12
|
detail?: string;
|
|
89
13
|
}
|
|
90
14
|
export interface ProbeDeps {
|
|
91
15
|
fetchImpl?: typeof fetch;
|
|
92
|
-
/** OAuth bearer source. Ignored when upstreamApiKey is set. */
|
|
93
16
|
getToken?: () => Promise<string>;
|
|
94
|
-
/** Per-token API mode — forwarded as x-api-key, mirroring request-path auth. */
|
|
95
17
|
upstreamApiKey?: string;
|
|
96
18
|
now?: () => number;
|
|
97
19
|
model?: string;
|
|
98
20
|
timeoutMs?: number;
|
|
99
21
|
ttlMs?: number;
|
|
100
22
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
23
|
+
export declare function classifyProbeResponse(status: number, body?: string): {
|
|
24
|
+
ok: boolean;
|
|
25
|
+
reason: ProbeReason;
|
|
26
|
+
detail?: string;
|
|
27
|
+
};
|
|
28
|
+
/** Backward-compatible status-only classifier; body-aware callers use classifyProbeResponse. */
|
|
105
29
|
export declare function classifyProbeStatus(status: number): {
|
|
106
30
|
ok: boolean;
|
|
107
31
|
reason: ProbeReason;
|
|
108
32
|
};
|
|
109
|
-
/**
|
|
110
|
-
* The cached verdict, refreshing it when stale. Never throws — a health
|
|
111
|
-
* surface that can 500 is worse than useless, so every failure path becomes a
|
|
112
|
-
* `ok: false` verdict with a reason instead of an exception.
|
|
113
|
-
*
|
|
114
|
-
* Concurrent callers past the TTL share one in-flight probe; the loser of the
|
|
115
|
-
* race gets the same result rather than sending a second billed request.
|
|
116
|
-
*/
|
|
117
33
|
export declare function getServingProbe(deps?: ProbeDeps): Promise<ProbeResult>;
|
|
118
|
-
/** Age of the cached verdict, for the `ageMs` field callers see. */
|
|
119
34
|
export declare function probeAgeMs(result: ProbeResult, now: number): number;
|
|
120
35
|
export declare function _resetServingProbeForTest(): void;
|