@askalf/dario 6.0.12 → 6.0.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,78 @@
1
+ /**
2
+ * Per-provider rate-limit cool-down for the failover chain (DEV-f66b131c).
3
+ *
4
+ * The chain used to have no memory. Every 429 was a fresh discovery, so with
5
+ * BOTH ends capped — a spent Claude 5h window and a 429-ing ChatGPT
6
+ * subscription — one request walked codex → claude → codex before dropping,
7
+ * and the next request did it again. 185 such lines in 38 minutes on
8
+ * 2026-09-02. That is the wrong shape twice over: at the moment quota is the
9
+ * scarce resource the chain spends TWICE as much of it discovering there is
10
+ * none, and whatever is cycling when a window rolls over consumes the fresh
11
+ * window ahead of legitimately queued work.
12
+ *
13
+ * Two rules, both enforced here so the request path can stay a straight line:
14
+ *
15
+ * WITHIN a request — an entry that already declined is never revisited
16
+ * ({@link canAttempt} against the attempted set).
17
+ * ACROSS requests — a 429 cools the entry for a bounded interval, honouring
18
+ * `retry-after` when the upstream sends one.
19
+ *
20
+ * When every entry is cooled the caller fails fast with one terminal verdict
21
+ * instead of a retry storm — see {@link allProvidersCooled}.
22
+ *
23
+ * Provider-granular, not entry-granular: a chain names MODELS
24
+ * (`gpt-5.6-sol,claude-sonnet-5`) but a 429 is a property of the ACCOUNT
25
+ * behind them, so cooling `codex` also cools every other codex slug in the
26
+ * chain, which is the correct blast radius.
27
+ *
28
+ * The clock is injectable so the whole thing is testable without sleeping.
29
+ */
30
+ /** Cool-down applied when the upstream gives no `retry-after`. */
31
+ export declare const DEFAULT_COOLDOWN_MS = 60000;
32
+ /**
33
+ * Ceiling on any cool-down. An upstream is free to say "come back in 4 hours";
34
+ * honouring that literally would keep a provider parked long after a window
35
+ * reset we cannot observe. One re-probe per 15 minutes is cheap.
36
+ */
37
+ export declare const MAX_COOLDOWN_MS: number;
38
+ /** Machine-readable terminal verdict. Distinct from `pool exhausted` (which
39
+ * implies a peer exists) and from the billing/credential classes. */
40
+ export declare const ALL_PROVIDERS_RATE_LIMITED = "all-providers-rate-limited";
41
+ /**
42
+ * `retry-after` as milliseconds. The header is either delta-seconds or an
43
+ * HTTP-date (RFC 9110 §10.2.3); both are accepted, anything else is null so
44
+ * the caller falls back to its default rather than trusting a parse it did not
45
+ * get. A date already in the past yields 0 — "retry now" — not a negative.
46
+ */
47
+ export declare function parseRetryAfterMs(value: string | null | undefined, now?: number): number | null;
48
+ /** A provider that declined, and for how long it should stay declined. */
49
+ export declare class ProviderCooldowns {
50
+ private readonly until;
51
+ private readonly now;
52
+ private readonly defaultMs;
53
+ constructor(now?: () => number, defaultMs?: number);
54
+ /**
55
+ * Record a 429 (or an equivalent "not right now") for `provider`. Returns the
56
+ * cool-down actually applied, so the caller can put it in its one log line.
57
+ * A `retryAfterMs` of 0 is honoured as "retry now" and cools nothing.
58
+ */
59
+ note(provider: string, retryAfterMs?: number | null): number;
60
+ /** Clear on a success — a provider that just served is not rate-limited. */
61
+ clear(provider: string): void;
62
+ remainingMs(provider: string): number;
63
+ isCooled(provider: string): boolean;
64
+ }
65
+ /**
66
+ * May this request try `provider`? False when it already declined during THIS
67
+ * request (the codex → claude → codex revisit) or when it is cooling from an
68
+ * earlier one.
69
+ */
70
+ export declare function canAttempt(provider: string, attempted: ReadonlySet<string>, cooldowns: ProviderCooldowns): boolean;
71
+ /**
72
+ * Every named provider is cooling — the fail-fast condition. An EMPTY list is
73
+ * NOT "all cooled": nothing configured is a configuration answer, not a rate
74
+ * limit, and must keep its existing message.
75
+ */
76
+ export declare function allProvidersCooled(providers: readonly string[], cooldowns: ProviderCooldowns): boolean;
77
+ /** Longest remaining cool-down across `providers` — what to put in `retry-after`. */
78
+ export declare function cooldownRetryAfterMs(providers: readonly string[], cooldowns: ProviderCooldowns): number;
@@ -0,0 +1,120 @@
1
+ /**
2
+ * Per-provider rate-limit cool-down for the failover chain (DEV-f66b131c).
3
+ *
4
+ * The chain used to have no memory. Every 429 was a fresh discovery, so with
5
+ * BOTH ends capped — a spent Claude 5h window and a 429-ing ChatGPT
6
+ * subscription — one request walked codex → claude → codex before dropping,
7
+ * and the next request did it again. 185 such lines in 38 minutes on
8
+ * 2026-09-02. That is the wrong shape twice over: at the moment quota is the
9
+ * scarce resource the chain spends TWICE as much of it discovering there is
10
+ * none, and whatever is cycling when a window rolls over consumes the fresh
11
+ * window ahead of legitimately queued work.
12
+ *
13
+ * Two rules, both enforced here so the request path can stay a straight line:
14
+ *
15
+ * WITHIN a request — an entry that already declined is never revisited
16
+ * ({@link canAttempt} against the attempted set).
17
+ * ACROSS requests — a 429 cools the entry for a bounded interval, honouring
18
+ * `retry-after` when the upstream sends one.
19
+ *
20
+ * When every entry is cooled the caller fails fast with one terminal verdict
21
+ * instead of a retry storm — see {@link allProvidersCooled}.
22
+ *
23
+ * Provider-granular, not entry-granular: a chain names MODELS
24
+ * (`gpt-5.6-sol,claude-sonnet-5`) but a 429 is a property of the ACCOUNT
25
+ * behind them, so cooling `codex` also cools every other codex slug in the
26
+ * chain, which is the correct blast radius.
27
+ *
28
+ * The clock is injectable so the whole thing is testable without sleeping.
29
+ */
30
+ /** Cool-down applied when the upstream gives no `retry-after`. */
31
+ export const DEFAULT_COOLDOWN_MS = 60_000;
32
+ /**
33
+ * Ceiling on any cool-down. An upstream is free to say "come back in 4 hours";
34
+ * honouring that literally would keep a provider parked long after a window
35
+ * reset we cannot observe. One re-probe per 15 minutes is cheap.
36
+ */
37
+ export const MAX_COOLDOWN_MS = 15 * 60_000;
38
+ /** Machine-readable terminal verdict. Distinct from `pool exhausted` (which
39
+ * implies a peer exists) and from the billing/credential classes. */
40
+ export const ALL_PROVIDERS_RATE_LIMITED = 'all-providers-rate-limited';
41
+ /**
42
+ * `retry-after` as milliseconds. The header is either delta-seconds or an
43
+ * HTTP-date (RFC 9110 §10.2.3); both are accepted, anything else is null so
44
+ * the caller falls back to its default rather than trusting a parse it did not
45
+ * get. A date already in the past yields 0 — "retry now" — not a negative.
46
+ */
47
+ export function parseRetryAfterMs(value, now = Date.now()) {
48
+ if (value == null)
49
+ return null;
50
+ const trimmed = value.trim();
51
+ if (trimmed.length === 0)
52
+ return null;
53
+ if (/^\d+$/.test(trimmed))
54
+ return Math.min(Number(trimmed) * 1000, MAX_COOLDOWN_MS);
55
+ const at = Date.parse(trimmed);
56
+ if (Number.isNaN(at))
57
+ return null;
58
+ return Math.min(Math.max(at - now, 0), MAX_COOLDOWN_MS);
59
+ }
60
+ /** A provider that declined, and for how long it should stay declined. */
61
+ export class ProviderCooldowns {
62
+ until = new Map();
63
+ now;
64
+ defaultMs;
65
+ constructor(now = () => Date.now(), defaultMs = DEFAULT_COOLDOWN_MS) {
66
+ this.now = now;
67
+ this.defaultMs = defaultMs;
68
+ }
69
+ /**
70
+ * Record a 429 (or an equivalent "not right now") for `provider`. Returns the
71
+ * cool-down actually applied, so the caller can put it in its one log line.
72
+ * A `retryAfterMs` of 0 is honoured as "retry now" and cools nothing.
73
+ */
74
+ note(provider, retryAfterMs) {
75
+ const ms = retryAfterMs == null ? this.defaultMs : Math.min(Math.max(retryAfterMs, 0), MAX_COOLDOWN_MS);
76
+ this.until.set(provider, this.now() + ms);
77
+ return ms;
78
+ }
79
+ /** Clear on a success — a provider that just served is not rate-limited. */
80
+ clear(provider) {
81
+ this.until.delete(provider);
82
+ }
83
+ remainingMs(provider) {
84
+ const at = this.until.get(provider);
85
+ if (at == null)
86
+ return 0;
87
+ const left = at - this.now();
88
+ if (left <= 0) {
89
+ // Expired entries are dropped on read; nothing else sweeps this map and
90
+ // a long-lived proxy would otherwise accumulate one entry per provider
91
+ // forever (bounded, but pointlessly).
92
+ this.until.delete(provider);
93
+ return 0;
94
+ }
95
+ return left;
96
+ }
97
+ isCooled(provider) {
98
+ return this.remainingMs(provider) > 0;
99
+ }
100
+ }
101
+ /**
102
+ * May this request try `provider`? False when it already declined during THIS
103
+ * request (the codex → claude → codex revisit) or when it is cooling from an
104
+ * earlier one.
105
+ */
106
+ export function canAttempt(provider, attempted, cooldowns) {
107
+ return !attempted.has(provider) && !cooldowns.isCooled(provider);
108
+ }
109
+ /**
110
+ * Every named provider is cooling — the fail-fast condition. An EMPTY list is
111
+ * NOT "all cooled": nothing configured is a configuration answer, not a rate
112
+ * limit, and must keep its existing message.
113
+ */
114
+ export function allProvidersCooled(providers, cooldowns) {
115
+ return providers.length > 0 && providers.every(p => cooldowns.isCooled(p));
116
+ }
117
+ /** Longest remaining cool-down across `providers` — what to put in `retry-after`. */
118
+ export function cooldownRetryAfterMs(providers, cooldowns) {
119
+ return providers.reduce((max, p) => Math.max(max, cooldowns.remainingMs(p)), 0);
120
+ }
package/dist/proxy.js CHANGED
@@ -28,6 +28,7 @@ import { RequestQueue, QueueFullError, QueueTimeoutError, DEFAULT_MAX_CONCURRENT
28
28
  import { redactSecrets } from './redact.js';
29
29
  import { BAKED_BASE_MODELS, withLongContextVariants, buildOpenAIModelsList, getModelCatalog, getCachedBases, resolveAliasAgainst, prewarmModelCatalog, retryModelCatalogNow, isSuspendedModel } from './model-catalog.js';
30
30
  import { classifyUpstreamRejection, diagnosticSnippet } from './upstream-rejection.js';
31
+ import { ProviderCooldowns, canAttempt, allProvidersCooled, cooldownRetryAfterMs, parseRetryAfterMs, ALL_PROVIDERS_RATE_LIMITED, } from './provider-cooldown.js';
31
32
  const ANTHROPIC_API = 'https://api.anthropic.com';
32
33
  const DEFAULT_PORT = 3456;
33
34
  const MAX_BODY_BYTES = 10 * 1024 * 1024; // 10 MB — generous for large prompts, prevents abuse
@@ -1328,6 +1329,10 @@ export async function startProxy(opts = {}) {
1328
1329
  // Per-alias request counts for GET /codex — the pool has requestCount per
1329
1330
  // account; the codex accounts had nothing until now.
1330
1331
  const codexRequestCounts = new Map();
1332
+ // Rate-limit memory for the failover chain (DEV-f66b131c). Process-wide and
1333
+ // deliberately NOT per-request: the point is that request #269 does not
1334
+ // re-discover the limit #267 already found. Cleared on any success.
1335
+ const providerCooldowns = new ProviderCooldowns();
1331
1336
  // Overage-guard (v4.1, dario#288). Resolved from opts with built-in
1332
1337
  // defaults (enabled=true, behavior='halt', cooldown=30min, notifyOs=true)
1333
1338
  // so an opts-less proxy still gets protection. The notifier is wired
@@ -1828,9 +1833,14 @@ export async function startProxy(opts = {}) {
1828
1833
  * a silently swapped model family is precisely the surprise this project
1829
1834
  * exists to avoid.
1830
1835
  */
1831
- const tryCodexPoolFallback = async (req, res, body, fallbackModels, shape, why) => {
1836
+ const tryCodexPoolFallback = async (req, res, body, fallbackModels, shape, why, attempted) => {
1832
1837
  if (fallbackModels.length === 0)
1833
1838
  return false;
1839
+ // Never a second codex attempt in the same request, and never one while it
1840
+ // is cooling from an earlier 429. This is the revisit the 20:13:40 log
1841
+ // sequence shows: codex -> claude -> codex, all inside request #267.
1842
+ if (!canAttempt('codex', attempted, providerCooldowns))
1843
+ return false;
1834
1844
  if (!(await hasAnyCodexAccount().catch(() => false)))
1835
1845
  return false;
1836
1846
  const stored = await selectCodexAccount().catch(() => null);
@@ -1852,6 +1862,7 @@ export async function startProxy(opts = {}) {
1852
1862
  return false;
1853
1863
  console.log(`[dario] #${requestCount} ${why} → codex account ${creds.alias} as ${fallbackModel}`);
1854
1864
  requestCount++;
1865
+ attempted.add('codex');
1855
1866
  // If an api-key backend could ALSO serve this request, let the subscription
1856
1867
  // decline a 429/5xx rather than answer with it, and report not-served so the
1857
1868
  // caller falls through to that backend. This helper's contract has always
@@ -1862,7 +1873,15 @@ export async function startProxy(opts = {}) {
1862
1873
  // With NO next option, do not defer: the real upstream error is more useful
1863
1874
  // to the client than replacing it with a generic 503.
1864
1875
  const hasNextOption = openaiBackend !== null && shape === 'openai';
1865
- return await forwardToCodex(req, res, fallbackBody, creds, corsOrigin, { ...SECURITY_HEADERS, 'x-dario-pool-fallback': fallbackModel }, upstreamTimeoutMs, verbose, shape, fetch, hasNextOption);
1876
+ const served = await forwardToCodex(req, res, fallbackBody, creds, corsOrigin, { ...SECURITY_HEADERS, 'x-dario-pool-fallback': fallbackModel }, upstreamTimeoutMs, verbose, shape, fetch, hasNextOption, undefined,
1877
+ // Only a rate limit cools the provider. A 5xx or a transport failure is
1878
+ // an outage, not quota — cooling it would park a provider that may be
1879
+ // back on the next request, which is the opposite of the fix.
1880
+ (d) => { if (d.status === 429)
1881
+ providerCooldowns.note('codex', d.retryAfterMs); });
1882
+ if (served)
1883
+ providerCooldowns.clear('codex');
1884
+ return served;
1866
1885
  };
1867
1886
  /**
1868
1887
  * Mid-flight 429, no peer account left to retry — the pool is genuinely
@@ -1885,8 +1904,8 @@ export async function startProxy(opts = {}) {
1885
1904
  * bug structurally impossible to reintroduce: a future third call site gets
1886
1905
  * this by construction, not by remembering to copy six lines correctly.
1887
1906
  */
1888
- const attemptPoolFallbackOn429 = async (req, res, body, isOpenAI) => {
1889
- if (await tryCodexPoolFallback(req, res, body, poolFallbackModels, isOpenAI ? 'openai' : 'anthropic', 'pool exhausted mid-flight (429, no peer)')) {
1907
+ const attemptPoolFallbackOn429 = async (req, res, body, isOpenAI, attempted) => {
1908
+ if (await tryCodexPoolFallback(req, res, body, poolFallbackModels, isOpenAI ? 'openai' : 'anthropic', 'pool exhausted mid-flight (429, no peer)', attempted)) {
1890
1909
  return true;
1891
1910
  }
1892
1911
  if (isOpenAI && poolFallbackModel && openaiBackend) {
@@ -2434,6 +2453,10 @@ export async function startProxy(opts = {}) {
2434
2453
  // form gives the declared type without seeding a narrowing.
2435
2454
  let poolAccount = null;
2436
2455
  let accessToken = '';
2456
+ // Providers this ONE request has already asked. Paired with the
2457
+ // process-wide cool-downs: the set stops a revisit inside the request,
2458
+ // the cool-downs stop the next request repeating the discovery.
2459
+ const attemptedProviders = new Set();
2437
2460
  /**
2438
2461
  * Take a Claude pool account for this request. Returns false when it has
2439
2462
  * already answered the client (empty or fully drained pool with no viable
@@ -2464,6 +2487,35 @@ export async function startProxy(opts = {}) {
2464
2487
  message: 'all accounts are rate-limited or in auth cool-down; retry shortly',
2465
2488
  }));
2466
2489
  };
2490
+ /**
2491
+ * Terminal verdict for DEV-f66b131c: every provider that could serve this
2492
+ * request is inside a 429 cool-down, so there is nothing left to try and
2493
+ * no reason to spend an upstream attempt proving it. ONE log line and one
2494
+ * response — the defect being fixed is a three-line-per-request storm.
2495
+ *
2496
+ * 429 rather than the 503 `writePoolUnavailable` gives, and distinct from
2497
+ * `pool exhausted` (which implies a peer exists): this IS a rate limit, it
2498
+ * self-clears, and `retry-after` says when. The
2499
+ * `x-dario-upstream-rejection` marker is the same channel the billing and
2500
+ * credential classes use, so the platform-side sustained-fallback tracker
2501
+ * reads one field for all of them.
2502
+ */
2503
+ const writeAllProvidersRateLimited = (providers) => {
2504
+ const retryAfterSec = Math.max(1, Math.ceil(cooldownRetryAfterMs(providers, providerCooldowns) / 1000));
2505
+ console.log(`[dario] #${requestCount} ${ALL_PROVIDERS_RATE_LIMITED} (${providers.join('+')}) — no attempt made, retry in ${retryAfterSec}s`);
2506
+ res.writeHead(429, {
2507
+ ...JSON_HEADERS,
2508
+ 'retry-after': String(retryAfterSec),
2509
+ 'x-dario-upstream-rejection': ALL_PROVIDERS_RATE_LIMITED,
2510
+ });
2511
+ res.end(JSON.stringify({
2512
+ error: {
2513
+ type: 'rate_limit_error',
2514
+ message: `All configured providers (${providers.join(', ')}) are rate-limited; retry in ${retryAfterSec}s.`,
2515
+ },
2516
+ reason: ALL_PROVIDERS_RATE_LIMITED,
2517
+ }));
2518
+ };
2467
2519
  const selectPoolAccount = () => {
2468
2520
  if (upstreamApiKey) {
2469
2521
  // Per-token API-key mode: no OAuth, no pool selection. `poolAccount`
@@ -2752,8 +2804,19 @@ export async function startProxy(opts = {}) {
2752
2804
  const claudeTarget = pickClaudeTarget(poolFallbackModels, codexModels, getCachedBases(), (m) => resolveClaudeAlias(applyModelAlias(m, modelAliases) ?? m));
2753
2805
  const claudeTargetModel = claudeTarget?.model ?? null;
2754
2806
  const canDefer = claudeTarget !== null && pool.size > 0 && !upstreamApiKey;
2807
+ // DEV-f66b131c: an account still inside its 429 cool-down is not
2808
+ // asked again. Skipping straight to the Claude half of the chain is
2809
+ // the whole saving — the doomed attempt was spending quota on BOTH
2810
+ // accounts to rediscover a limit the previous request already found.
2811
+ const codexAvailable = canAttempt('codex', attemptedProviders, providerCooldowns);
2812
+ if (!codexAvailable && !canDefer) {
2813
+ // Nothing else in the chain can pick this up, so an attempt could
2814
+ // only rediscover the limit. One verdict, no upstream request.
2815
+ writeAllProvidersRateLimited(['codex']);
2816
+ return;
2817
+ }
2755
2818
  const codexReq = requestCount;
2756
- const served = await forwardToCodex(req, res, body, codexCreds, corsOrigin, SECURITY_HEADERS, upstreamTimeoutMs, verbose, isOpenAI ? 'openai' : 'anthropic', fetch, canDefer,
2819
+ const served = codexAvailable && await forwardToCodex(req, res, body, codexCreds, corsOrigin, SECURITY_HEADERS, upstreamTimeoutMs, verbose, isOpenAI ? 'openai' : 'anthropic', fetch, canDefer,
2757
2820
  // Before this hook a codex request left no trace: nothing in
2758
2821
  // /analytics, nothing in the request log, no per-account count.
2759
2822
  // The dock (and anyone reading /analytics) saw a proxy that
@@ -2780,9 +2843,26 @@ export async function startProxy(opts = {}) {
2780
2843
  status: o.status, latency_ms: o.latencyMs, in_tokens: o.inputTokens, out_tokens: o.outputTokens,
2781
2844
  claim: CODEX_CLAIM, bucket: 'subscription', account: o.alias, stream: o.stream,
2782
2845
  });
2783
- });
2784
- if (served)
2846
+ },
2847
+ // Cool codex on a rate limit only — a 5xx or an unreachable backend
2848
+ // is an outage, and parking a provider for that would keep it out
2849
+ // of the chain while it was already coming back.
2850
+ (d) => { if (d.status === 429)
2851
+ providerCooldowns.note('codex', d.retryAfterMs); });
2852
+ if (served) {
2853
+ // A provider that just served is not rate-limited.
2854
+ providerCooldowns.clear('codex');
2855
+ return;
2856
+ }
2857
+ if (codexAvailable)
2858
+ attemptedProviders.add('codex');
2859
+ // Both ends of the chain limited — the case this ticket is about.
2860
+ // Fail fast with the machine-readable verdict instead of handing the
2861
+ // request on to a Claude pool that is itself cooling.
2862
+ if (providerCooldowns.isCooled('codex') && providerCooldowns.isCooled('claude')) {
2863
+ writeAllProvidersRateLimited(['codex', 'claude']);
2785
2864
  return;
2865
+ }
2786
2866
  const swapped = buildPoolFallbackBody(body, claudeTargetModel);
2787
2867
  if (!swapped) {
2788
2868
  res.writeHead(503, { 'Content-Type': 'application/json', ...SECURITY_HEADERS });
@@ -2836,7 +2916,12 @@ export async function startProxy(opts = {}) {
2836
2916
  // response carries `x-dario-pool-fallback` — a substituted model must
2837
2917
  // never be silent. GPT-bound requests never reach here (the routing
2838
2918
  // block above already forwarded them; they don't need the pool).
2839
- if (!upstreamApiKey && !poolAccount && await tryCodexPoolFallback(req, res, body, poolFallbackModels, isOpenAI ? 'openai' : 'anthropic', 'pool exhausted')) {
2919
+ // Selection returns null only for an empty pool or all-account auth-cooldown.
2920
+ // Genuine rate limits return an account and are cooled from the upstream 429.
2921
+ if (!upstreamApiKey && !poolAccount) {
2922
+ attemptedProviders.add('claude');
2923
+ }
2924
+ if (!upstreamApiKey && !poolAccount && await tryCodexPoolFallback(req, res, body, poolFallbackModels, isOpenAI ? 'openai' : 'anthropic', 'pool exhausted', attemptedProviders)) {
2840
2925
  return;
2841
2926
  }
2842
2927
  // `isOpenAI` is REQUIRED here and was not, before v6.0.0 — the selector's
@@ -2864,6 +2949,12 @@ export async function startProxy(opts = {}) {
2864
2949
  // would fall through to the Claude path with no account and an empty
2865
2950
  // bearer token, turning a clean 503 into a confusing upstream 401.
2866
2951
  if (!upstreamApiKey && !poolAccount) {
2952
+ // A chain where every entry is cooling is a rate limit, not a
2953
+ // misconfiguration — say so in the machine-readable way, once.
2954
+ if (allProvidersCooled(poolFallbackModels.length > 0 ? ['codex', 'claude'] : ['claude'], providerCooldowns)) {
2955
+ writeAllProvidersRateLimited(poolFallbackModels.length > 0 ? ['codex', 'claude'] : ['claude']);
2956
+ return;
2957
+ }
2867
2958
  console.log(`[dario] #${requestCount} pool exhausted and no fallback provider could serve ${poolFallbackModels.join(', ') || '(none configured)'}`);
2868
2959
  writePoolUnavailable();
2869
2960
  return;
@@ -3658,7 +3749,16 @@ export async function startProxy(opts = {}) {
3658
3749
  // attemptPoolFallbackOn429 below, so the file goes back to having
3659
3750
  // exactly ONE copy of this logic instead of two near-identical ones
3660
3751
  // — which is the exact shape of bug this incident already was.
3661
- if (await attemptPoolFallbackOn429(req, res, body, isOpenAI)) {
3752
+ // The Claude half is out of quota for this request AND for the next
3753
+ // one — cool it here, honouring the upstream `retry-after` when it
3754
+ // sent one, so the following request does not re-walk the chain.
3755
+ attemptedProviders.add('claude');
3756
+ providerCooldowns.note('claude', parseRetryAfterMs(upstream.headers.get('retry-after')));
3757
+ if (await attemptPoolFallbackOn429(req, res, body, isOpenAI, attemptedProviders)) {
3758
+ return;
3759
+ }
3760
+ if (allProvidersCooled(['codex', 'claude'], providerCooldowns)) {
3761
+ writeAllProvidersRateLimited(['codex', 'claude']);
3662
3762
  return;
3663
3763
  }
3664
3764
  const enriched = enrich429(peekedBody, upstream.headers);
@@ -3776,7 +3876,14 @@ export async function startProxy(opts = {}) {
3776
3876
  // translation (OpenAI-shape clients) went into finalBody, never back
3777
3877
  // into body. See attemptPoolFallbackOn429's doc comment for why this
3778
3878
  // is one shared function rather than an inline copy.
3779
- if (await attemptPoolFallbackOn429(req, res, body, isOpenAI)) {
3879
+ // Same bookkeeping as the other mid-flight site — see there.
3880
+ attemptedProviders.add('claude');
3881
+ providerCooldowns.note('claude', parseRetryAfterMs(upstream.headers.get('retry-after')));
3882
+ if (await attemptPoolFallbackOn429(req, res, body, isOpenAI, attemptedProviders)) {
3883
+ return;
3884
+ }
3885
+ if (allProvidersCooled(['codex', 'claude'], providerCooldowns)) {
3886
+ writeAllProvidersRateLimited(['codex', 'claude']);
3780
3887
  return;
3781
3888
  }
3782
3889
  const errBody = await upstream.text().catch(() => '');
@@ -3813,6 +3920,8 @@ export async function startProxy(opts = {}) {
3813
3920
  // so its consecutive-failure counter resets. dario#234.
3814
3921
  if (poolAccount && upstream.status >= 200 && upstream.status < 300) {
3815
3922
  pool.clearAuthFailure(poolAccount.alias);
3923
+ // Served — whatever cooled this provider has passed.
3924
+ providerCooldowns.clear('claude');
3816
3925
  }
3817
3926
  break;
3818
3927
  } // end dispatchLoop: while (true)
@@ -1,120 +1,35 @@
1
- /**
2
- * serving-probe.ts — the "can dario actually serve a request?" check.
3
- *
4
- * WHY THIS EXISTS (dario#905, dario#921)
5
- *
6
- * dario's existing health surfaces are both STRUCTURAL — they inspect state,
7
- * they never prove anything end-to-end:
8
- *
9
- * /livez the HTTP server is accepting connections. Always 200.
10
- * /health credentials/pool look sane: token not expired, refresh not broken,
11
- * accounts not all in auth-cooldown, queue snapshot attached.
12
- *
13
- * Neither can catch the failure class that actually took a user down for ~14h
14
- * across four episodes (#905): dario looking perfectly healthy from the outside
15
- * while every real request failed. The reporter's remediation was an external
16
- * watchdog that sends a real inference call and restarts the service when it
17
- * doesn't come back — i.e. the user had to write the liveness check dario
18
- * should own. This module is that check, brought in-house.
19
- *
20
- * It also covers a class /health provably cannot see: an account whose token is
21
- * unexpired and refreshable (so `status: healthy`) but whose accountUuid has
22
- * drifted, so upstream 401s every request. Structural inspection says fine;
23
- * only a round-trip says otherwise.
24
- *
25
- * WHAT IT PROVES, AND WHAT IT DELIBERATELY DOES NOT
26
- *
27
- * The probe is a minimal `POST /v1/messages` (max_tokens 1) sent DIRECTLY to
28
- * api.anthropic.com with the same auth the request path uses — pool bearer, or
29
- * `x-api-key` in upstream-API-key mode. That proves: a token can be acquired,
30
- * the network path is up, and upstream accepts our credential.
31
- *
32
- * It deliberately does NOT go through dario's own proxy path:
33
- * - No recursion, no self-deadlock, no port/auth assumptions.
34
- * - It must not take a concurrency slot. A probe that queues behind real
35
- * traffic would report "unhealthy" during a legitimate burst and hand a
36
- * watchdog a reason to restart a busy-but-fine dario. Slot exhaustion is
37
- * covered instead by `queue.saturatedSince` (request-queue.ts), which is
38
- * free, needs no tokens, and cannot false-positive on a short burst.
39
- *
40
- * So: the probe covers the credential/network axis, the queue snapshot covers
41
- * the concurrency axis. Neither claims to cover the transform path.
42
- *
43
- * COST AND EXPOSURE
44
- *
45
- * A probe is a real billed request. Three guards, all load-bearing:
46
- * 1. OPT-IN. Only runs when a caller explicitly asks (`/health?probe=1`).
47
- * A plain /health never spends a token — existing docker healthchecks and
48
- * uptime monitors keep their current cost profile, which is zero.
49
- * 2. TRUSTED CALLERS ONLY. proxy.ts honours `?probe=1` only for callers that
50
- * already pass shouldDiscloseHealthInternals. A world-readable /health
51
- * behind a Cloudflare tunnel bypass must not be a button the internet can
52
- * press to spend the operator's tokens.
53
- * 3. CACHED + SINGLE-FLIGHTED. Results are reused for `ttlMs` (default 60s)
54
- * and concurrent callers share one in-flight request, so a monitor polling
55
- * every second still costs at most one probe per minute.
56
- *
57
- * VERDICT SEMANTICS — why 429 is NOT a failure
58
- *
59
- * `ok` answers "would a restart or an alert help?", not "did the call return
60
- * 200". Rate-limited (429) and upstream-overloaded (529) are healthy states:
61
- * dario is working, the answer is legitimately "not right now", and a watchdog
62
- * that restarts on them just thrashes. That is the same lesson /livez already
63
- * encodes in proxy.ts — a shared-refresh-family outage once had dario restart
64
- * -looping for 4h+ because the healthcheck keyed on a condition a restart
65
- * cannot fix. Auth rejection, 5xx, network failure and timeout DO set ok=false.
66
- */
67
- /** Cheapest family, and the one doctor's own probe already uses. */
68
1
  export declare const DEFAULT_PROBE_MODEL = "claude-haiku-4-5";
69
2
  export declare const DEFAULT_PROBE_TTL_MS = 60000;
70
3
  export declare const DEFAULT_PROBE_TIMEOUT_MS = 15000;
71
- /**
72
- * Why the round-trip ended the way it did. Carried on both verdicts so an
73
- * operator reading a green probe can still tell "served" from "rate-limited".
74
- */
75
- export type ProbeReason = 'served' | 'rate-limited' | 'upstream-overloaded' | 'no-token' | 'auth-rejected' | 'upstream-error' | 'timeout' | 'network-error';
4
+ export type ProbeReason = 'served' | 'billing-required' | 'rate-limited' | 'upstream-overloaded' | 'no-token' | 'auth-rejected' | 'upstream-error' | 'timeout' | 'network-error';
76
5
  export interface ProbeResult {
77
- /** False only for conditions where dario is the problem — see module header. */
78
6
  ok: boolean;
79
7
  reason: ProbeReason;
80
- /** Epoch ms the round-trip completed. */
81
8
  checkedAt: number;
82
- /** Wall time of the round-trip, including token acquisition. */
83
9
  latencyMs: number;
84
10
  model: string;
85
- /** Upstream HTTP status, when upstream answered at all. */
86
11
  status?: number;
87
- /** Short failure detail. Never carries a token or a raw body. */
88
12
  detail?: string;
89
13
  }
90
14
  export interface ProbeDeps {
91
15
  fetchImpl?: typeof fetch;
92
- /** OAuth bearer source. Ignored when upstreamApiKey is set. */
93
16
  getToken?: () => Promise<string>;
94
- /** Per-token API mode — forwarded as x-api-key, mirroring request-path auth. */
95
17
  upstreamApiKey?: string;
96
18
  now?: () => number;
97
19
  model?: string;
98
20
  timeoutMs?: number;
99
21
  ttlMs?: number;
100
22
  }
101
- /**
102
- * Map an upstream HTTP status onto a verdict. Pure, so the whole policy
103
- * ("which statuses mean dario is broken") is testable without a network.
104
- */
23
+ export declare function classifyProbeResponse(status: number, body?: string): {
24
+ ok: boolean;
25
+ reason: ProbeReason;
26
+ detail?: string;
27
+ };
28
+ /** Backward-compatible status-only classifier; body-aware callers use classifyProbeResponse. */
105
29
  export declare function classifyProbeStatus(status: number): {
106
30
  ok: boolean;
107
31
  reason: ProbeReason;
108
32
  };
109
- /**
110
- * The cached verdict, refreshing it when stale. Never throws — a health
111
- * surface that can 500 is worse than useless, so every failure path becomes a
112
- * `ok: false` verdict with a reason instead of an exception.
113
- *
114
- * Concurrent callers past the TTL share one in-flight probe; the loser of the
115
- * race gets the same result rather than sending a second billed request.
116
- */
117
33
  export declare function getServingProbe(deps?: ProbeDeps): Promise<ProbeResult>;
118
- /** Age of the cached verdict, for the `ageMs` field callers see. */
119
34
  export declare function probeAgeMs(result: ProbeResult, now: number): number;
120
35
  export declare function _resetServingProbeForTest(): void;