@askalf/dario 5.4.31 → 5.5.1

package/dist/health-response.d.ts

@@ -45,7 +45,35 @@ export interface HealthStatusLike {
         queued: number;
         maxConcurrent: number;
         maxQueued: number;
+        /**
+         * Epoch ms since the queue has been at capacity with NO slot released
+         * (dario#905). Null when not at capacity; reset by any release, so
+         * sustained legitimate load never accumulates age here. See
+         * request-queue.ts for why turnover — not depth — is the wedge signal.
+         */
+        stalledSince?: number | null;
     };
+    /**
+     * Verdict from the opt-in serving probe (`/health?probe=1`), when the caller
+     * asked for one and was trusted enough to be given it. Absent otherwise —
+     * a plain /health never spends a token, so its absence means "not asked
+     * for", never "failed".
+     */
+    probe?: ServingProbeLike;
+}
+/**
+ * The subset of serving-probe.ts's ProbeResult that /health renders. Declared
+ * structurally rather than imported so this module stays free of the probe's
+ * network machinery and remains unit-testable as a pure function.
+ */
+export interface ServingProbeLike {
+    ok: boolean;
+    reason: string;
+    checkedAt: number;
+    latencyMs: number;
+    model: string;
+    status?: number;
+    detail?: string;
 }
 export interface HealthResponse {
     httpStatus: number;
@@ -65,22 +93,87 @@ export interface PoolDerivedStatus {
     accounts: number;
 }
 export declare function derivePoolStatus(accounts: readonly PoolAccountStatusLike[], now: number, adminEnabled: boolean): PoolDerivedStatus;
-export declare function buildHealthResponse(s: HealthStatusLike, requestCount: number, includeInternal: boolean): HealthResponse;
+export declare function buildHealthResponse(s: HealthStatusLike, requestCount: number, includeInternal: boolean, now?: number): HealthResponse;
+/**
+ * Did this caller ask for a serving probe? (`/health?probe=1`, dario#905.)
+ *
+ * Accepts `probe=1` / `probe=true` / bare `probe`, rejects `probe=0` and
+ * `probe=false` — a monitor templating the flag from a boolean config should
+ * get the behaviour it wrote, not a probe on every poll because the parameter
+ * was merely present. Anything unparseable is treated as "not asked": the
+ * failure direction for a token-spending flag has to be off.
+ */
+export declare function probeRequested(url: string | undefined): boolean;
+/**
+ * Decide whether to actually RUN a serving probe for this caller (dario#905).
+ *
+ * Deliberately stricter than shouldDiscloseHealthInternals, because this is
+ * not a disclosure decision — it spends the operator's money.
+ *
+ * The disclosure gate grants access to a caller that proved a configured
+ * DARIO_API_KEY, wherever it came from. That is the right answer for reading a
+ * field. It is not the right answer for an action that bills per call: a
+ * leaked or shared key becomes a metered spend endpoint reachable from the
+ * public internet, and the probe's own cache means an attacker needs only one
+ * request per TTL to keep it running indefinitely.
+ *
+ * So the probe additionally refuses anything that arrived through the tunnel,
+ * whatever the disclosure gate concluded. This only ever DENIES — it cannot
+ * widen access.
+ *
+ * (An unkeyed proxy is handled a layer up: shouldDiscloseHealthInternals now
+ * requires `keyConfigured`, so vacuous authentication no longer reaches here
+ * at all. This gate does not depend on that fix — it would refuse the tunnel
+ * caller either way — but the two are the same defence at different depths.)
+ *
+ * Accepted trade-off: an operator who authenticates THROUGH the tunnel is also
+ * refused, and has to probe from beside the proxy instead. For a flag whose
+ * failure direction is "silently spends money", off is the right default.
+ */
+export declare function shouldRunServingProbe(opts: {
+    requested: boolean;
+    discloseInternals: boolean;
+    viaCfRay: boolean;
+}): boolean;
 /**
  * Decide whether a /health caller may see the OAuth internals (#642).
  *
  * /health is intentionally auth-free (docker healthchecks need it before a
  * key is configured), so we cannot simply gate on the API key. Trust model:
- *   - authenticated (valid DARIO_API_KEY)      -> internal (an internal caller)
+ *   - PROVED a configured DARIO_API_KEY        -> internal (an internal caller)
  *   - came via the Cloudflare tunnel (cf-ray)  -> public   (world-reachable)
  *   - otherwise bare loopback                  -> internal (docker HC / doctor)
  *   - otherwise (LAN, other container, WAN)    -> public
- * The cf-ray check is now only ever used to DENY (force public), never to
- * grant, so spoofing it cannot widen disclosure — the previous fail-open
- * direction is closed.
+ * The cf-ray check is only ever used to DENY (force public), never to grant,
+ * so spoofing it cannot widen disclosure.
+ *
+ * `keyConfigured` is load-bearing and is why `authenticated` alone is not
+ * enough. `authenticateRequest()` short-circuits to TRUE when no
+ * DARIO_API_KEY is set — a deliberate convenience, since the common setup is
+ * loopback-only and requiring a key there would break `dario doctor` and every
+ * docker healthcheck. But it means "authenticated" is VACUOUS on an unkeyed
+ * proxy: every caller satisfies it, the first branch returns before cf-ray is
+ * ever consulted, and an unkeyed dario published through a Cloudflare tunnel
+ * hands its OAuth countdown, request volume and refresh-failure count to
+ * anyone who asks. That is the #642 fail-open re-entering through a side door
+ * — #642 closed the spoofable-header direction, not this one.
+ *
+ * Requiring both means the auth branch can only be taken by a caller that
+ * actually presented the operator's secret. Unkeyed proxies fall through to
+ * the transport rules, where loopback is still trusted (healthchecks and
+ * doctor keep working, unchanged) and the tunnel is not.
+ *
+ * `keyConfigured` is a REQUIRED field rather than an optional with a default:
+ * for a security predicate, every call site should be forced to state it.
+ *
+ * The HTTP status (200/503) is unaffected either way, so uptime monitoring
+ * that keys on the status code sees no change from this.
  */
 export declare function shouldDiscloseHealthInternals(opts: {
+    /** Passed authenticateRequest — which is vacuously true when unkeyed. */
     authenticated: boolean;
+    /** Whether a DARIO_API_KEY exists at all, i.e. whether `authenticated` means anything. */
+    keyConfigured: boolean;
     loopback: boolean;
     viaCfRay: boolean;
 }): boolean;

package/dist/health-response.js

@@ -59,10 +59,28 @@ export function derivePoolStatus(accounts, now, adminEnabled) {
         expiresIn: formatMsLeft(earliest - now),
     };
 }
-export function buildHealthResponse(s, requestCount, includeInternal) {
-    const dead = s.status === 'broken' ||
+/**
+ * Render `stalledSince` as an elapsed duration alongside the raw stamp. The
+ * stamp alone forces every consumer to subtract against its own clock, which
+ * is exactly the arithmetic a shell-based healthcheck can't do — and #905's
+ * reporter was writing his monitor in bash.
+ */
+function withStalledFor(q, now) {
+    const { stalledSince, ...rest } = q;
+    if (stalledSince === null || stalledSince === undefined)
+        return { ...rest, stalledSince: null };
+    return { ...rest, stalledSince, stalledForMs: Math.max(0, now - stalledSince) };
+}
+export function buildHealthResponse(s, requestCount, includeInternal, now = Date.now()) {
+    const structurallyDead = s.status === 'broken' ||
         s.status === 'none' ||
         (s.status === 'expired' && s.canRefresh === false);
+    // A failed round-trip is authoritative over a clean structural read: the
+    // whole point of the probe (dario#905) is the state where local inspection
+    // says healthy and every real request fails. When one was run and it came
+    // back false, /health must say degraded — that is what makes an existing
+    // status-code-only uptime monitor start seeing the outage it used to miss.
+    const dead = structurallyDead || s.probe?.ok === false;
     const httpStatus = dead ? 503 : 200;
     const liveness = { status: dead ? 'degraded' : 'ok' };
     // Only trusted callers (authenticated, or bare loopback not via the CF
@@ -82,27 +100,102 @@ export function buildHealthResponse(s, requestCount, includeInternal) {
             expiresIn: s.expiresIn,
             requests: requestCount,
             ...(s.sessions ? { sessions: s.sessions } : {}),
-            ...(s.queue ? { queue: s.queue } : {}),
+            ...(s.queue ? { queue: withStalledFor(s.queue, now) } : {}),
+            ...(s.probe ? { probe: { ...s.probe, ageMs: Math.max(0, now - s.probe.checkedAt) } } : {}),
             ...(s.refreshFailures ? { refreshFailures: s.refreshFailures } : {}),
         }
         : liveness;
     return { httpStatus, body };
 }
+/**
+ * Did this caller ask for a serving probe? (`/health?probe=1`, dario#905.)
+ *
+ * Accepts `probe=1` / `probe=true` / bare `probe`, rejects `probe=0` and
+ * `probe=false` — a monitor templating the flag from a boolean config should
+ * get the behaviour it wrote, not a probe on every poll because the parameter
+ * was merely present. Anything unparseable is treated as "not asked": the
+ * failure direction for a token-spending flag has to be off.
+ */
+export function probeRequested(url) {
+    const q = url?.indexOf('?') ?? -1;
+    if (q < 0)
+        return false;
+    const v = new URLSearchParams(url.slice(q + 1)).get('probe');
+    if (v === null)
+        return false;
+    if (v === '')
+        return true; // bare `?probe`
+    return v === '1' || v.toLowerCase() === 'true';
+}
+/**
+ * Decide whether to actually RUN a serving probe for this caller (dario#905).
+ *
+ * Deliberately stricter than shouldDiscloseHealthInternals, because this is
+ * not a disclosure decision — it spends the operator's money.
+ *
+ * The disclosure gate grants access to a caller that proved a configured
+ * DARIO_API_KEY, wherever it came from. That is the right answer for reading a
+ * field. It is not the right answer for an action that bills per call: a
+ * leaked or shared key becomes a metered spend endpoint reachable from the
+ * public internet, and the probe's own cache means an attacker needs only one
+ * request per TTL to keep it running indefinitely.
+ *
+ * So the probe additionally refuses anything that arrived through the tunnel,
+ * whatever the disclosure gate concluded. This only ever DENIES — it cannot
+ * widen access.
+ *
+ * (An unkeyed proxy is handled a layer up: shouldDiscloseHealthInternals now
+ * requires `keyConfigured`, so vacuous authentication no longer reaches here
+ * at all. This gate does not depend on that fix — it would refuse the tunnel
+ * caller either way — but the two are the same defence at different depths.)
+ *
+ * Accepted trade-off: an operator who authenticates THROUGH the tunnel is also
+ * refused, and has to probe from beside the proxy instead. For a flag whose
+ * failure direction is "silently spends money", off is the right default.
+ */
+export function shouldRunServingProbe(opts) {
+    if (!opts.requested)
+        return false;
+    if (opts.viaCfRay)
+        return false;
+    return opts.discloseInternals;
+}
 /**
  * Decide whether a /health caller may see the OAuth internals (#642).
  *
  * /health is intentionally auth-free (docker healthchecks need it before a
  * key is configured), so we cannot simply gate on the API key. Trust model:
- *   - authenticated (valid DARIO_API_KEY)      -> internal (an internal caller)
+ *   - PROVED a configured DARIO_API_KEY        -> internal (an internal caller)
  *   - came via the Cloudflare tunnel (cf-ray)  -> public   (world-reachable)
  *   - otherwise bare loopback                  -> internal (docker HC / doctor)
  *   - otherwise (LAN, other container, WAN)    -> public
- * The cf-ray check is now only ever used to DENY (force public), never to
- * grant, so spoofing it cannot widen disclosure — the previous fail-open
- * direction is closed.
+ * The cf-ray check is only ever used to DENY (force public), never to grant,
+ * so spoofing it cannot widen disclosure.
+ *
+ * `keyConfigured` is load-bearing and is why `authenticated` alone is not
+ * enough. `authenticateRequest()` short-circuits to TRUE when no
+ * DARIO_API_KEY is set — a deliberate convenience, since the common setup is
+ * loopback-only and requiring a key there would break `dario doctor` and every
+ * docker healthcheck. But it means "authenticated" is VACUOUS on an unkeyed
+ * proxy: every caller satisfies it, the first branch returns before cf-ray is
+ * ever consulted, and an unkeyed dario published through a Cloudflare tunnel
+ * hands its OAuth countdown, request volume and refresh-failure count to
+ * anyone who asks. That is the #642 fail-open re-entering through a side door
+ * — #642 closed the spoofable-header direction, not this one.
+ *
+ * Requiring both means the auth branch can only be taken by a caller that
+ * actually presented the operator's secret. Unkeyed proxies fall through to
+ * the transport rules, where loopback is still trusted (healthchecks and
+ * doctor keep working, unchanged) and the tunnel is not.
+ *
+ * `keyConfigured` is a REQUIRED field rather than an optional with a default:
+ * for a security predicate, every call site should be forced to state it.
+ *
+ * The HTTP status (200/503) is unaffected either way, so uptime monitoring
+ * that keys on the status code sees no change from this.
  */
 export function shouldDiscloseHealthInternals(opts) {
-    if (opts.authenticated)
+    if (opts.authenticated && opts.keyConfigured)
         return true;
     if (opts.viaCfRay)
         return false;

package/dist/proxy.js

@@ -6,7 +6,8 @@ import { homedir } from 'node:os';
 import { setDefaultResultOrder } from 'node:dns';
 import { arch, platform } from 'node:process';
 import { getAccessToken, getStatus, ignoreCcCredentials } from './oauth.js';
-import { buildHealthResponse, derivePoolStatus, shouldDiscloseHealthInternals } from './health-response.js';
+import { buildHealthResponse, derivePoolStatus, probeRequested, shouldDiscloseHealthInternals, shouldRunServingProbe } from './health-response.js';
+import { getServingProbe } from './serving-probe.js';
 import { darioVersion } from './version.js';
 import { buildCCRequest, applyCcPromptCaching, parseEffortSuffix, reverseMapResponse, createStreamingReverseMapper, orderHeadersForOutbound, overlayTemplateHeaderValues, forwardClientCCIdentityHeaders, isMcpToolName, CC_TEMPLATE, effectiveCacheControl, withForced1hBeta } from './cc-template.js';
 import { stampCch, hasCchSeed } from './cch.js';
@@ -1697,14 +1698,39 @@ export async function startProxy(opts = {}) {
             // key on the presence of the client-suppliable `cf-ray` header and failed
             // OPEN — a direct non-tunnel caller omits it and got the full internal
             // view. Now: authenticated, OR bare loopback that did not arrive via CF.
+            const viaCfRay = req.headers['cf-ray'] !== undefined;
             const includeInternal = shouldDiscloseHealthInternals({
                 authenticated: authenticateRequest(req.headers, apiKeyBuf),
+                // Without this, `authenticated` is vacuously true on an unkeyed proxy
+                // and the tunnel check below is never reached — see the gate's docs.
+                keyConfigured: apiKeyBuf !== null,
                 loopback: isLoopbackAddr(req.socket?.remoteAddress),
-                viaCfRay: req.headers['cf-ray'] !== undefined,
+                viaCfRay,
             });
+            // Opt-in serving probe (#905). A real upstream round-trip costs a real
+            // (tiny) billed request, so it runs only when explicitly asked for and
+            // only for callers that clear shouldRunServingProbe — which is stricter
+            // than the disclosure gate on purpose (see its docstring: an unkeyed
+            // dario authenticates everyone, which must not turn `?probe=1` into a
+            // spend button for the public internet). Cached and single-flighted
+            // inside getServingProbe, so a monitor polling every second still costs
+            // at most one probe per TTL.
+            const wantsProbe = shouldRunServingProbe({
+                requested: probeRequested(req.url),
+                discloseInternals: includeInternal,
+                viaCfRay,
+            });
+            const probe = wantsProbe
+                ? await getServingProbe({
+                    fetchImpl: upstreamFetch,
+                    getToken: catalogDeps.getToken,
+                    upstreamApiKey: upstreamApiKey || undefined,
+                })
+                : undefined;
             const { httpStatus, body } = buildHealthResponse({
                 ...s,
                 version: darioVersion(),
+                ...(probe ? { probe } : {}),
                 // pool.size === 0 is single-account mode (session-id registry drives
                 // the SESSION_ID slot); a loaded pool routes via sticky bindings.
                 sessions: pool.size === 0

package/dist/request-queue.d.ts

@@ -28,6 +28,31 @@ export interface QueueState {
     maxConcurrent: number;
     maxQueued: number;
 }
+/**
+ * QueueState plus the one derived field a monitor actually needs (dario#905).
+ *
+ * #910 put `active` / `queued` on /health so slot exhaustion stopped being
+ * invisible. But a raw sample cannot distinguish the 14h wedge from a healthy
+ * one-second burst — both read `active === maxConcurrent, queued > 0`. Polling
+ * fast enough to tell them apart is the monitor's problem, and it shouldn't be.
+ *
+ * The distinguishing signal is TURNOVER, not depth. A busy dario runs at its
+ * cap with a backlog all day and is perfectly healthy, because slots keep
+ * being released. The #905 wedge held `active` at `maxConcurrent` for hours
+ * with no release at all.
+ *
+ * So `stalledSince` is the epoch ms since which the queue has been at capacity
+ * with requests waiting AND NOT ONE SLOT HAS BEEN RELEASED. Any release resets
+ * it. Null when the queue isn't at capacity. A non-null value older than a
+ * request could plausibly take means slots are not turning over — one sample
+ * is enough to see it, and sustained legitimate load never trips it.
+ *
+ * This is also why the serving probe deliberately doesn't take a slot: the
+ * concurrency axis is covered here, for free and without false positives.
+ */
+export interface QueueSnapshot extends QueueState {
+    stalledSince: number | null;
+}
 export type AdmitDecision = {
     action: 'admit';
 } | {
@@ -60,6 +85,8 @@ export interface RequestQueueOptions {
      * test is waiting for never arrives.
      */
     unrefTimers?: boolean;
+    /** Clock source for `saturatedSince`. Injectable so tests need no timers. */
+    now?: () => number;
 }
 export declare const DEFAULT_MAX_CONCURRENT = 10;
 export declare const DEFAULT_MAX_QUEUED = 128;
@@ -71,7 +98,19 @@ export declare class RequestQueue {
     readonly unrefTimers: boolean;
     private active;
     private queue;
+    private readonly now;
+    private stalledSince;
     constructor(opts?: RequestQueueOptions);
+    /**
+     * Re-evaluate the stall stamp. Called after every state change, so it marks
+     * when the stall BEGAN rather than when it was last observed — a caller that
+     * never polls still reads an accurate duration.
+     *
+     * Arrivals must NOT refresh the stamp: under a steady arrival rate that
+     * would reset the clock continuously and hide a permanent wedge. Only
+     * `release()` refreshes it, by clearing first (see there).
+     */
+    private updateStall;
     /**
      * Acquire a concurrency slot. Resolves when admitted; throws
      * `QueueFullError` when the queue is at its `maxQueued` cap, throws
@@ -81,6 +120,6 @@ export declare class RequestQueue {
     acquire(): Promise<void>;
     /** Release a slot. The next queued entry (if any) is admitted in FIFO order. */
     release(): void;
-    /** Snapshot of queue state — exposed for /analytics + tests. */
-    snapshot(): QueueState;
+    /** Snapshot of queue state — exposed for /health + /analytics + tests. */
+    snapshot(): QueueSnapshot;
 }

package/dist/request-queue.js

@@ -50,11 +50,32 @@ export class RequestQueue {
     unrefTimers;
     active = 0;
     queue = [];
+    now;
+    stalledSince = null;
     constructor(opts = {}) {
         this.maxConcurrent = opts.maxConcurrent ?? DEFAULT_MAX_CONCURRENT;
         this.maxQueued = opts.maxQueued ?? DEFAULT_MAX_QUEUED;
         this.queueTimeoutMs = opts.queueTimeoutMs ?? DEFAULT_QUEUE_TIMEOUT_MS;
         this.unrefTimers = opts.unrefTimers ?? true;
+        this.now = opts.now ?? Date.now;
+    }
+    /**
+     * Re-evaluate the stall stamp. Called after every state change, so it marks
+     * when the stall BEGAN rather than when it was last observed — a caller that
+     * never polls still reads an accurate duration.
+     *
+     * Arrivals must NOT refresh the stamp: under a steady arrival rate that
+     * would reset the clock continuously and hide a permanent wedge. Only
+     * `release()` refreshes it, by clearing first (see there).
+     */
+    updateStall() {
+        const atCapacity = this.active >= this.maxConcurrent && this.queue.length > 0;
+        if (!atCapacity) {
+            this.stalledSince = null;
+            return;
+        }
+        if (this.stalledSince === null)
+            this.stalledSince = this.now();
     }
     /**
      * Acquire a concurrency slot. Resolves when admitted; throws
@@ -66,17 +87,19 @@ export class RequestQueue {
         const decision = decideAdmit(this.snapshot());
         if (decision.action === 'admit') {
             this.active++;
+            this.updateStall();
             return;
         }
         if (decision.action === 'reject') {
             throw new QueueFullError();
         }
         return new Promise((resolve, reject) => {
-            const enqueuedAt = Date.now();
+            const enqueuedAt = this.now();
             const timeoutHandle = setTimeout(() => {
                 const idx = this.queue.indexOf(entry);
                 if (idx >= 0) {
                     this.queue.splice(idx, 1);
+                    this.updateStall();
                     reject(new QueueTimeoutError());
                 }
             }, this.queueTimeoutMs);
@@ -87,6 +110,7 @@ export class RequestQueue {
                 timeoutHandle.unref?.();
             const entry = { resolve, reject, enqueuedAt, timeoutHandle };
             this.queue.push(entry);
+            this.updateStall();
         });
     }
     /** Release a slot. The next queued entry (if any) is admitted in FIFO order. */
@@ -99,14 +123,22 @@ export class RequestQueue {
             this.active++;
             next.resolve();
         }
+        // A release IS turnover — the thing whose absence defines the wedge — so
+        // clear the stamp unconditionally before re-evaluating. A queue that is
+        // still at capacity immediately starts a FRESH stall window, which is why
+        // a saturated-but-flowing dario never accumulates age here while a
+        // genuinely wedged one does.
+        this.stalledSince = null;
+        this.updateStall();
     }
-    /** Snapshot of queue state — exposed for /analytics + tests. */
+    /** Snapshot of queue state — exposed for /health + /analytics + tests. */
     snapshot() {
         return {
             active: this.active,
             queued: this.queue.length,
             maxConcurrent: this.maxConcurrent,
             maxQueued: this.maxQueued,
+            stalledSince: this.stalledSince,
         };
     }
 }

package/dist/serving-probe.d.ts

@@ -0,0 +1,120 @@
+/**
+ * serving-probe.ts — the "can dario actually serve a request?" check.
+ *
+ * WHY THIS EXISTS (dario#905, dario#921)
+ *
+ * dario's existing health surfaces are both STRUCTURAL — they inspect state,
+ * they never prove anything end-to-end:
+ *
+ *   /livez   the HTTP server is accepting connections. Always 200.
+ *   /health  credentials/pool look sane: token not expired, refresh not broken,
+ *            accounts not all in auth-cooldown, queue snapshot attached.
+ *
+ * Neither can catch the failure class that actually took a user down for ~14h
+ * across four episodes (#905): dario looking perfectly healthy from the outside
+ * while every real request failed. The reporter's remediation was an external
+ * watchdog that sends a real inference call and restarts the service when it
+ * doesn't come back — i.e. the user had to write the liveness check dario
+ * should own. This module is that check, brought in-house.
+ *
+ * It also covers a class /health provably cannot see: an account whose token is
+ * unexpired and refreshable (so `status: healthy`) but whose accountUuid has
+ * drifted, so upstream 401s every request. Structural inspection says fine;
+ * only a round-trip says otherwise.
+ *
+ * WHAT IT PROVES, AND WHAT IT DELIBERATELY DOES NOT
+ *
+ * The probe is a minimal `POST /v1/messages` (max_tokens 1) sent DIRECTLY to
+ * api.anthropic.com with the same auth the request path uses — pool bearer, or
+ * `x-api-key` in upstream-API-key mode. That proves: a token can be acquired,
+ * the network path is up, and upstream accepts our credential.
+ *
+ * It deliberately does NOT go through dario's own proxy path:
+ *   - No recursion, no self-deadlock, no port/auth assumptions.
+ *   - It must not take a concurrency slot. A probe that queues behind real
+ *     traffic would report "unhealthy" during a legitimate burst and hand a
+ *     watchdog a reason to restart a busy-but-fine dario. Slot exhaustion is
+ *     covered instead by `queue.saturatedSince` (request-queue.ts), which is
+ *     free, needs no tokens, and cannot false-positive on a short burst.
+ *
+ * So: the probe covers the credential/network axis, the queue snapshot covers
+ * the concurrency axis. Neither claims to cover the transform path.
+ *
+ * COST AND EXPOSURE
+ *
+ * A probe is a real billed request. Three guards, all load-bearing:
+ *   1. OPT-IN. Only runs when a caller explicitly asks (`/health?probe=1`).
+ *      A plain /health never spends a token — existing docker healthchecks and
+ *      uptime monitors keep their current cost profile, which is zero.
+ *   2. TRUSTED CALLERS ONLY. proxy.ts honours `?probe=1` only for callers that
+ *      already pass shouldDiscloseHealthInternals. A world-readable /health
+ *      behind a Cloudflare tunnel bypass must not be a button the internet can
+ *      press to spend the operator's tokens.
+ *   3. CACHED + SINGLE-FLIGHTED. Results are reused for `ttlMs` (default 60s)
+ *      and concurrent callers share one in-flight request, so a monitor polling
+ *      every second still costs at most one probe per minute.
+ *
+ * VERDICT SEMANTICS — why 429 is NOT a failure
+ *
+ * `ok` answers "would a restart or an alert help?", not "did the call return
+ * 200". Rate-limited (429) and upstream-overloaded (529) are healthy states:
+ * dario is working, the answer is legitimately "not right now", and a watchdog
+ * that restarts on them just thrashes. That is the same lesson /livez already
+ * encodes in proxy.ts — a shared-refresh-family outage once had dario restart
+ * -looping for 4h+ because the healthcheck keyed on a condition a restart
+ * cannot fix. Auth rejection, 5xx, network failure and timeout DO set ok=false.
+ */
+/** Cheapest family, and the one doctor's own probe already uses. */
+export declare const DEFAULT_PROBE_MODEL = "claude-haiku-4-5";
+export declare const DEFAULT_PROBE_TTL_MS = 60000;
+export declare const DEFAULT_PROBE_TIMEOUT_MS = 15000;
+/**
+ * Why the round-trip ended the way it did. Carried on both verdicts so an
+ * operator reading a green probe can still tell "served" from "rate-limited".
+ */
+export type ProbeReason = 'served' | 'rate-limited' | 'upstream-overloaded' | 'no-token' | 'auth-rejected' | 'upstream-error' | 'timeout' | 'network-error';
+export interface ProbeResult {
+    /** False only for conditions where dario is the problem — see module header. */
+    ok: boolean;
+    reason: ProbeReason;
+    /** Epoch ms the round-trip completed. */
+    checkedAt: number;
+    /** Wall time of the round-trip, including token acquisition. */
+    latencyMs: number;
+    model: string;
+    /** Upstream HTTP status, when upstream answered at all. */
+    status?: number;
+    /** Short failure detail. Never carries a token or a raw body. */
+    detail?: string;
+}
+export interface ProbeDeps {
+    fetchImpl?: typeof fetch;
+    /** OAuth bearer source. Ignored when upstreamApiKey is set. */
+    getToken?: () => Promise<string>;
+    /** Per-token API mode — forwarded as x-api-key, mirroring request-path auth. */
+    upstreamApiKey?: string;
+    now?: () => number;
+    model?: string;
+    timeoutMs?: number;
+    ttlMs?: number;
+}
+/**
+ * Map an upstream HTTP status onto a verdict. Pure, so the whole policy
+ * ("which statuses mean dario is broken") is testable without a network.
+ */
+export declare function classifyProbeStatus(status: number): {
+    ok: boolean;
+    reason: ProbeReason;
+};
+/**
+ * The cached verdict, refreshing it when stale. Never throws — a health
+ * surface that can 500 is worse than useless, so every failure path becomes a
+ * `ok: false` verdict with a reason instead of an exception.
+ *
+ * Concurrent callers past the TTL share one in-flight probe; the loser of the
+ * race gets the same result rather than sending a second billed request.
+ */
+export declare function getServingProbe(deps?: ProbeDeps): Promise<ProbeResult>;
+/** Age of the cached verdict, for the `ageMs` field callers see. */
+export declare function probeAgeMs(result: ProbeResult, now: number): number;
+export declare function _resetServingProbeForTest(): void;

package/dist/serving-probe.js

@@ -0,0 +1,204 @@
+/**
+ * serving-probe.ts — the "can dario actually serve a request?" check.
+ *
+ * WHY THIS EXISTS (dario#905, dario#921)
+ *
+ * dario's existing health surfaces are both STRUCTURAL — they inspect state,
+ * they never prove anything end-to-end:
+ *
+ *   /livez   the HTTP server is accepting connections. Always 200.
+ *   /health  credentials/pool look sane: token not expired, refresh not broken,
+ *            accounts not all in auth-cooldown, queue snapshot attached.
+ *
+ * Neither can catch the failure class that actually took a user down for ~14h
+ * across four episodes (#905): dario looking perfectly healthy from the outside
+ * while every real request failed. The reporter's remediation was an external
+ * watchdog that sends a real inference call and restarts the service when it
+ * doesn't come back — i.e. the user had to write the liveness check dario
+ * should own. This module is that check, brought in-house.
+ *
+ * It also covers a class /health provably cannot see: an account whose token is
+ * unexpired and refreshable (so `status: healthy`) but whose accountUuid has
+ * drifted, so upstream 401s every request. Structural inspection says fine;
+ * only a round-trip says otherwise.
+ *
+ * WHAT IT PROVES, AND WHAT IT DELIBERATELY DOES NOT
+ *
+ * The probe is a minimal `POST /v1/messages` (max_tokens 1) sent DIRECTLY to
+ * api.anthropic.com with the same auth the request path uses — pool bearer, or
+ * `x-api-key` in upstream-API-key mode. That proves: a token can be acquired,
+ * the network path is up, and upstream accepts our credential.
+ *
+ * It deliberately does NOT go through dario's own proxy path:
+ *   - No recursion, no self-deadlock, no port/auth assumptions.
+ *   - It must not take a concurrency slot. A probe that queues behind real
+ *     traffic would report "unhealthy" during a legitimate burst and hand a
+ *     watchdog a reason to restart a busy-but-fine dario. Slot exhaustion is
+ *     covered instead by `queue.saturatedSince` (request-queue.ts), which is
+ *     free, needs no tokens, and cannot false-positive on a short burst.
+ *
+ * So: the probe covers the credential/network axis, the queue snapshot covers
+ * the concurrency axis. Neither claims to cover the transform path.
+ *
+ * COST AND EXPOSURE
+ *
+ * A probe is a real billed request. Three guards, all load-bearing:
+ *   1. OPT-IN. Only runs when a caller explicitly asks (`/health?probe=1`).
+ *      A plain /health never spends a token — existing docker healthchecks and
+ *      uptime monitors keep their current cost profile, which is zero.
+ *   2. TRUSTED CALLERS ONLY. proxy.ts honours `?probe=1` only for callers that
+ *      already pass shouldDiscloseHealthInternals. A world-readable /health
+ *      behind a Cloudflare tunnel bypass must not be a button the internet can
+ *      press to spend the operator's tokens.
+ *   3. CACHED + SINGLE-FLIGHTED. Results are reused for `ttlMs` (default 60s)
+ *      and concurrent callers share one in-flight request, so a monitor polling
+ *      every second still costs at most one probe per minute.
+ *
+ * VERDICT SEMANTICS — why 429 is NOT a failure
+ *
+ * `ok` answers "would a restart or an alert help?", not "did the call return
+ * 200". Rate-limited (429) and upstream-overloaded (529) are healthy states:
+ * dario is working, the answer is legitimately "not right now", and a watchdog
+ * that restarts on them just thrashes. That is the same lesson /livez already
+ * encodes in proxy.ts — a shared-refresh-family outage once had dario restart
+ * -looping for 4h+ because the healthcheck keyed on a condition a restart
+ * cannot fix. Auth rejection, 5xx, network failure and timeout DO set ok=false.
+ */
+const ANTHROPIC_MESSAGES = 'https://api.anthropic.com/v1/messages';
+const ANTHROPIC_VERSION = '2023-06-01';
+const OAUTH_BETA = 'oauth-2025-04-20';
+/** Cheapest family, and the one doctor's own probe already uses. */
+export const DEFAULT_PROBE_MODEL = 'claude-haiku-4-5';
+export const DEFAULT_PROBE_TTL_MS = 60_000;
+export const DEFAULT_PROBE_TIMEOUT_MS = 15_000;
+let cache = null;
+let inflight = null;
+function envInt(name, dflt) {
+    const v = Number(process.env[name]);
+    return Number.isFinite(v) && v > 0 ? v : dflt;
+}
+/**
+ * Map an upstream HTTP status onto a verdict. Pure, so the whole policy
+ * ("which statuses mean dario is broken") is testable without a network.
+ */
+export function classifyProbeStatus(status) {
+    if (status >= 200 && status < 300)
+        return { ok: true, reason: 'served' };
+    // Serving fine, just throttled — restarting cannot help, so do not fail.
+    if (status === 429)
+        return { ok: true, reason: 'rate-limited' };
+    if (status === 529)
+        return { ok: true, reason: 'upstream-overloaded' };
+    // The credential is being rejected even though local state looks valid.
+    // This is the accountUuid-drift signature /health structurally cannot see.
+    if (status === 401 || status === 403)
+        return { ok: false, reason: 'auth-rejected' };
+    return { ok: false, reason: 'upstream-error' };
+}
+async function runProbe(deps) {
+    const f = deps.fetchImpl ?? fetch;
+    const now = deps.now ?? Date.now;
+    const model = deps.model ?? process.env.DARIO_PROBE_MODEL ?? DEFAULT_PROBE_MODEL;
+    const timeoutMs = deps.timeoutMs ?? envInt('DARIO_PROBE_TIMEOUT_MS', DEFAULT_PROBE_TIMEOUT_MS);
+    const startedAt = now();
+    // Arm the deadline BEFORE token acquisition so a wedged refresh is bounded
+    // too, not just the fetch — the same trap model-catalog.ts hit in #642: a
+    // getToken() that never settles would leave `inflight` non-null forever and
+    // permanently wedge every future probe on a stale cached verdict.
+    const ctl = new AbortController();
+    const timer = setTimeout(() => ctl.abort(), timeoutMs);
+    const finish = (r) => ({
+        ...r,
+        model,
+        checkedAt: now(),
+        latencyMs: now() - startedAt,
+    });
+    try {
+        const headers = {
+            'content-type': 'application/json',
+            'anthropic-version': ANTHROPIC_VERSION,
+        };
+        if (deps.upstreamApiKey) {
+            headers['x-api-key'] = deps.upstreamApiKey;
+        }
+        else {
+            if (!deps.getToken)
+                return finish({ ok: false, reason: 'no-token', detail: 'no token source configured' });
+            let token;
+            try {
+                token = await Promise.race([
+                    deps.getToken(),
+                    new Promise((_, rej) => {
+                        ctl.signal.addEventListener('abort', () => rej(new Error('token acquisition timed out')), { once: true });
+                    }),
+                ]);
+            }
+            catch (err) {
+                // An empty pool lands here — the message already says so (pool.ts /
+                // catalogDeps phrase it for the operator), so surface it verbatim.
+                return finish({ ok: false, reason: 'no-token', detail: errText(err) });
+            }
+            headers['authorization'] = `Bearer ${token}`;
+            headers['anthropic-beta'] = OAUTH_BETA;
+        }
+        const res = await f(ANTHROPIC_MESSAGES, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify({ model, max_tokens: 1, messages: [{ role: 'user', content: 'ping' }] }),
+            signal: ctl.signal,
+        });
+        // Drain so the socket is released back to the agent pool. The body is
+        // never inspected: a 200 is the whole signal, and the content could carry
+        // model output we have no reason to log.
+        await res.text().catch(() => '');
+        const verdict = classifyProbeStatus(res.status);
+        return finish({ ...verdict, status: res.status });
+    }
+    catch (err) {
+        const aborted = ctl.signal.aborted || err?.name === 'AbortError';
+        return finish({
+            ok: false,
+            reason: aborted ? 'timeout' : 'network-error',
+            detail: aborted ? `no upstream response within ${timeoutMs}ms` : errText(err),
+        });
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+function errText(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+/**
+ * The cached verdict, refreshing it when stale. Never throws — a health
+ * surface that can 500 is worse than useless, so every failure path becomes a
+ * `ok: false` verdict with a reason instead of an exception.
+ *
+ * Concurrent callers past the TTL share one in-flight probe; the loser of the
+ * race gets the same result rather than sending a second billed request.
+ */
+export async function getServingProbe(deps = {}) {
+    const now = (deps.now ?? Date.now)();
+    const ttl = deps.ttlMs ?? envInt('DARIO_PROBE_TTL_MS', DEFAULT_PROBE_TTL_MS);
+    if (cache !== null && now - cache.checkedAt < ttl)
+        return cache;
+    if (inflight !== null)
+        return inflight;
+    inflight = runProbe(deps)
+        .then((r) => {
+        cache = r;
+        return r;
+    })
+        .finally(() => {
+        inflight = null;
+    });
+    return inflight;
+}
+/** Age of the cached verdict, for the `ageMs` field callers see. */
+export function probeAgeMs(result, now) {
+    return Math.max(0, now - result.checkedAt);
+}
+export function _resetServingProbeForTest() {
+    cache = null;
+    inflight = null;
+}

package/docs/usage.md

@@ -121,3 +121,80 @@ const backends = await listBackends();
 ```bash
 curl http://localhost:3456/health
 ```
+
+dario has three health surfaces, and they answer different questions:
+
+| Endpoint | Answers | Costs |
+|---|---|---|
+| `GET /livez` | Is the HTTP server accepting connections? Always 200. | nothing |
+| `GET /health` | Do credentials and the pool look serviceable? 503 when not. | nothing |
+| `GET /health?probe=1` | Did a real request to Anthropic just succeed? | one tiny billed request per TTL |
+
+`/health` inspects state; it does not prove anything end-to-end. Adding
+`?probe=1` sends a real `max_tokens: 1` request upstream and folds the verdict
+into the response, so a proxy whose credentials look fine but whose requests all
+fail returns 503 instead of `ok`:
+
+```json
+{
+  "status": "degraded",
+  "oauth": "valid",
+  "probe": { "ok": false, "reason": "auth-rejected", "status": 401,
+             "latencyMs": 233, "ageMs": 4812, "model": "claude-haiku-4-5" },
+  "queue": { "active": 10, "queued": 4, "maxConcurrent": 10,
+             "stalledSince": 1754790000000, "stalledForMs": 28800000 }
+}
+```
+
+Notes that matter in production:
+
+- **The probe is opt-in and never runs on a plain `/health`.** Existing docker
+  healthchecks and uptime monitors keep costing nothing.
+- **Only trusted callers can trigger it** — and never a caller that arrived
+  through a Cloudflare tunnel, even an authenticated one. A `/health` reachable
+  from the internet is not a button for spending tokens, and the probe's own
+  cache means one request per TTL would be enough to keep it running.
+- **Results are cached and single-flighted** (`DARIO_PROBE_TTL_MS`, default
+  60000), so polling every second still costs at most one probe per minute.
+- **A rate-limited or overloaded upstream is not an outage.** 429 and 529 keep
+  `ok: true`; restarting dario cannot help either, and a watchdog that keys on
+  them just thrashes. Only auth rejection, 5xx, network failure and timeout set
+  `ok: false`.
+- **`queue.stalledForMs` is the slot-exhaustion signal**, not `active`/`queued`.
+  A busy proxy legitimately sits at its concurrency cap with a backlog; the
+  failure mode in dario#905 was slots that stopped turning over entirely. Any
+  release resets the stall clock, so sustained load never trips it.
+
+### Who sees what
+
+`/health` is auth-free by design — a docker healthcheck has to work before any
+key is configured. The response body is therefore split two ways, while the HTTP
+status (200/503) is identical for everyone, so uptime checks are unaffected:
+
+| Caller | Gets |
+|---|---|
+| Presented a configured `DARIO_API_KEY` | full detail |
+| Bare loopback (docker healthcheck, `dario doctor`) | full detail |
+| Arrived through a Cloudflare tunnel (`cf-ray`) | `{"status": "ok"}` only |
+| Anything else (LAN, another container, WAN) | `{"status": "ok"}` only |
+
+> **Changed in 5.5.1.** "Presented a configured key" previously read as "passed
+> the API-key check" — which every caller passes when **no** `DARIO_API_KEY` is
+> set. On an unkeyed proxy published through a tunnel, that disclosed the OAuth
+> countdown, request volume and refresh-failure count to anyone who asked. If
+> you monitor `/health` through a tunnel with no key configured, you now get the
+> liveness verdict only; set `DARIO_API_KEY` and send it, or query from
+> loopback, to keep the detail.
+
+A watchdog wants the probe; a container healthcheck usually does not:
+
+```bash
+# liveness — restart only if the process itself is gone
+curl -sf http://localhost:3456/livez
+
+# real serving check, e.g. every 5 minutes
+curl -sf 'http://localhost:3456/health?probe=1' >/dev/null || alert
+```
+
+Knobs: `DARIO_PROBE_MODEL` (default `claude-haiku-4-5`),
+`DARIO_PROBE_TTL_MS` (default `60000`), `DARIO_PROBE_TIMEOUT_MS` (default `15000`).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@askalf/dario",
-  "version": "5.4.31",
+  "version": "5.5.1",
   "description": "Use your Claude Pro/Max subscription in any tool — Cursor, Cline, Aider, the Agent SDK, your scripts — at subscription pricing, not per-token API bills. One local Anthropic + OpenAI-compatible endpoint.",
   "type": "module",
   "bin": {