ldrouter 1.17.8 → 1.17.9

package/CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project are documented here. The format follows
 [Keep a Changelog](https://keepachangelog.com/) and the project adheres to
 [Semantic Versioning](https://semver.org/).
 
+## [1.17.9] - 2026-09-22
+
+### Fixed
+
+- **A tripped provider circuit breaker locked the provider out until the process restarted.** The breaker's `open -> half_open` decay runs inside `getEffectiveState()`, which is only called from the attempt loop, but both candidate filters read the raw `isOpen()` flag instead — so once the flag was set no candidate ever reached the attempt loop that would have decayed it, and the provider answered `provider circuit is open` to every request until the container was restarted. `health_state` kept reading `healthy` throughout, so nothing in the UI showed the lockout. Live production: `code` and `qo` had each accumulated five consecutive upstream failures (a real upstream `503`) and were both in that state, while the upstream itself answered `200` to the same model and key when called directly. Candidate construction is now cooldown-aware through `circuitBlocks()`, so a candidate is admitted again as soon as the cooldown has elapsed and the half-open probe can close the breaker. Verified on a container against the released image: breaker open with the cooldown elapsed returned a zero-length `200`; with the fix it streams normally again.
+- **The admin "test model" button reported `Stream ended unexpectedly` instead of the actual reason.** The `test-stream` route hijacks the reply, so a `GatewayError` rethrown from its catch block could never reach Fastify's error handler — the response simply ended as `HTTP 200` with a zero-length body and no events, leaving the UI nothing to parse but that placeholder. The route now sends the reason as a `test_error` event. Measured on the released build with the breaker open: status `200`, `content-type` null, `body` 0 bytes; with the fix: `text/event-stream` carrying the real message.
+
 ## [1.17.8] - 2026-09-21
 
 ### Changed

package/dist/server/gateway/runner.js

@@ -11,7 +11,7 @@ import { qoderAttemptFailure, qoderCredentialsFor } from '../providers/qoder/cre
 import { refreshQoderCreditsIfStale } from '../providers/qoder/credits.js';
 import { callQoderNonStreaming, callQoderStreaming } from '../providers/qoder/client.js';
 import { refreshCodexUsageIfStale } from '../providers/codex-autostart.js';
-import { getEffectiveState, isOpen, recordSuccess, recordFailure, halfOpenProbeAllowed } from '../routing/circuit.js';
+import { getEffectiveState, recordSuccess, recordFailure, halfOpenProbeAllowed, circuitBlocks } from '../routing/circuit.js';
 import { checkRpm, checkTpm, acquireConcurrent, releaseConcurrent } from '../routing/ratelimit.js';
 import { checkDailyMonthly, consumeUsage } from '../routing/quota.js';
 import { keyAllowedFor } from '../auth/api-key.js';
@@ -436,7 +436,10 @@ export class GatewayRunner {
             enabled: m.enabled,
             providerEnabled: p.enabled,
             upstreamAvailable: m.upstreamAvailable,
-            circuitOpen: isOpen(m.providerId),
+            // Cooldown-aware, not the raw flag: `isOpen()` stays true until some request decays it,
+            // so a raw read here refused every candidate forever and the provider was unroutable
+            // until a restart.
+            circuitOpen: circuitBlocks(m.providerId, p.cbCooldownSeconds),
             capabilities: caps,
             providerType: p.type,
         };
@@ -479,6 +482,7 @@ export class GatewayRunner {
         const providers = db.select().from(schema.providers).all();
         const providerEnabled = new Map(providers.map((p) => [p.id, p.enabled]));
         const providerType = new Map(providers.map((p) => [p.id, p.type]));
+        const providerCooldown = new Map(providers.map((p) => [p.id, p.cbCooldownSeconds]));
         // Deliberately unfiltered: every combo member must reach selectCandidates so
         // it can report WHY it was skipped. Pre-filtering here erased the model rows
         // and turned every distinct reason into "model not found".
@@ -489,7 +493,10 @@ export class GatewayRunner {
             enabled: m.enabled,
             providerEnabled: providerEnabled.get(m.providerId),
             upstreamAvailable: m.upstreamAvailable,
-            circuitOpen: isOpen(m.providerId),
+            // Cooldown-aware: a raw `isOpen()` here froze the whole combo on one bad minute,
+            // because the flags are computed once per request while the breaker stays open
+            // until some request decays it — which this filter prevented from ever happening.
+            circuitOpen: circuitBlocks(m.providerId, providerCooldown.get(m.providerId) ?? 0),
             capabilities: safeJson(m.capabilitiesJson),
             providerType: providerType.get(m.providerId),
         }));

package/dist/server/routes/admin/models.js

@@ -312,8 +312,9 @@ export async function registerModelRoutes(app) {
         catch (e) {
             // Never let a raw non-GatewayError (e.g. MasterKeyError from credential
             // decryption) escape into the global handler as an opaque "Gateway error".
-            if (e instanceof GatewayError)
-                throw e;
+            // The reply is hijacked, so a rethrow here cannot reach the error handler:
+            // it ends the response having sent nothing, which the UI can only report as
+            // "Stream ended unexpectedly". Report the reason over the stream instead.
             send('test_error', { message: e.message });
         }
         finally {

package/dist/server/routing/circuit.js

@@ -35,3 +35,16 @@ export function getEffectiveState(providerId, cooldownSeconds) {
 export function halfOpenProbeAllowed(providerId) {
     return circuitState(providerId) === 'half_open';
 }
+/**
+ * Should the candidate filter refuse this provider outright?
+ *
+ * `isOpen()` reports the raw state, which stays `'open'` until a request walks the
+ * attempt loop — so a tripped breaker rejected every candidate for the lifetime of the
+ * process and made the provider unroutable until a restart (the combo path and the
+ * direct-model path both read the flag off the candidate record). A candidate must be
+ * allowed through once the cooldown has elapsed so the probe that closes the breaker
+ * can actually run; `getEffectiveState` is what performs that open -> half_open decay.
+ */
+export function circuitBlocks(providerId, cooldownSeconds) {
+    return getEffectiveState(providerId, cooldownSeconds) === 'open';
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ldrouter",
-  "version": "1.17.8",
+  "version": "1.17.9",
   "description": "LateDev Router — lightweight self-hosted LLM gateway with admin UI",
   "type": "module",
   "license": "MIT",