ldrouter 1.17.6 → 1.17.8

package/CHANGELOG.md

@@ -4,6 +4,19 @@ All notable changes to this project are documented here. The format follows
 [Keep a Changelog](https://keepachangelog.com/) and the project adheres to
 [Semantic Versioning](https://semver.org/).
 
+## [1.17.8] - 2026-09-21
+
+### Changed
+
+- Raised provider connection, first-token, stream-idle, and total request timeout defaults to 15s, 60s, 300s, and 600s. Existing providers still using the old defaults are upgraded by migration 0009.
+- Distinguish total request timeout errors from stream-idle timeout errors.
+
+## [1.17.7] - 2026-09-20
+
+### Fixed
+
+- **Codex accounts were refreshed at the last minute, in the window where a rotating grant is most fragile.** The refresh lead is a threshold on the *remaining* lifetime, not a token lifetime — `needsCodexRefresh` fires once `expires - now` drops below it — and it sat at 5 minutes, so the exchange ran as the access token expired. Because OpenAI rotates the refresh token on every exchange, that is exactly when another client holding the same grant can invalidate it, and the observed fleet showed every account at `consecutive_failures = 0`: the first failure was already being treated as final. Codex access tokens last 10 days (measured from the fleet: a rotation at 13:39:00 expired at 13:39:00 ten days later), so the lead is now 5 days — half a lifetime of margin, refreshing every 5 days instead of every 10. This matches the 5-day `refreshLeadMs` 9router ships for Codex, and it surfaces a dead grant on day 5 rather than day 10. The lead is deliberately far below the lifetime: any value above it makes `needsCodexRefresh` permanently true and refreshes on *every* request, which multiplies refresh-token rotation by the request count.
+
 ## [1.17.6] - 2026-09-20
 
 ### Fixed

package/dist/server/db/migrate.js

@@ -148,10 +148,10 @@ function buildInitialSchemaSql() {
       custom_headers_encrypted TEXT,
       custom_headers_nonce TEXT,
       enabled INTEGER NOT NULL DEFAULT 1,
-      connect_timeout_ms INTEGER NOT NULL DEFAULT 10000,
-      first_token_timeout_ms INTEGER NOT NULL DEFAULT 30000,
-      stream_idle_timeout_ms INTEGER NOT NULL DEFAULT 60000,
-      total_timeout_ms INTEGER NOT NULL DEFAULT 180000,
+      connect_timeout_ms INTEGER NOT NULL DEFAULT 15000,
+      first_token_timeout_ms INTEGER NOT NULL DEFAULT 60000,
+      stream_idle_timeout_ms INTEGER NOT NULL DEFAULT 300000,
+      total_timeout_ms INTEGER NOT NULL DEFAULT 600000,
       max_retries INTEGER NOT NULL DEFAULT 2,
       retry_base_ms INTEGER NOT NULL DEFAULT 500,
       retry_max_ms INTEGER NOT NULL DEFAULT 8000,

package/dist/server/db/schema.js

@@ -89,10 +89,10 @@ export const providers = sqliteTable('providers', {
     customHeadersEncrypted: text('custom_headers_encrypted'),
     customHeadersNonce: text('custom_headers_nonce'),
     enabled: integer('enabled', { mode: 'boolean' }).notNull().notNull().default(true),
-    connectTimeoutMs: integer('connect_timeout_ms').notNull().default(10000),
-    firstTokenTimeoutMs: integer('first_token_timeout_ms').notNull().default(30000),
-    streamIdleTimeoutMs: integer('stream_idle_timeout_ms').notNull().default(60000),
-    totalTimeoutMs: integer('total_timeout_ms').notNull().default(180000),
+    connectTimeoutMs: integer('connect_timeout_ms').notNull().default(15000),
+    firstTokenTimeoutMs: integer('first_token_timeout_ms').notNull().default(60000),
+    streamIdleTimeoutMs: integer('stream_idle_timeout_ms').notNull().default(300000),
+    totalTimeoutMs: integer('total_timeout_ms').notNull().default(600000),
     maxRetries: integer('max_retries').notNull().default(2),
     retryBaseMs: integer('retry_base_ms').notNull().default(500),
     retryMaxMs: integer('retry_max_ms').notNull().default(8000),

package/dist/server/providers/codex-refresh.js

@@ -28,7 +28,12 @@ const PERMANENT_REFRESH_FAILURES = new Set(['oauth_refresh_failed', 'credential_
 export function isPermanentRefreshFailure(error) {
     return PERMANENT_REFRESH_FAILURES.has(error);
 }
-const REFRESH_LEAD_MS = 5 * 60 * 1000;
+// Lead is measured against the remaining lifetime, and must stay well under it: a refresh that fires
+// at the last minute races the expiry, and because OpenAI rotates the refresh token on every
+// exchange, that is exactly the moment another client holding the same grant can invalidate it.
+// Codex access tokens last 10 days, so a 5-day lead keeps half the lifetime as margin and refreshes
+// every 5 days. Matches the 5-day `refreshLeadMs` 9router ships for Codex.
+const REFRESH_LEAD_MS = 5 * 24 * 60 * 60 * 1000;
 const REFRESH_TIMEOUT_MS = 10_000;
 const flights = new Map();
 let refreshClient = defaultRefreshClient;

package/dist/server/routes/admin/providers.js

@@ -19,10 +19,10 @@ const ProviderCreate = z.object({
     apiKey: z.string().min(1).max(20000).optional(),
     customHeaders: z.record(z.string(), z.string()).optional(),
     enabled: z.boolean().optional(),
-    connectTimeoutMs: z.number().int().min(100).max(60000).optional(),
-    firstTokenTimeoutMs: z.number().int().min(100).max(300000).optional(),
-    streamIdleTimeoutMs: z.number().int().min(100).max(600000).optional(),
-    totalTimeoutMs: z.number().int().min(1000).max(600000).optional(),
+    connectTimeoutMs: z.number().int().min(100).max(120000).optional(),
+    firstTokenTimeoutMs: z.number().int().min(100).max(600000).optional(),
+    streamIdleTimeoutMs: z.number().int().min(100).max(900000).optional(),
+    totalTimeoutMs: z.number().int().min(1000).max(1800000).optional(),
     maxRetries: z.number().int().min(0).max(8).optional(),
     cbFailureThreshold: z.number().int().min(1).max(50).optional(),
     cbCooldownSeconds: z.number().int().min(1).max(3600).optional(),
@@ -106,10 +106,10 @@ export async function registerProviderRoutes(app) {
             customHeadersEncrypted: headersEnc?.ciphertext ?? null,
             customHeadersNonce: headersEnc?.nonce ?? null,
             enabled: body.enabled ?? true,
-            connectTimeoutMs: body.connectTimeoutMs ?? 10000,
-            firstTokenTimeoutMs: body.firstTokenTimeoutMs ?? 30000,
-            streamIdleTimeoutMs: body.streamIdleTimeoutMs ?? 60000,
-            totalTimeoutMs: body.totalTimeoutMs ?? 180000,
+            connectTimeoutMs: body.connectTimeoutMs ?? 15000,
+            firstTokenTimeoutMs: body.firstTokenTimeoutMs ?? 60000,
+            streamIdleTimeoutMs: body.streamIdleTimeoutMs ?? 300000,
+            totalTimeoutMs: body.totalTimeoutMs ?? 600000,
             maxRetries: body.maxRetries ?? 2,
             cbFailureThreshold: body.cbFailureThreshold ?? 5,
             cbCooldownSeconds: body.cbCooldownSeconds ?? 60,

package/dist/server/upstream/client.js

@@ -148,15 +148,18 @@ function logUpstreamResponse(requestId, status, statusText, headers, durationMs,
  */
 export async function callUpstreamStreaming(cfg, url, payload, onChunk, requestId = '-') {
     const ctl = new AbortController();
-    const totalTimer = setTimeout(() => ctl.abort(), cfg.totalTimeoutMs);
     const start = Date.now();
     let ttft = null;
     let firstTokenTimer = null;
     let idleTimer = null;
+    // Which watchdog aborted, so the error names the limit that was actually hit: the total
+    // timer used to surface as "stream idle timeout", which sent operators tuning the wrong knob.
+    let abortedBy = 'total';
+    const totalTimer = setTimeout(() => { abortedBy = 'total'; ctl.abort(); }, cfg.totalTimeoutMs);
     const resetIdle = () => {
         if (idleTimer)
             clearTimeout(idleTimer);
-        idleTimer = setTimeout(() => ctl.abort(), cfg.streamIdleTimeoutMs);
+        idleTimer = setTimeout(() => { abortedBy = 'idle'; ctl.abort(); }, cfg.streamIdleTimeoutMs);
     };
     try {
         const res = await fetch(url, {
@@ -172,7 +175,7 @@ export async function callUpstreamStreaming(cfg, url, payload, onChunk, requestI
         }
         logUpstreamResponse(requestId, res.status, res.statusText, res.headers, Date.now() - start, '');
         // First-token watchdog
-        firstTokenTimer = setTimeout(() => ctl.abort(), cfg.firstTokenTimeoutMs);
+        firstTokenTimer = setTimeout(() => { abortedBy = 'first_token'; ctl.abort(); }, cfg.firstTokenTimeoutMs);
         resetIdle();
         const reader = res.body.getReader();
         const decoder = new TextDecoder();
@@ -233,7 +236,10 @@ export async function callUpstreamStreaming(cfg, url, payload, onChunk, requestI
             ...formatError(e),
         ]);
         if (err.name === 'AbortError') {
-            if (ttft === null && firstTokenTimer) {
+            if (abortedBy === 'total') {
+                throw new GatewayError('timeout_error', 'Upstream total timeout', { status: 504, cause: e });
+            }
+            if (abortedBy === 'first_token' && ttft === null) {
                 throw new GatewayError('timeout_error', 'Upstream first token timeout', { status: 504, cause: e });
             }
             throw new GatewayError('timeout_error', 'Upstream stream idle timeout', { status: 504, cause: e });

package/migrations/0009_raise_timeout_defaults.sql

@@ -0,0 +1,15 @@
+-- 0009_raise_timeout_defaults.sql
+-- Raise the upstream timeout defaults. Long reasoning streams sit idle well past 60s between
+-- SSE frames, and the 180s total ceiling cut them off: the production request log showed 55
+-- `timeout_error` rows all ending at exactly 180s with a first token already delivered, i.e.
+-- the total timer firing while the stream was still alive.
+--
+-- New defaults: connect 15s, first token 60s, stream idle 300s, total 600s.
+-- Only rows still on the previous defaults are touched, so an admin-tuned value survives.
+-- The column DEFAULT in `providers` stays at the old values (SQLite cannot ALTER a default
+-- without a full table rebuild) — the application always writes the timeout columns
+-- explicitly, so the column default only applies to raw SQL inserts.
+UPDATE providers SET connect_timeout_ms = 15000 WHERE connect_timeout_ms = 10000;
+UPDATE providers SET first_token_timeout_ms = 60000 WHERE first_token_timeout_ms = 30000;
+UPDATE providers SET stream_idle_timeout_ms = 300000 WHERE stream_idle_timeout_ms = 60000;
+UPDATE providers SET total_timeout_ms = 600000 WHERE total_timeout_ms = 180000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ldrouter",
-  "version": "1.17.6",
+  "version": "1.17.8",
   "description": "LateDev Router — lightweight self-hosted LLM gateway with admin UI",
   "type": "module",
   "license": "MIT",