n9router 0.4.42 → 0.4.44

package/app/src/mitm/tokenPool.js

@@ -7,27 +7,39 @@
 const fs = require("fs");
 const path = require("path");
 const http = require("http");
+const crypto = require("crypto");
+const { machineIdSync } = require("node-machine-id");
 const { DATA_DIR } = require("./paths");
 const { log } = require("./logger");
 const { updateJsonFileSync } = require("../lib/dbFileSafety.js");
 
 const DB_FILE = path.join(DATA_DIR, "db.json");
+const MACHINE_ID_FILE = path.join(DATA_DIR, "machine-id");
+const AUTH_DIR = path.join(DATA_DIR, "auth");
+const CLI_SECRET_FILE = path.join(AUTH_DIR, "cli-secret");
+const CLI_TOKEN_HEADER = "x-9r-cli-token";
+const CLI_TOKEN_SALT = "9r-cli-auth";
 const TOKEN_EXPIRY_BUFFER_MS = 5 * 60 * 1000; // refresh 5min before expiry
 const ROUTER_PORT = process.env.PORT || 20128;
 const DEFAULT_COOLDOWN_MS = 2 * 60 * 1000;
-const DEFAULT_AUTH_COOLDOWN_MS = 10 * 60 * 1000;
+const DEFAULT_AUTH_COOLDOWN_MS = 2 * 60 * 1000;
 const DEFAULT_STRIKE_THRESHOLD = 3; // consecutive 429s before hard cooldown
 const CAPACITY_EXHAUSTED_COOLDOWN_MS = 60 * 1000;
 const SONNET_46_MODEL_KEY = "claude-sonnet-4-6";
+const REFRESH_RESPONSE_BODY_LIMIT = 500;
 // Must match DEFAULT_AG_503_RETRY_COUNT in open-sse/config/runtimeConfig.js
-const DEFAULT_503_RETRY_COUNT = 3;  // default 503 retries per account before switching
+const DEFAULT_503_RETRY_COUNT = 3; // default 503 retries per account before switching
 
 // ── In-memory state ──────────────────────────────────────────
-const cooldownMap = {};        // { [connectionId]: expiresTimestamp } quota/general cooldown
-const authCooldownMap = {};    // { [connectionId]: expiresTimestamp } invalid_token/auth cooldown
-const modelCooldownMap = {};   // { [connectionId]: { [model]: expiresTimestamp } }
-const strikeMap = {};          // { [connectionId]: consecutiveHitCount }
-const modelStrikeMap = {};     // { [connectionId]: { [model]: consecutiveHitCount } }
+const cooldownMap = {}; // { [connectionId]: expiresTimestamp } quota/general cooldown
+const authCooldownMap = {}; // { [connectionId]: expiresTimestamp } invalid_token/auth cooldown
+const modelCooldownMap = {}; // { [connectionId]: { [model]: expiresTimestamp } }
+const strikeMap = {}; // { [connectionId]: consecutiveHitCount }
+const modelStrikeMap = {}; // { [connectionId]: { [model]: consecutiveHitCount } }
+const refreshRequestMap = new Map(); // { [connectionId:mode]: Promise<refreshResult> }
+let cachedRawMachineId = null;
+let cachedCliSecret = null;
+let cachedCliToken = null;
 // ── Strike + cooldown management ─────────────────────────────
 // Upstream often returns false-positive 429s. Instead of locking
 // an account on the first hit, we count consecutive strikes.
@@ -38,7 +50,9 @@ function getStrikeThreshold() {
     if (!fs.existsSync(DB_FILE)) return DEFAULT_STRIKE_THRESHOLD;
     const db = JSON.parse(fs.readFileSync(DB_FILE, "utf-8"));
     return db.settings?.cooldownStrikeThreshold || DEFAULT_STRIKE_THRESHOLD;
-  } catch { return DEFAULT_STRIKE_THRESHOLD; }
+  } catch {
+    return DEFAULT_STRIKE_THRESHOLD;
+  }
 }
 
 /**
@@ -55,11 +69,15 @@ function recordStrike(connId, durationMs) {
     const ms = durationMs || DEFAULT_COOLDOWN_MS;
     cooldownMap[connId] = Date.now() + ms;
     delete strikeMap[connId];
-    log(`⏸ [token-pool] cooldown: ${connId.slice(0, 8)}… for ${Math.ceil(ms / 60000)}m (after ${count} strikes)`);
+    log(
+      `⏸ [token-pool] cooldown: ${connId.slice(0, 8)}… for ${Math.ceil(ms / 60000)}m (after ${count} strikes)`,
+    );
     return true;
   }
 
-  log(`⚡ [token-pool] strike ${count}/${threshold}: ${connId.slice(0, 8)}… (not locked yet)`);
+  log(
+    `⚡ [token-pool] strike ${count}/${threshold}: ${connId.slice(0, 8)}… (not locked yet)`,
+  );
   return false;
 }
 
@@ -71,13 +89,17 @@ function setCooldown(connId, durationMs) {
   const ms = durationMs || DEFAULT_COOLDOWN_MS;
   cooldownMap[connId] = Date.now() + ms;
   delete strikeMap[connId];
-  log(`⏸ [token-pool] cooldown: ${connId.slice(0, 8)}… for ${Math.ceil(ms / 60000)}m`);
+  log(
+    `⏸ [token-pool] cooldown: ${connId.slice(0, 8)}… for ${Math.ceil(ms / 60000)}m`,
+  );
 }
 
 function setAuthCooldown(connId, durationMs) {
   const ms = durationMs || DEFAULT_AUTH_COOLDOWN_MS;
   authCooldownMap[connId] = Date.now() + ms;
-  log(`🔒 [token-pool] auth-cooldown: ${connId.slice(0, 8)}… for ${Math.ceil(ms / 60000)}m`);
+  log(
+    `🔒 [token-pool] auth-cooldown: ${connId.slice(0, 8)}… for ${Math.ceil(ms / 60000)}m`,
+  );
 }
 
 function getMapExpiry(map, connId) {
@@ -123,11 +145,15 @@ function recordModelStrike(connId, model, durationMs) {
     if (!modelCooldownMap[connId]) modelCooldownMap[connId] = {};
     modelCooldownMap[connId][key] = Date.now() + ms;
     delete modelStrikeMap[connId][key];
-    log(`⏸ [token-pool] model-cooldown: ${connId.slice(0, 8)}… model="${key}" for ${Math.ceil(ms / 60000)}m (after ${count} strikes)`);
+    log(
+      `⏸ [token-pool] model-cooldown: ${connId.slice(0, 8)}… model="${key}" for ${Math.ceil(ms / 60000)}m (after ${count} strikes)`,
+    );
     return true;
   }
 
-  log(`⚡ [token-pool] model-strike ${count}/${threshold}: ${connId.slice(0, 8)}… model="${key}" (not locked yet)`);
+  log(
+    `⚡ [token-pool] model-strike ${count}/${threshold}: ${connId.slice(0, 8)}… model="${key}" (not locked yet)`,
+  );
   return false;
 }
 
@@ -142,7 +168,9 @@ function setModelCooldown(connId, model, durationMs) {
   if (!modelCooldownMap[connId]) modelCooldownMap[connId] = {};
   modelCooldownMap[connId][key] = Date.now() + ms;
   if (modelStrikeMap[connId]?.[key]) delete modelStrikeMap[connId][key];
-  log(`⏸ [token-pool] model-cooldown: ${connId.slice(0, 8)}… model="${key}" for ${Math.ceil(ms / 60000)}m`);
+  log(
+    `⏸ [token-pool] model-cooldown: ${connId.slice(0, 8)}… model="${key}" for ${Math.ceil(ms / 60000)}m`,
+  );
 }
 
 function isModelExhausted(connId, model) {
@@ -166,7 +194,9 @@ function isStoredModelQuotaExhausted(connection, model) {
   const modelStatus = status[model];
   if (!modelStatus || typeof modelStatus !== "object") return false;
 
-  return modelStatus.exhausted === true || modelStatus.remainingPercentage === 0;
+  return (
+    modelStatus.exhausted === true || modelStatus.remainingPercentage === 0
+  );
 }
 
 function isAntigravitySonnetZeroAutoDisableEnabled(db) {
@@ -178,7 +208,9 @@ function autoDisableAccountIfSonnetQuotaZero(connection, failure = {}) {
 
   try {
     const now = new Date().toISOString();
-    const statusLabel = failure.statusCode ? `status ${failure.statusCode}` : "request failure";
+    const statusLabel = failure.statusCode
+      ? `status ${failure.statusCode}`
+      : "request failure";
     const disabled = updateConnectionInDb(connection.id, (conn, db) => {
       if (!isAntigravitySonnetZeroAutoDisableEnabled(db)) return false;
       if (conn.isActive === false) return false;
@@ -193,7 +225,10 @@ function autoDisableAccountIfSonnetQuotaZero(connection, failure = {}) {
       });
     });
 
-    if (disabled) log(`⏸ [token-pool] auto-disabled: ${connection.id.slice(0, 8)}… Claude Sonnet 4.6 quota is 0%`);
+    if (disabled)
+      log(
+        `⏸ [token-pool] auto-disabled: ${connection.id.slice(0, 8)}… Claude Sonnet 4.6 quota is 0%`,
+      );
     return disabled;
   } catch (e) {
     log(`⚠️ [token-pool] auto-disable check failed: ${e.message}`);
@@ -208,7 +243,9 @@ function getTokenSwapStrategy() {
     if (!fs.existsSync(DB_FILE)) return "round-robin";
     const db = JSON.parse(fs.readFileSync(DB_FILE, "utf-8"));
     return db.settings?.tokenSwapStrategy || "round-robin";
-  } catch { return "round-robin"; }
+  } catch {
+    return "round-robin";
+  }
 }
 
 // ── Quota cooldown parser ────────────────────────────────────
@@ -221,9 +258,9 @@ function parseQuotaCooldown(errorBody) {
     const msg = json?.error?.message || json?.message || "";
     const reason = String(
       json?.error?.details?.[0]?.reason ||
-      json?.error?.reason ||
-      json?.reason ||
-      ""
+        json?.error?.reason ||
+        json?.reason ||
+        "",
     ).toUpperCase();
     const match = msg.match(/reset after (\d+h)?(\d+m)?(\d+s)?/i);
     if (match) {
@@ -236,12 +273,18 @@ function parseQuotaCooldown(errorBody) {
 
     if (
       reason === "MODEL_CAPACITY_EXHAUSTED" ||
-      /no capacity available for model|model capacity exhausted|capacity exhausted/i.test(msg)
+      /no capacity available for model|model capacity exhausted|capacity exhausted/i.test(
+        msg,
+      )
     ) {
       return CAPACITY_EXHAUSTED_COOLDOWN_MS;
     }
   } catch {
-    if (/no capacity available for model|model capacity exhausted|capacity exhausted/i.test(raw)) {
+    if (
+      /no capacity available for model|model capacity exhausted|capacity exhausted/i.test(
+        raw,
+      )
+    ) {
       return CAPACITY_EXHAUSTED_COOLDOWN_MS;
     }
   }
@@ -270,7 +313,9 @@ function getAntigravity503RetryCount(connectionOverride) {
     if (!fs.existsSync(DB_FILE)) return DEFAULT_503_RETRY_COUNT;
     const db = JSON.parse(fs.readFileSync(DB_FILE, "utf-8"));
     return db.settings?.antigravity503RetryCount ?? DEFAULT_503_RETRY_COUNT;
-  } catch { return DEFAULT_503_RETRY_COUNT; }
+  } catch {
+    return DEFAULT_503_RETRY_COUNT;
+  }
 }
 
 // ── Read connections from db.json (sync) ─────────────────────
@@ -283,16 +328,23 @@ function getActiveConnections(provider) {
     const now = Date.now();
 
     return connections
-      .filter(c =>
-        c.provider === provider &&
-        c.isActive !== false &&
-        c.accessToken &&
-        !isInCooldown(c.id) &&
-        // Skip expired tokens that have no refresh token
-        !(c.expiresAt && new Date(c.expiresAt).getTime() < now && !c.refreshToken)
+      .filter(
+        (c) =>
+          c.provider === provider &&
+          c.isActive !== false &&
+          c.accessToken &&
+          !isInCooldown(c.id) &&
+          // Skip expired tokens that have no refresh token
+          !(
+            c.expiresAt &&
+            new Date(c.expiresAt).getTime() < now &&
+            !c.refreshToken
+          ),
       )
       .sort((a, b) => (a.priority || 999) - (b.priority || 999));
-  } catch { return []; }
+  } catch {
+    return [];
+  }
 }
 
 function getTokenSwapAvailabilitySummary(provider, model) {
@@ -303,12 +355,14 @@ function getTokenSwapAvailabilitySummary(provider, model) {
         eligible: 0,
         skipped: 0,
         reasons: {},
-        summaryText: "0/0 account(s) eligible"
+        summaryText: "0/0 account(s) eligible",
       };
     }
 
     const db = JSON.parse(fs.readFileSync(DB_FILE, "utf-8"));
-    const connections = (db.providerConnections || []).filter(c => c.provider === provider);
+    const connections = (db.providerConnections || []).filter(
+      (c) => c.provider === provider,
+    );
     const now = Date.now();
     const strategy = getTokenSwapStrategy();
     const reasons = {
@@ -331,7 +385,11 @@ function getTokenSwapAvailabilitySummary(provider, model) {
         reasons.noToken += 1;
         continue;
       }
-      if (connection.expiresAt && new Date(connection.expiresAt).getTime() < now && !connection.refreshToken) {
+      if (
+        connection.expiresAt &&
+        new Date(connection.expiresAt).getTime() < now &&
+        !connection.refreshToken
+      ) {
         reasons.expiredNoRefresh += 1;
         continue;
       }
@@ -351,7 +409,11 @@ function getTokenSwapAvailabilitySummary(provider, model) {
         continue;
       }
 
-      if (strategy === "sticky" && model && isModelExhausted(connection.id, model)) {
+      if (
+        strategy === "sticky" &&
+        model &&
+        isModelExhausted(connection.id, model)
+      ) {
         reasons.modelCooldown += 1;
         continue;
       }
@@ -361,25 +423,36 @@ function getTokenSwapAvailabilitySummary(provider, model) {
 
     const skipped = connections.length - eligible;
     const reasonParts = [];
-    if (reasons.quotaCooldown) reasonParts.push(`${reasons.quotaCooldown} quota cooldown`);
-    if (reasons.authCooldown) reasonParts.push(`${reasons.authCooldown} auth cooldown`);
-    if (reasons.modelCooldown) reasonParts.push(`${reasons.modelCooldown} model cooldown`);
+    if (reasons.quotaCooldown)
+      reasonParts.push(`${reasons.quotaCooldown} quota cooldown`);
+    if (reasons.authCooldown)
+      reasonParts.push(`${reasons.authCooldown} auth cooldown`);
+    if (reasons.modelCooldown)
+      reasonParts.push(`${reasons.modelCooldown} model cooldown`);
     if (reasons.inactive) reasonParts.push(`${reasons.inactive} inactive`);
     if (reasons.noToken) reasonParts.push(`${reasons.noToken} missing token`);
-    if (reasons.expiredNoRefresh) reasonParts.push(`${reasons.expiredNoRefresh} expired-no-refresh`);
+    if (reasons.expiredNoRefresh)
+      reasonParts.push(`${reasons.expiredNoRefresh} expired-no-refresh`);
 
-    const summaryText = skipped > 0
-      ? `${eligible}/${connections.length} account(s) eligible, skipped ${skipped} (${reasonParts.join(", ")})`
-      : `${eligible}/${connections.length} account(s) eligible`;
+    const summaryText =
+      skipped > 0
+        ? `${eligible}/${connections.length} account(s) eligible, skipped ${skipped} (${reasonParts.join(", ")})`
+        : `${eligible}/${connections.length} account(s) eligible`;
 
-    return { total: connections.length, eligible, skipped, reasons, summaryText };
+    return {
+      total: connections.length,
+      eligible,
+      skipped,
+      reasons,
+      summaryText,
+    };
   } catch {
     return {
       total: 0,
       eligible: 0,
       skipped: 0,
       reasons: {},
-      summaryText: "0/0 account(s) eligible"
+      summaryText: "0/0 account(s) eligible",
     };
   }
 }
@@ -389,7 +462,7 @@ function getConnectionById(connId) {
     if (!fs.existsSync(DB_FILE)) return null;
     const db = JSON.parse(fs.readFileSync(DB_FILE, "utf-8"));
     const connections = db.providerConnections || [];
-    return connections.find(c => c.id === connId) || null;
+    return connections.find((c) => c.id === connId) || null;
   } catch {
     return null;
   }
@@ -435,7 +508,9 @@ function isTokenSwapEnabled() {
     if (!fs.existsSync(DB_FILE)) return false;
     const db = JSON.parse(fs.readFileSync(DB_FILE, "utf-8"));
     return !!db.settings?.tokenSwapEnabled;
-  } catch { return false; }
+  } catch {
+    return false;
+  }
 }
 
 function getNextConnection(provider) {
@@ -445,12 +520,18 @@ function getNextConnection(provider) {
   const selected = connections[0];
   const strategy = getTokenSwapStrategy();
   if (connections.length === 1) {
-    log(`🎯 [token-pool] selected: "${getConnectionLabel(selected)}" (only account)`);
+    log(
+      `🎯 [token-pool] selected: "${getConnectionLabel(selected)}" (only account)`,
+    );
     return selected;
   }
 
-  const lastUsedTag = selected.lastUsedAt ? ` lastUsed=${selected.lastUsedAt}` : " lastUsed=never";
-  log(`🎯 [token-pool] ${strategy}[next/${connections.length}]: "${getConnectionLabel(selected)}"${lastUsedTag}`);
+  const lastUsedTag = selected.lastUsedAt
+    ? ` lastUsed=${selected.lastUsedAt}`
+    : " lastUsed=never";
+  log(
+    `🎯 [token-pool] ${strategy}[next/${connections.length}]: "${getConnectionLabel(selected)}"${lastUsedTag}`,
+  );
   return selected;
 }
 
@@ -460,7 +541,7 @@ function getAllActiveConnections(provider, model) {
 
   const strategy = getTokenSwapStrategy();
   const hardAvailable = model
-    ? connections.filter(c => !isStoredModelQuotaExhausted(c, model))
+    ? connections.filter((c) => !isStoredModelQuotaExhausted(c, model))
     : connections;
 
   if (model && hardAvailable.length === 0) {
@@ -471,7 +552,7 @@ function getAllActiveConnections(provider, model) {
     // Sticky strategy: filter out accounts exhausted for this specific model,
     // then sort most-recently-used first so the same account sticks across requests.
     const available = model
-      ? hardAvailable.filter(c => !isModelExhausted(c.id, model))
+      ? hardAvailable.filter((c) => !isModelExhausted(c.id, model))
       : hardAvailable;
 
     // Runtime model cooldowns are temporary and may be false positives,
@@ -480,7 +561,8 @@ function getAllActiveConnections(provider, model) {
 
     // Most-recently-used first (sticky within session)
     return [...pool].sort((a, b) => {
-      if (!a.lastUsedAt && !b.lastUsedAt) return (a.priority || 999) - (b.priority || 999);
+      if (!a.lastUsedAt && !b.lastUsedAt)
+        return (a.priority || 999) - (b.priority || 999);
       if (!a.lastUsedAt) return 1;
       if (!b.lastUsedAt) return -1;
       return new Date(b.lastUsedAt) - new Date(a.lastUsedAt);
@@ -491,7 +573,8 @@ function getAllActiveConnections(provider, model) {
   // update lastUsedAt, so the next selection naturally advances without
   // extra streak state or in-memory round-robin indexes.
   return [...hardAvailable].sort((a, b) => {
-    if (!a.lastUsedAt && !b.lastUsedAt) return (a.priority || 999) - (b.priority || 999);
+    if (!a.lastUsedAt && !b.lastUsedAt)
+      return (a.priority || 999) - (b.priority || 999);
     if (!a.lastUsedAt) return -1;
     if (!b.lastUsedAt) return 1;
     return new Date(a.lastUsedAt) - new Date(b.lastUsedAt);
@@ -505,7 +588,7 @@ function getAllActiveConnections(provider, model) {
 function updateConnectionInDb(connId, patchFn) {
   const result = updateJsonFileSync(DB_FILE, (db) => {
     const connections = db.providerConnections || [];
-    const index = connections.findIndex(c => c.id === connId);
+    const index = connections.findIndex((c) => c.id === connId);
     if (index === -1) return false;
     return patchFn(connections[index], db);
   });
@@ -526,84 +609,251 @@ function markAccountUsed(connId) {
   }
 }
 
+function truncateRefreshBody(body) {
+  if (!body) return "";
+  return body.length > REFRESH_RESPONSE_BODY_LIMIT
+    ? `${body.slice(0, REFRESH_RESPONSE_BODY_LIMIT)}...`
+    : body;
+}
+
+function getRefreshPayloadError(payload) {
+  if (!payload) return null;
+  if (typeof payload.error === "string") return payload.error;
+  if (payload.error?.message) return payload.error.message;
+  if (typeof payload.error_description === "string")
+    return payload.error_description;
+  if (typeof payload.message === "string") return payload.message;
+  return null;
+}
+
+function loadRawMachineId() {
+  if (cachedRawMachineId) return cachedRawMachineId;
+  try {
+    cachedRawMachineId = fs.readFileSync(MACHINE_ID_FILE, "utf8").trim();
+    if (cachedRawMachineId) return cachedRawMachineId;
+  } catch {}
+
+  try {
+    cachedRawMachineId = machineIdSync();
+  } catch {
+    cachedRawMachineId = crypto.randomUUID();
+  }
+
+  try {
+    fs.mkdirSync(DATA_DIR, { recursive: true });
+    fs.writeFileSync(MACHINE_ID_FILE, cachedRawMachineId, { mode: 0o600 });
+  } catch {}
+
+  return cachedRawMachineId;
+}
+
+function loadCliSecret() {
+  if (cachedCliSecret) return cachedCliSecret;
+  try {
+    cachedCliSecret = fs.readFileSync(CLI_SECRET_FILE, "utf8").trim();
+    if (cachedCliSecret) return cachedCliSecret;
+  } catch {}
+
+  cachedCliSecret = crypto.randomBytes(32).toString("hex");
+  try {
+    fs.mkdirSync(AUTH_DIR, { recursive: true });
+    fs.writeFileSync(CLI_SECRET_FILE, cachedCliSecret, { mode: 0o600 });
+  } catch {}
+
+  return cachedCliSecret;
+}
+
+function getInternalCliToken() {
+  if (!cachedCliToken) {
+    const raw = loadRawMachineId();
+    const secret = loadCliSecret();
+    cachedCliToken = crypto
+      .createHash("sha256")
+      .update(raw + CLI_TOKEN_SALT + secret)
+      .digest("hex")
+      .substring(0, 16);
+  }
+  return cachedCliToken;
+}
+
+function buildConnectionTestHeaders(payload) {
+  const headers = { [CLI_TOKEN_HEADER]: getInternalCliToken() };
+  if (payload) {
+    headers["Content-Type"] = "application/json";
+    headers["Content-Length"] = Buffer.byteLength(payload);
+  }
+  return headers;
+}
+
 // ── Token refresh trigger (refresh + reload persisted token) ─
 // Calls 9Router's existing POST /api/providers/:id/test which
 // checks expiry, refreshes token automatically, then returns the
 // latest persisted connection snapshot for the current request.
 
-async function runConnectionTest(connection, options = {}) {
+async function runConnectionTestRequest(connection, options = {}) {
   return await new Promise((resolve) => {
     if (!connection?.id) {
-      resolve({ connection, refreshed: false, valid: false });
+      resolve({
+        connection,
+        refreshed: false,
+        valid: false,
+        error: "Missing connection id",
+        errorType: "missing_connection_id",
+      });
       return;
     }
 
     let responseBody = "";
-    const payload = options.forceRefresh ? JSON.stringify({ forceRefresh: true }) : "";
-    const headers = payload
-      ? { "Content-Type": "application/json", "Content-Length": Buffer.byteLength(payload) }
-      : undefined;
+    const payload = options.forceRefresh
+      ? JSON.stringify({ forceRefresh: true })
+      : "";
+    const headers = buildConnectionTestHeaders(payload);
 
     try {
       const req = http.request(
-        { hostname: "127.0.0.1", port: ROUTER_PORT, path: `/api/providers/${connection.id}/test`, method: "POST", headers },
+        {
+          hostname: "127.0.0.1",
+          port: ROUTER_PORT,
+          path: `/api/providers/${connection.id}/test`,
+          method: "POST",
+          headers,
+        },
         (res) => {
           res.on("data", (chunk) => {
             responseBody += chunk.toString();
           });
           res.on("end", () => {
             let payload = {};
+            let parseError = null;
             try {
               payload = JSON.parse(responseBody || "{}");
-            } catch {
+            } catch (e) {
+              parseError = e;
               payload = {};
             }
 
             const refreshedConnection = getConnectionById(connection.id);
-            const tokenChanged = !!(refreshedConnection?.accessToken && refreshedConnection.accessToken !== connection.accessToken);
+            const tokenChanged = !!(
+              refreshedConnection?.accessToken &&
+              refreshedConnection.accessToken !== connection.accessToken
+            );
+            const payloadError = getRefreshPayloadError(payload);
             resolve({
               connection: refreshedConnection || connection,
               refreshed: !!payload.refreshed || tokenChanged,
               valid: !!payload.valid,
+              statusCode: res.statusCode || 0,
+              error:
+                payloadError ||
+                (parseError
+                  ? `Invalid JSON response: ${parseError.message}`
+                  : null),
+              errorType: payloadError
+                ? "api_error"
+                : parseError
+                  ? "parse_error"
+                  : null,
+              responseBody:
+                payloadError || parseError
+                  ? truncateRefreshBody(responseBody)
+                  : undefined,
             });
           });
-        }
+        },
+      );
+      req.on("error", (e) =>
+        resolve({
+          connection,
+          refreshed: false,
+          valid: false,
+          error: e.message,
+          errorType: "request_error",
+        }),
       );
-      req.on("error", () => resolve({ connection, refreshed: false, valid: false }));
       if (payload) req.write(payload);
       req.end();
-    } catch {
-      resolve({ connection, refreshed: false, valid: false });
+    } catch (e) {
+      resolve({
+        connection,
+        refreshed: false,
+        valid: false,
+        error: e.message,
+        errorType: "request_exception",
+      });
     }
   });
 }
 
+async function runConnectionTest(connection, options = {}) {
+  if (!connection?.id) {
+    return runConnectionTestRequest(connection, options);
+  }
+
+  const mode = options.forceRefresh ? "force" : "expiry";
+  const key = `${connection.id}:${mode}`;
+  const existing = refreshRequestMap.get(key);
+  if (existing) {
+    log(
+      `⏳ [token-pool] waiting for in-flight ${mode} refresh → ${getConnectionLabel(connection).slice(0, 20)}`,
+    );
+    return await existing;
+  }
+
+  const promise = runConnectionTestRequest(connection, options).finally(() => {
+    refreshRequestMap.delete(key);
+  });
+  refreshRequestMap.set(key, promise);
+  return await promise;
+}
+
 async function triggerRefreshIfNeeded(connection) {
   if (!connection?.refreshToken) return connection;
 
   const hasExpiry = !!connection?.expiresAt;
   const expiresAt = hasExpiry ? new Date(connection.expiresAt).getTime() : 0;
-  const isExpired = !hasExpiry || (Date.now() + TOKEN_EXPIRY_BUFFER_MS >= expiresAt);
+  const isExpired =
+    !hasExpiry || Date.now() + TOKEN_EXPIRY_BUFFER_MS >= expiresAt;
 
   if (!isExpired) return connection;
 
-  log(`🔄 [token-pool] ${!hasExpiry ? "missing-expiry" : "near-expiry"} refresh → ${getConnectionLabel(connection).slice(0, 20)}`);
+  log(
+    `🔄 [token-pool] ${!hasExpiry ? "missing-expiry" : "near-expiry"} refresh → ${getConnectionLabel(connection).slice(0, 20)}`,
+  );
   const result = await runConnectionTest(connection);
   if (result.refreshed) {
-    log(`♻️ [token-pool] refreshed token applied → ${getConnectionLabel(connection).slice(0, 20)}`);
+    log(
+      `♻️ [token-pool] refreshed token applied → ${getConnectionLabel(connection).slice(0, 20)}`,
+    );
   }
   return result.connection || connection;
 }
 
 async function forceRefreshConnection(connection) {
   if (!connection?.refreshToken) {
-    return { connection, refreshed: false, valid: false };
+    return {
+      connection,
+      refreshed: false,
+      valid: false,
+      error: "No refresh token",
+      errorType: "missing_refresh_token",
+    };
   }
 
-  log(`🔄 [token-pool] auth refresh → ${getConnectionLabel(connection).slice(0, 20)}`);
+  log(
+    `🔄 [token-pool] auth refresh → ${getConnectionLabel(connection).slice(0, 20)}`,
+  );
   const result = await runConnectionTest(connection, { forceRefresh: true });
   if (result.refreshed) {
-    log(`♻️ [token-pool] refreshed token applied → ${getConnectionLabel(connection).slice(0, 20)}`);
+    log(
+      `♻️ [token-pool] refreshed token applied → ${getConnectionLabel(connection).slice(0, 20)}`,
+    );
+  } else {
+    const statusTag = result.statusCode ? ` status=${result.statusCode}` : "";
+    const errorTag = result.error ? ` error="${result.error}"` : "";
+    const typeTag = result.errorType ? ` type=${result.errorType}` : "";
+    log(
+      `⚠️ [token-pool] auth refresh failed → ${getConnectionLabel(connection).slice(0, 20)}${statusTag}${typeTag}${errorTag}`,
+    );
   }
   return result;
 }