@drakon-systems/multi-clawd 2.4.0 → 2.5.1

package/dist/health.js

@@ -1,4 +1,5 @@
 import { modelWindowApplies, modelWindowKey } from "./shim-core.js";
+import { spreadBefore } from "./spread.js";
 export const DEFAULT_UTILIZATION_THRESHOLD = 0.85;
 const DEFAULT_STALE_AFTER_MS = 6 * 60 * 60 * 1000;
 export const MODEL_REJECTED_TTL_MS = 60 * 60 * 1000;
@@ -19,6 +20,9 @@ export function isPeriodWindow(window) {
 export function isOverageWindow(window) {
     return /overage/i.test(window);
 }
+export function logicalWindowKey(window) {
+    return window.startsWith("usage:") ? window.slice("usage:".length) : window;
+}
 export function isLongQuotaWindow(window) {
     return /(^|_)(days?|weeks?|months?)(_|$)/.test(window) && !isOverageWindow(window);
 }
@@ -89,6 +93,7 @@ export function classifyAccountHealth(state, options, nowMs, requestedModel) {
     let hasLiveEvidence = false;
     let overageUtilization;
     let quotaUtilization;
+    const freshestByWindow = new Map();
     for (const [window, w] of Object.entries(state.windows)) {
         const resetMs = typeof w.resetsAt === "number" ? w.resetsAt * 1000 : undefined;
         const resetBearing = resetMs !== undefined && resetMs > nowMs;
@@ -175,6 +180,14 @@ export function classifyAccountHealth(state, options, nowMs, requestedModel) {
         if (!isOverageWindow(window) && typeof w.utilization === "number") {
             quotaUtilization = Math.max(quotaUtilization ?? 0, w.utilization);
         }
+        if (typeof w.utilization === "number" &&
+            (resetBearing || resetMs === undefined) &&
+            (!isOverageWindow(window) || options.rotateOnOverage)) {
+            const logical = logicalWindowKey(window);
+            const seen = freshestByWindow.get(logical);
+            if (!seen || w.seenAt > seen.seenAt)
+                freshestByWindow.set(logical, { utilization: w.utilization, seenAt: w.seenAt });
+        }
         if (worst.verdict === "ok" &&
             typeof w.utilization !== "number" &&
             isShortWindow(window) &&
@@ -188,6 +201,12 @@ export function classifyAccountHealth(state, options, nowMs, requestedModel) {
     }
     if (worst.verdict === "ok" && !hasLiveEvidence)
         return { verdict: "no_data" };
+    let utilization;
+    for (const { utilization: u } of freshestByWindow.values()) {
+        utilization = utilization === undefined ? u : Math.max(utilization, u);
+    }
+    if (utilization !== undefined)
+        worst = { ...worst, utilization };
     if (worst.verdict === "ok" && overageUtilization !== undefined) {
         const pct = (n) => `${Math.round(n * 100)}%`;
         worst = {
@@ -223,7 +242,7 @@ export function summarizeWindowUsage(state, options, nowMs) {
     return usage.sort((a, b) => (b.resetsAt ?? 0) - (a.resetsAt ?? 0));
 }
 export function poolStrategy(value) {
-    return value === "reset-aware" ? "reset-aware" : "ordered";
+    return value === "reset-aware" || value === "spread" ? value : "ordered";
 }
 export function sevenDayReset(state, options, nowMs) {
     const usage = summarizeWindowUsage(state, options, nowMs);
@@ -243,10 +262,38 @@ export function choosePoolAccount(pool, strategy = "ordered") {
         if (best?.sevenDayResetsAt !== undefined)
             return best.id;
     }
+    if (strategy === "spread") {
+        let best;
+        for (const member of pool) {
+            if (member.verdict !== "ok" && member.verdict !== "no_data")
+                continue;
+            if (!best || spreadBefore(member, best))
+                best = member;
+        }
+        if (best)
+            return best.id;
+    }
     const usable = pool.find((a) => a.verdict === "ok" || a.verdict === "no_data");
     if (usable)
         return usable.id;
-    return pool.find((a) => a.verdict === "near_limit")?.id;
+    return nearLimitWithMostHeadroom(pool)?.id;
+}
+export function headroomBefore(a, b) {
+    if (a.utilization === undefined)
+        return false;
+    if (b.utilization === undefined)
+        return true;
+    return a.utilization < b.utilization;
+}
+export function nearLimitWithMostHeadroom(pool) {
+    let best;
+    for (const member of pool) {
+        if (member.verdict !== "near_limit")
+            continue;
+        if (!best || headroomBefore(member, best))
+            best = member;
+    }
+    return best;
 }
 export function allCredentialFailed(pool) {
     return pool.length > 0 && pool.every((a) => a.verdict === "credential_failed");

package/dist/history.js

@@ -0,0 +1,103 @@
+import { randomUUID } from "node:crypto";
+import { mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { acquireShimLock } from "./shim-lock.js";
+export const HISTORY_FILE = "history.jsonl";
+export const HISTORY_MAX_BYTES = 64 * 1024;
+export const HISTORY_MAX_EVENTS = 500;
+export function historyFile(stateFile) {
+    return join(dirname(stateFile), HISTORY_FILE);
+}
+const safeAccount = (value) => typeof value === "string" &&
+    value.length > 0 && value.length <= 100 && !/[\x00-\x1f\x7f]/.test(value);
+const safeModel = (value) => typeof value === "string" &&
+    value.length > 0 && value.length <= 100 && /^[a-zA-Z0-9][a-zA-Z0-9._:/-]*$/.test(value);
+function valid(raw) {
+    if (!raw || typeof raw !== "object")
+        return undefined;
+    const e = raw;
+    if (typeof e.at !== "string" || !Number.isFinite(Date.parse(e.at)) ||
+        !["served", "retry", "rotation"].includes(String(e.kind)) || !safeAccount(e.accountId) ||
+        (e.toAccountId !== undefined && !safeAccount(e.toAccountId)) ||
+        (e.reason !== undefined && !["model_limit", "auth_failure", "transient", "pool_selection"].includes(String(e.reason))))
+        return undefined;
+    return { at: e.at, kind: e.kind, accountId: e.accountId,
+        ...(e.toAccountId ? { toAccountId: e.toAccountId } : {}),
+        ...(safeModel(e.modelId) ? { modelId: e.modelId } : {}),
+        ...(e.reason ? { reason: e.reason } : {}) };
+}
+export function readHistory(file) {
+    let raw;
+    try {
+        raw = readFileSync(file, "utf8");
+    }
+    catch {
+        return [];
+    }
+    return raw.split("\n").flatMap((line) => {
+        try {
+            const event = valid(JSON.parse(line));
+            return event ? [event] : [];
+        }
+        catch {
+            return [];
+        }
+    });
+}
+export function recordHistory(stateFile, event) {
+    if (!stateFile)
+        return;
+    let release;
+    let tmp;
+    try {
+        const checked = valid(event);
+        if (!checked)
+            return;
+        const file = historyFile(stateFile);
+        release = acquireShimLock(file);
+        if (!release)
+            return;
+        const events = [...readHistory(file), checked].slice(-HISTORY_MAX_EVENTS);
+        let lines = events.map((e) => JSON.stringify(e) + "\n");
+        let bytes = lines.reduce((sum, line) => sum + Buffer.byteLength(line), 0);
+        while (bytes > HISTORY_MAX_BYTES && lines.length)
+            bytes -= Buffer.byteLength(lines.shift());
+        mkdirSync(dirname(file), { recursive: true });
+        tmp = `${file}.tmp-${process.pid}-${randomUUID()}`;
+        writeFileSync(tmp, lines.join(""), { mode: 0o600 });
+        renameSync(tmp, file);
+    }
+    catch { }
+    finally {
+        if (tmp)
+            try {
+                unlinkSync(tmp);
+            }
+            catch { }
+        release?.();
+    }
+}
+export function parseHistorySince(value) {
+    const match = /^(\d+)([dh])$/.exec(value);
+    if (!match || Number(match[1]) < 1 || Number(match[1]) > 3650) {
+        throw new Error("--since must be a positive duration such as 7d or 24h");
+    }
+    return Number(match[1]) * (match[2] === "d" ? 86_400_000 : 3_600_000);
+}
+export function renderHistory(events, accountIds, since, nowMs = Date.now()) {
+    const cutoff = nowMs - parseHistorySince(since);
+    const recent = events.filter((e) => Date.parse(e.at) >= cutoff && Date.parse(e.at) <= nowMs);
+    const ids = [...new Set([...accountIds, ...recent.map((e) => e.accountId)])];
+    const lines = [`Account history (last ${since}):`];
+    if (!ids.length)
+        return [...lines, "  No accounts configured."].join("\n");
+    for (const id of ids) {
+        const own = recent.filter((e) => e.accountId === id);
+        const count = (kind) => own.filter((e) => e.kind === kind).length;
+        const last = own.filter((e) => e.kind === "served")
+            .reduce((latest, e) => !latest || Date.parse(e.at) > Date.parse(latest) ? e.at : latest, undefined);
+        const n = (k, one, many) => `${k} ${k === 1 ? one : many}`;
+        lines.push(`  ${id}: ${n(count("served"), "served launch", "served launches")}, ${n(count("retry"), "retry", "retries")}, ${n(count("rotation"), "rotation", "rotations")}; last served ${last ?? "never in this window"}`);
+    }
+    return lines.join("\n");
+}

package/dist/index.js

@@ -7,8 +7,10 @@ import { MODEL_ALIASES, buildCatalogEntries, canonicalModelId, isModernClaudeMod
 import { decideDegradation, matchesPin } from "./degrade.js";
 import { resolveExecMode, permissionModeArgs } from "./exec-policy.js";
 import { resolveBaseModelIds } from "./catalog-source.js";
-import { allCredentialFailed, classifyAccountHealth, credentialFailureFor, lastAuthSuccessAt, overdueProbeAccount, sevenDayReset, } from "./health.js";
+import { allCredentialFailed, classifyAccountHealth, credentialFailureFor, lastAuthSuccessAt, overdueProbeAccount, poolStrategy, sevenDayReset, } from "./health.js";
 import { decideStickySelection } from "./sticky.js";
+import { HISTORY_FILE, readHistory, recordHistory } from "./history.js";
+import { spreadStatsFromHistory, spreadWindowMs, withSpreadStats } from "./spread.js";
 import { DEFAULT_SESSION_AFFINITY_TTL_MS, launchSessionId, readSessionPin, SESSION_AFFINITY_ENV, } from "./session-affinity.js";
 import { mergeHealthStates, parseStoredState, recordCredentialFailure, } from "./shim-core.js";
 import { createTokenRefResolver, isSecretRefShape, } from "./token-resolution.js";
@@ -18,6 +20,7 @@ import { RETRY_ROSTER_ENV } from "./retry-plan.js";
 import { SESSION_DIRS_ENV } from "./session-handover.js";
 import { addAlert, alertKeysWithPrefix, clearAlert, pendingAlertText, } from "./alerts.js";
 import { healthStateFile, usageStateFile, clearAccountCredentialFailure } from "./credential-state.js";
+import { persistHealthState, readHealthStateFile } from "./health-journal.js";
 import { createUsagePollController, effectiveUsagePollInterval, effectiveUsageWarnThreshold, usageAlertPrefix, usagePoolAlertKey, } from "./usage-poll.js";
 export { healthStateFile, usageStateFile, clearAccountCredentialFailure };
 import { accountConfigDir, AccountCredentialError, buildAccountChildEnv, credentialFailureText, parseSetupTokenFile, tokenFileModeWarning, validateAccountTokenSources, } from "./account-env.js";
@@ -188,22 +191,8 @@ const refProbeTrackers = new Map();
 const lastProbeStatus = new Map();
 function recordAccountCredentialFailure(accountId, reason, nowMs) {
     const file = healthStateFile(accountId);
-    let state;
-    try {
-        state = parseStoredState(readFileSync(file, "utf8")) ?? { accountId, windows: {} };
-    }
-    catch {
-        state = { accountId, windows: {} };
-    }
-    const next = mergeHealthStates(state, recordCredentialFailure(state, reason, nowMs), nowMs);
-    try {
-        mkdirSync(dirname(file), { recursive: true });
-        const tmp = `${file}.tmp-${process.pid}`;
-        writeFileSync(tmp, JSON.stringify(next, null, 2), { mode: 0o600 });
-        renameSync(tmp, file);
-    }
-    catch {
-    }
+    const state = readHealthStateFile(file) ?? { accountId, windows: {} };
+    persistHealthState(file, recordCredentialFailure(state, reason, nowMs));
 }
 export async function runLoginHealthProbe(accounts, logger, deps = {}) {
     const now = deps.nowMs ?? Date.now();
@@ -740,7 +729,7 @@ export default definePluginEntry({
     },
 });
 function readHealthState(accountId) {
-    return mergeStoredHealth(readStateFileLenient(healthStateFile(accountId)), readStateFileLenient(usageStateFile(accountId)));
+    return mergeStoredHealth(readHealthStateFile(healthStateFile(accountId)), readStateFileLenient(usageStateFile(accountId)));
 }
 function readStateFileLenient(file) {
     try {
@@ -757,6 +746,9 @@ function mergeStoredHealth(shim, usage) {
         return shim;
     return mergeHealthStates(shim, usage);
 }
+function poolHistoryFile() {
+    return join(homedir(), ".openclaw", "state", "multi-clawd", HISTORY_FILE);
+}
 function readStickyEntry(file) {
     try {
         const parsed = JSON.parse(readFileSync(file, "utf8"));
@@ -824,6 +816,11 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
         defaultModel: pool.defaultModel,
     };
     const minDwellMs = pool.minDwellMs;
+    const strategy = poolStrategy(pool.strategy);
+    const affinityOn = pool.sessionAffinity === true || strategy === "spread";
+    if (strategy === "spread" && pool.sessionAffinity === false) {
+        logger.warn(`[multi-clawd] pool "${poolId}": strategy "spread" keeps each session on the account that first served it — sessionAffinity: false is ignored`);
+    }
     const stickyFile = join(homedir(), ".openclaw", "state", "multi-clawd", `pool-${poolId}.sticky.json`);
     const backend = buildBackend(poolAccount, execMode, deps?.launch);
     const { launch, transport } = launchTransports.get(backend);
@@ -849,20 +846,24 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
             throw new Error(`[multi-clawd] ${text}`);
         }
         const previousSticky = readStickyEntry(stickyFile);
-        const poolVerdicts = verdicts.map((v) => ({
+        const rawVerdicts = verdicts.map((v) => ({
             id: v.id,
             verdict: v.health.verdict,
             observedAt: v.health.observedAt,
-            sevenDayResetsAt: pool.strategy === "reset-aware"
+            utilization: v.health.utilization,
+            sevenDayResetsAt: strategy === "reset-aware"
                 ? sevenDayReset(states.find((s) => s.id === v.id)?.state, options, now)
                 : undefined,
         }));
+        const poolVerdicts = strategy === "spread"
+            ? withSpreadStats(rawVerdicts, spreadStatsFromHistory(readHistory(poolHistoryFile()), now, spreadWindowMs(pool.spreadWindowMs)))
+            : rawVerdicts;
         const decision = decideStickySelection({
             verdicts: poolVerdicts,
             sticky: previousSticky,
             nowMs: now,
             minDwellMs,
-            strategy: pool.strategy,
+            strategy,
         });
         const chosen = members.find((a) => a.id === decision.account) ?? members[0];
         const previousAccount = previousSticky?.account ?? members[0].id;
@@ -878,7 +879,9 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
             logger.info(`[multi-clawd] pool ${poolId}: no proven account can serve ${requestedModel} — ` +
                 `launching on ${decision.account} to re-test a login last rejected by the Claude CLI`);
         }
-        else if (decision.account !== previousAccount) {
+        else if (strategy !== "spread" && decision.account !== previousAccount) {
+            recordHistory(join(homedir(), ".openclaw", "state", "multi-clawd", `${previousAccount}.json`), { at: new Date(now).toISOString(), kind: "rotation", accountId: previousAccount,
+                toAccountId: decision.account, modelId: requestedModel, reason: "pool_selection" });
             const home = verdicts[0];
             const line = decision.account === members[0].id
                 ? `pool ${poolId}: returning home to ${decision.account}`
@@ -975,7 +978,7 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
                 env[RETRY_ROSTER_ENV] = JSON.stringify(roster);
         }
         env[SESSION_DIRS_ENV] = JSON.stringify([...new Set(configuredMembers.map(accountConfigDir))]);
-        if (pool.sessionAffinity) {
+        if (affinityOn) {
             env[SESSION_AFFINITY_ENV] = JSON.stringify({
                 file: join(homedir(), ".openclaw", "state", "multi-clawd", `pool-${poolId}.sessions.json`),
                 ttlMs: typeof pool.sessionAffinityTtlMs === "number" &&
@@ -1012,7 +1015,7 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
                 logger.info(`[multi-clawd] pool ${poolId}: pinned lane keeps ${ctx.modelId} despite exhausted pool (will fail over via the chain)`);
             }
         }
-        const affinityTransport = pool.sessionAffinity && transport && launch.kind === "verified"
+        const affinityTransport = affinityOn && transport && launch.kind === "verified"
             ? async function* (execution) {
                 const id = launchSessionId(execution.args);
                 const pin = id ? readSessionPin(join(homedir(), ".openclaw", "state", "multi-clawd", `pool-${poolId}.sessions.json`), id, Date.now(), typeof pool.sessionAffinityTtlMs === "number" && pool.sessionAffinityTtlMs > 0
@@ -1112,6 +1115,7 @@ export function startDirectOrderSync(params) {
         healthOptions,
         minDwellMs: params.pool?.minDwellMs,
         strategy: params.pool?.strategy,
+        spreadWindowMs: params.pool?.spreadWindowMs,
         command: params.directRoute?.openclawCommand,
     });
     if (directController?.signature === signature && directTimer) {
@@ -1127,6 +1131,8 @@ export function startDirectOrderSync(params) {
         healthOptions,
         minDwellMs: params.pool?.minDwellMs,
         strategy: params.pool?.strategy,
+        readHistory: () => readHistory(poolHistoryFile()),
+        spreadWindowMs: params.pool?.spreadWindowMs,
         configOrder: params.configOrder,
         readSticky: () => readStickyEntry(stickyFile),
         writeSticky: (entry) => writeStickyEntry(stickyFile, entry, logger),
@@ -1228,17 +1234,7 @@ export function startUsagePoll(params) {
                 }
             },
             readHealth: (id) => {
-                let shim;
-                try {
-                    shim = parseStoredState(readFileSync(healthStateFile(id), "utf8"));
-                    if (!shim)
-                        throw new Error("not valid health-state JSON");
-                }
-                catch (err) {
-                    if (err.code !== "ENOENT")
-                        throw err;
-                    shim = undefined;
-                }
+                const shim = readHealthStateFile(healthStateFile(id), { strictSnapshot: true });
                 let usage;
                 try {
                     usage = parseStoredState(readFileSync(usageStateFile(id), "utf8"));

package/dist/retry-plan.js

@@ -66,6 +66,8 @@ export function chooseRetryAccount(params) {
             return account;
         verdicts.push({ id: account.id, verdict: health.verdict, observedAt: health.observedAt });
     }
+    if (params.allowOverdueProbe === false)
+        return undefined;
     const probe = overdueProbeAccount(verdicts, params.nowMs);
     return probe ? params.roster.find((a) => a.id === probe) : undefined;
 }

package/dist/shim-core.js

@@ -192,9 +192,9 @@ export function limitFamilyFor(displayName, modelId) {
     return displayName.trim().toLowerCase() === family ? family : undefined;
 }
 const LIMIT_TEXT_RE = /reached your (.{1,40}?) limit/i;
-export function parseModelLimitError(line) {
-    if (!/reached your /i.test(line))
-        return undefined;
+const USAGE_LIMIT_API_ERROR = "usage_limit_reached";
+const AUTH_FAILED_ERROR_KIND = "authentication_failed";
+function parseRecord(line) {
     let record;
     try {
         record = JSON.parse(line);
@@ -204,12 +204,15 @@ export function parseModelLimitError(line) {
     }
     if (typeof record !== "object" || record === null)
         return undefined;
-    const r = record;
-    const isErrorRecord = r.type === "error" ||
+    return record;
+}
+function isErrorRecord(r) {
+    return (r.type === "error" ||
         r.is_error === true ||
-        (typeof r.subtype === "string" && r.subtype.startsWith("error"));
-    if (!isErrorRecord)
-        return undefined;
+        (typeof r.subtype === "string" && r.subtype.startsWith("error")) ||
+        (r.type === "assistant" && r.is_api_error_message === true));
+}
+function errorRecordTexts(r) {
     const texts = [];
     if (typeof r.result === "string")
         texts.push(r.result);
@@ -220,13 +223,193 @@ export function parseModelLimitError(line) {
         if (typeof msg === "string")
             texts.push(msg);
     }
-    for (const text of texts) {
+    if (r.type === "assistant" && r.is_api_error_message === true && Array.isArray(r.message?.content)) {
+        for (const block of r.message.content) {
+            if (block && block.type === "text" && typeof block.text === "string")
+                texts.push(block.text);
+        }
+    }
+    return texts;
+}
+const TRANSIENT_TEXT_PATTERNS = [
+    { re: /repeated 529 overloaded errors/i, kind: "overloaded" },
+    { re: /overloaded_error/i, kind: "overloaded" },
+    { re: /the api is at capacity/i, kind: "overloaded" },
+    { re: /server-side issue, usually temporary/i, kind: "server_error" },
+    { re: /temporarily limiting requests \(not your usage limit\)/i, kind: "rate_limit_transient" },
+    { re: /request rejected \(429\)/i, kind: "rate_limit_transient" },
+];
+const TRANSIENT_TEXT_MAX_CHARS = 160;
+export function parseTransientApiError(line) {
+    if (!/("error"|is_error|api_error_status|overloaded|server-side|temporarily limiting|request rejected|at capacity)/i.test(line)) {
+        return undefined;
+    }
+    const r = parseRecord(line);
+    if (!r || !isErrorRecord(r))
+        return undefined;
+    if (parseModelLimitError(line) || parseAuthFailure(line))
+        return undefined;
+    const joined = errorRecordTexts(r).join("\n");
+    const status = typeof r.api_error_status === "number" ? r.api_error_status : undefined;
+    const text = joined.trim().slice(0, TRANSIENT_TEXT_MAX_CHARS);
+    const kindFromCli = typeof r.error === "string" ? r.error : undefined;
+    if (status === 529 || kindFromCli === "overloaded")
+        return { kind: "overloaded", status, text };
+    if ((status !== undefined && status >= 500) || kindFromCli === "server_error") {
+        return { kind: "server_error", status, text };
+    }
+    for (const { re, kind } of TRANSIENT_TEXT_PATTERNS) {
+        if (re.test(joined))
+            return { kind, status, text };
+    }
+    if (status === 429)
+        return { kind: "rate_limit_transient", status, text };
+    return undefined;
+}
+export function parseApiRetryNotice(line) {
+    if (!line.includes('"api_retry"'))
+        return undefined;
+    let record;
+    try {
+        record = JSON.parse(line);
+    }
+    catch {
+        return undefined;
+    }
+    if (typeof record !== "object" || record === null)
+        return undefined;
+    const r = record;
+    if (r.type !== "system" || r.subtype !== "api_retry")
+        return undefined;
+    const delay = typeof r.retry_delay_ms === "number" && r.retry_delay_ms >= 0 ? r.retry_delay_ms : 0;
+    return { retryDelayMs: delay };
+}
+export function parseModelLimitError(line) {
+    if (!/(reached your |usage_limit_reached)/i.test(line))
+        return undefined;
+    const r = parseRecord(line);
+    if (!r || !isErrorRecord(r))
+        return undefined;
+    for (const text of errorRecordTexts(r)) {
         const match = LIMIT_TEXT_RE.exec(text);
         if (match)
             return { displayName: match[1].trim() };
     }
+    if (r.api_error === USAGE_LIMIT_API_ERROR)
+        return { displayName: USAGE_LIMIT_API_ERROR };
     return undefined;
 }
+export const LIMIT_PASSTHROUGH_MARKER = " (rate limit: 429)";
+const HOST_DIAGNOSTIC_ERROR_PREFIX = "[ede_diagnostic]";
+export function markForwardedLimitLine(line) {
+    if (!parseModelLimitError(line))
+        return line;
+    let record;
+    try {
+        record = JSON.parse(line);
+    }
+    catch {
+        return line;
+    }
+    if (typeof record !== "object" || record === null || Array.isArray(record))
+        return line;
+    const r = record;
+    const subtype = typeof r.subtype === "string" ? r.subtype : "";
+    if (r.type !== "result" || !(r.is_error === true || subtype.startsWith("error")))
+        return line;
+    const mark = (text) => text.includes(LIMIT_PASSTHROUGH_MARKER) ? undefined : `${text.trimEnd()}${LIMIT_PASSTHROUGH_MARKER}`;
+    if (Array.isArray(r.errors)) {
+        const idx = r.errors.findIndex((e) => typeof e === "string" && e.trim() !== "" && !e.trim().startsWith(HOST_DIAGNOSTIC_ERROR_PREFIX));
+        if (idx >= 0) {
+            const marked = mark(r.errors[idx]);
+            if (marked === undefined)
+                return line;
+            const errors = [...r.errors];
+            errors[idx] = marked;
+            return JSON.stringify({ ...r, errors });
+        }
+    }
+    if (typeof r.result === "string" && r.result.trim() !== "") {
+        const marked = mark(r.result);
+        return marked === undefined ? line : JSON.stringify({ ...r, result: marked });
+    }
+    if (typeof r.error === "object" && r.error !== null && !Array.isArray(r.error)) {
+        const err = r.error;
+        if (typeof err.message === "string" && err.message.trim() !== "") {
+            const marked = mark(err.message);
+            return marked === undefined ? line : JSON.stringify({ ...r, error: { ...err, message: marked } });
+        }
+    }
+    if (typeof r.error === "string" && r.error.trim() !== "") {
+        const marked = mark(r.error);
+        return marked === undefined ? line : JSON.stringify({ ...r, error: marked });
+    }
+    return line;
+}
+const RESULT_LINE_PREFIX = Buffer.from('{"type":"result"', "utf8");
+const FORWARD_HOLD_CAP_BYTES = 256 * 1024;
+function couldBeResultLine(pending) {
+    let i = 0;
+    while (i < pending.length && (pending[i] === 0x20 || pending[i] === 0x09 || pending[i] === 0x0d))
+        i++;
+    const rest = pending.subarray(i);
+    const n = Math.min(rest.length, RESULT_LINE_PREFIX.length);
+    return rest.subarray(0, n).equals(RESULT_LINE_PREFIX.subarray(0, n));
+}
+function forwardCompleteLine(line, onMark) {
+    const text = line.toString("utf8");
+    const ending = text.endsWith("\r\n") ? "\r\n" : text.endsWith("\n") ? "\n" : "";
+    const body = text.slice(0, text.length - ending.length).trim();
+    const marked = markForwardedLimitLine(body);
+    if (marked === body)
+        return line;
+    onMark?.();
+    return Buffer.from(marked + ending, "utf8");
+}
+export function createForwardWriter(write, onMark) {
+    let pending;
+    let streaming = false;
+    return {
+        push(chunk) {
+            const out = [];
+            let start = 0;
+            while (start < chunk.length) {
+                const nl = chunk.indexOf(0x0a, start);
+                const end = nl < 0 ? chunk.length : nl + 1;
+                const piece = chunk.subarray(start, end);
+                start = end;
+                if (streaming) {
+                    out.push(piece);
+                    if (nl >= 0)
+                        streaming = false;
+                    continue;
+                }
+                const line = pending ? Buffer.concat([pending, piece]) : piece;
+                pending = undefined;
+                if (nl >= 0) {
+                    out.push(forwardCompleteLine(line, onMark));
+                }
+                else if (!couldBeResultLine(line) || line.length > FORWARD_HOLD_CAP_BYTES) {
+                    out.push(line);
+                    streaming = true;
+                }
+                else {
+                    pending = Buffer.from(line);
+                }
+            }
+            if (out.length > 0)
+                write(out.length === 1 ? out[0] : Buffer.concat(out));
+        },
+        flush() {
+            if (pending) {
+                const line = pending;
+                pending = undefined;
+                write(forwardCompleteLine(line, onMark));
+            }
+            streaming = false;
+        },
+    };
+}
 export function recordModelLimit(state, modelId, now, resetsAt, family) {
     return {
         ...state,
@@ -300,36 +483,16 @@ export function parseAuthFailure(line) {
     if (!/(authenticat|oauth|logged in|\/login|unauthori|api key|access token|bearer token)/i.test(line)) {
         return undefined;
     }
-    let record;
-    try {
-        record = JSON.parse(line);
-    }
-    catch {
-        return undefined;
-    }
-    if (typeof record !== "object" || record === null)
-        return undefined;
-    const r = record;
-    const isErrorRecord = r.type === "error" ||
-        r.is_error === true ||
-        (typeof r.subtype === "string" && r.subtype.startsWith("error"));
-    if (!isErrorRecord)
+    const r = parseRecord(line);
+    if (!r || !isErrorRecord(r))
         return undefined;
-    const texts = [];
-    if (typeof r.result === "string")
-        texts.push(r.result);
-    if (typeof r.error === "string")
-        texts.push(r.error);
-    if (typeof r.error === "object" && r.error !== null) {
-        const msg = r.error.message;
-        if (typeof msg === "string")
-            texts.push(msg);
-    }
-    for (const text of texts) {
+    for (const text of errorRecordTexts(r)) {
         if (AUTH_FAILURE_PATTERNS.some((p) => p.test(text))) {
             return { reason: text.trim().slice(0, AUTH_REASON_MAX_CHARS) };
         }
     }
+    if (r.error === AUTH_FAILED_ERROR_KIND)
+        return { reason: AUTH_FAILED_ERROR_KIND };
     return undefined;
 }
 export function recordCredentialFailure(state, reason, now) {