@drakon-systems/multi-clawd 1.7.3 → 1.8.0

package/README.md

@@ -364,7 +364,7 @@ openclaw plugins install (Get-Location).Path
 **Or let your agent install it.** Running an OpenClaw assistant or Claude
 Code on the target machine already? Paste it this and go make coffee:
 
-> Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v1.7.3/SETUP-AGENT.md
+> Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v1.8.0/SETUP-AGENT.md
 > and follow it to set up multi-clawd on this machine. I own a second
 > Claude account — ask me when you need me to log in.
 

package/dist/alerts.js

@@ -13,6 +13,9 @@ export function addAlert(state, alert, nowMs) {
 export function clearAlert(state, key) {
     return { alerts: state.alerts.filter((a) => a.key !== key) };
 }
+export function alertKeysWithPrefix(state, prefix) {
+    return state.alerts.filter((a) => a.key.startsWith(prefix)).map((a) => a.key);
+}
 function isLive(alert, nowMs) {
     const ttl = alert.ttlMs ?? DEFAULT_TTL_MS[alert.severity];
     return nowMs - alert.at <= ttl;

package/dist/explain-core.js

@@ -66,6 +66,7 @@ const VERDICT_WORDS = {
     no_data: "no recent telemetry — treated as healthy",
     near_limit: "NEAR ITS LIMIT — the pool will hand over before it hard-fails",
     exhausted: "EXHAUSTED",
+    credential_failed: "LOGIN REJECTED — excluded from the pool until it is re-authenticated",
 };
 export function renderExplanation(model) {
     const lines = [];

package/dist/health.js

@@ -2,6 +2,11 @@ import { modelWindowKey } from "./shim-core.js";
 const DEFAULT_UTILIZATION_THRESHOLD = 0.85;
 const DEFAULT_STALE_AFTER_MS = 6 * 60 * 60 * 1000;
 export const MODEL_REJECTED_TTL_MS = 60 * 60 * 1000;
+export const CREDENTIAL_FAILED_TTL_MS = 15 * 60 * 1000;
+export const REJECTION_REVALIDATE_AFTER_MS = 60 * 60 * 1000;
+function rejectionStillAssertable(seenAt, nowMs) {
+    return nowMs - seenAt <= REJECTION_REVALIDATE_AFTER_MS;
+}
 const MODEL_WINDOW_PREFIX = "model:";
 const SHORT_WINDOW_PATTERN = /(^|_)hours?(_|$)/;
 export function isShortWindow(window) {
@@ -11,18 +16,49 @@ const PERIOD_WINDOW_PATTERN = /(^|_)(minutes?|hours?|days?|weeks?|months?)(_|$)/
 export function isPeriodWindow(window) {
     return PERIOD_WINDOW_PATTERN.test(window);
 }
+export function isOverageWindow(window) {
+    return /overage/i.test(window);
+}
 export function isWarningStatus(status) {
     return /warning/i.test(status);
 }
 export const MAX_RESET_HORIZON_MS = 8 * 24 * 60 * 60 * 1000;
+export const PERIOD_RESET_LESS_BLOCK_MS = 6 * 60 * 60 * 1000;
+export function resetLessBlockMs(window) {
+    return isShortWindow(window) ? MODEL_REJECTED_TTL_MS : PERIOD_RESET_LESS_BLOCK_MS;
+}
+function windowAppliesToModel(w, requestedWindowKey) {
+    if (w.model === undefined)
+        return true;
+    if (requestedWindowKey === undefined)
+        return false;
+    return modelWindowKey(w.model) === requestedWindowKey;
+}
+export function credentialFailureFor(state, nowMs) {
+    const credential = state?.credential;
+    if (!credential || credential.status !== "failed")
+        return undefined;
+    if (nowMs - credential.seenAt > CREDENTIAL_FAILED_TTL_MS)
+        return undefined;
+    return {
+        verdict: "credential_failed",
+        resumeAt: credential.seenAt + CREDENTIAL_FAILED_TTL_MS,
+        reason: `login rejected by the Claude CLI ${Math.round((nowMs - credential.seenAt) / 60000)}m ago${credential.reason ? `: ${credential.reason}` : ""} — re-authenticate this account`,
+    };
+}
 export function classifyAccountHealth(state, options, nowMs, requestedModel) {
     const staleAfterMs = options.staleAfterMs ?? DEFAULT_STALE_AFTER_MS;
     const threshold = options.utilizationThreshold ?? DEFAULT_UTILIZATION_THRESHOLD;
     if (!state)
         return { verdict: "no_data" };
+    const credentialFailure = credentialFailureFor(state, nowMs);
+    if (credentialFailure)
+        return credentialFailure;
     const requestedWindowKey = requestedModel !== undefined ? modelWindowKey(requestedModel) : undefined;
     let worst = { verdict: "ok" };
     let hasLiveEvidence = false;
+    let overageUtilization;
+    let quotaUtilization;
     for (const [window, w] of Object.entries(state.windows)) {
         const resetMs = typeof w.resetsAt === "number" ? w.resetsAt * 1000 : undefined;
         const resetBearing = resetMs !== undefined && resetMs > nowMs;
@@ -58,32 +94,50 @@ export function classifyAccountHealth(state, options, nowMs, requestedModel) {
                 reason: `${requestedModel} limit hit ${Math.round((nowMs - w.seenAt) / 60000)}m ago (no reset time; TTL block)`,
             };
         }
-        const fresh = resetBearing || nowMs - w.seenAt <= staleAfterMs;
+        const ownBlockMs = w.status === "rejected" && resetMs === undefined && isPeriodWindow(window)
+            ? resetLessBlockMs(window)
+            : 0;
+        const fresh = resetBearing || nowMs - w.seenAt <= Math.max(staleAfterMs, ownBlockMs);
         if (!fresh)
             continue;
         hasLiveEvidence = true;
-        if (w.status === "rejected" && resetBearing && isPeriodWindow(window)) {
+        if (w.status === "rejected" &&
+            resetBearing &&
+            isPeriodWindow(window) &&
+            windowAppliesToModel(w, requestedWindowKey) &&
+            rejectionStillAssertable(w.seenAt, nowMs)) {
             return {
                 verdict: "exhausted",
                 resumeAt: resetMs,
                 reason: `${window} rejected until ${new Date(resetMs).toISOString()}`,
             };
         }
-        if (w.status === "rejected" && resetMs === undefined && isPeriodWindow(window)) {
+        if (w.status === "rejected" &&
+            resetMs === undefined &&
+            isPeriodWindow(window) &&
+            windowAppliesToModel(w, requestedWindowKey) &&
+            nowMs - w.seenAt <= resetLessBlockMs(window)) {
             return {
                 verdict: "exhausted",
-                resumeAt: w.seenAt + MODEL_REJECTED_TTL_MS,
-                reason: `${window} rejected ${Math.round((nowMs - w.seenAt) / 60000)}m ago (no reset time; TTL block)`,
+                resumeAt: w.seenAt + resetLessBlockMs(window),
+                reason: `${window} rejected ${Math.round((nowMs - w.seenAt) / 60000)}m ago (no reset time; ${Math.round(resetLessBlockMs(window) / 3600000)}h block)`,
             };
         }
-        if (worst.verdict === "ok" &&
-            typeof w.utilization === "number" &&
+        if (typeof w.utilization === "number" &&
             w.utilization >= threshold &&
             (resetBearing || resetMs === undefined)) {
-            worst = {
-                verdict: "near_limit",
-                reason: `${window} utilization ${w.utilization} >= ${threshold}`,
-            };
+            if (isOverageWindow(window) && !options.rotateOnOverage) {
+                overageUtilization = Math.max(overageUtilization ?? 0, w.utilization);
+            }
+            else if (worst.verdict === "ok") {
+                worst = {
+                    verdict: "near_limit",
+                    reason: `${window} utilization ${w.utilization} >= ${threshold}`,
+                };
+            }
+        }
+        if (!isOverageWindow(window) && typeof w.utilization === "number") {
+            quotaUtilization = Math.max(quotaUtilization ?? 0, w.utilization);
         }
         if (worst.verdict === "ok" &&
             typeof w.utilization !== "number" &&
@@ -98,6 +152,16 @@ export function classifyAccountHealth(state, options, nowMs, requestedModel) {
     }
     if (worst.verdict === "ok" && !hasLiveEvidence)
         return { verdict: "no_data" };
+    if (worst.verdict === "ok" && overageUtilization !== undefined) {
+        const pct = (n) => `${Math.round(n * 100)}%`;
+        worst = {
+            ...worst,
+            overageAdvisory: `paid overage is ${pct(overageUtilization)} spent while real quota is at ` +
+                `${quotaUtilization !== undefined ? pct(quotaUtilization) : "an unreported level"} — ` +
+                `not rotating on spill-over alone. Set pool.rotateOnOverage: true to rotate on it, ` +
+                `or leave it to keep using the overage you have paid for.`,
+        };
+    }
     return worst;
 }
 export function summarizeWindowUsage(state, options, nowMs) {
@@ -128,6 +192,12 @@ export function choosePoolAccount(pool) {
         return usable.id;
     return pool.find((a) => a.verdict === "near_limit")?.id;
 }
+export function allCredentialFailed(pool) {
+    return pool.length > 0 && pool.every((a) => a.verdict === "credential_failed");
+}
+export function fallbackPoolAccount(pool) {
+    return (pool.find((a) => a.verdict !== "credential_failed") ?? pool[0]).id;
+}
 export function pickPoolAccountForLaunch(pool) {
-    return choosePoolAccount(pool) ?? pool[0].id;
+    return choosePoolAccount(pool) ?? fallbackPoolAccount(pool);
 }

package/dist/index.js

@@ -9,11 +9,12 @@ import { MODEL_ALIASES, buildCatalogEntries, canonicalModelId, isModernClaudeMod
 import { decideDegradation, matchesPin } from "./degrade.js";
 import { resolveExecMode, permissionModeArgs } from "./exec-policy.js";
 import { resolveBaseModelIds } from "./catalog-source.js";
-import { classifyAccountHealth } from "./health.js";
+import { allCredentialFailed, classifyAccountHealth } from "./health.js";
 import { decideStickySelection } from "./sticky.js";
+import { clearCredentialFailure, mergeHealthStates, parseStoredState, recordCredentialFailure, } from "./shim-core.js";
 import { createTokenRefResolver, isSecretRefShape, } from "./token-resolution.js";
 import { resolveSecretRefValues } from "openclaw/plugin-sdk/secret-ref-runtime";
-import { addAlert, clearAlert, pendingAlertText } from "./alerts.js";
+import { addAlert, alertKeysWithPrefix, clearAlert, pendingAlertText, } from "./alerts.js";
 import { buildAccountChildEnv, tokenFileModeWarning, validateAccountTokenSources, } from "./account-env.js";
 import { diffCatalogModels, formatNewModelNotice, } from "./model-currency.js";
 import { checkAccountCredential, createRefProbeTracker, } from "./login-health.js";
@@ -68,6 +69,10 @@ let loginProbeTimer;
 function raiseAlert(alert) {
     alertState = addAlert(alertState, alert, Date.now());
 }
+export function pendingOperatorAlerts(nowMs) {
+    ingestAlertSpool();
+    return pendingAlertText(alertState, nowMs);
+}
 function ingestAlertSpool() {
     const spool = join(homedir(), ".openclaw", "state", "multi-clawd", "alerts-spool.jsonl");
     let raw;
@@ -116,53 +121,104 @@ const realCredentialIo = {
     },
     platform: process.platform,
 };
-function startLoginHealthProbe(accounts, logger) {
-    if (loginProbeTimer)
-        clearInterval(loginProbeTimer);
-    const lastStatus = new Map();
-    const refTrackers = new Map();
-    const probe = async () => {
-        const now = Date.now();
-        for (const account of accounts) {
-            let status;
-            let reason;
-            if (isSecretRefShape(account.oauthTokenRef) && !account.oauthTokenFile && !account.native) {
-                let tracker = refTrackers.get(account.id);
-                if (!tracker) {
-                    tracker = createRefProbeTracker();
-                    refTrackers.set(account.id, tracker);
-                }
-                const result = (await activeTokenResolver?.resolveDetailed(account.oauthTokenRef)) ?? {
-                    failure: "provider_error",
-                };
-                const outcome = tracker.observe(result, now);
-                status = outcome.status;
-                reason = outcome.reason;
-            }
-            else {
-                const check = checkAccountCredential(account, realCredentialIo);
-                status = check.status;
-                reason = check.reason;
+const refProbeTrackers = new Map();
+const lastProbeStatus = new Map();
+function recordAccountCredentialFailure(accountId, reason, nowMs) {
+    const file = healthStateFile(accountId);
+    let state;
+    try {
+        state = parseStoredState(readFileSync(file, "utf8")) ?? { accountId, windows: {} };
+    }
+    catch {
+        state = { accountId, windows: {} };
+    }
+    const next = mergeHealthStates(state, recordCredentialFailure(state, reason, nowMs), nowMs);
+    try {
+        mkdirSync(dirname(file), { recursive: true });
+        const tmp = `${file}.tmp-${process.pid}`;
+        writeFileSync(tmp, JSON.stringify(next, null, 2), { mode: 0o600 });
+        renameSync(tmp, file);
+    }
+    catch {
+    }
+}
+export async function runLoginHealthProbe(accounts, logger, deps = {}) {
+    const now = deps.nowMs ?? Date.now();
+    const resolver = deps.resolver ?? activeTokenResolver;
+    const io = deps.io ?? realCredentialIo;
+    for (const account of accounts) {
+        let status;
+        let reason;
+        let cause;
+        if (isSecretRefShape(account.oauthTokenRef) && !account.oauthTokenFile && !account.native) {
+            let tracker = refProbeTrackers.get(account.id);
+            if (!tracker) {
+                tracker = createRefProbeTracker();
+                refProbeTrackers.set(account.id, tracker);
             }
-            const previous = lastStatus.get(account.id);
-            lastStatus.set(account.id, status);
-            if (status === "broken" && previous !== "broken") {
-                const text = `account "${account.id}" login looks dead (${reason ?? "unknown"}) — turns on it will fail until fixed`;
+            const result = (await resolver?.resolveDetailed(account.oauthTokenRef)) ?? {
+                failure: "provider_error",
+            };
+            const outcome = tracker.observe(result, now);
+            status = outcome.status;
+            reason = outcome.reason;
+            cause = outcome.cause;
+        }
+        else {
+            const check = checkAccountCredential(account, io);
+            status = check.status;
+            reason = check.reason;
+        }
+        const previous = lastProbeStatus.get(account.id);
+        lastProbeStatus.set(account.id, status);
+        if (status === "broken" && cause === "credential") {
+            recordAccountCredentialFailure(account.id, reason ?? "login probe found no credential", now);
+            if (previous !== "broken") {
+                const text = `account "${account.id}" login looks dead (${reason ?? "unknown"}) — excluded from pool selection until it is fixed`;
                 logger.error(`[multi-clawd] ${text}`);
                 raiseAlert({ key: `login:${account.id}`, severity: "error", text });
             }
-            else if (status === "degraded" && previous !== "degraded") {
-                logger.info(`[multi-clawd] account "${account.id}" login degraded: ${reason ?? "resolver error"}`);
-            }
-            else if (status === "ok" && (previous === "broken" || previous === "degraded")) {
-                logger.info(`[multi-clawd] account "${account.id}" login recovered`);
-                alertState = clearAlert(alertState, `login:${account.id}`);
+        }
+        else if (status === "broken" && cause === "provider") {
+            if (previous !== "broken") {
+                const text = `account "${account.id}" credential resolver is unreachable (${reason ?? "unknown"}) — ` +
+                    `this looks like a host or network problem rather than a broken login, so account ` +
+                    `selection is unchanged. Check connectivity to the secret provider.`;
+                logger.error(`[multi-clawd] ${text}`);
+                raiseAlert({ key: `login-resolver:${account.id}`, severity: "error", text });
             }
         }
-    };
-    const initial = setTimeout(() => void probe().catch(() => { }), LOGIN_PROBE_INITIAL_DELAY_MS);
+        else if (status === "broken" && previous !== "broken") {
+            const text = `account "${account.id}" login looks dead (${reason ?? "unknown"}) — turns on it will fail until fixed`;
+            logger.error(`[multi-clawd] ${text}`);
+            raiseAlert({ key: `login:${account.id}`, severity: "error", text });
+        }
+        else if (status === "degraded" && previous !== "degraded") {
+            logger.info(`[multi-clawd] account "${account.id}" login degraded: ${reason ?? "resolver error"}`);
+        }
+        else if (status === "ok" && (previous === "broken" || previous === "degraded")) {
+            logger.info(`[multi-clawd] account "${account.id}" login recovered`);
+            alertState = clearAlert(alertState, `login:${account.id}`);
+            alertState = clearAlert(alertState, `login-resolver:${account.id}`);
+        }
+    }
+}
+export function startLoginHealthProbe(accounts, logger) {
+    if (loginProbeTimer)
+        clearInterval(loginProbeTimer);
+    const live = new Set(accounts.map((a) => a.id));
+    for (const id of [...refProbeTrackers.keys()]) {
+        if (!live.has(id))
+            refProbeTrackers.delete(id);
+    }
+    for (const id of [...lastProbeStatus.keys()]) {
+        if (!live.has(id))
+            lastProbeStatus.delete(id);
+    }
+    const probe = () => void runLoginHealthProbe(accounts, logger).catch(() => { });
+    const initial = setTimeout(probe, LOGIN_PROBE_INITIAL_DELAY_MS);
     initial.unref?.();
-    loginProbeTimer = setInterval(() => void probe().catch(() => { }), LOGIN_PROBE_INTERVAL_MS);
+    loginProbeTimer = setInterval(probe, LOGIN_PROBE_INTERVAL_MS);
     loginProbeTimer.unref?.();
 }
 const warnedTokenFileModes = new Set();
@@ -193,9 +249,9 @@ function peekToken(account) {
         return undefined;
     throw new Error(`[multi-clawd] account "${account.id}" needs oauthTokenFile, oauthTokenRef, or configDir`);
 }
-async function resolveTokenAsync(account) {
+async function resolveTokenAsync(account, resolver) {
     if (isSecretRefShape(account.oauthTokenRef) && !account.native && !account.oauthTokenFile) {
-        return activeTokenResolver?.resolve(account.oauthTokenRef);
+        return (resolver ?? activeTokenResolver)?.resolve(account.oauthTokenRef);
     }
     return peekToken(account);
 }
@@ -330,8 +386,8 @@ export function buildBackend(account, execMode) {
         },
     };
 }
-async function buildAccountEnv(account) {
-    const token = await resolveTokenAsync(account);
+async function buildAccountEnv(account, resolver) {
+    const token = await resolveTokenAsync(account, resolver);
     return buildAccountChildEnv(account, token, healthStateFile(account.id));
 }
 function buildCatalogProvider(account) {
@@ -424,8 +480,7 @@ export default definePluginEntry({
             const logger = api.logger;
             try {
                 api.on("heartbeat_prompt_contribution", () => {
-                    ingestAlertSpool();
-                    const text = pendingAlertText(alertState, Date.now());
+                    const text = pendingOperatorAlerts(Date.now());
                     return text ? { appendContext: text } : undefined;
                 });
             }
@@ -445,6 +500,32 @@ function readHealthState(accountId) {
         return undefined;
     }
 }
+export function clearAccountCredentialFailure(accountId) {
+    const file = healthStateFile(accountId);
+    let state;
+    try {
+        state = parseStoredState(readFileSync(file, "utf8")) ?? {
+            accountId,
+            windows: {},
+        };
+    }
+    catch {
+        return false;
+    }
+    if (state.credential?.status !== "failed")
+        return false;
+    const cleared = mergeHealthStates(state, clearCredentialFailure(state, Date.now()), Date.now());
+    try {
+        mkdirSync(dirname(file), { recursive: true });
+        const tmp = `${file}.tmp-${process.pid}`;
+        writeFileSync(tmp, JSON.stringify(cleared, null, 2), { mode: 0o600 });
+        renameSync(tmp, file);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
 function readStickyEntry(file) {
     try {
         const parsed = JSON.parse(readFileSync(file, "utf8"));
@@ -471,7 +552,7 @@ function writeStickyEntry(file, entry, logger) {
         logger.warn(`[multi-clawd] sticky state write failed: ${String(err)}`);
     }
 }
-export function registerPoolBackend(api, pool, accounts, registeredIds, execMode) {
+export function registerPoolBackend(api, pool, accounts, registeredIds, execMode, deps) {
     const logger = api.logger;
     if (!pool)
         return;
@@ -498,6 +579,7 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
     const options = {
         utilizationThreshold: pool.utilizationThreshold,
         staleAfterMs: pool.staleAfterMs,
+        rotateOnOverage: pool.rotateOnOverage,
     };
     const poolAccount = {
         id: poolId,
@@ -511,10 +593,22 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
     backend.prepareExecution = async (ctx) => {
         const now = Date.now();
         const requestedModel = canonicalModelId(ctx.modelId) ?? ctx.modelId;
-        const verdicts = members.map((a) => ({
-            id: a.id,
-            health: classifyAccountHealth(readHealthState(a.id), options, now, requestedModel),
+        const states = members.map((a) => ({ id: a.id, state: readHealthState(a.id) }));
+        const verdicts = states.map(({ id, state }) => ({
+            id,
+            health: classifyAccountHealth(state, options, now, requestedModel),
         }));
+        if (allCredentialFailed(verdicts.map((v) => ({ id: v.id, verdict: v.health.verdict })))) {
+            const detail = verdicts
+                .map((v) => `${v.id} (${v.health.reason ?? "credential rejected"})`)
+                .join("; ");
+            const text = `pool ${poolId}: every account's login is rejected by the Claude CLI — ` +
+                `re-authenticate with \`multi-clawd login <account>\`. ${detail}`;
+            logger.error(`[multi-clawd] ${text}`);
+            raiseAlert({ key: `pool-credentials:${poolId}`, severity: "error", text });
+            writeStickyEntry(stickyFile, undefined, logger);
+            throw new Error(`[multi-clawd] ${text}`);
+        }
         const previousSticky = readStickyEntry(stickyFile);
         const decision = decideStickySelection({
             verdicts: verdicts.map((v) => ({ id: v.id, verdict: v.health.verdict })),
@@ -532,15 +626,72 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
             logger.info(`[multi-clawd] ${line}`);
             raiseAlert({ key: `rotation:${poolId}`, severity: "info", text: line });
         }
+        const exhaustionPrefix = `pool-exhausted:${poolId}:`;
+        for (const key of alertKeysWithPrefix(alertState, exhaustionPrefix)) {
+            const alertedModel = key.slice(exhaustionPrefix.length);
+            const stillExhausted = states.every(({ state }) => classifyAccountHealth(state, options, now, alertedModel).verdict === "exhausted");
+            if (!stillExhausted)
+                alertState = clearAlert(alertState, key);
+        }
         if (verdicts.every((v) => v.health.verdict === "exhausted")) {
             raiseAlert({
-                key: `pool-exhausted:${poolId}:${requestedModel}`,
+                key: `${exhaustionPrefix}${requestedModel}`,
                 severity: "error",
                 text: `pool ${poolId}: every account is exhausted for ${requestedModel} — turns are degrading or falling through the chain`,
             });
         }
+        for (const v of verdicts) {
+            const key = `overage:${poolId}:${v.id}`;
+            if (v.health.overageAdvisory) {
+                raiseAlert({
+                    key,
+                    severity: "info",
+                    text: `pool ${poolId}: account "${v.id}" — ${v.health.overageAdvisory}`,
+                });
+            }
+            else {
+                alertState = clearAlert(alertState, key);
+            }
+        }
+        for (const v of verdicts) {
+            const key = `credential:${poolId}:${v.id}`;
+            if (v.health.verdict === "credential_failed") {
+                raiseAlert({
+                    key,
+                    severity: "error",
+                    text: `pool ${poolId}: account "${v.id}" is excluded — ${v.health.reason ?? "its login was rejected by the Claude CLI"}. Fix with \`multi-clawd login ${v.id}\`.`,
+                });
+            }
+            else {
+                alertState = clearAlert(alertState, key);
+            }
+        }
+        alertState = clearAlert(alertState, `pool-credentials:${poolId}`);
         writeStickyEntry(stickyFile, decision.sticky, logger);
-        const env = await buildAccountEnv(chosen);
+        const order = [chosen, ...members.filter((m) => m.id !== chosen.id)];
+        let env;
+        const unresolved = [];
+        for (const candidate of order) {
+            try {
+                env = await buildAccountEnv(candidate, deps?.resolver);
+                if (candidate.id !== chosen.id) {
+                    logger.warn(`[multi-clawd] pool ${poolId}: ${chosen.id}'s credential did not resolve — ` +
+                        `launching on ${candidate.id} instead (account not benched; secret provider may be down)`);
+                }
+                break;
+            }
+            catch (err) {
+                unresolved.push(`${candidate.id} (${err.message})`);
+            }
+        }
+        if (!env) {
+            const text = `pool ${poolId}: no account's credential could be resolved — ` +
+                `the secret provider is unreachable or every reference is empty. ${unresolved.join("; ")}`;
+            logger.error(`[multi-clawd] ${text}`);
+            raiseAlert({ key: `pool-unresolvable:${poolId}`, severity: "error", text });
+            throw new Error(`[multi-clawd] ${text}`);
+        }
+        alertState = clearAlert(alertState, `pool-unresolvable:${poolId}`);
         if (ladder.length > 0) {
             const pinned = matchesPin(pins, {
                 agentDir: ctx.agentDir ?? "",

package/dist/login-health.js

@@ -15,6 +15,7 @@ export function createRefProbeTracker(options = {}) {
                 firstFailureAt = undefined;
                 return {
                     status: "broken",
+                    cause: "credential",
                     reason: "oauthTokenRef resolved to nothing (credential problem)",
                 };
             }
@@ -25,6 +26,7 @@ export function createRefProbeTracker(options = {}) {
             if (consecutive >= deadAfterConsecutive && elapsed >= deadAfterMs) {
                 return {
                     status: "broken",
+                    cause: "provider",
                     reason: `resolver failing ${deadAfterConsecutive}+ consecutive probes over ${Math.round(deadAfterMs / 60000)}m`,
                 };
             }

package/dist/shim-core.js

@@ -89,12 +89,25 @@ export function parseStoredState(raw) {
             isUsingOverage: typeof w.isUsingOverage === "boolean" ? w.isUsingOverage : undefined,
             seenAt: w.seenAt,
             rawInfo: typeof w.rawInfo === "string" ? w.rawInfo : undefined,
+            model: typeof w.model === "string" ? w.model : undefined,
         };
     }
+    let credential;
+    if (typeof p.credential === "object" && p.credential !== null) {
+        const c = p.credential;
+        if ((c.status === "failed" || c.status === "ok") && typeof c.seenAt === "number") {
+            credential = {
+                status: c.status,
+                reason: typeof c.reason === "string" ? c.reason : undefined,
+                seenAt: c.seenAt,
+            };
+        }
+    }
     return {
         accountId: typeof p.accountId === "string" ? p.accountId : "unknown",
         updatedAt: typeof p.updatedAt === "number" ? p.updatedAt : undefined,
         windows,
+        credential,
     };
 }
 export function mergeHealthStates(disk, live, now, pruneAfterMs = PRUNE_AFTER_MS) {
@@ -119,11 +132,19 @@ export function mergeHealthStates(disk, live, now, pruneAfterMs = PRUNE_AFTER_MS
             }
         }
     }
+    let credential = disk.credential;
+    if (live.credential && (!credential || live.credential.seenAt >= credential.seenAt)) {
+        credential = live.credential;
+    }
+    if (now !== undefined && credential && now - credential.seenAt > pruneAfterMs) {
+        credential = undefined;
+    }
     const updatedAt = Math.max(disk.updatedAt ?? 0, live.updatedAt ?? 0);
     return {
         accountId: live.accountId,
         updatedAt: updatedAt > 0 ? updatedAt : undefined,
         windows,
+        credential,
     };
 }
 const MODEL_ID_PROVIDER_PREFIXES = [
@@ -207,7 +228,71 @@ export function recordModelLimit(state, modelId, now, resetsAt) {
         },
     };
 }
-export function updateHealthState(state, event, now) {
+const AUTH_FAILURE_PATTERNS = [
+    /oauth (session|token) (has )?(expired|been revoked)/i,
+    /failed to authenticate/i,
+    /invalid (api key|bearer token|access token)/i,
+    /authentication[_ ]?error/i,
+    /please run \/login/i,
+    /not logged in/i,
+    /unauthori[sz]ed/i,
+];
+const AUTH_REASON_MAX_CHARS = 200;
+export function parseAuthFailure(line) {
+    if (!/(authenticat|oauth|logged in|\/login|unauthori|api key|access token|bearer token)/i.test(line)) {
+        return undefined;
+    }
+    let record;
+    try {
+        record = JSON.parse(line);
+    }
+    catch {
+        return undefined;
+    }
+    if (typeof record !== "object" || record === null)
+        return undefined;
+    const r = record;
+    const isErrorRecord = r.type === "error" ||
+        r.is_error === true ||
+        (typeof r.subtype === "string" && r.subtype.startsWith("error"));
+    if (!isErrorRecord)
+        return undefined;
+    const texts = [];
+    if (typeof r.result === "string")
+        texts.push(r.result);
+    if (typeof r.error === "string")
+        texts.push(r.error);
+    if (typeof r.error === "object" && r.error !== null) {
+        const msg = r.error.message;
+        if (typeof msg === "string")
+            texts.push(msg);
+    }
+    for (const text of texts) {
+        if (AUTH_FAILURE_PATTERNS.some((p) => p.test(text))) {
+            return { reason: text.trim().slice(0, AUTH_REASON_MAX_CHARS) };
+        }
+    }
+    return undefined;
+}
+export function recordCredentialFailure(state, reason, now) {
+    return {
+        ...state,
+        updatedAt: now,
+        credential: {
+            status: "failed",
+            reason: reason.slice(0, AUTH_REASON_MAX_CHARS),
+            seenAt: now,
+        },
+    };
+}
+export function clearCredentialFailure(state, now) {
+    return {
+        ...state,
+        updatedAt: now,
+        credential: { status: "ok", seenAt: now },
+    };
+}
+export function updateHealthState(state, event, now, model) {
     const key = event.rateLimitType ?? "unknown";
     return {
         ...state,
@@ -221,6 +306,7 @@ export function updateHealthState(state, event, now) {
                 isUsingOverage: event.isUsingOverage,
                 seenAt: now,
                 rawInfo: event.rawInfo,
+                model: model === undefined ? undefined : canonicalizeModelIdForWindow(model),
             },
         },
     };

package/dist/shim.js

@@ -1,7 +1,7 @@
 import { spawn } from "node:child_process";
 import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
 import { dirname } from "node:path";
-import { classifyStateReadFailure, createLineScanner, mergeHealthStates, parseRateLimitEvent, parseStoredState, updateHealthState, } from "./shim-core.js";
+import { classifyStateReadFailure, clearCredentialFailure, createLineScanner, mergeHealthStates, parseAuthFailure, parseRateLimitEvent, parseStoredState, recordCredentialFailure, updateHealthState, } from "./shim-core.js";
 import { rewriteModelArg } from "./degrade.js";
 import { parseModelLimitError, recordModelLimit } from "./shim-core.js";
 import { canonicalModelId } from "./models.js";
@@ -108,11 +108,12 @@ function guessLimitResetsAt() {
     }
     return undefined;
 }
+let sawAuthFailure = false;
 const scanner = createLineScanner((line) => {
     try {
         const event = parseRateLimitEvent(line);
         if (event) {
-            state = updateHealthState(state, event, Date.now());
+            state = updateHealthState(state, event, Date.now(), effectiveModelId());
             persistState();
         }
         const limitHit = parseModelLimitError(line);
@@ -124,6 +125,13 @@ const scanner = createLineScanner((line) => {
                 process.stderr.write(`[multi-clawd shim] model limit hit recorded: ${model} (reported as "${limitHit.displayName}")\n`);
             }
         }
+        const authFailure = parseAuthFailure(line);
+        if (authFailure && !sawAuthFailure) {
+            sawAuthFailure = true;
+            state = recordCredentialFailure(state, authFailure.reason, Date.now());
+            persistState();
+            process.stderr.write(`[multi-clawd shim] auth failure recorded for ${accountId}: ${authFailure.reason}\n`);
+        }
     }
     catch {
     }
@@ -138,11 +146,26 @@ for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
         child.kill(signal);
     });
 }
+function clearRecordedAuthFailureOnSuccess() {
+    if (!stateFile || sawAuthFailure)
+        return;
+    try {
+        if (readPersistedState()?.credential?.status !== "failed")
+            return;
+        state = clearCredentialFailure(state, Date.now());
+        persistState();
+        process.stderr.write(`[multi-clawd shim] auth recovered for ${accountId} — credential exclusion cleared\n`);
+    }
+    catch {
+    }
+}
 child.on("close", (code, signal) => {
     if (signal) {
         process.kill(process.pid, signal);
         return;
     }
+    if ((code ?? 0) === 0)
+        clearRecordedAuthFailureOnSuccess();
     process.exit(code ?? 0);
 });
 child.on("error", (err) => {

package/dist/sticky.js

@@ -1,4 +1,4 @@
-import { choosePoolAccount } from "./health.js";
+import { choosePoolAccount, fallbackPoolAccount } from "./health.js";
 export const DEFAULT_MIN_DWELL_MS = 10 * 60 * 1000;
 export function decideStickySelection(params) {
     const { verdicts, sticky, nowMs } = params;
@@ -6,7 +6,7 @@ export function decideStickySelection(params) {
     const home = verdicts[0];
     const healthChoice = choosePoolAccount(verdicts);
     if (!healthChoice)
-        return { account: home.id };
+        return { account: fallbackPoolAccount(verdicts) };
     const stickyVerdict = sticky
         ? verdicts.find((v) => v.id === sticky.account)?.verdict
         : undefined;

package/openclaw.plugin.json

@@ -1,7 +1,7 @@
 {
   "id": "multi-clawd",
   "name": "multi-clawd",
-  "version": "1.7.3",
+  "version": "1.8.0",
   "description": "Register additional Claude Code logins (Max/Pro accounts) as first-class OpenClaw CLI backends for cross-account failover, keeping the full skills/MCP harness on every account.",
   "cliBackends": [
     "claw1",
@@ -51,6 +51,10 @@
             "type": "number",
             "description": "Ignore health data older than this. Default 21600000 (6h)."
           },
+          "rotateOnOverage": {
+            "type": "boolean",
+            "description": "Rotate when the seven_day_overage_included window (quota PLUS purchased spill-over) crosses utilizationThreshold. Default false: high utilization there means little paid overage left, not little quota left, so the pool keeps using the overage you have paid for and raises an operator alert instead of deciding for you."
+          },
           "minDwellMs": {
             "type": "number",
             "description": "After rotating away from home, stay on the rotated-to account at least this long before returning home (anti-flap hysteresis; health always overrides). Default 600000 (10min)."

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@drakon-systems/multi-clawd",
-  "version": "1.7.3",
+  "version": "1.8.0",
   "description": "Multi-account Claude Code failover for OpenClaw — register additional Claude (Max/Pro) logins as first-class CLI backends and keep the full skills/MCP harness across every account. Also imports those accounts' setup tokens into Hermes Agent's Anthropic credential pool.",
   "type": "module",
   "license": "MIT",

package/scripts/cli.mjs

@@ -461,10 +461,11 @@ async function login() {
   const { readFileSync: rf, existsSync, mkdirSync, chmodSync, statSync, mkdtempSync, rmSync } =
     await import("node:fs");
   const { homedir, tmpdir } = await import("node:os");
-  let lp, ec;
+  let lp, ec, idx;
   try {
     lp = await import(resolve(__dirname, "..", "dist", "login-plan.js"));
     ec = await import(resolve(__dirname, "..", "dist", "explain-core.js"));
+    idx = await import(resolve(__dirname, "..", "dist", "index.js"));
   } catch {
     console.error("login: built dist/ is missing — reinstall the package.");
     process.exit(1);
@@ -513,6 +514,17 @@ async function login() {
     console.error(`\n  ❌ ${plan.command.join(" ")} exited with ${r.status ?? "an error"}.`);
     process.exit(1);
   }
+  // Explicit re-authentication ends any recorded runtime credential failure
+  // immediately (#8) — without this the freshly re-authed account stays
+  // benched until the 15-minute TTL expires, which reads to the operator as
+  // "logging back in did nothing".
+  try {
+    if (idx.clearAccountCredentialFailure?.(acc.id)) {
+      console.log(`\n  ↻ cleared ${acc.id}'s recorded login failure — the pool can use it again.`);
+    }
+  } catch {
+    /* clearing is a courtesy; never fail a login over it */
+  }
   if (plan.verify === "auth-status") {
     try {
       const out = spawnSync("claude", ["auth", "status"], { encoding: "utf8", env }).stdout ?? "";

package/scripts/doctor.mjs

@@ -263,6 +263,20 @@ const io = {
 const accounts = pluginConfig.accounts ?? [];
 if (accounts.length === 0) warn("no accounts configured");
 for (const account of accounts) {
+  // RUNTIME credential health first: the source check below only proves a
+  // credential EXISTS, and #8 is exactly the case where a present credential
+  // is a session the Claude CLI has already rejected. A recorded runtime
+  // failure is the stronger evidence, so it is reported as such.
+  const recorded = readJson(join(STATE_DIR, `${account.id}.json`))?.credential;
+  if (recorded?.status === "failed") {
+    const ageMin = Math.round((Date.now() - recorded.seenAt) / 60000);
+    bad(
+      `${account.id}: the Claude CLI rejected this login ${ageMin}m ago${
+        recorded.reason ? ` (${recorded.reason})` : ""
+      } — excluded from the pool; fix with \`multi-clawd login ${account.id}\``,
+    );
+    continue;
+  }
   if (account.oauthTokenRef) {
     warn(`${account.id}: oauthTokenRef — validated by the gateway's async probe, not doctor`);
     continue;