@drakon-systems/multi-clawd 1.7.3 → 1.7.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -364,7 +364,7 @@ openclaw plugins install (Get-Location).Path
364
364
  **Or let your agent install it.** Running an OpenClaw assistant or Claude
365
365
  Code on the target machine already? Paste it this and go make coffee:
366
366
 
367
- > Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v1.7.3/SETUP-AGENT.md
367
+ > Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v1.7.4/SETUP-AGENT.md
368
368
  > and follow it to set up multi-clawd on this machine. I own a second
369
369
  > Claude account — ask me when you need me to log in.
370
370
 
@@ -66,6 +66,7 @@ const VERDICT_WORDS = {
66
66
  no_data: "no recent telemetry — treated as healthy",
67
67
  near_limit: "NEAR ITS LIMIT — the pool will hand over before it hard-fails",
68
68
  exhausted: "EXHAUSTED",
69
+ credential_failed: "LOGIN REJECTED — excluded from the pool until it is re-authenticated",
69
70
  };
70
71
  export function renderExplanation(model) {
71
72
  const lines = [];
package/dist/health.js CHANGED
@@ -2,6 +2,11 @@ import { modelWindowKey } from "./shim-core.js";
2
2
  const DEFAULT_UTILIZATION_THRESHOLD = 0.85;
3
3
  const DEFAULT_STALE_AFTER_MS = 6 * 60 * 60 * 1000;
4
4
  export const MODEL_REJECTED_TTL_MS = 60 * 60 * 1000;
5
+ export const CREDENTIAL_FAILED_TTL_MS = 15 * 60 * 1000;
6
+ export const REJECTION_REVALIDATE_AFTER_MS = 60 * 60 * 1000;
7
+ function rejectionStillAssertable(seenAt, nowMs) {
8
+ return nowMs - seenAt <= REJECTION_REVALIDATE_AFTER_MS;
9
+ }
5
10
  const MODEL_WINDOW_PREFIX = "model:";
6
11
  const SHORT_WINDOW_PATTERN = /(^|_)hours?(_|$)/;
7
12
  export function isShortWindow(window) {
@@ -15,11 +20,26 @@ export function isWarningStatus(status) {
15
20
  return /warning/i.test(status);
16
21
  }
17
22
  export const MAX_RESET_HORIZON_MS = 8 * 24 * 60 * 60 * 1000;
23
+ export function credentialFailureFor(state, nowMs) {
24
+ const credential = state?.credential;
25
+ if (!credential || credential.status !== "failed")
26
+ return undefined;
27
+ if (nowMs - credential.seenAt > CREDENTIAL_FAILED_TTL_MS)
28
+ return undefined;
29
+ return {
30
+ verdict: "credential_failed",
31
+ resumeAt: credential.seenAt + CREDENTIAL_FAILED_TTL_MS,
32
+ reason: `login rejected by the Claude CLI ${Math.round((nowMs - credential.seenAt) / 60000)}m ago${credential.reason ? `: ${credential.reason}` : ""} — re-authenticate this account`,
33
+ };
34
+ }
18
35
  export function classifyAccountHealth(state, options, nowMs, requestedModel) {
19
36
  const staleAfterMs = options.staleAfterMs ?? DEFAULT_STALE_AFTER_MS;
20
37
  const threshold = options.utilizationThreshold ?? DEFAULT_UTILIZATION_THRESHOLD;
21
38
  if (!state)
22
39
  return { verdict: "no_data" };
40
+ const credentialFailure = credentialFailureFor(state, nowMs);
41
+ if (credentialFailure)
42
+ return credentialFailure;
23
43
  const requestedWindowKey = requestedModel !== undefined ? modelWindowKey(requestedModel) : undefined;
24
44
  let worst = { verdict: "ok" };
25
45
  let hasLiveEvidence = false;
@@ -62,7 +82,10 @@ export function classifyAccountHealth(state, options, nowMs, requestedModel) {
62
82
  if (!fresh)
63
83
  continue;
64
84
  hasLiveEvidence = true;
65
- if (w.status === "rejected" && resetBearing && isPeriodWindow(window)) {
85
+ if (w.status === "rejected" &&
86
+ resetBearing &&
87
+ isPeriodWindow(window) &&
88
+ rejectionStillAssertable(w.seenAt, nowMs)) {
66
89
  return {
67
90
  verdict: "exhausted",
68
91
  resumeAt: resetMs,
@@ -128,6 +151,12 @@ export function choosePoolAccount(pool) {
128
151
  return usable.id;
129
152
  return pool.find((a) => a.verdict === "near_limit")?.id;
130
153
  }
154
+ export function allCredentialFailed(pool) {
155
+ return pool.length > 0 && pool.every((a) => a.verdict === "credential_failed");
156
+ }
157
+ export function fallbackPoolAccount(pool) {
158
+ return (pool.find((a) => a.verdict !== "credential_failed") ?? pool[0]).id;
159
+ }
131
160
  export function pickPoolAccountForLaunch(pool) {
132
- return choosePoolAccount(pool) ?? pool[0].id;
161
+ return choosePoolAccount(pool) ?? fallbackPoolAccount(pool);
133
162
  }
package/dist/index.js CHANGED
@@ -9,8 +9,9 @@ import { MODEL_ALIASES, buildCatalogEntries, canonicalModelId, isModernClaudeMod
9
9
  import { decideDegradation, matchesPin } from "./degrade.js";
10
10
  import { resolveExecMode, permissionModeArgs } from "./exec-policy.js";
11
11
  import { resolveBaseModelIds } from "./catalog-source.js";
12
- import { classifyAccountHealth } from "./health.js";
12
+ import { allCredentialFailed, classifyAccountHealth } from "./health.js";
13
13
  import { decideStickySelection } from "./sticky.js";
14
+ import { clearCredentialFailure, mergeHealthStates, parseStoredState, recordCredentialFailure, } from "./shim-core.js";
14
15
  import { createTokenRefResolver, isSecretRefShape, } from "./token-resolution.js";
15
16
  import { resolveSecretRefValues } from "openclaw/plugin-sdk/secret-ref-runtime";
16
17
  import { addAlert, clearAlert, pendingAlertText } from "./alerts.js";
@@ -116,53 +117,104 @@ const realCredentialIo = {
116
117
  },
117
118
  platform: process.platform,
118
119
  };
119
- function startLoginHealthProbe(accounts, logger) {
120
- if (loginProbeTimer)
121
- clearInterval(loginProbeTimer);
122
- const lastStatus = new Map();
123
- const refTrackers = new Map();
124
- const probe = async () => {
125
- const now = Date.now();
126
- for (const account of accounts) {
127
- let status;
128
- let reason;
129
- if (isSecretRefShape(account.oauthTokenRef) && !account.oauthTokenFile && !account.native) {
130
- let tracker = refTrackers.get(account.id);
131
- if (!tracker) {
132
- tracker = createRefProbeTracker();
133
- refTrackers.set(account.id, tracker);
134
- }
135
- const result = (await activeTokenResolver?.resolveDetailed(account.oauthTokenRef)) ?? {
136
- failure: "provider_error",
137
- };
138
- const outcome = tracker.observe(result, now);
139
- status = outcome.status;
140
- reason = outcome.reason;
141
- }
142
- else {
143
- const check = checkAccountCredential(account, realCredentialIo);
144
- status = check.status;
145
- reason = check.reason;
120
+ const refProbeTrackers = new Map();
121
+ const lastProbeStatus = new Map();
122
+ function recordAccountCredentialFailure(accountId, reason, nowMs) {
123
+ const file = healthStateFile(accountId);
124
+ let state;
125
+ try {
126
+ state = parseStoredState(readFileSync(file, "utf8")) ?? { accountId, windows: {} };
127
+ }
128
+ catch {
129
+ state = { accountId, windows: {} };
130
+ }
131
+ const next = mergeHealthStates(state, recordCredentialFailure(state, reason, nowMs), nowMs);
132
+ try {
133
+ mkdirSync(dirname(file), { recursive: true });
134
+ const tmp = `${file}.tmp-${process.pid}`;
135
+ writeFileSync(tmp, JSON.stringify(next, null, 2), { mode: 0o600 });
136
+ renameSync(tmp, file);
137
+ }
138
+ catch {
139
+ }
140
+ }
141
+ export async function runLoginHealthProbe(accounts, logger, deps = {}) {
142
+ const now = deps.nowMs ?? Date.now();
143
+ const resolver = deps.resolver ?? activeTokenResolver;
144
+ const io = deps.io ?? realCredentialIo;
145
+ for (const account of accounts) {
146
+ let status;
147
+ let reason;
148
+ let cause;
149
+ if (isSecretRefShape(account.oauthTokenRef) && !account.oauthTokenFile && !account.native) {
150
+ let tracker = refProbeTrackers.get(account.id);
151
+ if (!tracker) {
152
+ tracker = createRefProbeTracker();
153
+ refProbeTrackers.set(account.id, tracker);
146
154
  }
147
- const previous = lastStatus.get(account.id);
148
- lastStatus.set(account.id, status);
149
- if (status === "broken" && previous !== "broken") {
150
- const text = `account "${account.id}" login looks dead (${reason ?? "unknown"}) — turns on it will fail until fixed`;
155
+ const result = (await resolver?.resolveDetailed(account.oauthTokenRef)) ?? {
156
+ failure: "provider_error",
157
+ };
158
+ const outcome = tracker.observe(result, now);
159
+ status = outcome.status;
160
+ reason = outcome.reason;
161
+ cause = outcome.cause;
162
+ }
163
+ else {
164
+ const check = checkAccountCredential(account, io);
165
+ status = check.status;
166
+ reason = check.reason;
167
+ }
168
+ const previous = lastProbeStatus.get(account.id);
169
+ lastProbeStatus.set(account.id, status);
170
+ if (status === "broken" && cause === "credential") {
171
+ recordAccountCredentialFailure(account.id, reason ?? "login probe found no credential", now);
172
+ if (previous !== "broken") {
173
+ const text = `account "${account.id}" login looks dead (${reason ?? "unknown"}) — excluded from pool selection until it is fixed`;
151
174
  logger.error(`[multi-clawd] ${text}`);
152
175
  raiseAlert({ key: `login:${account.id}`, severity: "error", text });
153
176
  }
154
- else if (status === "degraded" && previous !== "degraded") {
155
- logger.info(`[multi-clawd] account "${account.id}" login degraded: ${reason ?? "resolver error"}`);
156
- }
157
- else if (status === "ok" && (previous === "broken" || previous === "degraded")) {
158
- logger.info(`[multi-clawd] account "${account.id}" login recovered`);
159
- alertState = clearAlert(alertState, `login:${account.id}`);
177
+ }
178
+ else if (status === "broken" && cause === "provider") {
179
+ if (previous !== "broken") {
180
+ const text = `account "${account.id}" credential resolver is unreachable (${reason ?? "unknown"}) — ` +
181
+ `this looks like a host or network problem rather than a broken login, so account ` +
182
+ `selection is unchanged. Check connectivity to the secret provider.`;
183
+ logger.error(`[multi-clawd] ${text}`);
184
+ raiseAlert({ key: `login-resolver:${account.id}`, severity: "error", text });
160
185
  }
161
186
  }
162
- };
163
- const initial = setTimeout(() => void probe().catch(() => { }), LOGIN_PROBE_INITIAL_DELAY_MS);
187
+ else if (status === "broken" && previous !== "broken") {
188
+ const text = `account "${account.id}" login looks dead (${reason ?? "unknown"}) — turns on it will fail until fixed`;
189
+ logger.error(`[multi-clawd] ${text}`);
190
+ raiseAlert({ key: `login:${account.id}`, severity: "error", text });
191
+ }
192
+ else if (status === "degraded" && previous !== "degraded") {
193
+ logger.info(`[multi-clawd] account "${account.id}" login degraded: ${reason ?? "resolver error"}`);
194
+ }
195
+ else if (status === "ok" && (previous === "broken" || previous === "degraded")) {
196
+ logger.info(`[multi-clawd] account "${account.id}" login recovered`);
197
+ alertState = clearAlert(alertState, `login:${account.id}`);
198
+ alertState = clearAlert(alertState, `login-resolver:${account.id}`);
199
+ }
200
+ }
201
+ }
202
+ export function startLoginHealthProbe(accounts, logger) {
203
+ if (loginProbeTimer)
204
+ clearInterval(loginProbeTimer);
205
+ const live = new Set(accounts.map((a) => a.id));
206
+ for (const id of [...refProbeTrackers.keys()]) {
207
+ if (!live.has(id))
208
+ refProbeTrackers.delete(id);
209
+ }
210
+ for (const id of [...lastProbeStatus.keys()]) {
211
+ if (!live.has(id))
212
+ lastProbeStatus.delete(id);
213
+ }
214
+ const probe = () => void runLoginHealthProbe(accounts, logger).catch(() => { });
215
+ const initial = setTimeout(probe, LOGIN_PROBE_INITIAL_DELAY_MS);
164
216
  initial.unref?.();
165
- loginProbeTimer = setInterval(() => void probe().catch(() => { }), LOGIN_PROBE_INTERVAL_MS);
217
+ loginProbeTimer = setInterval(probe, LOGIN_PROBE_INTERVAL_MS);
166
218
  loginProbeTimer.unref?.();
167
219
  }
168
220
  const warnedTokenFileModes = new Set();
@@ -193,9 +245,9 @@ function peekToken(account) {
193
245
  return undefined;
194
246
  throw new Error(`[multi-clawd] account "${account.id}" needs oauthTokenFile, oauthTokenRef, or configDir`);
195
247
  }
196
- async function resolveTokenAsync(account) {
248
+ async function resolveTokenAsync(account, resolver) {
197
249
  if (isSecretRefShape(account.oauthTokenRef) && !account.native && !account.oauthTokenFile) {
198
- return activeTokenResolver?.resolve(account.oauthTokenRef);
250
+ return (resolver ?? activeTokenResolver)?.resolve(account.oauthTokenRef);
199
251
  }
200
252
  return peekToken(account);
201
253
  }
@@ -330,8 +382,8 @@ export function buildBackend(account, execMode) {
330
382
  },
331
383
  };
332
384
  }
333
- async function buildAccountEnv(account) {
334
- const token = await resolveTokenAsync(account);
385
+ async function buildAccountEnv(account, resolver) {
386
+ const token = await resolveTokenAsync(account, resolver);
335
387
  return buildAccountChildEnv(account, token, healthStateFile(account.id));
336
388
  }
337
389
  function buildCatalogProvider(account) {
@@ -445,6 +497,32 @@ function readHealthState(accountId) {
445
497
  return undefined;
446
498
  }
447
499
  }
500
+ export function clearAccountCredentialFailure(accountId) {
501
+ const file = healthStateFile(accountId);
502
+ let state;
503
+ try {
504
+ state = parseStoredState(readFileSync(file, "utf8")) ?? {
505
+ accountId,
506
+ windows: {},
507
+ };
508
+ }
509
+ catch {
510
+ return false;
511
+ }
512
+ if (state.credential?.status !== "failed")
513
+ return false;
514
+ const cleared = mergeHealthStates(state, clearCredentialFailure(state, Date.now()), Date.now());
515
+ try {
516
+ mkdirSync(dirname(file), { recursive: true });
517
+ const tmp = `${file}.tmp-${process.pid}`;
518
+ writeFileSync(tmp, JSON.stringify(cleared, null, 2), { mode: 0o600 });
519
+ renameSync(tmp, file);
520
+ return true;
521
+ }
522
+ catch {
523
+ return false;
524
+ }
525
+ }
448
526
  function readStickyEntry(file) {
449
527
  try {
450
528
  const parsed = JSON.parse(readFileSync(file, "utf8"));
@@ -471,7 +549,7 @@ function writeStickyEntry(file, entry, logger) {
471
549
  logger.warn(`[multi-clawd] sticky state write failed: ${String(err)}`);
472
550
  }
473
551
  }
474
- export function registerPoolBackend(api, pool, accounts, registeredIds, execMode) {
552
+ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode, deps) {
475
553
  const logger = api.logger;
476
554
  if (!pool)
477
555
  return;
@@ -515,6 +593,17 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
515
593
  id: a.id,
516
594
  health: classifyAccountHealth(readHealthState(a.id), options, now, requestedModel),
517
595
  }));
596
+ if (allCredentialFailed(verdicts.map((v) => ({ id: v.id, verdict: v.health.verdict })))) {
597
+ const detail = verdicts
598
+ .map((v) => `${v.id} (${v.health.reason ?? "credential rejected"})`)
599
+ .join("; ");
600
+ const text = `pool ${poolId}: every account's login is rejected by the Claude CLI — ` +
601
+ `re-authenticate with \`multi-clawd login <account>\`. ${detail}`;
602
+ logger.error(`[multi-clawd] ${text}`);
603
+ raiseAlert({ key: `pool-credentials:${poolId}`, severity: "error", text });
604
+ writeStickyEntry(stickyFile, undefined, logger);
605
+ throw new Error(`[multi-clawd] ${text}`);
606
+ }
518
607
  const previousSticky = readStickyEntry(stickyFile);
519
608
  const decision = decideStickySelection({
520
609
  verdicts: verdicts.map((v) => ({ id: v.id, verdict: v.health.verdict })),
@@ -539,8 +628,45 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
539
628
  text: `pool ${poolId}: every account is exhausted for ${requestedModel} — turns are degrading or falling through the chain`,
540
629
  });
541
630
  }
631
+ for (const v of verdicts) {
632
+ const key = `credential:${poolId}:${v.id}`;
633
+ if (v.health.verdict === "credential_failed") {
634
+ raiseAlert({
635
+ key,
636
+ severity: "error",
637
+ text: `pool ${poolId}: account "${v.id}" is excluded — ${v.health.reason ?? "its login was rejected by the Claude CLI"}. Fix with \`multi-clawd login ${v.id}\`.`,
638
+ });
639
+ }
640
+ else {
641
+ alertState = clearAlert(alertState, key);
642
+ }
643
+ }
644
+ alertState = clearAlert(alertState, `pool-credentials:${poolId}`);
542
645
  writeStickyEntry(stickyFile, decision.sticky, logger);
543
- const env = await buildAccountEnv(chosen);
646
+ const order = [chosen, ...members.filter((m) => m.id !== chosen.id)];
647
+ let env;
648
+ const unresolved = [];
649
+ for (const candidate of order) {
650
+ try {
651
+ env = await buildAccountEnv(candidate, deps?.resolver);
652
+ if (candidate.id !== chosen.id) {
653
+ logger.warn(`[multi-clawd] pool ${poolId}: ${chosen.id}'s credential did not resolve — ` +
654
+ `launching on ${candidate.id} instead (account not benched; secret provider may be down)`);
655
+ }
656
+ break;
657
+ }
658
+ catch (err) {
659
+ unresolved.push(`${candidate.id} (${err.message})`);
660
+ }
661
+ }
662
+ if (!env) {
663
+ const text = `pool ${poolId}: no account's credential could be resolved — ` +
664
+ `the secret provider is unreachable or every reference is empty. ${unresolved.join("; ")}`;
665
+ logger.error(`[multi-clawd] ${text}`);
666
+ raiseAlert({ key: `pool-unresolvable:${poolId}`, severity: "error", text });
667
+ throw new Error(`[multi-clawd] ${text}`);
668
+ }
669
+ alertState = clearAlert(alertState, `pool-unresolvable:${poolId}`);
544
670
  if (ladder.length > 0) {
545
671
  const pinned = matchesPin(pins, {
546
672
  agentDir: ctx.agentDir ?? "",
@@ -15,6 +15,7 @@ export function createRefProbeTracker(options = {}) {
15
15
  firstFailureAt = undefined;
16
16
  return {
17
17
  status: "broken",
18
+ cause: "credential",
18
19
  reason: "oauthTokenRef resolved to nothing (credential problem)",
19
20
  };
20
21
  }
@@ -25,6 +26,7 @@ export function createRefProbeTracker(options = {}) {
25
26
  if (consecutive >= deadAfterConsecutive && elapsed >= deadAfterMs) {
26
27
  return {
27
28
  status: "broken",
29
+ cause: "provider",
28
30
  reason: `resolver failing ${deadAfterConsecutive}+ consecutive probes over ${Math.round(deadAfterMs / 60000)}m`,
29
31
  };
30
32
  }
package/dist/shim-core.js CHANGED
@@ -91,10 +91,22 @@ export function parseStoredState(raw) {
91
91
  rawInfo: typeof w.rawInfo === "string" ? w.rawInfo : undefined,
92
92
  };
93
93
  }
94
+ let credential;
95
+ if (typeof p.credential === "object" && p.credential !== null) {
96
+ const c = p.credential;
97
+ if ((c.status === "failed" || c.status === "ok") && typeof c.seenAt === "number") {
98
+ credential = {
99
+ status: c.status,
100
+ reason: typeof c.reason === "string" ? c.reason : undefined,
101
+ seenAt: c.seenAt,
102
+ };
103
+ }
104
+ }
94
105
  return {
95
106
  accountId: typeof p.accountId === "string" ? p.accountId : "unknown",
96
107
  updatedAt: typeof p.updatedAt === "number" ? p.updatedAt : undefined,
97
108
  windows,
109
+ credential,
98
110
  };
99
111
  }
100
112
  export function mergeHealthStates(disk, live, now, pruneAfterMs = PRUNE_AFTER_MS) {
@@ -119,11 +131,19 @@ export function mergeHealthStates(disk, live, now, pruneAfterMs = PRUNE_AFTER_MS
119
131
  }
120
132
  }
121
133
  }
134
+ let credential = disk.credential;
135
+ if (live.credential && (!credential || live.credential.seenAt >= credential.seenAt)) {
136
+ credential = live.credential;
137
+ }
138
+ if (now !== undefined && credential && now - credential.seenAt > pruneAfterMs) {
139
+ credential = undefined;
140
+ }
122
141
  const updatedAt = Math.max(disk.updatedAt ?? 0, live.updatedAt ?? 0);
123
142
  return {
124
143
  accountId: live.accountId,
125
144
  updatedAt: updatedAt > 0 ? updatedAt : undefined,
126
145
  windows,
146
+ credential,
127
147
  };
128
148
  }
129
149
  const MODEL_ID_PROVIDER_PREFIXES = [
@@ -207,6 +227,70 @@ export function recordModelLimit(state, modelId, now, resetsAt) {
207
227
  },
208
228
  };
209
229
  }
230
+ const AUTH_FAILURE_PATTERNS = [
231
+ /oauth (session|token) (has )?(expired|been revoked)/i,
232
+ /failed to authenticate/i,
233
+ /invalid (api key|bearer token|access token)/i,
234
+ /authentication[_ ]?error/i,
235
+ /please run \/login/i,
236
+ /not logged in/i,
237
+ /unauthori[sz]ed/i,
238
+ ];
239
+ const AUTH_REASON_MAX_CHARS = 200;
240
+ export function parseAuthFailure(line) {
241
+ if (!/(authenticat|oauth|logged in|\/login|unauthori|api key|access token|bearer token)/i.test(line)) {
242
+ return undefined;
243
+ }
244
+ let record;
245
+ try {
246
+ record = JSON.parse(line);
247
+ }
248
+ catch {
249
+ return undefined;
250
+ }
251
+ if (typeof record !== "object" || record === null)
252
+ return undefined;
253
+ const r = record;
254
+ const isErrorRecord = r.type === "error" ||
255
+ r.is_error === true ||
256
+ (typeof r.subtype === "string" && r.subtype.startsWith("error"));
257
+ if (!isErrorRecord)
258
+ return undefined;
259
+ const texts = [];
260
+ if (typeof r.result === "string")
261
+ texts.push(r.result);
262
+ if (typeof r.error === "string")
263
+ texts.push(r.error);
264
+ if (typeof r.error === "object" && r.error !== null) {
265
+ const msg = r.error.message;
266
+ if (typeof msg === "string")
267
+ texts.push(msg);
268
+ }
269
+ for (const text of texts) {
270
+ if (AUTH_FAILURE_PATTERNS.some((p) => p.test(text))) {
271
+ return { reason: text.trim().slice(0, AUTH_REASON_MAX_CHARS) };
272
+ }
273
+ }
274
+ return undefined;
275
+ }
276
+ export function recordCredentialFailure(state, reason, now) {
277
+ return {
278
+ ...state,
279
+ updatedAt: now,
280
+ credential: {
281
+ status: "failed",
282
+ reason: reason.slice(0, AUTH_REASON_MAX_CHARS),
283
+ seenAt: now,
284
+ },
285
+ };
286
+ }
287
+ export function clearCredentialFailure(state, now) {
288
+ return {
289
+ ...state,
290
+ updatedAt: now,
291
+ credential: { status: "ok", seenAt: now },
292
+ };
293
+ }
210
294
  export function updateHealthState(state, event, now) {
211
295
  const key = event.rateLimitType ?? "unknown";
212
296
  return {
package/dist/shim.js CHANGED
@@ -1,7 +1,7 @@
1
1
  import { spawn } from "node:child_process";
2
2
  import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
3
3
  import { dirname } from "node:path";
4
- import { classifyStateReadFailure, createLineScanner, mergeHealthStates, parseRateLimitEvent, parseStoredState, updateHealthState, } from "./shim-core.js";
4
+ import { classifyStateReadFailure, clearCredentialFailure, createLineScanner, mergeHealthStates, parseAuthFailure, parseRateLimitEvent, parseStoredState, recordCredentialFailure, updateHealthState, } from "./shim-core.js";
5
5
  import { rewriteModelArg } from "./degrade.js";
6
6
  import { parseModelLimitError, recordModelLimit } from "./shim-core.js";
7
7
  import { canonicalModelId } from "./models.js";
@@ -108,6 +108,7 @@ function guessLimitResetsAt() {
108
108
  }
109
109
  return undefined;
110
110
  }
111
+ let sawAuthFailure = false;
111
112
  const scanner = createLineScanner((line) => {
112
113
  try {
113
114
  const event = parseRateLimitEvent(line);
@@ -124,6 +125,13 @@ const scanner = createLineScanner((line) => {
124
125
  process.stderr.write(`[multi-clawd shim] model limit hit recorded: ${model} (reported as "${limitHit.displayName}")\n`);
125
126
  }
126
127
  }
128
+ const authFailure = parseAuthFailure(line);
129
+ if (authFailure && !sawAuthFailure) {
130
+ sawAuthFailure = true;
131
+ state = recordCredentialFailure(state, authFailure.reason, Date.now());
132
+ persistState();
133
+ process.stderr.write(`[multi-clawd shim] auth failure recorded for ${accountId}: ${authFailure.reason}\n`);
134
+ }
127
135
  }
128
136
  catch {
129
137
  }
@@ -138,11 +146,26 @@ for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
138
146
  child.kill(signal);
139
147
  });
140
148
  }
149
+ function clearRecordedAuthFailureOnSuccess() {
150
+ if (!stateFile || sawAuthFailure)
151
+ return;
152
+ try {
153
+ if (readPersistedState()?.credential?.status !== "failed")
154
+ return;
155
+ state = clearCredentialFailure(state, Date.now());
156
+ persistState();
157
+ process.stderr.write(`[multi-clawd shim] auth recovered for ${accountId} — credential exclusion cleared\n`);
158
+ }
159
+ catch {
160
+ }
161
+ }
141
162
  child.on("close", (code, signal) => {
142
163
  if (signal) {
143
164
  process.kill(process.pid, signal);
144
165
  return;
145
166
  }
167
+ if ((code ?? 0) === 0)
168
+ clearRecordedAuthFailureOnSuccess();
146
169
  process.exit(code ?? 0);
147
170
  });
148
171
  child.on("error", (err) => {
package/dist/sticky.js CHANGED
@@ -1,4 +1,4 @@
1
- import { choosePoolAccount } from "./health.js";
1
+ import { choosePoolAccount, fallbackPoolAccount } from "./health.js";
2
2
  export const DEFAULT_MIN_DWELL_MS = 10 * 60 * 1000;
3
3
  export function decideStickySelection(params) {
4
4
  const { verdicts, sticky, nowMs } = params;
@@ -6,7 +6,7 @@ export function decideStickySelection(params) {
6
6
  const home = verdicts[0];
7
7
  const healthChoice = choosePoolAccount(verdicts);
8
8
  if (!healthChoice)
9
- return { account: home.id };
9
+ return { account: fallbackPoolAccount(verdicts) };
10
10
  const stickyVerdict = sticky
11
11
  ? verdicts.find((v) => v.id === sticky.account)?.verdict
12
12
  : undefined;
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "id": "multi-clawd",
3
3
  "name": "multi-clawd",
4
- "version": "1.7.3",
4
+ "version": "1.7.4",
5
5
  "description": "Register additional Claude Code logins (Max/Pro accounts) as first-class OpenClaw CLI backends for cross-account failover, keeping the full skills/MCP harness on every account.",
6
6
  "cliBackends": [
7
7
  "claw1",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@drakon-systems/multi-clawd",
3
- "version": "1.7.3",
3
+ "version": "1.7.4",
4
4
  "description": "Multi-account Claude Code failover for OpenClaw — register additional Claude (Max/Pro) logins as first-class CLI backends and keep the full skills/MCP harness across every account. Also imports those accounts' setup tokens into Hermes Agent's Anthropic credential pool.",
5
5
  "type": "module",
6
6
  "license": "MIT",
package/scripts/cli.mjs CHANGED
@@ -461,10 +461,11 @@ async function login() {
461
461
  const { readFileSync: rf, existsSync, mkdirSync, chmodSync, statSync, mkdtempSync, rmSync } =
462
462
  await import("node:fs");
463
463
  const { homedir, tmpdir } = await import("node:os");
464
- let lp, ec;
464
+ let lp, ec, idx;
465
465
  try {
466
466
  lp = await import(resolve(__dirname, "..", "dist", "login-plan.js"));
467
467
  ec = await import(resolve(__dirname, "..", "dist", "explain-core.js"));
468
+ idx = await import(resolve(__dirname, "..", "dist", "index.js"));
468
469
  } catch {
469
470
  console.error("login: built dist/ is missing — reinstall the package.");
470
471
  process.exit(1);
@@ -513,6 +514,17 @@ async function login() {
513
514
  console.error(`\n ❌ ${plan.command.join(" ")} exited with ${r.status ?? "an error"}.`);
514
515
  process.exit(1);
515
516
  }
517
+ // Explicit re-authentication ends any recorded runtime credential failure
518
+ // immediately (#8) — without this the freshly re-authed account stays
519
+ // benched until the 15-minute TTL expires, which reads to the operator as
520
+ // "logging back in did nothing".
521
+ try {
522
+ if (idx.clearAccountCredentialFailure?.(acc.id)) {
523
+ console.log(`\n ↻ cleared ${acc.id}'s recorded login failure — the pool can use it again.`);
524
+ }
525
+ } catch {
526
+ /* clearing is a courtesy; never fail a login over it */
527
+ }
516
528
  if (plan.verify === "auth-status") {
517
529
  try {
518
530
  const out = spawnSync("claude", ["auth", "status"], { encoding: "utf8", env }).stdout ?? "";
@@ -263,6 +263,20 @@ const io = {
263
263
  const accounts = pluginConfig.accounts ?? [];
264
264
  if (accounts.length === 0) warn("no accounts configured");
265
265
  for (const account of accounts) {
266
+ // RUNTIME credential health first: the source check below only proves a
267
+ // credential EXISTS, and #8 is exactly the case where a present credential
268
+ // is a session the Claude CLI has already rejected. A recorded runtime
269
+ // failure is the stronger evidence, so it is reported as such.
270
+ const recorded = readJson(join(STATE_DIR, `${account.id}.json`))?.credential;
271
+ if (recorded?.status === "failed") {
272
+ const ageMin = Math.round((Date.now() - recorded.seenAt) / 60000);
273
+ bad(
274
+ `${account.id}: the Claude CLI rejected this login ${ageMin}m ago${
275
+ recorded.reason ? ` (${recorded.reason})` : ""
276
+ } — excluded from the pool; fix with \`multi-clawd login ${account.id}\``,
277
+ );
278
+ continue;
279
+ }
266
280
  if (account.oauthTokenRef) {
267
281
  warn(`${account.id}: oauthTokenRef — validated by the gateway's async probe, not doctor`);
268
282
  continue;