@drakon-systems/multi-clawd 1.7.3 → 1.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/explain-core.js +1 -0
- package/dist/health.js +31 -2
- package/dist/index.js +173 -47
- package/dist/login-health.js +2 -0
- package/dist/shim-core.js +84 -0
- package/dist/shim.js +24 -1
- package/dist/sticky.js +2 -2
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
- package/scripts/cli.mjs +13 -1
- package/scripts/doctor.mjs +14 -0
package/README.md
CHANGED
|
@@ -364,7 +364,7 @@ openclaw plugins install (Get-Location).Path
|
|
|
364
364
|
**Or let your agent install it.** Running an OpenClaw assistant or Claude
|
|
365
365
|
Code on the target machine already? Paste it this and go make coffee:
|
|
366
366
|
|
|
367
|
-
> Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v1.7.
|
|
367
|
+
> Read https://raw.githubusercontent.com/Drakon-Systems-Ltd/multi-clawd/v1.7.4/SETUP-AGENT.md
|
|
368
368
|
> and follow it to set up multi-clawd on this machine. I own a second
|
|
369
369
|
> Claude account — ask me when you need me to log in.
|
|
370
370
|
|
package/dist/explain-core.js
CHANGED
|
@@ -66,6 +66,7 @@ const VERDICT_WORDS = {
|
|
|
66
66
|
no_data: "no recent telemetry — treated as healthy",
|
|
67
67
|
near_limit: "NEAR ITS LIMIT — the pool will hand over before it hard-fails",
|
|
68
68
|
exhausted: "EXHAUSTED",
|
|
69
|
+
credential_failed: "LOGIN REJECTED — excluded from the pool until it is re-authenticated",
|
|
69
70
|
};
|
|
70
71
|
export function renderExplanation(model) {
|
|
71
72
|
const lines = [];
|
package/dist/health.js
CHANGED
|
@@ -2,6 +2,11 @@ import { modelWindowKey } from "./shim-core.js";
|
|
|
2
2
|
const DEFAULT_UTILIZATION_THRESHOLD = 0.85;
|
|
3
3
|
const DEFAULT_STALE_AFTER_MS = 6 * 60 * 60 * 1000;
|
|
4
4
|
export const MODEL_REJECTED_TTL_MS = 60 * 60 * 1000;
|
|
5
|
+
export const CREDENTIAL_FAILED_TTL_MS = 15 * 60 * 1000;
|
|
6
|
+
export const REJECTION_REVALIDATE_AFTER_MS = 60 * 60 * 1000;
|
|
7
|
+
function rejectionStillAssertable(seenAt, nowMs) {
|
|
8
|
+
return nowMs - seenAt <= REJECTION_REVALIDATE_AFTER_MS;
|
|
9
|
+
}
|
|
5
10
|
const MODEL_WINDOW_PREFIX = "model:";
|
|
6
11
|
const SHORT_WINDOW_PATTERN = /(^|_)hours?(_|$)/;
|
|
7
12
|
export function isShortWindow(window) {
|
|
@@ -15,11 +20,26 @@ export function isWarningStatus(status) {
|
|
|
15
20
|
return /warning/i.test(status);
|
|
16
21
|
}
|
|
17
22
|
export const MAX_RESET_HORIZON_MS = 8 * 24 * 60 * 60 * 1000;
|
|
23
|
+
export function credentialFailureFor(state, nowMs) {
|
|
24
|
+
const credential = state?.credential;
|
|
25
|
+
if (!credential || credential.status !== "failed")
|
|
26
|
+
return undefined;
|
|
27
|
+
if (nowMs - credential.seenAt > CREDENTIAL_FAILED_TTL_MS)
|
|
28
|
+
return undefined;
|
|
29
|
+
return {
|
|
30
|
+
verdict: "credential_failed",
|
|
31
|
+
resumeAt: credential.seenAt + CREDENTIAL_FAILED_TTL_MS,
|
|
32
|
+
reason: `login rejected by the Claude CLI ${Math.round((nowMs - credential.seenAt) / 60000)}m ago${credential.reason ? `: ${credential.reason}` : ""} — re-authenticate this account`,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
18
35
|
export function classifyAccountHealth(state, options, nowMs, requestedModel) {
|
|
19
36
|
const staleAfterMs = options.staleAfterMs ?? DEFAULT_STALE_AFTER_MS;
|
|
20
37
|
const threshold = options.utilizationThreshold ?? DEFAULT_UTILIZATION_THRESHOLD;
|
|
21
38
|
if (!state)
|
|
22
39
|
return { verdict: "no_data" };
|
|
40
|
+
const credentialFailure = credentialFailureFor(state, nowMs);
|
|
41
|
+
if (credentialFailure)
|
|
42
|
+
return credentialFailure;
|
|
23
43
|
const requestedWindowKey = requestedModel !== undefined ? modelWindowKey(requestedModel) : undefined;
|
|
24
44
|
let worst = { verdict: "ok" };
|
|
25
45
|
let hasLiveEvidence = false;
|
|
@@ -62,7 +82,10 @@ export function classifyAccountHealth(state, options, nowMs, requestedModel) {
|
|
|
62
82
|
if (!fresh)
|
|
63
83
|
continue;
|
|
64
84
|
hasLiveEvidence = true;
|
|
65
|
-
if (w.status === "rejected" &&
|
|
85
|
+
if (w.status === "rejected" &&
|
|
86
|
+
resetBearing &&
|
|
87
|
+
isPeriodWindow(window) &&
|
|
88
|
+
rejectionStillAssertable(w.seenAt, nowMs)) {
|
|
66
89
|
return {
|
|
67
90
|
verdict: "exhausted",
|
|
68
91
|
resumeAt: resetMs,
|
|
@@ -128,6 +151,12 @@ export function choosePoolAccount(pool) {
|
|
|
128
151
|
return usable.id;
|
|
129
152
|
return pool.find((a) => a.verdict === "near_limit")?.id;
|
|
130
153
|
}
|
|
154
|
+
export function allCredentialFailed(pool) {
|
|
155
|
+
return pool.length > 0 && pool.every((a) => a.verdict === "credential_failed");
|
|
156
|
+
}
|
|
157
|
+
export function fallbackPoolAccount(pool) {
|
|
158
|
+
return (pool.find((a) => a.verdict !== "credential_failed") ?? pool[0]).id;
|
|
159
|
+
}
|
|
131
160
|
export function pickPoolAccountForLaunch(pool) {
|
|
132
|
-
return choosePoolAccount(pool) ?? pool
|
|
161
|
+
return choosePoolAccount(pool) ?? fallbackPoolAccount(pool);
|
|
133
162
|
}
|
package/dist/index.js
CHANGED
|
@@ -9,8 +9,9 @@ import { MODEL_ALIASES, buildCatalogEntries, canonicalModelId, isModernClaudeMod
|
|
|
9
9
|
import { decideDegradation, matchesPin } from "./degrade.js";
|
|
10
10
|
import { resolveExecMode, permissionModeArgs } from "./exec-policy.js";
|
|
11
11
|
import { resolveBaseModelIds } from "./catalog-source.js";
|
|
12
|
-
import { classifyAccountHealth } from "./health.js";
|
|
12
|
+
import { allCredentialFailed, classifyAccountHealth } from "./health.js";
|
|
13
13
|
import { decideStickySelection } from "./sticky.js";
|
|
14
|
+
import { clearCredentialFailure, mergeHealthStates, parseStoredState, recordCredentialFailure, } from "./shim-core.js";
|
|
14
15
|
import { createTokenRefResolver, isSecretRefShape, } from "./token-resolution.js";
|
|
15
16
|
import { resolveSecretRefValues } from "openclaw/plugin-sdk/secret-ref-runtime";
|
|
16
17
|
import { addAlert, clearAlert, pendingAlertText } from "./alerts.js";
|
|
@@ -116,53 +117,104 @@ const realCredentialIo = {
|
|
|
116
117
|
},
|
|
117
118
|
platform: process.platform,
|
|
118
119
|
};
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
const
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
120
|
+
const refProbeTrackers = new Map();
|
|
121
|
+
const lastProbeStatus = new Map();
|
|
122
|
+
function recordAccountCredentialFailure(accountId, reason, nowMs) {
|
|
123
|
+
const file = healthStateFile(accountId);
|
|
124
|
+
let state;
|
|
125
|
+
try {
|
|
126
|
+
state = parseStoredState(readFileSync(file, "utf8")) ?? { accountId, windows: {} };
|
|
127
|
+
}
|
|
128
|
+
catch {
|
|
129
|
+
state = { accountId, windows: {} };
|
|
130
|
+
}
|
|
131
|
+
const next = mergeHealthStates(state, recordCredentialFailure(state, reason, nowMs), nowMs);
|
|
132
|
+
try {
|
|
133
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
134
|
+
const tmp = `${file}.tmp-${process.pid}`;
|
|
135
|
+
writeFileSync(tmp, JSON.stringify(next, null, 2), { mode: 0o600 });
|
|
136
|
+
renameSync(tmp, file);
|
|
137
|
+
}
|
|
138
|
+
catch {
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
export async function runLoginHealthProbe(accounts, logger, deps = {}) {
|
|
142
|
+
const now = deps.nowMs ?? Date.now();
|
|
143
|
+
const resolver = deps.resolver ?? activeTokenResolver;
|
|
144
|
+
const io = deps.io ?? realCredentialIo;
|
|
145
|
+
for (const account of accounts) {
|
|
146
|
+
let status;
|
|
147
|
+
let reason;
|
|
148
|
+
let cause;
|
|
149
|
+
if (isSecretRefShape(account.oauthTokenRef) && !account.oauthTokenFile && !account.native) {
|
|
150
|
+
let tracker = refProbeTrackers.get(account.id);
|
|
151
|
+
if (!tracker) {
|
|
152
|
+
tracker = createRefProbeTracker();
|
|
153
|
+
refProbeTrackers.set(account.id, tracker);
|
|
146
154
|
}
|
|
147
|
-
const
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
155
|
+
const result = (await resolver?.resolveDetailed(account.oauthTokenRef)) ?? {
|
|
156
|
+
failure: "provider_error",
|
|
157
|
+
};
|
|
158
|
+
const outcome = tracker.observe(result, now);
|
|
159
|
+
status = outcome.status;
|
|
160
|
+
reason = outcome.reason;
|
|
161
|
+
cause = outcome.cause;
|
|
162
|
+
}
|
|
163
|
+
else {
|
|
164
|
+
const check = checkAccountCredential(account, io);
|
|
165
|
+
status = check.status;
|
|
166
|
+
reason = check.reason;
|
|
167
|
+
}
|
|
168
|
+
const previous = lastProbeStatus.get(account.id);
|
|
169
|
+
lastProbeStatus.set(account.id, status);
|
|
170
|
+
if (status === "broken" && cause === "credential") {
|
|
171
|
+
recordAccountCredentialFailure(account.id, reason ?? "login probe found no credential", now);
|
|
172
|
+
if (previous !== "broken") {
|
|
173
|
+
const text = `account "${account.id}" login looks dead (${reason ?? "unknown"}) — excluded from pool selection until it is fixed`;
|
|
151
174
|
logger.error(`[multi-clawd] ${text}`);
|
|
152
175
|
raiseAlert({ key: `login:${account.id}`, severity: "error", text });
|
|
153
176
|
}
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
177
|
+
}
|
|
178
|
+
else if (status === "broken" && cause === "provider") {
|
|
179
|
+
if (previous !== "broken") {
|
|
180
|
+
const text = `account "${account.id}" credential resolver is unreachable (${reason ?? "unknown"}) — ` +
|
|
181
|
+
`this looks like a host or network problem rather than a broken login, so account ` +
|
|
182
|
+
`selection is unchanged. Check connectivity to the secret provider.`;
|
|
183
|
+
logger.error(`[multi-clawd] ${text}`);
|
|
184
|
+
raiseAlert({ key: `login-resolver:${account.id}`, severity: "error", text });
|
|
160
185
|
}
|
|
161
186
|
}
|
|
162
|
-
|
|
163
|
-
|
|
187
|
+
else if (status === "broken" && previous !== "broken") {
|
|
188
|
+
const text = `account "${account.id}" login looks dead (${reason ?? "unknown"}) — turns on it will fail until fixed`;
|
|
189
|
+
logger.error(`[multi-clawd] ${text}`);
|
|
190
|
+
raiseAlert({ key: `login:${account.id}`, severity: "error", text });
|
|
191
|
+
}
|
|
192
|
+
else if (status === "degraded" && previous !== "degraded") {
|
|
193
|
+
logger.info(`[multi-clawd] account "${account.id}" login degraded: ${reason ?? "resolver error"}`);
|
|
194
|
+
}
|
|
195
|
+
else if (status === "ok" && (previous === "broken" || previous === "degraded")) {
|
|
196
|
+
logger.info(`[multi-clawd] account "${account.id}" login recovered`);
|
|
197
|
+
alertState = clearAlert(alertState, `login:${account.id}`);
|
|
198
|
+
alertState = clearAlert(alertState, `login-resolver:${account.id}`);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
export function startLoginHealthProbe(accounts, logger) {
|
|
203
|
+
if (loginProbeTimer)
|
|
204
|
+
clearInterval(loginProbeTimer);
|
|
205
|
+
const live = new Set(accounts.map((a) => a.id));
|
|
206
|
+
for (const id of [...refProbeTrackers.keys()]) {
|
|
207
|
+
if (!live.has(id))
|
|
208
|
+
refProbeTrackers.delete(id);
|
|
209
|
+
}
|
|
210
|
+
for (const id of [...lastProbeStatus.keys()]) {
|
|
211
|
+
if (!live.has(id))
|
|
212
|
+
lastProbeStatus.delete(id);
|
|
213
|
+
}
|
|
214
|
+
const probe = () => void runLoginHealthProbe(accounts, logger).catch(() => { });
|
|
215
|
+
const initial = setTimeout(probe, LOGIN_PROBE_INITIAL_DELAY_MS);
|
|
164
216
|
initial.unref?.();
|
|
165
|
-
loginProbeTimer = setInterval(
|
|
217
|
+
loginProbeTimer = setInterval(probe, LOGIN_PROBE_INTERVAL_MS);
|
|
166
218
|
loginProbeTimer.unref?.();
|
|
167
219
|
}
|
|
168
220
|
const warnedTokenFileModes = new Set();
|
|
@@ -193,9 +245,9 @@ function peekToken(account) {
|
|
|
193
245
|
return undefined;
|
|
194
246
|
throw new Error(`[multi-clawd] account "${account.id}" needs oauthTokenFile, oauthTokenRef, or configDir`);
|
|
195
247
|
}
|
|
196
|
-
async function resolveTokenAsync(account) {
|
|
248
|
+
async function resolveTokenAsync(account, resolver) {
|
|
197
249
|
if (isSecretRefShape(account.oauthTokenRef) && !account.native && !account.oauthTokenFile) {
|
|
198
|
-
return activeTokenResolver?.resolve(account.oauthTokenRef);
|
|
250
|
+
return (resolver ?? activeTokenResolver)?.resolve(account.oauthTokenRef);
|
|
199
251
|
}
|
|
200
252
|
return peekToken(account);
|
|
201
253
|
}
|
|
@@ -330,8 +382,8 @@ export function buildBackend(account, execMode) {
|
|
|
330
382
|
},
|
|
331
383
|
};
|
|
332
384
|
}
|
|
333
|
-
async function buildAccountEnv(account) {
|
|
334
|
-
const token = await resolveTokenAsync(account);
|
|
385
|
+
async function buildAccountEnv(account, resolver) {
|
|
386
|
+
const token = await resolveTokenAsync(account, resolver);
|
|
335
387
|
return buildAccountChildEnv(account, token, healthStateFile(account.id));
|
|
336
388
|
}
|
|
337
389
|
function buildCatalogProvider(account) {
|
|
@@ -445,6 +497,32 @@ function readHealthState(accountId) {
|
|
|
445
497
|
return undefined;
|
|
446
498
|
}
|
|
447
499
|
}
|
|
500
|
+
export function clearAccountCredentialFailure(accountId) {
|
|
501
|
+
const file = healthStateFile(accountId);
|
|
502
|
+
let state;
|
|
503
|
+
try {
|
|
504
|
+
state = parseStoredState(readFileSync(file, "utf8")) ?? {
|
|
505
|
+
accountId,
|
|
506
|
+
windows: {},
|
|
507
|
+
};
|
|
508
|
+
}
|
|
509
|
+
catch {
|
|
510
|
+
return false;
|
|
511
|
+
}
|
|
512
|
+
if (state.credential?.status !== "failed")
|
|
513
|
+
return false;
|
|
514
|
+
const cleared = mergeHealthStates(state, clearCredentialFailure(state, Date.now()), Date.now());
|
|
515
|
+
try {
|
|
516
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
517
|
+
const tmp = `${file}.tmp-${process.pid}`;
|
|
518
|
+
writeFileSync(tmp, JSON.stringify(cleared, null, 2), { mode: 0o600 });
|
|
519
|
+
renameSync(tmp, file);
|
|
520
|
+
return true;
|
|
521
|
+
}
|
|
522
|
+
catch {
|
|
523
|
+
return false;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
448
526
|
function readStickyEntry(file) {
|
|
449
527
|
try {
|
|
450
528
|
const parsed = JSON.parse(readFileSync(file, "utf8"));
|
|
@@ -471,7 +549,7 @@ function writeStickyEntry(file, entry, logger) {
|
|
|
471
549
|
logger.warn(`[multi-clawd] sticky state write failed: ${String(err)}`);
|
|
472
550
|
}
|
|
473
551
|
}
|
|
474
|
-
export function registerPoolBackend(api, pool, accounts, registeredIds, execMode) {
|
|
552
|
+
export function registerPoolBackend(api, pool, accounts, registeredIds, execMode, deps) {
|
|
475
553
|
const logger = api.logger;
|
|
476
554
|
if (!pool)
|
|
477
555
|
return;
|
|
@@ -515,6 +593,17 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
515
593
|
id: a.id,
|
|
516
594
|
health: classifyAccountHealth(readHealthState(a.id), options, now, requestedModel),
|
|
517
595
|
}));
|
|
596
|
+
if (allCredentialFailed(verdicts.map((v) => ({ id: v.id, verdict: v.health.verdict })))) {
|
|
597
|
+
const detail = verdicts
|
|
598
|
+
.map((v) => `${v.id} (${v.health.reason ?? "credential rejected"})`)
|
|
599
|
+
.join("; ");
|
|
600
|
+
const text = `pool ${poolId}: every account's login is rejected by the Claude CLI — ` +
|
|
601
|
+
`re-authenticate with \`multi-clawd login <account>\`. ${detail}`;
|
|
602
|
+
logger.error(`[multi-clawd] ${text}`);
|
|
603
|
+
raiseAlert({ key: `pool-credentials:${poolId}`, severity: "error", text });
|
|
604
|
+
writeStickyEntry(stickyFile, undefined, logger);
|
|
605
|
+
throw new Error(`[multi-clawd] ${text}`);
|
|
606
|
+
}
|
|
518
607
|
const previousSticky = readStickyEntry(stickyFile);
|
|
519
608
|
const decision = decideStickySelection({
|
|
520
609
|
verdicts: verdicts.map((v) => ({ id: v.id, verdict: v.health.verdict })),
|
|
@@ -539,8 +628,45 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
539
628
|
text: `pool ${poolId}: every account is exhausted for ${requestedModel} — turns are degrading or falling through the chain`,
|
|
540
629
|
});
|
|
541
630
|
}
|
|
631
|
+
for (const v of verdicts) {
|
|
632
|
+
const key = `credential:${poolId}:${v.id}`;
|
|
633
|
+
if (v.health.verdict === "credential_failed") {
|
|
634
|
+
raiseAlert({
|
|
635
|
+
key,
|
|
636
|
+
severity: "error",
|
|
637
|
+
text: `pool ${poolId}: account "${v.id}" is excluded — ${v.health.reason ?? "its login was rejected by the Claude CLI"}. Fix with \`multi-clawd login ${v.id}\`.`,
|
|
638
|
+
});
|
|
639
|
+
}
|
|
640
|
+
else {
|
|
641
|
+
alertState = clearAlert(alertState, key);
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
alertState = clearAlert(alertState, `pool-credentials:${poolId}`);
|
|
542
645
|
writeStickyEntry(stickyFile, decision.sticky, logger);
|
|
543
|
-
const
|
|
646
|
+
const order = [chosen, ...members.filter((m) => m.id !== chosen.id)];
|
|
647
|
+
let env;
|
|
648
|
+
const unresolved = [];
|
|
649
|
+
for (const candidate of order) {
|
|
650
|
+
try {
|
|
651
|
+
env = await buildAccountEnv(candidate, deps?.resolver);
|
|
652
|
+
if (candidate.id !== chosen.id) {
|
|
653
|
+
logger.warn(`[multi-clawd] pool ${poolId}: ${chosen.id}'s credential did not resolve — ` +
|
|
654
|
+
`launching on ${candidate.id} instead (account not benched; secret provider may be down)`);
|
|
655
|
+
}
|
|
656
|
+
break;
|
|
657
|
+
}
|
|
658
|
+
catch (err) {
|
|
659
|
+
unresolved.push(`${candidate.id} (${err.message})`);
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
if (!env) {
|
|
663
|
+
const text = `pool ${poolId}: no account's credential could be resolved — ` +
|
|
664
|
+
`the secret provider is unreachable or every reference is empty. ${unresolved.join("; ")}`;
|
|
665
|
+
logger.error(`[multi-clawd] ${text}`);
|
|
666
|
+
raiseAlert({ key: `pool-unresolvable:${poolId}`, severity: "error", text });
|
|
667
|
+
throw new Error(`[multi-clawd] ${text}`);
|
|
668
|
+
}
|
|
669
|
+
alertState = clearAlert(alertState, `pool-unresolvable:${poolId}`);
|
|
544
670
|
if (ladder.length > 0) {
|
|
545
671
|
const pinned = matchesPin(pins, {
|
|
546
672
|
agentDir: ctx.agentDir ?? "",
|
package/dist/login-health.js
CHANGED
|
@@ -15,6 +15,7 @@ export function createRefProbeTracker(options = {}) {
|
|
|
15
15
|
firstFailureAt = undefined;
|
|
16
16
|
return {
|
|
17
17
|
status: "broken",
|
|
18
|
+
cause: "credential",
|
|
18
19
|
reason: "oauthTokenRef resolved to nothing (credential problem)",
|
|
19
20
|
};
|
|
20
21
|
}
|
|
@@ -25,6 +26,7 @@ export function createRefProbeTracker(options = {}) {
|
|
|
25
26
|
if (consecutive >= deadAfterConsecutive && elapsed >= deadAfterMs) {
|
|
26
27
|
return {
|
|
27
28
|
status: "broken",
|
|
29
|
+
cause: "provider",
|
|
28
30
|
reason: `resolver failing ${deadAfterConsecutive}+ consecutive probes over ${Math.round(deadAfterMs / 60000)}m`,
|
|
29
31
|
};
|
|
30
32
|
}
|
package/dist/shim-core.js
CHANGED
|
@@ -91,10 +91,22 @@ export function parseStoredState(raw) {
|
|
|
91
91
|
rawInfo: typeof w.rawInfo === "string" ? w.rawInfo : undefined,
|
|
92
92
|
};
|
|
93
93
|
}
|
|
94
|
+
let credential;
|
|
95
|
+
if (typeof p.credential === "object" && p.credential !== null) {
|
|
96
|
+
const c = p.credential;
|
|
97
|
+
if ((c.status === "failed" || c.status === "ok") && typeof c.seenAt === "number") {
|
|
98
|
+
credential = {
|
|
99
|
+
status: c.status,
|
|
100
|
+
reason: typeof c.reason === "string" ? c.reason : undefined,
|
|
101
|
+
seenAt: c.seenAt,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
}
|
|
94
105
|
return {
|
|
95
106
|
accountId: typeof p.accountId === "string" ? p.accountId : "unknown",
|
|
96
107
|
updatedAt: typeof p.updatedAt === "number" ? p.updatedAt : undefined,
|
|
97
108
|
windows,
|
|
109
|
+
credential,
|
|
98
110
|
};
|
|
99
111
|
}
|
|
100
112
|
export function mergeHealthStates(disk, live, now, pruneAfterMs = PRUNE_AFTER_MS) {
|
|
@@ -119,11 +131,19 @@ export function mergeHealthStates(disk, live, now, pruneAfterMs = PRUNE_AFTER_MS
|
|
|
119
131
|
}
|
|
120
132
|
}
|
|
121
133
|
}
|
|
134
|
+
let credential = disk.credential;
|
|
135
|
+
if (live.credential && (!credential || live.credential.seenAt >= credential.seenAt)) {
|
|
136
|
+
credential = live.credential;
|
|
137
|
+
}
|
|
138
|
+
if (now !== undefined && credential && now - credential.seenAt > pruneAfterMs) {
|
|
139
|
+
credential = undefined;
|
|
140
|
+
}
|
|
122
141
|
const updatedAt = Math.max(disk.updatedAt ?? 0, live.updatedAt ?? 0);
|
|
123
142
|
return {
|
|
124
143
|
accountId: live.accountId,
|
|
125
144
|
updatedAt: updatedAt > 0 ? updatedAt : undefined,
|
|
126
145
|
windows,
|
|
146
|
+
credential,
|
|
127
147
|
};
|
|
128
148
|
}
|
|
129
149
|
const MODEL_ID_PROVIDER_PREFIXES = [
|
|
@@ -207,6 +227,70 @@ export function recordModelLimit(state, modelId, now, resetsAt) {
|
|
|
207
227
|
},
|
|
208
228
|
};
|
|
209
229
|
}
|
|
230
|
+
const AUTH_FAILURE_PATTERNS = [
|
|
231
|
+
/oauth (session|token) (has )?(expired|been revoked)/i,
|
|
232
|
+
/failed to authenticate/i,
|
|
233
|
+
/invalid (api key|bearer token|access token)/i,
|
|
234
|
+
/authentication[_ ]?error/i,
|
|
235
|
+
/please run \/login/i,
|
|
236
|
+
/not logged in/i,
|
|
237
|
+
/unauthori[sz]ed/i,
|
|
238
|
+
];
|
|
239
|
+
const AUTH_REASON_MAX_CHARS = 200;
|
|
240
|
+
export function parseAuthFailure(line) {
|
|
241
|
+
if (!/(authenticat|oauth|logged in|\/login|unauthori|api key|access token|bearer token)/i.test(line)) {
|
|
242
|
+
return undefined;
|
|
243
|
+
}
|
|
244
|
+
let record;
|
|
245
|
+
try {
|
|
246
|
+
record = JSON.parse(line);
|
|
247
|
+
}
|
|
248
|
+
catch {
|
|
249
|
+
return undefined;
|
|
250
|
+
}
|
|
251
|
+
if (typeof record !== "object" || record === null)
|
|
252
|
+
return undefined;
|
|
253
|
+
const r = record;
|
|
254
|
+
const isErrorRecord = r.type === "error" ||
|
|
255
|
+
r.is_error === true ||
|
|
256
|
+
(typeof r.subtype === "string" && r.subtype.startsWith("error"));
|
|
257
|
+
if (!isErrorRecord)
|
|
258
|
+
return undefined;
|
|
259
|
+
const texts = [];
|
|
260
|
+
if (typeof r.result === "string")
|
|
261
|
+
texts.push(r.result);
|
|
262
|
+
if (typeof r.error === "string")
|
|
263
|
+
texts.push(r.error);
|
|
264
|
+
if (typeof r.error === "object" && r.error !== null) {
|
|
265
|
+
const msg = r.error.message;
|
|
266
|
+
if (typeof msg === "string")
|
|
267
|
+
texts.push(msg);
|
|
268
|
+
}
|
|
269
|
+
for (const text of texts) {
|
|
270
|
+
if (AUTH_FAILURE_PATTERNS.some((p) => p.test(text))) {
|
|
271
|
+
return { reason: text.trim().slice(0, AUTH_REASON_MAX_CHARS) };
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
return undefined;
|
|
275
|
+
}
|
|
276
|
+
export function recordCredentialFailure(state, reason, now) {
|
|
277
|
+
return {
|
|
278
|
+
...state,
|
|
279
|
+
updatedAt: now,
|
|
280
|
+
credential: {
|
|
281
|
+
status: "failed",
|
|
282
|
+
reason: reason.slice(0, AUTH_REASON_MAX_CHARS),
|
|
283
|
+
seenAt: now,
|
|
284
|
+
},
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
export function clearCredentialFailure(state, now) {
|
|
288
|
+
return {
|
|
289
|
+
...state,
|
|
290
|
+
updatedAt: now,
|
|
291
|
+
credential: { status: "ok", seenAt: now },
|
|
292
|
+
};
|
|
293
|
+
}
|
|
210
294
|
export function updateHealthState(state, event, now) {
|
|
211
295
|
const key = event.rateLimitType ?? "unknown";
|
|
212
296
|
return {
|
package/dist/shim.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
3
3
|
import { dirname } from "node:path";
|
|
4
|
-
import { classifyStateReadFailure, createLineScanner, mergeHealthStates, parseRateLimitEvent, parseStoredState, updateHealthState, } from "./shim-core.js";
|
|
4
|
+
import { classifyStateReadFailure, clearCredentialFailure, createLineScanner, mergeHealthStates, parseAuthFailure, parseRateLimitEvent, parseStoredState, recordCredentialFailure, updateHealthState, } from "./shim-core.js";
|
|
5
5
|
import { rewriteModelArg } from "./degrade.js";
|
|
6
6
|
import { parseModelLimitError, recordModelLimit } from "./shim-core.js";
|
|
7
7
|
import { canonicalModelId } from "./models.js";
|
|
@@ -108,6 +108,7 @@ function guessLimitResetsAt() {
|
|
|
108
108
|
}
|
|
109
109
|
return undefined;
|
|
110
110
|
}
|
|
111
|
+
let sawAuthFailure = false;
|
|
111
112
|
const scanner = createLineScanner((line) => {
|
|
112
113
|
try {
|
|
113
114
|
const event = parseRateLimitEvent(line);
|
|
@@ -124,6 +125,13 @@ const scanner = createLineScanner((line) => {
|
|
|
124
125
|
process.stderr.write(`[multi-clawd shim] model limit hit recorded: ${model} (reported as "${limitHit.displayName}")\n`);
|
|
125
126
|
}
|
|
126
127
|
}
|
|
128
|
+
const authFailure = parseAuthFailure(line);
|
|
129
|
+
if (authFailure && !sawAuthFailure) {
|
|
130
|
+
sawAuthFailure = true;
|
|
131
|
+
state = recordCredentialFailure(state, authFailure.reason, Date.now());
|
|
132
|
+
persistState();
|
|
133
|
+
process.stderr.write(`[multi-clawd shim] auth failure recorded for ${accountId}: ${authFailure.reason}\n`);
|
|
134
|
+
}
|
|
127
135
|
}
|
|
128
136
|
catch {
|
|
129
137
|
}
|
|
@@ -138,11 +146,26 @@ for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"]) {
|
|
|
138
146
|
child.kill(signal);
|
|
139
147
|
});
|
|
140
148
|
}
|
|
149
|
+
function clearRecordedAuthFailureOnSuccess() {
|
|
150
|
+
if (!stateFile || sawAuthFailure)
|
|
151
|
+
return;
|
|
152
|
+
try {
|
|
153
|
+
if (readPersistedState()?.credential?.status !== "failed")
|
|
154
|
+
return;
|
|
155
|
+
state = clearCredentialFailure(state, Date.now());
|
|
156
|
+
persistState();
|
|
157
|
+
process.stderr.write(`[multi-clawd shim] auth recovered for ${accountId} — credential exclusion cleared\n`);
|
|
158
|
+
}
|
|
159
|
+
catch {
|
|
160
|
+
}
|
|
161
|
+
}
|
|
141
162
|
child.on("close", (code, signal) => {
|
|
142
163
|
if (signal) {
|
|
143
164
|
process.kill(process.pid, signal);
|
|
144
165
|
return;
|
|
145
166
|
}
|
|
167
|
+
if ((code ?? 0) === 0)
|
|
168
|
+
clearRecordedAuthFailureOnSuccess();
|
|
146
169
|
process.exit(code ?? 0);
|
|
147
170
|
});
|
|
148
171
|
child.on("error", (err) => {
|
package/dist/sticky.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { choosePoolAccount } from "./health.js";
|
|
1
|
+
import { choosePoolAccount, fallbackPoolAccount } from "./health.js";
|
|
2
2
|
export const DEFAULT_MIN_DWELL_MS = 10 * 60 * 1000;
|
|
3
3
|
export function decideStickySelection(params) {
|
|
4
4
|
const { verdicts, sticky, nowMs } = params;
|
|
@@ -6,7 +6,7 @@ export function decideStickySelection(params) {
|
|
|
6
6
|
const home = verdicts[0];
|
|
7
7
|
const healthChoice = choosePoolAccount(verdicts);
|
|
8
8
|
if (!healthChoice)
|
|
9
|
-
return { account:
|
|
9
|
+
return { account: fallbackPoolAccount(verdicts) };
|
|
10
10
|
const stickyVerdict = sticky
|
|
11
11
|
? verdicts.find((v) => v.id === sticky.account)?.verdict
|
|
12
12
|
: undefined;
|
package/openclaw.plugin.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"id": "multi-clawd",
|
|
3
3
|
"name": "multi-clawd",
|
|
4
|
-
"version": "1.7.
|
|
4
|
+
"version": "1.7.4",
|
|
5
5
|
"description": "Register additional Claude Code logins (Max/Pro accounts) as first-class OpenClaw CLI backends for cross-account failover, keeping the full skills/MCP harness on every account.",
|
|
6
6
|
"cliBackends": [
|
|
7
7
|
"claw1",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@drakon-systems/multi-clawd",
|
|
3
|
-
"version": "1.7.
|
|
3
|
+
"version": "1.7.4",
|
|
4
4
|
"description": "Multi-account Claude Code failover for OpenClaw — register additional Claude (Max/Pro) logins as first-class CLI backends and keep the full skills/MCP harness across every account. Also imports those accounts' setup tokens into Hermes Agent's Anthropic credential pool.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/scripts/cli.mjs
CHANGED
|
@@ -461,10 +461,11 @@ async function login() {
|
|
|
461
461
|
const { readFileSync: rf, existsSync, mkdirSync, chmodSync, statSync, mkdtempSync, rmSync } =
|
|
462
462
|
await import("node:fs");
|
|
463
463
|
const { homedir, tmpdir } = await import("node:os");
|
|
464
|
-
let lp, ec;
|
|
464
|
+
let lp, ec, idx;
|
|
465
465
|
try {
|
|
466
466
|
lp = await import(resolve(__dirname, "..", "dist", "login-plan.js"));
|
|
467
467
|
ec = await import(resolve(__dirname, "..", "dist", "explain-core.js"));
|
|
468
|
+
idx = await import(resolve(__dirname, "..", "dist", "index.js"));
|
|
468
469
|
} catch {
|
|
469
470
|
console.error("login: built dist/ is missing — reinstall the package.");
|
|
470
471
|
process.exit(1);
|
|
@@ -513,6 +514,17 @@ async function login() {
|
|
|
513
514
|
console.error(`\n ❌ ${plan.command.join(" ")} exited with ${r.status ?? "an error"}.`);
|
|
514
515
|
process.exit(1);
|
|
515
516
|
}
|
|
517
|
+
// Explicit re-authentication ends any recorded runtime credential failure
|
|
518
|
+
// immediately (#8) — without this the freshly re-authed account stays
|
|
519
|
+
// benched until the 15-minute TTL expires, which reads to the operator as
|
|
520
|
+
// "logging back in did nothing".
|
|
521
|
+
try {
|
|
522
|
+
if (idx.clearAccountCredentialFailure?.(acc.id)) {
|
|
523
|
+
console.log(`\n ↻ cleared ${acc.id}'s recorded login failure — the pool can use it again.`);
|
|
524
|
+
}
|
|
525
|
+
} catch {
|
|
526
|
+
/* clearing is a courtesy; never fail a login over it */
|
|
527
|
+
}
|
|
516
528
|
if (plan.verify === "auth-status") {
|
|
517
529
|
try {
|
|
518
530
|
const out = spawnSync("claude", ["auth", "status"], { encoding: "utf8", env }).stdout ?? "";
|
package/scripts/doctor.mjs
CHANGED
|
@@ -263,6 +263,20 @@ const io = {
|
|
|
263
263
|
const accounts = pluginConfig.accounts ?? [];
|
|
264
264
|
if (accounts.length === 0) warn("no accounts configured");
|
|
265
265
|
for (const account of accounts) {
|
|
266
|
+
// RUNTIME credential health first: the source check below only proves a
|
|
267
|
+
// credential EXISTS, and #8 is exactly the case where a present credential
|
|
268
|
+
// is a session the Claude CLI has already rejected. A recorded runtime
|
|
269
|
+
// failure is the stronger evidence, so it is reported as such.
|
|
270
|
+
const recorded = readJson(join(STATE_DIR, `${account.id}.json`))?.credential;
|
|
271
|
+
if (recorded?.status === "failed") {
|
|
272
|
+
const ageMin = Math.round((Date.now() - recorded.seenAt) / 60000);
|
|
273
|
+
bad(
|
|
274
|
+
`${account.id}: the Claude CLI rejected this login ${ageMin}m ago${
|
|
275
|
+
recorded.reason ? ` (${recorded.reason})` : ""
|
|
276
|
+
} — excluded from the pool; fix with \`multi-clawd login ${account.id}\``,
|
|
277
|
+
);
|
|
278
|
+
continue;
|
|
279
|
+
}
|
|
266
280
|
if (account.oauthTokenRef) {
|
|
267
281
|
warn(`${account.id}: oauthTokenRef — validated by the gateway's async probe, not doctor`);
|
|
268
282
|
continue;
|