@drakon-systems/multi-clawd 1.10.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -19
- package/dist/account-env.js +41 -3
- package/dist/account-identity.js +5 -0
- package/dist/claude-command.js +69 -0
- package/dist/credential-store.js +222 -0
- package/dist/direct-route.js +2 -1
- package/dist/explain-core.js +1 -0
- package/dist/health.js +30 -5
- package/dist/index.js +173 -39
- package/dist/login-health.js +137 -40
- package/dist/retry-plan.js +38 -3
- package/dist/shim-core.js +3 -1
- package/dist/shim-transport.js +247 -0
- package/dist/shim.js +35 -15
- package/openclaw.plugin.json +1 -1
- package/package.json +4 -4
- package/scripts/cli.mjs +1 -1
- package/scripts/doctor.mjs +116 -49
package/dist/index.js
CHANGED
|
@@ -7,7 +7,7 @@ import { MODEL_ALIASES, buildCatalogEntries, canonicalModelId, isModernClaudeMod
|
|
|
7
7
|
import { decideDegradation, matchesPin } from "./degrade.js";
|
|
8
8
|
import { resolveExecMode, permissionModeArgs } from "./exec-policy.js";
|
|
9
9
|
import { resolveBaseModelIds } from "./catalog-source.js";
|
|
10
|
-
import { allCredentialFailed, classifyAccountHealth, overdueProbeAccount, } from "./health.js";
|
|
10
|
+
import { allCredentialFailed, classifyAccountHealth, lastAuthSuccessAt, overdueProbeAccount, } from "./health.js";
|
|
11
11
|
import { decideStickySelection } from "./sticky.js";
|
|
12
12
|
import { mergeHealthStates, parseStoredState, recordCredentialFailure, } from "./shim-core.js";
|
|
13
13
|
import { createTokenRefResolver, isSecretRefShape, } from "./token-resolution.js";
|
|
@@ -19,14 +19,16 @@ import { addAlert, alertKeysWithPrefix, clearAlert, pendingAlertText, } from "./
|
|
|
19
19
|
import { healthStateFile, usageStateFile, clearAccountCredentialFailure } from "./credential-state.js";
|
|
20
20
|
import { createUsagePollController, effectiveUsagePollInterval, effectiveUsageWarnThreshold, usageAlertPrefix, usagePoolAlertKey, } from "./usage-poll.js";
|
|
21
21
|
export { healthStateFile, usageStateFile, clearAccountCredentialFailure };
|
|
22
|
-
import { accountConfigDir, buildAccountChildEnv, tokenFileModeWarning, validateAccountTokenSources, } from "./account-env.js";
|
|
22
|
+
import { accountConfigDir, AccountCredentialError, buildAccountChildEnv, credentialFailureText, parseSetupTokenFile, tokenFileModeWarning, validateAccountTokenSources, } from "./account-env.js";
|
|
23
23
|
import { diffCatalogModels, formatNewModelNotice, } from "./model-currency.js";
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
24
|
+
import { assessAccountCredential, createRefProbeTracker, } from "./login-health.js";
|
|
25
|
+
import { createSystemCredentialIo } from "./credential-store.js";
|
|
26
26
|
import { collectDirectMembers } from "./direct-route.js";
|
|
27
27
|
import { createDirectOrderController, isValidAgentId } from "./direct-sync.js";
|
|
28
28
|
import { createOpenclawRunner } from "./openclaw-runner.js";
|
|
29
29
|
import { materialiseRuntime } from "./runtime-home.js";
|
|
30
|
+
import { CLAUDE_ARGV0_ENV, CLAUDE_BIN_ENV, resolveClaudeExecutable } from "./claude-command.js";
|
|
31
|
+
import { createShimExecute } from "./shim-transport.js";
|
|
30
32
|
const CLI_FRESH_WATCHDOG_DEFAULTS = {
|
|
31
33
|
noOutputTimeoutRatio: 0.8,
|
|
32
34
|
minMs: 180_000,
|
|
@@ -156,21 +158,7 @@ function ingestAlertSpool() {
|
|
|
156
158
|
}
|
|
157
159
|
const LOGIN_PROBE_INTERVAL_MS = 15 * 60 * 1000;
|
|
158
160
|
const LOGIN_PROBE_INITIAL_DELAY_MS = 45 * 1000;
|
|
159
|
-
|
|
160
|
-
readFile: (p) => readFileSync(expandHome(p), "utf8"),
|
|
161
|
-
keychainHasClaudeCredentials: () => keychainHasService("Claude Code-credentials"),
|
|
162
|
-
keychainHasClaudeCredentialsForDir: (dir) => keychainHasService(keychainServiceForConfigDir(expandHome(dir))),
|
|
163
|
-
platform: process.platform,
|
|
164
|
-
};
|
|
165
|
-
function keychainHasService(service) {
|
|
166
|
-
try {
|
|
167
|
-
execFileSync("security", ["find-generic-password", "-s", service], { stdio: "ignore" });
|
|
168
|
-
return true;
|
|
169
|
-
}
|
|
170
|
-
catch {
|
|
171
|
-
return false;
|
|
172
|
-
}
|
|
173
|
-
}
|
|
161
|
+
let realCredentialIo;
|
|
174
162
|
const refProbeTrackers = new Map();
|
|
175
163
|
const lastProbeStatus = new Map();
|
|
176
164
|
function recordAccountCredentialFailure(accountId, reason, nowMs) {
|
|
@@ -195,7 +183,8 @@ function recordAccountCredentialFailure(accountId, reason, nowMs) {
|
|
|
195
183
|
export async function runLoginHealthProbe(accounts, logger, deps = {}) {
|
|
196
184
|
const now = deps.nowMs ?? Date.now();
|
|
197
185
|
const resolver = deps.resolver ?? activeTokenResolver;
|
|
198
|
-
const io = deps.io ?? realCredentialIo;
|
|
186
|
+
const io = deps.io ?? (realCredentialIo ??= createSystemCredentialIo());
|
|
187
|
+
const readHealth = deps.readHealth ?? readHealthState;
|
|
199
188
|
for (const account of accounts) {
|
|
200
189
|
let status;
|
|
201
190
|
let reason;
|
|
@@ -215,9 +204,12 @@ export async function runLoginHealthProbe(accounts, logger, deps = {}) {
|
|
|
215
204
|
cause = outcome.cause;
|
|
216
205
|
}
|
|
217
206
|
else {
|
|
218
|
-
const check =
|
|
207
|
+
const check = assessAccountCredential(account, io, {
|
|
208
|
+
proof: { lastSuccessAt: lastAuthSuccessAt(readHealth(account.id)) },
|
|
209
|
+
nowMs: now,
|
|
210
|
+
});
|
|
219
211
|
status = check.status;
|
|
220
|
-
reason = check.reason;
|
|
212
|
+
reason = check.status === "ok" ? check.detail : check.reason;
|
|
221
213
|
}
|
|
222
214
|
const previous = lastProbeStatus.get(account.id);
|
|
223
215
|
lastProbeStatus.set(account.id, status);
|
|
@@ -243,14 +235,30 @@ export async function runLoginHealthProbe(accounts, logger, deps = {}) {
|
|
|
243
235
|
logger.error(`[multi-clawd] ${text}`);
|
|
244
236
|
raiseAlert({ key: `login:${account.id}`, severity: "error", text });
|
|
245
237
|
}
|
|
238
|
+
else if (status === "unverified" && previous !== "unverified") {
|
|
239
|
+
logger.info(`[multi-clawd] account "${account.id}" login unverified: ${reason ?? "unproven"}`);
|
|
240
|
+
}
|
|
241
|
+
else if (status === "unknown" && previous !== "unknown" && reason) {
|
|
242
|
+
logger.info(`[multi-clawd] account "${account.id}" login could not be checked: ${reason}`);
|
|
243
|
+
}
|
|
246
244
|
else if (status === "degraded" && previous !== "degraded") {
|
|
247
245
|
logger.info(`[multi-clawd] account "${account.id}" login degraded: ${reason ?? "resolver error"}`);
|
|
248
246
|
}
|
|
247
|
+
else if (status === "ok" && previous === "unverified") {
|
|
248
|
+
logger.info(`[multi-clawd] account "${account.id}" login verified: ${reason ?? "ok"}`);
|
|
249
|
+
}
|
|
249
250
|
else if (status === "ok" && (previous === "broken" || previous === "degraded")) {
|
|
250
251
|
logger.info(`[multi-clawd] account "${account.id}" login recovered`);
|
|
251
252
|
alertState = clearAlert(alertState, `login:${account.id}`);
|
|
252
253
|
alertState = clearAlert(alertState, `login-resolver:${account.id}`);
|
|
253
254
|
}
|
|
255
|
+
if (status === "ok") {
|
|
256
|
+
alertState = clearAlert(alertState, `login:${account.id}`);
|
|
257
|
+
alertState = clearAlert(alertState, `login-resolver:${account.id}`);
|
|
258
|
+
}
|
|
259
|
+
else if ((status === "unverified" || status === "unknown") && cause === undefined) {
|
|
260
|
+
alertState = clearAlert(alertState, `login:${account.id}`);
|
|
261
|
+
}
|
|
254
262
|
}
|
|
255
263
|
}
|
|
256
264
|
export function startLoginHealthProbe(accounts, logger) {
|
|
@@ -290,7 +298,20 @@ function peekToken(account) {
|
|
|
290
298
|
if (account.oauthTokenFile) {
|
|
291
299
|
const path = expandHome(account.oauthTokenFile);
|
|
292
300
|
warnIfTokenFileExposed(path);
|
|
293
|
-
|
|
301
|
+
let raw;
|
|
302
|
+
try {
|
|
303
|
+
raw = readFileSync(path, "utf8");
|
|
304
|
+
}
|
|
305
|
+
catch (err) {
|
|
306
|
+
const code = err.code;
|
|
307
|
+
throw new AccountCredentialError(`[multi-clawd] account "${account.id}": token file ${account.oauthTokenFile} unreadable` +
|
|
308
|
+
(typeof code === "string" ? ` (${code})` : ""));
|
|
309
|
+
}
|
|
310
|
+
const read = parseSetupTokenFile(raw, `token file ${account.oauthTokenFile}`);
|
|
311
|
+
if ("error" in read) {
|
|
312
|
+
throw new AccountCredentialError(`[multi-clawd] account "${account.id}": ${read.error}`);
|
|
313
|
+
}
|
|
314
|
+
return read.token;
|
|
294
315
|
}
|
|
295
316
|
if (isSecretRefShape(account.oauthTokenRef)) {
|
|
296
317
|
return activeTokenResolver?.peek(account.oauthTokenRef);
|
|
@@ -340,6 +361,63 @@ if (!SHIM_RUNTIME.stable) {
|
|
|
340
361
|
`using the load path, which dies if OpenClaw prunes its plugin capture (issue #31)`);
|
|
341
362
|
}
|
|
342
363
|
const SHIM_PATH = SHIM_RUNTIME.shimPath;
|
|
364
|
+
export const CLAUDE_RUNTIME_ARTIFACT = Object.freeze({
|
|
365
|
+
kind: "bundled-package-tree",
|
|
366
|
+
packageName: "@anthropic-ai/claude-code",
|
|
367
|
+
entrypoint: "command",
|
|
368
|
+
nativeExecutableNames: Object.freeze(["claude", "claude.exe"]),
|
|
369
|
+
});
|
|
370
|
+
export function hostRunsExecuteTransports(runtime) {
|
|
371
|
+
try {
|
|
372
|
+
const rt = runtime;
|
|
373
|
+
return (typeof rt?.agent?.runCommandFromIngress === "function" &&
|
|
374
|
+
typeof rt?.hooks?.dispatchHookAgentTurn === "function");
|
|
375
|
+
}
|
|
376
|
+
catch {
|
|
377
|
+
return false;
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
export function resolveBackendLaunch(params) {
|
|
381
|
+
const shimPath = params.shimPath ?? SHIM_PATH;
|
|
382
|
+
if (!params.hostSupportsExecute) {
|
|
383
|
+
return {
|
|
384
|
+
kind: "legacy",
|
|
385
|
+
shimPath,
|
|
386
|
+
reason: "this OpenClaw does not expose runtime.agent.runCommandFromIngress and " +
|
|
387
|
+
"runtime.hooks.dispatchHookAgentTurn, so it is treated as not running plugin execute " +
|
|
388
|
+
"transports (needs >= 2026.8.1)",
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
const claude = resolveClaudeExecutable(params.env ?? process.env, params.platform ?? process.platform);
|
|
392
|
+
if (!claude.ok)
|
|
393
|
+
return { kind: "legacy", shimPath, reason: claude.reason };
|
|
394
|
+
return {
|
|
395
|
+
kind: "verified",
|
|
396
|
+
claudeCommand: claude.command,
|
|
397
|
+
claudeSource: claude.source,
|
|
398
|
+
shimPath,
|
|
399
|
+
...(params.log ? { log: params.log } : {}),
|
|
400
|
+
};
|
|
401
|
+
}
|
|
402
|
+
const SHIM_ONLY_ENV = [
|
|
403
|
+
RETRY_ROSTER_ENV,
|
|
404
|
+
SESSION_DIRS_ENV,
|
|
405
|
+
"MULTI_CLAWD_MODEL_OVERRIDE",
|
|
406
|
+
CLAUDE_BIN_ENV,
|
|
407
|
+
CLAUDE_ARGV0_ENV,
|
|
408
|
+
];
|
|
409
|
+
function preparedForLaunch(launch, ctx, env, transport) {
|
|
410
|
+
if (launch.kind === "legacy")
|
|
411
|
+
return { env };
|
|
412
|
+
if (ctx.executionMode === "side-question") {
|
|
413
|
+
for (const key of SHIM_ONLY_ENV)
|
|
414
|
+
delete env[key];
|
|
415
|
+
return { env };
|
|
416
|
+
}
|
|
417
|
+
if (!transport)
|
|
418
|
+
throw new Error("[multi-clawd] verified launch is missing its shim transport");
|
|
419
|
+
return { env, execute: transport };
|
|
420
|
+
}
|
|
343
421
|
function knownModelsFile() {
|
|
344
422
|
return join(homedir(), ".openclaw", "state", "multi-clawd", "known-models.json");
|
|
345
423
|
}
|
|
@@ -388,7 +466,16 @@ function checkModelCurrency(catalogIds, chainRefs, poolId) {
|
|
|
388
466
|
catch {
|
|
389
467
|
}
|
|
390
468
|
}
|
|
391
|
-
export function buildBackend(account, execMode) {
|
|
469
|
+
export function buildBackend(account, execMode, launch = { kind: "legacy", shimPath: SHIM_PATH }) {
|
|
470
|
+
const verified = launch.kind === "verified";
|
|
471
|
+
const shimPrefix = verified ? [] : [launch.shimPath];
|
|
472
|
+
const transport = verified
|
|
473
|
+
? createShimExecute({
|
|
474
|
+
configuredCommand: launch.claudeCommand,
|
|
475
|
+
shimPath: launch.shimPath,
|
|
476
|
+
...(launch.log ? { log: launch.log } : {}),
|
|
477
|
+
})
|
|
478
|
+
: undefined;
|
|
392
479
|
const backend = {
|
|
393
480
|
id: account.id,
|
|
394
481
|
liveTest: {
|
|
@@ -400,6 +487,7 @@ export function buildBackend(account, execMode) {
|
|
|
400
487
|
binaryName: "claude",
|
|
401
488
|
},
|
|
402
489
|
},
|
|
490
|
+
...(verified ? { runtimeArtifact: CLAUDE_RUNTIME_ARTIFACT } : {}),
|
|
403
491
|
bundleMcp: true,
|
|
404
492
|
bundleMcpMode: "claude-config-file",
|
|
405
493
|
nativeToolMode: "selectable",
|
|
@@ -408,9 +496,9 @@ export function buildBackend(account, execMode) {
|
|
|
408
496
|
sideQuestionToolMode: "disabled",
|
|
409
497
|
ownsNativeCompaction: true,
|
|
410
498
|
config: {
|
|
411
|
-
command: process.execPath,
|
|
412
|
-
args: [
|
|
413
|
-
resumeArgs: [
|
|
499
|
+
command: verified ? launch.claudeCommand : process.execPath,
|
|
500
|
+
args: [...shimPrefix, ...BASE_ARGS, ...permissionModeArgs(execMode)],
|
|
501
|
+
resumeArgs: [...shimPrefix, ...BASE_ARGS, ...permissionModeArgs(execMode), "--resume", "{sessionId}"],
|
|
414
502
|
forkArg: "--fork-session",
|
|
415
503
|
resumeAtArg: "--resume-session-at",
|
|
416
504
|
output: "jsonl",
|
|
@@ -438,25 +526,30 @@ export function buildBackend(account, execMode) {
|
|
|
438
526
|
},
|
|
439
527
|
serialize: true,
|
|
440
528
|
},
|
|
441
|
-
async prepareExecution(
|
|
442
|
-
return
|
|
529
|
+
async prepareExecution(ctx) {
|
|
530
|
+
return preparedForLaunch(launch, ctx, await buildAccountEnv(account), transport);
|
|
443
531
|
},
|
|
444
532
|
};
|
|
533
|
+
launchTransports.set(backend, { launch, transport });
|
|
445
534
|
Object.assign(backend.config, { sessionArg: "--session-id" });
|
|
446
535
|
return backend;
|
|
447
536
|
}
|
|
537
|
+
const launchTransports = new WeakMap();
|
|
448
538
|
export function buildRetryRoster(members, launchedId) {
|
|
449
539
|
const roster = [];
|
|
450
540
|
for (const member of members) {
|
|
451
541
|
if (member.id === launchedId)
|
|
452
542
|
continue;
|
|
453
|
-
|
|
543
|
+
const tokenFile = !member.native && member.oauthTokenFile ? expandHome(member.oauthTokenFile) : undefined;
|
|
544
|
+
if (!member.native && !tokenFile && member.oauthTokenRef)
|
|
545
|
+
continue;
|
|
546
|
+
if (!member.native && !tokenFile && !member.configDir)
|
|
454
547
|
continue;
|
|
455
548
|
const stateFile = healthStateFile(member.id);
|
|
456
|
-
const env = buildAccountChildEnv(member, undefined, stateFile);
|
|
549
|
+
const env = buildAccountChildEnv({ ...member, oauthTokenFile: undefined, oauthTokenRef: undefined }, undefined, stateFile);
|
|
457
550
|
delete env.MULTI_CLAWD_ACCOUNT_ID;
|
|
458
551
|
delete env.MULTI_CLAWD_STATE_FILE;
|
|
459
|
-
roster.push({ id: member.id, stateFile, env });
|
|
552
|
+
roster.push({ id: member.id, stateFile, env, ...(tokenFile ? { tokenFile } : {}) });
|
|
460
553
|
}
|
|
461
554
|
return roster;
|
|
462
555
|
}
|
|
@@ -539,6 +632,20 @@ export default definePluginEntry({
|
|
|
539
632
|
}
|
|
540
633
|
const execMode = resolveExecMode(runtimeConfigLoader?.() ?? api.config);
|
|
541
634
|
api.logger.info(`[multi-clawd] exec policy: ${execMode ?? "unknown"} → permission-mode ${permissionModeArgs(execMode).length ? "bypassPermissions" : "default (no override)"}`);
|
|
635
|
+
const launchLogger = api.logger;
|
|
636
|
+
const launch = resolveBackendLaunch({
|
|
637
|
+
hostSupportsExecute: hostRunsExecuteTransports(api.runtime),
|
|
638
|
+
log: (line) => launchLogger.info(line),
|
|
639
|
+
});
|
|
640
|
+
if (launch.kind === "verified") {
|
|
641
|
+
api.logger.info(`[multi-clawd] launch mode: verified — claude ${launch.claudeCommand} (from ${launch.claudeSource === "override" ? "MULTI_CLAWD_CLAUDE_BIN" : "PATH"}) is the backend command; host runs execute transports ` +
|
|
642
|
+
`(runtime.agent.runCommandFromIngress + runtime.hooks.dispatchHookAgentTurn present), ` +
|
|
643
|
+
`so agent turns run through shim ${launch.shimPath} and side-questions run claude directly`);
|
|
644
|
+
}
|
|
645
|
+
else {
|
|
646
|
+
api.logger.warn(`[multi-clawd] launch mode: legacy node+shim — ${launch.reason ?? "unknown reason"}; ` +
|
|
647
|
+
`failover is unchanged, but OpenClaw cannot verify this backend as a setup owner`);
|
|
648
|
+
}
|
|
542
649
|
const seen = new Set();
|
|
543
650
|
for (const account of accounts) {
|
|
544
651
|
const id = account?.id?.trim();
|
|
@@ -555,10 +662,10 @@ export default definePluginEntry({
|
|
|
555
662
|
}
|
|
556
663
|
seen.add(id);
|
|
557
664
|
const normalized = { ...account, id };
|
|
558
|
-
api.registerCliBackend(buildBackend(normalized, execMode));
|
|
665
|
+
api.registerCliBackend(buildBackend(normalized, execMode, launch));
|
|
559
666
|
api.registerProvider(buildCatalogProvider(normalized));
|
|
560
667
|
}
|
|
561
|
-
registerPoolBackend(api, cfg.pool, accounts, seen, execMode);
|
|
668
|
+
registerPoolBackend(api, cfg.pool, accounts, seen, execMode, { launch });
|
|
562
669
|
{
|
|
563
670
|
const logger = api.logger;
|
|
564
671
|
try {
|
|
@@ -674,7 +781,8 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
674
781
|
};
|
|
675
782
|
const minDwellMs = pool.minDwellMs;
|
|
676
783
|
const stickyFile = join(homedir(), ".openclaw", "state", "multi-clawd", `pool-${poolId}.sticky.json`);
|
|
677
|
-
const backend = buildBackend(poolAccount, execMode);
|
|
784
|
+
const backend = buildBackend(poolAccount, execMode, deps?.launch);
|
|
785
|
+
const { launch, transport } = launchTransports.get(backend);
|
|
678
786
|
backend.prepareExecution = async (ctx) => {
|
|
679
787
|
const now = Date.now();
|
|
680
788
|
const requestedModel = canonicalModelId(ctx.modelId) ?? ctx.modelId;
|
|
@@ -716,6 +824,10 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
716
824
|
`launching on ${decision.account} to re-test a rejection last seen ` +
|
|
717
825
|
`${Math.round((now - observedAt) / 3600000)}h ago`);
|
|
718
826
|
}
|
|
827
|
+
else if (poolVerdicts.find((v) => v.id === decision.account)?.verdict === "credential_unverified") {
|
|
828
|
+
logger.info(`[multi-clawd] pool ${poolId}: no proven account can serve ${requestedModel} — ` +
|
|
829
|
+
`launching on ${decision.account} to re-test a login last rejected by the Claude CLI`);
|
|
830
|
+
}
|
|
719
831
|
else if (decision.account !== previousAccount) {
|
|
720
832
|
const home = verdicts[0];
|
|
721
833
|
const line = decision.account === members[0].id
|
|
@@ -760,6 +872,13 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
760
872
|
text: `pool ${poolId}: account "${v.id}" is excluded — ${v.health.reason ?? "its login was rejected by the Claude CLI"}. Fix with \`multi-clawd login ${v.id}\`.`,
|
|
761
873
|
});
|
|
762
874
|
}
|
|
875
|
+
else if (v.health.verdict === "credential_unverified") {
|
|
876
|
+
raiseAlert({
|
|
877
|
+
key,
|
|
878
|
+
severity: "error",
|
|
879
|
+
text: `pool ${poolId}: account "${v.id}" is unproven — ${v.health.reason ?? "its login was last rejected by the Claude CLI"}. Fix with \`multi-clawd login ${v.id}\`.`,
|
|
880
|
+
});
|
|
881
|
+
}
|
|
763
882
|
else {
|
|
764
883
|
alertState = clearAlert(alertState, key);
|
|
765
884
|
}
|
|
@@ -774,6 +893,7 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
774
893
|
try {
|
|
775
894
|
env = await buildAccountEnv(candidate, deps?.resolver);
|
|
776
895
|
launched = candidate;
|
|
896
|
+
alertState = clearAlert(alertState, `credential-unresolved:${candidate.id}`);
|
|
777
897
|
if (candidate.id !== chosen.id) {
|
|
778
898
|
logger.warn(`[multi-clawd] pool ${poolId}: ${chosen.id}'s credential did not resolve — ` +
|
|
779
899
|
`launching on ${candidate.id} instead (account not benched; secret provider may be down)`);
|
|
@@ -781,7 +901,14 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
781
901
|
break;
|
|
782
902
|
}
|
|
783
903
|
catch (err) {
|
|
784
|
-
|
|
904
|
+
const why = credentialFailureText(err);
|
|
905
|
+
unresolved.push(`${candidate.id} (${why})`);
|
|
906
|
+
raiseAlert({
|
|
907
|
+
key: `credential-unresolved:${candidate.id}`,
|
|
908
|
+
severity: "error",
|
|
909
|
+
text: `pool ${poolId}: account "${candidate.id}" was skipped — its declared credential did ` +
|
|
910
|
+
`not resolve, so it will not launch on any other login (${why})`,
|
|
911
|
+
});
|
|
785
912
|
}
|
|
786
913
|
}
|
|
787
914
|
if (!env) {
|
|
@@ -810,7 +937,12 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
810
937
|
requestedModel: ctx.modelId,
|
|
811
938
|
ladder,
|
|
812
939
|
});
|
|
813
|
-
|
|
940
|
+
const runsWithoutShim = launch.kind === "verified" && ctx.executionMode === "side-question";
|
|
941
|
+
if (degradation && runsWithoutShim) {
|
|
942
|
+
logger.warn(`[multi-clawd] pool ${poolId}: side-question runs ${ctx.modelId} undegraded on ${(launched ?? chosen).id} — every account is limited for it, and the degrade ladder (${degradation.model}) ` +
|
|
943
|
+
`can only be applied by the shim, which OpenClaw does not run for side-questions`);
|
|
944
|
+
}
|
|
945
|
+
else if (degradation) {
|
|
814
946
|
env.MULTI_CLAWD_MODEL_OVERRIDE = degradation.model;
|
|
815
947
|
const line = `pool ${poolId}: degrading ${ctx.modelId} → ${degradation.model} on ${chosen.id} (${degradation.reason})`;
|
|
816
948
|
logger.info(`[multi-clawd] ${line}`);
|
|
@@ -820,7 +952,7 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
820
952
|
logger.info(`[multi-clawd] pool ${poolId}: pinned lane keeps ${ctx.modelId} despite exhausted pool (will fail over via the chain)`);
|
|
821
953
|
}
|
|
822
954
|
}
|
|
823
|
-
return
|
|
955
|
+
return preparedForLaunch(launch, ctx, env, transport);
|
|
824
956
|
};
|
|
825
957
|
api.registerCliBackend(backend);
|
|
826
958
|
api.registerProvider(buildCatalogProvider(poolAccount));
|
|
@@ -927,9 +1059,11 @@ export function stopUsagePoll() {
|
|
|
927
1059
|
usageInitial = undefined;
|
|
928
1060
|
usageController = undefined;
|
|
929
1061
|
}
|
|
1062
|
+
export const USAGE_TOKEN_ACCOUNT_REASON = "setup-token login — the usage endpoint needs the user:profile scope and setup-tokens carry only " +
|
|
1063
|
+
"user:inference, so usage comes from stream telemetry only";
|
|
930
1064
|
export function usagePollCredentialsFile(account) {
|
|
931
1065
|
if (account.oauthTokenFile || account.oauthTokenRef) {
|
|
932
|
-
return { reason:
|
|
1066
|
+
return { reason: USAGE_TOKEN_ACCOUNT_REASON };
|
|
933
1067
|
}
|
|
934
1068
|
if (!account.native && !account.configDir) {
|
|
935
1069
|
return { reason: "no login dir to read credentials from" };
|
package/dist/login-health.js
CHANGED
|
@@ -1,8 +1,98 @@
|
|
|
1
|
+
import { parseSetupTokenFile } from "./account-env.js";
|
|
1
2
|
import { createHash } from "node:crypto";
|
|
3
|
+
export const CLAUDE_KEYCHAIN_SERVICE = "Claude Code-credentials";
|
|
4
|
+
export function claudeKeychainAccount(env, userInfo) {
|
|
5
|
+
try {
|
|
6
|
+
return env.USER || userInfo().username;
|
|
7
|
+
}
|
|
8
|
+
catch {
|
|
9
|
+
return "claude-code-user";
|
|
10
|
+
}
|
|
11
|
+
}
|
|
2
12
|
export function keychainServiceForConfigDir(absDir) {
|
|
3
13
|
const hash = createHash("sha256").update(absDir).digest("hex").slice(0, 8);
|
|
4
14
|
return `Claude Code-credentials-${hash}`;
|
|
5
15
|
}
|
|
16
|
+
export const AUTH_PROOF_FRESH_MS = 24 * 60 * 60 * 1000;
|
|
17
|
+
export const ACCESS_TOKEN_EXPIRY_SKEW_MS = 60 * 1000;
|
|
18
|
+
const MAX_PLAUSIBLE_EXPIRY_MS = 253402300799000;
|
|
19
|
+
export function parseOauthCredential(raw) {
|
|
20
|
+
let text = raw.trim();
|
|
21
|
+
if (/^(?:[0-9a-f]{2})+$/i.test(text)) {
|
|
22
|
+
text = Buffer.from(text, "hex").toString("utf8").trim();
|
|
23
|
+
}
|
|
24
|
+
let parsed;
|
|
25
|
+
try {
|
|
26
|
+
parsed = JSON.parse(text);
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
return undefined;
|
|
30
|
+
}
|
|
31
|
+
const oauth = parsed?.claudeAiOauth;
|
|
32
|
+
if (!oauth || typeof oauth !== "object")
|
|
33
|
+
return undefined;
|
|
34
|
+
const o = oauth;
|
|
35
|
+
const present = (v) => typeof v === "string" && v.trim().length > 0;
|
|
36
|
+
let expiresAt;
|
|
37
|
+
if (typeof o.expiresAt === "number" && Number.isFinite(o.expiresAt) && o.expiresAt > 0) {
|
|
38
|
+
const ms = o.expiresAt < 1e12 ? o.expiresAt * 1000 : o.expiresAt;
|
|
39
|
+
if (ms < MAX_PLAUSIBLE_EXPIRY_MS)
|
|
40
|
+
expiresAt = ms;
|
|
41
|
+
}
|
|
42
|
+
return {
|
|
43
|
+
hasAccessToken: present(o.accessToken),
|
|
44
|
+
hasRefreshToken: present(o.refreshToken),
|
|
45
|
+
expiresAt,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
function ago(ms) {
|
|
49
|
+
const min = Math.max(0, Math.round(ms / 60000));
|
|
50
|
+
return min < 90 ? `${min}m` : `${Math.round(min / 60)}h`;
|
|
51
|
+
}
|
|
52
|
+
export function judgeOauthCredential(meta, source, proof, nowMs) {
|
|
53
|
+
const base = { source, expiresAt: meta.expiresAt };
|
|
54
|
+
if (!meta.hasAccessToken) {
|
|
55
|
+
return { ...base, status: "broken", reason: `${source}: access token is blank` };
|
|
56
|
+
}
|
|
57
|
+
if (meta.expiresAt !== undefined && meta.expiresAt - ACCESS_TOKEN_EXPIRY_SKEW_MS > nowMs) {
|
|
58
|
+
return {
|
|
59
|
+
...base,
|
|
60
|
+
status: "ok",
|
|
61
|
+
detail: `access token valid until ${new Date(meta.expiresAt).toISOString()}`,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
if (meta.expiresAt === undefined) {
|
|
65
|
+
return {
|
|
66
|
+
...base,
|
|
67
|
+
status: "unverified",
|
|
68
|
+
reason: `${source}: records no access-token expiry, so it cannot be judged at rest`,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
const expiry = `access token expired ${new Date(meta.expiresAt).toISOString()}`;
|
|
72
|
+
const success = proof.lastSuccessAt;
|
|
73
|
+
if (success !== undefined &&
|
|
74
|
+
success <= nowMs &&
|
|
75
|
+
nowMs - success <= AUTH_PROOF_FRESH_MS &&
|
|
76
|
+
success >= meta.expiresAt) {
|
|
77
|
+
return {
|
|
78
|
+
...base,
|
|
79
|
+
status: "ok",
|
|
80
|
+
detail: `${expiry}; a successful turn ${ago(nowMs - success)} ago proves the refresh works`,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
if (!meta.hasRefreshToken) {
|
|
84
|
+
return {
|
|
85
|
+
...base,
|
|
86
|
+
status: "broken",
|
|
87
|
+
reason: `${source}: ${expiry} and no refresh token is stored`,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
return {
|
|
91
|
+
...base,
|
|
92
|
+
status: "unverified",
|
|
93
|
+
reason: `${source}: ${expiry} and no successful turn since proves the refresh works`,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
6
96
|
export function createRefProbeTracker(options = {}) {
|
|
7
97
|
const deadAfterConsecutive = options.deadAfterConsecutive ?? 3;
|
|
8
98
|
const deadAfterMs = options.deadAfterMs ?? 10 * 60 * 1000;
|
|
@@ -42,29 +132,40 @@ export function createRefProbeTracker(options = {}) {
|
|
|
42
132
|
},
|
|
43
133
|
};
|
|
44
134
|
}
|
|
45
|
-
function
|
|
46
|
-
return /^sk-ant-[a-z0-9]+-/.test(value.trim());
|
|
47
|
-
}
|
|
48
|
-
function checkCredentialsJson(io, dir) {
|
|
135
|
+
function assessCredentialsJson(io, path, proof, nowMs) {
|
|
49
136
|
let raw;
|
|
50
137
|
try {
|
|
51
|
-
raw = io.readFile(
|
|
138
|
+
raw = io.readFile(path);
|
|
52
139
|
}
|
|
53
140
|
catch {
|
|
54
|
-
return
|
|
141
|
+
return undefined;
|
|
55
142
|
}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
if (typeof token === "string" && token.trim().length > 0)
|
|
60
|
-
return { status: "ok" };
|
|
61
|
-
return { status: "broken", reason: `${dir}/.credentials.json access token is blank` };
|
|
143
|
+
const meta = parseOauthCredential(raw);
|
|
144
|
+
if (!meta) {
|
|
145
|
+
return { status: "broken", reason: `${path} holds no Claude OAuth credential`, source: path };
|
|
62
146
|
}
|
|
63
|
-
|
|
64
|
-
|
|
147
|
+
return judgeOauthCredential(meta, path, proof, nowMs);
|
|
148
|
+
}
|
|
149
|
+
function assessKeychain(io, service, proof, nowMs) {
|
|
150
|
+
const read = io.readKeychainItem(service, io.keychainAccount);
|
|
151
|
+
if (read.status === "absent")
|
|
152
|
+
return undefined;
|
|
153
|
+
const source = `keychain "${service}"`;
|
|
154
|
+
if (read.status === "unreadable") {
|
|
155
|
+
return {
|
|
156
|
+
status: "unknown",
|
|
157
|
+
reason: `${source} could not be read (Keychain locked, access refused or timed out) — not judged`,
|
|
158
|
+
source,
|
|
159
|
+
};
|
|
65
160
|
}
|
|
161
|
+
const meta = parseOauthCredential(read.data);
|
|
162
|
+
if (!meta)
|
|
163
|
+
return { status: "broken", reason: `${source} holds no Claude OAuth credential`, source };
|
|
164
|
+
return judgeOauthCredential(meta, source, proof, nowMs);
|
|
66
165
|
}
|
|
67
|
-
export function
|
|
166
|
+
export function assessAccountCredential(account, io, options = {}) {
|
|
167
|
+
const proof = options.proof ?? {};
|
|
168
|
+
const nowMs = options.nowMs ?? Date.now();
|
|
68
169
|
if (account.oauthTokenFile) {
|
|
69
170
|
let raw;
|
|
70
171
|
try {
|
|
@@ -73,35 +174,31 @@ export function checkAccountCredential(account, io) {
|
|
|
73
174
|
catch {
|
|
74
175
|
return { status: "broken", reason: `${account.oauthTokenFile} unreadable` };
|
|
75
176
|
}
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
reason: `${account.oauthTokenFile} does not contain a setup-token`,
|
|
81
|
-
};
|
|
177
|
+
const read = parseSetupTokenFile(raw, account.oauthTokenFile);
|
|
178
|
+
if ("token" in read)
|
|
179
|
+
return { status: "ok", detail: "setup-token present" };
|
|
180
|
+
return { status: "broken", reason: read.error };
|
|
82
181
|
}
|
|
83
182
|
if (account.oauthTokenRef) {
|
|
84
183
|
return { status: "unknown" };
|
|
85
184
|
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
return checkCredentialsJson(io, "~/.claude");
|
|
93
|
-
}
|
|
94
|
-
if (account.configDir) {
|
|
95
|
-
if (io.platform === "darwin" && io.keychainHasClaudeCredentialsForDir(account.configDir)) {
|
|
96
|
-
return { status: "ok" };
|
|
97
|
-
}
|
|
98
|
-
const file = checkCredentialsJson(io, account.configDir);
|
|
99
|
-
if (file.status === "ok" || io.platform !== "darwin")
|
|
100
|
-
return file;
|
|
101
|
-
return {
|
|
185
|
+
const dir = account.native ? "~/.claude" : account.configDir;
|
|
186
|
+
if (!dir)
|
|
187
|
+
return { status: "unknown" };
|
|
188
|
+
const path = `${dir.replace(/\/+$/, "")}/.credentials.json`;
|
|
189
|
+
if (io.platform !== "darwin") {
|
|
190
|
+
return (assessCredentialsJson(io, path, proof, nowMs) ?? {
|
|
102
191
|
status: "broken",
|
|
103
|
-
reason:
|
|
104
|
-
|
|
192
|
+
reason: `${path} unreadable`,
|
|
193
|
+
source: path,
|
|
194
|
+
});
|
|
105
195
|
}
|
|
106
|
-
|
|
196
|
+
const service = account.native
|
|
197
|
+
? CLAUDE_KEYCHAIN_SERVICE
|
|
198
|
+
: keychainServiceForConfigDir(io.expandHome(dir));
|
|
199
|
+
return (assessKeychain(io, service, proof, nowMs) ??
|
|
200
|
+
assessCredentialsJson(io, path, proof, nowMs) ?? {
|
|
201
|
+
status: "broken",
|
|
202
|
+
reason: `keychain has no Claude Code credentials for ${account.native ? "the default login" : dir} and ${path} unreadable`,
|
|
203
|
+
});
|
|
107
204
|
}
|
package/dist/retry-plan.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { classifyAccountHealth, overdueProbeAccount, } from "./health.js";
|
|
2
|
+
import { parseSetupTokenFile } from "./account-env.js";
|
|
2
3
|
import { resumeSessionId } from "./session-handover.js";
|
|
3
4
|
export const RETRY_ROSTER_ENV = "MULTI_CLAWD_RETRY_ACCOUNTS";
|
|
4
5
|
const CREDENTIAL_ENV_KEYS = ["CLAUDE_CONFIG_DIR", "CLAUDE_CODE_OAUTH_TOKEN"];
|
|
6
|
+
const ROSTER_ENV_KEYS = ["CLAUDE_CONFIG_DIR"];
|
|
5
7
|
export function parseRetryRoster(raw) {
|
|
6
8
|
if (!raw)
|
|
7
9
|
return [];
|
|
@@ -26,11 +28,12 @@ export function parseRetryRoster(raw) {
|
|
|
26
28
|
const env = {};
|
|
27
29
|
if (e.env && typeof e.env === "object") {
|
|
28
30
|
for (const [k, v] of Object.entries(e.env)) {
|
|
29
|
-
if (typeof v === "string")
|
|
31
|
+
if (typeof v === "string" && ROSTER_ENV_KEYS.includes(k))
|
|
30
32
|
env[k] = v;
|
|
31
33
|
}
|
|
32
34
|
}
|
|
33
|
-
|
|
35
|
+
const tokenFile = typeof e.tokenFile === "string" && e.tokenFile ? e.tokenFile : undefined;
|
|
36
|
+
out.push({ id: e.id, stateFile: e.stateFile, env, ...(tokenFile ? { tokenFile } : {}) });
|
|
34
37
|
}
|
|
35
38
|
return out;
|
|
36
39
|
}
|
|
@@ -40,7 +43,7 @@ export function isResumeLaunch(argv) {
|
|
|
40
43
|
}
|
|
41
44
|
export function retryArming(argv, roster, opts = {}) {
|
|
42
45
|
if (roster.length === 0) {
|
|
43
|
-
return { armed: false, reason: "no
|
|
46
|
+
return { armed: false, reason: "no sibling account the shim can retry onto" };
|
|
44
47
|
}
|
|
45
48
|
if (isResumeLaunch(argv) && !opts.resumeReady) {
|
|
46
49
|
return {
|
|
@@ -77,3 +80,35 @@ export function buildRetryEnv(base, account) {
|
|
|
77
80
|
env[RETRY_ROSTER_ENV] = "";
|
|
78
81
|
return env;
|
|
79
82
|
}
|
|
83
|
+
export function materializeRetryAccount(account, readFile) {
|
|
84
|
+
if (!account.tokenFile)
|
|
85
|
+
return { account };
|
|
86
|
+
let raw;
|
|
87
|
+
try {
|
|
88
|
+
raw = readFile(account.tokenFile);
|
|
89
|
+
}
|
|
90
|
+
catch (err) {
|
|
91
|
+
const code = err.code;
|
|
92
|
+
return { error: `token file unreadable${typeof code === "string" && /^E[A-Z]+$/.test(code) ? ` (${code})` : ""}` };
|
|
93
|
+
}
|
|
94
|
+
const read = parseSetupTokenFile(raw, "token file");
|
|
95
|
+
if ("error" in read)
|
|
96
|
+
return { error: read.error };
|
|
97
|
+
return {
|
|
98
|
+
account: { ...account, env: { ...account.env, CLAUDE_CODE_OAUTH_TOKEN: read.token } },
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
export function chooseLaunchableRetryAccount(params) {
|
|
102
|
+
let remaining = params.roster;
|
|
103
|
+
while (remaining.length > 0) {
|
|
104
|
+
const pick = chooseRetryAccount({ ...params, roster: remaining });
|
|
105
|
+
if (!pick)
|
|
106
|
+
return undefined;
|
|
107
|
+
const ready = params.materialize(pick);
|
|
108
|
+
if ("account" in ready)
|
|
109
|
+
return ready.account;
|
|
110
|
+
params.onSkip?.(pick.id, ready.error);
|
|
111
|
+
remaining = remaining.filter((a) => a !== pick);
|
|
112
|
+
}
|
|
113
|
+
return undefined;
|
|
114
|
+
}
|