@drakon-systems/multi-clawd 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +452 -0
- package/dist/account-env.js +35 -0
- package/dist/alerts.js +28 -0
- package/dist/catalog-source.js +34 -0
- package/dist/chain-audit.js +202 -0
- package/dist/degrade.js +32 -0
- package/dist/exec-policy.js +15 -0
- package/dist/health.js +83 -0
- package/dist/index.js +508 -0
- package/dist/login-health.js +91 -0
- package/dist/models.js +69 -0
- package/dist/setup-core.js +140 -0
- package/dist/shim-core.js +218 -0
- package/dist/shim.js +151 -0
- package/dist/sticky.js +24 -0
- package/dist/token-resolution.js +62 -0
- package/dist/watchdog-core.js +53 -0
- package/openclaw.plugin.json +134 -0
- package/package.json +65 -0
- package/scripts/doctor.mjs +340 -0
- package/scripts/eviction-watchdog.mjs +178 -0
- package/scripts/setup.mjs +231 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,508 @@
|
|
|
1
|
+
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
|
2
|
+
import { resolvePluginConfigObject, resolveLivePluginConfigObject, } from "openclaw/plugin-sdk/plugin-config-runtime";
|
|
3
|
+
import { CLI_FRESH_WATCHDOG_DEFAULTS, CLI_RESUME_WATCHDOG_DEFAULTS, } from "openclaw/plugin-sdk/cli-backend";
|
|
4
|
+
import { homedir } from "node:os";
|
|
5
|
+
import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
6
|
+
import { dirname, join, resolve } from "node:path";
|
|
7
|
+
import { fileURLToPath } from "node:url";
|
|
8
|
+
import { MODEL_ALIASES, buildCatalogEntries, canonicalModelId, isModernClaudeModelId, resolveModelSpec, } from "./models.js";
|
|
9
|
+
import { decideDegradation, matchesPin } from "./degrade.js";
|
|
10
|
+
import { resolveExecMode, permissionModeArgs } from "./exec-policy.js";
|
|
11
|
+
import { resolveBaseModelIds } from "./catalog-source.js";
|
|
12
|
+
import { classifyAccountHealth } from "./health.js";
|
|
13
|
+
import { decideStickySelection } from "./sticky.js";
|
|
14
|
+
import { createTokenRefResolver, isSecretRefShape, } from "./token-resolution.js";
|
|
15
|
+
import { resolveSecretRefValues } from "openclaw/plugin-sdk/secret-ref-runtime";
|
|
16
|
+
import { addAlert, clearAlert, pendingAlertText } from "./alerts.js";
|
|
17
|
+
import { buildAccountChildEnv, validateAccountTokenSources } from "./account-env.js";
|
|
18
|
+
import { checkAccountCredential, createRefProbeTracker, } from "./login-health.js";
|
|
19
|
+
import { execFileSync } from "node:child_process";
|
|
20
|
+
const BASE_ARGS = [
|
|
21
|
+
"-p",
|
|
22
|
+
"--output-format",
|
|
23
|
+
"stream-json",
|
|
24
|
+
"--include-partial-messages",
|
|
25
|
+
"--verbose",
|
|
26
|
+
"--setting-sources",
|
|
27
|
+
"user",
|
|
28
|
+
"--allowedTools",
|
|
29
|
+
"mcp__openclaw__*",
|
|
30
|
+
"--disallowedTools",
|
|
31
|
+
"ScheduleWakeup,CronCreate,Bash(run_in_background:true),Monitor",
|
|
32
|
+
];
|
|
33
|
+
const CLEAR_ENV = [
|
|
34
|
+
"ANTHROPIC_API_KEY",
|
|
35
|
+
"ANTHROPIC_API_KEY_OLD",
|
|
36
|
+
"ANTHROPIC_API_TOKEN",
|
|
37
|
+
"ANTHROPIC_AUTH_TOKEN",
|
|
38
|
+
"ANTHROPIC_BASE_URL",
|
|
39
|
+
"ANTHROPIC_CUSTOM_HEADERS",
|
|
40
|
+
"ANTHROPIC_OAUTH_TOKEN",
|
|
41
|
+
"ANTHROPIC_UNIX_SOCKET",
|
|
42
|
+
"CLAUDE_CONFIG_DIR",
|
|
43
|
+
"CLAUDE_CODE_API_KEY_FILE_DESCRIPTOR",
|
|
44
|
+
"CLAUDE_CODE_ENTRYPOINT",
|
|
45
|
+
"CLAUDE_CODE_OAUTH_REFRESH_TOKEN",
|
|
46
|
+
"CLAUDE_CODE_OAUTH_SCOPES",
|
|
47
|
+
"CLAUDE_CODE_OAUTH_TOKEN",
|
|
48
|
+
"CLAUDE_CODE_OAUTH_TOKEN_FILE_DESCRIPTOR",
|
|
49
|
+
"CLAUDE_CODE_PLUGIN_CACHE_DIR",
|
|
50
|
+
"CLAUDE_CODE_PLUGIN_SEED_DIR",
|
|
51
|
+
"CLAUDE_CODE_REMOTE",
|
|
52
|
+
"CLAUDE_CODE_USE_COWORK_PLUGINS",
|
|
53
|
+
"CLAUDE_CODE_USE_BEDROCK",
|
|
54
|
+
"CLAUDE_CODE_USE_FOUNDRY",
|
|
55
|
+
"CLAUDE_CODE_USE_VERTEX",
|
|
56
|
+
];
|
|
57
|
+
function expandHome(p) {
|
|
58
|
+
if (p === "~")
|
|
59
|
+
return homedir();
|
|
60
|
+
if (p.startsWith("~/"))
|
|
61
|
+
return resolve(homedir(), p.slice(2));
|
|
62
|
+
return resolve(p);
|
|
63
|
+
}
|
|
64
|
+
let activeTokenResolver;
|
|
65
|
+
let alertState = { alerts: [] };
|
|
66
|
+
let loginProbeTimer;
|
|
67
|
+
function raiseAlert(alert) {
|
|
68
|
+
alertState = addAlert(alertState, alert, Date.now());
|
|
69
|
+
}
|
|
70
|
+
function ingestAlertSpool() {
|
|
71
|
+
const spool = join(homedir(), ".openclaw", "state", "multi-clawd", "alerts-spool.jsonl");
|
|
72
|
+
let raw;
|
|
73
|
+
try {
|
|
74
|
+
raw = readFileSync(spool, "utf8");
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
try {
|
|
80
|
+
for (const line of raw.split("\n")) {
|
|
81
|
+
if (!line.trim())
|
|
82
|
+
continue;
|
|
83
|
+
try {
|
|
84
|
+
const alert = JSON.parse(line);
|
|
85
|
+
if (typeof alert.key === "string" && typeof alert.text === "string") {
|
|
86
|
+
alertState = addAlert(alertState, {
|
|
87
|
+
key: alert.key,
|
|
88
|
+
severity: alert.severity === "error" ? "error" : "info",
|
|
89
|
+
text: alert.text,
|
|
90
|
+
}, alert.at ?? Date.now());
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
rmSync(spool, { force: true });
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const LOGIN_PROBE_INTERVAL_MS = 15 * 60 * 1000;
|
|
102
|
+
const LOGIN_PROBE_INITIAL_DELAY_MS = 45 * 1000;
|
|
103
|
+
const realCredentialIo = {
|
|
104
|
+
readFile: (p) => readFileSync(expandHome(p), "utf8"),
|
|
105
|
+
keychainHasClaudeCredentials: () => {
|
|
106
|
+
try {
|
|
107
|
+
execFileSync("security", ["find-generic-password", "-s", "Claude Code-credentials"], {
|
|
108
|
+
stdio: "ignore",
|
|
109
|
+
});
|
|
110
|
+
return true;
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
return false;
|
|
114
|
+
}
|
|
115
|
+
},
|
|
116
|
+
platform: process.platform,
|
|
117
|
+
};
|
|
118
|
+
function startLoginHealthProbe(accounts, logger) {
|
|
119
|
+
if (loginProbeTimer)
|
|
120
|
+
clearInterval(loginProbeTimer);
|
|
121
|
+
const lastStatus = new Map();
|
|
122
|
+
const refTrackers = new Map();
|
|
123
|
+
const probe = async () => {
|
|
124
|
+
const now = Date.now();
|
|
125
|
+
for (const account of accounts) {
|
|
126
|
+
let status;
|
|
127
|
+
let reason;
|
|
128
|
+
if (isSecretRefShape(account.oauthTokenRef) && !account.oauthTokenFile && !account.native) {
|
|
129
|
+
let tracker = refTrackers.get(account.id);
|
|
130
|
+
if (!tracker) {
|
|
131
|
+
tracker = createRefProbeTracker();
|
|
132
|
+
refTrackers.set(account.id, tracker);
|
|
133
|
+
}
|
|
134
|
+
const result = (await activeTokenResolver?.resolveDetailed(account.oauthTokenRef)) ?? {
|
|
135
|
+
failure: "provider_error",
|
|
136
|
+
};
|
|
137
|
+
const outcome = tracker.observe(result, now);
|
|
138
|
+
status = outcome.status;
|
|
139
|
+
reason = outcome.reason;
|
|
140
|
+
}
|
|
141
|
+
else {
|
|
142
|
+
const check = checkAccountCredential(account, realCredentialIo);
|
|
143
|
+
status = check.status;
|
|
144
|
+
reason = check.reason;
|
|
145
|
+
}
|
|
146
|
+
const previous = lastStatus.get(account.id);
|
|
147
|
+
lastStatus.set(account.id, status);
|
|
148
|
+
if (status === "broken" && previous !== "broken") {
|
|
149
|
+
const text = `account "${account.id}" login looks dead (${reason ?? "unknown"}) — turns on it will fail until fixed`;
|
|
150
|
+
logger.error(`[multi-clawd] ${text}`);
|
|
151
|
+
raiseAlert({ key: `login:${account.id}`, severity: "error", text });
|
|
152
|
+
}
|
|
153
|
+
else if (status === "degraded" && previous !== "degraded") {
|
|
154
|
+
logger.info(`[multi-clawd] account "${account.id}" login degraded: ${reason ?? "resolver error"}`);
|
|
155
|
+
}
|
|
156
|
+
else if (status === "ok" && (previous === "broken" || previous === "degraded")) {
|
|
157
|
+
logger.info(`[multi-clawd] account "${account.id}" login recovered`);
|
|
158
|
+
alertState = clearAlert(alertState, `login:${account.id}`);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
};
|
|
162
|
+
const initial = setTimeout(() => void probe().catch(() => { }), LOGIN_PROBE_INITIAL_DELAY_MS);
|
|
163
|
+
initial.unref?.();
|
|
164
|
+
loginProbeTimer = setInterval(() => void probe().catch(() => { }), LOGIN_PROBE_INTERVAL_MS);
|
|
165
|
+
loginProbeTimer.unref?.();
|
|
166
|
+
}
|
|
167
|
+
function peekToken(account) {
|
|
168
|
+
if (account.native)
|
|
169
|
+
return undefined;
|
|
170
|
+
if (account.oauthTokenFile) {
|
|
171
|
+
return readFileSync(expandHome(account.oauthTokenFile), "utf8").trim();
|
|
172
|
+
}
|
|
173
|
+
if (isSecretRefShape(account.oauthTokenRef)) {
|
|
174
|
+
return activeTokenResolver?.peek(account.oauthTokenRef);
|
|
175
|
+
}
|
|
176
|
+
if (account.configDir)
|
|
177
|
+
return undefined;
|
|
178
|
+
throw new Error(`[multi-clawd] account "${account.id}" needs oauthTokenFile, oauthTokenRef, or configDir`);
|
|
179
|
+
}
|
|
180
|
+
async function resolveTokenAsync(account) {
|
|
181
|
+
if (isSecretRefShape(account.oauthTokenRef) && !account.native && !account.oauthTokenFile) {
|
|
182
|
+
return activeTokenResolver?.resolve(account.oauthTokenRef);
|
|
183
|
+
}
|
|
184
|
+
return peekToken(account);
|
|
185
|
+
}
|
|
186
|
+
function buildRuntimeModel(account, modelId) {
|
|
187
|
+
const id = canonicalModelId(modelId);
|
|
188
|
+
if (!id)
|
|
189
|
+
return undefined;
|
|
190
|
+
const label = account.label ?? account.id;
|
|
191
|
+
const spec = resolveModelSpec(id);
|
|
192
|
+
return {
|
|
193
|
+
id,
|
|
194
|
+
name: `${spec.name} (${label})`,
|
|
195
|
+
provider: account.id,
|
|
196
|
+
api: "anthropic-messages",
|
|
197
|
+
baseUrl: "https://api.anthropic.com",
|
|
198
|
+
reasoning: true,
|
|
199
|
+
input: ["text", "image"],
|
|
200
|
+
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
|
201
|
+
contextWindow: spec.contextWindow,
|
|
202
|
+
maxTokens: spec.maxTokens,
|
|
203
|
+
...(id === "claude-fable-5" || id === "claude-mythos-5"
|
|
204
|
+
? { thinkingLevelMap: { xhigh: "xhigh", max: "max" } }
|
|
205
|
+
: {}),
|
|
206
|
+
mediaInput: {
|
|
207
|
+
image: {
|
|
208
|
+
maxSidePx: spec.imageMaxSidePx,
|
|
209
|
+
preferredSidePx: spec.imageMaxSidePx,
|
|
210
|
+
tokenMode: "provider",
|
|
211
|
+
},
|
|
212
|
+
},
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
const SHIM_PATH = fileURLToPath(new URL("./shim.js", import.meta.url));
|
|
216
|
+
export function healthStateFile(accountId) {
|
|
217
|
+
return join(homedir(), ".openclaw", "state", "multi-clawd", `${accountId}.json`);
|
|
218
|
+
}
|
|
219
|
+
function buildBackend(account, execMode) {
|
|
220
|
+
return {
|
|
221
|
+
id: account.id,
|
|
222
|
+
liveTest: {
|
|
223
|
+
defaultModelRef: `${account.id}/${account.defaultModel ?? "claude-fable-5"}`,
|
|
224
|
+
defaultImageProbe: true,
|
|
225
|
+
defaultMcpProbe: true,
|
|
226
|
+
docker: {
|
|
227
|
+
npmPackage: "@anthropic-ai/claude-code",
|
|
228
|
+
binaryName: "claude",
|
|
229
|
+
},
|
|
230
|
+
},
|
|
231
|
+
bundleMcp: true,
|
|
232
|
+
bundleMcpMode: "claude-config-file",
|
|
233
|
+
nativeToolMode: "always-on",
|
|
234
|
+
sideQuestionToolMode: "disabled",
|
|
235
|
+
ownsNativeCompaction: true,
|
|
236
|
+
config: {
|
|
237
|
+
command: process.execPath,
|
|
238
|
+
args: [SHIM_PATH, ...BASE_ARGS, ...permissionModeArgs(execMode)],
|
|
239
|
+
resumeArgs: [SHIM_PATH, ...BASE_ARGS, ...permissionModeArgs(execMode), "--resume", "{sessionId}"],
|
|
240
|
+
output: "jsonl",
|
|
241
|
+
liveSession: "claude-stdio",
|
|
242
|
+
input: "stdin",
|
|
243
|
+
jsonlDialect: "claude-stream-json",
|
|
244
|
+
modelArg: "--model",
|
|
245
|
+
modelAliases: { ...MODEL_ALIASES },
|
|
246
|
+
imageArg: "@",
|
|
247
|
+
imagePathScope: "workspace",
|
|
248
|
+
sessionArg: "--session-id",
|
|
249
|
+
sessionMode: "always",
|
|
250
|
+
reseedFromRawTranscriptWhenUncompacted: false,
|
|
251
|
+
sessionIdFields: ["session_id", "sessionId", "conversation_id", "conversationId"],
|
|
252
|
+
systemPromptFileArg: "--append-system-prompt-file",
|
|
253
|
+
systemPromptMode: "append",
|
|
254
|
+
systemPromptWhen: "always",
|
|
255
|
+
clearEnv: [...CLEAR_ENV],
|
|
256
|
+
reliability: {
|
|
257
|
+
watchdog: {
|
|
258
|
+
fresh: { ...CLI_FRESH_WATCHDOG_DEFAULTS },
|
|
259
|
+
resume: { ...CLI_RESUME_WATCHDOG_DEFAULTS },
|
|
260
|
+
},
|
|
261
|
+
},
|
|
262
|
+
serialize: true,
|
|
263
|
+
},
|
|
264
|
+
async prepareExecution(_ctx) {
|
|
265
|
+
return { env: await buildAccountEnv(account) };
|
|
266
|
+
},
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
async function buildAccountEnv(account) {
|
|
270
|
+
const token = await resolveTokenAsync(account);
|
|
271
|
+
return buildAccountChildEnv(account, token, healthStateFile(account.id));
|
|
272
|
+
}
|
|
273
|
+
function buildCatalogProvider(account) {
|
|
274
|
+
return {
|
|
275
|
+
id: account.id,
|
|
276
|
+
label: account.label ?? `Claude Code (${account.id})`,
|
|
277
|
+
auth: [],
|
|
278
|
+
resolveSyntheticAuth: () => {
|
|
279
|
+
try {
|
|
280
|
+
const token = peekToken(account);
|
|
281
|
+
if (!token)
|
|
282
|
+
return undefined;
|
|
283
|
+
return {
|
|
284
|
+
apiKey: token,
|
|
285
|
+
source: `multi-clawd ${account.id} token`,
|
|
286
|
+
mode: "token",
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
return undefined;
|
|
291
|
+
}
|
|
292
|
+
},
|
|
293
|
+
augmentModelCatalog: async () => buildCatalogEntries(account, await resolveBaseModelIds()),
|
|
294
|
+
resolveDynamicModel: (ctx) => buildRuntimeModel(account, ctx.modelId),
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
export default definePluginEntry({
|
|
298
|
+
id: "multi-clawd",
|
|
299
|
+
name: "multi-clawd",
|
|
300
|
+
description: "Register additional Claude Code logins as first-class OpenClaw CLI backends for cross-account failover.",
|
|
301
|
+
register(api) {
|
|
302
|
+
const runtimeConfigLoader = (api.runtime?.config?.current
|
|
303
|
+
? () => api.runtime.config.current()
|
|
304
|
+
: undefined);
|
|
305
|
+
const candidates = [
|
|
306
|
+
resolveLivePluginConfigObject(runtimeConfigLoader, "multi-clawd", api.pluginConfig),
|
|
307
|
+
resolvePluginConfigObject(api.config, "multi-clawd"),
|
|
308
|
+
api.pluginConfig,
|
|
309
|
+
];
|
|
310
|
+
const hasAccounts = (c) => Array.isArray(c?.accounts) &&
|
|
311
|
+
(c.accounts?.length ?? 0) > 0;
|
|
312
|
+
const cfg = (candidates.find(hasAccounts) ?? {});
|
|
313
|
+
const accounts = Array.isArray(cfg.accounts) ? cfg.accounts : [];
|
|
314
|
+
const sourceNames = [
|
|
315
|
+
"runtime-live",
|
|
316
|
+
"static-api-config",
|
|
317
|
+
"startup-pluginConfig",
|
|
318
|
+
];
|
|
319
|
+
if (accounts.length === 0) {
|
|
320
|
+
api.logger.warn(`[multi-clawd] no accounts configured — nothing to register (sources: ${candidates
|
|
321
|
+
.map((c, i) => `${sourceNames[i]}=${hasAccounts(c) ? "ok" : c ? "empty" : "absent"}`)
|
|
322
|
+
.join(", ")})`);
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
api.logger.info(`[multi-clawd] register() pass — config source: ${sourceNames[candidates.findIndex(hasAccounts)] ?? "unknown"}, accounts: ${accounts.length}`);
|
|
326
|
+
{
|
|
327
|
+
const logger = api.logger;
|
|
328
|
+
const currentConfig = api.runtime?.config?.current;
|
|
329
|
+
activeTokenResolver = createTokenRefResolver({
|
|
330
|
+
resolveRefs: (refs) => resolveSecretRefValues(refs, {
|
|
331
|
+
config: (currentConfig ? currentConfig() : api.config),
|
|
332
|
+
}),
|
|
333
|
+
redact: true,
|
|
334
|
+
onError: (_ref, error) => logger.error(`[multi-clawd] ${String(error)}`),
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
const execMode = resolveExecMode(runtimeConfigLoader?.() ?? api.config);
|
|
338
|
+
api.logger.info(`[multi-clawd] exec policy: ${execMode ?? "unknown"} → permission-mode ${permissionModeArgs(execMode).length ? "bypassPermissions" : "default (no override)"}`);
|
|
339
|
+
const seen = new Set();
|
|
340
|
+
for (const account of accounts) {
|
|
341
|
+
const id = account?.id?.trim();
|
|
342
|
+
if (!id) {
|
|
343
|
+
api.logger.warn("[multi-clawd] skipping account without id");
|
|
344
|
+
continue;
|
|
345
|
+
}
|
|
346
|
+
for (const warning of validateAccountTokenSources(account)) {
|
|
347
|
+
api.logger.warn(`[multi-clawd] ${warning}`);
|
|
348
|
+
}
|
|
349
|
+
if (id === "claude-cli" || seen.has(id)) {
|
|
350
|
+
api.logger.warn(`[multi-clawd] skipping account "${id}" — id collides with an existing backend`);
|
|
351
|
+
continue;
|
|
352
|
+
}
|
|
353
|
+
seen.add(id);
|
|
354
|
+
const normalized = { ...account, id };
|
|
355
|
+
api.registerCliBackend(buildBackend(normalized, execMode));
|
|
356
|
+
api.registerProvider(buildCatalogProvider(normalized));
|
|
357
|
+
}
|
|
358
|
+
registerPoolBackend(api, cfg.pool, accounts, seen, execMode);
|
|
359
|
+
{
|
|
360
|
+
const logger = api.logger;
|
|
361
|
+
try {
|
|
362
|
+
api.on("heartbeat_prompt_contribution", () => {
|
|
363
|
+
ingestAlertSpool();
|
|
364
|
+
const text = pendingAlertText(alertState, Date.now());
|
|
365
|
+
return text ? { appendContext: text } : undefined;
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
catch (err) {
|
|
369
|
+
logger.warn(`[multi-clawd] heartbeat alert hook unavailable: ${String(err)}`);
|
|
370
|
+
}
|
|
371
|
+
startLoginHealthProbe(accounts.filter((a) => seen.has(a.id.trim())), logger);
|
|
372
|
+
}
|
|
373
|
+
api.logger.info(`[multi-clawd] registered ${seen.size} backend(s)+provider(s): ${[...seen].join(", ")}`);
|
|
374
|
+
},
|
|
375
|
+
});
|
|
376
|
+
function readHealthState(accountId) {
|
|
377
|
+
try {
|
|
378
|
+
return JSON.parse(readFileSync(healthStateFile(accountId), "utf8"));
|
|
379
|
+
}
|
|
380
|
+
catch {
|
|
381
|
+
return undefined;
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
function readStickyEntry(file) {
|
|
385
|
+
try {
|
|
386
|
+
const parsed = JSON.parse(readFileSync(file, "utf8"));
|
|
387
|
+
if (typeof parsed?.account === "string" && typeof parsed?.since === "number") {
|
|
388
|
+
return parsed;
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
catch {
|
|
392
|
+
}
|
|
393
|
+
return undefined;
|
|
394
|
+
}
|
|
395
|
+
function writeStickyEntry(file, entry, logger) {
|
|
396
|
+
try {
|
|
397
|
+
if (!entry) {
|
|
398
|
+
rmSync(file, { force: true });
|
|
399
|
+
return;
|
|
400
|
+
}
|
|
401
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
402
|
+
const tmp = `${file}.tmp-${process.pid}`;
|
|
403
|
+
writeFileSync(tmp, JSON.stringify(entry), { mode: 0o600 });
|
|
404
|
+
renameSync(tmp, file);
|
|
405
|
+
}
|
|
406
|
+
catch (err) {
|
|
407
|
+
logger.warn(`[multi-clawd] sticky state write failed: ${String(err)}`);
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
function registerPoolBackend(api, pool, accounts, registeredIds, execMode) {
|
|
411
|
+
const logger = api.logger;
|
|
412
|
+
if (!pool)
|
|
413
|
+
return;
|
|
414
|
+
const poolId = pool.id?.trim() || "clawd";
|
|
415
|
+
const memberIds = (pool.accounts ?? []).filter((id) => registeredIds.has(id));
|
|
416
|
+
const members = memberIds
|
|
417
|
+
.map((id) => accounts.find((a) => a.id.trim() === id))
|
|
418
|
+
.filter((a) => a !== undefined);
|
|
419
|
+
const ladder = (pool.degrade?.ladder ?? []).filter((m) => {
|
|
420
|
+
if (isModernClaudeModelId(m))
|
|
421
|
+
return true;
|
|
422
|
+
logger.warn(`[multi-clawd] pool "${poolId}": ignoring invalid degrade ladder entry "${m}"`);
|
|
423
|
+
return false;
|
|
424
|
+
});
|
|
425
|
+
const pins = pool.degrade?.pins ?? [];
|
|
426
|
+
if (members.length < 2 && !(members.length === 1 && ladder.length > 0)) {
|
|
427
|
+
logger.warn(`[multi-clawd] pool "${poolId}" has ${members.length} registered account(s) — need at least 2 (or 1 with a degrade ladder); pool not registered`);
|
|
428
|
+
return;
|
|
429
|
+
}
|
|
430
|
+
if (poolId === "claude-cli" || registeredIds.has(poolId)) {
|
|
431
|
+
logger.warn(`[multi-clawd] pool id "${poolId}" collides with an existing backend — pool not registered`);
|
|
432
|
+
return;
|
|
433
|
+
}
|
|
434
|
+
const options = {
|
|
435
|
+
utilizationThreshold: pool.utilizationThreshold,
|
|
436
|
+
staleAfterMs: pool.staleAfterMs,
|
|
437
|
+
};
|
|
438
|
+
const poolAccount = {
|
|
439
|
+
id: poolId,
|
|
440
|
+
label: pool.label ?? `Claude pool (${memberIds.join("+")})`,
|
|
441
|
+
models: pool.models,
|
|
442
|
+
defaultModel: pool.defaultModel,
|
|
443
|
+
};
|
|
444
|
+
const minDwellMs = pool.minDwellMs;
|
|
445
|
+
const stickyFile = join(homedir(), ".openclaw", "state", "multi-clawd", `pool-${poolId}.sticky.json`);
|
|
446
|
+
const backend = buildBackend(poolAccount, execMode);
|
|
447
|
+
backend.prepareExecution = async (ctx) => {
|
|
448
|
+
const now = Date.now();
|
|
449
|
+
const requestedModel = canonicalModelId(ctx.modelId) ?? ctx.modelId;
|
|
450
|
+
const verdicts = members.map((a) => ({
|
|
451
|
+
id: a.id,
|
|
452
|
+
health: classifyAccountHealth(readHealthState(a.id), options, now, requestedModel),
|
|
453
|
+
}));
|
|
454
|
+
const previousSticky = readStickyEntry(stickyFile);
|
|
455
|
+
const decision = decideStickySelection({
|
|
456
|
+
verdicts: verdicts.map((v) => ({ id: v.id, verdict: v.health.verdict })),
|
|
457
|
+
sticky: previousSticky,
|
|
458
|
+
nowMs: now,
|
|
459
|
+
minDwellMs,
|
|
460
|
+
});
|
|
461
|
+
const chosen = members.find((a) => a.id === decision.account) ?? members[0];
|
|
462
|
+
const previousAccount = previousSticky?.account ?? members[0].id;
|
|
463
|
+
if (decision.account !== previousAccount) {
|
|
464
|
+
const home = verdicts[0];
|
|
465
|
+
const line = decision.account === members[0].id
|
|
466
|
+
? `pool ${poolId}: returning home to ${decision.account}`
|
|
467
|
+
: `pool ${poolId}: rotated to ${decision.account} from ${previousAccount} (${home.health.reason ?? home.health.verdict})`;
|
|
468
|
+
logger.info(`[multi-clawd] ${line}`);
|
|
469
|
+
raiseAlert({ key: `rotation:${poolId}`, severity: "info", text: line });
|
|
470
|
+
}
|
|
471
|
+
if (verdicts.every((v) => v.health.verdict === "exhausted")) {
|
|
472
|
+
raiseAlert({
|
|
473
|
+
key: `pool-exhausted:${poolId}:${requestedModel}`,
|
|
474
|
+
severity: "error",
|
|
475
|
+
text: `pool ${poolId}: every account is exhausted for ${requestedModel} — turns are degrading or falling through the chain`,
|
|
476
|
+
});
|
|
477
|
+
}
|
|
478
|
+
writeStickyEntry(stickyFile, decision.sticky, logger);
|
|
479
|
+
const env = await buildAccountEnv(chosen);
|
|
480
|
+
if (ladder.length > 0) {
|
|
481
|
+
const pinned = matchesPin(pins, {
|
|
482
|
+
agentDir: ctx.agentDir ?? "",
|
|
483
|
+
workspaceDir: ctx.workspaceDir,
|
|
484
|
+
});
|
|
485
|
+
const degradation = pinned
|
|
486
|
+
? undefined
|
|
487
|
+
: decideDegradation({
|
|
488
|
+
verdicts: verdicts.map((v) => ({ id: v.id, verdict: v.health.verdict })),
|
|
489
|
+
requestedModel: ctx.modelId,
|
|
490
|
+
ladder,
|
|
491
|
+
});
|
|
492
|
+
if (degradation) {
|
|
493
|
+
env.MULTI_CLAWD_MODEL_OVERRIDE = degradation.model;
|
|
494
|
+
const line = `pool ${poolId}: degrading ${ctx.modelId} → ${degradation.model} on ${chosen.id} (${degradation.reason})`;
|
|
495
|
+
logger.info(`[multi-clawd] ${line}`);
|
|
496
|
+
raiseAlert({ key: `degrade:${poolId}`, severity: "info", text: line });
|
|
497
|
+
}
|
|
498
|
+
else if (pinned && verdicts.every((v) => v.health.verdict === "exhausted")) {
|
|
499
|
+
logger.info(`[multi-clawd] pool ${poolId}: pinned lane keeps ${ctx.modelId} despite exhausted pool (will fail over via the chain)`);
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
return { env };
|
|
503
|
+
};
|
|
504
|
+
api.registerCliBackend(backend);
|
|
505
|
+
api.registerProvider(buildCatalogProvider(poolAccount));
|
|
506
|
+
registeredIds.add(poolId);
|
|
507
|
+
logger.info(`[multi-clawd] pool "${poolId}" active — accounts: ${memberIds.join(" → ")}, threshold: ${options.utilizationThreshold ?? 0.85}`);
|
|
508
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
export function createRefProbeTracker(options = {}) {
|
|
2
|
+
const deadAfterConsecutive = options.deadAfterConsecutive ?? 3;
|
|
3
|
+
const deadAfterMs = options.deadAfterMs ?? 10 * 60 * 1000;
|
|
4
|
+
let consecutive = 0;
|
|
5
|
+
let firstFailureAt;
|
|
6
|
+
return {
|
|
7
|
+
observe(result, nowMs) {
|
|
8
|
+
if (result.value !== undefined) {
|
|
9
|
+
consecutive = 0;
|
|
10
|
+
firstFailureAt = undefined;
|
|
11
|
+
return { status: "ok" };
|
|
12
|
+
}
|
|
13
|
+
if (result.failure === "empty_result") {
|
|
14
|
+
consecutive = 0;
|
|
15
|
+
firstFailureAt = undefined;
|
|
16
|
+
return {
|
|
17
|
+
status: "broken",
|
|
18
|
+
reason: "oauthTokenRef resolved to nothing (credential problem)",
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
if (consecutive === 0)
|
|
22
|
+
firstFailureAt = nowMs;
|
|
23
|
+
consecutive += 1;
|
|
24
|
+
const elapsed = nowMs - (firstFailureAt ?? nowMs);
|
|
25
|
+
if (consecutive >= deadAfterConsecutive && elapsed >= deadAfterMs) {
|
|
26
|
+
return {
|
|
27
|
+
status: "broken",
|
|
28
|
+
reason: `resolver failing ${deadAfterConsecutive}+ consecutive probes over ${Math.round(deadAfterMs / 60000)}m`,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
return {
|
|
32
|
+
status: "degraded",
|
|
33
|
+
reason: `resolver error (network?) — streak ${consecutive}/${deadAfterConsecutive}`,
|
|
34
|
+
};
|
|
35
|
+
},
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
function looksLikeSetupToken(value) {
|
|
39
|
+
return /^sk-ant-[a-z0-9]+-/.test(value.trim());
|
|
40
|
+
}
|
|
41
|
+
function checkCredentialsJson(io, dir) {
|
|
42
|
+
let raw;
|
|
43
|
+
try {
|
|
44
|
+
raw = io.readFile(`${dir}/.credentials.json`);
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
return { status: "broken", reason: `${dir}/.credentials.json unreadable` };
|
|
48
|
+
}
|
|
49
|
+
try {
|
|
50
|
+
const parsed = JSON.parse(raw);
|
|
51
|
+
const token = parsed?.claudeAiOauth?.accessToken;
|
|
52
|
+
if (typeof token === "string" && token.trim().length > 0)
|
|
53
|
+
return { status: "ok" };
|
|
54
|
+
return { status: "broken", reason: `${dir}/.credentials.json access token is blank` };
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
return { status: "broken", reason: `${dir}/.credentials.json is not valid JSON` };
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
export function checkAccountCredential(account, io) {
|
|
61
|
+
if (account.oauthTokenFile) {
|
|
62
|
+
let raw;
|
|
63
|
+
try {
|
|
64
|
+
raw = io.readFile(account.oauthTokenFile);
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
return { status: "broken", reason: `${account.oauthTokenFile} unreadable` };
|
|
68
|
+
}
|
|
69
|
+
if (looksLikeSetupToken(raw))
|
|
70
|
+
return { status: "ok" };
|
|
71
|
+
return {
|
|
72
|
+
status: "broken",
|
|
73
|
+
reason: `${account.oauthTokenFile} does not contain a setup-token`,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
if (account.oauthTokenRef) {
|
|
77
|
+
return { status: "unknown" };
|
|
78
|
+
}
|
|
79
|
+
if (account.native) {
|
|
80
|
+
if (io.platform === "darwin") {
|
|
81
|
+
return io.keychainHasClaudeCredentials()
|
|
82
|
+
? { status: "ok" }
|
|
83
|
+
: { status: "broken", reason: "keychain has no Claude Code credentials" };
|
|
84
|
+
}
|
|
85
|
+
return checkCredentialsJson(io, "~/.claude");
|
|
86
|
+
}
|
|
87
|
+
if (account.configDir) {
|
|
88
|
+
return checkCredentialsJson(io, account.configDir);
|
|
89
|
+
}
|
|
90
|
+
return { status: "unknown" };
|
|
91
|
+
}
|
package/dist/models.js
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
export const MODEL_ALIASES = {
|
|
2
|
+
opus: "opus",
|
|
3
|
+
"opus-4.8": "claude-opus-4-8",
|
|
4
|
+
"opus-4.7": "claude-opus-4-7",
|
|
5
|
+
"opus-4.6": "claude-opus-4-6",
|
|
6
|
+
sonnet: "sonnet",
|
|
7
|
+
"sonnet-4.6": "claude-sonnet-4-6",
|
|
8
|
+
haiku: "haiku",
|
|
9
|
+
};
|
|
10
|
+
const KNOWN_SPECS = {
|
|
11
|
+
"claude-opus-4-8": { name: "Claude Opus 4.8", contextWindow: 1048576, maxTokens: 128000 },
|
|
12
|
+
"claude-opus-4-7": { name: "Claude Opus 4.7", contextWindow: 1048576, maxTokens: 64000 },
|
|
13
|
+
"claude-opus-4-6": { name: "Claude Opus 4.6", contextWindow: 1048576, maxTokens: 64000 },
|
|
14
|
+
"claude-sonnet-4-6": { name: "Claude Sonnet 4.6", contextWindow: 1048576, maxTokens: 64000 },
|
|
15
|
+
"claude-sonnet-5": { name: "Claude Sonnet 5", contextWindow: 200000, maxTokens: 64000 },
|
|
16
|
+
"claude-fable-5": { name: "Claude Fable 5", contextWindow: 1000000, maxTokens: 128000 },
|
|
17
|
+
"claude-haiku-4-5": { name: "Claude Haiku 4.5", contextWindow: 200000, maxTokens: 64000 },
|
|
18
|
+
};
|
|
19
|
+
export const FALLBACK_MODEL_IDS = Object.keys(KNOWN_SPECS);
|
|
20
|
+
const MODERN_CLAUDE_ID_RE = /^claude(-[a-z0-9]+(\.[a-z0-9]+)*)+$/;
|
|
21
|
+
export function isModernClaudeModelId(id) {
|
|
22
|
+
return MODERN_CLAUDE_ID_RE.test(id);
|
|
23
|
+
}
|
|
24
|
+
const CLI_SHORT_ALIASES = new Set(["opus", "sonnet", "haiku"]);
|
|
25
|
+
export function canonicalModelId(modelId) {
|
|
26
|
+
const trimmed = modelId?.trim();
|
|
27
|
+
if (!trimmed)
|
|
28
|
+
return undefined;
|
|
29
|
+
const aliased = MODEL_ALIASES[trimmed] ?? trimmed;
|
|
30
|
+
if (CLI_SHORT_ALIASES.has(aliased))
|
|
31
|
+
return aliased;
|
|
32
|
+
return isModernClaudeModelId(aliased) ? aliased : undefined;
|
|
33
|
+
}
|
|
34
|
+
function deriveName(id) {
|
|
35
|
+
return id
|
|
36
|
+
.split("-")
|
|
37
|
+
.map((part) => (/^[a-z]/.test(part) ? part[0].toUpperCase() + part.slice(1) : part))
|
|
38
|
+
.join(" ")
|
|
39
|
+
.replace(/^Claude (\w+) (\d+) (\d)$/, "Claude $1 $2.$3");
|
|
40
|
+
}
|
|
41
|
+
export function resolveModelSpec(id) {
|
|
42
|
+
const known = KNOWN_SPECS[id];
|
|
43
|
+
const imageMaxSidePx = id === "claude-opus-4-8" || id === "claude-opus-4-7" ? 2576 : 1568;
|
|
44
|
+
if (known)
|
|
45
|
+
return { ...known, imageMaxSidePx };
|
|
46
|
+
return { name: deriveName(id), contextWindow: 200000, maxTokens: 64000, imageMaxSidePx };
|
|
47
|
+
}
|
|
48
|
+
export function buildCatalogEntries(account, baseIds) {
|
|
49
|
+
const label = account.label ?? account.id;
|
|
50
|
+
const ids = [...new Set([...baseIds, ...(account.models ?? [])])].filter(isModernClaudeModelId);
|
|
51
|
+
return ids.map((id) => {
|
|
52
|
+
const spec = resolveModelSpec(id);
|
|
53
|
+
return {
|
|
54
|
+
id,
|
|
55
|
+
name: `${spec.name} (${label})`,
|
|
56
|
+
provider: account.id,
|
|
57
|
+
reasoning: true,
|
|
58
|
+
input: ["text", "image"],
|
|
59
|
+
contextWindow: spec.contextWindow,
|
|
60
|
+
mediaInput: {
|
|
61
|
+
image: {
|
|
62
|
+
maxSidePx: spec.imageMaxSidePx,
|
|
63
|
+
preferredSidePx: spec.imageMaxSidePx,
|
|
64
|
+
tokenMode: "provider",
|
|
65
|
+
},
|
|
66
|
+
},
|
|
67
|
+
};
|
|
68
|
+
});
|
|
69
|
+
}
|