@drakon-systems/multi-clawd 1.8.11 → 1.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -1
- package/SECURITY.md +1 -0
- package/dist/account-env.js +5 -0
- package/dist/chain-audit.js +9 -7
- package/dist/direct-report.js +119 -0
- package/dist/direct-route.js +156 -0
- package/dist/direct-sync.js +335 -0
- package/dist/explain-core.js +44 -2
- package/dist/index.js +102 -1
- package/dist/openclaw-runner.js +42 -0
- package/dist/session-handover.js +117 -0
- package/dist/setup-core.js +25 -0
- package/dist/shim.js +6 -0
- package/openclaw.plugin.json +66 -1
- package/package.json +1 -1
- package/scripts/cli.mjs +236 -4
- package/scripts/doctor.mjs +117 -2
- package/scripts/setup.mjs +74 -1
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
import { classifyAccountHealth } from "./health.js";
|
|
2
|
+
import { DIRECT_PROVIDER, planDirectOrder } from "./direct-route.js";
|
|
3
|
+
import { parseClaudeSetupToken } from "./hermes-core.js";
|
|
4
|
+
export function safeCliError(result) {
|
|
5
|
+
const text = `${result.stderr ?? ""}\n${result.stdout ?? ""}`;
|
|
6
|
+
const line = text
|
|
7
|
+
.split("\n")
|
|
8
|
+
.map((l) => l.trim())
|
|
9
|
+
.find((l) => /error|fail|refus|not found|missing|invalid|denied/i.test(l)) ??
|
|
10
|
+
text.split("\n").map((l) => l.trim()).find(Boolean) ??
|
|
11
|
+
"";
|
|
12
|
+
const masked = line.replace(/sk-ant-[A-Za-z0-9_-]+/g, "sk-ant-…").replace(/[A-Za-z0-9_-]{40,}/g, "…");
|
|
13
|
+
return `exit ${result.code}${masked ? `: ${masked.slice(0, 200)}` : ""}`;
|
|
14
|
+
}
|
|
15
|
+
function parseJson(stdout) {
|
|
16
|
+
const start = stdout.indexOf("{");
|
|
17
|
+
if (start < 0)
|
|
18
|
+
return undefined;
|
|
19
|
+
try {
|
|
20
|
+
return JSON.parse(stdout.slice(start));
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
return undefined;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
export function parseAuthList(stdout) {
|
|
27
|
+
const doc = parseJson(stdout);
|
|
28
|
+
if (!doc || !Array.isArray(doc.profiles))
|
|
29
|
+
return undefined;
|
|
30
|
+
return doc.profiles
|
|
31
|
+
.filter((p) => typeof p?.id === "string")
|
|
32
|
+
.filter((p) => p.provider === undefined || p.provider === DIRECT_PROVIDER)
|
|
33
|
+
.map((p) => ({ id: p.id, type: typeof p.type === "string" ? p.type : undefined }));
|
|
34
|
+
}
|
|
35
|
+
export function parseOrderGet(stdout) {
|
|
36
|
+
const doc = parseJson(stdout);
|
|
37
|
+
if (!doc || !("order" in doc))
|
|
38
|
+
return undefined;
|
|
39
|
+
if (doc.order === null || doc.order === undefined)
|
|
40
|
+
return { order: null };
|
|
41
|
+
if (!Array.isArray(doc.order))
|
|
42
|
+
return undefined;
|
|
43
|
+
return { order: doc.order.filter((v) => typeof v === "string") };
|
|
44
|
+
}
|
|
45
|
+
export function parseUnusableProfiles(stdout) {
|
|
46
|
+
const doc = parseJson(stdout);
|
|
47
|
+
const rows = doc?.auth?.unusableProfiles;
|
|
48
|
+
if (!Array.isArray(rows))
|
|
49
|
+
return undefined;
|
|
50
|
+
const out = [];
|
|
51
|
+
for (const row of rows) {
|
|
52
|
+
if (typeof row?.profileId !== "string" || typeof row.until !== "number")
|
|
53
|
+
continue;
|
|
54
|
+
if (row.provider !== undefined && row.provider !== DIRECT_PROVIDER)
|
|
55
|
+
continue;
|
|
56
|
+
out.push({
|
|
57
|
+
profileId: row.profileId,
|
|
58
|
+
kind: typeof row.kind === "string" ? row.kind : "cooldown",
|
|
59
|
+
reason: typeof row.reason === "string" ? row.reason : undefined,
|
|
60
|
+
until: row.until,
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
return out;
|
|
64
|
+
}
|
|
65
|
+
const AGENT_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/;
|
|
66
|
+
export function isValidAgentId(agentId) {
|
|
67
|
+
return typeof agentId === "string" && AGENT_ID_RE.test(agentId);
|
|
68
|
+
}
|
|
69
|
+
export function assertAgentId(agentId) {
|
|
70
|
+
if (!AGENT_ID_RE.test(agentId))
|
|
71
|
+
throw new Error(`invalid agent id "${agentId}"`);
|
|
72
|
+
return agentId;
|
|
73
|
+
}
|
|
74
|
+
export function authListArgs(agentId) {
|
|
75
|
+
return ["models", "auth", "list", "--agent", assertAgentId(agentId), "--provider", DIRECT_PROVIDER, "--json"];
|
|
76
|
+
}
|
|
77
|
+
export function orderGetArgs(agentId) {
|
|
78
|
+
return ["models", "auth", "order", "get", "--agent", assertAgentId(agentId), "--provider", DIRECT_PROVIDER, "--json"];
|
|
79
|
+
}
|
|
80
|
+
export function orderSetArgs(agentId, order) {
|
|
81
|
+
if (order.length === 0)
|
|
82
|
+
throw new Error("refusing to set an empty order");
|
|
83
|
+
for (const id of order) {
|
|
84
|
+
if (!id.startsWith(`${DIRECT_PROVIDER}:`))
|
|
85
|
+
throw new Error(`"${id}" is not an ${DIRECT_PROVIDER} profile id`);
|
|
86
|
+
}
|
|
87
|
+
return ["models", "auth", "order", "set", "--agent", assertAgentId(agentId), "--provider", DIRECT_PROVIDER, ...order];
|
|
88
|
+
}
|
|
89
|
+
export function pasteTokenArgs(agentId, profileId) {
|
|
90
|
+
return [
|
|
91
|
+
"models",
|
|
92
|
+
"auth",
|
|
93
|
+
"paste-token",
|
|
94
|
+
"--agent",
|
|
95
|
+
assertAgentId(agentId),
|
|
96
|
+
"--provider",
|
|
97
|
+
DIRECT_PROVIDER,
|
|
98
|
+
"--profile-id",
|
|
99
|
+
profileId,
|
|
100
|
+
];
|
|
101
|
+
}
|
|
102
|
+
export function buildTokenRefPlan(params) {
|
|
103
|
+
const { profileId } = params;
|
|
104
|
+
if (profileId.includes(".")) {
|
|
105
|
+
throw new Error(`profile id "${profileId}" contains "." and cannot be targeted by a secrets plan — choose a direct.profileId without dots`);
|
|
106
|
+
}
|
|
107
|
+
return {
|
|
108
|
+
version: 1,
|
|
109
|
+
protocolVersion: 1,
|
|
110
|
+
scrubEnv: false,
|
|
111
|
+
scrubAuthProfilesForProviderTargets: false,
|
|
112
|
+
targets: [
|
|
113
|
+
{
|
|
114
|
+
type: "auth-profiles.token.token",
|
|
115
|
+
path: `profiles.${profileId}.token`,
|
|
116
|
+
pathSegments: ["profiles", profileId, "token"],
|
|
117
|
+
agentId: assertAgentId(params.agentId),
|
|
118
|
+
authProfileProvider: DIRECT_PROVIDER,
|
|
119
|
+
ref: { source: params.ref.source, provider: params.ref.provider, id: params.ref.id },
|
|
120
|
+
},
|
|
121
|
+
],
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
export async function syncDirectProfiles(params) {
|
|
125
|
+
const { runner, agentId } = params;
|
|
126
|
+
const list = await runner(authListArgs(agentId), { timeoutMs: 60_000 });
|
|
127
|
+
const listed = list.code === 0 ? parseAuthList(list.stdout) : undefined;
|
|
128
|
+
if (!listed)
|
|
129
|
+
return { results: [], listError: safeCliError(list) };
|
|
130
|
+
const stored = new Set(listed.map((p) => p.id));
|
|
131
|
+
const results = [];
|
|
132
|
+
for (const member of params.members) {
|
|
133
|
+
const base = { accountId: member.accountId, profileId: member.profileId };
|
|
134
|
+
const { source } = member;
|
|
135
|
+
if (source.kind === "existing") {
|
|
136
|
+
results.push({ ...base, action: stored.has(member.profileId) ? "adopted" : "adopt-missing" });
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
if (source.kind !== "file" && source.kind !== "ref")
|
|
140
|
+
continue;
|
|
141
|
+
if (stored.has(member.profileId) && !params.resync) {
|
|
142
|
+
results.push({ ...base, action: "present" });
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
if (source.kind === "file") {
|
|
146
|
+
let token;
|
|
147
|
+
try {
|
|
148
|
+
token = parseClaudeSetupToken(params.readTokenFile(source.path));
|
|
149
|
+
}
|
|
150
|
+
catch (err) {
|
|
151
|
+
results.push({ ...base, action: "failed", detail: `token file: ${err.message}` });
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
if (params.dryRun) {
|
|
155
|
+
results.push({ ...base, action: "would-store", detail: "paste-token from the token file" });
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
const r = await runner(pasteTokenArgs(agentId, member.profileId), { input: token, timeoutMs: 120_000 });
|
|
159
|
+
results.push(r.code === 0
|
|
160
|
+
? { ...base, action: "stored", detail: "paste-token" }
|
|
161
|
+
: { ...base, action: "failed", detail: `paste-token ${safeCliError(r)}` });
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
let plan;
|
|
165
|
+
try {
|
|
166
|
+
plan = buildTokenRefPlan({ agentId, profileId: member.profileId, ref: source.ref });
|
|
167
|
+
}
|
|
168
|
+
catch (err) {
|
|
169
|
+
results.push({ ...base, action: "failed", detail: err.message });
|
|
170
|
+
continue;
|
|
171
|
+
}
|
|
172
|
+
const execFlag = source.ref.source === "exec" ? ["--allow-exec"] : [];
|
|
173
|
+
const file = params.writePlanFile(plan);
|
|
174
|
+
try {
|
|
175
|
+
const check = await runner(["secrets", "apply", "--from", file.path, "--dry-run", ...execFlag], {
|
|
176
|
+
timeoutMs: 120_000,
|
|
177
|
+
});
|
|
178
|
+
if (check.code !== 0) {
|
|
179
|
+
results.push({ ...base, action: "failed", detail: `secrets apply --dry-run ${safeCliError(check)}` });
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
if (params.dryRun) {
|
|
183
|
+
results.push({ ...base, action: "would-store", detail: "secrets apply (tokenRef, validated)" });
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
const apply = await runner(["secrets", "apply", "--from", file.path, ...execFlag], { timeoutMs: 120_000 });
|
|
187
|
+
results.push(apply.code === 0
|
|
188
|
+
? { ...base, action: "stored", detail: "secrets apply (tokenRef — no copy of the secret)" }
|
|
189
|
+
: { ...base, action: "failed", detail: `secrets apply ${safeCliError(apply)}` });
|
|
190
|
+
}
|
|
191
|
+
finally {
|
|
192
|
+
file.cleanup();
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return { results };
|
|
196
|
+
}
|
|
197
|
+
export async function readDirectRouteSnapshot(runner, agentId) {
|
|
198
|
+
const list = await runner(authListArgs(agentId), { timeoutMs: 60_000 });
|
|
199
|
+
const listed = list.code === 0 ? parseAuthList(list.stdout) : undefined;
|
|
200
|
+
if (!listed)
|
|
201
|
+
return { error: `auth list ${safeCliError(list)}` };
|
|
202
|
+
const order = await runner(orderGetArgs(agentId), { timeoutMs: 60_000 });
|
|
203
|
+
const parsed = order.code === 0 ? parseOrderGet(order.stdout) : undefined;
|
|
204
|
+
if (!parsed)
|
|
205
|
+
return { error: `order get ${safeCliError(order)}` };
|
|
206
|
+
return { snapshot: { stored: new Set(listed.map((p) => p.id)), storedOrder: parsed.order } };
|
|
207
|
+
}
|
|
208
|
+
function sameOrder(a, b) {
|
|
209
|
+
if (!a || !b)
|
|
210
|
+
return false;
|
|
211
|
+
return a.length === b.length && a.every((id, i) => id === b[i]);
|
|
212
|
+
}
|
|
213
|
+
export function planAgentOrder(params) {
|
|
214
|
+
const { snapshot } = params;
|
|
215
|
+
const present = params.members.filter((m) => snapshot.stored.has(m.profileId));
|
|
216
|
+
const missing = params.members.filter((m) => !snapshot.stored.has(m.profileId)).map((m) => m.profileId);
|
|
217
|
+
if (present.length === 0)
|
|
218
|
+
return { missing };
|
|
219
|
+
const explicit = snapshot.storedOrder ?? (params.configOrder?.length ? [...params.configOrder] : null);
|
|
220
|
+
const base = (explicit ?? [...snapshot.stored]).filter((id) => snapshot.stored.has(id));
|
|
221
|
+
const plan = planDirectOrder({
|
|
222
|
+
members: present,
|
|
223
|
+
currentOrder: base,
|
|
224
|
+
sticky: params.sticky,
|
|
225
|
+
nowMs: params.nowMs,
|
|
226
|
+
minDwellMs: params.minDwellMs,
|
|
227
|
+
});
|
|
228
|
+
if (!plan)
|
|
229
|
+
return { missing };
|
|
230
|
+
const inForce = snapshot.storedOrder !== null ? sameOrder(snapshot.storedOrder, plan.order) : sameOrder(explicit, plan.order);
|
|
231
|
+
return { order: inForce ? undefined : plan.order, missing, firstAccount: plan.firstAccount };
|
|
232
|
+
}
|
|
233
|
+
export function createDirectOrderController(deps) {
|
|
234
|
+
const reassertMs = deps.reassertMs ?? 60 * 60 * 1000;
|
|
235
|
+
const backoffMs = deps.backoffMs ?? 10 * 60 * 1000;
|
|
236
|
+
const applied = new Map();
|
|
237
|
+
const failed = new Map();
|
|
238
|
+
const warnedMissing = new Set();
|
|
239
|
+
let running = false;
|
|
240
|
+
return {
|
|
241
|
+
async tick() {
|
|
242
|
+
const nowMs = (deps.now ?? Date.now)();
|
|
243
|
+
if (running)
|
|
244
|
+
return { managedOrder: [], agents: [] };
|
|
245
|
+
running = true;
|
|
246
|
+
try {
|
|
247
|
+
const verdicts = deps.members.map((m) => ({
|
|
248
|
+
accountId: m.accountId,
|
|
249
|
+
profileId: m.profileId,
|
|
250
|
+
verdict: classifyAccountHealth(deps.readHealth(m.accountId), deps.healthOptions, nowMs).verdict,
|
|
251
|
+
}));
|
|
252
|
+
const sticky = deps.readSticky();
|
|
253
|
+
const managed = planDirectOrder({ members: verdicts, sticky, nowMs, minDwellMs: deps.minDwellMs });
|
|
254
|
+
if (!managed)
|
|
255
|
+
return { managedOrder: [], agents: [] };
|
|
256
|
+
if (JSON.stringify(managed.sticky) !== JSON.stringify(sticky))
|
|
257
|
+
deps.writeSticky(managed.sticky);
|
|
258
|
+
const key = managed.order.join(" ");
|
|
259
|
+
const report = { managedOrder: managed.order, agents: [] };
|
|
260
|
+
for (const agentId of deps.agents) {
|
|
261
|
+
try {
|
|
262
|
+
const done = applied.get(agentId);
|
|
263
|
+
if (done && done.key === key && nowMs - done.at < reassertMs) {
|
|
264
|
+
report.agents.push({ agentId, outcome: "skipped" });
|
|
265
|
+
continue;
|
|
266
|
+
}
|
|
267
|
+
const fail = failed.get(agentId);
|
|
268
|
+
if (fail && fail.key === key && nowMs - fail.at < backoffMs) {
|
|
269
|
+
report.agents.push({ agentId, outcome: "skipped", detail: "backing off after a failure" });
|
|
270
|
+
continue;
|
|
271
|
+
}
|
|
272
|
+
const read = await readDirectRouteSnapshot(deps.runner, agentId);
|
|
273
|
+
if (!read.snapshot) {
|
|
274
|
+
if (!fail || fail.key !== key) {
|
|
275
|
+
deps.logger.warn(`[multi-clawd] direct route: cannot read agent ${agentId}'s anthropic profiles (${read.error})`);
|
|
276
|
+
}
|
|
277
|
+
failed.set(agentId, { key, at: nowMs });
|
|
278
|
+
report.agents.push({ agentId, outcome: "failed", detail: read.error });
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
const plan = planAgentOrder({
|
|
282
|
+
members: verdicts,
|
|
283
|
+
snapshot: read.snapshot,
|
|
284
|
+
configOrder: deps.configOrder(),
|
|
285
|
+
sticky,
|
|
286
|
+
nowMs,
|
|
287
|
+
minDwellMs: deps.minDwellMs,
|
|
288
|
+
});
|
|
289
|
+
for (const id of plan.missing) {
|
|
290
|
+
const k = `${agentId}:${id}`;
|
|
291
|
+
if (warnedMissing.has(k))
|
|
292
|
+
continue;
|
|
293
|
+
warnedMissing.add(k);
|
|
294
|
+
deps.logger.warn(`[multi-clawd] direct route: profile ${id} is not stored for agent ${agentId} — run \`multi-clawd direct sync\``);
|
|
295
|
+
}
|
|
296
|
+
if (!plan.order) {
|
|
297
|
+
applied.set(agentId, { key, at: nowMs });
|
|
298
|
+
failed.delete(agentId);
|
|
299
|
+
report.agents.push({ agentId, outcome: "unchanged" });
|
|
300
|
+
continue;
|
|
301
|
+
}
|
|
302
|
+
const write = await deps.runner(orderSetArgs(agentId, plan.order), { timeoutMs: 60_000 });
|
|
303
|
+
if (write.code === 0) {
|
|
304
|
+
applied.set(agentId, { key, at: nowMs });
|
|
305
|
+
failed.delete(agentId);
|
|
306
|
+
deps.logger.info(`[multi-clawd] direct route: agent ${agentId} anthropic order → ${plan.order.join(" → ")}`);
|
|
307
|
+
report.agents.push({ agentId, outcome: "written", order: plan.order });
|
|
308
|
+
}
|
|
309
|
+
else {
|
|
310
|
+
const detail = `order set ${safeCliError(write)}`;
|
|
311
|
+
if (!fail || fail.key !== key) {
|
|
312
|
+
deps.logger.warn(`[multi-clawd] direct route: agent ${agentId}: ${detail}`);
|
|
313
|
+
}
|
|
314
|
+
failed.set(agentId, { key, at: nowMs });
|
|
315
|
+
report.agents.push({ agentId, outcome: "failed", detail });
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
catch (err) {
|
|
319
|
+
const detail = String(err?.message ?? err).slice(0, 200);
|
|
320
|
+
const fail = failed.get(agentId);
|
|
321
|
+
if (!fail || fail.key !== key) {
|
|
322
|
+
deps.logger.warn(`[multi-clawd] direct route: agent ${agentId}: ${detail}`);
|
|
323
|
+
}
|
|
324
|
+
failed.set(agentId, { key, at: nowMs });
|
|
325
|
+
report.agents.push({ agentId, outcome: "failed", detail });
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
return report;
|
|
329
|
+
}
|
|
330
|
+
finally {
|
|
331
|
+
running = false;
|
|
332
|
+
}
|
|
333
|
+
},
|
|
334
|
+
};
|
|
335
|
+
}
|
package/dist/explain-core.js
CHANGED
|
@@ -40,7 +40,7 @@ export function describeAccount(acc) {
|
|
|
40
40
|
}
|
|
41
41
|
return parts.join("; ") || "no credential source configured";
|
|
42
42
|
}
|
|
43
|
-
export function annotateChainRef(ref, pool) {
|
|
43
|
+
export function annotateChainRef(ref, pool, direct) {
|
|
44
44
|
const slash = ref.indexOf("/");
|
|
45
45
|
const provider = slash > 0 ? ref.slice(0, slash) : undefined;
|
|
46
46
|
if (pool && provider === pool.id) {
|
|
@@ -53,6 +53,10 @@ export function annotateChainRef(ref, pool) {
|
|
|
53
53
|
if (provider && /^claw\d+$/.test(provider)) {
|
|
54
54
|
return `pinned to only ${provider} — no cross-account failover on this rung`;
|
|
55
55
|
}
|
|
56
|
+
if (provider === "anthropic" && direct && direct.members.length > 0) {
|
|
57
|
+
const order = direct.members.map((m) => m.accountId).join(", then ");
|
|
58
|
+
return `direct to Anthropic — pooled by auth-profile order (${order}), health-sorted by the plugin`;
|
|
59
|
+
}
|
|
56
60
|
if (provider === "anthropic" || provider === "claude-cli") {
|
|
57
61
|
return "direct to Anthropic — bypasses the pool (no cross-account failover)";
|
|
58
62
|
}
|
|
@@ -94,12 +98,16 @@ export function renderExplanation(model) {
|
|
|
94
98
|
lines.push("POOL (no pool configured — each account is a standalone backend)");
|
|
95
99
|
}
|
|
96
100
|
lines.push("");
|
|
101
|
+
if (model.direct) {
|
|
102
|
+
lines.push(...renderDirectSection(model.direct, model.nowMs ?? Date.now()));
|
|
103
|
+
lines.push("");
|
|
104
|
+
}
|
|
97
105
|
if (model.chain?.primary || model.chain?.fallbacks?.length) {
|
|
98
106
|
lines.push("FAILOVER CHAIN (agents.defaults)");
|
|
99
107
|
const rungs = [model.chain.primary, ...(model.chain.fallbacks ?? [])].filter((r) => typeof r === "string");
|
|
100
108
|
rungs.forEach((ref, i) => {
|
|
101
109
|
lines.push(` ${i + 1}. ${ref}`);
|
|
102
|
-
lines.push(` ${annotateChainRef(ref, model.pool)}`);
|
|
110
|
+
lines.push(` ${annotateChainRef(ref, model.pool, model.direct)}`);
|
|
103
111
|
});
|
|
104
112
|
}
|
|
105
113
|
else {
|
|
@@ -122,3 +130,37 @@ export function renderExplanation(model) {
|
|
|
122
130
|
}
|
|
123
131
|
return lines.join("\n");
|
|
124
132
|
}
|
|
133
|
+
export function renderDirectSection(direct, nowMs) {
|
|
134
|
+
const lines = [];
|
|
135
|
+
lines.push("DIRECT ROUTE anthropic/* (gateway → Anthropic API with each account's setup-token)");
|
|
136
|
+
if (direct.members.length === 0) {
|
|
137
|
+
lines.push(" no account has a usable direct credential yet");
|
|
138
|
+
}
|
|
139
|
+
for (const m of direct.members) {
|
|
140
|
+
const bits = [m.source];
|
|
141
|
+
if (m.stored === true)
|
|
142
|
+
bits.push("stored in OpenClaw");
|
|
143
|
+
else if (m.stored === false) {
|
|
144
|
+
bits.push(m.adopted
|
|
145
|
+
? "NOT STORED — this profile is yours to store (`openclaw models auth paste-token --provider anthropic --profile-id " +
|
|
146
|
+
`${m.profileId}\`) or fix direct.profileId`
|
|
147
|
+
: "NOT STORED — run `multi-clawd direct sync`");
|
|
148
|
+
}
|
|
149
|
+
if (m.cooldownUntil !== undefined && m.cooldownUntil > nowMs) {
|
|
150
|
+
bits.push(`COOLING DOWN${m.cooldownReason ? ` (${m.cooldownReason})` : ""} for ${relativeUntil(m.cooldownUntil, nowMs)}`);
|
|
151
|
+
}
|
|
152
|
+
lines.push(` ${m.accountId} → ${m.profileId}`);
|
|
153
|
+
lines.push(` ${bits.join(" · ")}`);
|
|
154
|
+
}
|
|
155
|
+
for (const p of direct.problems) {
|
|
156
|
+
lines.push(` ${p.accountId}: NOT on the direct route — ${p.reason}`);
|
|
157
|
+
}
|
|
158
|
+
if (direct.order) {
|
|
159
|
+
const managed = new Set(direct.members.map((m) => m.profileId));
|
|
160
|
+
const rendered = direct.order.map((id) => (managed.has(id) ? id : `${id} (not managed)`));
|
|
161
|
+
lines.push(` order${direct.orderSource ? ` (${direct.orderSource})` : ""}: ${rendered.length > 0 ? rendered.join(" → ") : "none set — OpenClaw's round-robin decides"}`);
|
|
162
|
+
}
|
|
163
|
+
lines.push(" The plugin keeps this order in pool-health order, so a nearly-maxed account");
|
|
164
|
+
lines.push(" drops back BEFORE it errors; OpenClaw's own rotation covers the rest in-turn.");
|
|
165
|
+
return lines;
|
|
166
|
+
}
|
package/dist/index.js
CHANGED
|
@@ -14,13 +14,17 @@ import { createTokenRefResolver, isSecretRefShape, } from "./token-resolution.js
|
|
|
14
14
|
import { resolveSecretRefValues } from "openclaw/plugin-sdk/secret-ref-runtime";
|
|
15
15
|
import { resolvePoolExecutionArgs } from "./tool-cap.js";
|
|
16
16
|
import { RETRY_ROSTER_ENV } from "./retry-plan.js";
|
|
17
|
+
import { SESSION_DIRS_ENV } from "./session-handover.js";
|
|
17
18
|
import { addAlert, alertKeysWithPrefix, clearAlert, pendingAlertText, } from "./alerts.js";
|
|
18
19
|
import { healthStateFile, clearAccountCredentialFailure } from "./credential-state.js";
|
|
19
20
|
export { healthStateFile, clearAccountCredentialFailure };
|
|
20
|
-
import { buildAccountChildEnv, tokenFileModeWarning, validateAccountTokenSources, } from "./account-env.js";
|
|
21
|
+
import { accountConfigDir, buildAccountChildEnv, tokenFileModeWarning, validateAccountTokenSources, } from "./account-env.js";
|
|
21
22
|
import { diffCatalogModels, formatNewModelNotice, } from "./model-currency.js";
|
|
22
23
|
import { checkAccountCredential, keychainServiceForConfigDir, createRefProbeTracker, } from "./login-health.js";
|
|
23
24
|
import { execFileSync } from "node:child_process";
|
|
25
|
+
import { collectDirectMembers } from "./direct-route.js";
|
|
26
|
+
import { createDirectOrderController, isValidAgentId } from "./direct-sync.js";
|
|
27
|
+
import { createOpenclawRunner } from "./openclaw-runner.js";
|
|
24
28
|
const CLI_FRESH_WATCHDOG_DEFAULTS = {
|
|
25
29
|
noOutputTimeoutRatio: 0.8,
|
|
26
30
|
minMs: 180_000,
|
|
@@ -506,6 +510,9 @@ export default definePluginEntry({
|
|
|
506
510
|
"startup-pluginConfig",
|
|
507
511
|
];
|
|
508
512
|
if (accounts.length === 0) {
|
|
513
|
+
const mode = api.registrationMode;
|
|
514
|
+
if (mode === undefined || mode === "full")
|
|
515
|
+
stopDirectOrderSync();
|
|
509
516
|
api.logger.warn(`[multi-clawd] no accounts configured — nothing to register (sources: ${candidates
|
|
510
517
|
.map((c, i) => `${sourceNames[i]}=${hasAccounts(c) ? "ok" : c ? "empty" : "absent"}`)
|
|
511
518
|
.join(", ")})`);
|
|
@@ -557,6 +564,18 @@ export default definePluginEntry({
|
|
|
557
564
|
logger.warn(`[multi-clawd] heartbeat alert hook unavailable: ${String(err)}`);
|
|
558
565
|
}
|
|
559
566
|
startLoginHealthProbe(accounts.filter((a) => seen.has(a.id.trim())), logger);
|
|
567
|
+
startDirectOrderSync({
|
|
568
|
+
accounts: accounts.filter((a) => seen.has(a.id.trim())),
|
|
569
|
+
pool: cfg.pool,
|
|
570
|
+
directRoute: cfg.directRoute,
|
|
571
|
+
registrationMode: api.registrationMode,
|
|
572
|
+
configOrder: () => {
|
|
573
|
+
const root = (runtimeConfigLoader?.() ?? api.config);
|
|
574
|
+
const order = root?.auth?.order?.anthropic;
|
|
575
|
+
return Array.isArray(order) ? order.filter((v) => typeof v === "string") : undefined;
|
|
576
|
+
},
|
|
577
|
+
logger,
|
|
578
|
+
});
|
|
560
579
|
}
|
|
561
580
|
api.logger.info(`[multi-clawd] registered ${seen.size} backend(s)+provider(s): ${[...seen].join(", ")}`);
|
|
562
581
|
},
|
|
@@ -742,6 +761,7 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
742
761
|
if (roster.length > 0)
|
|
743
762
|
env[RETRY_ROSTER_ENV] = JSON.stringify(roster);
|
|
744
763
|
}
|
|
764
|
+
env[SESSION_DIRS_ENV] = JSON.stringify([...new Set(members.map(accountConfigDir))]);
|
|
745
765
|
if (ladder.length > 0) {
|
|
746
766
|
const pinned = matchesPin(pins, {
|
|
747
767
|
agentDir: ctx.agentDir ?? "",
|
|
@@ -778,3 +798,84 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
|
|
|
778
798
|
}
|
|
779
799
|
})();
|
|
780
800
|
}
|
|
801
|
+
let directTimer;
|
|
802
|
+
let directInitial;
|
|
803
|
+
let directController;
|
|
804
|
+
export function stopDirectOrderSync() {
|
|
805
|
+
if (directTimer)
|
|
806
|
+
clearInterval(directTimer);
|
|
807
|
+
if (directInitial)
|
|
808
|
+
clearTimeout(directInitial);
|
|
809
|
+
directTimer = undefined;
|
|
810
|
+
directInitial = undefined;
|
|
811
|
+
directController = undefined;
|
|
812
|
+
}
|
|
813
|
+
export function startDirectOrderSync(params) {
|
|
814
|
+
const { logger } = params;
|
|
815
|
+
const mode = params.registrationMode;
|
|
816
|
+
if (mode !== undefined && mode !== "full")
|
|
817
|
+
return { active: false, members: [] };
|
|
818
|
+
const { members, problems } = collectDirectMembers(params.accounts, params.pool?.accounts ?? []);
|
|
819
|
+
for (const p of problems) {
|
|
820
|
+
logger.warn(`[multi-clawd] direct route: account "${p.accountId}" skipped — ${p.reason}`);
|
|
821
|
+
}
|
|
822
|
+
if (members.length === 0 || params.directRoute?.manageOrder === false) {
|
|
823
|
+
stopDirectOrderSync();
|
|
824
|
+
return { active: false, members: members.map((m) => m.accountId) };
|
|
825
|
+
}
|
|
826
|
+
const requested = params.directRoute?.agents?.length ? params.directRoute.agents : ["main"];
|
|
827
|
+
const agents = requested.filter((a) => {
|
|
828
|
+
if (isValidAgentId(a))
|
|
829
|
+
return true;
|
|
830
|
+
logger.warn(`[multi-clawd] direct route: ignoring agent id ${JSON.stringify(a)} in directRoute.agents — letters, digits, "_" and "-" only`);
|
|
831
|
+
return false;
|
|
832
|
+
});
|
|
833
|
+
if (agents.length === 0) {
|
|
834
|
+
stopDirectOrderSync();
|
|
835
|
+
return { active: false, members: members.map((m) => m.accountId) };
|
|
836
|
+
}
|
|
837
|
+
const intervalMs = Math.max(15_000, params.directRoute?.intervalMs ?? 60_000);
|
|
838
|
+
const healthOptions = {
|
|
839
|
+
utilizationThreshold: params.pool?.utilizationThreshold,
|
|
840
|
+
staleAfterMs: params.pool?.staleAfterMs,
|
|
841
|
+
rotateOnOverage: params.pool?.rotateOnOverage,
|
|
842
|
+
};
|
|
843
|
+
const signature = JSON.stringify({
|
|
844
|
+
members: members.map((m) => m.profileId),
|
|
845
|
+
agents,
|
|
846
|
+
intervalMs,
|
|
847
|
+
healthOptions,
|
|
848
|
+
minDwellMs: params.pool?.minDwellMs,
|
|
849
|
+
command: params.directRoute?.openclawCommand,
|
|
850
|
+
});
|
|
851
|
+
if (directController?.signature === signature && directTimer) {
|
|
852
|
+
return { active: true, members: members.map((m) => m.accountId) };
|
|
853
|
+
}
|
|
854
|
+
stopDirectOrderSync();
|
|
855
|
+
const stickyFile = join(homedir(), ".openclaw", "state", "multi-clawd", "direct-order.sticky.json");
|
|
856
|
+
const controller = createDirectOrderController({
|
|
857
|
+
members,
|
|
858
|
+
agents,
|
|
859
|
+
runner: createOpenclawRunner(params.directRoute?.openclawCommand ?? "openclaw"),
|
|
860
|
+
readHealth: (id) => readHealthState(id),
|
|
861
|
+
healthOptions,
|
|
862
|
+
minDwellMs: params.pool?.minDwellMs,
|
|
863
|
+
configOrder: params.configOrder,
|
|
864
|
+
readSticky: () => readStickyEntry(stickyFile),
|
|
865
|
+
writeSticky: (entry) => writeStickyEntry(stickyFile, entry, logger),
|
|
866
|
+
logger,
|
|
867
|
+
});
|
|
868
|
+
directController = { signature, tick: controller.tick };
|
|
869
|
+
const tick = () => void controller.tick().catch((err) => logger.warn(`[multi-clawd] direct route tick failed: ${String(err)}`));
|
|
870
|
+
directInitial = setTimeout(tick, Math.min(30_000, intervalMs));
|
|
871
|
+
directInitial.unref?.();
|
|
872
|
+
directTimer = setInterval(tick, intervalMs);
|
|
873
|
+
directTimer.unref?.();
|
|
874
|
+
logger.info(`[multi-clawd] direct route: keeping anthropic order for agent(s) ${agents.join(", ")} in pool-health order — ${members
|
|
875
|
+
.map((m) => `${m.accountId}=${m.profileId}`)
|
|
876
|
+
.join(", ")}`);
|
|
877
|
+
return { active: true, members: members.map((m) => m.accountId) };
|
|
878
|
+
}
|
|
879
|
+
export async function runDirectOrderTickNow() {
|
|
880
|
+
return directController?.tick();
|
|
881
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
const MAX_CAPTURE = 1024 * 1024;
|
|
3
|
+
export function createOpenclawRunner(command = "openclaw") {
|
|
4
|
+
return (args, opts) => new Promise((resolve) => {
|
|
5
|
+
let stdout = "";
|
|
6
|
+
let stderr = "";
|
|
7
|
+
let settled = false;
|
|
8
|
+
const finish = (result) => {
|
|
9
|
+
if (settled)
|
|
10
|
+
return;
|
|
11
|
+
settled = true;
|
|
12
|
+
clearTimeout(timer);
|
|
13
|
+
resolve(result);
|
|
14
|
+
};
|
|
15
|
+
let child;
|
|
16
|
+
try {
|
|
17
|
+
child = spawn(command, args, { stdio: ["pipe", "pipe", "pipe"], shell: false });
|
|
18
|
+
}
|
|
19
|
+
catch (err) {
|
|
20
|
+
resolve({ code: 127, stdout: "", stderr: String(err.message) });
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
const timer = setTimeout(() => {
|
|
24
|
+
child.kill("SIGTERM");
|
|
25
|
+
finish({ code: 124, stdout, stderr: `${stderr}\ntimed out` });
|
|
26
|
+
}, opts?.timeoutMs ?? 60_000);
|
|
27
|
+
timer.unref?.();
|
|
28
|
+
child.stdout?.on("data", (d) => {
|
|
29
|
+
if (stdout.length < MAX_CAPTURE)
|
|
30
|
+
stdout += d.toString("utf8");
|
|
31
|
+
});
|
|
32
|
+
child.stderr?.on("data", (d) => {
|
|
33
|
+
if (stderr.length < MAX_CAPTURE)
|
|
34
|
+
stderr += d.toString("utf8");
|
|
35
|
+
});
|
|
36
|
+
child.on("error", (err) => finish({ code: 127, stdout, stderr: `${stderr}\n${err.message}` }));
|
|
37
|
+
child.on("close", (code) => finish({ code: code ?? 1, stdout, stderr }));
|
|
38
|
+
child.stdin?.on("error", () => {
|
|
39
|
+
});
|
|
40
|
+
child.stdin?.end(opts?.input ?? "");
|
|
41
|
+
});
|
|
42
|
+
}
|