@drakon-systems/multi-clawd 2.3.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +121 -31
- package/SECURITY.md +3 -0
- package/dist/credential-state.js +8 -24
- package/dist/direct-report.js +12 -1
- package/dist/direct-route.js +12 -6
- package/dist/direct-sync.js +12 -4
- package/dist/explain-core.js +33 -10
- package/dist/health-journal.js +123 -0
- package/dist/health.js +13 -1
- package/dist/history.js +103 -0
- package/dist/index.js +89 -38
- package/dist/retry-plan.js +2 -0
- package/dist/session-affinity.js +108 -0
- package/dist/shim-core.js +197 -34
- package/dist/shim-lock.js +106 -0
- package/dist/shim.js +170 -79
- package/dist/spread.js +54 -0
- package/dist/sticky.js +2 -0
- package/dist/update-core.js +75 -1
- package/openclaw.plugin.json +26 -3
- package/package.json +1 -1
- package/scripts/account-config.mjs +115 -0
- package/scripts/cli.mjs +131 -25
- package/scripts/doctor.mjs +113 -6
package/dist/shim.js
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
2
|
+
import { readFileSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { persistHealthState, readHealthStateFile } from "./health-journal.js";
|
|
4
|
+
import { acquireShimLock } from "./shim-lock.js";
|
|
5
|
+
import { recordHistory } from "./history.js";
|
|
6
|
+
import { clearCredentialFailure, clearModelRejection, createForwardWriter, createLineScanner, limitFamilyFor, parseAuthFailure, parseRateLimitEvent, parseSuccessResult, recordCredentialFailure, updateHealthState, } from "./shim-core.js";
|
|
5
7
|
import { rewriteModelArg } from "./degrade.js";
|
|
6
|
-
import { parseModelLimitError, recordModelLimit } from "./shim-core.js";
|
|
8
|
+
import { parseApiRetryNotice, parseModelLimitError, parseTransientApiError, recordModelLimit, } from "./shim-core.js";
|
|
7
9
|
import { canonicalModelId } from "./models.js";
|
|
8
10
|
import { buildRetryEnv, chooseLaunchableRetryAccount, materializeRetryAccount, parseRetryRoster, retryArming, RETRY_ROSTER_ENV, } from "./retry-plan.js";
|
|
11
|
+
import { eligibleSessionAccount, launchSessionId, readSessionPin, recordSessionPin, SESSION_AFFINITY_ENV, } from "./session-affinity.js";
|
|
9
12
|
import { CLAUDE_ARGV0_ENV, CLAUDE_BIN_ENV, parseClaudeBinOverride, resolveExecutable, } from "./claude-command.js";
|
|
10
13
|
import { effectiveConfigDir, handoverForLaunch, handoverSnapshotTo, parseSessionDirs, SESSION_DIRS_ENV, snapshotResumeTranscript, } from "./session-handover.js";
|
|
11
14
|
function resolveClaudeCommand() {
|
|
@@ -59,40 +62,12 @@ function preserveCorruptState(raw) {
|
|
|
59
62
|
process.stderr.write(`[multi-clawd shim] state file unreadable/corrupt — starting fresh (${note})\n`);
|
|
60
63
|
}
|
|
61
64
|
function readPersistedState() {
|
|
62
|
-
|
|
63
|
-
return undefined;
|
|
64
|
-
let raw;
|
|
65
|
-
try {
|
|
66
|
-
raw = readFileSync(stateFile, "utf8");
|
|
67
|
-
}
|
|
68
|
-
catch (err) {
|
|
69
|
-
if (classifyStateReadFailure(err) === "absent")
|
|
70
|
-
return undefined;
|
|
71
|
-
preserveCorruptState(undefined);
|
|
72
|
-
return undefined;
|
|
73
|
-
}
|
|
74
|
-
const parsed = parseStoredState(raw);
|
|
75
|
-
if (parsed === undefined) {
|
|
76
|
-
preserveCorruptState(raw);
|
|
77
|
-
return undefined;
|
|
78
|
-
}
|
|
79
|
-
return parsed;
|
|
65
|
+
return stateFile ? readHealthStateFile(stateFile, { onSnapshotError: preserveCorruptState }) : undefined;
|
|
80
66
|
}
|
|
81
67
|
function persistState() {
|
|
82
68
|
if (!stateFile)
|
|
83
69
|
return;
|
|
84
|
-
|
|
85
|
-
const disk = readPersistedState();
|
|
86
|
-
if (disk)
|
|
87
|
-
state = mergeHealthStates(disk, state, Date.now());
|
|
88
|
-
mkdirSync(dirname(stateFile), { recursive: true });
|
|
89
|
-
const tmp = `${stateFile}.tmp-${process.pid}`;
|
|
90
|
-
writeFileSync(tmp, JSON.stringify(state, null, 2), { mode: 0o600 });
|
|
91
|
-
renameSync(tmp, stateFile);
|
|
92
|
-
}
|
|
93
|
-
catch (err) {
|
|
94
|
-
process.stderr.write(`[multi-clawd shim] state write failed: ${String(err)}\n`);
|
|
95
|
-
}
|
|
70
|
+
state = persistHealthState(stateFile, state, { onSnapshotError: preserveCorruptState });
|
|
96
71
|
}
|
|
97
72
|
const { command, prependArgs, argv0: claudeArgv0 } = resolveClaudeCommand();
|
|
98
73
|
let childArgs = [...prependArgs, ...process.argv.slice(2)];
|
|
@@ -104,15 +79,53 @@ if (modelOverride) {
|
|
|
104
79
|
process.stderr.write(`[multi-clawd shim] degrading model for this launch → ${modelOverride}\n`);
|
|
105
80
|
}
|
|
106
81
|
}
|
|
82
|
+
const sessionId = launchSessionId(childArgs);
|
|
83
|
+
let affinity;
|
|
84
|
+
let launchEnv = process.env;
|
|
85
|
+
try {
|
|
86
|
+
const raw = process.env[SESSION_AFFINITY_ENV];
|
|
87
|
+
if (raw && sessionId) {
|
|
88
|
+
const config = JSON.parse(raw);
|
|
89
|
+
if (typeof config.file === "string" && config.file && typeof config.ttlMs === "number" &&
|
|
90
|
+
Number.isFinite(config.ttlMs) && config.ttlMs > 0 && Array.isArray(config.verdicts)) {
|
|
91
|
+
affinity = {
|
|
92
|
+
file: config.file,
|
|
93
|
+
ttlMs: config.ttlMs,
|
|
94
|
+
verdicts: config.verdicts,
|
|
95
|
+
roster: parseRetryRoster(JSON.stringify(config.roster)),
|
|
96
|
+
};
|
|
97
|
+
const pin = readSessionPin(affinity.file, sessionId, Date.now(), affinity.ttlMs);
|
|
98
|
+
const eligible = eligibleSessionAccount(pin, accountId, affinity.verdicts, affinity.roster.map((r) => r.id));
|
|
99
|
+
if (eligible && eligible !== accountId) {
|
|
100
|
+
const member = affinity.roster.find((r) => r.id === eligible);
|
|
101
|
+
if (member) {
|
|
102
|
+
const ready = materializeRetryAccount(member, (path) => readFileSync(path, "utf8"));
|
|
103
|
+
if ("account" in ready) {
|
|
104
|
+
launchEnv = buildRetryEnv(process.env, ready.account);
|
|
105
|
+
accountId = ready.account.id;
|
|
106
|
+
stateFile = ready.account.stateFile;
|
|
107
|
+
state = { accountId, windows: {} };
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
affinity = undefined;
|
|
116
|
+
}
|
|
117
|
+
delete launchEnv[SESSION_AFFINITY_ENV];
|
|
107
118
|
{
|
|
108
|
-
const note = handoverForLaunch(childArgs,
|
|
119
|
+
const note = handoverForLaunch(childArgs, launchEnv);
|
|
109
120
|
if (note)
|
|
110
121
|
process.stderr.write(`[multi-clawd shim] ${note}\n`);
|
|
111
122
|
}
|
|
112
|
-
const retryRoster =
|
|
123
|
+
const retryRoster = affinity && launchEnv !== process.env
|
|
124
|
+
? affinity.roster.filter((r) => r.id !== accountId)
|
|
125
|
+
: parseRetryRoster(process.env[RETRY_ROSTER_ENV]);
|
|
113
126
|
delete process.env[RETRY_ROSTER_ENV];
|
|
114
|
-
const resumeSnapshot = retryRoster.length > 0 && parseSessionDirs(
|
|
115
|
-
? snapshotResumeTranscript(childArgs,
|
|
127
|
+
const resumeSnapshot = retryRoster.length > 0 && parseSessionDirs(launchEnv[SESSION_DIRS_ENV]).length > 0
|
|
128
|
+
? snapshotResumeTranscript(childArgs, launchEnv)
|
|
116
129
|
: undefined;
|
|
117
130
|
const arming = retryArming(childArgs, retryRoster, { resumeReady: resumeSnapshot !== undefined });
|
|
118
131
|
let retryArmed = arming.armed;
|
|
@@ -121,9 +134,11 @@ if (!retryArmed && retryRoster.length > 0 && arming.reason) {
|
|
|
121
134
|
}
|
|
122
135
|
let child = spawn(command, childArgs, {
|
|
123
136
|
stdio: ["pipe", "pipe", "inherit"],
|
|
124
|
-
env:
|
|
137
|
+
env: launchEnv,
|
|
125
138
|
...(claudeArgv0 ? { argv0: claudeArgv0 } : {}),
|
|
126
139
|
});
|
|
140
|
+
child.once("spawn", () => recordHistory(stateFile, { at: new Date().toISOString(),
|
|
141
|
+
kind: "served", accountId, modelId: effectiveModelId() }));
|
|
127
142
|
const STDIN_REPLAY_CAP_BYTES = 8 * 1024 * 1024;
|
|
128
143
|
let stdinReplay = [];
|
|
129
144
|
let stdinReplayBytes = 0;
|
|
@@ -170,6 +185,7 @@ function guessLimitResetsAt() {
|
|
|
170
185
|
let sawAuthFailure = false;
|
|
171
186
|
let sawModelLimit = false;
|
|
172
187
|
let sawSuccessResult = false;
|
|
188
|
+
let recordedLimitDisplayName;
|
|
173
189
|
function observeLine(line) {
|
|
174
190
|
try {
|
|
175
191
|
if (parseSuccessResult(line))
|
|
@@ -180,7 +196,8 @@ function observeLine(line) {
|
|
|
180
196
|
persistState();
|
|
181
197
|
}
|
|
182
198
|
const limitHit = parseModelLimitError(line);
|
|
183
|
-
if (limitHit) {
|
|
199
|
+
if (limitHit && limitHit.displayName !== recordedLimitDisplayName) {
|
|
200
|
+
recordedLimitDisplayName = limitHit.displayName;
|
|
184
201
|
sawModelLimit = true;
|
|
185
202
|
const model = effectiveModelId();
|
|
186
203
|
if (model) {
|
|
@@ -203,12 +220,17 @@ function observeLine(line) {
|
|
|
203
220
|
}
|
|
204
221
|
}
|
|
205
222
|
const scanner = createLineScanner(observeLine);
|
|
223
|
+
const forward = createForwardWriter((bytes) => process.stdout.write(bytes), () => process.stderr.write(`[multi-clawd shim] model limit on ${accountId} forwarded to the host marked as a rate limit\n`));
|
|
206
224
|
const HOLD_CAP_BYTES = 256 * 1024;
|
|
207
225
|
const HOLD_MAX_MS = Number(process.env.MULTI_CLAWD_HOLD_MAX_MS ?? 10_000);
|
|
226
|
+
const HOLD_RETRY_MAX_MS = Math.min(Number(process.env.MULTI_CLAWD_HOLD_RETRY_MAX_MS ?? 120_000), 150_000);
|
|
227
|
+
const HOLD_RETRY_GRACE_MS = 5_000;
|
|
208
228
|
let held = [];
|
|
209
229
|
let heldBytes = 0;
|
|
210
230
|
let holdTail = "";
|
|
211
231
|
let retryUsed = false;
|
|
232
|
+
let pendingRetryCause;
|
|
233
|
+
let sawTerminalFailure = false;
|
|
212
234
|
function isPreambleRecord(line) {
|
|
213
235
|
if (line.length === 0)
|
|
214
236
|
return true;
|
|
@@ -226,12 +248,12 @@ function isPreambleRecord(line) {
|
|
|
226
248
|
}
|
|
227
249
|
function releaseHeldOutput() {
|
|
228
250
|
if (held.length > 0) {
|
|
229
|
-
|
|
251
|
+
forward.push(Buffer.from(held.join(""), "utf8"));
|
|
230
252
|
held = [];
|
|
231
253
|
heldBytes = 0;
|
|
232
254
|
}
|
|
233
255
|
if (holdTail.length > 0) {
|
|
234
|
-
|
|
256
|
+
forward.push(Buffer.from(holdTail, "utf8"));
|
|
235
257
|
scanner.push(holdTail);
|
|
236
258
|
holdTail = "";
|
|
237
259
|
}
|
|
@@ -239,14 +261,15 @@ function releaseHeldOutput() {
|
|
|
239
261
|
stdinReplay = [];
|
|
240
262
|
}
|
|
241
263
|
function readSiblingState(file) {
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
return
|
|
247
|
-
|
|
264
|
+
return readHealthStateFile(file);
|
|
265
|
+
}
|
|
266
|
+
function describeCause(cause) {
|
|
267
|
+
if (cause.kind === "model_limit")
|
|
268
|
+
return "model limit";
|
|
269
|
+
const d = cause.detail;
|
|
270
|
+
return `transient provider error (${d.kind}${d.status !== undefined ? ` ${d.status}` : ""})`;
|
|
248
271
|
}
|
|
249
|
-
function attemptRetry() {
|
|
272
|
+
function attemptRetry(cause) {
|
|
250
273
|
if (!retryArmed || retryUsed)
|
|
251
274
|
return false;
|
|
252
275
|
const model = effectiveModelId();
|
|
@@ -255,13 +278,13 @@ function attemptRetry() {
|
|
|
255
278
|
readState: readSiblingState,
|
|
256
279
|
modelId: model,
|
|
257
280
|
nowMs: Date.now(),
|
|
281
|
+
allowOverdueProbe: cause.kind !== "transient",
|
|
258
282
|
materialize: (account) => materializeRetryAccount(account, (path) => readFileSync(path, "utf8")),
|
|
259
283
|
onSkip: (id, reason) => process.stderr.write(`[multi-clawd shim] not retrying onto ${id}: its declared credential did not resolve (${reason})\n`),
|
|
260
284
|
});
|
|
261
285
|
if (!target) {
|
|
262
|
-
process.stderr.write(`[multi-clawd shim]
|
|
286
|
+
process.stderr.write(`[multi-clawd shim] ${describeCause(cause)} on ${accountId} with no healthy sibling to retry onto — ` +
|
|
263
287
|
`passing the failure through to the host's chain\n`);
|
|
264
|
-
releaseHeldOutput();
|
|
265
288
|
return false;
|
|
266
289
|
}
|
|
267
290
|
if (resumeSnapshot) {
|
|
@@ -269,41 +292,40 @@ function attemptRetry() {
|
|
|
269
292
|
handoverSnapshotTo(resumeSnapshot, effectiveConfigDir(target.env));
|
|
270
293
|
}
|
|
271
294
|
catch (err) {
|
|
272
|
-
process.stderr.write(`[multi-clawd shim]
|
|
295
|
+
process.stderr.write(`[multi-clawd shim] ${describeCause(cause)} on ${accountId}, but session ` +
|
|
273
296
|
`${resumeSnapshot.sessionId.slice(0, 8)} could not be handed to ${target.id} ` +
|
|
274
297
|
`(${err instanceof Error ? err.message : String(err)}) — ` +
|
|
275
298
|
`passing the failure through to the host's chain\n`);
|
|
276
|
-
releaseHeldOutput();
|
|
277
299
|
return false;
|
|
278
300
|
}
|
|
279
301
|
}
|
|
280
302
|
retryUsed = true;
|
|
281
303
|
retryArmed = false;
|
|
282
|
-
|
|
304
|
+
const reason = cause.kind === "model_limit" ? "model_limit" : "transient";
|
|
305
|
+
recordHistory(stateFile, { at: new Date().toISOString(), kind: "retry", accountId,
|
|
306
|
+
toAccountId: target.id, modelId: model, reason });
|
|
307
|
+
if (target.id !== accountId)
|
|
308
|
+
recordHistory(stateFile, { at: new Date().toISOString(),
|
|
309
|
+
kind: "rotation", accountId, toAccountId: target.id, modelId: model, reason });
|
|
310
|
+
process.stderr.write(`[multi-clawd shim] ${describeCause(cause)} on ${accountId}${model ? ` for ${model}` : ""} — ` +
|
|
283
311
|
`retrying this turn on ${target.id}${resumeSnapshot ? `, session ${resumeSnapshot.sessionId.slice(0, 8)} handed over` : ""} (nothing forwarded downstream yet)\n`);
|
|
284
312
|
held = [];
|
|
285
313
|
heldBytes = 0;
|
|
286
314
|
holdTail = "";
|
|
287
|
-
const previous = child;
|
|
288
|
-
previous.stdout.removeAllListeners();
|
|
289
|
-
previous.removeAllListeners("close");
|
|
290
|
-
previous.removeAllListeners("error");
|
|
291
|
-
try {
|
|
292
|
-
previous.kill("SIGTERM");
|
|
293
|
-
}
|
|
294
|
-
catch {
|
|
295
|
-
}
|
|
296
315
|
accountId = target.id;
|
|
297
316
|
stateFile = target.stateFile;
|
|
298
317
|
state = { accountId, windows: {} };
|
|
299
318
|
sawAuthFailure = false;
|
|
300
319
|
sawModelLimit = false;
|
|
320
|
+
recordedLimitDisplayName = undefined;
|
|
301
321
|
sawSuccessResult = false;
|
|
302
322
|
child = spawn(command, childArgs, {
|
|
303
323
|
stdio: ["pipe", "pipe", "inherit"],
|
|
304
|
-
env: buildRetryEnv(
|
|
324
|
+
env: buildRetryEnv(launchEnv, target),
|
|
305
325
|
...(claudeArgv0 ? { argv0: claudeArgv0 } : {}),
|
|
306
326
|
});
|
|
327
|
+
child.once("spawn", () => recordHistory(stateFile, { at: new Date().toISOString(),
|
|
328
|
+
kind: "served", accountId, modelId: effectiveModelId() }));
|
|
307
329
|
attachChildHandlers();
|
|
308
330
|
for (const chunk of stdinReplay)
|
|
309
331
|
child.stdin.write(chunk);
|
|
@@ -321,34 +343,84 @@ function onArmedChunk(chunk) {
|
|
|
321
343
|
const raw = holdTail.slice(0, idx + 1);
|
|
322
344
|
holdTail = holdTail.slice(idx + 1);
|
|
323
345
|
const line = raw.trim();
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
346
|
+
const releaseDecision = stateFile && (parseModelLimitError(line) || parseTransientApiError(line))
|
|
347
|
+
? acquireShimLock(`${stateFile}.retry`)
|
|
348
|
+
: undefined;
|
|
349
|
+
try {
|
|
350
|
+
observeLine(line);
|
|
351
|
+
let cause;
|
|
352
|
+
if (parseModelLimitError(line)) {
|
|
353
|
+
cause = { kind: "model_limit" };
|
|
354
|
+
}
|
|
355
|
+
else if (!sawModelLimit && !sawAuthFailure) {
|
|
356
|
+
const transient = parseTransientApiError(line);
|
|
357
|
+
if (transient)
|
|
358
|
+
cause = { kind: "transient", detail: transient };
|
|
359
|
+
}
|
|
360
|
+
if (cause && (!pendingRetryCause || cause.kind === "model_limit")) {
|
|
361
|
+
pendingRetryCause = cause;
|
|
362
|
+
}
|
|
363
|
+
let terminalFailure = false;
|
|
364
|
+
try {
|
|
365
|
+
const record = JSON.parse(line);
|
|
366
|
+
terminalFailure = record.type === "result" && record.is_error === true;
|
|
367
|
+
}
|
|
368
|
+
catch { }
|
|
369
|
+
if (terminalFailure)
|
|
370
|
+
sawTerminalFailure = true;
|
|
371
|
+
const retryNotice = parseApiRetryNotice(line);
|
|
372
|
+
if (retryNotice)
|
|
373
|
+
extendHold(retryNotice.retryDelayMs + HOLD_RETRY_GRACE_MS);
|
|
374
|
+
held.push(raw);
|
|
375
|
+
heldBytes += Buffer.byteLength(raw);
|
|
376
|
+
if ((!isPreambleRecord(line) && !(terminalFailure && pendingRetryCause && !sawAuthFailure)) || heldBytes > HOLD_CAP_BYTES) {
|
|
377
|
+
releaseHeldOutput();
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
finally {
|
|
382
|
+
releaseDecision?.();
|
|
332
383
|
}
|
|
333
384
|
}
|
|
385
|
+
if (retryArmed && heldBytes + Buffer.byteLength(holdTail) > HOLD_CAP_BYTES) {
|
|
386
|
+
releaseHeldOutput();
|
|
387
|
+
}
|
|
334
388
|
if (!retryArmed && holdTail.length > 0) {
|
|
335
389
|
const rest = holdTail;
|
|
336
390
|
holdTail = "";
|
|
337
|
-
|
|
391
|
+
forward.push(Buffer.from(rest, "utf8"));
|
|
338
392
|
scanner.push(rest);
|
|
339
393
|
}
|
|
340
394
|
}
|
|
341
|
-
|
|
342
|
-
|
|
395
|
+
let holdTimer;
|
|
396
|
+
const holdStartedAt = Date.now();
|
|
397
|
+
let holdDeadline = holdStartedAt + HOLD_MAX_MS;
|
|
398
|
+
function armHoldTimer() {
|
|
399
|
+
if (holdTimer)
|
|
400
|
+
clearTimeout(holdTimer);
|
|
401
|
+
holdTimer = setTimeout(() => {
|
|
343
402
|
if (!retryArmed)
|
|
344
403
|
return;
|
|
345
|
-
process.stderr.write(`[multi-clawd shim] in-turn retry window closed after ${
|
|
404
|
+
process.stderr.write(`[multi-clawd shim] in-turn retry window closed after ${Date.now() - holdStartedAt}ms — releasing the stream\n`);
|
|
346
405
|
releaseHeldOutput();
|
|
347
|
-
},
|
|
406
|
+
}, Math.max(0, holdDeadline - Date.now()));
|
|
348
407
|
holdTimer.unref?.();
|
|
349
408
|
}
|
|
409
|
+
function extendHold(ms) {
|
|
410
|
+
if (!retryArmed)
|
|
411
|
+
return;
|
|
412
|
+
const wanted = Date.now() + ms;
|
|
413
|
+
const ceiling = holdStartedAt + HOLD_RETRY_MAX_MS;
|
|
414
|
+
const next = Math.min(wanted, ceiling);
|
|
415
|
+
if (next <= holdDeadline)
|
|
416
|
+
return;
|
|
417
|
+
holdDeadline = next;
|
|
418
|
+
armHoldTimer();
|
|
419
|
+
}
|
|
420
|
+
if (retryArmed)
|
|
421
|
+
armHoldTimer();
|
|
350
422
|
function onPassthroughChunk(chunk) {
|
|
351
|
-
|
|
423
|
+
forward.push(chunk);
|
|
352
424
|
scanner.push(chunk.toString("utf8"));
|
|
353
425
|
}
|
|
354
426
|
function attachChildHandlers() {
|
|
@@ -360,8 +432,10 @@ function attachChildHandlers() {
|
|
|
360
432
|
onPassthroughChunk(chunk);
|
|
361
433
|
});
|
|
362
434
|
child.stdout.on("end", () => {
|
|
363
|
-
if (retryArmed)
|
|
435
|
+
if (retryArmed && (!sawTerminalFailure || holdTail.length > 0))
|
|
364
436
|
releaseHeldOutput();
|
|
437
|
+
if (!retryArmed)
|
|
438
|
+
forward.flush();
|
|
365
439
|
scanner.flush();
|
|
366
440
|
});
|
|
367
441
|
child.on("close", onChildClose);
|
|
@@ -412,6 +486,20 @@ function clearRecordedModelRejectionOnSuccess() {
|
|
|
412
486
|
}
|
|
413
487
|
}
|
|
414
488
|
function onChildClose(code, signal) {
|
|
489
|
+
if (!signal && retryArmed && sawTerminalFailure && !sawSuccessResult &&
|
|
490
|
+
!sawAuthFailure && pendingRetryCause) {
|
|
491
|
+
const releaseDecision = stateFile ? acquireShimLock(`${stateFile}.retry`) : undefined;
|
|
492
|
+
try {
|
|
493
|
+
if (attemptRetry(pendingRetryCause))
|
|
494
|
+
return;
|
|
495
|
+
}
|
|
496
|
+
finally {
|
|
497
|
+
releaseDecision?.();
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
if (retryArmed)
|
|
501
|
+
releaseHeldOutput();
|
|
502
|
+
forward.flush();
|
|
415
503
|
if (signal) {
|
|
416
504
|
process.kill(process.pid, signal);
|
|
417
505
|
return;
|
|
@@ -419,6 +507,9 @@ function onChildClose(code, signal) {
|
|
|
419
507
|
if ((code ?? 0) === 0) {
|
|
420
508
|
clearRecordedAuthFailureOnSuccess();
|
|
421
509
|
clearRecordedModelRejectionOnSuccess();
|
|
510
|
+
if (affinity && sessionId && sawSuccessResult && !sawAuthFailure && !sawModelLimit) {
|
|
511
|
+
recordSessionPin(affinity.file, sessionId, accountId, Date.now(), affinity.ttlMs);
|
|
512
|
+
}
|
|
422
513
|
}
|
|
423
514
|
process.exit(code ?? 0);
|
|
424
515
|
}
|
package/dist/spread.js
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
export const DEFAULT_SPREAD_WINDOW_MS = 60 * 60 * 1000;
|
|
2
|
+
export const MIN_SPREAD_WINDOW_MS = 60 * 1000;
|
|
3
|
+
export const MAX_SPREAD_WINDOW_MS = 7 * 24 * 60 * 60 * 1000;
|
|
4
|
+
export function spreadWindowMs(value) {
|
|
5
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0)
|
|
6
|
+
return DEFAULT_SPREAD_WINDOW_MS;
|
|
7
|
+
return Math.min(MAX_SPREAD_WINDOW_MS, Math.max(MIN_SPREAD_WINDOW_MS, value));
|
|
8
|
+
}
|
|
9
|
+
export function spreadStatsFromHistory(events, nowMs, windowMs = DEFAULT_SPREAD_WINDOW_MS) {
|
|
10
|
+
const cutoff = nowMs - windowMs;
|
|
11
|
+
const stats = new Map();
|
|
12
|
+
for (const event of events) {
|
|
13
|
+
if (event.kind !== "served")
|
|
14
|
+
continue;
|
|
15
|
+
const at = Date.parse(event.at);
|
|
16
|
+
if (!Number.isFinite(at) || at < cutoff || at > nowMs)
|
|
17
|
+
continue;
|
|
18
|
+
const current = stats.get(event.accountId) ?? { servedInWindow: 0 };
|
|
19
|
+
current.servedInWindow += 1;
|
|
20
|
+
if (current.lastServedAt === undefined || at > current.lastServedAt)
|
|
21
|
+
current.lastServedAt = at;
|
|
22
|
+
stats.set(event.accountId, current);
|
|
23
|
+
}
|
|
24
|
+
return stats;
|
|
25
|
+
}
|
|
26
|
+
export function spreadStatsFor(stats, accountId) {
|
|
27
|
+
const s = stats.get(accountId);
|
|
28
|
+
return { servedInWindow: s?.servedInWindow ?? 0, ...(s?.lastServedAt !== undefined ? { lastServedAt: s.lastServedAt } : {}) };
|
|
29
|
+
}
|
|
30
|
+
export function withSpreadStats(verdicts, stats) {
|
|
31
|
+
return verdicts.map((v) => ({ ...v, ...spreadStatsFor(stats, v.id) }));
|
|
32
|
+
}
|
|
33
|
+
export function spreadBefore(a, b) {
|
|
34
|
+
const served = (a.servedInWindow ?? 0) - (b.servedInWindow ?? 0);
|
|
35
|
+
if (served !== 0)
|
|
36
|
+
return served < 0;
|
|
37
|
+
return (a.lastServedAt ?? -Infinity) < (b.lastServedAt ?? -Infinity);
|
|
38
|
+
}
|
|
39
|
+
export function describeSpreadPick(picked, windowMs) {
|
|
40
|
+
const window = describeWindow(windowMs);
|
|
41
|
+
const n = picked.servedInWindow ?? 0;
|
|
42
|
+
if (n === 0)
|
|
43
|
+
return `not served in the last ${window}`;
|
|
44
|
+
return `fewest served launches in the last ${window} (${n})`;
|
|
45
|
+
}
|
|
46
|
+
export function describeWindow(windowMs) {
|
|
47
|
+
const minutes = Math.round(windowMs / 60_000);
|
|
48
|
+
if (minutes < 60)
|
|
49
|
+
return `${minutes}m`;
|
|
50
|
+
const hours = minutes / 60;
|
|
51
|
+
if (hours < 48)
|
|
52
|
+
return Number.isInteger(hours) ? `${hours}h` : `${hours.toFixed(1)}h`;
|
|
53
|
+
return `${Math.round(hours / 24)}d`;
|
|
54
|
+
}
|
package/dist/sticky.js
CHANGED
|
@@ -7,6 +7,8 @@ export function decideStickySelection(params) {
|
|
|
7
7
|
const healthChoice = choosePoolAccount(verdicts, params.strategy);
|
|
8
8
|
if (!healthChoice)
|
|
9
9
|
return { account: fallbackPoolAccount(verdicts, nowMs) };
|
|
10
|
+
if (params.strategy === "spread")
|
|
11
|
+
return { account: healthChoice };
|
|
10
12
|
if (params.strategy === "reset-aware") {
|
|
11
13
|
const current = sticky && verdicts.find((v) => v.id === sticky.account);
|
|
12
14
|
const stillUsable = current?.verdict === "ok" || current?.verdict === "no_data";
|
package/dist/update-core.js
CHANGED
|
@@ -15,6 +15,80 @@ export function decideUpdateAction(opts) {
|
|
|
15
15
|
return "unknown";
|
|
16
16
|
return compareVersions(opts.installed, opts.latest) < 0 ? "update" : "up-to-date";
|
|
17
17
|
}
|
|
18
|
+
export function pluginInstallAttempts(pkg, version) {
|
|
19
|
+
return [
|
|
20
|
+
["plugins", "install", `clawhub:${pkg}@${version}`, "--force"],
|
|
21
|
+
["plugins", "install", `${pkg}@${version}`, "--pin", "--force"],
|
|
22
|
+
];
|
|
23
|
+
}
|
|
24
|
+
export function exactTargetVersion(version) {
|
|
25
|
+
return version && /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-(?:0|[1-9]\d*|[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:0|[1-9]\d*|[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-]*))*)?$/.test(version)
|
|
26
|
+
? version : undefined;
|
|
27
|
+
}
|
|
28
|
+
export function classifyClawhubFailure(result, output) {
|
|
29
|
+
if (result.status === 0 || result.signal || result.status === null || result.error != null || result.status >= 128)
|
|
30
|
+
return "stop";
|
|
31
|
+
if (/moderation|integrity|trust|policy|signature|security|malicious|quarantin|revoked|blocked|cancel(?:led|ed)?|abort|clawhub_(?:download_blocked|security_unavailable|archive_integrity_mismatch)/i.test(output))
|
|
32
|
+
return "stop";
|
|
33
|
+
if (/package not found on clawhub|version not found on clawhub|\b(?:package_not_found|version_not_found)\b/i.test(output))
|
|
34
|
+
return "fallback";
|
|
35
|
+
if (/clawhub.{0,80}(?:unreachable|temporarily unavailable|network error|fetch failed|connection (?:failed|refused|timed out))|ClawHub .* failed \(5\d\d\)|(?:ENOTFOUND|EAI_AGAIN|ECONNREFUSED|ECONNRESET|ETIMEDOUT)/i.test(output))
|
|
36
|
+
return "fallback";
|
|
37
|
+
if (/unknown (?:option|command)|unsupported (?:spec|source|protocol)|invalid (?:spec|source).{0,80}clawhub:|clawhub:.{0,80}(?:not supported|unsupported)/i.test(output))
|
|
38
|
+
return "fallback";
|
|
39
|
+
return "stop";
|
|
40
|
+
}
|
|
41
|
+
function asInstallRecord(value) {
|
|
42
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
43
|
+
return undefined;
|
|
44
|
+
const record = value;
|
|
45
|
+
if (typeof record.source !== "string")
|
|
46
|
+
return undefined;
|
|
47
|
+
return {
|
|
48
|
+
source: record.source,
|
|
49
|
+
...(typeof record.version === "string" ? { version: record.version } : {}),
|
|
50
|
+
...(typeof record.clawhubChannel === "string" ? { clawhubChannel: record.clawhubChannel } : {}),
|
|
51
|
+
...(typeof record.installPath === "string" ? { installPath: record.installPath } : {}),
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
export function parseInstallRecord(inspectJsonText, openclawConfig) {
|
|
55
|
+
let inspected;
|
|
56
|
+
try {
|
|
57
|
+
inspected = JSON.parse(inspectJsonText ?? "");
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
}
|
|
61
|
+
const inspectRecord = inspected && typeof inspected === "object" && !Array.isArray(inspected)
|
|
62
|
+
? asInstallRecord(inspected.install)
|
|
63
|
+
: undefined;
|
|
64
|
+
if (inspectRecord)
|
|
65
|
+
return inspectRecord;
|
|
66
|
+
const config = openclawConfig && typeof openclawConfig === "object" && !Array.isArray(openclawConfig)
|
|
67
|
+
? openclawConfig
|
|
68
|
+
: undefined;
|
|
69
|
+
return asInstallRecord(config?.plugins?.installs?.["multi-clawd"]);
|
|
70
|
+
}
|
|
71
|
+
export function classifyInstallSource(record) {
|
|
72
|
+
switch (record?.source) {
|
|
73
|
+
case "clawhub":
|
|
74
|
+
return {
|
|
75
|
+
level: "ok",
|
|
76
|
+
text: `install source: ClawHub${record.version ? ` v${record.version}` : ""}${record.clawhubChannel ? ` (${record.clawhubChannel} channel)` : ""}`,
|
|
77
|
+
};
|
|
78
|
+
case "npm":
|
|
79
|
+
return {
|
|
80
|
+
level: "warn",
|
|
81
|
+
text: "install source: npm — OpenClaw shows an untrusted-source caution for npm installs. Fix: multi-clawd update moves it to ClawHub",
|
|
82
|
+
};
|
|
83
|
+
case "path":
|
|
84
|
+
case "archive":
|
|
85
|
+
case "git":
|
|
86
|
+
case "marketplace":
|
|
87
|
+
return { level: "note", text: `install source: ${record.source} (source/dev install)` };
|
|
88
|
+
default:
|
|
89
|
+
return undefined;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
18
92
|
export function classifyCliSkew(opts) {
|
|
19
93
|
if (opts.pluginVersion === undefined)
|
|
20
94
|
return "plugin-missing";
|
|
@@ -80,7 +154,7 @@ export function formatUpdateBanner(opts) {
|
|
|
80
154
|
const action = decideUpdateAction(opts);
|
|
81
155
|
switch (action) {
|
|
82
156
|
case "install":
|
|
83
|
-
return `not installed — latest is v${opts.latest}
|
|
157
|
+
return opts.latest ? `not installed — latest is v${opts.latest}` : "not installed — could not determine the latest version";
|
|
84
158
|
case "update":
|
|
85
159
|
return `update available: v${opts.installed} → v${opts.latest}`;
|
|
86
160
|
case "up-to-date":
|
package/openclaw.plugin.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"id": "multi-clawd",
|
|
3
3
|
"name": "multi-clawd",
|
|
4
|
-
"version": "2.
|
|
4
|
+
"version": "2.5.0",
|
|
5
5
|
"description": "Register additional Claude Code logins (Max/Pro accounts) as first-class OpenClaw CLI backends for cross-account failover, keeping the full skills/MCP harness on every account.",
|
|
6
6
|
"cliBackends": [
|
|
7
7
|
"claw1",
|
|
@@ -45,8 +45,17 @@
|
|
|
45
45
|
},
|
|
46
46
|
"strategy": {
|
|
47
47
|
"type": "string",
|
|
48
|
-
"enum": [
|
|
49
|
-
|
|
48
|
+
"enum": [
|
|
49
|
+
"ordered",
|
|
50
|
+
"reset-aware",
|
|
51
|
+
"spread"
|
|
52
|
+
],
|
|
53
|
+
"description": "Healthy account selection: ordered (default) keeps pool order; reset-aware prefers the healthy account whose live seven-day quota window resets soonest; spread starts each new session on the healthy account with the fewest served launches in the recent window (spreadWindowMs) and keeps sessions on their serving account (implies sessionAffinity). Near-limit, exhausted and credential-failed accounts are never chosen by any strategy."
|
|
54
|
+
},
|
|
55
|
+
"spreadWindowMs": {
|
|
56
|
+
"type": "number",
|
|
57
|
+
"exclusiveMinimum": 0,
|
|
58
|
+
"description": "Spread strategy only: how far back served launches count when placing a new session. Default 3600000 (1h); clamped to 60000..604800000."
|
|
50
59
|
},
|
|
51
60
|
"utilizationThreshold": {
|
|
52
61
|
"type": "number",
|
|
@@ -70,6 +79,15 @@
|
|
|
70
79
|
"type": "number",
|
|
71
80
|
"description": "After rotating away from home, stay on the rotated-to account at least this long before returning home (anti-flap hysteresis; health always overrides). Default 600000 (10min)."
|
|
72
81
|
},
|
|
82
|
+
"sessionAffinity": {
|
|
83
|
+
"type": "boolean",
|
|
84
|
+
"description": "Pin a Claude CLI session to the account that last served it, while that account remains healthy. CLI launches only. Default false."
|
|
85
|
+
},
|
|
86
|
+
"sessionAffinityTtlMs": {
|
|
87
|
+
"type": "number",
|
|
88
|
+
"exclusiveMinimum": 0,
|
|
89
|
+
"description": "Successful session pin lifetime in milliseconds. Default 2592000000 (30 days); only used when sessionAffinity is true."
|
|
90
|
+
},
|
|
73
91
|
"usagePoll": {
|
|
74
92
|
"type": "object",
|
|
75
93
|
"additionalProperties": false,
|
|
@@ -152,6 +170,11 @@
|
|
|
152
170
|
"type": "string",
|
|
153
171
|
"description": "Human-friendly label shown in status/catalog."
|
|
154
172
|
},
|
|
173
|
+
"disabled": {
|
|
174
|
+
"type": "boolean",
|
|
175
|
+
"default": false,
|
|
176
|
+
"description": "Exclude this account from pool selection, in-turn retries, session affinity and direct Anthropic order while keeping it configured and visible in status. Takes effect when the plugin next reads config."
|
|
177
|
+
},
|
|
155
178
|
"native": {
|
|
156
179
|
"type": "boolean",
|
|
157
180
|
"description": "Use the machine's native Claude login (default config dir; OS keychain on macOS). Sets neither CLAUDE_CONFIG_DIR nor a token. Use this to include the main account in the rotation pool."
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@drakon-systems/multi-clawd",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.5.0",
|
|
4
4
|
"description": "Multi-account Claude Code failover for OpenClaw — register additional Claude (Max/Pro) logins as first-class CLI backends and keep the full skills/MCP harness across every account. Also imports those accounts' setup tokens into Hermes Agent's Anthropic credential pool.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|