@drakon-systems/multi-clawd 2.4.0 → 2.5.0

package/dist/shim.js

@@ -1,9 +1,11 @@
 import { spawn } from "node:child_process";
-import { mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
-import { dirname } from "node:path";
-import { classifyStateReadFailure, clearCredentialFailure, clearModelRejection, createLineScanner, limitFamilyFor, mergeHealthStates, parseAuthFailure, parseRateLimitEvent, parseStoredState, parseSuccessResult, recordCredentialFailure, updateHealthState, } from "./shim-core.js";
+import { readFileSync, writeFileSync } from "node:fs";
+import { persistHealthState, readHealthStateFile } from "./health-journal.js";
+import { acquireShimLock } from "./shim-lock.js";
+import { recordHistory } from "./history.js";
+import { clearCredentialFailure, clearModelRejection, createForwardWriter, createLineScanner, limitFamilyFor, parseAuthFailure, parseRateLimitEvent, parseSuccessResult, recordCredentialFailure, updateHealthState, } from "./shim-core.js";
 import { rewriteModelArg } from "./degrade.js";
-import { parseModelLimitError, recordModelLimit } from "./shim-core.js";
+import { parseApiRetryNotice, parseModelLimitError, parseTransientApiError, recordModelLimit, } from "./shim-core.js";
 import { canonicalModelId } from "./models.js";
 import { buildRetryEnv, chooseLaunchableRetryAccount, materializeRetryAccount, parseRetryRoster, retryArming, RETRY_ROSTER_ENV, } from "./retry-plan.js";
 import { eligibleSessionAccount, launchSessionId, readSessionPin, recordSessionPin, SESSION_AFFINITY_ENV, } from "./session-affinity.js";
@@ -60,40 +62,12 @@ function preserveCorruptState(raw) {
     process.stderr.write(`[multi-clawd shim] state file unreadable/corrupt — starting fresh (${note})\n`);
 }
 function readPersistedState() {
-    if (!stateFile)
-        return undefined;
-    let raw;
-    try {
-        raw = readFileSync(stateFile, "utf8");
-    }
-    catch (err) {
-        if (classifyStateReadFailure(err) === "absent")
-            return undefined;
-        preserveCorruptState(undefined);
-        return undefined;
-    }
-    const parsed = parseStoredState(raw);
-    if (parsed === undefined) {
-        preserveCorruptState(raw);
-        return undefined;
-    }
-    return parsed;
+    return stateFile ? readHealthStateFile(stateFile, { onSnapshotError: preserveCorruptState }) : undefined;
 }
 function persistState() {
     if (!stateFile)
         return;
-    try {
-        const disk = readPersistedState();
-        if (disk)
-            state = mergeHealthStates(disk, state, Date.now());
-        mkdirSync(dirname(stateFile), { recursive: true });
-        const tmp = `${stateFile}.tmp-${process.pid}`;
-        writeFileSync(tmp, JSON.stringify(state, null, 2), { mode: 0o600 });
-        renameSync(tmp, stateFile);
-    }
-    catch (err) {
-        process.stderr.write(`[multi-clawd shim] state write failed: ${String(err)}\n`);
-    }
+    state = persistHealthState(stateFile, state, { onSnapshotError: preserveCorruptState });
 }
 const { command, prependArgs, argv0: claudeArgv0 } = resolveClaudeCommand();
 let childArgs = [...prependArgs, ...process.argv.slice(2)];
@@ -163,6 +137,8 @@ let child = spawn(command, childArgs, {
     env: launchEnv,
     ...(claudeArgv0 ? { argv0: claudeArgv0 } : {}),
 });
+child.once("spawn", () => recordHistory(stateFile, { at: new Date().toISOString(),
+    kind: "served", accountId, modelId: effectiveModelId() }));
 const STDIN_REPLAY_CAP_BYTES = 8 * 1024 * 1024;
 let stdinReplay = [];
 let stdinReplayBytes = 0;
@@ -209,6 +185,7 @@ function guessLimitResetsAt() {
 let sawAuthFailure = false;
 let sawModelLimit = false;
 let sawSuccessResult = false;
+let recordedLimitDisplayName;
 function observeLine(line) {
     try {
         if (parseSuccessResult(line))
@@ -219,7 +196,8 @@ function observeLine(line) {
             persistState();
         }
         const limitHit = parseModelLimitError(line);
-        if (limitHit) {
+        if (limitHit && limitHit.displayName !== recordedLimitDisplayName) {
+            recordedLimitDisplayName = limitHit.displayName;
             sawModelLimit = true;
             const model = effectiveModelId();
             if (model) {
@@ -242,12 +220,17 @@ function observeLine(line) {
     }
 }
 const scanner = createLineScanner(observeLine);
+const forward = createForwardWriter((bytes) => process.stdout.write(bytes), () => process.stderr.write(`[multi-clawd shim] model limit on ${accountId} forwarded to the host marked as a rate limit\n`));
 const HOLD_CAP_BYTES = 256 * 1024;
 const HOLD_MAX_MS = Number(process.env.MULTI_CLAWD_HOLD_MAX_MS ?? 10_000);
+const HOLD_RETRY_MAX_MS = Math.min(Number(process.env.MULTI_CLAWD_HOLD_RETRY_MAX_MS ?? 120_000), 150_000);
+const HOLD_RETRY_GRACE_MS = 5_000;
 let held = [];
 let heldBytes = 0;
 let holdTail = "";
 let retryUsed = false;
+let pendingRetryCause;
+let sawTerminalFailure = false;
 function isPreambleRecord(line) {
     if (line.length === 0)
         return true;
@@ -265,12 +248,12 @@ function isPreambleRecord(line) {
 }
 function releaseHeldOutput() {
     if (held.length > 0) {
-        process.stdout.write(held.join(""));
+        forward.push(Buffer.from(held.join(""), "utf8"));
         held = [];
         heldBytes = 0;
     }
     if (holdTail.length > 0) {
-        process.stdout.write(holdTail);
+        forward.push(Buffer.from(holdTail, "utf8"));
         scanner.push(holdTail);
         holdTail = "";
     }
@@ -278,14 +261,15 @@ function releaseHeldOutput() {
     stdinReplay = [];
 }
 function readSiblingState(file) {
-    try {
-        return parseStoredState(readFileSync(file, "utf8"));
-    }
-    catch {
-        return undefined;
-    }
+    return readHealthStateFile(file);
+}
+function describeCause(cause) {
+    if (cause.kind === "model_limit")
+        return "model limit";
+    const d = cause.detail;
+    return `transient provider error (${d.kind}${d.status !== undefined ? ` ${d.status}` : ""})`;
 }
-function attemptRetry() {
+function attemptRetry(cause) {
     if (!retryArmed || retryUsed)
         return false;
     const model = effectiveModelId();
@@ -294,13 +278,13 @@ function attemptRetry() {
         readState: readSiblingState,
         modelId: model,
         nowMs: Date.now(),
+        allowOverdueProbe: cause.kind !== "transient",
         materialize: (account) => materializeRetryAccount(account, (path) => readFileSync(path, "utf8")),
         onSkip: (id, reason) => process.stderr.write(`[multi-clawd shim] not retrying onto ${id}: its declared credential did not resolve (${reason})\n`),
     });
     if (!target) {
-        process.stderr.write(`[multi-clawd shim] model limit on ${accountId} with no healthy sibling to retry onto — ` +
+        process.stderr.write(`[multi-clawd shim] ${describeCause(cause)} on ${accountId} with no healthy sibling to retry onto — ` +
             `passing the failure through to the host's chain\n`);
-        releaseHeldOutput();
         return false;
     }
     if (resumeSnapshot) {
@@ -308,41 +292,40 @@ function attemptRetry() {
             handoverSnapshotTo(resumeSnapshot, effectiveConfigDir(target.env));
         }
         catch (err) {
-            process.stderr.write(`[multi-clawd shim] model limit on ${accountId}, but session ` +
+            process.stderr.write(`[multi-clawd shim] ${describeCause(cause)} on ${accountId}, but session ` +
                 `${resumeSnapshot.sessionId.slice(0, 8)} could not be handed to ${target.id} ` +
                 `(${err instanceof Error ? err.message : String(err)}) — ` +
                 `passing the failure through to the host's chain\n`);
-            releaseHeldOutput();
             return false;
         }
     }
     retryUsed = true;
     retryArmed = false;
-    process.stderr.write(`[multi-clawd shim] model limit on ${accountId}${model ? ` for ${model}` : ""} — ` +
+    const reason = cause.kind === "model_limit" ? "model_limit" : "transient";
+    recordHistory(stateFile, { at: new Date().toISOString(), kind: "retry", accountId,
+        toAccountId: target.id, modelId: model, reason });
+    if (target.id !== accountId)
+        recordHistory(stateFile, { at: new Date().toISOString(),
+            kind: "rotation", accountId, toAccountId: target.id, modelId: model, reason });
+    process.stderr.write(`[multi-clawd shim] ${describeCause(cause)} on ${accountId}${model ? ` for ${model}` : ""} — ` +
         `retrying this turn on ${target.id}${resumeSnapshot ? `, session ${resumeSnapshot.sessionId.slice(0, 8)} handed over` : ""} (nothing forwarded downstream yet)\n`);
     held = [];
     heldBytes = 0;
     holdTail = "";
-    const previous = child;
-    previous.stdout.removeAllListeners();
-    previous.removeAllListeners("close");
-    previous.removeAllListeners("error");
-    try {
-        previous.kill("SIGTERM");
-    }
-    catch {
-    }
     accountId = target.id;
     stateFile = target.stateFile;
     state = { accountId, windows: {} };
     sawAuthFailure = false;
     sawModelLimit = false;
+    recordedLimitDisplayName = undefined;
     sawSuccessResult = false;
     child = spawn(command, childArgs, {
         stdio: ["pipe", "pipe", "inherit"],
         env: buildRetryEnv(launchEnv, target),
         ...(claudeArgv0 ? { argv0: claudeArgv0 } : {}),
     });
+    child.once("spawn", () => recordHistory(stateFile, { at: new Date().toISOString(),
+        kind: "served", accountId, modelId: effectiveModelId() }));
     attachChildHandlers();
     for (const chunk of stdinReplay)
         child.stdin.write(chunk);
@@ -360,34 +343,84 @@ function onArmedChunk(chunk) {
         const raw = holdTail.slice(0, idx + 1);
         holdTail = holdTail.slice(idx + 1);
         const line = raw.trim();
-        observeLine(line);
-        if (parseModelLimitError(line) && attemptRetry())
-            return;
-        held.push(raw);
-        heldBytes += raw.length;
-        if (!isPreambleRecord(line) || heldBytes > HOLD_CAP_BYTES) {
-            releaseHeldOutput();
-            return;
+        const releaseDecision = stateFile && (parseModelLimitError(line) || parseTransientApiError(line))
+            ? acquireShimLock(`${stateFile}.retry`)
+            : undefined;
+        try {
+            observeLine(line);
+            let cause;
+            if (parseModelLimitError(line)) {
+                cause = { kind: "model_limit" };
+            }
+            else if (!sawModelLimit && !sawAuthFailure) {
+                const transient = parseTransientApiError(line);
+                if (transient)
+                    cause = { kind: "transient", detail: transient };
+            }
+            if (cause && (!pendingRetryCause || cause.kind === "model_limit")) {
+                pendingRetryCause = cause;
+            }
+            let terminalFailure = false;
+            try {
+                const record = JSON.parse(line);
+                terminalFailure = record.type === "result" && record.is_error === true;
+            }
+            catch { }
+            if (terminalFailure)
+                sawTerminalFailure = true;
+            const retryNotice = parseApiRetryNotice(line);
+            if (retryNotice)
+                extendHold(retryNotice.retryDelayMs + HOLD_RETRY_GRACE_MS);
+            held.push(raw);
+            heldBytes += Buffer.byteLength(raw);
+            if ((!isPreambleRecord(line) && !(terminalFailure && pendingRetryCause && !sawAuthFailure)) || heldBytes > HOLD_CAP_BYTES) {
+                releaseHeldOutput();
+                return;
+            }
+        }
+        finally {
+            releaseDecision?.();
         }
     }
+    if (retryArmed && heldBytes + Buffer.byteLength(holdTail) > HOLD_CAP_BYTES) {
+        releaseHeldOutput();
+    }
     if (!retryArmed && holdTail.length > 0) {
         const rest = holdTail;
         holdTail = "";
-        process.stdout.write(rest);
+        forward.push(Buffer.from(rest, "utf8"));
         scanner.push(rest);
     }
 }
-if (retryArmed) {
-    const holdTimer = setTimeout(() => {
+let holdTimer;
+const holdStartedAt = Date.now();
+let holdDeadline = holdStartedAt + HOLD_MAX_MS;
+function armHoldTimer() {
+    if (holdTimer)
+        clearTimeout(holdTimer);
+    holdTimer = setTimeout(() => {
         if (!retryArmed)
             return;
-        process.stderr.write(`[multi-clawd shim] in-turn retry window closed after ${HOLD_MAX_MS}ms — releasing the stream\n`);
+        process.stderr.write(`[multi-clawd shim] in-turn retry window closed after ${Date.now() - holdStartedAt}ms — releasing the stream\n`);
         releaseHeldOutput();
-    }, HOLD_MAX_MS);
+    }, Math.max(0, holdDeadline - Date.now()));
     holdTimer.unref?.();
 }
+function extendHold(ms) {
+    if (!retryArmed)
+        return;
+    const wanted = Date.now() + ms;
+    const ceiling = holdStartedAt + HOLD_RETRY_MAX_MS;
+    const next = Math.min(wanted, ceiling);
+    if (next <= holdDeadline)
+        return;
+    holdDeadline = next;
+    armHoldTimer();
+}
+if (retryArmed)
+    armHoldTimer();
 function onPassthroughChunk(chunk) {
-    process.stdout.write(chunk);
+    forward.push(chunk);
     scanner.push(chunk.toString("utf8"));
 }
 function attachChildHandlers() {
@@ -399,8 +432,10 @@ function attachChildHandlers() {
             onPassthroughChunk(chunk);
     });
     child.stdout.on("end", () => {
-        if (retryArmed)
+        if (retryArmed && (!sawTerminalFailure || holdTail.length > 0))
             releaseHeldOutput();
+        if (!retryArmed)
+            forward.flush();
         scanner.flush();
     });
     child.on("close", onChildClose);
@@ -451,6 +486,20 @@ function clearRecordedModelRejectionOnSuccess() {
     }
 }
 function onChildClose(code, signal) {
+    if (!signal && retryArmed && sawTerminalFailure && !sawSuccessResult &&
+        !sawAuthFailure && pendingRetryCause) {
+        const releaseDecision = stateFile ? acquireShimLock(`${stateFile}.retry`) : undefined;
+        try {
+            if (attemptRetry(pendingRetryCause))
+                return;
+        }
+        finally {
+            releaseDecision?.();
+        }
+    }
+    if (retryArmed)
+        releaseHeldOutput();
+    forward.flush();
     if (signal) {
         process.kill(process.pid, signal);
         return;

package/dist/spread.js

@@ -0,0 +1,54 @@
+export const DEFAULT_SPREAD_WINDOW_MS = 60 * 60 * 1000;
+export const MIN_SPREAD_WINDOW_MS = 60 * 1000;
+export const MAX_SPREAD_WINDOW_MS = 7 * 24 * 60 * 60 * 1000;
+export function spreadWindowMs(value) {
+    if (typeof value !== "number" || !Number.isFinite(value) || value <= 0)
+        return DEFAULT_SPREAD_WINDOW_MS;
+    return Math.min(MAX_SPREAD_WINDOW_MS, Math.max(MIN_SPREAD_WINDOW_MS, value));
+}
+export function spreadStatsFromHistory(events, nowMs, windowMs = DEFAULT_SPREAD_WINDOW_MS) {
+    const cutoff = nowMs - windowMs;
+    const stats = new Map();
+    for (const event of events) {
+        if (event.kind !== "served")
+            continue;
+        const at = Date.parse(event.at);
+        if (!Number.isFinite(at) || at < cutoff || at > nowMs)
+            continue;
+        const current = stats.get(event.accountId) ?? { servedInWindow: 0 };
+        current.servedInWindow += 1;
+        if (current.lastServedAt === undefined || at > current.lastServedAt)
+            current.lastServedAt = at;
+        stats.set(event.accountId, current);
+    }
+    return stats;
+}
+export function spreadStatsFor(stats, accountId) {
+    const s = stats.get(accountId);
+    return { servedInWindow: s?.servedInWindow ?? 0, ...(s?.lastServedAt !== undefined ? { lastServedAt: s.lastServedAt } : {}) };
+}
+export function withSpreadStats(verdicts, stats) {
+    return verdicts.map((v) => ({ ...v, ...spreadStatsFor(stats, v.id) }));
+}
+export function spreadBefore(a, b) {
+    const served = (a.servedInWindow ?? 0) - (b.servedInWindow ?? 0);
+    if (served !== 0)
+        return served < 0;
+    return (a.lastServedAt ?? -Infinity) < (b.lastServedAt ?? -Infinity);
+}
+export function describeSpreadPick(picked, windowMs) {
+    const window = describeWindow(windowMs);
+    const n = picked.servedInWindow ?? 0;
+    if (n === 0)
+        return `not served in the last ${window}`;
+    return `fewest served launches in the last ${window} (${n})`;
+}
+export function describeWindow(windowMs) {
+    const minutes = Math.round(windowMs / 60_000);
+    if (minutes < 60)
+        return `${minutes}m`;
+    const hours = minutes / 60;
+    if (hours < 48)
+        return Number.isInteger(hours) ? `${hours}h` : `${hours.toFixed(1)}h`;
+    return `${Math.round(hours / 24)}d`;
+}

package/dist/sticky.js

@@ -7,6 +7,8 @@ export function decideStickySelection(params) {
     const healthChoice = choosePoolAccount(verdicts, params.strategy);
     if (!healthChoice)
         return { account: fallbackPoolAccount(verdicts, nowMs) };
+    if (params.strategy === "spread")
+        return { account: healthChoice };
     if (params.strategy === "reset-aware") {
         const current = sticky && verdicts.find((v) => v.id === sticky.account);
         const stillUsable = current?.verdict === "ok" || current?.verdict === "no_data";

package/dist/update-core.js

@@ -15,6 +15,80 @@ export function decideUpdateAction(opts) {
         return "unknown";
     return compareVersions(opts.installed, opts.latest) < 0 ? "update" : "up-to-date";
 }
+export function pluginInstallAttempts(pkg, version) {
+    return [
+        ["plugins", "install", `clawhub:${pkg}@${version}`, "--force"],
+        ["plugins", "install", `${pkg}@${version}`, "--pin", "--force"],
+    ];
+}
+export function exactTargetVersion(version) {
+    return version && /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-(?:0|[1-9]\d*|[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:0|[1-9]\d*|[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-]*))*)?$/.test(version)
+        ? version : undefined;
+}
+export function classifyClawhubFailure(result, output) {
+    if (result.status === 0 || result.signal || result.status === null || result.error != null || result.status >= 128)
+        return "stop";
+    if (/moderation|integrity|trust|policy|signature|security|malicious|quarantin|revoked|blocked|cancel(?:led|ed)?|abort|clawhub_(?:download_blocked|security_unavailable|archive_integrity_mismatch)/i.test(output))
+        return "stop";
+    if (/package not found on clawhub|version not found on clawhub|\b(?:package_not_found|version_not_found)\b/i.test(output))
+        return "fallback";
+    if (/clawhub.{0,80}(?:unreachable|temporarily unavailable|network error|fetch failed|connection (?:failed|refused|timed out))|ClawHub .* failed \(5\d\d\)|(?:ENOTFOUND|EAI_AGAIN|ECONNREFUSED|ECONNRESET|ETIMEDOUT)/i.test(output))
+        return "fallback";
+    if (/unknown (?:option|command)|unsupported (?:spec|source|protocol)|invalid (?:spec|source).{0,80}clawhub:|clawhub:.{0,80}(?:not supported|unsupported)/i.test(output))
+        return "fallback";
+    return "stop";
+}
+function asInstallRecord(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        return undefined;
+    const record = value;
+    if (typeof record.source !== "string")
+        return undefined;
+    return {
+        source: record.source,
+        ...(typeof record.version === "string" ? { version: record.version } : {}),
+        ...(typeof record.clawhubChannel === "string" ? { clawhubChannel: record.clawhubChannel } : {}),
+        ...(typeof record.installPath === "string" ? { installPath: record.installPath } : {}),
+    };
+}
+export function parseInstallRecord(inspectJsonText, openclawConfig) {
+    let inspected;
+    try {
+        inspected = JSON.parse(inspectJsonText ?? "");
+    }
+    catch {
+    }
+    const inspectRecord = inspected && typeof inspected === "object" && !Array.isArray(inspected)
+        ? asInstallRecord(inspected.install)
+        : undefined;
+    if (inspectRecord)
+        return inspectRecord;
+    const config = openclawConfig && typeof openclawConfig === "object" && !Array.isArray(openclawConfig)
+        ? openclawConfig
+        : undefined;
+    return asInstallRecord(config?.plugins?.installs?.["multi-clawd"]);
+}
+export function classifyInstallSource(record) {
+    switch (record?.source) {
+        case "clawhub":
+            return {
+                level: "ok",
+                text: `install source: ClawHub${record.version ? ` v${record.version}` : ""}${record.clawhubChannel ? ` (${record.clawhubChannel} channel)` : ""}`,
+            };
+        case "npm":
+            return {
+                level: "warn",
+                text: "install source: npm — OpenClaw shows an untrusted-source caution for npm installs. Fix: multi-clawd update moves it to ClawHub",
+            };
+        case "path":
+        case "archive":
+        case "git":
+        case "marketplace":
+            return { level: "note", text: `install source: ${record.source} (source/dev install)` };
+        default:
+            return undefined;
+    }
+}
 export function classifyCliSkew(opts) {
     if (opts.pluginVersion === undefined)
         return "plugin-missing";
@@ -80,7 +154,7 @@ export function formatUpdateBanner(opts) {
     const action = decideUpdateAction(opts);
     switch (action) {
         case "install":
-            return `not installed — latest is v${opts.latest}`;
+            return opts.latest ? `not installed — latest is v${opts.latest}` : "not installed — could not determine the latest version";
         case "update":
             return `update available: v${opts.installed} → v${opts.latest}`;
         case "up-to-date":

package/openclaw.plugin.json

@@ -1,7 +1,7 @@
 {
   "id": "multi-clawd",
   "name": "multi-clawd",
-  "version": "2.4.0",
+  "version": "2.5.0",
   "description": "Register additional Claude Code logins (Max/Pro accounts) as first-class OpenClaw CLI backends for cross-account failover, keeping the full skills/MCP harness on every account.",
   "cliBackends": [
     "claw1",
@@ -45,8 +45,17 @@
           },
           "strategy": {
             "type": "string",
-            "enum": ["ordered", "reset-aware"],
-            "description": "Healthy account selection: ordered (default) keeps pool order; reset-aware prefers the healthy account whose live seven-day quota window resets soonest."
+            "enum": [
+              "ordered",
+              "reset-aware",
+              "spread"
+            ],
+            "description": "Healthy account selection: ordered (default) keeps pool order; reset-aware prefers the healthy account whose live seven-day quota window resets soonest; spread starts each new session on the healthy account with the fewest served launches in the recent window (spreadWindowMs) and keeps sessions on their serving account (implies sessionAffinity). Near-limit, exhausted and credential-failed accounts are never chosen by any strategy."
+          },
+          "spreadWindowMs": {
+            "type": "number",
+            "exclusiveMinimum": 0,
+            "description": "Spread strategy only: how far back served launches count when placing a new session. Default 3600000 (1h); clamped to 60000..604800000."
           },
           "utilizationThreshold": {
             "type": "number",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@drakon-systems/multi-clawd",
-  "version": "2.4.0",
+  "version": "2.5.0",
   "description": "Multi-account Claude Code failover for OpenClaw — register additional Claude (Max/Pro) logins as first-class CLI backends and keep the full skills/MCP harness across every account. Also imports those accounts' setup tokens into Hermes Agent's Anthropic credential pool.",
   "type": "module",
   "license": "MIT",