@drakon-systems/multi-clawd 2.4.0 → 2.5.0

package/dist/history.js

@@ -0,0 +1,103 @@
+import { randomUUID } from "node:crypto";
+import { mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { acquireShimLock } from "./shim-lock.js";
+export const HISTORY_FILE = "history.jsonl";
+export const HISTORY_MAX_BYTES = 64 * 1024;
+export const HISTORY_MAX_EVENTS = 500;
+export function historyFile(stateFile) {
+    return join(dirname(stateFile), HISTORY_FILE);
+}
+const safeAccount = (value) => typeof value === "string" &&
+    value.length > 0 && value.length <= 100 && !/[\x00-\x1f\x7f]/.test(value);
+const safeModel = (value) => typeof value === "string" &&
+    value.length > 0 && value.length <= 100 && /^[a-zA-Z0-9][a-zA-Z0-9._:/-]*$/.test(value);
+function valid(raw) {
+    if (!raw || typeof raw !== "object")
+        return undefined;
+    const e = raw;
+    if (typeof e.at !== "string" || !Number.isFinite(Date.parse(e.at)) ||
+        !["served", "retry", "rotation"].includes(String(e.kind)) || !safeAccount(e.accountId) ||
+        (e.toAccountId !== undefined && !safeAccount(e.toAccountId)) ||
+        (e.reason !== undefined && !["model_limit", "auth_failure", "transient", "pool_selection"].includes(String(e.reason))))
+        return undefined;
+    return { at: e.at, kind: e.kind, accountId: e.accountId,
+        ...(e.toAccountId ? { toAccountId: e.toAccountId } : {}),
+        ...(safeModel(e.modelId) ? { modelId: e.modelId } : {}),
+        ...(e.reason ? { reason: e.reason } : {}) };
+}
+export function readHistory(file) {
+    let raw;
+    try {
+        raw = readFileSync(file, "utf8");
+    }
+    catch {
+        return [];
+    }
+    return raw.split("\n").flatMap((line) => {
+        try {
+            const event = valid(JSON.parse(line));
+            return event ? [event] : [];
+        }
+        catch {
+            return [];
+        }
+    });
+}
+export function recordHistory(stateFile, event) {
+    if (!stateFile)
+        return;
+    let release;
+    let tmp;
+    try {
+        const checked = valid(event);
+        if (!checked)
+            return;
+        const file = historyFile(stateFile);
+        release = acquireShimLock(file);
+        if (!release)
+            return;
+        const events = [...readHistory(file), checked].slice(-HISTORY_MAX_EVENTS);
+        let lines = events.map((e) => JSON.stringify(e) + "\n");
+        let bytes = lines.reduce((sum, line) => sum + Buffer.byteLength(line), 0);
+        while (bytes > HISTORY_MAX_BYTES && lines.length)
+            bytes -= Buffer.byteLength(lines.shift());
+        mkdirSync(dirname(file), { recursive: true });
+        tmp = `${file}.tmp-${process.pid}-${randomUUID()}`;
+        writeFileSync(tmp, lines.join(""), { mode: 0o600 });
+        renameSync(tmp, file);
+    }
+    catch { }
+    finally {
+        if (tmp)
+            try {
+                unlinkSync(tmp);
+            }
+            catch { }
+        release?.();
+    }
+}
+export function parseHistorySince(value) {
+    const match = /^(\d+)([dh])$/.exec(value);
+    if (!match || Number(match[1]) < 1 || Number(match[1]) > 3650) {
+        throw new Error("--since must be a positive duration such as 7d or 24h");
+    }
+    return Number(match[1]) * (match[2] === "d" ? 86_400_000 : 3_600_000);
+}
+export function renderHistory(events, accountIds, since, nowMs = Date.now()) {
+    const cutoff = nowMs - parseHistorySince(since);
+    const recent = events.filter((e) => Date.parse(e.at) >= cutoff && Date.parse(e.at) <= nowMs);
+    const ids = [...new Set([...accountIds, ...recent.map((e) => e.accountId)])];
+    const lines = [`Account history (last ${since}):`];
+    if (!ids.length)
+        return [...lines, "  No accounts configured."].join("\n");
+    for (const id of ids) {
+        const own = recent.filter((e) => e.accountId === id);
+        const count = (kind) => own.filter((e) => e.kind === kind).length;
+        const last = own.filter((e) => e.kind === "served")
+            .reduce((latest, e) => !latest || Date.parse(e.at) > Date.parse(latest) ? e.at : latest, undefined);
+        const n = (k, one, many) => `${k} ${k === 1 ? one : many}`;
+        lines.push(`  ${id}: ${n(count("served"), "served launch", "served launches")}, ${n(count("retry"), "retry", "retries")}, ${n(count("rotation"), "rotation", "rotations")}; last served ${last ?? "never in this window"}`);
+    }
+    return lines.join("\n");
+}

package/dist/index.js

@@ -7,8 +7,10 @@ import { MODEL_ALIASES, buildCatalogEntries, canonicalModelId, isModernClaudeMod
 import { decideDegradation, matchesPin } from "./degrade.js";
 import { resolveExecMode, permissionModeArgs } from "./exec-policy.js";
 import { resolveBaseModelIds } from "./catalog-source.js";
-import { allCredentialFailed, classifyAccountHealth, credentialFailureFor, lastAuthSuccessAt, overdueProbeAccount, sevenDayReset, } from "./health.js";
+import { allCredentialFailed, classifyAccountHealth, credentialFailureFor, lastAuthSuccessAt, overdueProbeAccount, poolStrategy, sevenDayReset, } from "./health.js";
 import { decideStickySelection } from "./sticky.js";
+import { HISTORY_FILE, readHistory, recordHistory } from "./history.js";
+import { spreadStatsFromHistory, spreadWindowMs, withSpreadStats } from "./spread.js";
 import { DEFAULT_SESSION_AFFINITY_TTL_MS, launchSessionId, readSessionPin, SESSION_AFFINITY_ENV, } from "./session-affinity.js";
 import { mergeHealthStates, parseStoredState, recordCredentialFailure, } from "./shim-core.js";
 import { createTokenRefResolver, isSecretRefShape, } from "./token-resolution.js";
@@ -18,6 +20,7 @@ import { RETRY_ROSTER_ENV } from "./retry-plan.js";
 import { SESSION_DIRS_ENV } from "./session-handover.js";
 import { addAlert, alertKeysWithPrefix, clearAlert, pendingAlertText, } from "./alerts.js";
 import { healthStateFile, usageStateFile, clearAccountCredentialFailure } from "./credential-state.js";
+import { persistHealthState, readHealthStateFile } from "./health-journal.js";
 import { createUsagePollController, effectiveUsagePollInterval, effectiveUsageWarnThreshold, usageAlertPrefix, usagePoolAlertKey, } from "./usage-poll.js";
 export { healthStateFile, usageStateFile, clearAccountCredentialFailure };
 import { accountConfigDir, AccountCredentialError, buildAccountChildEnv, credentialFailureText, parseSetupTokenFile, tokenFileModeWarning, validateAccountTokenSources, } from "./account-env.js";
@@ -188,22 +191,8 @@ const refProbeTrackers = new Map();
 const lastProbeStatus = new Map();
 function recordAccountCredentialFailure(accountId, reason, nowMs) {
     const file = healthStateFile(accountId);
-    let state;
-    try {
-        state = parseStoredState(readFileSync(file, "utf8")) ?? { accountId, windows: {} };
-    }
-    catch {
-        state = { accountId, windows: {} };
-    }
-    const next = mergeHealthStates(state, recordCredentialFailure(state, reason, nowMs), nowMs);
-    try {
-        mkdirSync(dirname(file), { recursive: true });
-        const tmp = `${file}.tmp-${process.pid}`;
-        writeFileSync(tmp, JSON.stringify(next, null, 2), { mode: 0o600 });
-        renameSync(tmp, file);
-    }
-    catch {
-    }
+    const state = readHealthStateFile(file) ?? { accountId, windows: {} };
+    persistHealthState(file, recordCredentialFailure(state, reason, nowMs));
 }
 export async function runLoginHealthProbe(accounts, logger, deps = {}) {
     const now = deps.nowMs ?? Date.now();
@@ -740,7 +729,7 @@ export default definePluginEntry({
     },
 });
 function readHealthState(accountId) {
-    return mergeStoredHealth(readStateFileLenient(healthStateFile(accountId)), readStateFileLenient(usageStateFile(accountId)));
+    return mergeStoredHealth(readHealthStateFile(healthStateFile(accountId)), readStateFileLenient(usageStateFile(accountId)));
 }
 function readStateFileLenient(file) {
     try {
@@ -757,6 +746,9 @@ function mergeStoredHealth(shim, usage) {
         return shim;
     return mergeHealthStates(shim, usage);
 }
+function poolHistoryFile() {
+    return join(homedir(), ".openclaw", "state", "multi-clawd", HISTORY_FILE);
+}
 function readStickyEntry(file) {
     try {
         const parsed = JSON.parse(readFileSync(file, "utf8"));
@@ -824,6 +816,11 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
         defaultModel: pool.defaultModel,
     };
     const minDwellMs = pool.minDwellMs;
+    const strategy = poolStrategy(pool.strategy);
+    const affinityOn = pool.sessionAffinity === true || strategy === "spread";
+    if (strategy === "spread" && pool.sessionAffinity === false) {
+        logger.warn(`[multi-clawd] pool "${poolId}": strategy "spread" keeps each session on the account that first served it — sessionAffinity: false is ignored`);
+    }
     const stickyFile = join(homedir(), ".openclaw", "state", "multi-clawd", `pool-${poolId}.sticky.json`);
     const backend = buildBackend(poolAccount, execMode, deps?.launch);
     const { launch, transport } = launchTransports.get(backend);
@@ -849,20 +846,23 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
             throw new Error(`[multi-clawd] ${text}`);
         }
         const previousSticky = readStickyEntry(stickyFile);
-        const poolVerdicts = verdicts.map((v) => ({
+        const rawVerdicts = verdicts.map((v) => ({
             id: v.id,
             verdict: v.health.verdict,
             observedAt: v.health.observedAt,
-            sevenDayResetsAt: pool.strategy === "reset-aware"
+            sevenDayResetsAt: strategy === "reset-aware"
                 ? sevenDayReset(states.find((s) => s.id === v.id)?.state, options, now)
                 : undefined,
         }));
+        const poolVerdicts = strategy === "spread"
+            ? withSpreadStats(rawVerdicts, spreadStatsFromHistory(readHistory(poolHistoryFile()), now, spreadWindowMs(pool.spreadWindowMs)))
+            : rawVerdicts;
         const decision = decideStickySelection({
             verdicts: poolVerdicts,
             sticky: previousSticky,
             nowMs: now,
             minDwellMs,
-            strategy: pool.strategy,
+            strategy,
         });
         const chosen = members.find((a) => a.id === decision.account) ?? members[0];
         const previousAccount = previousSticky?.account ?? members[0].id;
@@ -878,7 +878,9 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
             logger.info(`[multi-clawd] pool ${poolId}: no proven account can serve ${requestedModel} — ` +
                 `launching on ${decision.account} to re-test a login last rejected by the Claude CLI`);
         }
-        else if (decision.account !== previousAccount) {
+        else if (strategy !== "spread" && decision.account !== previousAccount) {
+            recordHistory(join(homedir(), ".openclaw", "state", "multi-clawd", `${previousAccount}.json`), { at: new Date(now).toISOString(), kind: "rotation", accountId: previousAccount,
+                toAccountId: decision.account, modelId: requestedModel, reason: "pool_selection" });
             const home = verdicts[0];
             const line = decision.account === members[0].id
                 ? `pool ${poolId}: returning home to ${decision.account}`
@@ -975,7 +977,7 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
                 env[RETRY_ROSTER_ENV] = JSON.stringify(roster);
         }
         env[SESSION_DIRS_ENV] = JSON.stringify([...new Set(configuredMembers.map(accountConfigDir))]);
-        if (pool.sessionAffinity) {
+        if (affinityOn) {
             env[SESSION_AFFINITY_ENV] = JSON.stringify({
                 file: join(homedir(), ".openclaw", "state", "multi-clawd", `pool-${poolId}.sessions.json`),
                 ttlMs: typeof pool.sessionAffinityTtlMs === "number" &&
@@ -1012,7 +1014,7 @@ export function registerPoolBackend(api, pool, accounts, registeredIds, execMode
                 logger.info(`[multi-clawd] pool ${poolId}: pinned lane keeps ${ctx.modelId} despite exhausted pool (will fail over via the chain)`);
             }
         }
-        const affinityTransport = pool.sessionAffinity && transport && launch.kind === "verified"
+        const affinityTransport = affinityOn && transport && launch.kind === "verified"
             ? async function* (execution) {
                 const id = launchSessionId(execution.args);
                 const pin = id ? readSessionPin(join(homedir(), ".openclaw", "state", "multi-clawd", `pool-${poolId}.sessions.json`), id, Date.now(), typeof pool.sessionAffinityTtlMs === "number" && pool.sessionAffinityTtlMs > 0
@@ -1112,6 +1114,7 @@ export function startDirectOrderSync(params) {
         healthOptions,
         minDwellMs: params.pool?.minDwellMs,
         strategy: params.pool?.strategy,
+        spreadWindowMs: params.pool?.spreadWindowMs,
         command: params.directRoute?.openclawCommand,
     });
     if (directController?.signature === signature && directTimer) {
@@ -1127,6 +1130,8 @@ export function startDirectOrderSync(params) {
         healthOptions,
         minDwellMs: params.pool?.minDwellMs,
         strategy: params.pool?.strategy,
+        readHistory: () => readHistory(poolHistoryFile()),
+        spreadWindowMs: params.pool?.spreadWindowMs,
         configOrder: params.configOrder,
         readSticky: () => readStickyEntry(stickyFile),
         writeSticky: (entry) => writeStickyEntry(stickyFile, entry, logger),
@@ -1228,17 +1233,7 @@ export function startUsagePoll(params) {
                 }
             },
             readHealth: (id) => {
-                let shim;
-                try {
-                    shim = parseStoredState(readFileSync(healthStateFile(id), "utf8"));
-                    if (!shim)
-                        throw new Error("not valid health-state JSON");
-                }
-                catch (err) {
-                    if (err.code !== "ENOENT")
-                        throw err;
-                    shim = undefined;
-                }
+                const shim = readHealthStateFile(healthStateFile(id), { strictSnapshot: true });
                 let usage;
                 try {
                     usage = parseStoredState(readFileSync(usageStateFile(id), "utf8"));

package/dist/retry-plan.js

@@ -66,6 +66,8 @@ export function chooseRetryAccount(params) {
             return account;
         verdicts.push({ id: account.id, verdict: health.verdict, observedAt: health.observedAt });
     }
+    if (params.allowOverdueProbe === false)
+        return undefined;
     const probe = overdueProbeAccount(verdicts, params.nowMs);
     return probe ? params.roster.find((a) => a.id === probe) : undefined;
 }

package/dist/shim-core.js

@@ -192,9 +192,9 @@ export function limitFamilyFor(displayName, modelId) {
     return displayName.trim().toLowerCase() === family ? family : undefined;
 }
 const LIMIT_TEXT_RE = /reached your (.{1,40}?) limit/i;
-export function parseModelLimitError(line) {
-    if (!/reached your /i.test(line))
-        return undefined;
+const USAGE_LIMIT_API_ERROR = "usage_limit_reached";
+const AUTH_FAILED_ERROR_KIND = "authentication_failed";
+function parseRecord(line) {
     let record;
     try {
         record = JSON.parse(line);
@@ -204,12 +204,15 @@ export function parseModelLimitError(line) {
     }
     if (typeof record !== "object" || record === null)
         return undefined;
-    const r = record;
-    const isErrorRecord = r.type === "error" ||
+    return record;
+}
+function isErrorRecord(r) {
+    return (r.type === "error" ||
         r.is_error === true ||
-        (typeof r.subtype === "string" && r.subtype.startsWith("error"));
-    if (!isErrorRecord)
-        return undefined;
+        (typeof r.subtype === "string" && r.subtype.startsWith("error")) ||
+        (r.type === "assistant" && r.is_api_error_message === true));
+}
+function errorRecordTexts(r) {
     const texts = [];
     if (typeof r.result === "string")
         texts.push(r.result);
@@ -220,13 +223,193 @@ export function parseModelLimitError(line) {
         if (typeof msg === "string")
             texts.push(msg);
     }
-    for (const text of texts) {
+    if (r.type === "assistant" && r.is_api_error_message === true && Array.isArray(r.message?.content)) {
+        for (const block of r.message.content) {
+            if (block && block.type === "text" && typeof block.text === "string")
+                texts.push(block.text);
+        }
+    }
+    return texts;
+}
+const TRANSIENT_TEXT_PATTERNS = [
+    { re: /repeated 529 overloaded errors/i, kind: "overloaded" },
+    { re: /overloaded_error/i, kind: "overloaded" },
+    { re: /the api is at capacity/i, kind: "overloaded" },
+    { re: /server-side issue, usually temporary/i, kind: "server_error" },
+    { re: /temporarily limiting requests \(not your usage limit\)/i, kind: "rate_limit_transient" },
+    { re: /request rejected \(429\)/i, kind: "rate_limit_transient" },
+];
+const TRANSIENT_TEXT_MAX_CHARS = 160;
+export function parseTransientApiError(line) {
+    if (!/("error"|is_error|api_error_status|overloaded|server-side|temporarily limiting|request rejected|at capacity)/i.test(line)) {
+        return undefined;
+    }
+    const r = parseRecord(line);
+    if (!r || !isErrorRecord(r))
+        return undefined;
+    if (parseModelLimitError(line) || parseAuthFailure(line))
+        return undefined;
+    const joined = errorRecordTexts(r).join("\n");
+    const status = typeof r.api_error_status === "number" ? r.api_error_status : undefined;
+    const text = joined.trim().slice(0, TRANSIENT_TEXT_MAX_CHARS);
+    const kindFromCli = typeof r.error === "string" ? r.error : undefined;
+    if (status === 529 || kindFromCli === "overloaded")
+        return { kind: "overloaded", status, text };
+    if ((status !== undefined && status >= 500) || kindFromCli === "server_error") {
+        return { kind: "server_error", status, text };
+    }
+    for (const { re, kind } of TRANSIENT_TEXT_PATTERNS) {
+        if (re.test(joined))
+            return { kind, status, text };
+    }
+    if (status === 429)
+        return { kind: "rate_limit_transient", status, text };
+    return undefined;
+}
+export function parseApiRetryNotice(line) {
+    if (!line.includes('"api_retry"'))
+        return undefined;
+    let record;
+    try {
+        record = JSON.parse(line);
+    }
+    catch {
+        return undefined;
+    }
+    if (typeof record !== "object" || record === null)
+        return undefined;
+    const r = record;
+    if (r.type !== "system" || r.subtype !== "api_retry")
+        return undefined;
+    const delay = typeof r.retry_delay_ms === "number" && r.retry_delay_ms >= 0 ? r.retry_delay_ms : 0;
+    return { retryDelayMs: delay };
+}
+export function parseModelLimitError(line) {
+    if (!/(reached your |usage_limit_reached)/i.test(line))
+        return undefined;
+    const r = parseRecord(line);
+    if (!r || !isErrorRecord(r))
+        return undefined;
+    for (const text of errorRecordTexts(r)) {
         const match = LIMIT_TEXT_RE.exec(text);
         if (match)
             return { displayName: match[1].trim() };
     }
+    if (r.api_error === USAGE_LIMIT_API_ERROR)
+        return { displayName: USAGE_LIMIT_API_ERROR };
     return undefined;
 }
+export const LIMIT_PASSTHROUGH_MARKER = " (rate limit: 429)";
+const HOST_DIAGNOSTIC_ERROR_PREFIX = "[ede_diagnostic]";
+export function markForwardedLimitLine(line) {
+    if (!parseModelLimitError(line))
+        return line;
+    let record;
+    try {
+        record = JSON.parse(line);
+    }
+    catch {
+        return line;
+    }
+    if (typeof record !== "object" || record === null || Array.isArray(record))
+        return line;
+    const r = record;
+    const subtype = typeof r.subtype === "string" ? r.subtype : "";
+    if (r.type !== "result" || !(r.is_error === true || subtype.startsWith("error")))
+        return line;
+    const mark = (text) => text.includes(LIMIT_PASSTHROUGH_MARKER) ? undefined : `${text.trimEnd()}${LIMIT_PASSTHROUGH_MARKER}`;
+    if (Array.isArray(r.errors)) {
+        const idx = r.errors.findIndex((e) => typeof e === "string" && e.trim() !== "" && !e.trim().startsWith(HOST_DIAGNOSTIC_ERROR_PREFIX));
+        if (idx >= 0) {
+            const marked = mark(r.errors[idx]);
+            if (marked === undefined)
+                return line;
+            const errors = [...r.errors];
+            errors[idx] = marked;
+            return JSON.stringify({ ...r, errors });
+        }
+    }
+    if (typeof r.result === "string" && r.result.trim() !== "") {
+        const marked = mark(r.result);
+        return marked === undefined ? line : JSON.stringify({ ...r, result: marked });
+    }
+    if (typeof r.error === "object" && r.error !== null && !Array.isArray(r.error)) {
+        const err = r.error;
+        if (typeof err.message === "string" && err.message.trim() !== "") {
+            const marked = mark(err.message);
+            return marked === undefined ? line : JSON.stringify({ ...r, error: { ...err, message: marked } });
+        }
+    }
+    if (typeof r.error === "string" && r.error.trim() !== "") {
+        const marked = mark(r.error);
+        return marked === undefined ? line : JSON.stringify({ ...r, error: marked });
+    }
+    return line;
+}
+const RESULT_LINE_PREFIX = Buffer.from('{"type":"result"', "utf8");
+const FORWARD_HOLD_CAP_BYTES = 256 * 1024;
+function couldBeResultLine(pending) {
+    let i = 0;
+    while (i < pending.length && (pending[i] === 0x20 || pending[i] === 0x09 || pending[i] === 0x0d))
+        i++;
+    const rest = pending.subarray(i);
+    const n = Math.min(rest.length, RESULT_LINE_PREFIX.length);
+    return rest.subarray(0, n).equals(RESULT_LINE_PREFIX.subarray(0, n));
+}
+function forwardCompleteLine(line, onMark) {
+    const text = line.toString("utf8");
+    const ending = text.endsWith("\r\n") ? "\r\n" : text.endsWith("\n") ? "\n" : "";
+    const body = text.slice(0, text.length - ending.length).trim();
+    const marked = markForwardedLimitLine(body);
+    if (marked === body)
+        return line;
+    onMark?.();
+    return Buffer.from(marked + ending, "utf8");
+}
+export function createForwardWriter(write, onMark) {
+    let pending;
+    let streaming = false;
+    return {
+        push(chunk) {
+            const out = [];
+            let start = 0;
+            while (start < chunk.length) {
+                const nl = chunk.indexOf(0x0a, start);
+                const end = nl < 0 ? chunk.length : nl + 1;
+                const piece = chunk.subarray(start, end);
+                start = end;
+                if (streaming) {
+                    out.push(piece);
+                    if (nl >= 0)
+                        streaming = false;
+                    continue;
+                }
+                const line = pending ? Buffer.concat([pending, piece]) : piece;
+                pending = undefined;
+                if (nl >= 0) {
+                    out.push(forwardCompleteLine(line, onMark));
+                }
+                else if (!couldBeResultLine(line) || line.length > FORWARD_HOLD_CAP_BYTES) {
+                    out.push(line);
+                    streaming = true;
+                }
+                else {
+                    pending = Buffer.from(line);
+                }
+            }
+            if (out.length > 0)
+                write(out.length === 1 ? out[0] : Buffer.concat(out));
+        },
+        flush() {
+            if (pending) {
+                const line = pending;
+                pending = undefined;
+                write(forwardCompleteLine(line, onMark));
+            }
+            streaming = false;
+        },
+    };
+}
 export function recordModelLimit(state, modelId, now, resetsAt, family) {
     return {
         ...state,
@@ -300,36 +483,16 @@ export function parseAuthFailure(line) {
     if (!/(authenticat|oauth|logged in|\/login|unauthori|api key|access token|bearer token)/i.test(line)) {
         return undefined;
     }
-    let record;
-    try {
-        record = JSON.parse(line);
-    }
-    catch {
-        return undefined;
-    }
-    if (typeof record !== "object" || record === null)
-        return undefined;
-    const r = record;
-    const isErrorRecord = r.type === "error" ||
-        r.is_error === true ||
-        (typeof r.subtype === "string" && r.subtype.startsWith("error"));
-    if (!isErrorRecord)
+    const r = parseRecord(line);
+    if (!r || !isErrorRecord(r))
         return undefined;
-    const texts = [];
-    if (typeof r.result === "string")
-        texts.push(r.result);
-    if (typeof r.error === "string")
-        texts.push(r.error);
-    if (typeof r.error === "object" && r.error !== null) {
-        const msg = r.error.message;
-        if (typeof msg === "string")
-            texts.push(msg);
-    }
-    for (const text of texts) {
+    for (const text of errorRecordTexts(r)) {
         if (AUTH_FAILURE_PATTERNS.some((p) => p.test(text))) {
             return { reason: text.trim().slice(0, AUTH_REASON_MAX_CHARS) };
         }
     }
+    if (r.error === AUTH_FAILED_ERROR_KIND)
+        return { reason: AUTH_FAILED_ERROR_KIND };
     return undefined;
 }
 export function recordCredentialFailure(state, reason, now) {

package/dist/shim-lock.js

@@ -0,0 +1,106 @@
+import { randomUUID } from "node:crypto";
+import { closeSync, fstatSync, mkdirSync, openSync, readFileSync, statSync, unlinkSync, writeFileSync } from "node:fs";
+import { dirname, isAbsolute } from "node:path";
+const WAIT_MS = 150;
+const LEASE_MS = 5_000;
+const pause = new Int32Array(new SharedArrayBuffer(4));
+export function acquireShimLock(file, waitMs = WAIT_MS, hooks = {}) {
+    if (!file || !isAbsolute(file))
+        return undefined;
+    const lock = `${file}.lock`;
+    const deadline = Date.now() + waitMs;
+    try {
+        mkdirSync(dirname(lock), { recursive: true });
+    }
+    catch {
+        return undefined;
+    }
+    do {
+        try {
+            const fd = openSync(lock, "wx", 0o600);
+            const token = randomUUID();
+            const content = Buffer.from(`${process.pid} ${Date.now()} ${token}\n`);
+            let ino;
+            try {
+                ino = fstatSync(fd).ino;
+                writeFileSync(fd, content);
+            }
+            catch {
+                try {
+                    if (ino !== undefined && statSync(lock).ino === ino) {
+                        const found = readFileSync(lock);
+                        if (found.includes(Buffer.from(token)) || found.length === 0)
+                            unlinkSync(lock);
+                    }
+                }
+                catch { }
+                try {
+                    closeSync(fd);
+                }
+                catch { }
+                return undefined;
+            }
+            const release = () => {
+                try {
+                    if (statSync(lock).ino === ino && readFileSync(lock).equals(content))
+                        unlinkSync(lock);
+                }
+                catch { }
+            };
+            try {
+                (hooks.close ?? closeSync)(fd);
+            }
+            catch {
+                try {
+                    closeSync(fd);
+                }
+                catch { }
+                return release;
+            }
+            return release;
+        }
+        catch (err) {
+            if (err.code !== "EEXIST")
+                return undefined;
+        }
+        let staleFd;
+        try {
+            staleFd = openSync(lock, "r");
+            const before = fstatSync(staleFd);
+            hooks.beforeStaleSnapshot?.();
+            const observed = readFileSync(staleFd);
+            const [pidText, timeText] = observed.toString("utf8").trim().split(" ");
+            const pid = Number(pidText);
+            const age = Date.now() - Number(timeText);
+            let dead = false;
+            if (Number.isSafeInteger(pid) && pid > 0) {
+                try {
+                    process.kill(pid, 0);
+                }
+                catch (err) {
+                    dead = err.code === "ESRCH";
+                }
+            }
+            if (dead || (Number.isFinite(age) && age > LEASE_MS) || Date.now() - before.mtimeMs > LEASE_MS) {
+                hooks.beforeStaleUnlink?.();
+                const current = statSync(lock);
+                if (current.ino === before.ino && current.dev === before.dev && readFileSync(lock).equals(observed)) {
+                    unlinkSync(lock);
+                    continue;
+                }
+            }
+        }
+        catch { }
+        finally {
+            if (staleFd !== undefined)
+                try {
+                    closeSync(staleFd);
+                }
+                catch { }
+        }
+        if (Date.now() >= deadline)
+            break;
+        Atomics.wait(pause, 0, 0, Math.min(10, deadline - Date.now()));
+    } while (Date.now() <= deadline);
+    return undefined;
+}