@drakon-systems/multi-clawd 2.3.0 → 2.5.0

package/dist/shim-core.js

@@ -192,9 +192,9 @@ export function limitFamilyFor(displayName, modelId) {
     return displayName.trim().toLowerCase() === family ? family : undefined;
 }
 const LIMIT_TEXT_RE = /reached your (.{1,40}?) limit/i;
-export function parseModelLimitError(line) {
-    if (!/reached your /i.test(line))
-        return undefined;
+const USAGE_LIMIT_API_ERROR = "usage_limit_reached";
+const AUTH_FAILED_ERROR_KIND = "authentication_failed";
+function parseRecord(line) {
     let record;
     try {
         record = JSON.parse(line);
@@ -204,12 +204,15 @@ export function parseModelLimitError(line) {
     }
     if (typeof record !== "object" || record === null)
         return undefined;
-    const r = record;
-    const isErrorRecord = r.type === "error" ||
+    return record;
+}
+function isErrorRecord(r) {
+    return (r.type === "error" ||
         r.is_error === true ||
-        (typeof r.subtype === "string" && r.subtype.startsWith("error"));
-    if (!isErrorRecord)
-        return undefined;
+        (typeof r.subtype === "string" && r.subtype.startsWith("error")) ||
+        (r.type === "assistant" && r.is_api_error_message === true));
+}
+function errorRecordTexts(r) {
     const texts = [];
     if (typeof r.result === "string")
         texts.push(r.result);
@@ -220,13 +223,193 @@ export function parseModelLimitError(line) {
         if (typeof msg === "string")
             texts.push(msg);
     }
-    for (const text of texts) {
+    if (r.type === "assistant" && r.is_api_error_message === true && Array.isArray(r.message?.content)) {
+        for (const block of r.message.content) {
+            if (block && block.type === "text" && typeof block.text === "string")
+                texts.push(block.text);
+        }
+    }
+    return texts;
+}
+const TRANSIENT_TEXT_PATTERNS = [
+    { re: /repeated 529 overloaded errors/i, kind: "overloaded" },
+    { re: /overloaded_error/i, kind: "overloaded" },
+    { re: /the api is at capacity/i, kind: "overloaded" },
+    { re: /server-side issue, usually temporary/i, kind: "server_error" },
+    { re: /temporarily limiting requests \(not your usage limit\)/i, kind: "rate_limit_transient" },
+    { re: /request rejected \(429\)/i, kind: "rate_limit_transient" },
+];
+const TRANSIENT_TEXT_MAX_CHARS = 160;
+export function parseTransientApiError(line) {
+    if (!/("error"|is_error|api_error_status|overloaded|server-side|temporarily limiting|request rejected|at capacity)/i.test(line)) {
+        return undefined;
+    }
+    const r = parseRecord(line);
+    if (!r || !isErrorRecord(r))
+        return undefined;
+    if (parseModelLimitError(line) || parseAuthFailure(line))
+        return undefined;
+    const joined = errorRecordTexts(r).join("\n");
+    const status = typeof r.api_error_status === "number" ? r.api_error_status : undefined;
+    const text = joined.trim().slice(0, TRANSIENT_TEXT_MAX_CHARS);
+    const kindFromCli = typeof r.error === "string" ? r.error : undefined;
+    if (status === 529 || kindFromCli === "overloaded")
+        return { kind: "overloaded", status, text };
+    if ((status !== undefined && status >= 500) || kindFromCli === "server_error") {
+        return { kind: "server_error", status, text };
+    }
+    for (const { re, kind } of TRANSIENT_TEXT_PATTERNS) {
+        if (re.test(joined))
+            return { kind, status, text };
+    }
+    if (status === 429)
+        return { kind: "rate_limit_transient", status, text };
+    return undefined;
+}
+export function parseApiRetryNotice(line) {
+    if (!line.includes('"api_retry"'))
+        return undefined;
+    let record;
+    try {
+        record = JSON.parse(line);
+    }
+    catch {
+        return undefined;
+    }
+    if (typeof record !== "object" || record === null)
+        return undefined;
+    const r = record;
+    if (r.type !== "system" || r.subtype !== "api_retry")
+        return undefined;
+    const delay = typeof r.retry_delay_ms === "number" && r.retry_delay_ms >= 0 ? r.retry_delay_ms : 0;
+    return { retryDelayMs: delay };
+}
+export function parseModelLimitError(line) {
+    if (!/(reached your |usage_limit_reached)/i.test(line))
+        return undefined;
+    const r = parseRecord(line);
+    if (!r || !isErrorRecord(r))
+        return undefined;
+    for (const text of errorRecordTexts(r)) {
         const match = LIMIT_TEXT_RE.exec(text);
         if (match)
             return { displayName: match[1].trim() };
     }
+    if (r.api_error === USAGE_LIMIT_API_ERROR)
+        return { displayName: USAGE_LIMIT_API_ERROR };
     return undefined;
 }
+export const LIMIT_PASSTHROUGH_MARKER = " (rate limit: 429)";
+const HOST_DIAGNOSTIC_ERROR_PREFIX = "[ede_diagnostic]";
+export function markForwardedLimitLine(line) {
+    if (!parseModelLimitError(line))
+        return line;
+    let record;
+    try {
+        record = JSON.parse(line);
+    }
+    catch {
+        return line;
+    }
+    if (typeof record !== "object" || record === null || Array.isArray(record))
+        return line;
+    const r = record;
+    const subtype = typeof r.subtype === "string" ? r.subtype : "";
+    if (r.type !== "result" || !(r.is_error === true || subtype.startsWith("error")))
+        return line;
+    const mark = (text) => text.includes(LIMIT_PASSTHROUGH_MARKER) ? undefined : `${text.trimEnd()}${LIMIT_PASSTHROUGH_MARKER}`;
+    if (Array.isArray(r.errors)) {
+        const idx = r.errors.findIndex((e) => typeof e === "string" && e.trim() !== "" && !e.trim().startsWith(HOST_DIAGNOSTIC_ERROR_PREFIX));
+        if (idx >= 0) {
+            const marked = mark(r.errors[idx]);
+            if (marked === undefined)
+                return line;
+            const errors = [...r.errors];
+            errors[idx] = marked;
+            return JSON.stringify({ ...r, errors });
+        }
+    }
+    if (typeof r.result === "string" && r.result.trim() !== "") {
+        const marked = mark(r.result);
+        return marked === undefined ? line : JSON.stringify({ ...r, result: marked });
+    }
+    if (typeof r.error === "object" && r.error !== null && !Array.isArray(r.error)) {
+        const err = r.error;
+        if (typeof err.message === "string" && err.message.trim() !== "") {
+            const marked = mark(err.message);
+            return marked === undefined ? line : JSON.stringify({ ...r, error: { ...err, message: marked } });
+        }
+    }
+    if (typeof r.error === "string" && r.error.trim() !== "") {
+        const marked = mark(r.error);
+        return marked === undefined ? line : JSON.stringify({ ...r, error: marked });
+    }
+    return line;
+}
+const RESULT_LINE_PREFIX = Buffer.from('{"type":"result"', "utf8");
+const FORWARD_HOLD_CAP_BYTES = 256 * 1024;
+function couldBeResultLine(pending) {
+    let i = 0;
+    while (i < pending.length && (pending[i] === 0x20 || pending[i] === 0x09 || pending[i] === 0x0d))
+        i++;
+    const rest = pending.subarray(i);
+    const n = Math.min(rest.length, RESULT_LINE_PREFIX.length);
+    return rest.subarray(0, n).equals(RESULT_LINE_PREFIX.subarray(0, n));
+}
+function forwardCompleteLine(line, onMark) {
+    const text = line.toString("utf8");
+    const ending = text.endsWith("\r\n") ? "\r\n" : text.endsWith("\n") ? "\n" : "";
+    const body = text.slice(0, text.length - ending.length).trim();
+    const marked = markForwardedLimitLine(body);
+    if (marked === body)
+        return line;
+    onMark?.();
+    return Buffer.from(marked + ending, "utf8");
+}
+export function createForwardWriter(write, onMark) {
+    let pending;
+    let streaming = false;
+    return {
+        push(chunk) {
+            const out = [];
+            let start = 0;
+            while (start < chunk.length) {
+                const nl = chunk.indexOf(0x0a, start);
+                const end = nl < 0 ? chunk.length : nl + 1;
+                const piece = chunk.subarray(start, end);
+                start = end;
+                if (streaming) {
+                    out.push(piece);
+                    if (nl >= 0)
+                        streaming = false;
+                    continue;
+                }
+                const line = pending ? Buffer.concat([pending, piece]) : piece;
+                pending = undefined;
+                if (nl >= 0) {
+                    out.push(forwardCompleteLine(line, onMark));
+                }
+                else if (!couldBeResultLine(line) || line.length > FORWARD_HOLD_CAP_BYTES) {
+                    out.push(line);
+                    streaming = true;
+                }
+                else {
+                    pending = Buffer.from(line);
+                }
+            }
+            if (out.length > 0)
+                write(out.length === 1 ? out[0] : Buffer.concat(out));
+        },
+        flush() {
+            if (pending) {
+                const line = pending;
+                pending = undefined;
+                write(forwardCompleteLine(line, onMark));
+            }
+            streaming = false;
+        },
+    };
+}
 export function recordModelLimit(state, modelId, now, resetsAt, family) {
     return {
         ...state,
@@ -300,36 +483,16 @@ export function parseAuthFailure(line) {
     if (!/(authenticat|oauth|logged in|\/login|unauthori|api key|access token|bearer token)/i.test(line)) {
         return undefined;
     }
-    let record;
-    try {
-        record = JSON.parse(line);
-    }
-    catch {
-        return undefined;
-    }
-    if (typeof record !== "object" || record === null)
-        return undefined;
-    const r = record;
-    const isErrorRecord = r.type === "error" ||
-        r.is_error === true ||
-        (typeof r.subtype === "string" && r.subtype.startsWith("error"));
-    if (!isErrorRecord)
+    const r = parseRecord(line);
+    if (!r || !isErrorRecord(r))
         return undefined;
-    const texts = [];
-    if (typeof r.result === "string")
-        texts.push(r.result);
-    if (typeof r.error === "string")
-        texts.push(r.error);
-    if (typeof r.error === "object" && r.error !== null) {
-        const msg = r.error.message;
-        if (typeof msg === "string")
-            texts.push(msg);
-    }
-    for (const text of texts) {
+    for (const text of errorRecordTexts(r)) {
         if (AUTH_FAILURE_PATTERNS.some((p) => p.test(text))) {
             return { reason: text.trim().slice(0, AUTH_REASON_MAX_CHARS) };
         }
     }
+    if (r.error === AUTH_FAILED_ERROR_KIND)
+        return { reason: AUTH_FAILED_ERROR_KIND };
     return undefined;
 }
 export function recordCredentialFailure(state, reason, now) {

package/dist/shim-lock.js

@@ -0,0 +1,106 @@
+import { randomUUID } from "node:crypto";
+import { closeSync, fstatSync, mkdirSync, openSync, readFileSync, statSync, unlinkSync, writeFileSync } from "node:fs";
+import { dirname, isAbsolute } from "node:path";
+const WAIT_MS = 150;
+const LEASE_MS = 5_000;
+const pause = new Int32Array(new SharedArrayBuffer(4));
+export function acquireShimLock(file, waitMs = WAIT_MS, hooks = {}) {
+    if (!file || !isAbsolute(file))
+        return undefined;
+    const lock = `${file}.lock`;
+    const deadline = Date.now() + waitMs;
+    try {
+        mkdirSync(dirname(lock), { recursive: true });
+    }
+    catch {
+        return undefined;
+    }
+    do {
+        try {
+            const fd = openSync(lock, "wx", 0o600);
+            const token = randomUUID();
+            const content = Buffer.from(`${process.pid} ${Date.now()} ${token}\n`);
+            let ino;
+            try {
+                ino = fstatSync(fd).ino;
+                writeFileSync(fd, content);
+            }
+            catch {
+                try {
+                    if (ino !== undefined && statSync(lock).ino === ino) {
+                        const found = readFileSync(lock);
+                        if (found.includes(Buffer.from(token)) || found.length === 0)
+                            unlinkSync(lock);
+                    }
+                }
+                catch { }
+                try {
+                    closeSync(fd);
+                }
+                catch { }
+                return undefined;
+            }
+            const release = () => {
+                try {
+                    if (statSync(lock).ino === ino && readFileSync(lock).equals(content))
+                        unlinkSync(lock);
+                }
+                catch { }
+            };
+            try {
+                (hooks.close ?? closeSync)(fd);
+            }
+            catch {
+                try {
+                    closeSync(fd);
+                }
+                catch { }
+                return release;
+            }
+            return release;
+        }
+        catch (err) {
+            if (err.code !== "EEXIST")
+                return undefined;
+        }
+        let staleFd;
+        try {
+            staleFd = openSync(lock, "r");
+            const before = fstatSync(staleFd);
+            hooks.beforeStaleSnapshot?.();
+            const observed = readFileSync(staleFd);
+            const [pidText, timeText] = observed.toString("utf8").trim().split(" ");
+            const pid = Number(pidText);
+            const age = Date.now() - Number(timeText);
+            let dead = false;
+            if (Number.isSafeInteger(pid) && pid > 0) {
+                try {
+                    process.kill(pid, 0);
+                }
+                catch (err) {
+                    dead = err.code === "ESRCH";
+                }
+            }
+            if (dead || (Number.isFinite(age) && age > LEASE_MS) || Date.now() - before.mtimeMs > LEASE_MS) {
+                hooks.beforeStaleUnlink?.();
+                const current = statSync(lock);
+                if (current.ino === before.ino && current.dev === before.dev && readFileSync(lock).equals(observed)) {
+                    unlinkSync(lock);
+                    continue;
+                }
+            }
+        }
+        catch { }
+        finally {
+            if (staleFd !== undefined)
+                try {
+                    closeSync(staleFd);
+                }
+                catch { }
+        }
+        if (Date.now() >= deadline)
+            break;
+        Atomics.wait(pause, 0, 0, Math.min(10, deadline - Date.now()));
+    } while (Date.now() <= deadline);
+    return undefined;
+}