@tea-agent/loop-agent 0.25.1 → 0.25.3

package/dist/commands/client-recovery.js

@@ -0,0 +1,657 @@
+import { mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+export const CLIENT_RECOVERY_MODES = ["auto", "project", "user", "off"];
+export const OPENCODE_TRANSIENT_RETRY_PLUGIN_PATH = ".opencode/plugins/loop-agent-transient-retry.js";
+export const PERMANENT_ERROR_INTERACTION = "plugin-ignore-permanent-error";
+export const BACKOFF_MS = [2000, 4000, 8000, 16000, 30000];
+export const MAX_SESSION_RETRIES = 5;
+export const PI_RECOMMENDED_RETRY = {
+    enabled: true,
+    maxRetries: 5,
+    baseDelayMs: 3000,
+    provider: {
+        maxRetries: 0,
+        maxRetryDelayMs: 60000,
+    },
+};
+const PERMANENT_MESSAGE_PATTERNS = [
+    { reason: "auth", pattern: /\b(401|unauthorized|invalid api key|authentication)\b/i },
+    { reason: "permission", pattern: /\b(403|forbidden|permission denied|not allowed)\b/i },
+    { reason: "quota", pattern: /\b(quota|rate.?limit|billing|insufficient.?credit)\b/i },
+    {
+        reason: "context-overflow",
+        pattern: /\b(context (length )?overflow|too many tokens|maximum context|context window)\b/i,
+    },
+    { reason: "cancelled", pattern: /\b(cancelled|canceled|aborted by user|user cancel)\b/i },
+    {
+        reason: "business-validation",
+        pattern: /\b(business validation|invalid task|schema validation)\b/i,
+    },
+];
+const TRANSIENT_MESSAGE_PATTERNS = [
+    /\bcode[:\s]*502\b/i,
+    /\b502\b/,
+    /\bLLMRequestError\b/i,
+    /\bTypeValidationError\b/i,
+    /\bnetwork fluctuation\b/i,
+    /\btimeout\b/i,
+    /\btransient provider failure\b/i,
+    /\bnon-standard (payload|response)\b/i,
+    /\bupstream returned\b/i,
+    /\bmodel processing timeout\b/i,
+];
+export function isClientRecoveryMode(value) {
+    return (typeof value === "string" &&
+        CLIENT_RECOVERY_MODES.includes(value));
+}
+export function parseClientRecoveryMode(value) {
+    if (value === undefined || value === null || value === "")
+        return "auto";
+    if (isClientRecoveryMode(value))
+        return value;
+    throw new Error(`init --client-recovery must be one of ${CLIENT_RECOVERY_MODES.join("|")}`);
+}
+function asText(value) {
+    if (typeof value === "string")
+        return value;
+    if (value == null)
+        return "";
+    try {
+        return JSON.stringify(value);
+    }
+    catch {
+        return String(value);
+    }
+}
+function collectErrorText(error) {
+    if (typeof error === "string")
+        return error;
+    if (!error || typeof error !== "object")
+        return asText(error);
+    const record = error;
+    const parts = [
+        asText(record.name),
+        asText(record.message),
+        asText(record.status),
+        asText(record.code),
+        asText(record.data),
+    ];
+    return parts.filter(Boolean).join(" ");
+}
+function hasUnknownErrorShape(error) {
+    const text = collectErrorText(error);
+    if (/\bUnknownError\b/i.test(text))
+        return true;
+    if (error && typeof error === "object") {
+        const record = error;
+        if (typeof record.name === "string" && /UnknownError/i.test(record.name))
+            return true;
+    }
+    return false;
+}
+function isStandardApiError(error) {
+    if (!error || typeof error !== "object")
+        return false;
+    const record = error;
+    if (typeof record.name === "string" && /^APIError$/i.test(record.name))
+        return true;
+    if (typeof record.name === "string" && /^(AuthError|PermissionError)$/i.test(record.name)) {
+        return true;
+    }
+    return false;
+}
+function permanentReason(error) {
+    if (isStandardApiError(error)) {
+        const record = error;
+        if (typeof record.name === "string" && /AuthError/i.test(record.name))
+            return "auth";
+        if (typeof record.name === "string" && /PermissionError/i.test(record.name)) {
+            return "permission";
+        }
+        return "api-error";
+    }
+    const text = collectErrorText(error);
+    for (const entry of PERMANENT_MESSAGE_PATTERNS) {
+        if (entry.pattern.test(text))
+            return entry.reason;
+    }
+    return undefined;
+}
+function looksTransient(error) {
+    const text = collectErrorText(error);
+    if (TRANSIENT_MESSAGE_PATTERNS.some((pattern) => pattern.test(text)))
+        return true;
+    if (error && typeof error === "object") {
+        const record = error;
+        if (record.code === 502 || record.status === 502)
+            return true;
+        if (record.data && typeof record.data === "object") {
+            const data = record.data;
+            if (data.code === 502 || data.status === 502)
+                return true;
+        }
+    }
+    return false;
+}
+/**
+ * Decide whether an OpenCode error should be resumed by the project plugin.
+ * Only compensates transient UnknownError paths that built-in APIError retry misses.
+ */
+export function classifyTransientUnknownError(error) {
+    const permanent = permanentReason(error);
+    if (permanent) {
+        return {
+            resumable: false,
+            reason: permanent,
+            interaction: PERMANENT_ERROR_INTERACTION,
+        };
+    }
+    if (!hasUnknownErrorShape(error)) {
+        return {
+            resumable: false,
+            reason: "not-unknown-error",
+            interaction: PERMANENT_ERROR_INTERACTION,
+        };
+    }
+    if (!looksTransient(error)) {
+        return {
+            resumable: false,
+            reason: "unknown-non-transient",
+            interaction: PERMANENT_ERROR_INTERACTION,
+        };
+    }
+    return { resumable: true, reason: "transient-unknown-error" };
+}
+export function nextBackoffMs(attempt) {
+    if (!Number.isInteger(attempt) || attempt < 0 || attempt >= BACKOFF_MS.length) {
+        return undefined;
+    }
+    return BACKOFF_MS[attempt];
+}
+export function canResumeSession(input) {
+    if (input.sessionStatus === "retry") {
+        return { allowed: false, reason: "builtin-retry-active" };
+    }
+    if (input.locked) {
+        return { allowed: false, reason: "session-locked" };
+    }
+    if (!input.idle || input.sessionStatus === "busy" || input.sessionStatus === "running") {
+        return { allowed: false, reason: "session-busy" };
+    }
+    if (input.attempt >= MAX_SESSION_RETRIES) {
+        return { allowed: false, reason: "max-retries" };
+    }
+    return { allowed: true, reason: "ready" };
+}
+export function buildResumePrompt(input) {
+    return [
+        `[loop-agent transient recovery] session=${input.sessionId} attempt=${input.attempt}`,
+        `Previous model turn failed with a transient UnknownError: ${input.errorSummary}`,
+        "Before continuing any work, first inspect already completed tool calls and existing file modifications in this session.",
+        "Do not re-run side-effecting tools or rewrite files that already reflect successful prior work.",
+        "Resume only the remaining unfinished work after that check.",
+        "续接前请先检查本 session 已有工具调用与文件改动，避免重复执行有副作用的操作。",
+    ].join("\n");
+}
+export function createSessionRecoveryTracker() {
+    const states = new Map();
+    function getOrCreate(sessionId) {
+        let state = states.get(sessionId);
+        if (!state) {
+            state = { attempt: 0, locked: false };
+            states.set(sessionId, state);
+        }
+        return state;
+    }
+    return {
+        getOrCreate,
+        isLocked(sessionId) {
+            return getOrCreate(sessionId).locked;
+        },
+        tryAcquire(sessionId) {
+            const state = getOrCreate(sessionId);
+            if (state.locked)
+                return false;
+            state.locked = true;
+            return true;
+        },
+        release(sessionId) {
+            const state = getOrCreate(sessionId);
+            state.locked = false;
+        },
+        markSuccess(sessionId) {
+            const state = getOrCreate(sessionId);
+            state.attempt = 0;
+            state.locked = false;
+        },
+        incrementAttempt(sessionId) {
+            const state = getOrCreate(sessionId);
+            state.attempt += 1;
+            return state.attempt;
+        },
+    };
+}
+function isRecord(value) {
+    return Boolean(value) && typeof value === "object" && !Array.isArray(value);
+}
+function readRetryEnabled(existing) {
+    if (!existing || !isRecord(existing.retry))
+        return undefined;
+    if (typeof existing.retry.enabled !== "boolean")
+        return undefined;
+    return existing.retry.enabled;
+}
+/**
+ * Field-level plan for ~/.pi/agent/settings.json retry recommendations.
+ * Never clobbers existing values; never overrides enabled:false.
+ */
+export function planPiRetryMerge(existing) {
+    const settingsPath = "~/.pi/agent/settings.json";
+    if (existing === undefined) {
+        return {
+            action: "write",
+            reason: "missing-file",
+            path: settingsPath,
+            next: { retry: { ...PI_RECOMMENDED_RETRY, provider: { ...PI_RECOMMENDED_RETRY.provider } } },
+        };
+    }
+    if (!isRecord(existing)) {
+        return {
+            action: "report",
+            reason: "invalid-json",
+            path: settingsPath,
+            message: "Pi settings root must be a JSON object",
+        };
+    }
+    const enabled = readRetryEnabled(existing);
+    if (enabled === false) {
+        return {
+            action: "report",
+            reason: "enabled-false",
+            path: settingsPath,
+            next: existing,
+            message: "retry.enabled=false is preserved; human decision required to enable recovery",
+        };
+    }
+    const retry = isRecord(existing.retry) ? { ...existing.retry } : {};
+    const provider = isRecord(retry.provider) ? { ...retry.provider } : {};
+    let changed = !isRecord(existing.retry) || !isRecord(existing.retry.provider);
+    if (retry.enabled === undefined) {
+        retry.enabled = PI_RECOMMENDED_RETRY.enabled;
+        changed = true;
+    }
+    if (retry.maxRetries === undefined) {
+        retry.maxRetries = PI_RECOMMENDED_RETRY.maxRetries;
+        changed = true;
+    }
+    if (retry.baseDelayMs === undefined) {
+        retry.baseDelayMs = PI_RECOMMENDED_RETRY.baseDelayMs;
+        changed = true;
+    }
+    if (provider.maxRetries === undefined) {
+        provider.maxRetries = PI_RECOMMENDED_RETRY.provider.maxRetries;
+        changed = true;
+    }
+    if (provider.maxRetryDelayMs === undefined) {
+        provider.maxRetryDelayMs = PI_RECOMMENDED_RETRY.provider.maxRetryDelayMs;
+        changed = true;
+    }
+    retry.provider = provider;
+    const next = { ...existing, retry };
+    if (!changed) {
+        return {
+            action: "noop",
+            reason: "matches",
+            path: settingsPath,
+            next,
+        };
+    }
+    return {
+        action: "write",
+        reason: "missing-fields",
+        path: settingsPath,
+        next,
+    };
+}
+export function piSettingsPath(homeDir) {
+    return path.join(homeDir, ".pi", "agent", "settings.json");
+}
+async function writeJsonAtomicHome(filePath, value) {
+    await mkdir(path.dirname(filePath), { recursive: true });
+    const tempPath = path.join(path.dirname(filePath), `.${path.basename(filePath)}.${process.pid}.${Date.now()}.tmp`);
+    try {
+        await writeFile(tempPath, `${JSON.stringify(value, null, 2)}\n`, "utf-8");
+        await rename(tempPath, filePath);
+    }
+    catch (error) {
+        await rm(tempPath, { force: true }).catch(() => undefined);
+        throw error;
+    }
+}
+function isMissingFileError(error) {
+    return (error instanceof Error &&
+        "code" in error &&
+        error.code === "ENOENT");
+}
+export async function applyPiRetryMerge(input) {
+    const settingsPath = piSettingsPath(input.homeDir);
+    let raw;
+    try {
+        raw = await readFile(settingsPath, "utf-8");
+    }
+    catch (error) {
+        if (!isMissingFileError(error))
+            throw error;
+        raw = undefined;
+    }
+    if (raw === undefined) {
+        const plan = planPiRetryMerge(undefined);
+        if (plan.action === "write" && plan.next) {
+            await writeJsonAtomicHome(settingsPath, plan.next);
+            return { ...plan, path: settingsPath, wrote: true };
+        }
+        return { ...plan, path: settingsPath, wrote: false };
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        return {
+            action: "report",
+            reason: "invalid-json",
+            path: settingsPath,
+            wrote: false,
+            message: "Pi settings.json is not valid JSON; refusing to write",
+        };
+    }
+    if (!isRecord(parsed)) {
+        return {
+            action: "report",
+            reason: "invalid-json",
+            path: settingsPath,
+            wrote: false,
+            message: "Pi settings.json root must be a JSON object; refusing to write",
+        };
+    }
+    const plan = planPiRetryMerge(parsed);
+    if (plan.action === "write" && plan.next) {
+        await writeJsonAtomicHome(settingsPath, plan.next);
+        // Best-effort cleanup of any stray temp files from interrupted prior runs.
+        try {
+            const dir = path.dirname(settingsPath);
+            const entries = await readdir(dir);
+            await Promise.all(entries
+                .filter((name) => name.startsWith(`.${path.basename(settingsPath)}.`) && name.endsWith(".tmp"))
+                .map((name) => rm(path.join(dir, name), { force: true })));
+        }
+        catch {
+            // ignore cleanup failures
+        }
+        return { ...plan, path: settingsPath, wrote: true };
+    }
+    return { ...plan, path: settingsPath, wrote: false };
+}
+/**
+ * Stable generated source for the OpenCode project plugin.
+ * The plugin compensates only transient UnknownError after built-in retry is idle.
+ */
+export function buildOpenCodeTransientRetryPluginSource() {
+    // Keep this template deterministic: no timestamps, no random ids.
+    return `/**
+ * loop-agent OpenCode transient recovery plugin
+ * Path: ${OPENCODE_TRANSIENT_RETRY_PLUGIN_PATH}
+ *
+ * Compensates transient UnknownError cases that OpenCode built-in APIError
+ * retry does not cover. Never runs concurrently with session.status === "retry".
+ * Permanent failures use interaction "${PERMANENT_ERROR_INTERACTION}".
+ */
+const PLUGIN_PATH = ${JSON.stringify(OPENCODE_TRANSIENT_RETRY_PLUGIN_PATH)};
+const PERMANENT_INTERACTION = ${JSON.stringify(PERMANENT_ERROR_INTERACTION)};
+const BACKOFF_MS = ${JSON.stringify([...BACKOFF_MS])};
+const MAX_RETRIES = ${MAX_SESSION_RETRIES};
+
+const sessionState = new Map();
+const sessionStatus = new Map();
+
+function getState(sessionId) {
+  let state = sessionState.get(sessionId);
+  if (!state) {
+    state = { attempt: 0, locked: false };
+    sessionState.set(sessionId, state);
+  }
+  return state;
+}
+
+function textOf(error) {
+  if (!error) return "";
+  if (typeof error === "string") return error;
+  try {
+    return JSON.stringify(error);
+  } catch {
+    return String(error);
+  }
+}
+
+function isPermanent(error) {
+  const text = textOf(error);
+  if (error && typeof error === "object" && typeof error.name === "string") {
+    if (/^APIError$/i.test(error.name)) return true;
+    if (/^(AuthError|PermissionError)$/i.test(error.name)) return true;
+  }
+  return /\\b(401|403|unauthorized|forbidden|quota|rate.?limit|context (length )?overflow|too many tokens|cancelled|canceled|business validation|invalid task|schema validation)\\b/i.test(
+    text,
+  );
+}
+
+function isTransientUnknown(error) {
+  if (isPermanent(error)) return false;
+  const text = textOf(error);
+  const hasUnknown = /UnknownError/i.test(text) || (error && /UnknownError/i.test(String(error.name || "")));
+  if (!hasUnknown) return false;
+  return (
+    /\\b(code[:\\s]*502|502|LLMRequestError|TypeValidationError|Type validation failed|network fluctuation|timeout|transient provider failure|non-standard)\\b/i.test(
+      text,
+    ) ||
+    (error && (error.code === 502 || error.status === 502))
+  );
+}
+
+function buildResumePrompt(sessionId, error, attempt) {
+  return [
+    "[loop-agent transient recovery] session=" + sessionId + " attempt=" + attempt,
+    "Previous model turn failed with a transient UnknownError: " + textOf(error),
+    "Before continuing any work, first inspect already completed tool calls and existing file modifications in this session.",
+    "Do not re-run side-effecting tools or rewrite files that already reflect successful prior work.",
+    "Resume only the remaining unfinished work after that check.",
+    "续接前请先检查本 session 已有工具调用与文件改动，避免重复执行有副作用的操作。",
+  ].join("\\n");
+}
+
+async function wait(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function statusType(status) {
+  return status && typeof status.type === "string" ? status.type : undefined;
+}
+
+async function readSessionStatus(client, sessionId) {
+  const cached = sessionStatus.get(sessionId);
+  if (cached === "retry") return cached;
+  try {
+    const response = await client.session.status();
+    const current = statusType(response?.data?.[sessionId]);
+    if (current) sessionStatus.set(sessionId, current);
+    return current || cached || "idle";
+  } catch {
+    return cached || "idle";
+  }
+}
+
+export default async function loopAgentTransientRetryPlugin({ client, $ }) {
+  void $;
+  void PLUGIN_PATH;
+
+  async function maybeResume(sessionId, error) {
+    if (!sessionId) return { interaction: PERMANENT_INTERACTION, reason: "missing-session" };
+    if (!isTransientUnknown(error)) {
+      return { interaction: PERMANENT_INTERACTION, reason: "plugin-ignore-permanent-error" };
+    }
+
+    const state = getState(sessionId);
+    if (state.locked) {
+      return { interaction: PERMANENT_INTERACTION, reason: "session-locked" };
+    }
+    if (state.attempt >= MAX_RETRIES) {
+      return { interaction: PERMANENT_INTERACTION, reason: "max-retries" };
+    }
+
+    state.locked = true;
+    try {
+      let status = await readSessionStatus(client, sessionId);
+      // Never race OpenCode built-in retry.
+      if (status === "retry") {
+        return { interaction: PERMANENT_INTERACTION, reason: "builtin-retry-active" };
+      }
+      if (status === "busy" || status === "running") {
+        return { interaction: PERMANENT_INTERACTION, reason: "session-busy" };
+      }
+
+      const delay = BACKOFF_MS[state.attempt];
+      const nextAttempt = state.attempt + 1;
+      if (typeof delay === "number") await wait(delay);
+      // Re-check idle after backoff.
+      status = await readSessionStatus(client, sessionId);
+      if (status === "retry" || status === "busy" || status === "running") {
+        return { interaction: PERMANENT_INTERACTION, reason: "session-not-idle" };
+      }
+      const prompt = buildResumePrompt(sessionId, error, nextAttempt);
+      state.attempt = nextAttempt;
+      await client.session.promptAsync({
+        path: { id: sessionId },
+        body: { parts: [{ type: "text", text: prompt }] },
+      });
+      return { resumed: true, attempt: state.attempt };
+    } finally {
+      state.locked = false;
+    }
+  }
+
+  return {
+    event: async ({ event }) => {
+      if (!event || typeof event !== "object") return;
+      const properties = event.properties || {};
+      const sessionId =
+        properties.sessionID || properties.sessionId || properties.id;
+
+      if (event.type === "session.status") {
+        const current = statusType(properties.status);
+        if (sessionId && current) sessionStatus.set(sessionId, current);
+        return;
+      }
+
+      if (event.type === "message.updated") {
+        const info = properties.info;
+        if (
+          info?.role === "assistant" &&
+          info.sessionID &&
+          info.time?.completed != null &&
+          info.error == null
+        ) {
+          getState(info.sessionID).attempt = 0;
+        }
+        return;
+      }
+
+      if (event.type === "session.error") {
+        return maybeResume(sessionId, properties.error || properties);
+      }
+    },
+  };
+}
+`;
+}
+export async function installProjectOpenCodePlugin(input) {
+    const relativePath = OPENCODE_TRANSIENT_RETRY_PLUGIN_PATH;
+    const target = path.join(input.repoRoot, relativePath);
+    const content = buildOpenCodeTransientRetryPluginSource();
+    if (input.onlyIfMissing) {
+        try {
+            await readFile(target, "utf-8");
+            return { path: relativePath, written: false, reason: "unchanged" };
+        }
+        catch {
+            // missing → write
+        }
+    }
+    await mkdir(path.dirname(target), { recursive: true });
+    await writeFile(target, content, "utf-8");
+    return { path: relativePath, written: true, reason: "written" };
+}
+export async function inspectPiRetryConfig(input) {
+    const homeDir = input.homeDir ?? os.homedir();
+    const settingsPath = piSettingsPath(homeDir);
+    let raw;
+    try {
+        raw = await readFile(settingsPath, "utf-8");
+    }
+    catch (error) {
+        if (!isMissingFileError(error))
+            throw error;
+        return {
+            action: "report",
+            reason: "missing-file",
+            path: settingsPath,
+            wrote: false,
+        };
+    }
+    try {
+        const parsed = JSON.parse(raw);
+        if (!isRecord(parsed)) {
+            return {
+                action: "report",
+                reason: "invalid-json",
+                path: settingsPath,
+                wrote: false,
+            };
+        }
+        const plan = planPiRetryMerge(parsed);
+        return { ...plan, path: settingsPath, wrote: false };
+    }
+    catch {
+        return {
+            action: "report",
+            reason: "invalid-json",
+            path: settingsPath,
+            wrote: false,
+        };
+    }
+}
+/**
+ * Install project OpenCode plugin and optionally merge Pi user settings.
+ * - auto/project: project plugin only (no home writes)
+ * - user: project plugin + explicit Pi merge
+ * - off: skip all
+ */
+export async function runClientRecovery(input) {
+    const mode = input.mode ?? "auto";
+    if (mode === "off") {
+        return {
+            mode,
+            plugin: {
+                path: OPENCODE_TRANSIENT_RETRY_PLUGIN_PATH,
+                written: false,
+                reason: "skipped",
+            },
+        };
+    }
+    const plugin = await installProjectOpenCodePlugin({ repoRoot: input.repoRoot });
+    if (mode !== "user") {
+        return { mode, plugin };
+    }
+    const homeDir = input.homeDir ?? os.homedir();
+    const pi = await applyPiRetryMerge({ homeDir });
+    return { mode, plugin, pi };
+}