helloloop 0.3.1 → 0.7.0

package/src/process.mjs

@@ -1,58 +1,45 @@
 import fs from "node:fs";
 import path from "node:path";
-import { spawn } from "node:child_process";
 
-import { ensureDir, nowIso, tailText, writeText } from "./common.mjs";
-import { resolveCodexInvocation, resolveVerifyShellInvocation } from "./shell_invocation.mjs";
+import { ensureDir, nowIso, tailText, writeJson, writeText } from "./common.mjs";
+import { getEngineDisplayName, normalizeEngineName } from "./engine_metadata.mjs";
+import {
+  buildClaudeArgs,
+  buildCodexArgs,
+  buildGeminiArgs,
+  resolveEngineInvocation,
+  resolveVerifyInvocation,
+  runChild,
+} from "./engine_process_support.mjs";
+import { sendRuntimeStopNotification } from "./email_notification.mjs";
+import { loadGlobalConfig } from "./global_config.mjs";
+import {
+  buildEngineHealthProbePrompt,
+  buildRuntimeRecoveryPrompt,
+  classifyRuntimeRecoveryFailure,
+  renderRuntimeRecoverySummary,
+  resolveRuntimeRecoveryPolicy,
+  selectRuntimeRecoveryDelayMs,
+} from "./runtime_recovery.mjs";
 
-function runChild(command, args, options = {}) {
+function sleep(ms) {
   return new Promise((resolve) => {
-    const child = spawn(command, args, {
-      cwd: options.cwd,
-      env: {
-        ...process.env,
-        ...(options.env || {}),
-      },
-      stdio: ["pipe", "pipe", "pipe"],
-      shell: Boolean(options.shell),
-    });
-
-    let stdout = "";
-    let stderr = "";
-
-    child.stdout.on("data", (chunk) => {
-      stdout += chunk.toString();
-    });
-    child.stderr.on("data", (chunk) => {
-      stderr += chunk.toString();
-    });
-
-    if (options.stdin) {
-      child.stdin.write(options.stdin);
-    }
-    child.stdin.end();
-
-    child.on("error", (error) => {
-      resolve({
-        ok: false,
-        code: 1,
-        stdout,
-        stderr: String(error?.stack || error || ""),
-      });
-    });
+    setTimeout(resolve, ms);
+  });
+}
 
-    child.on("close", (code) => {
-      resolve({
-        ok: code === 0,
-        code: code ?? 1,
-        stdout,
-        stderr,
-      });
+function createRuntimeStatusWriter(runtimeStatusFile, baseState) {
+  return function writeRuntimeStatus(status, extra = {}) {
+    writeJson(runtimeStatusFile, {
+      ...baseState,
+      ...extra,
+      status,
+      updatedAt: nowIso(),
     });
-  });
+  };
 }
 
-function writeCodexRunArtifacts(runDir, prefix, result, finalMessage) {
+function writeEngineRunArtifacts(runDir, prefix, result, finalMessage) {
   writeText(path.join(runDir, `${prefix}-stdout.log`), result.stdout);
   writeText(path.join(runDir, `${prefix}-stderr.log`), result.stderr);
   writeText(path.join(runDir, `${prefix}-summary.txt`), [
@@ -64,48 +51,90 @@ function writeCodexRunArtifacts(runDir, prefix, result, finalMessage) {
   ].join("\n"));
 }
 
-export async function runCodexTask({
-  context,
-  prompt,
-  runDir,
-  model = "",
-  executable = "",
-  sandbox = "workspace-write",
-  dangerouslyBypassSandbox = false,
-  jsonOutput = true,
-  outputSchemaFile = "",
-  outputPrefix = "codex",
-  ephemeral = false,
-  skipGitRepoCheck = false,
-  env = {},
-}) {
-  ensureDir(runDir);
-
-  const lastMessageFile = path.join(runDir, `${outputPrefix}-last-message.txt`);
-  const invocation = resolveCodexInvocation({ explicitExecutable: executable });
-  const codexArgs = ["exec", "-C", context.repoRoot];
-
-  if (model) {
-    codexArgs.push("--model", model);
+function resolveEnginePolicy(policy = {}, engine) {
+  if (engine === "codex") {
+    return policy.codex || {};
   }
-  if (dangerouslyBypassSandbox) {
-    codexArgs.push("--dangerously-bypass-approvals-and-sandbox");
-  } else {
-    codexArgs.push("--sandbox", sandbox);
+  if (engine === "claude") {
+    return policy.claude || {};
   }
-  if (skipGitRepoCheck) {
-    codexArgs.push("--skip-git-repo-check");
+  if (engine === "gemini") {
+    return policy.gemini || {};
   }
-  if (ephemeral) {
-    codexArgs.push("--ephemeral");
+  return {};
+}
+
+function buildEngineArgs({
+  engine,
+  context,
+  resolvedPolicy,
+  executionMode,
+  outputSchemaFile,
+  ephemeral,
+  skipGitRepoCheck,
+  lastMessageFile,
+  probeMode = false,
+}) {
+  if (engine === "codex") {
+    return buildCodexArgs({
+      context,
+      model: resolvedPolicy.model,
+      sandbox: resolvedPolicy.sandbox,
+      dangerouslyBypassSandbox: resolvedPolicy.dangerouslyBypassSandbox,
+      jsonOutput: probeMode ? false : (resolvedPolicy.jsonOutput !== false),
+      outputSchemaFile: probeMode ? "" : outputSchemaFile,
+      ephemeral,
+      skipGitRepoCheck,
+      lastMessageFile,
+    });
   }
-  if (outputSchemaFile) {
-    codexArgs.push("--output-schema", outputSchemaFile);
+
+  if (engine === "claude") {
+    return buildClaudeArgs({
+      model: resolvedPolicy.model,
+      outputSchemaFile: probeMode ? "" : outputSchemaFile,
+      executionMode: probeMode ? "execute" : executionMode,
+      policy: resolvedPolicy,
+    });
   }
-  if (jsonOutput) {
-    codexArgs.push("--json");
+
+  return buildGeminiArgs({
+    model: resolvedPolicy.model,
+    executionMode: probeMode ? "execute" : executionMode,
+    policy: resolvedPolicy,
+  });
+}
+
+function readEngineFinalMessage(engine, lastMessageFile, result) {
+  if (engine === "codex") {
+    return fs.existsSync(lastMessageFile)
+      ? fs.readFileSync(lastMessageFile, "utf8").trim()
+      : "";
   }
-  codexArgs.push("-o", lastMessageFile, "-");
+  return String(result.stdout || "").trim();
+}
+
+async function runEngineAttempt({
+  engine,
+  invocation,
+  context,
+  prompt,
+  runDir,
+  attemptPrefix,
+  resolvedPolicy,
+  executionMode,
+  outputSchemaFile,
+  env,
+  recoveryPolicy,
+  writeRuntimeStatus,
+  recoveryCount,
+  recoveryHistory,
+  ephemeral = false,
+  skipGitRepoCheck = false,
+  probeMode = false,
+}) {
+  const attemptPromptFile = path.join(runDir, `${attemptPrefix}-prompt.md`);
+  const attemptLastMessageFile = path.join(runDir, `${attemptPrefix}-last-message.txt`);
 
   if (invocation.error) {
     const result = {
@@ -113,46 +142,484 @@ export async function runCodexTask({
       code: 1,
       stdout: "",
       stderr: invocation.error,
+      signal: "",
+      startedAt: nowIso(),
+      finishedAt: nowIso(),
+      idleTimeout: false,
+      watchdogTriggered: false,
+      watchdogReason: "",
+    };
+    writeText(attemptPromptFile, prompt);
+    writeEngineRunArtifacts(runDir, attemptPrefix, result, "");
+    return {
+      result,
+      finalMessage: "",
+      attemptPrefix,
     };
-    writeText(path.join(runDir, `${outputPrefix}-prompt.md`), prompt);
-    writeCodexRunArtifacts(runDir, outputPrefix, result, "");
-    return { ...result, finalMessage: "" };
   }
 
-  const args = [...invocation.argsPrefix, ...codexArgs];
+  const finalArgs = [
+    ...invocation.argsPrefix,
+    ...buildEngineArgs({
+      engine,
+      context,
+      resolvedPolicy,
+      executionMode,
+      outputSchemaFile,
+      ephemeral,
+      skipGitRepoCheck,
+      lastMessageFile: attemptLastMessageFile,
+      probeMode,
+    }),
+  ];
+
+  writeRuntimeStatus(probeMode ? "probe_running" : (recoveryCount > 0 ? "recovering" : "running"), {
+    attemptPrefix,
+    recoveryCount,
+    recoveryHistory,
+  });
 
-  const result = await runChild(invocation.command, args, {
+  const result = await runChild(invocation.command, finalArgs, {
     cwd: context.repoRoot,
     stdin: prompt,
     env,
     shell: invocation.shell,
+    heartbeatIntervalMs: recoveryPolicy.heartbeatIntervalSeconds * 1000,
+    stallWarningMs: recoveryPolicy.stallWarningSeconds * 1000,
+    maxIdleMs: recoveryPolicy.maxIdleSeconds * 1000,
+    killGraceMs: recoveryPolicy.killGraceSeconds * 1000,
+    onHeartbeat(payload) {
+      writeRuntimeStatus(payload.status, {
+        attemptPrefix,
+        recoveryCount,
+        recoveryHistory,
+        heartbeat: payload,
+      });
+    },
+  });
+  const finalMessage = readEngineFinalMessage(engine, attemptLastMessageFile, result);
+
+  writeText(attemptPromptFile, prompt);
+  writeEngineRunArtifacts(runDir, attemptPrefix, result, finalMessage);
+
+  return {
+    result,
+    finalMessage,
+    attemptPrefix,
+  };
+}
+
+async function runEngineHealthProbe({
+  engine,
+  invocation,
+  context,
+  runDir,
+  resolvedPolicy,
+  recoveryPolicy,
+  writeRuntimeStatus,
+  recoveryCount,
+  recoveryHistory,
+  env,
+  probeIndex,
+}) {
+  const probePrompt = buildEngineHealthProbePrompt(engine);
+  const attemptPrefix = `${engine}-probe-${String(probeIndex).padStart(2, "0")}`;
+  writeRuntimeStatus("probe_waiting", {
+    attemptPrefix,
+    recoveryCount,
+    recoveryHistory,
+  });
+  const attempt = await runEngineAttempt({
+    engine,
+    invocation,
+    context,
+    prompt: probePrompt,
+    runDir,
+    attemptPrefix,
+    resolvedPolicy,
+    executionMode: "execute",
+    outputSchemaFile: "",
+    env,
+    recoveryPolicy: {
+      ...recoveryPolicy,
+      maxIdleSeconds: recoveryPolicy.healthProbeTimeoutSeconds,
+    },
+    writeRuntimeStatus,
+    recoveryCount,
+    recoveryHistory,
+    ephemeral: true,
+    skipGitRepoCheck: true,
+    probeMode: true,
   });
-  const finalMessage = fs.existsSync(lastMessageFile)
-    ? fs.readFileSync(lastMessageFile, "utf8").trim()
-    : "";
 
-  writeText(path.join(runDir, `${outputPrefix}-prompt.md`), prompt);
-  writeCodexRunArtifacts(runDir, outputPrefix, result, finalMessage);
+  return {
+    ...attempt,
+    failure: classifyRuntimeRecoveryFailure({
+      result: {
+        ...attempt.result,
+        finalMessage: attempt.finalMessage,
+      },
+    }),
+  };
+}
 
-  return { ...result, finalMessage };
+async function maybeSendStopNotification({
+  context,
+  runDir,
+  engine,
+  executionMode,
+  failure,
+  result,
+  recoveryHistory,
+}) {
+  try {
+    return await sendRuntimeStopNotification({
+      globalConfig: loadGlobalConfig(),
+      context,
+      engine: getEngineDisplayName(engine),
+      phase: executionMode === "analyze" ? "分析/复核" : "执行",
+      failure,
+      result,
+      recoveryHistory,
+      runDir,
+    });
+  } catch (error) {
+    return {
+      attempted: true,
+      delivered: false,
+      reason: String(error?.message || error || "邮件发送失败。"),
+    };
+  }
+}
+
+function buildNotificationNote(notificationResult) {
+  if (!notificationResult) {
+    return "";
+  }
+  if (notificationResult.delivered) {
+    return `告警邮件已发送：${(notificationResult.recipients || []).join(", ")}`;
+  }
+  if (notificationResult.attempted) {
+    return `告警邮件发送失败：${notificationResult.reason || "未知原因"}`;
+  }
+  return `未发送告警邮件：${notificationResult.reason || "未启用"}`;
+}
+
+export async function runEngineTask({
+  engine = "codex",
+  context,
+  prompt,
+  runDir,
+  policy = {},
+  executionMode = "analyze",
+  outputSchemaFile = "",
+  outputPrefix = "",
+  ephemeral = false,
+  skipGitRepoCheck = false,
+  env = {},
+}) {
+  ensureDir(runDir);
+
+  const normalizedEngine = normalizeEngineName(engine) || "codex";
+  const resolvedPolicy = resolveEnginePolicy(policy, normalizedEngine);
+  const prefix = outputPrefix || normalizedEngine;
+  const invocation = resolveEngineInvocation(normalizedEngine, resolvedPolicy.executable);
+  const recoveryPolicy = resolveRuntimeRecoveryPolicy(policy);
+  const runtimeStatusFile = path.join(runDir, `${prefix}-runtime.json`);
+  const writeRuntimeStatus = createRuntimeStatusWriter(runtimeStatusFile, {
+    engine: normalizedEngine,
+    engineDisplayName: getEngineDisplayName(normalizedEngine),
+    phase: executionMode,
+    outputPrefix: prefix,
+    hardRetryBudget: recoveryPolicy.hardRetryDelaysSeconds.length,
+    softRetryBudget: recoveryPolicy.softRetryDelaysSeconds.length,
+  });
+
+  const recoveryHistory = [];
+  let currentPrompt = prompt;
+  let currentRecoveryCount = 0;
+  let activeFailure = null;
+
+  while (true) {
+    const attemptPrefix = currentRecoveryCount === 0
+      ? prefix
+      : `${prefix}-recovery-${String(currentRecoveryCount).padStart(2, "0")}`;
+    const taskAttempt = await runEngineAttempt({
+      engine: normalizedEngine,
+      invocation,
+      context,
+      prompt: currentPrompt,
+      runDir,
+      attemptPrefix,
+      resolvedPolicy,
+      executionMode,
+      outputSchemaFile,
+      env,
+      recoveryPolicy,
+      writeRuntimeStatus,
+      recoveryCount: currentRecoveryCount,
+      recoveryHistory,
+      ephemeral,
+      skipGitRepoCheck,
+      probeMode: false,
+    });
+
+    const taskFailure = classifyRuntimeRecoveryFailure({
+      result: {
+        ...taskAttempt.result,
+        finalMessage: taskAttempt.finalMessage,
+      },
+    });
+
+    if (taskAttempt.result.ok || !recoveryPolicy.enabled) {
+      const finalRecoverySummary = taskAttempt.result.ok
+        ? ""
+        : renderRuntimeRecoverySummary(recoveryHistory, taskFailure);
+      const notification = taskAttempt.result.ok
+        ? null
+        : await maybeSendStopNotification({
+          context,
+          runDir,
+          engine: normalizedEngine,
+          executionMode,
+          failure: taskFailure,
+          result: taskAttempt.result,
+          recoveryHistory,
+        });
+      const notificationNote = taskAttempt.result.ok ? "" : buildNotificationNote(notification);
+      const finalizedResult = taskAttempt.result.ok
+        ? taskAttempt.result
+        : {
+          ...taskAttempt.result,
+          stderr: [
+            taskAttempt.result.stderr,
+            "",
+            finalRecoverySummary,
+            notificationNote,
+          ].filter(Boolean).join("\n").trim(),
+        };
+
+      writeText(path.join(runDir, `${prefix}-prompt.md`), currentPrompt);
+      writeEngineRunArtifacts(runDir, prefix, finalizedResult, taskAttempt.finalMessage);
+      if (normalizedEngine === "codex" && taskAttempt.finalMessage) {
+        writeText(path.join(runDir, `${prefix}-last-message.txt`), taskAttempt.finalMessage);
+      }
+      writeRuntimeStatus(taskAttempt.result.ok ? "completed" : "paused_manual", {
+        attemptPrefix,
+        recoveryCount: recoveryHistory.length,
+        recoveryHistory,
+        recoverySummary: finalRecoverySummary,
+        finalMessage: taskAttempt.finalMessage,
+        code: finalizedResult.code,
+        failureCode: taskFailure.code,
+        failureFamily: taskFailure.family,
+        failureReason: taskFailure.reason,
+        notification,
+      });
+
+      return {
+        ...finalizedResult,
+        finalMessage: taskAttempt.finalMessage,
+        recoveryCount: recoveryHistory.length,
+        recoveryHistory,
+        recoverySummary: finalRecoverySummary,
+        recoveryFailure: taskAttempt.result.ok
+          ? null
+          : {
+            ...taskFailure,
+            shouldStopTask: true,
+            exhausted: true,
+          },
+        notification,
+      };
+    }
+
+    activeFailure = taskFailure;
+    while (true) {
+      const nextRecoveryIndex = recoveryHistory.length + 1;
+      const recoveryPrompt = buildRuntimeRecoveryPrompt({
+        basePrompt: prompt,
+        engine: normalizedEngine,
+        phaseLabel: executionMode === "analyze" ? "分析/复核" : "执行",
+        failure: activeFailure,
+        result: {
+          ...taskAttempt.result,
+          finalMessage: taskAttempt.finalMessage,
+        },
+        nextRecoveryIndex,
+        maxRecoveries: recoveryPolicy[activeFailure.family === "hard" ? "hardRetryDelaysSeconds" : "softRetryDelaysSeconds"].length,
+      });
+      writeText(
+        path.join(runDir, `${prefix}-auto-recovery-${String(nextRecoveryIndex).padStart(2, "0")}-prompt.md`),
+        recoveryPrompt,
+      );
+      const delayMs = selectRuntimeRecoveryDelayMs(recoveryPolicy, activeFailure.family, nextRecoveryIndex);
+      if (delayMs < 0) {
+        const finalRecoverySummary = renderRuntimeRecoverySummary(recoveryHistory, activeFailure);
+        const notification = await maybeSendStopNotification({
+          context,
+          runDir,
+          engine: normalizedEngine,
+          executionMode,
+          failure: activeFailure,
+          result: taskAttempt.result,
+          recoveryHistory,
+        });
+        const notificationNote = buildNotificationNote(notification);
+        const finalizedResult = {
+          ...taskAttempt.result,
+          stderr: [
+            taskAttempt.result.stderr,
+            "",
+            finalRecoverySummary,
+            notificationNote,
+          ].filter(Boolean).join("\n").trim(),
+        };
+
+        writeText(path.join(runDir, `${prefix}-prompt.md`), currentPrompt);
+        writeEngineRunArtifacts(runDir, prefix, finalizedResult, taskAttempt.finalMessage);
+        writeRuntimeStatus("paused_manual", {
+          attemptPrefix,
+          recoveryCount: recoveryHistory.length,
+          recoveryHistory,
+          recoverySummary: finalRecoverySummary,
+          finalMessage: taskAttempt.finalMessage,
+          code: finalizedResult.code,
+          failureCode: activeFailure.code,
+          failureFamily: activeFailure.family,
+          failureReason: activeFailure.reason,
+          notification,
+        });
+
+        return {
+          ...finalizedResult,
+          finalMessage: taskAttempt.finalMessage,
+          recoveryCount: recoveryHistory.length,
+          recoveryHistory,
+          recoverySummary: finalRecoverySummary,
+          recoveryFailure: {
+            ...activeFailure,
+            shouldStopTask: true,
+            exhausted: true,
+          },
+          notification,
+        };
+      }
+
+      writeRuntimeStatus("retry_waiting", {
+        attemptPrefix,
+        recoveryCount: nextRecoveryIndex,
+        recoveryHistory,
+        nextRetryDelayMs: delayMs,
+        nextRetryAt: new Date(Date.now() + delayMs).toISOString(),
+        failureCode: activeFailure.code,
+        failureFamily: activeFailure.family,
+        failureReason: activeFailure.reason,
+      });
+      if (delayMs > 0) {
+        await sleep(delayMs);
+      }
+
+      const probeAttempt = await runEngineHealthProbe({
+        engine: normalizedEngine,
+        invocation,
+        context,
+        runDir,
+        resolvedPolicy,
+        recoveryPolicy,
+        writeRuntimeStatus,
+        recoveryCount: nextRecoveryIndex,
+        recoveryHistory,
+        env,
+        probeIndex: nextRecoveryIndex,
+      });
+      const recoveryRecord = {
+        recoveryIndex: nextRecoveryIndex,
+        family: activeFailure.family,
+        code: activeFailure.code,
+        reason: activeFailure.reason,
+        delaySeconds: Math.floor(delayMs / 1000),
+        taskStatus: "failed",
+        taskCode: taskAttempt.result.code,
+        taskAttemptPrefix: attemptPrefix,
+        probeStatus: probeAttempt.result.ok ? "ok" : "failed",
+        probeCode: probeAttempt.result.code,
+        probeAttemptPrefix: probeAttempt.attemptPrefix,
+        probeFailureCode: probeAttempt.failure?.code || "",
+        probeFailureFamily: probeAttempt.failure?.family || "",
+        probeFailureReason: probeAttempt.failure?.reason || "",
+        watchdogTriggered: taskAttempt.result.watchdogTriggered === true || probeAttempt.result.watchdogTriggered === true,
+      };
+      recoveryHistory.push(recoveryRecord);
+      writeJson(path.join(
+        runDir,
+        `${prefix}-auto-recovery-${String(nextRecoveryIndex).padStart(2, "0")}.json`,
+      ), {
+        ...recoveryRecord,
+        engine: normalizedEngine,
+        phase: executionMode,
+        stdoutTail: tailText(taskAttempt.result.stdout, 20),
+        stderrTail: tailText(taskAttempt.result.stderr, 20),
+        finalMessageTail: tailText(taskAttempt.finalMessage, 20),
+        probeStdoutTail: tailText(probeAttempt.result.stdout, 20),
+        probeStderrTail: tailText(probeAttempt.result.stderr, 20),
+        probeFinalMessageTail: tailText(probeAttempt.finalMessage, 20),
+        createdAt: nowIso(),
+      });
+
+      if (!probeAttempt.result.ok) {
+        activeFailure = probeAttempt.failure;
+        writeRuntimeStatus("probe_failed", {
+          attemptPrefix: probeAttempt.attemptPrefix,
+          recoveryCount: nextRecoveryIndex,
+          recoveryHistory,
+          failureCode: activeFailure.code,
+          failureFamily: activeFailure.family,
+          failureReason: activeFailure.reason,
+        });
+        continue;
+      }
+
+      currentPrompt = recoveryPrompt;
+      currentRecoveryCount = nextRecoveryIndex;
+      break;
+    }
+  }
+}
+
+export async function runCodexTask(options) {
+  return runEngineTask({
+    ...options,
+    engine: "codex",
+  });
 }
 
 export async function runCodexExec({ context, prompt, runDir, policy }) {
-  return runCodexTask({
+  return runEngineTask({
+    engine: "codex",
     context,
     prompt,
     runDir,
-    model: policy.codex.model,
-    executable: policy.codex.executable,
-    sandbox: policy.codex.sandbox,
-    dangerouslyBypassSandbox: policy.codex.dangerouslyBypassSandbox,
-    jsonOutput: policy.codex.jsonOutput,
+    policy,
+    executionMode: "execute",
     outputPrefix: "codex",
   });
 }
 
+export async function runEngineExec({ engine, context, prompt, runDir, policy }) {
+  return runEngineTask({
+    engine,
+    context,
+    prompt,
+    runDir,
+    policy,
+    executionMode: "execute",
+    outputPrefix: engine,
+  });
+}
+
 export async function runShellCommand(context, commandLine, runDir, index) {
-  const shellInvocation = resolveVerifyShellInvocation();
+  const shellInvocation = resolveVerifyInvocation();
   if (shellInvocation.error) {
     const result = {
       command: commandLine,