helloloop 0.8.2 → 0.8.5

package/src/runtime_engine_task.mjs

@@ -0,0 +1,395 @@
+import path from "node:path";
+
+import { ensureDir, nowIso, tailText, writeJson, writeText } from "./common.mjs";
+import { getEngineDisplayName, normalizeEngineName } from "./engine_metadata.mjs";
+import { resolveEngineInvocation } from "./engine_process_support.mjs";
+import { isHostLeaseAlive } from "./host_lease.mjs";
+import {
+  buildRuntimeRecoveryPrompt,
+  classifyRuntimeRecoveryFailure,
+  renderRuntimeRecoverySummary,
+  resolveRuntimeRecoveryPolicy,
+  selectRuntimeRecoveryDelayMs,
+} from "./runtime_recovery.mjs";
+import {
+  buildHostLeaseStoppedResult,
+  buildNotificationNote,
+  createRuntimeStatusWriter,
+  maybeSendStopNotification,
+  resolveEnginePolicy,
+  runEngineAttempt,
+  runEngineHealthProbe,
+  sleepWithLease,
+  writeEngineRunArtifacts,
+} from "./runtime_engine_support.mjs";
+
+export async function runEngineTask({
+  engine = "codex",
+  context,
+  prompt,
+  runDir,
+  policy = {},
+  executionMode = "analyze",
+  outputSchemaFile = "",
+  outputPrefix = "",
+  ephemeral = false,
+  skipGitRepoCheck = false,
+  env = {},
+  hostLease = null,
+}) {
+  ensureDir(runDir);
+
+  const normalizedEngine = normalizeEngineName(engine) || "codex";
+  const resolvedPolicy = resolveEnginePolicy(policy, normalizedEngine);
+  const prefix = outputPrefix || normalizedEngine;
+  const invocation = resolveEngineInvocation(normalizedEngine, resolvedPolicy.executable);
+  const recoveryPolicy = resolveRuntimeRecoveryPolicy(policy);
+  const runtimeStatusFile = path.join(runDir, `${prefix}-runtime.json`);
+  const writeRuntimeStatus = createRuntimeStatusWriter(runtimeStatusFile, {
+    engine: normalizedEngine,
+    engineDisplayName: getEngineDisplayName(normalizedEngine),
+    phase: executionMode,
+    outputPrefix: prefix,
+    hardRetryBudget: recoveryPolicy.hardRetryDelaysSeconds.length,
+    softRetryBudget: recoveryPolicy.softRetryDelaysSeconds.length,
+  });
+
+  const recoveryHistory = [];
+  let currentPrompt = prompt;
+  let currentRecoveryCount = 0;
+  let activeFailure = null;
+
+  while (true) {
+    if (!isHostLeaseAlive(hostLease)) {
+      const stopped = buildHostLeaseStoppedResult("检测到宿主窗口已关闭，HelloLoop 已停止本轮自动执行。");
+      writeRuntimeStatus("stopped_host_closed", {
+        attemptPrefix: prefix,
+        recoveryCount: recoveryHistory.length,
+        recoveryHistory,
+        code: stopped.code,
+        failureCode: "host_closed",
+        failureFamily: "host_lease",
+        failureReason: stopped.leaseReason,
+      });
+      return {
+        ...stopped,
+        finalMessage: "",
+        recoveryCount: recoveryHistory.length,
+        recoveryHistory,
+        recoverySummary: "",
+        recoveryFailure: null,
+        notification: null,
+      };
+    }
+
+    const attemptPrefix = currentRecoveryCount === 0
+      ? prefix
+      : `${prefix}-recovery-${String(currentRecoveryCount).padStart(2, "0")}`;
+    const taskAttempt = await runEngineAttempt({
+      engine: normalizedEngine,
+      invocation,
+      context,
+      prompt: currentPrompt,
+      runDir,
+      attemptPrefix,
+      resolvedPolicy,
+      executionMode,
+      outputSchemaFile,
+      env,
+      recoveryPolicy,
+      writeRuntimeStatus,
+      recoveryCount: currentRecoveryCount,
+      recoveryHistory,
+      hostLease,
+      ephemeral,
+      skipGitRepoCheck,
+      probeMode: false,
+    });
+    if (taskAttempt.result.leaseExpired) {
+      writeText(path.join(runDir, `${prefix}-prompt.md`), currentPrompt);
+      writeEngineRunArtifacts(runDir, prefix, taskAttempt.result, taskAttempt.finalMessage);
+      writeRuntimeStatus("stopped_host_closed", {
+        attemptPrefix,
+        recoveryCount: recoveryHistory.length,
+        recoveryHistory,
+        finalMessage: taskAttempt.finalMessage,
+        code: taskAttempt.result.code,
+        failureCode: "host_closed",
+        failureFamily: "host_lease",
+        failureReason: taskAttempt.result.leaseReason,
+      });
+      return {
+        ...taskAttempt.result,
+        finalMessage: taskAttempt.finalMessage,
+        recoveryCount: recoveryHistory.length,
+        recoveryHistory,
+        recoverySummary: "",
+        recoveryFailure: null,
+        notification: null,
+      };
+    }
+
+    const taskFailure = classifyRuntimeRecoveryFailure({
+      result: {
+        ...taskAttempt.result,
+        finalMessage: taskAttempt.finalMessage,
+      },
+    });
+
+    if (taskAttempt.result.ok || !recoveryPolicy.enabled) {
+      const finalRecoverySummary = taskAttempt.result.ok
+        ? ""
+        : renderRuntimeRecoverySummary(recoveryHistory, taskFailure);
+      const notification = taskAttempt.result.ok
+        ? null
+        : await maybeSendStopNotification({
+          context,
+          runDir,
+          engine: normalizedEngine,
+          executionMode,
+          failure: taskFailure,
+          result: taskAttempt.result,
+          recoveryHistory,
+        });
+      const notificationNote = taskAttempt.result.ok ? "" : buildNotificationNote(notification);
+      const finalizedResult = taskAttempt.result.ok
+        ? taskAttempt.result
+        : {
+          ...taskAttempt.result,
+          stderr: [
+            taskAttempt.result.stderr,
+            "",
+            finalRecoverySummary,
+            notificationNote,
+          ].filter(Boolean).join("\n").trim(),
+        };
+
+      writeText(path.join(runDir, `${prefix}-prompt.md`), currentPrompt);
+      writeEngineRunArtifacts(runDir, prefix, finalizedResult, taskAttempt.finalMessage);
+      if (normalizedEngine === "codex" && taskAttempt.finalMessage) {
+        writeText(path.join(runDir, `${prefix}-last-message.txt`), taskAttempt.finalMessage);
+      }
+      writeRuntimeStatus(taskAttempt.result.ok ? "completed" : "paused_manual", {
+        attemptPrefix,
+        recoveryCount: recoveryHistory.length,
+        recoveryHistory,
+        recoverySummary: finalRecoverySummary,
+        finalMessage: taskAttempt.finalMessage,
+        code: finalizedResult.code,
+        failureCode: taskFailure.code,
+        failureFamily: taskFailure.family,
+        failureReason: taskFailure.reason,
+        notification,
+      });
+
+      return {
+        ...finalizedResult,
+        finalMessage: taskAttempt.finalMessage,
+        recoveryCount: recoveryHistory.length,
+        recoveryHistory,
+        recoverySummary: finalRecoverySummary,
+        recoveryFailure: taskAttempt.result.ok
+          ? null
+          : {
+            ...taskFailure,
+            shouldStopTask: true,
+            exhausted: true,
+          },
+        notification,
+      };
+    }
+
+    activeFailure = taskFailure;
+    while (true) {
+      const nextRecoveryIndex = recoveryHistory.length + 1;
+      const recoveryPrompt = buildRuntimeRecoveryPrompt({
+        basePrompt: prompt,
+        engine: normalizedEngine,
+        phaseLabel: executionMode === "analyze" ? "分析/复核" : "执行",
+        failure: activeFailure,
+        result: {
+          ...taskAttempt.result,
+          finalMessage: taskAttempt.finalMessage,
+        },
+        nextRecoveryIndex,
+        maxRecoveries: recoveryPolicy[activeFailure.family === "hard" ? "hardRetryDelaysSeconds" : "softRetryDelaysSeconds"].length,
+      });
+      writeText(
+        path.join(runDir, `${prefix}-auto-recovery-${String(nextRecoveryIndex).padStart(2, "0")}-prompt.md`),
+        recoveryPrompt,
+      );
+      const delayMs = selectRuntimeRecoveryDelayMs(recoveryPolicy, activeFailure.family, nextRecoveryIndex);
+      if (delayMs < 0) {
+        const finalRecoverySummary = renderRuntimeRecoverySummary(recoveryHistory, activeFailure);
+        const notification = await maybeSendStopNotification({
+          context,
+          runDir,
+          engine: normalizedEngine,
+          executionMode,
+          failure: activeFailure,
+          result: taskAttempt.result,
+          recoveryHistory,
+        });
+        const notificationNote = buildNotificationNote(notification);
+        const finalizedResult = {
+          ...taskAttempt.result,
+          stderr: [
+            taskAttempt.result.stderr,
+            "",
+            finalRecoverySummary,
+            notificationNote,
+          ].filter(Boolean).join("\n").trim(),
+        };
+
+        writeText(path.join(runDir, `${prefix}-prompt.md`), currentPrompt);
+        writeEngineRunArtifacts(runDir, prefix, finalizedResult, taskAttempt.finalMessage);
+        writeRuntimeStatus("paused_manual", {
+          attemptPrefix,
+          recoveryCount: recoveryHistory.length,
+          recoveryHistory,
+          recoverySummary: finalRecoverySummary,
+          finalMessage: taskAttempt.finalMessage,
+          code: finalizedResult.code,
+          failureCode: activeFailure.code,
+          failureFamily: activeFailure.family,
+          failureReason: activeFailure.reason,
+          notification,
+        });
+
+        return {
+          ...finalizedResult,
+          finalMessage: taskAttempt.finalMessage,
+          recoveryCount: recoveryHistory.length,
+          recoveryHistory,
+          recoverySummary: finalRecoverySummary,
+          recoveryFailure: {
+            ...activeFailure,
+            shouldStopTask: true,
+            exhausted: true,
+          },
+          notification,
+        };
+      }
+
+      writeRuntimeStatus("retry_waiting", {
+        attemptPrefix,
+        recoveryCount: nextRecoveryIndex,
+        recoveryHistory,
+        nextRetryDelayMs: delayMs,
+        nextRetryAt: new Date(Date.now() + delayMs).toISOString(),
+        failureCode: activeFailure.code,
+        failureFamily: activeFailure.family,
+        failureReason: activeFailure.reason,
+      });
+      if (delayMs > 0) {
+        const canContinue = await sleepWithLease(delayMs, hostLease);
+        if (!canContinue) {
+          const stopped = buildHostLeaseStoppedResult("检测到宿主窗口已关闭，HelloLoop 已停止等待中的自动恢复。");
+          writeRuntimeStatus("stopped_host_closed", {
+            attemptPrefix,
+            recoveryCount: recoveryHistory.length,
+            recoveryHistory,
+            code: stopped.code,
+            failureCode: "host_closed",
+            failureFamily: "host_lease",
+            failureReason: stopped.leaseReason,
+          });
+          return {
+            ...stopped,
+            finalMessage: taskAttempt.finalMessage,
+            recoveryCount: recoveryHistory.length,
+            recoveryHistory,
+            recoverySummary: "",
+            recoveryFailure: null,
+            notification: null,
+          };
+        }
+      }
+
+      const probeAttempt = await runEngineHealthProbe({
+        engine: normalizedEngine,
+        invocation,
+        context,
+        runDir,
+        resolvedPolicy,
+        recoveryPolicy,
+        writeRuntimeStatus,
+        recoveryCount: nextRecoveryIndex,
+        recoveryHistory,
+        hostLease,
+        env,
+        probeIndex: nextRecoveryIndex,
+      });
+      if (probeAttempt.result.leaseExpired) {
+        writeRuntimeStatus("stopped_host_closed", {
+          attemptPrefix: probeAttempt.attemptPrefix,
+          recoveryCount: recoveryHistory.length,
+          recoveryHistory,
+          code: probeAttempt.result.code,
+          failureCode: "host_closed",
+          failureFamily: "host_lease",
+          failureReason: probeAttempt.result.leaseReason,
+        });
+        return {
+          ...probeAttempt.result,
+          finalMessage: probeAttempt.finalMessage,
+          recoveryCount: recoveryHistory.length,
+          recoveryHistory,
+          recoverySummary: "",
+          recoveryFailure: null,
+          notification: null,
+        };
+      }
+      const recoveryRecord = {
+        recoveryIndex: nextRecoveryIndex,
+        family: activeFailure.family,
+        code: activeFailure.code,
+        reason: activeFailure.reason,
+        delaySeconds: Math.floor(delayMs / 1000),
+        taskStatus: "failed",
+        taskCode: taskAttempt.result.code,
+        taskAttemptPrefix: attemptPrefix,
+        probeStatus: probeAttempt.result.ok ? "ok" : "failed",
+        probeCode: probeAttempt.result.code,
+        probeAttemptPrefix: probeAttempt.attemptPrefix,
+        probeFailureCode: probeAttempt.failure?.code || "",
+        probeFailureFamily: probeAttempt.failure?.family || "",
+        probeFailureReason: probeAttempt.failure?.reason || "",
+        watchdogTriggered: taskAttempt.result.watchdogTriggered === true || probeAttempt.result.watchdogTriggered === true,
+      };
+      recoveryHistory.push(recoveryRecord);
+      writeJson(path.join(
+        runDir,
+        `${prefix}-auto-recovery-${String(nextRecoveryIndex).padStart(2, "0")}.json`,
+      ), {
+        ...recoveryRecord,
+        engine: normalizedEngine,
+        phase: executionMode,
+        stdoutTail: tailText(taskAttempt.result.stdout, 20),
+        stderrTail: tailText(taskAttempt.result.stderr, 20),
+        finalMessageTail: tailText(taskAttempt.finalMessage, 20),
+        probeStdoutTail: tailText(probeAttempt.result.stdout, 20),
+        probeStderrTail: tailText(probeAttempt.result.stderr, 20),
+        probeFinalMessageTail: tailText(probeAttempt.finalMessage, 20),
+        createdAt: nowIso(),
+      });
+
+      if (!probeAttempt.result.ok) {
+        activeFailure = probeAttempt.failure;
+        writeRuntimeStatus("probe_failed", {
+          attemptPrefix: probeAttempt.attemptPrefix,
+          recoveryCount: nextRecoveryIndex,
+          recoveryHistory,
+          failureCode: activeFailure.code,
+          failureFamily: activeFailure.family,
+          failureReason: activeFailure.reason,
+        });
+        continue;
+      }
+
+      currentPrompt = recoveryPrompt;
+      currentRecoveryCount = nextRecoveryIndex;
+      break;
+    }
+  }
+}

package/src/runtime_recovery.mjs

@@ -21,6 +21,8 @@ const HARD_STOP_MATCHERS = [
       "400 bad request",
       "bad request",
       "invalid request",
+      "invalid schema",
+      "invalid_json_schema",
       "invalid argument",
       "invalid_argument",
       "failed to parse",
@@ -28,6 +30,7 @@ const HARD_STOP_MATCHERS = [
       "malformed",
       "schema validation",
       "json schema",
+      "response_format",
       "unexpected argument",
       "unknown option",
     ],

package/src/supervisor_runtime.mjs

@@ -0,0 +1,285 @@
+import fs from "node:fs";
+import path from "node:path";
+import { spawn } from "node:child_process";
+
+import { createContext } from "./context.mjs";
+import { nowIso, readJson, writeJson, readTextIfExists, timestampForFile } from "./common.mjs";
+import { isHostLeaseAlive, renderHostLeaseLabel, resolveHostLease } from "./host_lease.mjs";
+import { runLoop, runOnce } from "./runner.mjs";
+
+const ACTIVE_STATUSES = new Set(["launching", "running"]);
+const FINAL_STATUSES = new Set(["completed", "failed", "stopped"]);
+
+function removeIfExists(filePath) {
+  try {
+    fs.rmSync(filePath, { force: true });
+  } catch {
+    // ignore cleanup failures
+  }
+}
+
+function readJsonIfExists(filePath) {
+  return fs.existsSync(filePath) ? readJson(filePath) : null;
+}
+
+function isPidAlive(pid) {
+  const numberPid = Number(pid || 0);
+  if (!Number.isFinite(numberPid) || numberPid <= 0) {
+    return false;
+  }
+  try {
+    process.kill(numberPid, 0);
+    return true;
+  } catch (error) {
+    return String(error?.code || "") === "EPERM";
+  }
+}
+
+function buildState(context, patch = {}) {
+  const current = readJsonIfExists(context.supervisorStateFile) || {};
+  return {
+    ...current,
+    ...patch,
+    updatedAt: nowIso(),
+  };
+}
+
+function writeState(context, patch) {
+  writeJson(context.supervisorStateFile, buildState(context, patch));
+}
+
+function toSerializableOptions(options = {}) {
+  return JSON.parse(JSON.stringify(options));
+}
+
+export function readSupervisorState(context) {
+  return readJsonIfExists(context.supervisorStateFile);
+}
+
+export function hasActiveSupervisor(context) {
+  const state = readSupervisorState(context);
+  return Boolean(state?.status && ACTIVE_STATUSES.has(String(state.status)) && isPidAlive(state.pid));
+}
+
+export function renderSupervisorLaunchSummary(session) {
+  return [
+    `HelloLoop supervisor 已启动：${session.sessionId}`,
+    `- 宿主租约：${renderHostLeaseLabel(session.lease)}`,
+    "- 当前 turn 若被中断，只要当前宿主窗口仍存活，本轮自动执行会继续。",
+    "- 如需主动停止，直接关闭当前 CLI 窗口即可。",
+  ].join("\n");
+}
+
+export function launchSupervisedCommand(context, command, options = {}) {
+  const existing = readSupervisorState(context);
+  if (existing?.status && ACTIVE_STATUSES.has(String(existing.status)) && isPidAlive(existing.pid)) {
+    throw new Error(`已有 HelloLoop supervisor 正在运行：${existing.sessionId || "unknown"}`);
+  }
+
+  const sessionId = timestampForFile();
+  const lease = resolveHostLease({ hostContext: options.hostContext });
+  const request = {
+    sessionId,
+    command,
+    context: {
+      repoRoot: context.repoRoot,
+      configDirName: context.configDirName,
+    },
+    options: toSerializableOptions(options),
+    lease,
+  };
+
+  fs.mkdirSync(context.supervisorRoot, { recursive: true });
+  removeIfExists(context.supervisorResultFile);
+  removeIfExists(context.supervisorStdoutFile);
+  removeIfExists(context.supervisorStderrFile);
+  writeJson(context.supervisorRequestFile, request);
+  writeState(context, {
+    sessionId,
+    status: "launching",
+    command,
+    lease,
+    startedAt: nowIso(),
+    pid: 0,
+  });
+
+  const stdoutFd = fs.openSync(context.supervisorStdoutFile, "w");
+  const stderrFd = fs.openSync(context.supervisorStderrFile, "w");
+  try {
+    const child = spawn(process.execPath, [
+      path.join(context.bundleRoot, "bin", "helloloop.js"),
+      "__supervise",
+      "--session-file",
+      context.supervisorRequestFile,
+    ], {
+      cwd: context.repoRoot,
+      detached: true,
+      shell: false,
+      windowsHide: true,
+      stdio: ["ignore", stdoutFd, stderrFd],
+      env: {
+        ...process.env,
+        HELLOLOOP_SUPERVISOR_ACTIVE: "1",
+      },
+    });
+    child.unref();
+    writeState(context, {
+      sessionId,
+      status: "running",
+      command,
+      lease,
+      startedAt: nowIso(),
+      pid: child.pid ?? 0,
+    });
+
+    return {
+      sessionId,
+      pid: child.pid ?? 0,
+      lease,
+    };
+  } finally {
+    fs.closeSync(stdoutFd);
+    fs.closeSync(stderrFd);
+  }
+}
+
+export async function waitForSupervisedResult(context, session, options = {}) {
+  const pollMs = Math.max(100, Number(options.pollMs || 500));
+
+  while (true) {
+    const result = readJsonIfExists(context.supervisorResultFile);
+    if (result?.sessionId === session.sessionId) {
+      return result;
+    }
+
+    const state = readSupervisorState(context);
+    if (state?.sessionId === session.sessionId && FINAL_STATUSES.has(String(state.status || ""))) {
+      return {
+        sessionId: session.sessionId,
+        command: state.command || "",
+        exitCode: Number(state.exitCode || 1),
+        ok: state.status === "completed",
+        error: state.message || readTextIfExists(context.supervisorStderrFile, "").trim() || "HelloLoop supervisor 异常结束。",
+      };
+    }
+
+    if (!isPidAlive(session.pid)) {
+      return {
+        sessionId: session.sessionId,
+        command: state?.command || "",
+        exitCode: 1,
+        ok: false,
+        error: readTextIfExists(context.supervisorStderrFile, "").trim() || "HelloLoop supervisor 已退出，但未生成结果文件。",
+      };
+    }
+
+    await new Promise((resolve) => {
+      setTimeout(resolve, pollMs);
+    });
+  }
+}
+
+export async function runSupervisedCommandFromSessionFile(sessionFile) {
+  const request = readJson(sessionFile);
+  const context = createContext(request.context || {});
+  const command = String(request.command || "").trim();
+  const lease = request.lease || {};
+  const commandOptions = {
+    ...(request.options || {}),
+    hostLease: lease,
+  };
+
+  writeState(context, {
+    sessionId: request.sessionId,
+    command,
+    status: "running",
+    lease,
+    pid: process.pid,
+    startedAt: nowIso(),
+  });
+
+  try {
+    if (!isHostLeaseAlive(lease)) {
+      const stopped = {
+        sessionId: request.sessionId,
+        command,
+        exitCode: 1,
+        ok: false,
+        stopped: true,
+        error: "检测到宿主窗口已关闭，HelloLoop supervisor 未继续执行。",
+      };
+      writeJson(context.supervisorResultFile, stopped);
+      writeState(context, {
+        sessionId: request.sessionId,
+        command,
+        status: "stopped",
+        exitCode: stopped.exitCode,
+        message: stopped.error,
+        completedAt: nowIso(),
+      });
+      return;
+    }
+
+    if (command === "run-loop") {
+      const results = await runLoop(context, commandOptions);
+      const exitCode = results.some((item) => !item.ok) ? 1 : 0;
+      const payload = {
+        sessionId: request.sessionId,
+        command,
+        ok: exitCode === 0,
+        exitCode,
+        results,
+      };
+      writeJson(context.supervisorResultFile, payload);
+      writeState(context, {
+        sessionId: request.sessionId,
+        command,
+        status: payload.ok ? "completed" : (payload.results.some((item) => item.kind === "host-lease-stopped") ? "stopped" : "failed"),
+        exitCode,
+        message: payload.ok ? "" : (payload.results.find((item) => !item.ok)?.summary || ""),
+        completedAt: nowIso(),
+      });
+      return;
+    }
+
+    if (command === "run-once") {
+      const result = await runOnce(context, commandOptions);
+      const payload = {
+        sessionId: request.sessionId,
+        command,
+        ok: result.ok,
+        exitCode: result.ok ? 0 : 1,
+        result,
+      };
+      writeJson(context.supervisorResultFile, payload);
+      writeState(context, {
+        sessionId: request.sessionId,
+        command,
+        status: payload.ok ? "completed" : (result.kind === "host-lease-stopped" ? "stopped" : "failed"),
+        exitCode: payload.exitCode,
+        message: result.summary || result.finalMessage || "",
+        completedAt: nowIso(),
+      });
+      return;
+    }
+
+    throw new Error(`不支持的 supervisor 命令：${command}`);
+  } catch (error) {
+    const payload = {
+      sessionId: request.sessionId,
+      command,
+      ok: false,
+      exitCode: 1,
+      error: String(error?.stack || error || ""),
+    };
+    writeJson(context.supervisorResultFile, payload);
+    writeState(context, {
+      sessionId: request.sessionId,
+      command,
+      status: "failed",
+      exitCode: 1,
+      message: payload.error,
+      completedAt: nowIso(),
+    });
+  }
+}