@chllming/wave-orchestration 0.9.0 → 0.9.2

package/scripts/wave-orchestrator/agent-process-runner.mjs

@@ -0,0 +1,344 @@
+import { spawn } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  REPO_ROOT,
+  ensureDirectory,
+  readFileTail,
+  readJsonOrNull,
+  sleep,
+  toIsoTimestamp,
+  writeJsonAtomic,
+} from "./shared.mjs";
+
+export const AGENT_RUNTIME_HEARTBEAT_INTERVAL_MS = 10_000;
+
+function isProcessAlive(pid) {
+  if (!Number.isInteger(pid) || pid <= 0) {
+    return false;
+  }
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function parsePositiveInt(value, fallback = null) {
+  const parsed = Number.parseInt(String(value ?? ""), 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+function readRuntimeRecord(runtimePath) {
+  return runtimePath ? readJsonOrNull(runtimePath) || {} : {};
+}
+
+function writeRuntimeRecord(runtimePath, payload) {
+  if (!runtimePath) {
+    return payload;
+  }
+  ensureDirectory(path.dirname(runtimePath));
+  writeJsonAtomic(runtimePath, payload);
+  return payload;
+}
+
+function updateRuntimeRecord(runtimePath, transform) {
+  const current = readRuntimeRecord(runtimePath);
+  const next = transform(current) || current;
+  return writeRuntimeRecord(runtimePath, next);
+}
+
+function appendLogLine(logPath, message) {
+  ensureDirectory(path.dirname(logPath));
+  fs.appendFileSync(logPath, `${String(message || "").trimEnd()}\n`, "utf8");
+}
+
+function normalizeExitCode(code, signal) {
+  if (Number.isInteger(code)) {
+    return code;
+  }
+  if (!signal) {
+    return 1;
+  }
+  if (signal === "SIGTERM") {
+    return 143;
+  }
+  if (signal === "SIGINT") {
+    return 130;
+  }
+  if (signal === "SIGKILL") {
+    return 137;
+  }
+  return 1;
+}
+
+function exitReasonForOutcome(code, signal) {
+  if (signal) {
+    return signal.toLowerCase();
+  }
+  return Number(code) === 0 ? "completed" : "failed";
+}
+
+function terminalDispositionForOutcome(code, signal) {
+  if (signal === "SIGTERM" || signal === "SIGINT" || signal === "SIGKILL") {
+    return "terminated";
+  }
+  return Number(code) === 0 ? "completed" : "failed";
+}
+
+export async function terminateAgentProcessRuntime(runtimeRecord, { graceMs = 1000 } = {}) {
+  const pgid = parsePositiveInt(runtimeRecord?.pgid, null);
+  const candidatePid = parsePositiveInt(
+    runtimeRecord?.executorPid ?? runtimeRecord?.pid ?? runtimeRecord?.runnerPid,
+    null,
+  );
+  if (!pgid && !candidatePid) {
+    return false;
+  }
+  const terminate = (signal) => {
+    if (pgid) {
+      try {
+        process.kill(-pgid, signal);
+        return true;
+      } catch {
+        // fall through
+      }
+    }
+    if (candidatePid) {
+      try {
+        process.kill(candidatePid, signal);
+        return true;
+      } catch {
+        // no-op
+      }
+    }
+    return false;
+  };
+  const sent = terminate("SIGTERM");
+  if (!sent) {
+    return false;
+  }
+  await sleep(graceMs);
+  if (
+    (pgid && isProcessAlive(pgid)) ||
+    (candidatePid && isProcessAlive(candidatePid))
+  ) {
+    terminate("SIGKILL");
+  }
+  return true;
+}
+
+export function buildAgentAttachInfo(runtimeRecord) {
+  return {
+    sessionBackend: String(runtimeRecord?.sessionBackend || "process").trim() || "process",
+    attachMode: String(runtimeRecord?.attachMode || "log-tail").trim() || "log-tail",
+    sessionName: String(runtimeRecord?.sessionName || "").trim() || null,
+    tmuxSessionName: String(runtimeRecord?.tmuxSessionName || "").trim() || null,
+    logPath: String(runtimeRecord?.logPath || "").trim() || null,
+    statusPath: String(runtimeRecord?.statusPath || "").trim() || null,
+    terminalDisposition: String(runtimeRecord?.terminalDisposition || "").trim() || null,
+  };
+}
+
+export function renderAgentAttachFallback(runtimeRecord, { terminal = false } = {}) {
+  const logPath = String(runtimeRecord?.logPath || "").trim();
+  if (!logPath || !fs.existsSync(logPath)) {
+    return "";
+  }
+  return terminal ? readFileTail(logPath, 12000) : logPath;
+}
+
+export function spawnAgentProcessRunner(payload, { env = process.env } = {}) {
+  const payloadPath = String(payload?.payloadPath || "").trim();
+  if (!payloadPath) {
+    throw new Error("Detached agent runner requires payloadPath.");
+  }
+  ensureDirectory(path.dirname(payloadPath));
+  writeJsonAtomic(payloadPath, payload);
+  const runnerPath = fileURLToPath(new URL("./agent-process-runner.mjs", import.meta.url));
+  const child = spawn(process.execPath, [runnerPath, "--payload-file", payloadPath], {
+    cwd: REPO_ROOT,
+    detached: true,
+    stdio: "ignore",
+    env,
+  });
+  child.unref();
+  return {
+    runnerPid: child.pid,
+    payloadPath,
+  };
+}
+
+function parseArgs(argv) {
+  const options = {
+    payloadFile: "",
+  };
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === "--payload-file") {
+      options.payloadFile = String(argv[++index] || "").trim();
+    } else if (arg && arg !== "--") {
+      throw new Error(`Unknown agent-process-runner argument: ${arg}`);
+    }
+  }
+  if (!options.payloadFile) {
+    throw new Error("--payload-file is required");
+  }
+  return options;
+}
+
+async function runAgentProcessRunner(payloadFile) {
+  const payload = readJsonOrNull(payloadFile);
+  if (!payload || typeof payload !== "object") {
+    throw new Error(`Invalid detached agent runner payload: ${payloadFile}`);
+  }
+  const runtimePath = String(payload.runtimePath || "").trim();
+  const statusPath = String(payload.statusPath || "").trim();
+  const logPath = String(payload.logPath || "").trim();
+  const command = String(payload.command || "").trim();
+  if (!statusPath || !logPath || !command) {
+    throw new Error("Detached agent runner payload is missing statusPath, logPath, or command.");
+  }
+
+  const startedAt = toIsoTimestamp();
+  writeRuntimeRecord(runtimePath, {
+    runId: payload.runId || null,
+    waveNumber: Number(payload.waveNumber) || null,
+    attempt: Number(payload.attempt) || 1,
+    agentId: payload.agentId || null,
+    sessionName: payload.sessionName || null,
+    tmuxSessionName: null,
+    sessionBackend: "process",
+    attachMode: "log-tail",
+    runnerPid: process.pid,
+    executorPid: null,
+    pid: null,
+    pgid: null,
+    startedAt,
+    lastHeartbeatAt: startedAt,
+    statusPath,
+    logPath,
+    exitCode: null,
+    exitReason: null,
+    terminalDisposition: "launching",
+  });
+
+  ensureDirectory(path.dirname(logPath));
+  const child = spawn("bash", ["-lc", command], {
+    cwd: REPO_ROOT,
+    detached: true,
+    stdio: "ignore",
+    env: {
+      ...process.env,
+      ...(payload.env && typeof payload.env === "object" ? payload.env : {}),
+      WAVE_ORCHESTRATOR_ID: String(payload.orchestratorId || ""),
+      WAVE_EXECUTOR_MODE: String(payload.executorId || ""),
+    },
+  });
+  const executorPid = parsePositiveInt(child.pid, null);
+  const pgid = executorPid;
+  const markRuntime = (patch = {}) =>
+    updateRuntimeRecord(runtimePath, (current) => ({
+      ...current,
+      ...patch,
+      lastHeartbeatAt: toIsoTimestamp(),
+    }));
+  markRuntime({
+    runnerPid: process.pid,
+    executorPid,
+    pid: executorPid,
+    pgid,
+    terminalDisposition: "running",
+  });
+
+  const heartbeat = setInterval(() => {
+    if (fs.existsSync(statusPath)) {
+      return;
+    }
+    markRuntime({
+      terminalDisposition: "running",
+    });
+  }, AGENT_RUNTIME_HEARTBEAT_INTERVAL_MS);
+  heartbeat.unref?.();
+
+  let forwardedSignal = "";
+  const handleSignal = async (signal) => {
+    forwardedSignal = signal;
+    try {
+      await terminateAgentProcessRuntime({ pgid, executorPid, pid: executorPid });
+    } catch {
+      // best-effort only
+    }
+  };
+  process.once("SIGTERM", () => {
+    void handleSignal("SIGTERM");
+  });
+  process.once("SIGINT", () => {
+    void handleSignal("SIGINT");
+  });
+
+  await new Promise((resolve, reject) => {
+    child.once("error", (error) => {
+      reject(error);
+    });
+    child.once("close", (code, signal) => {
+      clearInterval(heartbeat);
+      const completedAt = toIsoTimestamp();
+      const exitCode = normalizeExitCode(code, signal || forwardedSignal);
+      const exitReason = exitReasonForOutcome(exitCode, signal || forwardedSignal);
+      const terminalDisposition = terminalDispositionForOutcome(exitCode, signal || forwardedSignal);
+      writeJsonAtomic(statusPath, {
+        code: exitCode,
+        promptHash: payload.promptHash || null,
+        orchestratorId: payload.orchestratorId || null,
+        attempt: Number(payload.attempt) || 1,
+        completedAt,
+      });
+      markRuntime({
+        exitCode,
+        exitReason,
+        terminalDisposition,
+      });
+      appendLogLine(
+        logPath,
+        `[${payload.lane || "wave"}-wave-launcher] ${payload.sessionName || payload.agentId || "agent"} finished with code ${exitCode}`,
+      );
+      resolve();
+    });
+  });
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
+  const options = parseArgs(process.argv.slice(2));
+  runAgentProcessRunner(options.payloadFile).catch((error) => {
+    const payload = readJsonOrNull(options.payloadFile);
+    const runtimePath = String(payload?.runtimePath || "").trim();
+    const statusPath = String(payload?.statusPath || "").trim();
+    const logPath = String(payload?.logPath || "").trim();
+    if (logPath) {
+      appendLogLine(logPath, `[wave-agent-runner] ${error instanceof Error ? error.message : String(error)}`);
+    }
+    if (runtimePath) {
+      updateRuntimeRecord(runtimePath, (current) => ({
+        ...current,
+        runnerPid: process.pid,
+        exitCode: 1,
+        exitReason: error instanceof Error ? error.message : String(error),
+        terminalDisposition: "failed",
+        lastHeartbeatAt: toIsoTimestamp(),
+      }));
+    }
+    if (statusPath && !fs.existsSync(statusPath)) {
+      writeJsonAtomic(statusPath, {
+        code: 1,
+        promptHash: payload?.promptHash || null,
+        orchestratorId: payload?.orchestratorId || null,
+        attempt: Number(payload?.attempt) || 1,
+        completedAt: toIsoTimestamp(),
+      });
+    }
+    process.exit(1);
+  });
+}

package/scripts/wave-orchestrator/agent-state.mjs

@@ -350,7 +350,6 @@ function detectTermination(agent, logText, statusRecord) {
   const patterns = [
     { reason: "max-turns", regex: /Reached max turns \((\d+)\)/i },
     { reason: "timeout", regex: /(timed out(?: after [^\n.]+)?)/i },
-    { reason: "session-missing", regex: /(session [^\n]+ disappeared before [^\n]+ was written)/i },
   ];
   for (const pattern of patterns) {
     const match = String(logText || "").match(pattern.regex);

package/scripts/wave-orchestrator/artifact-schemas.mjs

@@ -101,6 +101,13 @@ export function normalizeRelaunchPlan(payload, defaults = {}) {
     attempt: normalizeInteger(source.attempt, null),
     phase: normalizeText(source.phase, null),
     selectedAgentIds: Array.isArray(source.selectedAgentIds) ? source.selectedAgentIds : [],
+    resumeFromPhase: normalizeText(source.resumeFromPhase, null),
+    invalidatedAgentIds: normalizeStringArray(source.invalidatedAgentIds),
+    reusableAgentIds: normalizeStringArray(source.reusableAgentIds),
+    reusableProofBundleIds: normalizeStringArray(source.reusableProofBundleIds),
+    forwardedClosureGaps: Array.isArray(source.forwardedClosureGaps)
+      ? cloneJson(source.forwardedClosureGaps)
+      : [],
     reasonBuckets: isPlainObject(source.reasonBuckets) ? source.reasonBuckets : {},
     executorStates: isPlainObject(source.executorStates) ? source.executorStates : {},
     fallbackHistory: isPlainObject(source.fallbackHistory) ? source.fallbackHistory : {},

package/scripts/wave-orchestrator/autonomous.mjs

@@ -2,7 +2,6 @@ import { spawnSync } from "node:child_process";
 import fs from "node:fs";
 import path from "node:path";
 import {
-  DEFAULT_CODEX_SANDBOX_MODE,
   DEFAULT_EXECUTOR_MODE,
   loadWaveConfig,
   normalizeCodexSandboxMode,
@@ -36,6 +35,10 @@ import {
   readMaterializedCoordinationState,
 } from "./coordination-store.mjs";
 import { readWaveLedger } from "./ledger.mjs";
+import {
+  submitLauncherRun,
+  waitForRunState,
+} from "./supervisor-cli.mjs";
 
 const AUTONOMOUS_EXECUTOR_MODES = SUPPORTED_EXECUTOR_MODES.filter((mode) => mode !== "local");
 
@@ -58,7 +61,7 @@ Options:
   --orchestrator-id <id>        Orchestrator ID for coordination board
   --resident-orchestrator       Launch a resident orchestrator session for each live wave
   --executor <mode>             Default executor passed to launcher: ${AUTONOMOUS_EXECUTOR_MODES.join(" | ")} (default: lane config)
-  --codex-sandbox <mode>        Default Codex sandbox mode passed to launcher (default: ${DEFAULT_CODEX_SANDBOX_MODE})
+  --codex-sandbox <mode>        Codex sandbox mode override passed to launcher (default: lane config)
   --dashboard                   Enable dashboards (default: disabled)
   --keep-sessions               Keep tmux sessions between waves
   --keep-terminals              Keep temporary terminal entries between waves
@@ -80,7 +83,7 @@ export function parseArgs(argv) {
     orchestratorId: null,
     residentOrchestrator: false,
     executorMode: DEFAULT_EXECUTOR_MODE,
-    codexSandboxMode: DEFAULT_CODEX_SANDBOX_MODE,
+    codexSandboxMode: null,
     noDashboard: true,
     keepSessions: false,
     keepTerminals: false,
@@ -234,9 +237,8 @@ function listPendingFeedback(lane, project) {
   ]);
 }
 
-function launchSingleWave(params) {
-  const args = [
-    path.join(PACKAGE_ROOT, "scripts", "wave-launcher.mjs"),
+export function buildSingleWaveLauncherArgs(params) {
+  const launcherArgs = [
     "--project",
     params.project,
     "--lane",
@@ -259,26 +261,57 @@ function launchSingleWave(params) {
     String(params.agentLaunchStaggerMs),
     "--executor",
     params.executorMode,
-    "--codex-sandbox",
-    params.codexSandboxMode,
     "--orchestrator-id",
     params.orchestratorId,
     "--coordination-note",
     `autonomous single-wave run wave=${params.wave} attempt=${params.attempt}`,
   ];
   if (params.noDashboard) {
-    args.push("--no-dashboard");
+    launcherArgs.push("--no-dashboard");
+  }
+  if (params.codexSandboxMode) {
+    launcherArgs.push("--codex-sandbox", params.codexSandboxMode);
   }
   if (params.keepSessions) {
-    args.push("--keep-sessions");
+    launcherArgs.push("--keep-sessions");
   }
   if (params.keepTerminals) {
-    args.push("--keep-terminals");
+    launcherArgs.push("--keep-terminals");
   }
   if (params.residentOrchestrator) {
-    args.push("--resident-orchestrator");
+    launcherArgs.push("--resident-orchestrator");
   }
-  return runCommand(args, { [WAVE_SUPPRESS_UPDATE_NOTICE_ENV]: "1" });
+  return launcherArgs;
+}
+
+function launchSingleWave(params) {
+  const launcherArgs = buildSingleWaveLauncherArgs(params);
+  const submission = submitLauncherRun(launcherArgs);
+  console.log(
+    `[autonomous] submitted wave ${params.wave} as run_id=${submission.runId} lane=${submission.lane} project=${submission.project}`,
+  );
+  const observeTimeoutSeconds = Math.max(30, Math.min(60, params.timeoutMinutes * 60));
+  return (async () => {
+    while (true) {
+      const located = await waitForRunState({
+        project: submission.project,
+        lane: submission.lane,
+        adhocRunId: submission.adhocRunId,
+        runId: submission.runId,
+        timeoutSeconds: observeTimeoutSeconds,
+      });
+      if (located.state.status === "completed") {
+        return 0;
+      }
+      if (located.state.status === "failed") {
+        return Number.isInteger(located.state.exitCode) ? located.state.exitCode : 1;
+      }
+      const reconcileStatus = reconcile(params.lane, params.project);
+      if (reconcileStatus !== 0) {
+        return reconcileStatus;
+      }
+    }
+  })();
 }
 
 function requiredInboundDependenciesOpen(lanePaths, lane) {
@@ -434,7 +467,7 @@ export async function runAutonomousCli(argv) {
       console.log(
         `\n[autonomous] launching wave ${wave} (attempt ${attempt}/${options.maxAttemptsPerWave})`,
       );
-      const status = launchSingleWave({
+      const status = await launchSingleWave({
         ...options,
         wave,
         attempt,