@chllming/wave-orchestration 0.9.0 → 0.9.1

package/scripts/wave-orchestrator/retry-engine.mjs

@@ -36,7 +36,7 @@ import {
   isDocsOnlyDesignAgent,
   isDesignAgent,
   isImplementationOwningDesignAgent,
-  isSecurityReviewAgent,
+  isSecurityReviewAgentForLane,
   resolveWaveRoleBindings,
 } from "./role-helpers.mjs";
 import {
@@ -283,7 +283,10 @@ export function reconcileFailuresAgainstSharedComponentState(wave, agentRuns, fa
   const summariesByAgentId = Object.fromEntries(
     (agentRuns || []).map((runInfo) => [
       runInfo.agent.agentId,
-      readRunExecutionSummary(runInfo, wave, { mode: "live" }),
+      readRunExecutionSummary(runInfo, wave, {
+        mode: "live",
+        securityRolePromptPath: lanePaths?.securityRolePromptPath,
+      }),
     ]),
   );
   const failureAgentIds = new Set(failures.map((failure) => failure.agentId).filter(Boolean));
@@ -407,7 +410,7 @@ function isClosureAgentId(agent, lanePaths, waveDefinition = null) {
   return (
     resolveWaveRoleBindings(waveDefinition, lanePaths, waveDefinition?.agents).closureAgentIds.includes(
       agent?.agentId,
-    ) || isSecurityReviewAgent(agent)
+    ) || isSecurityReviewAgentForLane(agent, lanePaths)
   );
 }
 
@@ -658,7 +661,7 @@ function resolveRunsForResumePhase(agentRuns, lanePaths, resumePhase, waveDefini
     return runsFromAgentIds(agentRuns, [roleBindings.integrationAgentId]);
   }
   if (resumePhase === "security-review") {
-    return (agentRuns || []).filter((run) => isSecurityReviewAgent(run.agent));
+    return (agentRuns || []).filter((run) => isSecurityReviewAgentForLane(run.agent, lanePaths));
   }
   if (resumePhase === "docs-closure") {
     return runsFromAgentIds(agentRuns, [roleBindings.documentationAgentId]);
@@ -820,7 +823,7 @@ function resolveRelaunchRunsLegacy(agentRuns, failures, derivedState, lanePaths,
   }
   if (derivedState?.ledger?.phase === "security-review") {
     return {
-      runs: agentRuns.filter((run) => isSecurityReviewAgent(run.agent)),
+      runs: agentRuns.filter((run) => isSecurityReviewAgentForLane(run.agent, lanePaths)),
       barrier: null,
     };
   }
@@ -1196,6 +1199,66 @@ function collectResumeExecutorChanges(waveState) {
     }));
 }
 
+const CLOSURE_STAGE_ORDER = [
+  "cont-eval",
+  "security-review",
+  "integrating",
+  "docs-closure",
+  "cont-qa-closure",
+];
+
+function closureStageForAgentId(agentId, roleBindings) {
+  if (!agentId) {
+    return null;
+  }
+  if (agentId === roleBindings?.contEvalAgentId) {
+    return "cont-eval";
+  }
+  if ((roleBindings?.securityReviewerAgentIds || []).includes(agentId)) {
+    return "security-review";
+  }
+  if (agentId === roleBindings?.integrationAgentId) {
+    return "integrating";
+  }
+  if (agentId === roleBindings?.documentationAgentId) {
+    return "docs-closure";
+  }
+  if (agentId === roleBindings?.contQaAgentId) {
+    return "cont-qa-closure";
+  }
+  return null;
+}
+
+function collectForwardedClosureGaps(waveState, waveDefinition, lanePaths) {
+  const roleBindings = resolveWaveRoleBindings(waveDefinition, lanePaths, waveDefinition?.agents);
+  return normalizeRetryTargets(waveState?.retryTargetSet)
+    .filter((target) => target?.statusCode === "wave-proof-gap" || target?.reason === "wave-proof-gap")
+    .map((target) => {
+      const stageKey = closureStageForAgentId(target.agentId, roleBindings);
+      if (!stageKey) {
+        return null;
+      }
+      return {
+        stageKey,
+        agentId: target.agentId,
+        attempt: waveState?.attempt ?? null,
+        detail: target.detail || target.reason || target.statusCode || null,
+        targets: normalizeRetryTargets(waveState?.retryTargetSet)
+          .filter((entry) => closureStageForAgentId(entry.agentId, roleBindings))
+          .map((entry) => entry.agentId)
+          .filter((agentId) => {
+            const agentStage = closureStageForAgentId(agentId, roleBindings);
+            return CLOSURE_STAGE_ORDER.indexOf(agentStage) >= CLOSURE_STAGE_ORDER.indexOf(stageKey);
+          }),
+        resolved: false,
+      };
+    })
+    .filter(Boolean)
+    .sort(
+      (left, right) => CLOSURE_STAGE_ORDER.indexOf(left.stageKey) - CLOSURE_STAGE_ORDER.indexOf(right.stageKey),
+    );
+}
+
 /**
  * Deterministic resume planner operating on reducer output (WaveState).
  * Pure function — no file I/O.
@@ -1207,8 +1270,29 @@ export function buildResumePlan(waveState, options = {}) {
   const canResume = reason !== "all-gates-pass";
   const pendingAgentIds = waveState.closureEligibility?.pendingAgentIds || [];
   const provenAgentIds = waveState.closureEligibility?.ownedSliceProvenAgentIds || [];
+  const forwardedClosureGaps = collectForwardedClosureGaps(waveState, waveDefinition, lanePaths);
   const invalidatedAgentIds = [...pendingAgentIds].sort();
   const reusableAgentIds = [...provenAgentIds].sort();
+  if (forwardedClosureGaps.length > 0) {
+    const roleBindings = resolveWaveRoleBindings(waveDefinition, lanePaths, waveDefinition?.agents);
+    const earliestGap = forwardedClosureGaps[0];
+    const invalidatedClosureAgentIds = roleBindings.closureAgentIds.filter((agentId) => {
+      const stageKey = closureStageForAgentId(agentId, roleBindings);
+      return CLOSURE_STAGE_ORDER.indexOf(stageKey) >= CLOSURE_STAGE_ORDER.indexOf(earliestGap.stageKey);
+    });
+    for (const agentId of invalidatedClosureAgentIds) {
+      if (!invalidatedAgentIds.includes(agentId)) {
+        invalidatedAgentIds.push(agentId);
+      }
+    }
+    invalidatedAgentIds.sort();
+    for (const agentId of invalidatedClosureAgentIds) {
+      const index = reusableAgentIds.indexOf(agentId);
+      if (index >= 0) {
+        reusableAgentIds.splice(index, 1);
+      }
+    }
+  }
   const proofBundles =
     waveState.closureEligibility?.proofBundles ||
     waveState.proofAvailability?.activeProofBundles ||
@@ -1219,7 +1303,9 @@ export function buildResumePlan(waveState, options = {}) {
     .sort();
   const gateSnapshot = waveState.gateSnapshot || {};
   let resumeFromPhase = "completed";
-  if (canResume && gateSnapshot.overall && !gateSnapshot.overall.ok) {
+  if (forwardedClosureGaps.length > 0) {
+    resumeFromPhase = forwardedClosureGaps[0].stageKey;
+  } else if (canResume && gateSnapshot.overall && !gateSnapshot.overall.ok) {
     resumeFromPhase = phaseFromGate(gateSnapshot.overall.gate);
   } else if (canResume) {
     resumeFromPhase = "implementation";
@@ -1239,6 +1325,7 @@ export function buildResumePlan(waveState, options = {}) {
     humanInputBlockers: collectResumeHumanInputBlockers(waveState),
     gateBlockers: collectResumeGateBlockers(gateSnapshot),
     closureEligibility: waveState.closureEligibility || null,
+    forwardedClosureGaps,
     deterministic: true,
     createdAt: toIsoTimestamp(),
   };

package/scripts/wave-orchestrator/role-helpers.mjs

@@ -105,6 +105,12 @@ export function isSecurityReviewAgent(
   return capabilities.includes("security-review");
 }
 
+export function isSecurityReviewAgentForLane(agent, lanePaths = null) {
+  return isSecurityReviewAgent(agent, {
+    securityRolePromptPath: lanePaths?.securityRolePromptPath,
+  });
+}
+
 export function isDesignAgent(
   agent,
   { designRolePromptPath = DEFAULT_DESIGN_ROLE_PROMPT_PATH } = {},
@@ -202,6 +208,30 @@ export function resolveWaveRoleBindings(wave = {}, lanePaths = {}, agents = wave
   };
 }
 
+export function resolveAgentClosureRoleKeys(agent, roleBindings = {}, lanePaths = {}) {
+  const roles = [];
+  if (agent?.agentId === roleBindings?.contEvalAgentId) {
+    roles.push("cont-eval");
+  }
+  if (
+    isSecurityReviewAgent(agent, {
+      securityRolePromptPath: lanePaths?.securityRolePromptPath,
+    })
+  ) {
+    roles.push("security-review");
+  }
+  if (agent?.agentId === roleBindings?.integrationAgentId) {
+    roles.push("integration");
+  }
+  if (agent?.agentId === roleBindings?.documentationAgentId) {
+    roles.push("documentation");
+  }
+  if (agent?.agentId === roleBindings?.contQaAgentId) {
+    roles.push("cont-qa");
+  }
+  return roles;
+}
+
 export function isClosureRoleAgentId(agentId, roleBindings) {
   return (roleBindings?.closureAgentIds || []).includes(agentId);
 }

package/scripts/wave-orchestrator/session-supervisor.mjs

@@ -1,4 +1,3 @@
-import { spawnSync } from "node:child_process";
 import fs from "node:fs";
 import path from "node:path";
 import {
@@ -22,7 +21,6 @@ import {
   ensureDirectory,
   shellQuote,
   PACKAGE_ROOT,
-  TMUX_COMMAND_TIMEOUT_MS,
   toIsoTimestamp,
   writeJsonAtomic,
 } from "./shared.mjs";
@@ -32,15 +30,25 @@ import {
   terminalSurfaceUsesTerminalRegistry,
   pruneOrphanLaneTemporaryTerminalEntries,
 } from "./terminals.mjs";
+import {
+  createSession as createTmuxSession,
+  listSessions as listTmuxSessions,
+  runTmuxCommand,
+} from "./tmux-adapter.mjs";
 import {
   recordGlobalDashboardEvent,
 } from "./dashboard-state.mjs";
 import { buildHumanFeedbackWorkflowUpdate } from "./human-input-workflow.mjs";
 import {
-  collectUnexpectedSessionFailures as collectUnexpectedSessionFailuresImpl,
+  collectUnexpectedSessionWarnings as collectUnexpectedSessionWarningsImpl,
   launchAgentSession as launchAgentSessionImpl,
   waitForWaveCompletion as waitForWaveCompletionImpl,
 } from "./launcher-runtime.mjs";
+import { terminateAgentProcessRuntime } from "./agent-process-runner.mjs";
+import {
+  buildSupervisorPaths,
+  supervisorAgentRuntimePathForRun,
+} from "./supervisor-cli.mjs";
 import {
   agentUsesSignalHygiene,
   buildSignalStatusLine,
@@ -65,6 +73,13 @@ function relativeArtifactPath(filePath) {
   return filePath ? path.relative(REPO_ROOT, filePath) : null;
 }
 
+function readRuntimeRecord(run) {
+  if (!run?.runtimePath || !fs.existsSync(run.runtimePath)) {
+    return null;
+  }
+  return readJsonOrNull(run.runtimePath);
+}
+
 export function recordWaveRunState(lanePaths, waveNumber, state, data = {}) {
   return appendWaveControlEvent(lanePaths, waveNumber, {
     entityType: "wave_run",
@@ -235,8 +250,8 @@ function isLaneSessionName(lanePaths, sessionName) {
   );
 }
 
-function listLaneTmuxSessionNames(lanePaths) {
-  return listTmuxSessionNames(lanePaths).filter((sessionName) =>
+async function listLaneTmuxSessionNames(lanePaths) {
+  return (await listTmuxSessionNames(lanePaths)).filter((sessionName) =>
     isLaneSessionName(lanePaths, sessionName),
   );
 }
@@ -362,6 +377,7 @@ export function buildResidentOrchestratorRun({
       promptPath: path.join(lanePaths.promptsDir, `${baseName}.prompt.md`),
       logPath: path.join(lanePaths.logsDir, `${baseName}.log`),
       statusPath: path.join(lanePaths.statusDir, `${baseName}.status`),
+      runtimePath: path.join(lanePaths.statusDir, `${baseName}.runtime.json`),
       promptOverride: buildResidentOrchestratorPrompt({
         lane: lanePaths.lane,
         wave: wave.wave,
@@ -415,20 +431,28 @@ export function monitorResidentOrchestratorSession({
     });
     return true;
   }
-  const activeSessions = new Set(listLaneTmuxSessionNames(lanePaths));
-  if (!activeSessions.has(run.sessionName)) {
+  const runtimeRecord = readRuntimeRecord(run);
+  if (
+    runtimeRecord &&
+    ["completed", "failed", "terminated"].includes(
+      String(runtimeRecord.terminalDisposition || ""),
+    )
+  ) {
     sessionState.closed = true;
+    const exitCode = Number.parseInt(String(runtimeRecord.exitCode ?? ""), 10);
     recordCombinedEvent({
-      level: "warn",
+      level: Number.isFinite(exitCode) && exitCode === 0 ? "info" : "warn",
       agentId: run.agent.agentId,
       message:
-        "Resident orchestrator session disappeared before writing a status file; launcher continues as the control plane.",
+        Number.isFinite(exitCode) && exitCode === 0
+          ? "Resident orchestrator ended via runtime record before writing a status file; launcher continues as the control plane."
+          : "Resident orchestrator ended via runtime record before writing a status file; launcher continues as the control plane.",
     });
     appendCoordination({
-      event: "resident_orchestrator_missing",
+      event: "resident_orchestrator_runtime_terminal",
       waves: [waveNumber],
-      status: "warn",
-      details: `tmux session ${run.sessionName} disappeared before ${path.relative(REPO_ROOT, run.statusPath)} was written.`,
+      status: Number.isFinite(exitCode) && exitCode === 0 ? "resolved" : "warn",
+      details: `runtime record reached ${runtimeRecord.terminalDisposition} before ${path.relative(REPO_ROOT, run.statusPath)} was written.`,
       actionRequested: "None",
     });
     return true;
@@ -505,7 +529,7 @@ export function pruneDryRunExecutorPreviewDirs(lanePaths, waves) {
   return removedPaths.toSorted();
 }
 
-export function reconcileStaleLauncherArtifacts(lanePaths, options = {}) {
+export async function reconcileStaleLauncherArtifacts(lanePaths, options = {}) {
   const outcome = {
     removedLock: false,
     removedSessions: [],
@@ -527,8 +551,8 @@ export function reconcileStaleLauncherArtifacts(lanePaths, options = {}) {
     outcome.removedLock = true;
   }
 
-  outcome.removedSessions = cleanupLaneTmuxSessions(lanePaths);
-  const activeSessionNames = new Set(listLaneTmuxSessionNames(lanePaths));
+  outcome.removedSessions = await cleanupLaneTmuxSessions(lanePaths);
+  const activeSessionNames = new Set(await listLaneTmuxSessionNames(lanePaths));
   if (terminalSurfaceUsesTerminalRegistry(options.terminalSurface || "vscode")) {
     const terminalCleanup = pruneOrphanLaneTemporaryTerminalEntries(
       lanePaths.terminalsPath,
@@ -562,87 +586,35 @@ export function reconcileStaleLauncherArtifacts(lanePaths, options = {}) {
 }
 
 export function runTmux(lanePaths, args, description) {
-  const result = spawnSync("tmux", ["-L", lanePaths.tmuxSocketName, ...args], {
-    cwd: REPO_ROOT,
-    encoding: "utf8",
-    env: { ...process.env, TMUX: "" },
-    timeout: TMUX_COMMAND_TIMEOUT_MS,
+  return runTmuxCommand(lanePaths.tmuxSocketName, args, {
+    description,
+    mutate: ["new-session", "kill-session"].includes(String(args?.[0] || "")),
   });
-  if (result.error) {
-    if (result.error.code === "ETIMEDOUT") {
-      throw new Error(
-        `${description} failed: tmux command timed out after ${TMUX_COMMAND_TIMEOUT_MS}ms`,
-      );
-    }
-    throw new Error(`${description} failed: ${result.error.message}`);
-  }
-  if (result.status !== 0) {
-    throw new Error(
-      `${description} failed: ${(result.stderr || "").trim() || "tmux command failed"}`,
-    );
-  }
 }
 
 function listTmuxSessionNames(lanePaths) {
-  const result = spawnSync(
-    "tmux",
-    ["-L", lanePaths.tmuxSocketName, "list-sessions", "-F", "#{session_name}"],
-    {
-      cwd: REPO_ROOT,
-      encoding: "utf8",
-      env: { ...process.env, TMUX: "" },
-      timeout: TMUX_COMMAND_TIMEOUT_MS,
-    },
-  );
-  if (result.error) {
-    if (result.error.code === "ENOENT") {
-      return [];
-    }
-    if (result.error.code === "ETIMEDOUT") {
-      throw new Error(`list tmux sessions failed: timed out after ${TMUX_COMMAND_TIMEOUT_MS}ms`);
-    }
-    throw new Error(`list tmux sessions failed: ${result.error.message}`);
-  }
-  if (result.status !== 0) {
-    const combined = `${String(result.stderr || "").toLowerCase()}\n${String(result.stdout || "").toLowerCase()}`;
-    if (
-      combined.includes("no server running") ||
-      combined.includes("failed to connect") ||
-      combined.includes("error connecting")
-    ) {
-      return [];
-    }
-    throw new Error(
-      `list tmux sessions failed: ${(result.stderr || "").trim() || "unknown error"}`,
-    );
-  }
-  return String(result.stdout || "")
-    .split(/\r?\n/)
-    .map((line) => line.trim())
-    .filter(Boolean);
+  return listTmuxSessions(lanePaths.tmuxSocketName);
 }
 
-export function cleanupLaneTmuxSessions(lanePaths, { excludeSessionNames = new Set() } = {}) {
-  const sessionNames = listTmuxSessionNames(lanePaths);
+export async function cleanupLaneTmuxSessions(lanePaths, { excludeSessionNames = new Set() } = {}) {
+  const sessionNames = await listTmuxSessionNames(lanePaths);
   const killed = [];
   for (const sessionName of sessionNames) {
     if (excludeSessionNames.has(sessionName) || !isLaneSessionName(lanePaths, sessionName)) {
       continue;
     }
-    killTmuxSessionIfExists(lanePaths.tmuxSocketName, sessionName);
+    await killTmuxSessionIfExists(lanePaths.tmuxSocketName, sessionName);
     killed.push(sessionName);
   }
   return killed;
 }
 
-export function collectUnexpectedSessionFailures(lanePaths, agentRuns, pendingAgentIds) {
-  return collectUnexpectedSessionFailuresImpl(lanePaths, agentRuns, pendingAgentIds, {
-    listLaneTmuxSessionNamesFn: listLaneTmuxSessionNames,
-  });
+export function collectUnexpectedSessionWarnings(lanePaths, agentRuns, pendingAgentIds) {
+  return collectUnexpectedSessionWarningsImpl(lanePaths, agentRuns, pendingAgentIds, {});
 }
 
-export function launchWaveDashboardSession(lanePaths, { sessionName, dashboardPath, messageBoardPath }) {
-  killTmuxSessionIfExists(lanePaths.tmuxSocketName, sessionName);
+export async function launchWaveDashboardSession(lanePaths, { sessionName, dashboardPath, messageBoardPath }) {
+  await killTmuxSessionIfExists(lanePaths.tmuxSocketName, sessionName);
   const messageBoardArg = messageBoardPath
     ? ` --message-board ${shellQuote(messageBoardPath)}`
     : "";
@@ -653,15 +625,31 @@ export function launchWaveDashboardSession(lanePaths, { sessionName, dashboardPa
     )}${messageBoardArg} --lane ${shellQuote(lanePaths.lane)} --watch`,
     "exec bash -l",
   ].join("; ");
-  runTmux(
-    lanePaths,
-    ["new-session", "-d", "-s", sessionName, `bash -lc ${shellQuote(command)}`],
-    `launch dashboard session ${sessionName}`,
+  await createTmuxSession(
+    lanePaths.tmuxSocketName,
+    sessionName,
+    `bash -lc ${shellQuote(command)}`,
+    { description: `launch dashboard session ${sessionName}` },
   );
 }
 
 export async function launchAgentSession(lanePaths, params) {
-  const result = await launchAgentSessionImpl(lanePaths, params, { runTmuxFn: runTmux });
+  const supervisorRunId = String(process.env.WAVE_SUPERVISOR_RUN_ID || "").trim();
+  const runtimePath = supervisorRunId
+    ? supervisorAgentRuntimePathForRun(
+      buildSupervisorPaths(lanePaths),
+      supervisorRunId,
+      params?.agent?.agentId || "unknown-agent",
+    )
+    : params?.runtimePath || null;
+  const result = await launchAgentSessionImpl(
+    lanePaths,
+    {
+      ...params,
+      runtimePath,
+    },
+    { runTmuxFn: runTmux },
+  );
   const controlPlane = params?.controlPlane || null;
   if (!params?.dryRun && controlPlane?.waveNumber !== undefined && controlPlane?.attempt) {
     recordAgentRunStarted(lanePaths, {
@@ -669,11 +657,32 @@ export async function launchAgentSession(lanePaths, params) {
       attempt: controlPlane.attempt,
       runInfo: {
         ...params,
+        runtimePath,
         lastExecutorId: result?.executorId || params?.agent?.executorResolved?.id || null,
       },
     });
   }
-  return result;
+  return {
+    ...result,
+    runtimePath,
+  };
+}
+
+export async function cleanupLaunchedRun(
+  lanePaths,
+  run,
+  {
+    terminateRuntimeFn = terminateAgentProcessRuntime,
+    killSessionFn = killTmuxSessionIfExists,
+  } = {},
+) {
+  const runtimeRecord = readRuntimeRecord(run);
+  if (runtimeRecord && typeof runtimeRecord === "object") {
+    await terminateRuntimeFn(runtimeRecord);
+  }
+  if (run?.sessionName) {
+    await killSessionFn(lanePaths.tmuxSocketName, run.sessionName);
+  }
 }
 
 export async function waitForWaveCompletion(
@@ -684,7 +693,7 @@ export async function waitForWaveCompletion(
   options = {},
 ) {
   const result = await waitForWaveCompletionImpl(lanePaths, agentRuns, timeoutMinutes, onProgress, {
-    collectUnexpectedSessionFailuresFn: collectUnexpectedSessionFailures,
+    collectUnexpectedSessionWarningsFn: collectUnexpectedSessionWarnings,
   });
   const controlPlane = options?.controlPlane || null;
   if (controlPlane?.waveNumber !== undefined && controlPlane?.attempt) {