@chllming/wave-orchestration 0.9.0 → 0.9.2

package/scripts/wave-orchestrator/launcher-runtime.mjs

@@ -1,20 +1,31 @@
 import fs from "node:fs";
 import path from "node:path";
 import { buildExecutionPrompt } from "./coordination.mjs";
+import {
+  materializeWaveCorridorContext,
+  renderCorridorPromptContext,
+  waveCorridorContextPath,
+} from "./corridor.mjs";
 import {
   DEFAULT_AGENT_RATE_LIMIT_BASE_DELAY_SECONDS,
   DEFAULT_AGENT_RATE_LIMIT_MAX_DELAY_SECONDS,
   DEFAULT_WAIT_PROGRESS_INTERVAL_MS,
   REPO_ROOT,
   ensureDirectory,
+  readJsonOrNull,
   shellQuote,
+  sleep,
   writeJsonAtomic,
 } from "./shared.mjs";
 import { readStatusCodeIfPresent } from "./dashboard-state.mjs";
 import { buildExecutorLaunchSpec } from "./executors.mjs";
 import { hashAgentPromptFingerprint, prefetchContext7ForSelection } from "./context7.mjs";
-import { killTmuxSessionIfExists } from "./terminals.mjs";
-import { isDesignAgent, resolveDesignReportPath, resolveWaveRoleBindings } from "./role-helpers.mjs";
+import {
+  isDesignAgent,
+  isSecurityReviewAgent,
+  resolveDesignReportPath,
+  resolveWaveRoleBindings,
+} from "./role-helpers.mjs";
 import {
   resolveAgentSkills,
   summarizeResolvedSkills,
@@ -25,6 +36,48 @@ import {
   agentSignalPath,
   agentUsesSignalHygiene,
 } from "./signals.mjs";
+import {
+  spawnAgentProcessRunner,
+  terminateAgentProcessRuntime,
+} from "./agent-process-runner.mjs";
+import {
+  requestWaveControlCredentialEnv,
+  requestWaveControlProviderEnv,
+} from "./provider-runtime.mjs";
+
+function redactPreviewEnv(env = {}, redactedKeys = []) {
+  const output = { ...(env || {}) };
+  for (const key of redactedKeys) {
+    if (Object.prototype.hasOwnProperty.call(output, key)) {
+      output[key] = "[redacted]";
+    }
+  }
+  return output;
+}
+
+function buildDryRunContext7Preview(selection) {
+  if (
+    !selection ||
+    selection.bundleId === "none" ||
+    !Array.isArray(selection.libraries) ||
+    selection.libraries.length === 0
+  ) {
+    return {
+      mode: "none",
+      selection,
+      promptText: "",
+      snippetHash: "",
+      warning: "",
+    };
+  }
+  return {
+    mode: "dry-run",
+    selection,
+    promptText: "",
+    snippetHash: "",
+    warning: "Context7 prefetch skipped during dry-run preview.",
+  };
+}
 
 export function refreshResolvedSkillsForRun(runInfo, waveDefinition, lanePaths) {
   runInfo.agent.skillsResolved = resolveAgentSkills(
@@ -35,32 +88,67 @@ export function refreshResolvedSkillsForRun(runInfo, waveDefinition, lanePaths)
   return runInfo.agent.skillsResolved;
 }
 
-export function collectUnexpectedSessionFailures(
+export function applyLaunchResultToRun(
+  runInfo,
+  launchResult,
+  {
+    attempt = null,
+    fallbackExecutorId = null,
+    fallbackSkills = null,
+  } = {},
+) {
+  if (!runInfo || !launchResult) {
+    return runInfo;
+  }
+  if (attempt !== null && attempt !== undefined) {
+    runInfo.lastLaunchAttempt = attempt;
+  }
+  runInfo.lastPromptHash = launchResult.promptHash || null;
+  runInfo.lastContext7 = launchResult.context7 || null;
+  runInfo.lastExecutorId = launchResult.executorId || fallbackExecutorId || null;
+  runInfo.lastSkillProjection = launchResult.skills || fallbackSkills || null;
+  runInfo.runtimePath = launchResult.runtimePath || runInfo.runtimePath || null;
+  runInfo.sessionBackend = launchResult.sessionBackend || runInfo.sessionBackend || "process";
+  runInfo.attachMode = launchResult.attachMode || runInfo.attachMode || "log-tail";
+  return runInfo;
+}
+
+export function collectUnexpectedSessionWarnings(
   lanePaths,
   agentRuns,
   pendingAgentIds,
   { listLaneTmuxSessionNamesFn },
 ) {
-  const activeSessionNames = new Set(listLaneTmuxSessionNamesFn(lanePaths));
-  const failures = [];
+  const warnings = [];
   for (const run of agentRuns) {
     if (!pendingAgentIds.has(run.agent.agentId) || fs.existsSync(run.statusPath)) {
       continue;
     }
-    if (activeSessionNames.has(run.sessionName)) {
+    if (!run.runtimePath || !fs.existsSync(run.runtimePath)) {
       continue;
     }
-    failures.push({
+    const runtimeRecord = JSON.parse(fs.readFileSync(run.runtimePath, "utf8"));
+    if (!runtimeRecord || typeof runtimeRecord !== "object") {
+      continue;
+    }
+    if (runtimeRecord.terminalDisposition !== "projection-missing") {
+      continue;
+    }
+    warnings.push({
       agentId: run.agent.agentId,
-      statusCode: "session-missing",
+      statusCode: "terminal-session-missing",
       logPath: path.relative(REPO_ROOT, run.logPath),
-      detail: `tmux session ${run.sessionName} disappeared before ${path.relative(REPO_ROOT, run.statusPath)} was written.`,
+      detail: `terminal projection for ${run.sessionName} disappeared before ${path.relative(REPO_ROOT, run.statusPath)} was written.`,
     });
   }
-  return failures;
+  return warnings;
 }
 
-export async function launchAgentSession(lanePaths, params, { runTmuxFn }) {
+export async function launchAgentSession(
+  lanePaths,
+  params,
+  { spawnRunnerFn = spawnAgentProcessRunner } = {},
+) {
   const {
     wave,
     waveDefinition = null,
@@ -84,19 +172,48 @@ export async function launchAgentSession(lanePaths, params, { runTmuxFn }) {
     context7Enabled,
     designExecutionMode = null,
     dryRun = false,
+    runtimePath = null,
   } = params;
   ensureDirectory(path.dirname(promptPath));
   ensureDirectory(path.dirname(logPath));
   ensureDirectory(path.dirname(statusPath));
+  if (runtimePath && fs.existsSync(runtimePath)) {
+    const priorRuntime = readJsonOrNull(runtimePath);
+    if (priorRuntime && typeof priorRuntime === "object") {
+      await terminateAgentProcessRuntime(priorRuntime);
+    }
+  }
   fs.rmSync(statusPath, { force: true });
+  if (runtimePath) {
+    ensureDirectory(path.dirname(runtimePath));
+    fs.rmSync(runtimePath, { force: true });
+  }
 
-  const context7 = await prefetchContext7ForSelection(agent.context7Resolved, {
-    cacheDir: lanePaths.context7CacheDir,
-    disabled: !context7Enabled,
-  });
+  const resolvedWaveDefinition = waveDefinition || { deployEnvironments: [] };
+  const context7 = dryRun
+    ? buildDryRunContext7Preview(agent.context7Resolved || null)
+    : await prefetchContext7ForSelection(agent.context7Resolved, {
+        lanePaths,
+        cacheDir: lanePaths.context7CacheDir,
+        disabled: !context7Enabled,
+      });
+  const integrationAgentId =
+    waveDefinition?.integrationAgentId || lanePaths.integrationAgentId || "A8";
+  const shouldLoadCorridorContext =
+    lanePaths.externalProviders?.corridor?.enabled === true &&
+    (isSecurityReviewAgent(agent) || agent.agentId === integrationAgentId);
+  const corridorContext = !dryRun && shouldLoadCorridorContext
+    ? await materializeWaveCorridorContext(lanePaths, resolvedWaveDefinition)
+    : null;
+  const corridorContextPath = !dryRun && shouldLoadCorridorContext
+    ? waveCorridorContextPath(lanePaths, wave)
+    : null;
+  const corridorContextText =
+    dryRun && shouldLoadCorridorContext
+      ? "Corridor context omitted in dry-run preview."
+      : renderCorridorPromptContext(corridorContext);
   const overlayDir = path.join(lanePaths.executorOverlaysDir, `wave-${wave}`, agent.slug);
   ensureDirectory(overlayDir);
-  const resolvedWaveDefinition = waveDefinition || { deployEnvironments: [] };
   const skillsResolved =
     agent.skillsResolved ||
     resolveAgentSkills(agent, resolvedWaveDefinition, {
@@ -125,6 +242,8 @@ export async function launchAgentSession(lanePaths, params, { runTmuxFn }) {
       inboxPath,
       inboxText,
       context7,
+      corridorContextPath,
+      corridorContextText,
       componentPromotions: resolvedWaveDefinition.componentPromotions,
       evalTargets: resolvedWaveDefinition.evalTargets,
       benchmarkCatalogPath: lanePaths.laneProfile?.paths?.benchmarkCatalogPath,
@@ -150,11 +269,45 @@ export async function launchAgentSession(lanePaths, params, { runTmuxFn }) {
     overlayDir,
     skillProjection: agent.skillsResolved,
   });
+  const requestedCredentialProviders = Array.isArray(lanePaths.waveControl?.credentialProviders)
+    ? lanePaths.waveControl.credentialProviders
+    : [];
+  const requestedCredentials = Array.isArray(lanePaths.waveControl?.credentials)
+    ? lanePaths.waveControl.credentials
+    : [];
+  const leasedProviderEnv =
+    !dryRun && requestedCredentialProviders.length > 0
+      ? await requestWaveControlProviderEnv(fetch, lanePaths.waveControl, requestedCredentialProviders)
+      : {};
+  const leasedCredentialEnv =
+    !dryRun && requestedCredentials.length > 0
+      ? await requestWaveControlCredentialEnv(fetch, lanePaths.waveControl, requestedCredentials)
+      : {};
+  const overlappingLeasedEnvVars = Object.keys(leasedProviderEnv).filter((key) =>
+    Object.prototype.hasOwnProperty.call(leasedCredentialEnv, key),
+  );
+  if (overlappingLeasedEnvVars.length > 0) {
+    throw new Error(
+      `Wave Control leased duplicate environment variables: ${overlappingLeasedEnvVars.join(", ")}.`,
+    );
+  }
+  const leasedEnv = {
+    ...leasedProviderEnv,
+    ...leasedCredentialEnv,
+  };
+  if (Object.keys(leasedEnv).length > 0) {
+    launchSpec.env = {
+      ...(launchSpec.env || {}),
+      ...leasedEnv,
+    };
+  }
   const resolvedExecutorMode = launchSpec.executorId || agent.executorResolved?.id || "codex";
   writeJsonAtomic(path.join(overlayDir, "launch-preview.json"), {
     executorId: resolvedExecutorMode,
     command: launchSpec.command,
-    env: launchSpec.env || {},
+    env: redactPreviewEnv(launchSpec.env || {}, Object.keys(leasedEnv)),
+    credentialProviders: requestedCredentialProviders,
+    credentials: requestedCredentials,
     useRateLimitRetries: launchSpec.useRateLimitRetries === true,
     invocationLines: launchSpec.invocationLines,
     limits: launchSpec.limits || null,
@@ -164,13 +317,13 @@ export async function launchAgentSession(lanePaths, params, { runTmuxFn }) {
     return {
       promptHash,
       context7,
+      corridorContext,
       executorId: resolvedExecutorMode,
       launchSpec,
       dryRun: true,
       skills: summarizeResolvedSkills(agent.skillsResolved),
     };
   }
-  killTmuxSessionIfExists(lanePaths.tmuxSocketName, sessionName);
 
   const executionLines = [];
   if (launchSpec.env) {
@@ -195,6 +348,9 @@ export async function launchAgentSession(lanePaths, params, { runTmuxFn }) {
     executionLines.push("rate_attempt=1");
     executionLines.push("status=1");
     executionLines.push('while [ "$rate_attempt" -le "$max_rate_attempts" ]; do');
+    executionLines.push(
+      `  attempt_log_offset=$(wc -c < ${shellQuote(logPath)} 2>/dev/null || echo 0)`,
+    );
     for (const line of launchSpec.invocationLines) {
       executionLines.push(`  ${line}`);
     }
@@ -206,7 +362,7 @@ export async function launchAgentSession(lanePaths, params, { runTmuxFn }) {
     executionLines.push("    break");
     executionLines.push("  fi");
     executionLines.push(
-      `  if tail -n 120 ${shellQuote(logPath)} | grep -Eqi '429 Too Many Requests|exceeded retry limit|last status: 429|rate limit'; then`,
+      `  if tail -c +$((attempt_log_offset + 1)) ${shellQuote(logPath)} | grep -Eqi '429 Too Many Requests|exceeded retry limit|last status: 429|rate limit'; then`,
     );
     executionLines.push("    sleep_seconds=$((rate_delay_base * (2 ** (rate_attempt - 1))))");
     executionLines.push(
@@ -231,23 +387,67 @@ export async function launchAgentSession(lanePaths, params, { runTmuxFn }) {
     `export WAVE_ORCHESTRATOR_ID=${shellQuote(orchestratorId || "")}`,
     `export WAVE_EXECUTOR_MODE=${shellQuote(resolvedExecutorMode)}`,
     ...executionLines,
-    `node -e ${shellQuote(
-      "const fs=require('node:fs'); const statusPath=process.argv[1]; const payload={code:Number(process.argv[2]),promptHash:process.argv[3]||null,orchestratorId:process.argv[4]||null,attempt:Number(process.argv[5])||1,completedAt:new Date().toISOString()}; fs.writeFileSync(statusPath, JSON.stringify(payload, null, 2)+'\\n', 'utf8');",
-    )} ${shellQuote(statusPath)} "$status" ${shellQuote(promptHash)} ${shellQuote(orchestratorId || "")} ${shellQuote(String(attempt || 1))}`,
-    `echo "[${lanePaths.lane}-wave-launcher] ${sessionName} finished with code $status"`,
-    "exit \"$status\"",
   ].join("\n");
-
-  runTmuxFn(
-    lanePaths,
-    ["new-session", "-d", "-s", sessionName, `bash -lc ${shellQuote(command)}`],
-    `launch session ${sessionName}`,
-  );
+  const payloadPath = path.join(overlayDir, "runner-payload.json");
+  const initialRuntimeRecord = runtimePath
+    ? {
+      runId: process.env.WAVE_SUPERVISOR_RUN_ID || null,
+      waveNumber: wave,
+      attempt: Number(attempt || 1),
+      agentId: agent.agentId,
+      sessionName,
+      tmuxSessionName: null,
+      sessionBackend: "process",
+      attachMode: "log-tail",
+      runnerPid: null,
+      executorPid: null,
+      pid: null,
+      pgid: null,
+      startedAt: new Date().toISOString(),
+      lastHeartbeatAt: new Date().toISOString(),
+      statusPath,
+      logPath,
+      exitCode: null,
+      exitReason: null,
+      terminalDisposition: "launching",
+    }
+    : null;
+  if (runtimePath && initialRuntimeRecord) {
+    writeJsonAtomic(runtimePath, initialRuntimeRecord);
+  }
+  const runner = spawnRunnerFn({
+    payloadPath,
+    runId: process.env.WAVE_SUPERVISOR_RUN_ID || null,
+    lane: lanePaths.lane,
+    waveNumber: wave,
+    attempt: Number(attempt || 1),
+    agentId: agent.agentId,
+    sessionName,
+    runtimePath,
+    statusPath,
+    logPath,
+    promptHash,
+    orchestratorId: orchestratorId || "",
+    executorId: resolvedExecutorMode,
+    env: launchSpec.env || {},
+    command,
+  });
+  if (runtimePath && initialRuntimeRecord) {
+    writeJsonAtomic(runtimePath, {
+      ...initialRuntimeRecord,
+      runnerPid: runner?.runnerPid || null,
+      lastHeartbeatAt: new Date().toISOString(),
+    });
+  }
   return {
     promptHash,
     context7,
+    corridorContext,
     executorId: resolvedExecutorMode,
     skills: summarizeResolvedSkills(agent.skillsResolved),
+    runtimePath,
+    sessionBackend: "process",
+    attachMode: "log-tail",
   };
 }
 
@@ -256,7 +456,7 @@ export async function waitForWaveCompletion(
   agentRuns,
   timeoutMinutes,
   onProgress = null,
-  { collectUnexpectedSessionFailuresFn },
+  { collectUnexpectedSessionWarningsFn = () => [] },
 ) {
   const defaultTimeoutMs = timeoutMinutes * 60 * 1000;
   const startedAt = Date.now();
@@ -272,8 +472,7 @@ export async function waitForWaveCompletion(
   );
   const pending = new Set(agentRuns.map((run) => run.agent.agentId));
   const timedOutAgentIds = new Set();
-  let sessionFailures = [];
-
+  let sessionWarnings = [];
   const refreshPending = () => {
     for (const run of agentRuns) {
       if (pending.has(run.agent.agentId) && fs.existsSync(run.statusPath)) {
@@ -282,51 +481,58 @@ export async function waitForWaveCompletion(
     }
   };
 
-  await new Promise((resolve) => {
-    const interval = setInterval(() => {
-      refreshPending();
-      onProgress?.({ pendingAgentIds: new Set(pending), timedOut: false });
-      if (pending.size === 0) {
-        clearInterval(interval);
-        resolve();
-        return;
-      }
-      sessionFailures = collectUnexpectedSessionFailuresFn(lanePaths, agentRuns, pending);
-      if (sessionFailures.length > 0) {
-        onProgress?.({
-          pendingAgentIds: new Set(pending),
-          timedOut: false,
-          failures: sessionFailures,
-        });
-        clearInterval(interval);
-        resolve();
-        return;
+  while (true) {
+    refreshPending();
+    onProgress?.({ pendingAgentIds: new Set(pending), timedOut: false });
+    if (pending.size === 0) {
+      break;
+    }
+    sessionWarnings = collectUnexpectedSessionWarningsFn(lanePaths, agentRuns, pending);
+    if (sessionWarnings.length > 0) {
+      onProgress?.({
+        pendingAgentIds: new Set(pending),
+        timedOut: false,
+        warnings: sessionWarnings,
+      });
+    }
+    const now = Date.now();
+    for (const run of agentRuns) {
+      if (!pending.has(run.agent.agentId)) {
+        continue;
       }
-      const now = Date.now();
-      for (const run of agentRuns) {
-        if (!pending.has(run.agent.agentId)) {
-          continue;
-        }
-        const deadline = timeoutAtByAgentId.get(run.agent.agentId) || startedAt + defaultTimeoutMs;
-        if (now <= deadline) {
-          continue;
+      if (run.runtimePath && fs.existsSync(run.runtimePath)) {
+        try {
+          const runtimeRecord = readJsonOrNull(run.runtimePath);
+          if (
+            runtimeRecord &&
+            typeof runtimeRecord === "object" &&
+            ["completed", "failed", "terminated"].includes(
+              String(runtimeRecord.terminalDisposition || ""),
+            )
+          ) {
+            pending.delete(run.agent.agentId);
+            continue;
+          }
+        } catch {
+          // best-effort runtime observation only
         }
-        timedOutAgentIds.add(run.agent.agentId);
-        pending.delete(run.agent.agentId);
-        killTmuxSessionIfExists(lanePaths.tmuxSocketName, run.sessionName);
       }
-      if (pending.size === 0) {
-        clearInterval(interval);
-        resolve();
+      const deadline = timeoutAtByAgentId.get(run.agent.agentId) || startedAt + defaultTimeoutMs;
+      if (now <= deadline) {
+        continue;
       }
-    }, DEFAULT_WAIT_PROGRESS_INTERVAL_MS);
-    refreshPending();
-    onProgress?.({ pendingAgentIds: new Set(pending), timedOut: false });
-  });
-
-  if (sessionFailures.length > 0) {
-    onProgress?.({ pendingAgentIds: new Set(), timedOut: false, failures: sessionFailures });
-    return { failures: sessionFailures, timedOut: false };
+      timedOutAgentIds.add(run.agent.agentId);
+      pending.delete(run.agent.agentId);
+      const runtimeRecord =
+        run.runtimePath && fs.existsSync(run.runtimePath) ? readJsonOrNull(run.runtimePath) : null;
+      if (runtimeRecord) {
+        await terminateAgentProcessRuntime(runtimeRecord);
+      }
+    }
+    if (pending.size === 0) {
+      break;
+    }
+    await sleep(DEFAULT_WAIT_PROGRESS_INTERVAL_MS);
   }
 
   const failures = [];
@@ -336,10 +542,19 @@ export async function waitForWaveCompletion(
       continue;
     }
     if (code === null || timedOutAgentIds.has(run.agent.agentId)) {
+      let runtimeRecord = null;
+      if (run.runtimePath && fs.existsSync(run.runtimePath)) {
+        runtimeRecord = readJsonOrNull(run.runtimePath);
+      }
       failures.push({
         agentId: run.agent.agentId,
-        statusCode: timedOutAgentIds.has(run.agent.agentId) ? "timeout-no-status" : "missing-status",
+        statusCode: timedOutAgentIds.has(run.agent.agentId)
+          ? "timeout-no-status"
+          : runtimeRecord?.terminalDisposition === "failed"
+            ? "runtime-failed-before-status"
+            : "missing-status",
         logPath: path.relative(REPO_ROOT, run.logPath),
+        detail: runtimeRecord?.exitReason || null,
       });
       continue;
     }