@moretea-labs/forge 1.6.0 → 1.7.1

package/src/runtime/control-plane/launcher/launch-reservation-store.ts

@@ -5,6 +5,7 @@ import {
   writeControlPlaneRecord,
 } from '../persistence/sqlite-store';
 import type { ControllerType } from '../facade/types';
+import { redactProcessOutput } from '../../../effects/process-runner';
 
 const NAMESPACE = 'external_controller_launch_reservation';
 const DEFAULT_TTL_MS = 2 * 60_000;
@@ -19,8 +20,14 @@ export interface ExternalControllerLaunchReservation {
   createdAt: string;
   expiresAt: string;
   pid?: number;
+  exitedAt?: string;
+  exitCode?: number;
+  exitSignal?: string;
   releasedAt?: string;
   releaseReason?: string;
+  /** Bounded, redacted provider output captured only during launcher startup. */
+  stdoutTail?: string;
+  stderrTail?: string;
 }
 
 export interface LaunchReservationStoreOptions {
@@ -54,6 +61,14 @@ export function getExternalControllerLaunchReservation(
   return active(record(options, workId)?.value);
 }
 
+/** Read the latest reservation record including released/failed startup evidence. */
+export function readExternalControllerLaunchReservationRecord(
+  options: LaunchReservationStoreOptions,
+  workId: string,
+): ExternalControllerLaunchReservation | undefined {
+  return record(options, workId)?.value;
+}
+
 export function reserveExternalControllerLaunch(
   options: LaunchReservationStoreOptions,
   input: {
@@ -128,6 +143,80 @@ export function attachExternalControllerLaunchPid(
   );
 }
 
+export function recordExternalControllerLaunchDiagnostics(
+  options: LaunchReservationStoreOptions,
+  workId: string,
+  reservationId: string,
+  diagnostics: { stdoutTail?: string; stderrTail?: string },
+): ExternalControllerLaunchReservation | undefined {
+  return withControllerLock(
+    options.controllerHome,
+    { scope: 'task', repoId: options.repoId, taskId: `controller-launch-${workId}` },
+    `controller-launch-diagnostics:${reservationId}`,
+    () => {
+      const existing = record(options, workId);
+      if (!existing || existing.value.reservationId !== reservationId) return existing?.value;
+      const next: ExternalControllerLaunchReservation = {
+        ...existing.value,
+        ...(diagnostics.stdoutTail ? { stdoutTail: redactProcessOutput(diagnostics.stdoutTail).slice(-8 * 1024) } : {}),
+        ...(diagnostics.stderrTail ? { stderrTail: redactProcessOutput(diagnostics.stderrTail).slice(-8 * 1024) } : {}),
+      };
+      writeControlPlaneRecord(options.controllerHome, {
+        namespace: NAMESPACE,
+        scope: options.repoId,
+        key: workId,
+        schemaVersion: 1,
+        value: next,
+        action: 'external_controller_launch_diagnostics',
+        expectedRevision: existing.revision,
+      });
+      return next;
+    },
+  );
+}
+
+export function recordExternalControllerLaunchExit(
+  options: LaunchReservationStoreOptions,
+  workId: string,
+  reservationId: string,
+  exit: { exitCode: number | null; signal: string | null; stdoutTail?: string; stderrTail?: string },
+): ExternalControllerLaunchReservation | undefined {
+  return withControllerLock(
+    options.controllerHome,
+    { scope: 'task', repoId: options.repoId, taskId: `controller-launch-${workId}` },
+    `controller-launch-exit:${reservationId}`,
+    () => {
+      const existing = record(options, workId);
+      if (!existing || existing.value.reservationId !== reservationId) return existing?.value;
+      const exitedAt = nowIso(options);
+      const alreadyReleased = Boolean(existing.value.releasedAt);
+      const next: ExternalControllerLaunchReservation = {
+        ...existing.value,
+        expiresAt: alreadyReleased ? existing.value.expiresAt : exitedAt,
+        exitedAt,
+        ...(exit.exitCode === null ? {} : { exitCode: exit.exitCode }),
+        ...(exit.signal ? { exitSignal: exit.signal } : {}),
+        ...(exit.stdoutTail ? { stdoutTail: redactProcessOutput(exit.stdoutTail).slice(-8 * 1024) } : {}),
+        ...(exit.stderrTail ? { stderrTail: redactProcessOutput(exit.stderrTail).slice(-8 * 1024) } : {}),
+        ...(alreadyReleased ? {} : {
+          releasedAt: exitedAt,
+          releaseReason: `process_exit:code=${String(exit.exitCode ?? 'null')}:signal=${exit.signal ?? 'none'}`.slice(0, 240),
+        }),
+      };
+      writeControlPlaneRecord(options.controllerHome, {
+        namespace: NAMESPACE,
+        scope: options.repoId,
+        key: workId,
+        schemaVersion: 1,
+        value: next,
+        action: 'external_controller_launch_exit',
+        expectedRevision: existing.revision,
+      });
+      return next;
+    },
+  );
+}
+
 export function releaseExternalControllerLaunchReservation(
   options: LaunchReservationStoreOptions,
   workId: string,

package/src/runtime/control-plane/launcher/thin-launcher.ts

@@ -1,4 +1,4 @@
-import { spawn, spawnSync } from 'child_process';
+import { spawn, spawnSync, type ChildProcess } from 'child_process';
 import {
   getHandoffItem,
   getWorkContract,
@@ -7,12 +7,17 @@ import {
 } from '../facade';
 import {
   attachExternalControllerLaunchPid,
+  recordExternalControllerLaunchDiagnostics,
+  recordExternalControllerLaunchExit,
   releaseExternalControllerLaunchReservation,
   reserveExternalControllerLaunch,
 } from './launch-reservation-store';
 import type { ControllerType } from '../facade/types';
+import { getControllerSession } from '../facade/controller-session-store';
 import { codexMcpConfigArgs, resolveProviderMcpBootstrap, type ProviderMcpBootstrap } from './provider-mcp-bootstrap';
 import { getChatgptWorkConversationBinding } from './chatgpt-work-binding-store';
+import { repositoryChildProcessEnvironment } from '../../shared/process-environment';
+import { redactProcessOutput } from '../../../effects/process-runner';
 
 export interface ThinLauncherRequest {
   controllerType: Exclude<ControllerType, 'human'>;
@@ -42,7 +47,51 @@ export interface ThinLauncherResult {
   executable: string;
 }
 
-function resolveLauncherExecutable(request: ThinLauncherRequest): string {
+const LAUNCHER_STARTUP_GRACE_MS = 250;
+const CODEX_WORK_CLAIM_TIMEOUT_MS = 30_000;
+const CODEX_WORK_CLAIM_SETTLEMENT_GRACE_MS = 1_500;
+const CODEX_WORK_CLAIM_POLL_INTERVAL_MS = 100;
+const STARTUP_DIAGNOSTIC_BYTES = 8 * 1024;
+
+function appendStartupDiagnosticTail(current: string, chunk: unknown): string {
+  const text = redactProcessOutput(Buffer.isBuffer(chunk) ? chunk.toString('utf8') : String(chunk ?? ''));
+  const buffer = Buffer.from(`${current}${text}`, 'utf8');
+  return buffer.length <= STARTUP_DIAGNOSTIC_BYTES
+    ? buffer.toString('utf8')
+    : buffer.subarray(buffer.length - STARTUP_DIAGNOSTIC_BYTES).toString('utf8');
+}
+
+function startupDiagnosticSummary(stdoutTail: string, stderrTail: string): string {
+  const value = redactProcessOutput(stderrTail.trim() || stdoutTail.trim()).replace(/\s+/g, ' ').slice(0, 1200);
+  return value ? `; startup_output=${value}` : '';
+}
+
+interface ExternalControllerClaimExpectation {
+  controllerType: 'codex';
+  controllerId: string;
+  principalId: string;
+  sessionId: string;
+}
+
+export interface ThinLauncherDependencies {
+  resolveProviderMcpBootstrap?: typeof resolveProviderMcpBootstrap;
+  claimTimeoutMs?: number;
+  claimSettlementGraceMs?: number;
+  claimPollIntervalMs?: number;
+}
+
+function launcherProcessEnvironment(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
+  const repositoryEnv = repositoryChildProcessEnvironment(env);
+  return {
+    ...env,
+    PATH: repositoryEnv.PATH,
+  };
+}
+
+export function resolveLauncherExecutable(
+  request: ThinLauncherRequest,
+  env: NodeJS.ProcessEnv = process.env,
+): string {
   const configured = request.executable?.trim();
   if (configured) return configured;
   const executable = request.controllerType === 'codex'
@@ -50,13 +99,14 @@ function resolveLauncherExecutable(request: ThinLauncherRequest): string {
     : request.controllerType === 'claude'
       ? 'claude'
       : request.controllerType === 'chatgpt'
-        ? process.env.FORGE_CLI_EXECUTABLE?.trim() || 'forge'
+        ? env.FORGE_CLI_EXECUTABLE?.trim() || 'forge'
         : '';
   if (!executable) throw new Error(`LAUNCHER_EXECUTABLE_REQUIRED: ${request.controllerType} requires an external launcher executable`);
   const probe = spawnSync(executable, ['--version'], {
     cwd: request.cwd,
     stdio: 'ignore',
     timeout: 5_000,
+    env: launcherProcessEnvironment(env),
   });
   if (probe.error || probe.status !== 0) {
     throw new Error(`LAUNCHER_EXECUTABLE_UNAVAILABLE: ${executable}`);
@@ -64,6 +114,171 @@ function resolveLauncherExecutable(request: ThinLauncherRequest): string {
   return executable;
 }
 
+async function awaitExternalControllerStartup(
+  child: ChildProcess,
+  stores: { work: WorkContractStoreOptions & { controllerHome: string; repoId: string } },
+  workId: string,
+  reservationId: string,
+  claimExpectation?: ExternalControllerClaimExpectation,
+  options: { claimTimeoutMs?: number; claimSettlementGraceMs?: number; claimPollIntervalMs?: number } = {},
+): Promise<void> {
+  await new Promise<void>((resolve, reject) => {
+    let startupSettled = false;
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    let stdoutTail = '';
+    let stderrTail = '';
+    const captureStdout = (chunk: unknown) => { stdoutTail = appendStartupDiagnosticTail(stdoutTail, chunk); };
+    const captureStderr = (chunk: unknown) => { stderrTail = appendStartupDiagnosticTail(stderrTail, chunk); };
+    child.stdout?.on('data', captureStdout);
+    child.stderr?.on('data', captureStderr);
+    const closeStartupPipes = () => {
+      child.stdout?.off('data', captureStdout);
+      child.stderr?.off('data', captureStderr);
+      child.stdout?.destroy();
+      child.stderr?.destroy();
+    };
+    const persistDiagnostics = () => {
+      if (!stdoutTail && !stderrTail) return;
+      try {
+        recordExternalControllerLaunchDiagnostics(stores.work, workId, reservationId, { stdoutTail, stderrTail });
+      } catch {
+        // Startup diagnostics are evidence only; preserve the primary ownership/claim outcome.
+      }
+    };
+    const clearTimer = () => {
+      if (timer) clearTimeout(timer);
+      timer = undefined;
+    };
+    const releaseFailure = (reason: string) => {
+      try {
+        releaseExternalControllerLaunchReservation(stores.work, workId, reservationId, reason);
+      } catch {
+        // Preserve the primary startup failure if diagnostic persistence itself races or fails.
+      }
+    };
+    const terminateUnclaimedChild = () => {
+      if (!child.pid || child.exitCode !== null || child.signalCode !== null) return;
+      try { child.kill('SIGTERM'); } catch { /* process may already be exiting */ }
+    };
+    const fail = (error: Error, reason: string, terminate = false) => {
+      if (startupSettled) return;
+      startupSettled = true;
+      clearTimer();
+      persistDiagnostics();
+      closeStartupPipes();
+      if (terminate) terminateUnclaimedChild();
+      releaseFailure(reason);
+      const diagnostics = startupDiagnosticSummary(stdoutTail, stderrTail);
+      reject(diagnostics ? new Error(`${error.message}${diagnostics}`) : error);
+    };
+    const expectedClaimMatches = (): { matches: boolean; mismatch?: string } => {
+      if (!claimExpectation) return { matches: true };
+      const owner = getControllerSession(stores.work, workId);
+      if (!owner) return { matches: false };
+      const ownerPrincipal = owner.principalId?.trim() || owner.controllerId;
+      const matches = owner.controllerType === claimExpectation.controllerType
+        && owner.controllerId === claimExpectation.controllerId
+        && ownerPrincipal === claimExpectation.principalId
+        && owner.sessionId === claimExpectation.sessionId;
+      return matches
+        ? { matches: true }
+        : {
+          matches: false,
+          mismatch: `observed type=${owner.controllerType} controller=${owner.controllerId} principal=${ownerPrincipal} session=${owner.sessionId}`,
+        };
+    };
+
+    child.once('error', (error) => {
+      fail(new Error(`LAUNCHER_STARTUP_FAILED: ${error.message}`), `spawn_error:${error.message}`);
+    });
+
+    child.once('exit', (exitCode, signal) => {
+      try {
+        recordExternalControllerLaunchExit(stores.work, workId, reservationId, {
+          exitCode,
+          signal: signal ? String(signal) : null,
+          stdoutTail,
+          stderrTail,
+        });
+      } catch {
+        // Exit evidence is best-effort after another authority has already released the reservation.
+      }
+      if (startupSettled) return;
+      startupSettled = true;
+      clearTimer();
+      closeStartupPipes();
+      const phase = claimExpectation ? 'before exact Work claim became live' : 'during startup grace';
+      reject(new Error(`LAUNCHER_STARTUP_FAILED: external Controller exited ${phase} (code=${String(exitCode ?? 'null')}, signal=${signal ?? 'none'})${startupDiagnosticSummary(stdoutTail, stderrTail)}`));
+    });
+
+    if (!claimExpectation) {
+      timer = setTimeout(() => {
+        if (startupSettled) return;
+        startupSettled = true;
+        closeStartupPipes();
+        resolve();
+      }, LAUNCHER_STARTUP_GRACE_MS);
+      return;
+    }
+
+    const claimTimeoutMs = Math.max(100, Math.min(options.claimTimeoutMs ?? CODEX_WORK_CLAIM_TIMEOUT_MS, 60_000));
+    const claimSettlementGraceMs = Math.max(0, Math.min(options.claimSettlementGraceMs ?? CODEX_WORK_CLAIM_SETTLEMENT_GRACE_MS, 5_000));
+    const claimPollIntervalMs = Math.max(10, Math.min(options.claimPollIntervalMs ?? CODEX_WORK_CLAIM_POLL_INTERVAL_MS, 1_000));
+    const claimDeadline = Date.now() + claimTimeoutMs;
+    const claimSettlementDeadline = claimDeadline + claimSettlementGraceMs;
+    const pollClaim = () => {
+      if (startupSettled) return;
+      let observation: ReturnType<typeof expectedClaimMatches>;
+      try {
+        observation = expectedClaimMatches();
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        fail(new Error(`LAUNCHER_CLAIM_OBSERVATION_FAILED: ${message}`), `claim_observation_failed:${message}`, true);
+        return;
+      }
+      if (observation.mismatch) {
+        fail(
+          new Error(`LAUNCHER_CLAIM_MISMATCH: Work ${workId} expected type=${claimExpectation.controllerType} controller=${claimExpectation.controllerId} principal=${claimExpectation.principalId} session=${claimExpectation.sessionId}; ${observation.mismatch}`),
+          'claim_mismatch',
+          true,
+        );
+        return;
+      }
+      if (observation.matches) {
+        timer = setTimeout(() => {
+          if (startupSettled) return;
+          try {
+            const confirmation = expectedClaimMatches();
+            if (!confirmation.matches || confirmation.mismatch) {
+              fail(new Error(`LAUNCHER_CLAIM_LOST: Codex claim for exact Work ${workId} was not live after startup grace`), 'claim_lost', true);
+              return;
+            }
+          } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            fail(new Error(`LAUNCHER_CLAIM_OBSERVATION_FAILED: ${message}`), `claim_observation_failed:${message}`, true);
+            return;
+          }
+          startupSettled = true;
+          timer = undefined;
+          closeStartupPipes();
+          resolve();
+        }, LAUNCHER_STARTUP_GRACE_MS);
+        return;
+      }
+      if (Date.now() >= claimSettlementDeadline) {
+        fail(
+          new Error(`LAUNCHER_CLAIM_TIMEOUT: Codex pid=${String(child.pid ?? 'unknown')} did not claim exact Work ${workId} through Forge MCP within ${claimTimeoutMs}ms plus ${claimSettlementGraceMs}ms settlement grace`),
+          `claim_timeout:${claimTimeoutMs}ms+${claimSettlementGraceMs}ms_settlement`,
+          true,
+        );
+        return;
+      }
+      timer = setTimeout(pollClaim, claimPollIntervalMs);
+    };
+    pollClaim();
+  });
+}
+
 function assertChatgptConversationUrl(value: string | undefined): string | undefined {
   const raw = value?.trim();
   if (!raw) return undefined;
@@ -134,13 +349,14 @@ export function buildSuperControllerInvocation(
  * provider process selection and prompt construction; the execution Kernel
  * receives neither provider-specific arguments nor model output.
  */
-export function launchSuperController(
+export async function launchSuperController(
   stores: {
     work: WorkContractStoreOptions & { controllerHome: string; repoId: string };
     handoff: HandoffInboxStoreOptions;
   },
   request: ThinLauncherRequest,
-): ThinLauncherResult {
+  dependencies: ThinLauncherDependencies = {},
+): Promise<ThinLauncherResult> {
   const executable = resolveLauncherExecutable(request);
   const work = getWorkContract(stores.work, request.workId);
   if (!work) throw new Error(`WORK_NOT_FOUND: ${request.workId}`);
@@ -158,7 +374,7 @@ export function launchSuperController(
       `Continue Forge Work ${work.workId} in repo ${work.repoId}.`,
       handoff ? `Handoff: ${handoff.summary}\nNext: ${handoff.recommendedContinuationPrompt ?? handoff.recommendedPrompt}` : '',
       request.continuationPrompt?.trim() ? `Continuation: ${request.continuationPrompt.trim()}` : '',
-      `First call rh_work continue with repo_id=${work.repoId} and work_id=${work.workId}. Treat Forge Work/Plan/evidence as source of truth; do not invent new scope from chat history. If the claim fails, do not mutate. Continue the next safe action, finalize only when acceptance passes, and create a HandoffItem when judgement is required.`,
+      `No Controller ownership was preclaimed for this continued conversation. First call rh_work operation=controller_claim with repo_id=${work.repoId}, work_id=${work.workId}, and controller_type=chatgpt through your authenticated MCP session; do not invent controller_id/session_id. Only after that exact Work claim succeeds, call rh_work operation=continue with the same repo_id/work_id. Treat Forge Work/Plan/evidence as source of truth; do not invent new scope from chat history. If the claim does not succeed, do not mutate. Continue the next safe action, finalize only when acceptance passes, and create a HandoffItem when judgement is required.`,
     ].filter(Boolean).join('\n')
     : [
       `Work: ${work.workId}`,
@@ -167,21 +383,38 @@ export function launchSuperController(
       `Current status: ${work.status}`,
       handoff ? `Handoff: ${handoff.summary}\nNext: ${handoff.recommendedContinuationPrompt ?? handoff.recommendedPrompt}` : '',
       request.continuationPrompt?.trim() ? `Continuation: ${request.continuationPrompt.trim()}` : '',
-      `No Controller ownership was preclaimed for you. First call rh_work continue with repo_id=${work.repoId} and work_id=${work.workId} through your authenticated MCP session; do not invent controller_id/session_id. If that claim does not succeed, do not mutate the repository: create no patch, command, commit, or test run until ownership is established. Then use the repository MCP facade, record verification evidence, finalize only when acceptance passes, and create a HandoffItem when judgement is required.`,
+      `No Controller ownership was preclaimed for you. First call rh_work operation=controller_claim with repo_id=${work.repoId}, work_id=${work.workId}, and controller_type=${request.controllerType} through your authenticated MCP session; do not invent controller_id/session_id. Only after that exact Work claim succeeds, call rh_work operation=continue with the same repo_id/work_id and continue the returned next safe action. If the claim does not succeed, do not mutate the repository: create no patch, command, commit, or test run until ownership is established. Then use the repository MCP facade, record verification evidence, finalize only when acceptance passes, and create a HandoffItem when judgement is required.`, 
     ].filter(Boolean).join('\n');
   try {
     const mcpBootstrap = request.controllerType === 'codex'
-      ? resolveProviderMcpBootstrap(stores.work.controllerHome, 'codex', reservation.reservationId)
+      ? (dependencies.resolveProviderMcpBootstrap ?? resolveProviderMcpBootstrap)(stores.work.controllerHome, 'codex', reservation.reservationId)
       : undefined;
     const invocation = buildSuperControllerInvocation({ ...request, controllerHome: stores.work.controllerHome, repoId: work.repoId }, executable, prompt, mcpBootstrap);
     const child = spawn(invocation.executable, invocation.args, {
       cwd: request.cwd,
       detached: true,
-      stdio: 'ignore',
-      env: mcpBootstrap?.env ?? process.env,
+      stdio: ['ignore', 'pipe', 'pipe'],
+      env: launcherProcessEnvironment(mcpBootstrap?.env ?? process.env),
     });
-    child.unref();
     attachExternalControllerLaunchPid(stores.work, work.workId, reservation.reservationId, child.pid);
+    await awaitExternalControllerStartup(
+      child,
+      stores,
+      work.workId,
+      reservation.reservationId,
+      mcpBootstrap ? {
+        controllerType: 'codex',
+        controllerId: mcpBootstrap.principalId,
+        principalId: mcpBootstrap.principalId,
+        sessionId: mcpBootstrap.sessionId,
+      } : undefined,
+      {
+        claimTimeoutMs: dependencies.claimTimeoutMs,
+        claimSettlementGraceMs: dependencies.claimSettlementGraceMs,
+        claimPollIntervalMs: dependencies.claimPollIntervalMs,
+      },
+    );
+    child.unref();
     return { controllerType: request.controllerType, reservationId: reservation.reservationId, pid: child.pid, prompt, executable };
   } catch (error) {
     releaseExternalControllerLaunchReservation(stores.work, work.workId, reservation.reservationId, 'spawn_failed');