@moretea-labs/forge 1.8.0 → 1.8.2

package/src/runtime/root/mcp-transport.ts

@@ -13,11 +13,19 @@ interface ManagedSession {
   activeStreams: number;
 }
 
+type RuntimeMcpRequestId = string | number;
+interface ActiveRuntimeMcpPost {
+  response: Response;
+  requestIds: RuntimeMcpRequestId[];
+  transport?: NodeStreamableHTTPServerTransport;
+}
+
 export interface RuntimeMcpSessionSnapshot {
   active: number;
   maximum: number;
   initializing: number;
   protected: number;
+  activeRequests: number;
   capacityAvailable: number;
   capacityEvictions: number;
 }
@@ -47,6 +55,8 @@ export interface StartRuntimeMcpTransportOptions {
   onFatal?: (error: Error) => void;
   /** Internal Runtime sessions are bounded even if a Gateway dies without DELETE. */
   maximumSessions?: number;
+  /** Test/embedded override; production defaults to the bounded cutover grace. */
+  requestDrainTimeoutMs?: number;
 }
 
 function authorized(request: Request, configuredToken: string): boolean {
@@ -76,6 +86,15 @@ function observesToolSurface(body: unknown): boolean {
   });
 }
 
+function runtimeMcpRequestIds(body: unknown): RuntimeMcpRequestId[] {
+  const values = Array.isArray(body) ? body : [body];
+  return values.flatMap((value) => {
+    if (!value || typeof value !== 'object') return [];
+    const id = (value as Record<string, unknown>).id;
+    return typeof id === 'string' || typeof id === 'number' ? [id] : [];
+  });
+}
+
 function parseBody(body: unknown): unknown {
   if (!Buffer.isBuffer(body)) return body;
   try { return JSON.parse(body.toString('utf8')); } catch {
@@ -95,7 +114,16 @@ function authMiddleware(token: string) {
 }
 
 const SESSION_CLOSE_TIMEOUT_MS = 1_000;
-const REQUEST_DRAIN_TIMEOUT_MS = 250;
+/**
+ * Cutover first withdraws admission, then gives already-admitted requests a
+ * bounded grace period that still fits inside the primary service stop budget.
+ * Requests that outlive it are explicitly settled as outcome-unknown before
+ * session/socket teardown, so an outer controller turn is never left waiting on
+ * a connection the Runtime is intentionally destroying.
+ */
+export const RUNTIME_MCP_REQUEST_DRAIN_TIMEOUT_MS = 5_000;
+const CUTOVER_SETTLEMENT_DRAIN_TIMEOUT_MS = 250;
+export const RUNTIME_MCP_CUTOVER_OUTCOME_UNKNOWN = 'MCP_RUNTIME_CUTOVER_OUTCOME_UNKNOWN';
 export const DEFAULT_RUNTIME_MCP_MAX_SESSIONS = 64;
 
 function boundedRuntimeMcpMaximumSessions(value: number | undefined): number {
@@ -112,31 +140,41 @@ function closeServer(server: NodeHttpServer): Promise<void> {
   });
 }
 
-function boundedDrain(wait: Promise<void>, timeoutMs: number): Promise<void> {
-  if (timeoutMs <= 0) return Promise.resolve();
-  return Promise.race([
-    wait.catch(() => undefined),
+async function boundedDrain(wait: Promise<void>, timeoutMs: number): Promise<boolean> {
+  if (timeoutMs <= 0) return false;
+  let drained = false;
+  await Promise.race([
+    wait.then(() => { drained = true; }).catch(() => undefined),
     new Promise<void>((resolve) => setTimeout(resolve, timeoutMs)),
   ]);
+  return drained;
 }
 
 export async function closeRuntimeMcpTransportResources(input: {
   closeListener: () => Promise<void>;
   closeSessions: Array<() => Promise<void>>;
   waitForRequestDrain?: () => Promise<void>;
+  settleActiveRequests?: () => void | Promise<void>;
   forceCloseConnections?: () => void;
   requestDrainTimeoutMs?: number;
   sessionCloseTimeoutMs?: number;
 }): Promise<void> {
-  // Withdraw the TCP listener first so no new request can enter. Allow ordinary
-  // short MCP calls to finish before closing their session transport; long SSE
-  // streams remain bounded and are force-closed after the drain window.
+  // Withdraw the listener first so no new request can enter. Existing calls get
+  // one bounded grace window. If that expires, settle still-open POST responses
+  // as explicit cutover/outcome-unknown before closing session transports. This
+  // preserves the no-blind-replay contract while preventing a controller turn
+  // from being stranded on a socket the Runtime is about to destroy.
   const listenerClose = input.closeListener();
   void listenerClose.catch(() => undefined);
-  await boundedDrain(
-    input.waitForRequestDrain?.() ?? Promise.resolve(),
-    Math.max(0, input.requestDrainTimeoutMs ?? REQUEST_DRAIN_TIMEOUT_MS),
+  const requestDrain = input.waitForRequestDrain?.() ?? Promise.resolve();
+  const drained = await boundedDrain(
+    requestDrain,
+    Math.max(0, input.requestDrainTimeoutMs ?? RUNTIME_MCP_REQUEST_DRAIN_TIMEOUT_MS),
   );
+  if (!drained) {
+    await input.settleActiveRequests?.();
+    await boundedDrain(requestDrain, CUTOVER_SETTLEMENT_DRAIN_TIMEOUT_MS);
+  }
   const sessionDrain = Promise.allSettled(input.closeSessions.map(async (close) => await close()));
   await boundedDrain(sessionDrain.then(() => undefined), Math.max(0, input.sessionCloseTimeoutMs ?? SESSION_CLOSE_TIMEOUT_MS));
   input.forceCloseConnections?.();
@@ -152,12 +190,14 @@ export async function startRuntimeMcpTransport(
   let initializingSessions = 0;
   let capacityEvictions = 0;
   let activeRequests = 0;
+  const activePosts = new Map<Response, ActiveRuntimeMcpPost>();
 
   const sessionSnapshot = (): RuntimeMcpSessionSnapshot => ({
     active: sessions.size,
     maximum: maximumSessions,
     initializing: initializingSessions,
     protected: [...sessions.values()].filter((session) => session.activePosts > 0).length,
+    activeRequests,
     capacityAvailable: Math.max(0, maximumSessions - sessions.size - initializingSessions),
     capacityEvictions,
   });
@@ -192,7 +232,7 @@ export async function startRuntimeMcpTransport(
       managed.lastActivityAt = Date.now();
     }
   };
-  const requestDrainWaiters = new Set<() => void>();
+  const postDrainWaiters = new Set<() => void>();
   const app = express();
   app.disable('x-powered-by');
   app.use((_req, res, next) => {
@@ -202,18 +242,17 @@ export async function startRuntimeMcpTransport(
       if (settled) return;
       settled = true;
       activeRequests = Math.max(0, activeRequests - 1);
-      if (activeRequests === 0) {
-        for (const resolve of requestDrainWaiters) resolve();
-        requestDrainWaiters.clear();
-      }
     };
     res.once('finish', settle);
     res.once('close', settle);
     next();
   });
-  const waitForRequestDrain = (): Promise<void> => activeRequests === 0
+  // Only POST requests can represent an in-flight tool mutation/read result that
+  // must settle before cutover. Long-lived GET/SSE streams are transport-only and
+  // must not consume the cutover grace window.
+  const waitForRequestDrain = (): Promise<void> => activePosts.size === 0
     ? Promise.resolve()
-    : new Promise<void>((resolve) => requestDrainWaiters.add(resolve));
+    : new Promise<void>((resolve) => postDrainWaiters.add(resolve));
   app.get('/ready', (_req, res) => {
     const readiness = options.readiness();
     res.status(readiness.ready ? 200 : 503).json(readiness);
@@ -221,12 +260,24 @@ export async function startRuntimeMcpTransport(
 
   const requireAuth = authMiddleware(options.authToken);
   app.post('/mcp', requireAuth, express.raw({ type: '*/*', limit: '1mb' }), (req, res) => {
+    const activePost: ActiveRuntimeMcpPost = { response: res, requestIds: [] };
+    activePosts.set(res, activePost);
+    const releasePostResponse = (): void => {
+      activePosts.delete(res);
+      if (activePosts.size === 0) {
+        for (const resolve of postDrainWaiters) resolve();
+        postDrainWaiters.clear();
+      }
+    };
+    res.once('finish', releasePostResponse);
+    res.once('close', releasePostResponse);
     void (async () => {
       let body: unknown;
       try { body = parseBody(req.body); } catch (error) {
         res.status(400).json({ error: error instanceof Error ? error.message : String(error) });
         return;
       }
+      activePost.requestIds = runtimeMcpRequestIds(body);
       if (observesToolSurface(body)) options.onToolSurfaceObservation?.();
       const requestedSessionId = req.headers['mcp-session-id'];
       const sessionId = typeof requestedSessionId === 'string' ? requestedSessionId : undefined;
@@ -265,6 +316,7 @@ export async function startRuntimeMcpTransport(
             });
           },
         });
+        activePost.transport = transport;
         transport.onclose = () => {
           if (transport.sessionId) sessions.delete(transport.sessionId);
         };
@@ -293,10 +345,15 @@ export async function startRuntimeMcpTransport(
         res.status(404).json({ error: 'mcp_session_not_found' });
         return;
       }
+      activePost.transport = managed.transport;
       await withManagedSessionRequest(managed, 'post', async () => await managed.transport.handleRequest(req, res, body));
     })().catch((error: unknown) => {
       if (!res.headersSent) res.status(500).json({ error: 'mcp_request_failed' });
       console.error('[forge-runtime mcp] request failed:', error);
+    }).finally(() => {
+      releasePostResponse();
+      res.off('finish', releasePostResponse);
+      res.off('close', releasePostResponse);
     });
   });
   app.get('/mcp', requireAuth, (req, res) => {
@@ -362,6 +419,7 @@ export async function startRuntimeMcpTransport(
       await closeRuntimeMcpTransportResources({
         closeListener: async () => await closeServer(httpServer),
         waitForRequestDrain,
+        requestDrainTimeoutMs: options.requestDrainTimeoutMs,
         // Snapshot sessions only after the request drain: an initialize request
         // already in flight may register its session while shutdown is starting.
         closeSessions: [async () => {
@@ -369,6 +427,41 @@ export async function startRuntimeMcpTransport(
           await Promise.allSettled(activeSessions.map(async ({ transport }) => await transport.close().catch(() => undefined)));
           sessions.clear();
         }],
+        settleActiveRequests: async () => {
+          for (const active of activePosts.values()) {
+            let protocolSettled = false;
+            if (active.transport && active.requestIds.length > 0) {
+              const outcomes = await Promise.allSettled(active.requestIds.map(async (id) => {
+                await active.transport!.send({
+                  jsonrpc: '2.0',
+                  id,
+                  error: {
+                    code: -32000,
+                    message: RUNTIME_MCP_CUTOVER_OUTCOME_UNKNOWN,
+                    data: {
+                      outcome: 'unknown',
+                      recoverable: true,
+                      retryable: false,
+                      action: 'reconcile',
+                    },
+                  },
+                });
+              }));
+              protocolSettled = outcomes.some((outcome) => outcome.status === 'fulfilled');
+            }
+            if (protocolSettled) continue;
+            const response = active.response;
+            if (response.destroyed || response.writableEnded || response.headersSent) continue;
+            response.status(409).json({
+              error: 'runtime_cutover_outcome_unknown',
+              code: RUNTIME_MCP_CUTOVER_OUTCOME_UNKNOWN,
+              message: 'Canonical Runtime entered cutover after this request was admitted. The request outcome is unknown; reconcile by request/effect identity before any retry.',
+              recoverable: true,
+              retryable: false,
+              action: 'reconcile',
+            });
+          }
+        },
         forceCloseConnections: () => httpServer.closeAllConnections?.(),
       });
     },

package/src/runtime/root/ownership.ts

@@ -10,13 +10,28 @@ import {
 } from 'fs';
 import { dirname, join } from 'path';
 import { resolveControllerHome } from '../../cli/repositories/controller-home';
-import { isProcessAlive } from '../shared/process-tree';
+import { isProcessAlive, terminateProcessTree, type ProcessTreeTerminationResult } from '../shared/process-tree';
+import { defaultProcessIdentityProbe, executableFingerprint, processIdentityMatches } from '../shared/process-identity';
 
 export interface RuntimeOwnerRecord {
-  schemaVersion: 1;
+  schemaVersion: 1 | 2;
   runtimeInstanceId: string;
   pid: number;
   acquiredAt: string;
+  /** Monotonic Controller-Home writer epoch. Schema v1 owners predate it. */
+  fencingGeneration?: number;
+  /** Optional OS-process identity evidence used only for bounded crash recovery. */
+  processStartTime?: string;
+  executableFingerprint?: string;
+}
+
+export interface RuntimeIncarnationRecord {
+  schemaVersion: 1;
+  controllerHome: string;
+  runtimeInstanceId: string;
+  pid: number;
+  fencingGeneration: number;
+  activatedAt: string;
 }
 
 export interface RuntimeOwnershipHandle {
@@ -28,11 +43,18 @@ export function runtimeOwnerPath(controllerHome: string): string {
   return join(resolveControllerHome(controllerHome), 'runtime', 'active-runtime-owner.json');
 }
 
+export function runtimeIncarnationPath(controllerHome: string): string {
+  return join(resolveControllerHome(controllerHome), 'runtime', 'runtime-incarnation.json');
+}
+
 function readOwnerPath(path: string): RuntimeOwnerRecord | undefined {
   if (!existsSync(path)) return undefined;
   try {
     const value = JSON.parse(readFileSync(path, 'utf8')) as RuntimeOwnerRecord;
-    if (value.schemaVersion !== 1 || !value.runtimeInstanceId || !Number.isInteger(value.pid)) return undefined;
+    if ((value.schemaVersion !== 1 && value.schemaVersion !== 2) || !value.runtimeInstanceId || !Number.isInteger(value.pid)) return undefined;
+    if (value.schemaVersion === 2 && (!Number.isInteger(value.fencingGeneration) || value.fencingGeneration! < 1)) return undefined;
+    if ((value.processStartTime === undefined) !== (value.executableFingerprint === undefined)) return undefined;
+    if (value.processStartTime !== undefined && (!value.processStartTime.trim() || !value.executableFingerprint?.trim())) return undefined;
     return value;
   } catch {
     return undefined;
@@ -43,6 +65,208 @@ export function readRuntimeOwner(controllerHome: string): RuntimeOwnerRecord | u
   return readOwnerPath(runtimeOwnerPath(controllerHome));
 }
 
+export function readRuntimeIncarnation(controllerHome: string): RuntimeIncarnationRecord | undefined {
+  const path = runtimeIncarnationPath(controllerHome);
+  if (!existsSync(path)) return undefined;
+  try {
+    const value = JSON.parse(readFileSync(path, 'utf8')) as RuntimeIncarnationRecord;
+    if (
+      value.schemaVersion !== 1
+      || resolveControllerHome(value.controllerHome) !== resolveControllerHome(controllerHome)
+      || !value.runtimeInstanceId
+      || !Number.isInteger(value.pid)
+      || !Number.isInteger(value.fencingGeneration)
+      || value.fencingGeneration < 1
+      || !Number.isFinite(Date.parse(value.activatedAt))
+    ) return undefined;
+    return value;
+  } catch {
+    return undefined;
+  }
+}
+
+function nextFencingGeneration(controllerHome: string, minimum = 1): number {
+  const incarnationGeneration = readRuntimeIncarnation(controllerHome)?.fencingGeneration ?? 0;
+  const ownerGeneration = readRuntimeOwner(controllerHome)?.fencingGeneration ?? 0;
+  return Math.max(minimum - 1, incarnationGeneration, ownerGeneration) + 1;
+}
+
+function writeRuntimeIncarnation(record: RuntimeIncarnationRecord): void {
+  const path = runtimeIncarnationPath(record.controllerHome);
+  const temporary = `${path}.${process.pid}.${Date.now()}.tmp`;
+  writeFileSync(temporary, `${JSON.stringify(record, null, 2)}\n`, { encoding: 'utf8', mode: 0o600 });
+  renameSync(temporary, path);
+}
+
+
+export interface RuntimeOwnershipInspection {
+  owner?: RuntimeOwnerRecord;
+  incarnation?: RuntimeIncarnationRecord;
+  ownerAlive: boolean;
+  incarnationAlive: boolean;
+  coherent: boolean;
+}
+
+export function inspectRuntimeOwnership(controllerHome: string): RuntimeOwnershipInspection {
+  const owner = readRuntimeOwner(controllerHome);
+  const incarnation = readRuntimeIncarnation(controllerHome);
+  const coherent = Boolean(
+    owner
+    && incarnation
+    && owner.runtimeInstanceId === incarnation.runtimeInstanceId
+    && owner.pid === incarnation.pid
+    && (owner.fencingGeneration ?? incarnation.fencingGeneration) === incarnation.fencingGeneration,
+  );
+  // Schema-v2 owners carry OS-process identity. Once present, raw PID liveness
+  // is insufficient because a stopped Runtime PID may be reused by another
+  // process before Recovery reconciles durable ownership.
+  const ownerAlive = Boolean(owner && (
+    owner.processStartTime && owner.executableFingerprint
+      ? runtimeOwnerProcessIdentityMatches(owner)
+      : isProcessAlive(owner.pid)
+  ));
+  // A coherent incarnation is the same Runtime owner epoch, so inherit the
+  // stronger owner identity result instead of re-introducing PID-only liveness.
+  // Incarnation-only legacy evidence remains conservative because it has no
+  // independent process identity fields.
+  const incarnationAlive = Boolean(incarnation && (
+    coherent
+      ? ownerAlive
+      : isProcessAlive(incarnation.pid)
+  ));
+  return { owner, incarnation, ownerAlive, incarnationAlive, coherent };
+}
+
+export function runtimeOwnerProcessIdentityMatches(record: RuntimeOwnerRecord | undefined): boolean {
+  if (!record?.processStartTime || !record.executableFingerprint) return false;
+  return processIdentityMatches({
+    pid: record.pid,
+    processStartTime: record.processStartTime,
+    executableFingerprint: record.executableFingerprint,
+  }, record.pid).matches;
+}
+
+/**
+ * Recovery calls this only after the service manager and Runtime listener have
+ * been proven stopped. It never kills a live process. Dead owner evidence is
+ * renamed, not discarded, so fencing generations and forensic evidence survive.
+ */
+export function reconcileStoppedRuntimeOwnership(controllerHome: string): {
+  ok: boolean;
+  changed: boolean;
+  detail: string;
+  inspection: RuntimeOwnershipInspection;
+  staleOwnerPath?: string;
+} {
+  const inspection = inspectRuntimeOwnership(controllerHome);
+  if (inspection.ownerAlive || inspection.incarnationAlive) {
+    return {
+      ok: false,
+      changed: false,
+      detail: 'RUNTIME_OWNERSHIP_STILL_LIVE',
+      inspection,
+    };
+  }
+  const ownerPath = runtimeOwnerPath(controllerHome);
+  if (!inspection.owner || !existsSync(ownerPath)) {
+    return {
+      ok: true,
+      changed: false,
+      detail: 'Runtime ownership is already quiescent',
+      inspection,
+    };
+  }
+  const staleOwnerPath = `${ownerPath}.stale-${Date.now()}`;
+  renameSync(ownerPath, staleOwnerPath);
+  return {
+    ok: true,
+    changed: true,
+    detail: 'dead Runtime owner evidence was fenced out after service shutdown',
+    inspection,
+    staleOwnerPath,
+  };
+}
+
+
+export async function terminateVerifiedRuntimeOwner(
+  controllerHome: string,
+  expectedOwner?: RuntimeOwnerRecord | null,
+): Promise<{
+  ok: boolean;
+  attempted: boolean;
+  detail: string;
+  owner?: RuntimeOwnerRecord;
+  termination?: ProcessTreeTerminationResult;
+}> {
+  const owner = readRuntimeOwner(controllerHome);
+  if (expectedOwner) {
+    if (!owner) {
+      return {
+        ok: true,
+        attempted: false,
+        detail: 'Runtime owner disappeared during stop',
+        owner: expectedOwner,
+      };
+    }
+    if (
+      owner.pid !== expectedOwner.pid
+      || owner.runtimeInstanceId !== expectedOwner.runtimeInstanceId
+      || owner.fencingGeneration !== expectedOwner.fencingGeneration
+      || owner.processStartTime !== expectedOwner.processStartTime
+      || owner.executableFingerprint !== expectedOwner.executableFingerprint
+    ) {
+      return {
+        ok: false,
+        attempted: false,
+        detail: 'RUNTIME_OWNER_CHANGED_DURING_STOP',
+        owner,
+      };
+    }
+  } else if (owner) {
+    return {
+      ok: false,
+      attempted: false,
+      detail: 'RUNTIME_OWNER_APPEARED_DURING_STOP',
+      owner,
+    };
+  }
+  if (!owner) return { ok: true, attempted: false, detail: 'Runtime owner is already absent' };
+  if (!isProcessAlive(owner.pid)) {
+    return { ok: true, attempted: false, detail: 'Runtime owner process is already dead', owner };
+  }
+  if (!owner.processStartTime || !owner.executableFingerprint) {
+    return {
+      ok: false,
+      attempted: false,
+      detail: 'RUNTIME_OWNER_IDENTITY_EVIDENCE_MISSING',
+      owner,
+    };
+  }
+  if (!runtimeOwnerProcessIdentityMatches(owner)) {
+    return {
+      ok: false,
+      attempted: false,
+      detail: 'RUNTIME_OWNER_IDENTITY_CHANGED',
+      owner,
+    };
+  }
+  const termination = await terminateProcessTree(owner.pid, {
+    gracePeriodMs: 1_500,
+    killAfterMs: 5_000,
+    pollIntervalMs: 50,
+  });
+  const exited = termination.exited && !isProcessAlive(owner.pid);
+  return {
+    ok: exited,
+    attempted: true,
+    detail: exited
+      ? 'verified orphan Runtime owner process tree terminated after service shutdown'
+      : `verified orphan Runtime owner did not fully exit; remaining pids=${termination.remainingPids.join(',')}`,
+    owner,
+    termination,
+  };
+}
+
 export function acquireRuntimeOwnership(
   controllerHome: string,
   runtimeInstanceId: string,
@@ -51,14 +275,23 @@ export function acquireRuntimeOwnership(
   if (!runtimeInstanceId.trim()) throw new Error('RUNTIME_INSTANCE_ID_REQUIRED');
   const path = runtimeOwnerPath(controllerHome);
   mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
-  const record: RuntimeOwnerRecord = {
-    schemaVersion: 1,
-    runtimeInstanceId,
-    pid: process.pid,
-    acquiredAt: now(),
-  };
+  let minimumGeneration = nextFencingGeneration(controllerHome);
 
   for (let attempt = 0; attempt < 3; attempt += 1) {
+    const generation = nextFencingGeneration(controllerHome, minimumGeneration);
+    const processStartTime = defaultProcessIdentityProbe.startTime(process.pid);
+    const processCommand = defaultProcessIdentityProbe.command(process.pid);
+    const record: RuntimeOwnerRecord = {
+      schemaVersion: 2,
+      runtimeInstanceId,
+      pid: process.pid,
+      acquiredAt: now(),
+      fencingGeneration: generation,
+      ...(processStartTime && processCommand ? {
+        processStartTime,
+        executableFingerprint: executableFingerprint(processCommand),
+      } : {}),
+    };
     try {
       const fd = openSync(path, 'wx', 0o600);
       try {
@@ -66,6 +299,19 @@ export function acquireRuntimeOwnership(
       } finally {
         closeSync(fd);
       }
+      try {
+        writeRuntimeIncarnation({
+          schemaVersion: 1,
+          controllerHome: resolveControllerHome(controllerHome),
+          runtimeInstanceId,
+          pid: process.pid,
+          fencingGeneration: generation,
+          activatedAt: record.acquiredAt,
+        });
+      } catch (writeError) {
+        try { unlinkSync(path); } catch { /* ownership did not become usable */ }
+        throw writeError;
+      }
       return {
         record,
         release: () => {
@@ -84,6 +330,12 @@ export function acquireRuntimeOwnership(
           `RUNTIME_OWNERSHIP_CONFLICT: controller home is owned by ${current.runtimeInstanceId} pid=${current.pid}`,
         );
       }
+      minimumGeneration = Math.max(
+        minimumGeneration,
+        generation + 1,
+        (current?.fencingGeneration ?? 0) + 1,
+        (readRuntimeIncarnation(controllerHome)?.fencingGeneration ?? 0) + 1,
+      );
       const stalePath = `${path}.stale-${Date.now()}-${attempt}`;
       try { renameSync(path, stalePath); } catch (renameError) {
         if ((renameError as NodeJS.ErrnoException).code !== 'ENOENT') throw renameError;